LLVM  15.0.0git
VPlanTransforms.cpp
Go to the documentation of this file.
1 //===-- VPlanTransforms.cpp - Utility VPlan to VPlan transforms -----------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 ///
9 /// \file
10 /// This file implements a set of utility VPlan to VPlan transformations.
11 ///
12 //===----------------------------------------------------------------------===//
13 
14 #include "VPlanTransforms.h"
16 #include "llvm/ADT/SetVector.h"
18 
19 using namespace llvm;
20 
22  Loop *OrigLoop, VPlanPtr &Plan,
24  GetIntOrFpInductionDescriptor,
25  SmallPtrSetImpl<Instruction *> &DeadInstructions, ScalarEvolution &SE) {
26 
27  auto *TopRegion = cast<VPRegionBlock>(Plan->getEntry());
28  ReversePostOrderTraversal<VPBlockBase *> RPOT(TopRegion->getEntry());
29 
30  for (VPBlockBase *Base : RPOT) {
31  // Do not widen instructions in pre-header and exit blocks.
32  if (Base->getNumPredecessors() == 0 || Base->getNumSuccessors() == 0)
33  continue;
34 
35  VPBasicBlock *VPBB = Base->getEntryBasicBlock();
36  // Introduce each ingredient into VPlan.
37  for (VPRecipeBase &Ingredient : llvm::make_early_inc_range(*VPBB)) {
38  VPValue *VPV = Ingredient.getVPSingleValue();
39  Instruction *Inst = cast<Instruction>(VPV->getUnderlyingValue());
40  if (DeadInstructions.count(Inst)) {
41  VPValue DummyValue;
42  VPV->replaceAllUsesWith(&DummyValue);
43  Ingredient.eraseFromParent();
44  continue;
45  }
46 
47  VPRecipeBase *NewRecipe = nullptr;
48  if (auto *VPPhi = dyn_cast<VPWidenPHIRecipe>(&Ingredient)) {
49  auto *Phi = cast<PHINode>(VPPhi->getUnderlyingValue());
50  if (const auto *II = GetIntOrFpInductionDescriptor(Phi)) {
51  VPValue *Start = Plan->getOrAddVPValue(II->getStartValue());
52  VPValue *Step =
53  vputils::getOrCreateVPValueForSCEVExpr(*Plan, II->getStep(), SE);
54  NewRecipe = new VPWidenIntOrFpInductionRecipe(Phi, Start, Step, *II,
55  false, true);
56  } else {
57  Plan->addVPValue(Phi, VPPhi);
58  continue;
59  }
60  } else {
61  assert(isa<VPInstruction>(&Ingredient) &&
62  "only VPInstructions expected here");
63  assert(!isa<PHINode>(Inst) && "phis should be handled above");
64  // Create VPWidenMemoryInstructionRecipe for loads and stores.
65  if (LoadInst *Load = dyn_cast<LoadInst>(Inst)) {
66  NewRecipe = new VPWidenMemoryInstructionRecipe(
67  *Load, Plan->getOrAddVPValue(getLoadStorePointerOperand(Inst)),
68  nullptr /*Mask*/, false /*Consecutive*/, false /*Reverse*/);
69  } else if (StoreInst *Store = dyn_cast<StoreInst>(Inst)) {
70  NewRecipe = new VPWidenMemoryInstructionRecipe(
71  *Store, Plan->getOrAddVPValue(getLoadStorePointerOperand(Inst)),
72  Plan->getOrAddVPValue(Store->getValueOperand()), nullptr /*Mask*/,
73  false /*Consecutive*/, false /*Reverse*/);
74  } else if (GetElementPtrInst *GEP = dyn_cast<GetElementPtrInst>(Inst)) {
75  NewRecipe = new VPWidenGEPRecipe(
76  GEP, Plan->mapToVPValues(GEP->operands()), OrigLoop);
77  } else if (CallInst *CI = dyn_cast<CallInst>(Inst)) {
78  NewRecipe =
79  new VPWidenCallRecipe(*CI, Plan->mapToVPValues(CI->args()));
80  } else if (SelectInst *SI = dyn_cast<SelectInst>(Inst)) {
81  bool InvariantCond =
82  SE.isLoopInvariant(SE.getSCEV(SI->getOperand(0)), OrigLoop);
83  NewRecipe = new VPWidenSelectRecipe(
84  *SI, Plan->mapToVPValues(SI->operands()), InvariantCond);
85  } else {
86  NewRecipe =
87  new VPWidenRecipe(*Inst, Plan->mapToVPValues(Inst->operands()));
88  }
89  }
90 
91  NewRecipe->insertBefore(&Ingredient);
92  if (NewRecipe->getNumDefinedValues() == 1)
93  VPV->replaceAllUsesWith(NewRecipe->getVPSingleValue());
94  else
95  assert(NewRecipe->getNumDefinedValues() == 0 &&
96  "Only recpies with zero or one defined values expected");
97  Ingredient.eraseFromParent();
98  Plan->removeVPValueFor(Inst);
99  for (auto *Def : NewRecipe->definedValues()) {
100  Plan->addVPValue(Inst, Def);
101  }
102  }
103  }
104 }
105 
107  auto Iter = depth_first(
109  bool Changed = false;
110  // First, collect the operands of all predicated replicate recipes as seeds
111  // for sinking.
113  for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(Iter)) {
114  for (auto &Recipe : *VPBB) {
115  auto *RepR = dyn_cast<VPReplicateRecipe>(&Recipe);
116  if (!RepR || !RepR->isPredicated())
117  continue;
118  for (VPValue *Op : RepR->operands())
119  WorkList.insert(std::make_pair(RepR->getParent(), Op));
120  }
121  }
122 
123  // Try to sink each replicate recipe in the worklist.
124  while (!WorkList.empty()) {
125  VPBasicBlock *SinkTo;
126  VPValue *C;
127  std::tie(SinkTo, C) = WorkList.pop_back_val();
128  auto *SinkCandidate = dyn_cast_or_null<VPReplicateRecipe>(C->Def);
129  if (!SinkCandidate || SinkCandidate->isUniform() ||
130  SinkCandidate->getParent() == SinkTo ||
131  SinkCandidate->mayHaveSideEffects() ||
132  SinkCandidate->mayReadOrWriteMemory())
133  continue;
134 
135  bool NeedsDuplicating = false;
136  // All recipe users of the sink candidate must be in the same block SinkTo
137  // or all users outside of SinkTo must be uniform-after-vectorization (
138  // i.e., only first lane is used) . In the latter case, we need to duplicate
139  // SinkCandidate. At the moment, we identify such UAV's by looking for the
140  // address operands of widened memory recipes.
141  auto CanSinkWithUser = [SinkTo, &NeedsDuplicating,
142  SinkCandidate](VPUser *U) {
143  auto *UI = dyn_cast<VPRecipeBase>(U);
144  if (!UI)
145  return false;
146  if (UI->getParent() == SinkTo)
147  return true;
148  auto *WidenI = dyn_cast<VPWidenMemoryInstructionRecipe>(UI);
149  if (WidenI && WidenI->getAddr() == SinkCandidate) {
150  NeedsDuplicating = true;
151  return true;
152  }
153  return false;
154  };
155  if (!all_of(SinkCandidate->users(), CanSinkWithUser))
156  continue;
157 
158  if (NeedsDuplicating) {
159  Instruction *I = cast<Instruction>(SinkCandidate->getUnderlyingValue());
160  auto *Clone =
161  new VPReplicateRecipe(I, SinkCandidate->operands(), true, false);
162  // TODO: add ".cloned" suffix to name of Clone's VPValue.
163 
164  Clone->insertBefore(SinkCandidate);
165  SmallVector<VPUser *, 4> Users(SinkCandidate->users());
166  for (auto *U : Users) {
167  auto *UI = cast<VPRecipeBase>(U);
168  if (UI->getParent() == SinkTo)
169  continue;
170 
171  for (unsigned Idx = 0; Idx != UI->getNumOperands(); Idx++) {
172  if (UI->getOperand(Idx) != SinkCandidate)
173  continue;
174  UI->setOperand(Idx, Clone);
175  }
176  }
177  }
178  SinkCandidate->moveBefore(*SinkTo, SinkTo->getFirstNonPhi());
179  for (VPValue *Op : SinkCandidate->operands())
180  WorkList.insert(std::make_pair(SinkTo, Op));
181  Changed = true;
182  }
183  return Changed;
184 }
185 
186 /// If \p R is a region with a VPBranchOnMaskRecipe in the entry block, return
187 /// the mask.
189  auto *EntryBB = dyn_cast<VPBasicBlock>(R->getEntry());
190  if (!EntryBB || EntryBB->size() != 1 ||
191  !isa<VPBranchOnMaskRecipe>(EntryBB->begin()))
192  return nullptr;
193 
194  return cast<VPBranchOnMaskRecipe>(&*EntryBB->begin())->getOperand(0);
195 }
196 
197 /// If \p R is a triangle region, return the 'then' block of the triangle.
199  auto *EntryBB = cast<VPBasicBlock>(R->getEntry());
200  if (EntryBB->getNumSuccessors() != 2)
201  return nullptr;
202 
203  auto *Succ0 = dyn_cast<VPBasicBlock>(EntryBB->getSuccessors()[0]);
204  auto *Succ1 = dyn_cast<VPBasicBlock>(EntryBB->getSuccessors()[1]);
205  if (!Succ0 || !Succ1)
206  return nullptr;
207 
208  if (Succ0->getNumSuccessors() + Succ1->getNumSuccessors() != 1)
209  return nullptr;
210  if (Succ0->getSingleSuccessor() == Succ1)
211  return Succ0;
212  if (Succ1->getSingleSuccessor() == Succ0)
213  return Succ1;
214  return nullptr;
215 }
216 
218  SetVector<VPRegionBlock *> DeletedRegions;
219  bool Changed = false;
220 
221  // Collect region blocks to process up-front, to avoid iterator invalidation
222  // issues while merging regions.
223  SmallVector<VPRegionBlock *, 8> CandidateRegions(
224  VPBlockUtils::blocksOnly<VPRegionBlock>(depth_first(
226 
227  // Check if Base is a predicated triangle, followed by an empty block,
228  // followed by another predicate triangle. If that's the case, move the
229  // recipes from the first to the second triangle.
230  for (VPRegionBlock *Region1 : CandidateRegions) {
231  if (DeletedRegions.contains(Region1))
232  continue;
233  auto *MiddleBasicBlock =
234  dyn_cast_or_null<VPBasicBlock>(Region1->getSingleSuccessor());
235  if (!MiddleBasicBlock || !MiddleBasicBlock->empty())
236  continue;
237 
238  auto *Region2 =
239  dyn_cast_or_null<VPRegionBlock>(MiddleBasicBlock->getSingleSuccessor());
240  if (!Region2)
241  continue;
242 
243  VPValue *Mask1 = getPredicatedMask(Region1);
244  VPValue *Mask2 = getPredicatedMask(Region2);
245  if (!Mask1 || Mask1 != Mask2)
246  continue;
247  VPBasicBlock *Then1 = getPredicatedThenBlock(Region1);
248  VPBasicBlock *Then2 = getPredicatedThenBlock(Region2);
249  if (!Then1 || !Then2)
250  continue;
251 
252  assert(Mask1 && Mask2 && "both region must have conditions");
253 
254  // Note: No fusion-preventing memory dependencies are expected in either
255  // region. Such dependencies should be rejected during earlier dependence
256  // checks, which guarantee accesses can be re-ordered for vectorization.
257  //
258  // Move recipes to the successor region.
259  for (VPRecipeBase &ToMove : make_early_inc_range(reverse(*Then1)))
260  ToMove.moveBefore(*Then2, Then2->getFirstNonPhi());
261 
262  auto *Merge1 = cast<VPBasicBlock>(Then1->getSingleSuccessor());
263  auto *Merge2 = cast<VPBasicBlock>(Then2->getSingleSuccessor());
264 
265  // Move VPPredInstPHIRecipes from the merge block to the successor region's
266  // merge block. Update all users inside the successor region to use the
267  // original values.
268  for (VPRecipeBase &Phi1ToMove : make_early_inc_range(reverse(*Merge1))) {
269  VPValue *PredInst1 =
270  cast<VPPredInstPHIRecipe>(&Phi1ToMove)->getOperand(0);
271  VPValue *Phi1ToMoveV = Phi1ToMove.getVPSingleValue();
272  SmallVector<VPUser *> Users(Phi1ToMoveV->users());
273  for (VPUser *U : Users) {
274  auto *UI = dyn_cast<VPRecipeBase>(U);
275  if (!UI || UI->getParent() != Then2)
276  continue;
277  for (unsigned I = 0, E = U->getNumOperands(); I != E; ++I) {
278  if (Phi1ToMoveV != U->getOperand(I))
279  continue;
280  U->setOperand(I, PredInst1);
281  }
282  }
283 
284  Phi1ToMove.moveBefore(*Merge2, Merge2->begin());
285  }
286 
287  // Finally, remove the first region.
288  for (VPBlockBase *Pred : make_early_inc_range(Region1->getPredecessors())) {
289  VPBlockUtils::disconnectBlocks(Pred, Region1);
290  VPBlockUtils::connectBlocks(Pred, MiddleBasicBlock);
291  }
292  VPBlockUtils::disconnectBlocks(Region1, MiddleBasicBlock);
293  DeletedRegions.insert(Region1);
294  }
295 
296  for (VPRegionBlock *ToDelete : DeletedRegions)
297  delete ToDelete;
298  return Changed;
299 }
300 
302  for (auto &Phi : Plan.getVectorLoopRegion()->getEntryBasicBlock()->phis()) {
303  auto *IV = dyn_cast<VPWidenIntOrFpInductionRecipe>(&Phi);
304  if (!IV || IV->getTruncInst())
305  continue;
306 
307  // A sequence of IR Casts has potentially been recorded for IV, which
308  // *must be bypassed* when the IV is vectorized, because the vectorized IV
309  // will produce the desired casted value. This sequence forms a def-use
310  // chain and is provided in reverse order, ending with the cast that uses
311  // the IV phi. Search for the recipe of the last cast in the chain and
312  // replace it with the original IV. Note that only the final cast is
313  // expected to have users outside the cast-chain and the dead casts left
314  // over will be cleaned up later.
315  auto &Casts = IV->getInductionDescriptor().getCastInsts();
316  VPValue *FindMyCast = IV;
317  for (Instruction *IRCast : reverse(Casts)) {
318  VPRecipeBase *FoundUserCast = nullptr;
319  for (auto *U : FindMyCast->users()) {
320  auto *UserCast = cast<VPRecipeBase>(U);
321  if (UserCast->getNumDefinedValues() == 1 &&
322  UserCast->getVPSingleValue()->getUnderlyingValue() == IRCast) {
323  FoundUserCast = UserCast;
324  break;
325  }
326  }
327  FindMyCast = FoundUserCast->getVPSingleValue();
328  }
329  FindMyCast->replaceAllUsesWith(IV);
330  }
331 }
332 
334  VPCanonicalIVPHIRecipe *CanonicalIV = Plan.getCanonicalIV();
335  VPWidenCanonicalIVRecipe *WidenNewIV = nullptr;
336  for (VPUser *U : CanonicalIV->users()) {
337  WidenNewIV = dyn_cast<VPWidenCanonicalIVRecipe>(U);
338  if (WidenNewIV)
339  break;
340  }
341 
342  if (!WidenNewIV)
343  return;
344 
345  VPBasicBlock *HeaderVPBB = Plan.getVectorLoopRegion()->getEntryBasicBlock();
346  for (VPRecipeBase &Phi : HeaderVPBB->phis()) {
347  auto *WidenOriginalIV = dyn_cast<VPWidenIntOrFpInductionRecipe>(&Phi);
348 
349  if (!WidenOriginalIV || !WidenOriginalIV->isCanonical() ||
350  WidenOriginalIV->getScalarType() != WidenNewIV->getScalarType())
351  continue;
352 
353  // Replace WidenNewIV with WidenOriginalIV if WidenOriginalIV provides
354  // everything WidenNewIV's users need. That is, WidenOriginalIV will
355  // generate a vector phi or all users of WidenNewIV demand the first lane
356  // only.
357  if (WidenOriginalIV->needsVectorIV() ||
358  vputils::onlyFirstLaneUsed(WidenNewIV)) {
359  WidenNewIV->replaceAllUsesWith(WidenOriginalIV);
360  WidenNewIV->eraseFromParent();
361  return;
362  }
363  }
364 }
365 
368  // Remove dead recipes in header block. The recipes in the block are processed
369  // in reverse order, to catch chains of dead recipes.
370  // TODO: Remove dead recipes across whole plan.
371  for (VPRecipeBase &R : make_early_inc_range(reverse(*Header))) {
372  if (R.mayHaveSideEffects() || any_of(R.definedValues(), [](VPValue *V) {
373  return V->getNumUsers() > 0;
374  }))
375  continue;
376  R.eraseFromParent();
377  }
378 }
379 
382  VPBasicBlock *HeaderVPBB = Plan.getVectorLoopRegion()->getEntryBasicBlock();
383  for (VPRecipeBase &Phi : HeaderVPBB->phis()) {
384  auto *IV = dyn_cast<VPWidenIntOrFpInductionRecipe>(&Phi);
385  if (!IV || !IV->needsScalarIV())
386  continue;
387 
388  const InductionDescriptor &ID = IV->getInductionDescriptor();
389  VPValue *Step =
390  vputils::getOrCreateVPValueForSCEVExpr(Plan, ID.getStep(), SE);
391  Instruction *TruncI = IV->getTruncInst();
393  IV->getPHINode()->getType(), ID, Plan.getCanonicalIV(),
394  IV->getStartValue(), Step, TruncI ? TruncI->getType() : nullptr);
395  HeaderVPBB->insert(Steps, HeaderVPBB->getFirstNonPhi());
396 
397  // If there are no vector users of IV, simply update all users to use Step
398  // instead.
399  if (!IV->needsVectorIV()) {
400  IV->replaceAllUsesWith(Steps);
401  continue;
402  }
403 
404  // Otherwise only update scalar users of IV to use Step instead. Use
405  // SetVector to ensure the list of users doesn't contain duplicates.
406  SetVector<VPUser *> Users(IV->user_begin(), IV->user_end());
407  for (VPUser *U : Users) {
408  if (!U->usesScalars(IV))
409  continue;
410  for (unsigned I = 0, E = U->getNumOperands(); I != E; I++) {
411  if (U->getOperand(I) != IV)
412  continue;
413  U->setOperand(I, Steps);
414  }
415  }
416  }
417 }
418 
421 
422  for (VPRecipeBase &R :
424  auto *ExpR = dyn_cast<VPExpandSCEVRecipe>(&R);
425  if (!ExpR)
426  continue;
427 
428  auto I = SCEV2VPV.insert({ExpR->getSCEV(), ExpR});
429  if (I.second)
430  continue;
431  ExpR->replaceAllUsesWith(I.first->second);
432  ExpR->eraseFromParent();
433  }
434 }
llvm
This is an optimization pass for GlobalISel generic memory operations.
Definition: AddressRanges.h:17
llvm::VPlanTransforms::VPInstructionsToVPRecipes
static void VPInstructionsToVPRecipes(Loop *OrigLoop, VPlanPtr &Plan, function_ref< const InductionDescriptor *(PHINode *)> GetIntOrFpInductionDescriptor, SmallPtrSetImpl< Instruction * > &DeadInstructions, ScalarEvolution &SE)
Replaces the VPInstructions in Plan with corresponding widen recipes.
Definition: VPlanTransforms.cpp:21
llvm::tgtok::Def
@ Def
Definition: TGLexer.h:50
llvm::VPCanonicalIVPHIRecipe
Canonical scalar induction phi of the vector loop.
Definition: VPlan.h:1852
llvm::VPRecipeBase::eraseFromParent
iplist< VPRecipeBase >::iterator eraseFromParent()
This method unlinks 'this' from the containing basic block and deletes it.
Definition: VPlan.cpp:689
llvm::User::operands
op_range operands()
Definition: User.h:242
getPredicatedMask
VPValue * getPredicatedMask(VPRegionBlock *R)
If R is a region with a VPBranchOnMaskRecipe in the entry block, return the mask.
Definition: VPlanTransforms.cpp:188
llvm::VPBasicBlock
VPBasicBlock serves as the leaf of the Hierarchical Control-Flow Graph.
Definition: VPlan.h:1999
llvm::Loop
Represents a single loop in the control flow graph.
Definition: LoopInfo.h:530
llvm::SmallVector
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
Definition: SmallVector.h:1185
llvm::VPDef::getVPSingleValue
VPValue * getVPSingleValue()
Returns the only VPValue defined by the VPDef.
Definition: VPlanValue.h:389
ToRemove
ReachingDefAnalysis InstSet & ToRemove
Definition: ARMLowOverheadLoops.cpp:542
llvm::ScalarEvolution
The main scalar evolution driver.
Definition: ScalarEvolution.h:449
llvm::VPlan::getCanonicalIV
VPCanonicalIVPHIRecipe * getCanonicalIV()
Returns the canonical induction recipe of the vector loop.
Definition: VPlan.h:2683
llvm::VPlanPtr
std::unique_ptr< VPlan > VPlanPtr
Definition: VPlan.h:97
llvm::reverse
auto reverse(ContainerTy &&C, std::enable_if_t< has_rbegin< ContainerTy >::value > *=nullptr)
Definition: STLExtras.h:380
llvm::SPII::Load
@ Load
Definition: SparcInstrInfo.h:32
llvm::VPWidenCallRecipe
A recipe for widening Call instructions.
Definition: VPlan.h:975
llvm::VPValue::users
user_range users()
Definition: VPlanValue.h:162
llvm::VPReplicateRecipe
VPReplicateRecipe replicates a given instruction producing multiple scalar copies of the original sca...
Definition: VPlan.h:1574
llvm::VPBlockBase
VPBlockBase is the building block of the Hierarchical Control-Flow Graph.
Definition: VPlan.h:394
llvm::VPlanTransforms::removeRedundantCanonicalIVs
static void removeRedundantCanonicalIVs(VPlan &Plan)
Try to replace VPWidenCanonicalIVRecipes with a widened canonical IV recipe, if it exists.
Definition: VPlanTransforms.cpp:333
llvm::all_of
bool all_of(R &&range, UnaryPredicate P)
Provide wrappers to std::all_of which take ranges instead of having to pass begin/end explicitly.
Definition: STLExtras.h:1607
llvm::InductionDescriptor
A struct for saving information about induction variables.
Definition: IVDescriptors.h:306
llvm::vputils::getOrCreateVPValueForSCEVExpr
VPValue * getOrCreateVPValueForSCEVExpr(VPlan &Plan, const SCEV *Expr, ScalarEvolution &SE)
Get or create a VPValue that corresponds to the expansion of Expr.
Definition: VPlan.cpp:1801
E
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
C
(vector float) vec_cmpeq(*A, *B) C
Definition: README_ALTIVEC.txt:86
llvm::Instruction
Definition: Instruction.h:42
llvm::SPII::Store
@ Store
Definition: SparcInstrInfo.h:33
llvm::VPlanTransforms::optimizeInductions
static void optimizeInductions(VPlan &Plan, ScalarEvolution &SE)
If any user of a VPWidenIntOrFpInductionRecipe needs scalar values, provide them by building scalar s...
Definition: VPlanTransforms.cpp:380
llvm::VPValue::replaceAllUsesWith
void replaceAllUsesWith(VPValue *New)
Definition: VPlan.cpp:1685
llvm::VPValue
Definition: VPlanValue.h:44
llvm::VPWidenCanonicalIVRecipe
A Recipe for widening the canonical induction variable of the vector loop.
Definition: VPlan.h:1897
llvm::VPDef::getNumDefinedValues
unsigned getNumDefinedValues() const
Returns the number of values defined by the VPDef.
Definition: VPlanValue.h:416
IVDescriptors.h
llvm::SetVector::empty
bool empty() const
Determine if the SetVector is empty or not.
Definition: SetVector.h:72
llvm::CallingConv::ID
unsigned ID
LLVM IR allows to use arbitrary numbers as calling convention identifiers.
Definition: CallingConv.h:24
llvm::VPRegionBlock
VPRegionBlock represents a collection of VPBasicBlocks and VPRegionBlocks which form a Single-Entry-S...
Definition: VPlan.h:2115
llvm::ScalarEvolution::getSCEV
const SCEV * getSCEV(Value *V)
Return a SCEV expression for the full generality of the specified expression.
Definition: ScalarEvolution.cpp:4407
llvm::function_ref
An efficient, type-erasing, non-owning reference to a callable.
Definition: STLFunctionalExtras.h:36
llvm::StoreInst
An instruction for storing to memory.
Definition: Instructions.h:305
llvm::SetVector::contains
bool contains(const key_type &key) const
Check if the SetVector contains the given key.
Definition: SetVector.h:209
llvm::VPWidenRecipe
VPWidenRecipe is a recipe for producing a copy of vector type its ingredient.
Definition: VPlan.h:947
llvm::DenseMap
Definition: DenseMap.h:716
llvm::VPlanTransforms::sinkScalarOperands
static bool sinkScalarOperands(VPlan &Plan)
Definition: VPlanTransforms.cpp:106
I
#define I(x, y, z)
Definition: MD5.cpp:58
llvm::VPBlockUtils::connectBlocks
static void connectBlocks(VPBlockBase *From, VPBlockBase *To)
Connect VPBlockBases From and To bi-directionally.
Definition: VPlan.h:2839
llvm::GetElementPtrInst
an instruction for type-safe pointer arithmetic to access elements of arrays and structs
Definition: Instructions.h:929
llvm::make_early_inc_range
iterator_range< early_inc_iterator_impl< detail::IterOfRange< RangeT > > > make_early_inc_range(RangeT &&Range)
Make a range that does early increment to allow mutation of the underlying range without disrupting i...
Definition: STLExtras.h:608
llvm::VPBlockUtils::disconnectBlocks
static void disconnectBlocks(VPBlockBase *From, VPBlockBase *To)
Disconnect VPBlockBases From and To bi-directionally.
Definition: VPlan.h:2850
llvm::VPlan::getEntry
VPBlockBase * getEntry()
Definition: VPlan.h:2548
assert
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
VPlanTransforms.h
llvm::VPWidenIntOrFpInductionRecipe
A recipe for handling phi nodes of integer and floating-point inductions, producing their vector valu...
Definition: VPlan.h:1073
SI
StandardInstrumentations SI(Debug, VerifyEach)
llvm::VPWidenGEPRecipe
A recipe for handling GEP instructions.
Definition: VPlan.h:1032
llvm::SelectInst
This class represents the LLVM 'select' instruction.
Definition: Instructions.h:1737
llvm::VPWidenMemoryInstructionRecipe
A Recipe for widening load/store operations.
Definition: VPlan.h:1724
llvm::VPWidenSelectRecipe
A recipe for widening select instructions.
Definition: VPlan.h:1001
llvm::SmallPtrSetImpl::count
size_type count(ConstPtrType Ptr) const
count - Return 1 if the specified pointer is in the set, 0 otherwise.
Definition: SmallPtrSet.h:383
llvm::SetVector::insert
bool insert(const value_type &X)
Insert a new element into the SetVector.
Definition: SetVector.h:141
llvm::VPlan::getVectorLoopRegion
VPRegionBlock * getVectorLoopRegion()
Returns the VPRegionBlock of the vector loop.
Definition: VPlan.h:2671
getPredicatedThenBlock
static VPBasicBlock * getPredicatedThenBlock(VPRegionBlock *R)
If R is a triangle region, return the 'then' block of the triangle.
Definition: VPlanTransforms.cpp:198
llvm::any_of
bool any_of(R &&range, UnaryPredicate P)
Provide wrappers to std::any_of which take ranges instead of having to pass begin/end explicitly.
Definition: STLExtras.h:1614
llvm::VPBlockBase::getSingleSuccessor
VPBlockBase * getSingleSuccessor() const
Definition: VPlan.h:511
llvm::Value::getType
Type * getType() const
All values are typed, get the type of this value.
Definition: Value.h:255
llvm::LoadInst
An instruction for reading from memory.
Definition: Instructions.h:176
llvm::DenseMapBase< DenseMap< KeyT, ValueT, DenseMapInfo< KeyT >, llvm::detail::DenseMapPair< KeyT, ValueT > >, KeyT, ValueT, DenseMapInfo< KeyT >, llvm::detail::DenseMapPair< KeyT, ValueT > >::insert
std::pair< iterator, bool > insert(const std::pair< KeyT, ValueT > &KV)
Definition: DenseMap.h:209
llvm::VPBlockBase::getEntryBasicBlock
const VPBasicBlock * getEntryBasicBlock() const
Definition: VPlan.cpp:140
llvm::vputils::onlyFirstLaneUsed
bool onlyFirstLaneUsed(VPValue *Def)
Returns true if only the first lane of Def is used.
Definition: VPlan.cpp:1796
llvm::ScalarEvolution::isLoopInvariant
bool isLoopInvariant(const SCEV *S, const Loop *L)
Return true if the value of the given SCEV is unchanging in the specified loop.
Definition: ScalarEvolution.cpp:13248
llvm::depth_first
iterator_range< df_iterator< T > > depth_first(const T &G)
Definition: DepthFirstIterator.h:230
llvm::VPlanTransforms::removeRedundantExpandSCEVRecipes
static void removeRedundantExpandSCEVRecipes(VPlan &Plan)
Remove redundant EpxandSCEVRecipes in Plan's entry block by replacing them with already existing reci...
Definition: VPlanTransforms.cpp:419
llvm::VPlanTransforms::removeDeadRecipes
static void removeDeadRecipes(VPlan &Plan, Loop &OrigLoop)
Try to remove dead recipes.
Definition: VPlanTransforms.cpp:366
llvm::AMDGPU::SendMsg::Op
Op
Definition: SIDefines.h:341
llvm::VPScalarIVStepsRecipe
A recipe for handling phi nodes of integer and floating-point inductions, producing their scalar valu...
Definition: VPlan.h:1940
llvm::VPRecipeBase
VPRecipeBase is a base class modeling a sequence of one or more output IR instructions.
Definition: VPlan.h:698
llvm::VPWidenCanonicalIVRecipe::getScalarType
const Type * getScalarType() const
Returns the scalar type of the induction.
Definition: VPlan.h:1932
llvm::ReversePostOrderTraversal
Definition: PostOrderIterator.h:291
llvm::VPBasicBlock::insert
void insert(VPRecipeBase *Recipe, iterator InsertPt)
Definition: VPlan.h:2058
llvm::VPUser
This class augments VPValue with operands which provide the inverse def-use edges from VPValue's user...
Definition: VPlanValue.h:206
llvm::VPlanTransforms::removeRedundantInductionCasts
static void removeRedundantInductionCasts(VPlan &Plan)
Remove redundant casts of inductions.
Definition: VPlanTransforms.cpp:301
PostOrderIterator.h
Users
iv Induction Variable Users
Definition: IVUsers.cpp:48
llvm::PHINode
Definition: Instructions.h:2664
llvm::VPDef::definedValues
ArrayRef< VPValue * > definedValues()
Returns an ArrayRef of the values defined by the VPDef.
Definition: VPlanValue.h:411
IV
static const uint32_t IV[8]
Definition: blake3_impl.h:85
llvm::SmallPtrSetImpl< Instruction * >
llvm::getLoadStorePointerOperand
const Value * getLoadStorePointerOperand(const Value *V)
A helper function that returns the pointer operand of a load or store instruction.
Definition: Instructions.h:5330
llvm::CallInst
This class represents a function call, abstracting a target machine's calling convention.
Definition: Instructions.h:1474
GEP
Hexagon Common GEP
Definition: HexagonCommonGEP.cpp:172
llvm::VPBasicBlock::phis
iterator_range< iterator > phis()
Returns an iterator range over the PHI-like recipes in the block.
Definition: VPlan.h:2077
llvm::VPBasicBlock::getFirstNonPhi
iterator getFirstNonPhi()
Return the position of the first non-phi node recipe in the block.
Definition: VPlan.cpp:220
llvm::VPBlockRecursiveTraversalWrapper
Helper for GraphTraits specialization that traverses through VPRegionBlocks.
Definition: VPlan.h:2413
llvm::SetVector
A vector that has set insertion semantics.
Definition: SetVector.h:40
llvm::VPlanTransforms::mergeReplicateRegions
static bool mergeReplicateRegions(VPlan &Plan)
Definition: VPlanTransforms.cpp:217
llvm::VPValue::getUnderlyingValue
Value * getUnderlyingValue()
Return the underlying Value attached to this VPValue.
Definition: VPlanValue.h:84
llvm::VPlan
VPlan models a candidate for vectorization, encoding various decisions take to produce efficient outp...
Definition: VPlan.h:2469
llvm::SetVector::pop_back_val
LLVM_NODISCARD T pop_back_val()
Definition: SetVector.h:232
llvm::VPRecipeBase::insertBefore
void insertBefore(VPRecipeBase *InsertPos)
Insert an unlinked recipe into a basic block immediately before the specified recipe.
Definition: VPlan.cpp:659
SetVector.h
llvm::sampleprof::Base
@ Base
Definition: Discriminator.h:58