LLVM  16.0.0git
VPlanTransforms.cpp
Go to the documentation of this file.
1 //===-- VPlanTransforms.cpp - Utility VPlan to VPlan transforms -----------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 ///
9 /// \file
10 /// This file implements a set of utility VPlan to VPlan transformations.
11 ///
12 //===----------------------------------------------------------------------===//
13 
14 #include "VPlanTransforms.h"
16 #include "llvm/ADT/SetVector.h"
19 #include "llvm/IR/Intrinsics.h"
20 
21 using namespace llvm;
22 
24  Loop *OrigLoop, VPlanPtr &Plan,
26  GetIntOrFpInductionDescriptor,
27  SmallPtrSetImpl<Instruction *> &DeadInstructions, ScalarEvolution &SE,
28  const TargetLibraryInfo &TLI) {
29 
31  RPOT(Plan->getEntry());
32  for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(RPOT)) {
33  VPRecipeBase *Term = VPBB->getTerminator();
34  auto EndIter = Term ? Term->getIterator() : VPBB->end();
35  // Introduce each ingredient into VPlan.
36  for (VPRecipeBase &Ingredient :
37  make_early_inc_range(make_range(VPBB->begin(), EndIter))) {
38 
39  VPValue *VPV = Ingredient.getVPSingleValue();
40  Instruction *Inst = cast<Instruction>(VPV->getUnderlyingValue());
41  if (DeadInstructions.count(Inst)) {
42  VPValue DummyValue;
43  VPV->replaceAllUsesWith(&DummyValue);
44  Ingredient.eraseFromParent();
45  continue;
46  }
47 
48  VPRecipeBase *NewRecipe = nullptr;
49  if (auto *VPPhi = dyn_cast<VPWidenPHIRecipe>(&Ingredient)) {
50  auto *Phi = cast<PHINode>(VPPhi->getUnderlyingValue());
51  if (const auto *II = GetIntOrFpInductionDescriptor(Phi)) {
52  VPValue *Start = Plan->getOrAddVPValue(II->getStartValue());
53  VPValue *Step =
54  vputils::getOrCreateVPValueForSCEVExpr(*Plan, II->getStep(), SE);
55  NewRecipe =
56  new VPWidenIntOrFpInductionRecipe(Phi, Start, Step, *II, true);
57  } else {
58  Plan->addVPValue(Phi, VPPhi);
59  continue;
60  }
61  } else {
62  assert(isa<VPInstruction>(&Ingredient) &&
63  "only VPInstructions expected here");
64  assert(!isa<PHINode>(Inst) && "phis should be handled above");
65  // Create VPWidenMemoryInstructionRecipe for loads and stores.
66  if (LoadInst *Load = dyn_cast<LoadInst>(Inst)) {
67  NewRecipe = new VPWidenMemoryInstructionRecipe(
68  *Load, Plan->getOrAddVPValue(getLoadStorePointerOperand(Inst)),
69  nullptr /*Mask*/, false /*Consecutive*/, false /*Reverse*/);
70  } else if (StoreInst *Store = dyn_cast<StoreInst>(Inst)) {
71  NewRecipe = new VPWidenMemoryInstructionRecipe(
72  *Store, Plan->getOrAddVPValue(getLoadStorePointerOperand(Inst)),
73  Plan->getOrAddVPValue(Store->getValueOperand()), nullptr /*Mask*/,
74  false /*Consecutive*/, false /*Reverse*/);
75  } else if (GetElementPtrInst *GEP = dyn_cast<GetElementPtrInst>(Inst)) {
76  NewRecipe = new VPWidenGEPRecipe(
77  GEP, Plan->mapToVPValues(GEP->operands()), OrigLoop);
78  } else if (CallInst *CI = dyn_cast<CallInst>(Inst)) {
79  NewRecipe =
80  new VPWidenCallRecipe(*CI, Plan->mapToVPValues(CI->args()),
81  getVectorIntrinsicIDForCall(CI, &TLI));
82  } else if (SelectInst *SI = dyn_cast<SelectInst>(Inst)) {
83  bool InvariantCond =
84  SE.isLoopInvariant(SE.getSCEV(SI->getOperand(0)), OrigLoop);
85  NewRecipe = new VPWidenSelectRecipe(
86  *SI, Plan->mapToVPValues(SI->operands()), InvariantCond);
87  } else {
88  NewRecipe =
89  new VPWidenRecipe(*Inst, Plan->mapToVPValues(Inst->operands()));
90  }
91  }
92 
93  NewRecipe->insertBefore(&Ingredient);
94  if (NewRecipe->getNumDefinedValues() == 1)
95  VPV->replaceAllUsesWith(NewRecipe->getVPSingleValue());
96  else
97  assert(NewRecipe->getNumDefinedValues() == 0 &&
98  "Only recpies with zero or one defined values expected");
99  Ingredient.eraseFromParent();
100  Plan->removeVPValueFor(Inst);
101  for (auto *Def : NewRecipe->definedValues()) {
102  Plan->addVPValue(Inst, Def);
103  }
104  }
105  }
106 }
107 
109  auto Iter = depth_first(
111  bool Changed = false;
112  // First, collect the operands of all predicated replicate recipes as seeds
113  // for sinking.
115  for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(Iter)) {
116  for (auto &Recipe : *VPBB) {
117  auto *RepR = dyn_cast<VPReplicateRecipe>(&Recipe);
118  if (!RepR || !RepR->isPredicated())
119  continue;
120  for (VPValue *Op : RepR->operands())
121  if (auto *Def = Op->getDefiningRecipe())
122  WorkList.insert(std::make_pair(RepR->getParent(), Def));
123  }
124  }
125 
126  // Try to sink each replicate or scalar IV steps recipe in the worklist.
127  while (!WorkList.empty()) {
128  VPBasicBlock *SinkTo;
129  VPRecipeBase *SinkCandidate;
130  std::tie(SinkTo, SinkCandidate) = WorkList.pop_back_val();
131  if (SinkCandidate->getParent() == SinkTo ||
132  SinkCandidate->mayHaveSideEffects() ||
133  SinkCandidate->mayReadOrWriteMemory())
134  continue;
135  if (auto *RepR = dyn_cast<VPReplicateRecipe>(SinkCandidate)) {
136  if (RepR->isUniform())
137  continue;
138  } else if (!isa<VPScalarIVStepsRecipe>(SinkCandidate))
139  continue;
140 
141  bool NeedsDuplicating = false;
142  // All recipe users of the sink candidate must be in the same block SinkTo
143  // or all users outside of SinkTo must be uniform-after-vectorization (
144  // i.e., only first lane is used) . In the latter case, we need to duplicate
145  // SinkCandidate.
146  auto CanSinkWithUser = [SinkTo, &NeedsDuplicating,
147  SinkCandidate](VPUser *U) {
148  auto *UI = dyn_cast<VPRecipeBase>(U);
149  if (!UI)
150  return false;
151  if (UI->getParent() == SinkTo)
152  return true;
153  NeedsDuplicating =
154  UI->onlyFirstLaneUsed(SinkCandidate->getVPSingleValue());
155  // We only know how to duplicate VPRecipeRecipes for now.
156  return NeedsDuplicating && isa<VPReplicateRecipe>(SinkCandidate);
157  };
158  if (!all_of(SinkCandidate->getVPSingleValue()->users(), CanSinkWithUser))
159  continue;
160 
161  if (NeedsDuplicating) {
162  Instruction *I = cast<Instruction>(
163  cast<VPReplicateRecipe>(SinkCandidate)->getUnderlyingValue());
164  auto *Clone =
165  new VPReplicateRecipe(I, SinkCandidate->operands(), true, false);
166  // TODO: add ".cloned" suffix to name of Clone's VPValue.
167 
168  Clone->insertBefore(SinkCandidate);
170  SinkCandidate->getVPSingleValue()->users());
171  for (auto *U : Users) {
172  auto *UI = cast<VPRecipeBase>(U);
173  if (UI->getParent() == SinkTo)
174  continue;
175 
176  for (unsigned Idx = 0; Idx != UI->getNumOperands(); Idx++) {
177  if (UI->getOperand(Idx) != SinkCandidate->getVPSingleValue())
178  continue;
179  UI->setOperand(Idx, Clone);
180  }
181  }
182  }
183  SinkCandidate->moveBefore(*SinkTo, SinkTo->getFirstNonPhi());
184  for (VPValue *Op : SinkCandidate->operands())
185  if (auto *Def = Op->getDefiningRecipe())
186  WorkList.insert(std::make_pair(SinkTo, Def));
187  Changed = true;
188  }
189  return Changed;
190 }
191 
192 /// If \p R is a region with a VPBranchOnMaskRecipe in the entry block, return
193 /// the mask.
195  auto *EntryBB = dyn_cast<VPBasicBlock>(R->getEntry());
196  if (!EntryBB || EntryBB->size() != 1 ||
197  !isa<VPBranchOnMaskRecipe>(EntryBB->begin()))
198  return nullptr;
199 
200  return cast<VPBranchOnMaskRecipe>(&*EntryBB->begin())->getOperand(0);
201 }
202 
203 /// If \p R is a triangle region, return the 'then' block of the triangle.
205  auto *EntryBB = cast<VPBasicBlock>(R->getEntry());
206  if (EntryBB->getNumSuccessors() != 2)
207  return nullptr;
208 
209  auto *Succ0 = dyn_cast<VPBasicBlock>(EntryBB->getSuccessors()[0]);
210  auto *Succ1 = dyn_cast<VPBasicBlock>(EntryBB->getSuccessors()[1]);
211  if (!Succ0 || !Succ1)
212  return nullptr;
213 
214  if (Succ0->getNumSuccessors() + Succ1->getNumSuccessors() != 1)
215  return nullptr;
216  if (Succ0->getSingleSuccessor() == Succ1)
217  return Succ0;
218  if (Succ1->getSingleSuccessor() == Succ0)
219  return Succ1;
220  return nullptr;
221 }
222 
224  SetVector<VPRegionBlock *> DeletedRegions;
225  bool Changed = false;
226 
227  // Collect region blocks to process up-front, to avoid iterator invalidation
228  // issues while merging regions.
229  SmallVector<VPRegionBlock *, 8> CandidateRegions(
230  VPBlockUtils::blocksOnly<VPRegionBlock>(depth_first(
232 
233  // Check if Base is a predicated triangle, followed by an empty block,
234  // followed by another predicate triangle. If that's the case, move the
235  // recipes from the first to the second triangle.
236  for (VPRegionBlock *Region1 : CandidateRegions) {
237  if (DeletedRegions.contains(Region1))
238  continue;
239  auto *MiddleBasicBlock =
240  dyn_cast_or_null<VPBasicBlock>(Region1->getSingleSuccessor());
241  if (!MiddleBasicBlock || !MiddleBasicBlock->empty())
242  continue;
243 
244  auto *Region2 =
245  dyn_cast_or_null<VPRegionBlock>(MiddleBasicBlock->getSingleSuccessor());
246  if (!Region2)
247  continue;
248 
249  VPValue *Mask1 = getPredicatedMask(Region1);
250  VPValue *Mask2 = getPredicatedMask(Region2);
251  if (!Mask1 || Mask1 != Mask2)
252  continue;
253  VPBasicBlock *Then1 = getPredicatedThenBlock(Region1);
254  VPBasicBlock *Then2 = getPredicatedThenBlock(Region2);
255  if (!Then1 || !Then2)
256  continue;
257 
258  assert(Mask1 && Mask2 && "both region must have conditions");
259 
260  // Note: No fusion-preventing memory dependencies are expected in either
261  // region. Such dependencies should be rejected during earlier dependence
262  // checks, which guarantee accesses can be re-ordered for vectorization.
263  //
264  // Move recipes to the successor region.
265  for (VPRecipeBase &ToMove : make_early_inc_range(reverse(*Then1)))
266  ToMove.moveBefore(*Then2, Then2->getFirstNonPhi());
267 
268  auto *Merge1 = cast<VPBasicBlock>(Then1->getSingleSuccessor());
269  auto *Merge2 = cast<VPBasicBlock>(Then2->getSingleSuccessor());
270 
271  // Move VPPredInstPHIRecipes from the merge block to the successor region's
272  // merge block. Update all users inside the successor region to use the
273  // original values.
274  for (VPRecipeBase &Phi1ToMove : make_early_inc_range(reverse(*Merge1))) {
275  VPValue *PredInst1 =
276  cast<VPPredInstPHIRecipe>(&Phi1ToMove)->getOperand(0);
277  VPValue *Phi1ToMoveV = Phi1ToMove.getVPSingleValue();
278  SmallVector<VPUser *> Users(Phi1ToMoveV->users());
279  for (VPUser *U : Users) {
280  auto *UI = dyn_cast<VPRecipeBase>(U);
281  if (!UI || UI->getParent() != Then2)
282  continue;
283  for (unsigned I = 0, E = U->getNumOperands(); I != E; ++I) {
284  if (Phi1ToMoveV != U->getOperand(I))
285  continue;
286  U->setOperand(I, PredInst1);
287  }
288  }
289 
290  Phi1ToMove.moveBefore(*Merge2, Merge2->begin());
291  }
292 
293  // Finally, remove the first region.
294  for (VPBlockBase *Pred : make_early_inc_range(Region1->getPredecessors())) {
295  VPBlockUtils::disconnectBlocks(Pred, Region1);
296  VPBlockUtils::connectBlocks(Pred, MiddleBasicBlock);
297  }
298  VPBlockUtils::disconnectBlocks(Region1, MiddleBasicBlock);
299  DeletedRegions.insert(Region1);
300  }
301 
302  for (VPRegionBlock *ToDelete : DeletedRegions)
303  delete ToDelete;
304  return Changed;
305 }
306 
308  for (auto &Phi : Plan.getVectorLoopRegion()->getEntryBasicBlock()->phis()) {
309  auto *IV = dyn_cast<VPWidenIntOrFpInductionRecipe>(&Phi);
310  if (!IV || IV->getTruncInst())
311  continue;
312 
313  // A sequence of IR Casts has potentially been recorded for IV, which
314  // *must be bypassed* when the IV is vectorized, because the vectorized IV
315  // will produce the desired casted value. This sequence forms a def-use
316  // chain and is provided in reverse order, ending with the cast that uses
317  // the IV phi. Search for the recipe of the last cast in the chain and
318  // replace it with the original IV. Note that only the final cast is
319  // expected to have users outside the cast-chain and the dead casts left
320  // over will be cleaned up later.
321  auto &Casts = IV->getInductionDescriptor().getCastInsts();
322  VPValue *FindMyCast = IV;
323  for (Instruction *IRCast : reverse(Casts)) {
324  VPRecipeBase *FoundUserCast = nullptr;
325  for (auto *U : FindMyCast->users()) {
326  auto *UserCast = cast<VPRecipeBase>(U);
327  if (UserCast->getNumDefinedValues() == 1 &&
328  UserCast->getVPSingleValue()->getUnderlyingValue() == IRCast) {
329  FoundUserCast = UserCast;
330  break;
331  }
332  }
333  FindMyCast = FoundUserCast->getVPSingleValue();
334  }
335  FindMyCast->replaceAllUsesWith(IV);
336  }
337 }
338 
340  VPCanonicalIVPHIRecipe *CanonicalIV = Plan.getCanonicalIV();
341  VPWidenCanonicalIVRecipe *WidenNewIV = nullptr;
342  for (VPUser *U : CanonicalIV->users()) {
343  WidenNewIV = dyn_cast<VPWidenCanonicalIVRecipe>(U);
344  if (WidenNewIV)
345  break;
346  }
347 
348  if (!WidenNewIV)
349  return;
350 
351  VPBasicBlock *HeaderVPBB = Plan.getVectorLoopRegion()->getEntryBasicBlock();
352  for (VPRecipeBase &Phi : HeaderVPBB->phis()) {
353  auto *WidenOriginalIV = dyn_cast<VPWidenIntOrFpInductionRecipe>(&Phi);
354 
355  if (!WidenOriginalIV || !WidenOriginalIV->isCanonical() ||
356  WidenOriginalIV->getScalarType() != WidenNewIV->getScalarType())
357  continue;
358 
359  // Replace WidenNewIV with WidenOriginalIV if WidenOriginalIV provides
360  // everything WidenNewIV's users need. That is, WidenOriginalIV will
361  // generate a vector phi or all users of WidenNewIV demand the first lane
362  // only.
363  if (WidenOriginalIV->needsVectorIV() ||
364  vputils::onlyFirstLaneUsed(WidenNewIV)) {
365  WidenNewIV->replaceAllUsesWith(WidenOriginalIV);
366  WidenNewIV->eraseFromParent();
367  return;
368  }
369  }
370 }
371 
374  RPOT(Plan.getEntry());
375 
376  for (VPBasicBlock *VPBB : reverse(VPBlockUtils::blocksOnly<VPBasicBlock>(RPOT))) {
377  // The recipes in the block are processed in reverse order, to catch chains
378  // of dead recipes.
379  for (VPRecipeBase &R : make_early_inc_range(reverse(*VPBB))) {
380  if (R.mayHaveSideEffects() || any_of(R.definedValues(), [](VPValue *V) {
381  return V->getNumUsers() > 0;
382  }))
383  continue;
384  R.eraseFromParent();
385  }
386  }
387 }
388 
391  VPBasicBlock *HeaderVPBB = Plan.getVectorLoopRegion()->getEntryBasicBlock();
392  bool HasOnlyVectorVFs = !Plan.hasVF(ElementCount::getFixed(1));
393  for (VPRecipeBase &Phi : HeaderVPBB->phis()) {
394  auto *WideIV = dyn_cast<VPWidenIntOrFpInductionRecipe>(&Phi);
395  if (!WideIV)
396  continue;
397  if (HasOnlyVectorVFs && none_of(WideIV->users(), [WideIV](VPUser *U) {
398  return U->usesScalars(WideIV);
399  }))
400  continue;
401 
402  auto IP = HeaderVPBB->getFirstNonPhi();
403  VPCanonicalIVPHIRecipe *CanonicalIV = Plan.getCanonicalIV();
404  Type *ResultTy = WideIV->getPHINode()->getType();
405  if (Instruction *TruncI = WideIV->getTruncInst())
406  ResultTy = TruncI->getType();
407  const InductionDescriptor &ID = WideIV->getInductionDescriptor();
408  VPValue *Step =
409  vputils::getOrCreateVPValueForSCEVExpr(Plan, ID.getStep(), SE);
410  VPValue *BaseIV = CanonicalIV;
411  if (!CanonicalIV->isCanonical(ID, ResultTy)) {
412  BaseIV = new VPDerivedIVRecipe(ID, WideIV->getStartValue(), CanonicalIV,
413  Step, ResultTy);
414  HeaderVPBB->insert(BaseIV->getDefiningRecipe(), IP);
415  }
416 
417  VPScalarIVStepsRecipe *Steps = new VPScalarIVStepsRecipe(ID, BaseIV, Step);
418  HeaderVPBB->insert(Steps, IP);
419 
420  // Update scalar users of IV to use Step instead. Use SetVector to ensure
421  // the list of users doesn't contain duplicates.
422  SetVector<VPUser *> Users(WideIV->user_begin(), WideIV->user_end());
423  for (VPUser *U : Users) {
424  if (HasOnlyVectorVFs && !U->usesScalars(WideIV))
425  continue;
426  for (unsigned I = 0, E = U->getNumOperands(); I != E; I++) {
427  if (U->getOperand(I) != WideIV)
428  continue;
429  U->setOperand(I, Steps);
430  }
431  }
432  }
433 }
434 
437 
438  for (VPRecipeBase &R :
440  auto *ExpR = dyn_cast<VPExpandSCEVRecipe>(&R);
441  if (!ExpR)
442  continue;
443 
444  auto I = SCEV2VPV.insert({ExpR->getSCEV(), ExpR});
445  if (I.second)
446  continue;
447  ExpR->replaceAllUsesWith(I.first->second);
448  ExpR->eraseFromParent();
449  }
450 }
llvm
This is an optimization pass for GlobalISel generic memory operations.
Definition: AddressRanges.h:18
llvm::tgtok::Def
@ Def
Definition: TGLexer.h:50
llvm::VPCanonicalIVPHIRecipe
Canonical scalar induction phi of the vector loop.
Definition: VPlan.h:1830
llvm::none_of
bool none_of(R &&Range, UnaryPredicate P)
Provide wrappers to std::none_of which take ranges instead of having to pass begin/end explicitly.
Definition: STLExtras.h:1748
llvm::make_range
iterator_range< T > make_range(T x, T y)
Convenience function for iterating over sub-ranges.
Definition: iterator_range.h:53
llvm::VPRecipeBase::eraseFromParent
iplist< VPRecipeBase >::iterator eraseFromParent()
This method unlinks 'this' from the containing basic block and deletes it.
Definition: VPlanRecipes.cpp:176
llvm::User::operands
op_range operands()
Definition: User.h:242
getPredicatedMask
VPValue * getPredicatedMask(VPRegionBlock *R)
If R is a region with a VPBranchOnMaskRecipe in the entry block, return the mask.
Definition: VPlanTransforms.cpp:194
llvm::getVectorIntrinsicIDForCall
Intrinsic::ID getVectorIntrinsicIDForCall(const CallInst *CI, const TargetLibraryInfo *TLI)
Returns intrinsic ID for call.
Definition: VectorUtils.cpp:135
llvm::VPBasicBlock
VPBasicBlock serves as the leaf of the Hierarchical Control-Flow Graph.
Definition: VPlan.h:2065
llvm::Loop
Represents a single loop in the control flow graph.
Definition: LoopInfo.h:547
llvm::VPlanTransforms::removeDeadRecipes
static void removeDeadRecipes(VPlan &Plan)
Definition: VPlanTransforms.cpp:372
llvm::SmallVector
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
Definition: SmallVector.h:1199
llvm::VPDef::getVPSingleValue
VPValue * getVPSingleValue()
Returns the only VPValue defined by the VPDef.
Definition: VPlanValue.h:404
ToRemove
ReachingDefAnalysis InstSet & ToRemove
Definition: ARMLowOverheadLoops.cpp:547
llvm::ScalarEvolution
The main scalar evolution driver.
Definition: ScalarEvolution.h:449
llvm::VPlan::getCanonicalIV
VPCanonicalIVPHIRecipe * getCanonicalIV()
Returns the canonical induction recipe of the vector loop.
Definition: VPlan.h:2742
llvm::Type
The instances of the Type class are immutable: once they are created, they are never changed.
Definition: Type.h:45
llvm::VPlanPtr
std::unique_ptr< VPlan > VPlanPtr
Definition: VPlan.h:103
llvm::VPWidenCallRecipe
A recipe for widening Call instructions.
Definition: VPlan.h:952
llvm::VPUser::operands
operand_range operands()
Definition: VPlanValue.h:296
llvm::SPII::Load
@ Load
Definition: SparcInstrInfo.h:32
llvm::VPValue::users
user_range users()
Definition: VPlanValue.h:167
llvm::VPReplicateRecipe
VPReplicateRecipe replicates a given instruction producing multiple scalar copies of the original sca...
Definition: VPlan.h:1545
llvm::VPBlockBase
VPBlockBase is the building block of the Hierarchical Control-Flow Graph.
Definition: VPlan.h:389
llvm::VPlanTransforms::removeRedundantCanonicalIVs
static void removeRedundantCanonicalIVs(VPlan &Plan)
Try to replace VPWidenCanonicalIVRecipes with a widened canonical IV recipe, if it exists.
Definition: VPlanTransforms.cpp:339
llvm::VPlan::hasVF
bool hasVF(ElementCount VF)
Definition: VPlan.h:2653
llvm::all_of
bool all_of(R &&range, UnaryPredicate P)
Provide wrappers to std::all_of which take ranges instead of having to pass begin/end explicitly.
Definition: STLExtras.h:1734
llvm::InductionDescriptor
A struct for saving information about induction variables.
Definition: IVDescriptors.h:306
llvm::vputils::getOrCreateVPValueForSCEVExpr
VPValue * getOrCreateVPValueForSCEVExpr(VPlan &Plan, const SCEV *Expr, ScalarEvolution &SE)
Get or create a VPValue that corresponds to the expansion of Expr.
Definition: VPlan.cpp:1108
E
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
Intrinsics.h
SI
@ SI
Definition: SIInstrInfo.cpp:7966
llvm::VPRecipeBase::moveBefore
void moveBefore(VPBasicBlock &BB, iplist< VPRecipeBase >::iterator I)
Unlink this recipe and insert into BB before I.
Definition: VPlanRecipes.cpp:186
IP
Definition: NVPTXLowerArgs.cpp:168
llvm::SPII::Store
@ Store
Definition: SparcInstrInfo.h:33
llvm::Instruction
Definition: Instruction.h:42
llvm::VPlanTransforms::optimizeInductions
static void optimizeInductions(VPlan &Plan, ScalarEvolution &SE)
If any user of a VPWidenIntOrFpInductionRecipe needs scalar values, provide them by building scalar s...
Definition: VPlanTransforms.cpp:389
llvm::VPValue::replaceAllUsesWith
void replaceAllUsesWith(VPValue *New)
Definition: VPlan.cpp:989
llvm::VPValue
Definition: VPlanValue.h:44
llvm::VPWidenCanonicalIVRecipe
A Recipe for widening the canonical induction variable of the vector loop.
Definition: VPlan.h:1915
llvm::VPDef::getNumDefinedValues
unsigned getNumDefinedValues() const
Returns the number of values defined by the VPDef.
Definition: VPlanValue.h:431
IVDescriptors.h
llvm::SetVector::empty
bool empty() const
Determine if the SetVector is empty or not.
Definition: SetVector.h:72
llvm::VPUser::getNumOperands
unsigned getNumOperands() const
Definition: VPlanValue.h:270
llvm::LinearPolySize< ElementCount >::getFixed
static ElementCount getFixed(ScalarTy MinVal)
Definition: TypeSize.h:283
llvm::CallingConv::ID
unsigned ID
LLVM IR allows to use arbitrary numbers as calling convention identifiers.
Definition: CallingConv.h:24
llvm::VPRegionBlock
VPRegionBlock represents a collection of VPBasicBlocks and VPRegionBlocks which form a Single-Entry-S...
Definition: VPlan.h:2189
llvm::ScalarEvolution::getSCEV
const SCEV * getSCEV(Value *V)
Return a SCEV expression for the full generality of the specified expression.
Definition: ScalarEvolution.cpp:4442
llvm::function_ref
An efficient, type-erasing, non-owning reference to a callable.
Definition: STLFunctionalExtras.h:36
llvm::VPUser::setOperand
void setOperand(unsigned I, VPValue *New)
Definition: VPlanValue.h:276
VectorUtils.h
llvm::VPRecipeBase::mayHaveSideEffects
bool mayHaveSideEffects() const
Returns true if the recipe may have side-effects.
Definition: VPlanRecipes.cpp:106
llvm::StoreInst
An instruction for storing to memory.
Definition: Instructions.h:298
llvm::SetVector::contains
bool contains(const key_type &key) const
Check if the SetVector contains the given key.
Definition: SetVector.h:209
llvm::VPWidenRecipe
VPWidenRecipe is a recipe for producing a copy of vector type its ingredient.
Definition: VPlan.h:924
llvm::DenseMap
Definition: DenseMap.h:714
llvm::VPlanTransforms::sinkScalarOperands
static bool sinkScalarOperands(VPlan &Plan)
Definition: VPlanTransforms.cpp:108
I
#define I(x, y, z)
Definition: MD5.cpp:58
llvm::VPBlockUtils::connectBlocks
static void connectBlocks(VPBlockBase *From, VPBlockBase *To)
Connect VPBlockBases From and To bi-directionally.
Definition: VPlan.h:2899
llvm::GetElementPtrInst
an instruction for type-safe pointer arithmetic to access elements of arrays and structs
Definition: Instructions.h:929
llvm::make_early_inc_range
iterator_range< early_inc_iterator_impl< detail::IterOfRange< RangeT > > > make_early_inc_range(RangeT &&Range)
Make a range that does early increment to allow mutation of the underlying range without disrupting i...
Definition: STLExtras.h:716
llvm::VPlanTransforms::VPInstructionsToVPRecipes
static void VPInstructionsToVPRecipes(Loop *OrigLoop, VPlanPtr &Plan, function_ref< const InductionDescriptor *(PHINode *)> GetIntOrFpInductionDescriptor, SmallPtrSetImpl< Instruction * > &DeadInstructions, ScalarEvolution &SE, const TargetLibraryInfo &TLI)
Replaces the VPInstructions in Plan with corresponding widen recipes.
Definition: VPlanTransforms.cpp:23
llvm::VPBlockUtils::disconnectBlocks
static void disconnectBlocks(VPBlockBase *From, VPBlockBase *To)
Disconnect VPBlockBases From and To bi-directionally.
Definition: VPlan.h:2910
llvm::VPlan::getEntry
VPBlockBase * getEntry()
Definition: VPlan.h:2621
assert
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
VPlanTransforms.h
llvm::VPWidenIntOrFpInductionRecipe
A recipe for handling phi nodes of integer and floating-point inductions, producing their vector valu...
Definition: VPlan.h:1055
llvm::VPWidenGEPRecipe
A recipe for handling GEP instructions.
Definition: VPlan.h:1014
llvm::SelectInst
This class represents the LLVM 'select' instruction.
Definition: Instructions.h:1735
llvm::VPWidenMemoryInstructionRecipe
A Recipe for widening load/store operations.
Definition: VPlan.h:1702
llvm::VPWidenSelectRecipe
A recipe for widening select instructions.
Definition: VPlan.h:983
llvm::SmallPtrSetImpl::count
size_type count(ConstPtrType Ptr) const
count - Return 1 if the specified pointer is in the set, 0 otherwise.
Definition: SmallPtrSet.h:383
llvm::SetVector::insert
bool insert(const value_type &X)
Insert a new element into the SetVector.
Definition: SetVector.h:141
llvm::VPlan::getVectorLoopRegion
VPRegionBlock * getVectorLoopRegion()
Returns the VPRegionBlock of the vector loop.
Definition: VPlan.h:2734
llvm::SetVector::pop_back_val
T pop_back_val()
Definition: SetVector.h:232
getPredicatedThenBlock
static VPBasicBlock * getPredicatedThenBlock(VPRegionBlock *R)
If R is a triangle region, return the 'then' block of the triangle.
Definition: VPlanTransforms.cpp:204
llvm::any_of
bool any_of(R &&range, UnaryPredicate P)
Provide wrappers to std::any_of which take ranges instead of having to pass begin/end explicitly.
Definition: STLExtras.h:1741
llvm::VPBlockBase::getSingleSuccessor
VPBlockBase * getSingleSuccessor() const
Definition: VPlan.h:496
llvm::VPCanonicalIVPHIRecipe::isCanonical
bool isCanonical(const InductionDescriptor &ID, Type *Ty) const
Check if the induction described by ID is canonical, i.e.
Definition: VPlanRecipes.cpp:1056
llvm::VPRecipeBase::getParent
VPBasicBlock * getParent()
Definition: VPlan.h:682
llvm::LoadInst
An instruction for reading from memory.
Definition: Instructions.h:174
llvm::DenseMapBase< DenseMap< KeyT, ValueT, DenseMapInfo< KeyT >, llvm::detail::DenseMapPair< KeyT, ValueT > >, KeyT, ValueT, DenseMapInfo< KeyT >, llvm::detail::DenseMapPair< KeyT, ValueT > >::insert
std::pair< iterator, bool > insert(const std::pair< KeyT, ValueT > &KV)
Definition: DenseMap.h:207
llvm::VPBlockBase::getEntryBasicBlock
const VPBasicBlock * getEntryBasicBlock() const
Definition: VPlan.cpp:147
llvm::vputils::onlyFirstLaneUsed
bool onlyFirstLaneUsed(VPValue *Def)
Returns true if only the first lane of Def is used.
Definition: VPlan.cpp:1103
llvm::ScalarEvolution::isLoopInvariant
bool isLoopInvariant(const SCEV *S, const Loop *L)
Return true if the value of the given SCEV is unchanging in the specified loop.
Definition: ScalarEvolution.cpp:13637
llvm::depth_first
iterator_range< df_iterator< T > > depth_first(const T &G)
Definition: DepthFirstIterator.h:230
llvm::VPlanTransforms::removeRedundantExpandSCEVRecipes
static void removeRedundantExpandSCEVRecipes(VPlan &Plan)
Remove redundant EpxandSCEVRecipes in Plan's entry block by replacing them with already existing reci...
Definition: VPlanTransforms.cpp:435
llvm::AMDGPU::SendMsg::Op
Op
Definition: SIDefines.h:351
llvm::VPScalarIVStepsRecipe
A recipe for handling phi nodes of integer and floating-point inductions, producing their scalar valu...
Definition: VPlan.h:2018
llvm::VPRecipeBase
VPRecipeBase is a base class modeling a sequence of one or more output IR instructions.
Definition: VPlan.h:663
llvm::TargetLibraryInfo
Provides information about what library functions are available for the current target.
Definition: TargetLibraryInfo.h:226
llvm::VPRecipeBase::mayReadOrWriteMemory
bool mayReadOrWriteMemory() const
Returns true if the recipe may read from or write to memory.
Definition: VPlan.h:752
llvm::VPWidenCanonicalIVRecipe::getScalarType
const Type * getScalarType() const
Returns the scalar type of the induction.
Definition: VPlan.h:1950
llvm::ReversePostOrderTraversal
Definition: PostOrderIterator.h:291
llvm::VPBasicBlock::insert
void insert(VPRecipeBase *Recipe, iterator InsertPt)
Definition: VPlan.h:2124
llvm::VPUser
This class augments VPValue with operands which provide the inverse def-use edges from VPValue's user...
Definition: VPlanValue.h:221
llvm::VPlanTransforms::removeRedundantInductionCasts
static void removeRedundantInductionCasts(VPlan &Plan)
Remove redundant casts of inductions.
Definition: VPlanTransforms.cpp:307
PostOrderIterator.h
llvm::VPUser::getOperand
VPValue * getOperand(unsigned N) const
Definition: VPlanValue.h:271
Users
iv Induction Variable Users
Definition: IVUsers.cpp:48
llvm::PHINode
Definition: Instructions.h:2697
llvm::VPDef::definedValues
ArrayRef< VPValue * > definedValues()
Returns an ArrayRef of the values defined by the VPDef.
Definition: VPlanValue.h:426
IV
static const uint32_t IV[8]
Definition: blake3_impl.h:85
llvm::reverse
auto reverse(ContainerTy &&C)
Definition: STLExtras.h:485
llvm::SmallPtrSetImpl< Instruction * >
llvm::getLoadStorePointerOperand
const Value * getLoadStorePointerOperand(const Value *V)
A helper function that returns the pointer operand of a load or store instruction.
Definition: Instructions.h:5358
llvm::VPUser::usesScalars
virtual bool usesScalars(const VPValue *Op) const
Returns true if the VPUser uses scalars of operand Op.
Definition: VPlanValue.h:306
llvm::CallInst
This class represents a function call, abstracting a target machine's calling convention.
Definition: Instructions.h:1474
GEP
Hexagon Common GEP
Definition: HexagonCommonGEP.cpp:171
llvm::VPValue::getDefiningRecipe
VPRecipeBase * getDefiningRecipe()
Returns the recipe defining this VPValue or nullptr if it is not defined by a recipe,...
Definition: VPlan.cpp:112
llvm::VPBasicBlock::phis
iterator_range< iterator > phis()
Returns an iterator range over the PHI-like recipes in the block.
Definition: VPlan.h:2143
llvm::M68kBeads::Term
@ Term
Definition: M68kBaseInfo.h:71
llvm::VPBasicBlock::getFirstNonPhi
iterator getFirstNonPhi()
Return the position of the first non-phi node recipe in the block.
Definition: VPlan.cpp:205
llvm::VPBlockRecursiveTraversalWrapper
Helper for GraphTraits specialization that traverses through VPRegionBlocks.
Definition: VPlan.h:2488
llvm::SetVector
A vector that has set insertion semantics.
Definition: SetVector.h:40
llvm::VPlanTransforms::mergeReplicateRegions
static bool mergeReplicateRegions(VPlan &Plan)
Definition: VPlanTransforms.cpp:223
llvm::VPValue::getUnderlyingValue
Value * getUnderlyingValue()
Return the underlying Value attached to this VPValue.
Definition: VPlanValue.h:84
llvm::VPlan
VPlan models a candidate for vectorization, encoding various decisions take to produce efficient outp...
Definition: VPlan.h:2544
llvm::VPRecipeBase::insertBefore
void insertBefore(VPRecipeBase *InsertPos)
Insert an unlinked recipe into a basic block immediately before the specified recipe.
Definition: VPlanRecipes.cpp:146
SetVector.h
llvm::VPDerivedIVRecipe
A recipe for converting the canonical IV value to the corresponding value of an IV with different sta...
Definition: VPlan.h:1959