LLVM  16.0.0git
VPlan.cpp
Go to the documentation of this file.
1 //===- VPlan.cpp - Vectorizer Plan ----------------------------------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 ///
9 /// \file
10 /// This is the LLVM vectorization plan. It represents a candidate for
11 /// vectorization, allowing to plan and optimize how to vectorize a given loop
12 /// before generating LLVM-IR.
13 /// The vectorizer uses vectorization plans to estimate the costs of potential
14 /// candidates and if profitable to execute the desired plan, generating vector
15 /// LLVM-IR code.
16 ///
17 //===----------------------------------------------------------------------===//
18 
19 #include "VPlan.h"
20 #include "VPlanDominatorTree.h"
23 #include "llvm/ADT/STLExtras.h"
24 #include "llvm/ADT/SmallVector.h"
25 #include "llvm/ADT/Twine.h"
26 #include "llvm/Analysis/LoopInfo.h"
27 #include "llvm/IR/BasicBlock.h"
28 #include "llvm/IR/CFG.h"
29 #include "llvm/IR/IRBuilder.h"
30 #include "llvm/IR/Instruction.h"
31 #include "llvm/IR/Instructions.h"
32 #include "llvm/IR/Type.h"
33 #include "llvm/IR/Value.h"
34 #include "llvm/Support/Casting.h"
36 #include "llvm/Support/Debug.h"
43 #include <cassert>
44 #include <string>
45 #include <vector>
46 
47 using namespace llvm;
49 
50 #define DEBUG_TYPE "vplan"
51 
52 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
54  const VPInstruction *Instr = dyn_cast<VPInstruction>(&V);
56  (Instr && Instr->getParent()) ? Instr->getParent()->getPlan() : nullptr);
57  V.print(OS, SlotTracker);
58  return OS;
59 }
60 #endif
61 
63  const ElementCount &VF) const {
64  switch (LaneKind) {
66  // Lane = RuntimeVF - VF.getKnownMinValue() + Lane
67  return Builder.CreateSub(getRuntimeVF(Builder, Builder.getInt32Ty(), VF),
68  Builder.getInt32(VF.getKnownMinValue() - Lane));
70  return Builder.getInt32(Lane);
71  }
72  llvm_unreachable("Unknown lane kind");
73 }
74 
75 VPValue::VPValue(const unsigned char SC, Value *UV, VPDef *Def)
76  : SubclassID(SC), UnderlyingVal(UV), Def(Def) {
77  if (Def)
78  Def->addDefinedValue(this);
79 }
80 
82  assert(Users.empty() && "trying to delete a VPValue with remaining users");
83  if (Def)
84  Def->removeDefinedValue(this);
85 }
86 
87 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
89  if (const VPRecipeBase *R = dyn_cast_or_null<VPRecipeBase>(Def))
90  R->print(OS, "", SlotTracker);
91  else
93 }
94 
95 void VPValue::dump() const {
96  const VPRecipeBase *Instr = dyn_cast_or_null<VPRecipeBase>(this->Def);
98  (Instr && Instr->getParent()) ? Instr->getParent()->getPlan() : nullptr);
100  dbgs() << "\n";
101 }
102 
103 void VPDef::dump() const {
104  const VPRecipeBase *Instr = dyn_cast_or_null<VPRecipeBase>(this);
106  (Instr && Instr->getParent()) ? Instr->getParent()->getPlan() : nullptr);
107  print(dbgs(), "", SlotTracker);
108  dbgs() << "\n";
109 }
110 #endif
111 
113  return cast_or_null<VPRecipeBase>(Def);
114 }
115 
117  return cast_or_null<VPRecipeBase>(Def);
118 }
119 
120 // Get the top-most entry block of \p Start. This is the entry block of the
121 // containing VPlan. This function is templated to support both const and non-const blocks
122 template <typename T> static T *getPlanEntry(T *Start) {
123  T *Next = Start;
124  T *Current = Start;
125  while ((Next = Next->getParent()))
126  Current = Next;
127 
128  SmallSetVector<T *, 8> WorkList;
129  WorkList.insert(Current);
130 
131  for (unsigned i = 0; i < WorkList.size(); i++) {
132  T *Current = WorkList[i];
133  if (Current->getNumPredecessors() == 0)
134  return Current;
135  auto &Predecessors = Current->getPredecessors();
136  WorkList.insert(Predecessors.begin(), Predecessors.end());
137  }
138 
139  llvm_unreachable("VPlan without any entry node without predecessors");
140 }
141 
142 VPlan *VPBlockBase::getPlan() { return getPlanEntry(this)->Plan; }
143 
144 const VPlan *VPBlockBase::getPlan() const { return getPlanEntry(this)->Plan; }
145 
146 /// \return the VPBasicBlock that is the entry of Block, possibly indirectly.
148  const VPBlockBase *Block = this;
149  while (const VPRegionBlock *Region = dyn_cast<VPRegionBlock>(Block))
150  Block = Region->getEntry();
151  return cast<VPBasicBlock>(Block);
152 }
153 
155  VPBlockBase *Block = this;
156  while (VPRegionBlock *Region = dyn_cast<VPRegionBlock>(Block))
157  Block = Region->getEntry();
158  return cast<VPBasicBlock>(Block);
159 }
160 
161 void VPBlockBase::setPlan(VPlan *ParentPlan) {
162  assert(ParentPlan->getEntry() == this &&
163  "Can only set plan on its entry block.");
164  Plan = ParentPlan;
165 }
166 
167 /// \return the VPBasicBlock that is the exit of Block, possibly indirectly.
169  const VPBlockBase *Block = this;
170  while (const VPRegionBlock *Region = dyn_cast<VPRegionBlock>(Block))
171  Block = Region->getExiting();
172  return cast<VPBasicBlock>(Block);
173 }
174 
176  VPBlockBase *Block = this;
177  while (VPRegionBlock *Region = dyn_cast<VPRegionBlock>(Block))
178  Block = Region->getExiting();
179  return cast<VPBasicBlock>(Block);
180 }
181 
183  if (!Successors.empty() || !Parent)
184  return this;
185  assert(Parent->getExiting() == this &&
186  "Block w/o successors not the exiting block of its parent.");
187  return Parent->getEnclosingBlockWithSuccessors();
188 }
189 
191  if (!Predecessors.empty() || !Parent)
192  return this;
193  assert(Parent->getEntry() == this &&
194  "Block w/o predecessors not the entry of its parent.");
195  return Parent->getEnclosingBlockWithPredecessors();
196 }
197 
200 
201  for (VPBlockBase *Block : Blocks)
202  delete Block;
203 }
204 
206  iterator It = begin();
207  while (It != end() && It->isPhi())
208  It++;
209  return It;
210 }
211 
213  if (!Def->hasDefiningRecipe())
214  return Def->getLiveInIRValue();
215 
216  if (hasScalarValue(Def, Instance)) {
217  return Data
218  .PerPartScalars[Def][Instance.Part][Instance.Lane.mapToCacheIndex(VF)];
219  }
220 
222  auto *VecPart = Data.PerPartOutput[Def][Instance.Part];
223  if (!VecPart->getType()->isVectorTy()) {
224  assert(Instance.Lane.isFirstLane() && "cannot get lane > 0 for scalar");
225  return VecPart;
226  }
227  // TODO: Cache created scalar values.
228  Value *Lane = Instance.Lane.getAsRuntimeExpr(Builder, VF);
229  auto *Extract = Builder.CreateExtractElement(VecPart, Lane);
230  // set(Def, Extract, Instance);
231  return Extract;
232 }
234  VPRegionBlock *LoopRegion = R->getParent()->getEnclosingLoopRegion();
235  return VPBB2IRBB[LoopRegion->getPreheaderVPBB()];
236 }
237 
239  const Instruction *Orig) {
240  // If the loop was versioned with memchecks, add the corresponding no-alias
241  // metadata.
242  if (LVer && (isa<LoadInst>(Orig) || isa<StoreInst>(Orig)))
243  LVer->annotateInstWithNoAlias(To, Orig);
244 }
245 
247  propagateMetadata(To, From);
248  addNewMetadata(To, From);
249 }
250 
252  for (Value *V : To) {
253  if (Instruction *I = dyn_cast<Instruction>(V))
254  addMetadata(I, From);
255  }
256 }
257 
259  const Instruction *Inst = dyn_cast<Instruction>(V);
260  if (!Inst) {
262  return;
263  }
264 
265  const DILocation *DIL = Inst->getDebugLoc();
266  // When a FSDiscriminator is enabled, we don't need to add the multiply
267  // factors to the discriminators.
268  if (DIL && Inst->getFunction()->isDebugInfoForProfiling() &&
269  !isa<DbgInfoIntrinsic>(Inst) && !EnableFSDiscriminator) {
270  // FIXME: For scalable vectors, assume vscale=1.
271  auto NewDIL =
273  if (NewDIL)
275  else
276  LLVM_DEBUG(dbgs() << "Failed to create new discriminator: "
277  << DIL->getFilename() << " Line: " << DIL->getLine());
278  } else
280 }
281 
282 BasicBlock *
283 VPBasicBlock::createEmptyBasicBlock(VPTransformState::CFGState &CFG) {
284  // BB stands for IR BasicBlocks. VPBB stands for VPlan VPBasicBlocks.
285  // Pred stands for Predessor. Prev stands for Previous - last visited/created.
286  BasicBlock *PrevBB = CFG.PrevBB;
287  BasicBlock *NewBB = BasicBlock::Create(PrevBB->getContext(), getName(),
288  PrevBB->getParent(), CFG.ExitBB);
289  LLVM_DEBUG(dbgs() << "LV: created " << NewBB->getName() << '\n');
290 
291  // Hook up the new basic block to its predecessors.
292  for (VPBlockBase *PredVPBlock : getHierarchicalPredecessors()) {
293  VPBasicBlock *PredVPBB = PredVPBlock->getExitingBasicBlock();
294  auto &PredVPSuccessors = PredVPBB->getHierarchicalSuccessors();
295  BasicBlock *PredBB = CFG.VPBB2IRBB[PredVPBB];
296 
297  assert(PredBB && "Predecessor basic-block not found building successor.");
298  auto *PredBBTerminator = PredBB->getTerminator();
299  LLVM_DEBUG(dbgs() << "LV: draw edge from" << PredBB->getName() << '\n');
300 
301  auto *TermBr = dyn_cast<BranchInst>(PredBBTerminator);
302  if (isa<UnreachableInst>(PredBBTerminator)) {
303  assert(PredVPSuccessors.size() == 1 &&
304  "Predecessor ending w/o branch must have single successor.");
305  DebugLoc DL = PredBBTerminator->getDebugLoc();
306  PredBBTerminator->eraseFromParent();
307  auto *Br = BranchInst::Create(NewBB, PredBB);
308  Br->setDebugLoc(DL);
309  } else if (TermBr && !TermBr->isConditional()) {
310  TermBr->setSuccessor(0, NewBB);
311  } else {
312  // Set each forward successor here when it is created, excluding
313  // backedges. A backward successor is set when the branch is created.
314  unsigned idx = PredVPSuccessors.front() == this ? 0 : 1;
315  assert(!TermBr->getSuccessor(idx) &&
316  "Trying to reset an existing successor block.");
317  TermBr->setSuccessor(idx, NewBB);
318  }
319  }
320  return NewBB;
321 }
322 
324  bool Replica = State->Instance && !State->Instance->isFirstIteration();
325  VPBasicBlock *PrevVPBB = State->CFG.PrevVPBB;
326  VPBlockBase *SingleHPred = nullptr;
327  BasicBlock *NewBB = State->CFG.PrevBB; // Reuse it if possible.
328 
329  auto IsLoopRegion = [](VPBlockBase *BB) {
330  auto *R = dyn_cast<VPRegionBlock>(BB);
331  return R && !R->isReplicator();
332  };
333 
334  // 1. Create an IR basic block, or reuse the last one or ExitBB if possible.
335  if (getPlan()->getVectorLoopRegion()->getSingleSuccessor() == this) {
336  // ExitBB can be re-used for the exit block of the Plan.
337  NewBB = State->CFG.ExitBB;
338  State->CFG.PrevBB = NewBB;
339 
340  // Update the branch instruction in the predecessor to branch to ExitBB.
341  VPBlockBase *PredVPB = getSingleHierarchicalPredecessor();
342  VPBasicBlock *ExitingVPBB = PredVPB->getExitingBasicBlock();
343  assert(PredVPB->getSingleSuccessor() == this &&
344  "predecessor must have the current block as only successor");
345  BasicBlock *ExitingBB = State->CFG.VPBB2IRBB[ExitingVPBB];
346  // The Exit block of a loop is always set to be successor 0 of the Exiting
347  // block.
348  cast<BranchInst>(ExitingBB->getTerminator())->setSuccessor(0, NewBB);
349  } else if (PrevVPBB && /* A */
350  !((SingleHPred = getSingleHierarchicalPredecessor()) &&
351  SingleHPred->getExitingBasicBlock() == PrevVPBB &&
352  PrevVPBB->getSingleHierarchicalSuccessor() &&
353  (SingleHPred->getParent() == getEnclosingLoopRegion() &&
354  !IsLoopRegion(SingleHPred))) && /* B */
355  !(Replica && getPredecessors().empty())) { /* C */
356  // The last IR basic block is reused, as an optimization, in three cases:
357  // A. the first VPBB reuses the loop pre-header BB - when PrevVPBB is null;
358  // B. when the current VPBB has a single (hierarchical) predecessor which
359  // is PrevVPBB and the latter has a single (hierarchical) successor which
360  // both are in the same non-replicator region; and
361  // C. when the current VPBB is an entry of a region replica - where PrevVPBB
362  // is the exiting VPBB of this region from a previous instance, or the
363  // predecessor of this region.
364 
365  NewBB = createEmptyBasicBlock(State->CFG);
366  State->Builder.SetInsertPoint(NewBB);
367  // Temporarily terminate with unreachable until CFG is rewired.
369  // Register NewBB in its loop. In innermost loops its the same for all
370  // BB's.
371  if (State->CurrentVectorLoop)
372  State->CurrentVectorLoop->addBasicBlockToLoop(NewBB, *State->LI);
374  State->CFG.PrevBB = NewBB;
375  }
376 
377  // 2. Fill the IR basic block with IR instructions.
378  LLVM_DEBUG(dbgs() << "LV: vectorizing VPBB:" << getName()
379  << " in BB:" << NewBB->getName() << '\n');
380 
381  State->CFG.VPBB2IRBB[this] = NewBB;
382  State->CFG.PrevVPBB = this;
383 
384  for (VPRecipeBase &Recipe : Recipes)
385  Recipe.execute(*State);
386 
387  LLVM_DEBUG(dbgs() << "LV: filled BB:" << *NewBB);
388 }
389 
391  for (VPRecipeBase &R : Recipes) {
392  for (auto *Def : R.definedValues())
393  Def->replaceAllUsesWith(NewValue);
394 
395  for (unsigned I = 0, E = R.getNumOperands(); I != E; I++)
396  R.setOperand(I, NewValue);
397  }
398 }
399 
401  assert((SplitAt == end() || SplitAt->getParent() == this) &&
402  "can only split at a position in the same block");
403 
405  // First, disconnect the current block from its successors.
406  for (VPBlockBase *Succ : Succs)
407  VPBlockUtils::disconnectBlocks(this, Succ);
408 
409  // Create new empty block after the block to split.
410  auto *SplitBlock = new VPBasicBlock(getName() + ".split");
412 
413  // Add successors for block to split to new block.
414  for (VPBlockBase *Succ : Succs)
416 
417  // Finally, move the recipes starting at SplitAt to new block.
418  for (VPRecipeBase &ToMove :
419  make_early_inc_range(make_range(SplitAt, this->end())))
420  ToMove.moveBefore(*SplitBlock, SplitBlock->end());
421 
422  return SplitBlock;
423 }
424 
427  if (P && P->isReplicator()) {
428  P = P->getParent();
429  assert(!cast<VPRegionBlock>(P)->isReplicator() &&
430  "unexpected nested replicate regions");
431  }
432  return P;
433 }
434 
435 static bool hasConditionalTerminator(const VPBasicBlock *VPBB) {
436  if (VPBB->empty()) {
437  assert(
438  VPBB->getNumSuccessors() < 2 &&
439  "block with multiple successors doesn't have a recipe as terminator");
440  return false;
441  }
442 
443  const VPRecipeBase *R = &VPBB->back();
444  auto *VPI = dyn_cast<VPInstruction>(R);
445  bool IsCondBranch =
446  isa<VPBranchOnMaskRecipe>(R) ||
447  (VPI && (VPI->getOpcode() == VPInstruction::BranchOnCond ||
448  VPI->getOpcode() == VPInstruction::BranchOnCount));
449  (void)IsCondBranch;
450 
451  if (VPBB->getNumSuccessors() >= 2 || VPBB->isExiting()) {
452  assert(IsCondBranch && "block with multiple successors not terminated by "
453  "conditional branch recipe");
454 
455  return true;
456  }
457 
458  assert(
459  !IsCondBranch &&
460  "block with 0 or 1 successors terminated by conditional branch recipe");
461  return false;
462 }
463 
465  if (hasConditionalTerminator(this))
466  return &back();
467  return nullptr;
468 }
469 
471  if (hasConditionalTerminator(this))
472  return &back();
473  return nullptr;
474 }
475 
477  return getParent()->getExitingBasicBlock() == this;
478 }
479 
480 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
481 void VPBlockBase::printSuccessors(raw_ostream &O, const Twine &Indent) const {
482  if (getSuccessors().empty()) {
483  O << Indent << "No successors\n";
484  } else {
485  O << Indent << "Successor(s): ";
486  ListSeparator LS;
487  for (auto *Succ : getSuccessors())
488  O << LS << Succ->getName();
489  O << '\n';
490  }
491 }
492 
493 void VPBasicBlock::print(raw_ostream &O, const Twine &Indent,
494  VPSlotTracker &SlotTracker) const {
495  O << Indent << getName() << ":\n";
496 
497  auto RecipeIndent = Indent + " ";
498  for (const VPRecipeBase &Recipe : *this) {
499  Recipe.print(O, RecipeIndent, SlotTracker);
500  O << '\n';
501  }
502 
503  printSuccessors(O, Indent);
504 }
505 #endif
506 
508  for (VPBlockBase *Block : depth_first(Entry))
509  // Drop all references in VPBasicBlocks and replace all uses with
510  // DummyValue.
511  Block->dropAllReferences(NewValue);
512 }
513 
516 
517  if (!isReplicator()) {
518  // Create and register the new vector loop.
519  Loop *PrevLoop = State->CurrentVectorLoop;
520  State->CurrentVectorLoop = State->LI->AllocateLoop();
521  BasicBlock *VectorPH = State->CFG.VPBB2IRBB[getPreheaderVPBB()];
522  Loop *ParentLoop = State->LI->getLoopFor(VectorPH);
523 
524  // Insert the new loop into the loop nest and register the new basic blocks
525  // before calling any utilities such as SCEV that require valid LoopInfo.
526  if (ParentLoop)
527  ParentLoop->addChildLoop(State->CurrentVectorLoop);
528  else
529  State->LI->addTopLevelLoop(State->CurrentVectorLoop);
530 
531  // Visit the VPBlocks connected to "this", starting from it.
532  for (VPBlockBase *Block : RPOT) {
533  LLVM_DEBUG(dbgs() << "LV: VPBlock in RPO " << Block->getName() << '\n');
534  Block->execute(State);
535  }
536 
537  State->CurrentVectorLoop = PrevLoop;
538  return;
539  }
540 
541  assert(!State->Instance && "Replicating a Region with non-null instance.");
542 
543  // Enter replicating mode.
544  State->Instance = VPIteration(0, 0);
545 
546  for (unsigned Part = 0, UF = State->UF; Part < UF; ++Part) {
547  State->Instance->Part = Part;
548  assert(!State->VF.isScalable() && "VF is assumed to be non scalable.");
549  for (unsigned Lane = 0, VF = State->VF.getKnownMinValue(); Lane < VF;
550  ++Lane) {
551  State->Instance->Lane = VPLane(Lane, VPLane::Kind::First);
552  // Visit the VPBlocks connected to \p this, starting from it.
553  for (VPBlockBase *Block : RPOT) {
554  LLVM_DEBUG(dbgs() << "LV: VPBlock in RPO " << Block->getName() << '\n');
555  Block->execute(State);
556  }
557  }
558  }
559 
560  // Exit replicating mode.
561  State->Instance.reset();
562 }
563 
564 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
566  VPSlotTracker &SlotTracker) const {
567  O << Indent << (isReplicator() ? "<xVFxUF> " : "<x1> ") << getName() << ": {";
568  auto NewIndent = Indent + " ";
569  for (auto *BlockBase : depth_first(Entry)) {
570  O << '\n';
571  BlockBase->print(O, NewIndent, SlotTracker);
572  }
573  O << Indent << "}\n";
574 
575  printSuccessors(O, Indent);
576 }
577 #endif
578 
580  VPBasicBlock *Header = getVectorLoopRegion()->getEntryBasicBlock();
581  for (VPRecipeBase &R : Header->phis()) {
582  if (isa<VPActiveLaneMaskPHIRecipe>(&R))
583  return cast<VPActiveLaneMaskPHIRecipe>(&R);
584  }
585  return nullptr;
586 }
587 
589  VPInstruction *Not = dyn_cast<VPInstruction>(Term->getOperand(0));
590  if (!Not || Not->getOpcode() != VPInstruction::Not)
591  return false;
592 
593  VPInstruction *ALM = dyn_cast<VPInstruction>(Not->getOperand(0));
594  return ALM && ALM->getOpcode() == VPInstruction::ActiveLaneMask;
595 }
596 
597 void VPlan::prepareToExecute(Value *TripCountV, Value *VectorTripCountV,
598  Value *CanonicalIVStartValue,
599  VPTransformState &State,
600  bool IsEpilogueVectorization) {
601 
602  VPBasicBlock *ExitingVPBB = getVectorLoopRegion()->getExitingBasicBlock();
603  auto *Term = dyn_cast<VPInstruction>(&ExitingVPBB->back());
604  // Try to simplify the branch condition if TC <= VF * UF when preparing to
605  // execute the plan for the main vector loop. We only do this if the
606  // terminator is:
607  // 1. BranchOnCount, or
608  // 2. BranchOnCond where the input is Not(ActiveLaneMask).
609  if (!IsEpilogueVectorization && Term && isa<ConstantInt>(TripCountV) &&
610  (Term->getOpcode() == VPInstruction::BranchOnCount ||
611  (Term->getOpcode() == VPInstruction::BranchOnCond &&
613  ConstantInt *C = cast<ConstantInt>(TripCountV);
614  uint64_t TCVal = C->getZExtValue();
615  if (TCVal && TCVal <= State.VF.getKnownMinValue() * State.UF) {
616  auto *BOC =
618  {getOrAddExternalDef(State.Builder.getTrue())});
619  Term->eraseFromParent();
620  ExitingVPBB->appendRecipe(BOC);
621  // TODO: Further simplifications are possible
622  // 1. Replace inductions with constants.
623  // 2. Replace vector loop region with VPBasicBlock.
624  }
625  }
626 
627  // Check if the trip count is needed, and if so build it.
628  if (TripCount && TripCount->getNumUsers()) {
629  for (unsigned Part = 0, UF = State.UF; Part < UF; ++Part)
630  State.set(TripCount, TripCountV, Part);
631  }
632 
633  // Check if the backedge taken count is needed, and if so build it.
634  if (BackedgeTakenCount && BackedgeTakenCount->getNumUsers()) {
636  auto *TCMO = Builder.CreateSub(TripCountV,
637  ConstantInt::get(TripCountV->getType(), 1),
638  "trip.count.minus.1");
639  auto VF = State.VF;
640  Value *VTCMO =
641  VF.isScalar() ? TCMO : Builder.CreateVectorSplat(VF, TCMO, "broadcast");
642  for (unsigned Part = 0, UF = State.UF; Part < UF; ++Part)
643  State.set(BackedgeTakenCount, VTCMO, Part);
644  }
645 
646  for (unsigned Part = 0, UF = State.UF; Part < UF; ++Part)
647  State.set(&VectorTripCount, VectorTripCountV, Part);
648 
649  // When vectorizing the epilogue loop, the canonical induction start value
650  // needs to be changed from zero to the value after the main vector loop.
651  if (CanonicalIVStartValue) {
652  VPValue *VPV = getOrAddExternalDef(CanonicalIVStartValue);
653  auto *IV = getCanonicalIV();
654  assert(all_of(IV->users(),
655  [](const VPUser *U) {
656  if (isa<VPScalarIVStepsRecipe>(U))
657  return true;
658  auto *VPI = cast<VPInstruction>(U);
659  return VPI->getOpcode() ==
660  VPInstruction::CanonicalIVIncrement ||
661  VPI->getOpcode() ==
662  VPInstruction::CanonicalIVIncrementNUW;
663  }) &&
664  "the canonical IV should only be used by its increments or "
665  "ScalarIVSteps when "
666  "resetting the start value");
667  IV->setOperand(0, VPV);
668  }
669 }
670 
671 /// Generate the code inside the preheader and body of the vectorized loop.
672 /// Assumes a single pre-header basic-block was created for this. Introduce
673 /// additional basic-blocks as needed, and fill them all.
675  // Set the reverse mapping from VPValues to Values for code generation.
676  for (auto &Entry : Value2VPValue)
677  State->VPValue2Value[Entry.second] = Entry.first;
678 
679  // Initialize CFG state.
680  State->CFG.PrevVPBB = nullptr;
681  State->CFG.ExitBB = State->CFG.PrevBB->getSingleSuccessor();
682  BasicBlock *VectorPreHeader = State->CFG.PrevBB;
683  State->Builder.SetInsertPoint(VectorPreHeader->getTerminator());
684 
685  // Generate code in the loop pre-header and body.
686  for (VPBlockBase *Block : depth_first(Entry))
687  Block->execute(State);
688 
689  VPBasicBlock *LatchVPBB = getVectorLoopRegion()->getExitingBasicBlock();
690  BasicBlock *VectorLatchBB = State->CFG.VPBB2IRBB[LatchVPBB];
691 
692  // Fix the latch value of canonical, reduction and first-order recurrences
693  // phis in the vector loop.
694  VPBasicBlock *Header = getVectorLoopRegion()->getEntryBasicBlock();
695  for (VPRecipeBase &R : Header->phis()) {
696  // Skip phi-like recipes that generate their backedege values themselves.
697  if (isa<VPWidenPHIRecipe>(&R))
698  continue;
699 
700  if (isa<VPWidenPointerInductionRecipe>(&R) ||
701  isa<VPWidenIntOrFpInductionRecipe>(&R)) {
702  PHINode *Phi = nullptr;
703  if (isa<VPWidenIntOrFpInductionRecipe>(&R)) {
704  Phi = cast<PHINode>(State->get(R.getVPSingleValue(), 0));
705  } else {
706  auto *WidenPhi = cast<VPWidenPointerInductionRecipe>(&R);
707  // TODO: Split off the case that all users of a pointer phi are scalar
708  // from the VPWidenPointerInductionRecipe.
709  if (WidenPhi->onlyScalarsGenerated(State->VF))
710  continue;
711 
712  auto *GEP = cast<GetElementPtrInst>(State->get(WidenPhi, 0));
713  Phi = cast<PHINode>(GEP->getPointerOperand());
714  }
715 
716  Phi->setIncomingBlock(1, VectorLatchBB);
717 
718  // Move the last step to the end of the latch block. This ensures
719  // consistent placement of all induction updates.
720  Instruction *Inc = cast<Instruction>(Phi->getIncomingValue(1));
721  Inc->moveBefore(VectorLatchBB->getTerminator()->getPrevNode());
722  continue;
723  }
724 
725  auto *PhiR = cast<VPHeaderPHIRecipe>(&R);
726  // For canonical IV, first-order recurrences and in-order reduction phis,
727  // only a single part is generated, which provides the last part from the
728  // previous iteration. For non-ordered reductions all UF parts are
729  // generated.
730  bool SinglePartNeeded = isa<VPCanonicalIVPHIRecipe>(PhiR) ||
731  isa<VPFirstOrderRecurrencePHIRecipe>(PhiR) ||
732  (isa<VPReductionPHIRecipe>(PhiR) &&
733  cast<VPReductionPHIRecipe>(PhiR)->isOrdered());
734  unsigned LastPartForNewPhi = SinglePartNeeded ? 1 : State->UF;
735 
736  for (unsigned Part = 0; Part < LastPartForNewPhi; ++Part) {
737  Value *Phi = State->get(PhiR, Part);
738  Value *Val = State->get(PhiR->getBackedgeValue(),
739  SinglePartNeeded ? State->UF - 1 : Part);
740  cast<PHINode>(Phi)->addIncoming(Val, VectorLatchBB);
741  }
742  }
743 
744  // We do not attempt to preserve DT for outer loop vectorization currently.
745  if (!EnableVPlanNativePath) {
746  BasicBlock *VectorHeaderBB = State->CFG.VPBB2IRBB[Header];
747  State->DT->addNewBlock(VectorHeaderBB, VectorPreHeader);
748  updateDominatorTree(State->DT, VectorHeaderBB, VectorLatchBB,
749  State->CFG.ExitBB);
750  }
751 }
752 
753 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
755 void VPlan::print(raw_ostream &O) const {
757 
758  O << "VPlan '" << Name << "' {";
759 
760  if (VectorTripCount.getNumUsers() > 0) {
761  O << "\nLive-in ";
762  VectorTripCount.printAsOperand(O, SlotTracker);
763  O << " = vector-trip-count\n";
764  }
765 
766  if (BackedgeTakenCount && BackedgeTakenCount->getNumUsers()) {
767  O << "\nLive-in ";
768  BackedgeTakenCount->printAsOperand(O, SlotTracker);
769  O << " = backedge-taken count\n";
770  }
771 
772  for (const VPBlockBase *Block : depth_first(getEntry())) {
773  O << '\n';
774  Block->print(O, "", SlotTracker);
775  }
776 
777  if (!LiveOuts.empty())
778  O << "\n";
779  for (const auto &KV : LiveOuts) {
780  O << "Live-out ";
781  KV.second->getPhi()->printAsOperand(O);
782  O << " = ";
783  KV.second->getOperand(0)->printAsOperand(O, SlotTracker);
784  O << "\n";
785  }
786 
787  O << "}\n";
788 }
789 
792  VPlanPrinter Printer(O, *this);
793  Printer.dump();
794 }
795 
797 void VPlan::dump() const { print(dbgs()); }
798 #endif
799 
801  assert(LiveOuts.count(PN) == 0 && "an exit value for PN already exists");
802  LiveOuts.insert({PN, new VPLiveOut(PN, V)});
803 }
804 
805 void VPlan::updateDominatorTree(DominatorTree *DT, BasicBlock *LoopHeaderBB,
806  BasicBlock *LoopLatchBB,
807  BasicBlock *LoopExitBB) {
808  // The vector body may be more than a single basic-block by this point.
809  // Update the dominator tree information inside the vector body by propagating
810  // it from header to latch, expecting only triangular control-flow, if any.
811  BasicBlock *PostDomSucc = nullptr;
812  for (auto *BB = LoopHeaderBB; BB != LoopLatchBB; BB = PostDomSucc) {
813  // Get the list of successors of this block.
814  std::vector<BasicBlock *> Succs(succ_begin(BB), succ_end(BB));
815  assert(Succs.size() <= 2 &&
816  "Basic block in vector loop has more than 2 successors.");
817  PostDomSucc = Succs[0];
818  if (Succs.size() == 1) {
819  assert(PostDomSucc->getSinglePredecessor() &&
820  "PostDom successor has more than one predecessor.");
821  DT->addNewBlock(PostDomSucc, BB);
822  continue;
823  }
824  BasicBlock *InterimSucc = Succs[1];
825  if (PostDomSucc->getSingleSuccessor() == InterimSucc) {
826  PostDomSucc = Succs[1];
827  InterimSucc = Succs[0];
828  }
829  assert(InterimSucc->getSingleSuccessor() == PostDomSucc &&
830  "One successor of a basic block does not lead to the other.");
831  assert(InterimSucc->getSinglePredecessor() &&
832  "Interim successor has more than one predecessor.");
833  assert(PostDomSucc->hasNPredecessors(2) &&
834  "PostDom successor has more than two predecessors.");
835  DT->addNewBlock(InterimSucc, BB);
836  DT->addNewBlock(PostDomSucc, BB);
837  }
838  // Latch block is a new dominator for the loop exit.
839  DT->changeImmediateDominator(LoopExitBB, LoopLatchBB);
841 }
842 
843 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
844 
845 Twine VPlanPrinter::getUID(const VPBlockBase *Block) {
846  return (isa<VPRegionBlock>(Block) ? "cluster_N" : "N") +
847  Twine(getOrCreateBID(Block));
848 }
849 
850 Twine VPlanPrinter::getOrCreateName(const VPBlockBase *Block) {
851  const std::string &Name = Block->getName();
852  if (!Name.empty())
853  return Name;
854  return "VPB" + Twine(getOrCreateBID(Block));
855 }
856 
858  Depth = 1;
859  bumpIndent(0);
860  OS << "digraph VPlan {\n";
861  OS << "graph [labelloc=t, fontsize=30; label=\"Vectorization Plan";
862  if (!Plan.getName().empty())
863  OS << "\\n" << DOT::EscapeString(Plan.getName());
864  if (Plan.BackedgeTakenCount) {
865  OS << ", where:\\n";
866  Plan.BackedgeTakenCount->print(OS, SlotTracker);
867  OS << " := BackedgeTakenCount";
868  }
869  OS << "\"]\n";
870  OS << "node [shape=rect, fontname=Courier, fontsize=30]\n";
871  OS << "edge [fontname=Courier, fontsize=30]\n";
872  OS << "compound=true\n";
873 
874  for (const VPBlockBase *Block : depth_first(Plan.getEntry()))
875  dumpBlock(Block);
876 
877  OS << "}\n";
878 }
879 
880 void VPlanPrinter::dumpBlock(const VPBlockBase *Block) {
881  if (const VPBasicBlock *BasicBlock = dyn_cast<VPBasicBlock>(Block))
882  dumpBasicBlock(BasicBlock);
883  else if (const VPRegionBlock *Region = dyn_cast<VPRegionBlock>(Block))
884  dumpRegion(Region);
885  else
886  llvm_unreachable("Unsupported kind of VPBlock.");
887 }
888 
889 void VPlanPrinter::drawEdge(const VPBlockBase *From, const VPBlockBase *To,
890  bool Hidden, const Twine &Label) {
891  // Due to "dot" we print an edge between two regions as an edge between the
892  // exiting basic block and the entry basic of the respective regions.
893  const VPBlockBase *Tail = From->getExitingBasicBlock();
894  const VPBlockBase *Head = To->getEntryBasicBlock();
895  OS << Indent << getUID(Tail) << " -> " << getUID(Head);
896  OS << " [ label=\"" << Label << '\"';
897  if (Tail != From)
898  OS << " ltail=" << getUID(From);
899  if (Head != To)
900  OS << " lhead=" << getUID(To);
901  if (Hidden)
902  OS << "; splines=none";
903  OS << "]\n";
904 }
905 
906 void VPlanPrinter::dumpEdges(const VPBlockBase *Block) {
907  auto &Successors = Block->getSuccessors();
908  if (Successors.size() == 1)
909  drawEdge(Block, Successors.front(), false, "");
910  else if (Successors.size() == 2) {
911  drawEdge(Block, Successors.front(), false, "T");
912  drawEdge(Block, Successors.back(), false, "F");
913  } else {
914  unsigned SuccessorNumber = 0;
915  for (auto *Successor : Successors)
916  drawEdge(Block, Successor, false, Twine(SuccessorNumber++));
917  }
918 }
919 
920 void VPlanPrinter::dumpBasicBlock(const VPBasicBlock *BasicBlock) {
921  // Implement dot-formatted dump by performing plain-text dump into the
922  // temporary storage followed by some post-processing.
923  OS << Indent << getUID(BasicBlock) << " [label =\n";
924  bumpIndent(1);
925  std::string Str;
926  raw_string_ostream SS(Str);
927  // Use no indentation as we need to wrap the lines into quotes ourselves.
929 
930  // We need to process each line of the output separately, so split
931  // single-string plain-text dump.
933  StringRef(Str).rtrim('\n').split(Lines, "\n");
934 
935  auto EmitLine = [&](StringRef Line, StringRef Suffix) {
936  OS << Indent << '"' << DOT::EscapeString(Line.str()) << "\\l\"" << Suffix;
937  };
938 
939  // Don't need the "+" after the last line.
940  for (auto Line : make_range(Lines.begin(), Lines.end() - 1))
941  EmitLine(Line, " +\n");
942  EmitLine(Lines.back(), "\n");
943 
944  bumpIndent(-1);
945  OS << Indent << "]\n";
946 
948 }
949 
950 void VPlanPrinter::dumpRegion(const VPRegionBlock *Region) {
951  OS << Indent << "subgraph " << getUID(Region) << " {\n";
952  bumpIndent(1);
953  OS << Indent << "fontname=Courier\n"
954  << Indent << "label=\""
955  << DOT::EscapeString(Region->isReplicator() ? "<xVFxUF> " : "<x1> ")
956  << DOT::EscapeString(Region->getName()) << "\"\n";
957  // Dump the blocks of the region.
958  assert(Region->getEntry() && "Region contains no inner blocks.");
959  for (const VPBlockBase *Block : depth_first(Region->getEntry()))
960  dumpBlock(Block);
961  bumpIndent(-1);
962  OS << Indent << "}\n";
963  dumpEdges(Region);
964 }
965 
967  if (auto *Inst = dyn_cast<Instruction>(V)) {
968  if (!Inst->getType()->isVoidTy()) {
969  Inst->printAsOperand(O, false);
970  O << " = ";
971  }
972  O << Inst->getOpcodeName() << " ";
973  unsigned E = Inst->getNumOperands();
974  if (E > 0) {
975  Inst->getOperand(0)->printAsOperand(O, false);
976  for (unsigned I = 1; I < E; ++I)
977  Inst->getOperand(I)->printAsOperand(O << ", ", false);
978  }
979  } else // !Inst
980  V->printAsOperand(O, false);
981 }
982 
983 #endif
984 
985 template void DomTreeBuilder::Calculate<VPDominatorTree>(VPDominatorTree &DT);
986 
988  for (unsigned J = 0; J < getNumUsers();) {
989  VPUser *User = Users[J];
990  unsigned NumUsers = getNumUsers();
991  for (unsigned I = 0, E = User->getNumOperands(); I < E; ++I)
992  if (User->getOperand(I) == this)
993  User->setOperand(I, New);
994  // If a user got removed after updating the current user, the next user to
995  // update will be moved to the current position, so we only need to
996  // increment the index if the number of users did not change.
997  if (NumUsers == getNumUsers())
998  J++;
999  }
1000 }
1001 
1002 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
1004  if (const Value *UV = getUnderlyingValue()) {
1005  OS << "ir<";
1006  UV->printAsOperand(OS, false);
1007  OS << ">";
1008  return;
1009  }
1010 
1011  unsigned Slot = Tracker.getSlot(this);
1012  if (Slot == unsigned(-1))
1013  OS << "<badref>";
1014  else
1015  OS << "vp<%" << Tracker.getSlot(this) << ">";
1016 }
1017 
1019  interleaveComma(operands(), O, [&O, &SlotTracker](VPValue *Op) {
1020  Op->printAsOperand(O, SlotTracker);
1021  });
1022 }
1023 #endif
1024 
1025 void VPInterleavedAccessInfo::visitRegion(VPRegionBlock *Region,
1026  Old2NewTy &Old2New,
1027  InterleavedAccessInfo &IAI) {
1029  for (VPBlockBase *Base : RPOT) {
1030  visitBlock(Base, Old2New, IAI);
1031  }
1032 }
1033 
1034 void VPInterleavedAccessInfo::visitBlock(VPBlockBase *Block, Old2NewTy &Old2New,
1035  InterleavedAccessInfo &IAI) {
1036  if (VPBasicBlock *VPBB = dyn_cast<VPBasicBlock>(Block)) {
1037  for (VPRecipeBase &VPI : *VPBB) {
1038  if (isa<VPHeaderPHIRecipe>(&VPI))
1039  continue;
1040  assert(isa<VPInstruction>(&VPI) && "Can only handle VPInstructions");
1041  auto *VPInst = cast<VPInstruction>(&VPI);
1042 
1043  auto *Inst = dyn_cast_or_null<Instruction>(VPInst->getUnderlyingValue());
1044  if (!Inst)
1045  continue;
1046  auto *IG = IAI.getInterleaveGroup(Inst);
1047  if (!IG)
1048  continue;
1049 
1050  auto NewIGIter = Old2New.find(IG);
1051  if (NewIGIter == Old2New.end())
1052  Old2New[IG] = new InterleaveGroup<VPInstruction>(
1053  IG->getFactor(), IG->isReverse(), IG->getAlign());
1054 
1055  if (Inst == IG->getInsertPos())
1056  Old2New[IG]->setInsertPos(VPInst);
1057 
1058  InterleaveGroupMap[VPInst] = Old2New[IG];
1059  InterleaveGroupMap[VPInst]->insertMember(
1060  VPInst, IG->getIndex(Inst),
1061  Align(IG->isReverse() ? (-1) * int(IG->getFactor())
1062  : IG->getFactor()));
1063  }
1064  } else if (VPRegionBlock *Region = dyn_cast<VPRegionBlock>(Block))
1065  visitRegion(Region, Old2New, IAI);
1066  else
1067  llvm_unreachable("Unsupported kind of VPBlock.");
1068 }
1069 
1071  InterleavedAccessInfo &IAI) {
1072  Old2NewTy Old2New;
1073  visitRegion(Plan.getVectorLoopRegion(), Old2New, IAI);
1074 }
1075 
1076 void VPSlotTracker::assignSlot(const VPValue *V) {
1077  assert(Slots.find(V) == Slots.end() && "VPValue already has a slot!");
1078  Slots[V] = NextSlot++;
1079 }
1080 
1081 void VPSlotTracker::assignSlots(const VPlan &Plan) {
1082 
1083  for (const auto &P : Plan.VPExternalDefs)
1084  assignSlot(P.second);
1085 
1086  assignSlot(&Plan.VectorTripCount);
1087  if (Plan.BackedgeTakenCount)
1088  assignSlot(Plan.BackedgeTakenCount);
1089 
1093  Plan.getEntry()));
1094  for (const VPBasicBlock *VPBB :
1095  VPBlockUtils::blocksOnly<const VPBasicBlock>(RPOT))
1096  for (const VPRecipeBase &Recipe : *VPBB)
1097  for (VPValue *Def : Recipe.definedValues())
1098  assignSlot(Def);
1099 }
1100 
1102  return all_of(Def->users(),
1103  [Def](VPUser *U) { return U->onlyFirstLaneUsed(Def); });
1104 }
1105 
1107  ScalarEvolution &SE) {
1108  if (auto *E = dyn_cast<SCEVConstant>(Expr))
1109  return Plan.getOrAddExternalDef(E->getValue());
1110  if (auto *E = dyn_cast<SCEVUnknown>(Expr))
1111  return Plan.getOrAddExternalDef(E->getValue());
1112 
1113  VPBasicBlock *Preheader = Plan.getEntry()->getEntryBasicBlock();
1114  VPExpandSCEVRecipe *Step = new VPExpandSCEVRecipe(Expr, SE);
1115  Preheader->appendRecipe(Step);
1116  return Step;
1117 }
llvm::VPInstruction::getOpcode
unsigned getOpcode() const
Definition: VPlan.h:851
i
i
Definition: README.txt:29
llvm::DILocation::cloneByMultiplyingDuplicationFactor
Optional< const DILocation * > cloneByMultiplyingDuplicationFactor(unsigned DF) const
Returns a new DILocation with duplication factor DF * current duplication factor encoded in the discr...
Definition: DebugInfoMetadata.h:2282
llvm::M68kBeads::Term
@ Term
Definition: M68kBaseInfo.h:71
llvm::VPlan::getActiveLaneMaskPhi
VPActiveLaneMaskPHIRecipe * getActiveLaneMaskPhi()
Find and return the VPActiveLaneMaskPHIRecipe from the header - there be only one at most.
Definition: VPlan.cpp:579
llvm::BasicBlock::end
iterator end()
Definition: BasicBlock.h:308
llvm::VPBlockBase::getPlan
VPlan * getPlan()
Definition: VPlan.cpp:142
llvm::BasicBlock::print
void print(raw_ostream &OS, AssemblyAnnotationWriter *AAW=nullptr, bool ShouldPreserveUseListOrder=false, bool IsForDebug=false) const
Print the basic block to an output stream with an optional AssemblyAnnotationWriter.
Definition: AsmWriter.cpp:4582
llvm::VPlanIngredient::print
void print(raw_ostream &O) const
Definition: VPlan.cpp:966
getName
static StringRef getName(Value *V)
Definition: ProvenanceAnalysisEvaluator.cpp:20
LLVM_DUMP_METHOD
#define LLVM_DUMP_METHOD
Mark debug helper function definitions like dump() that should not be stripped from debug builds.
Definition: Compiler.h:492
llvm::IRBuilderBase::SetInsertPoint
void SetInsertPoint(BasicBlock *TheBB)
This specifies that created instructions should be appended to the end of the specified block.
Definition: IRBuilder.h:179
llvm
This is an optimization pass for GlobalISel generic memory operations.
Definition: AddressRanges.h:18
llvm::tgtok::Def
@ Def
Definition: TGLexer.h:50
llvm::make_range
iterator_range< T > make_range(T x, T y)
Convenience function for iterating over sub-ranges.
Definition: iterator_range.h:53
llvm::VPTransformState::CFG
struct llvm::VPTransformState::CFGState CFG
llvm::VPRegionBlock::execute
void execute(VPTransformState *State) override
The method which generates the output IR instructions that correspond to this VPRegionBlock,...
Definition: VPlan.cpp:514
llvm::VPBlockBase::getEnclosingBlockWithPredecessors
VPBlockBase * getEnclosingBlockWithPredecessors()
Definition: VPlan.cpp:190
llvm::VPBasicBlock::back
const VPRecipeBase & back() const
Definition: VPlan.h:2056
llvm::VPTransformState::Builder
IRBuilderBase & Builder
Hold a reference to the IRBuilder used to generate output IR code.
Definition: VPlan.h:359
llvm::BasicBlock::getParent
const Function * getParent() const
Return the enclosing method, or null if none.
Definition: BasicBlock.h:104
ScalarEvolutionExpander.h
llvm::ElementCount
Definition: TypeSize.h:404
llvm::succ_end
Interval::succ_iterator succ_end(Interval *I)
Definition: Interval.h:102
llvm::codeview::DebugSubsectionKind::Lines
@ Lines
llvm::VPBasicBlock
VPBasicBlock serves as the leaf of the Hierarchical Control-Flow Graph.
Definition: VPlan.h:2013
llvm::Loop
Represents a single loop in the control flow graph.
Definition: LoopInfo.h:547
P
This currently compiles esp xmm0 movsd esp eax eax esp ret We should use not the dag combiner This is because dagcombine2 needs to be able to see through the X86ISD::Wrapper which DAGCombine can t really do The code for turning x load into a single vector load is target independent and should be moved to the dag combiner The code for turning x load into a vector load can only handle a direct load from a global or a direct load from the stack It should be generalized to handle any load from P
Definition: README-SSE.txt:411
llvm::raw_string_ostream
A raw_ostream that writes to an std::string.
Definition: raw_ostream.h:628
llvm::SetVector< T, SmallVector< T, N >, SmallDenseSet< T, N > >::size
size_type size() const
Determine the number of elements in the SetVector.
Definition: SetVector.h:77
llvm::VPlanPrinter::dump
LLVM_DUMP_METHOD void dump()
Definition: VPlan.cpp:857
llvm::SmallVector
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
Definition: SmallVector.h:1199
llvm::VPTransformState::get
Value * get(VPValue *Def, unsigned Part)
Get the generated Value for a given VPValue and a given Part.
Definition: LoopVectorize.cpp:9865
llvm::VPlan::getOrAddExternalDef
VPValue * getOrAddExternalDef(Value *V)
Get the existing or add a new external definition for V.
Definition: VPlan.h:2608
llvm::VPRegionBlock::dropAllReferences
void dropAllReferences(VPValue *NewValue) override
Replace all operands of VPUsers in the block with NewValue and also replaces all uses of VPValues def...
Definition: VPlan.cpp:507
llvm::VPDef::print
virtual void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const =0
Each concrete VPDef prints itself.
llvm::DOT::EscapeString
std::string EscapeString(const std::string &Label)
Definition: GraphWriter.cpp:55
llvm::VPLane::Kind::First
@ First
For First, Lane is the index into the first N elements of a fixed-vector <N x <ElTy>> or a scalable v...
llvm::VPLane::Kind::ScalableLast
@ ScalableLast
For ScalableLast, Lane is the offset from the start of the last N-element subvector in a scalable vec...
llvm::IRBuilder<>
llvm::ScalarEvolution
The main scalar evolution driver.
Definition: ScalarEvolution.h:449
llvm::VPTransformState::set
void set(VPValue *Def, Value *V, unsigned Part)
Set the generated Value for a given VPValue and a given Part.
Definition: VPlan.h:264
llvm::DominatorTree
Concrete subclass of DominatorTreeBase that is used to compute a normal dominator tree.
Definition: Dominators.h:166
llvm::VPRegionBlock::getPreheaderVPBB
VPBasicBlock * getPreheaderVPBB()
Returns the pre-header VPBasicBlock of the loop region.
Definition: VPlan.h:2207
llvm::EnableFSDiscriminator
cl::opt< bool > EnableFSDiscriminator
Definition: TargetPassConfig.cpp:390
llvm::VPTransformState::CFGState::getPreheaderBBFor
BasicBlock * getPreheaderBBFor(VPRecipeBase *R)
Returns the BasicBlock* mapped to the pre-header of the loop region containing R.
Definition: VPlan.cpp:233
llvm::interleaveComma
void interleaveComma(const Container &c, StreamT &os, UnaryFunctor each_fn)
Definition: STLExtras.h:2072
llvm::Depth
@ Depth
Definition: SIMachineScheduler.h:36
llvm::DILocation
Debug location.
Definition: DebugInfoMetadata.h:1595
llvm::VPBasicBlock::dropAllReferences
void dropAllReferences(VPValue *NewValue) override
Replace all operands of VPUsers in the block with NewValue and also replaces all uses of VPValues def...
Definition: VPlan.cpp:390
llvm::VPInstruction::BranchOnCond
@ BranchOnCond
Definition: VPlan.h:796
llvm::sys::path::end
const_iterator end(StringRef path)
Get end iterator over path.
Definition: Path.cpp:235
llvm::VPValue::dump
void dump() const
Dump the value to stderr (for debugging).
Definition: VPlan.cpp:95
dumpEdges
static void dumpEdges(CFGMST< Edge, BBInfo > &MST, GCOVFunction &GF)
Definition: GCOVProfiling.cpp:747
llvm::BasicBlock::getSingleSuccessor
const BasicBlock * getSingleSuccessor() const
Return the successor of this block if it has a single successor.
Definition: BasicBlock.cpp:315
llvm::VPBasicBlock::getTerminator
VPRecipeBase * getTerminator()
If the block has multiple successors, return the branch recipe terminating the block.
Definition: VPlan.cpp:464
hasConditionalTerminator
static bool hasConditionalTerminator(const VPBasicBlock *VPBB)
Definition: VPlan.cpp:435
canSimplifyBranchOnCond
static bool canSimplifyBranchOnCond(VPInstruction *Term)
Definition: VPlan.cpp:588
T
#define T
Definition: Mips16ISelLowering.cpp:341
llvm::VPValue::print
void print(raw_ostream &OS, VPSlotTracker &Tracker) const
Definition: VPlan.cpp:88
llvm::successors
auto successors(MachineBasicBlock *BB)
Definition: MachineSSAContext.h:29
llvm::VPBasicBlock::isExiting
bool isExiting() const
Returns true if the block is exiting it's parent region.
Definition: VPlan.cpp:476
llvm::BasicBlock::getSinglePredecessor
const BasicBlock * getSinglePredecessor() const
Return the predecessor of this block if it has a single predecessor block.
Definition: BasicBlock.cpp:285
STLExtras.h
llvm::BasicBlock::hasNPredecessors
bool hasNPredecessors(unsigned N) const
Return true if this block has exactly N predecessors.
Definition: BasicBlock.cpp:307
llvm::VPBasicBlock::execute
void execute(VPTransformState *State) override
The method which generates the output IR instructions that correspond to this VPBasicBlock,...
Definition: VPlan.cpp:323
llvm::propagateMetadata
Instruction * propagateMetadata(Instruction *I, ArrayRef< Value * > VL)
Specifically, let Kinds = [MD_tbaa, MD_alias_scope, MD_noalias, MD_fpmath, MD_nontemporal,...
Definition: VectorUtils.cpp:877
llvm::VPRegionBlock::getEntry
const VPBlockBase * getEntry() const
Definition: VPlan.h:2176
llvm::Sched::Fast
@ Fast
Definition: TargetLowering.h:104
llvm::VPTransformState::hasScalarValue
bool hasScalarValue(VPValue *Def, VPIteration Instance)
Definition: VPlan.h:253
llvm::VPValue::VPValue
VPValue(const unsigned char SC, Value *UV=nullptr, VPDef *Def=nullptr)
Definition: VPlan.cpp:75
llvm::InterleaveGroup
The group of interleaved loads/stores sharing the same stride and close to each other.
Definition: VectorUtils.h:284
llvm::LinearPolySize::isScalable
bool isScalable() const
Returns whether the size is scaled by a runtime quantity (vscale).
Definition: TypeSize.h:298
llvm::VPBasicBlock::getEnclosingLoopRegion
VPRegionBlock * getEnclosingLoopRegion()
Definition: VPlan.cpp:425
LLVM_DEBUG
#define LLVM_DEBUG(X)
Definition: Debug.h:101
DepthFirstIterator.h
llvm::BasicBlock
LLVM Basic Block Representation.
Definition: BasicBlock.h:55
llvm::VPDef
This class augments a recipe with a set of VPValues defined by the recipe.
Definition: VPlanValue.h:325
llvm::dbgs
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
Definition: Debug.cpp:163
VPlanDominatorTree.h
llvm::VPTransformState::DataState::PerPartOutput
DenseMap< VPValue *, PerPartValuesTy > PerPartOutput
Definition: VPlan.h:227
llvm::ElementCount::isScalar
bool isScalar() const
Counting predicates.
Definition: TypeSize.h:414
llvm::X86AS::SS
@ SS
Definition: X86.h:201
Instruction.h
llvm::VPTransformState
VPTransformState holds information passed down when "executing" a VPlan, needed for generating the ou...
Definition: VPlan.h:205
CommandLine.h
llvm::VPBlockBase
VPBlockBase is the building block of the Hierarchical Control-Flow Graph.
Definition: VPlan.h:389
llvm::ConstantInt
This is the shared class of boolean and integer constants.
Definition: Constants.h:79
llvm::VPBasicBlock::appendRecipe
void appendRecipe(VPRecipeBase *Recipe)
Augment the existing recipes of a VPBasicBlock with an additional Recipe as the last recipe.
Definition: VPlan.h:2081
llvm::all_of
bool all_of(R &&range, UnaryPredicate P)
Provide wrappers to std::all_of which take ranges instead of having to pass begin/end explicitly.
Definition: STLExtras.h:1709
llvm::VPInstruction::ActiveLaneMask
@ ActiveLaneMask
Definition: VPlan.h:788
llvm::VPInstruction
This is a concrete Recipe that models a single VPlan-level instruction.
Definition: VPlan.h:775
llvm::PPCISD::SC
@ SC
CHAIN = SC CHAIN, Imm128 - System call.
Definition: PPCISelLowering.h:419
llvm::VPlan::print
void print(raw_ostream &O) const
Print this VPlan to O.
Definition: VPlan.cpp:755
llvm::vputils::getOrCreateVPValueForSCEVExpr
VPValue * getOrCreateVPValueForSCEVExpr(VPlan &Plan, const SCEV *Expr, ScalarEvolution &SE)
Get or create a VPValue that corresponds to the expansion of Expr.
Definition: VPlan.cpp:1106
llvm::PHINode::getIncomingValue
Value * getIncomingValue(unsigned i) const
Return incoming value number x.
Definition: Instructions.h:2794
E
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
llvm::VPTransformState::LI
LoopInfo * LI
Hold a pointer to LoopInfo to register new basic blocks in the loop.
Definition: VPlan.h:353
llvm::User
Definition: User.h:44
C
(vector float) vec_cmpeq(*A, *B) C
Definition: README_ALTIVEC.txt:86
Twine.h
llvm::VPBlockUtils::insertBlockAfter
static void insertBlockAfter(VPBlockBase *NewBlock, VPBlockBase *BlockPtr)
Insert disconnected VPBlockBase NewBlock after BlockPtr.
Definition: VPlan.h:2811
llvm::LoopBase::addChildLoop
void addChildLoop(LoopT *NewChild)
Add the specified loop to be a child of this loop.
Definition: LoopInfo.h:412
llvm::VPInstruction::Not
@ Not
Definition: VPlan.h:784
llvm::MachineBasicBlock::eraseFromParent
void eraseFromParent()
This method unlinks 'this' from the containing function and deletes it.
Definition: MachineBasicBlock.cpp:1347
llvm::LoopInfoBase::addTopLevelLoop
void addTopLevelLoop(LoopT *New)
This adds the specified loop to the collection of top-level loops.
Definition: LoopInfo.h:1049
llvm::IRBuilderBase::SetCurrentDebugLocation
void SetCurrentDebugLocation(DebugLoc L)
Set location information used by debugging information.
Definition: IRBuilder.h:211
llvm::VPBlockBase::getEnclosingBlockWithSuccessors
VPBlockBase * getEnclosingBlockWithSuccessors()
An Enclosing Block of a block B is any block containing B, including B itself.
Definition: VPlan.cpp:182
llvm::logicalview::LVSortMode::Line
@ Line
llvm::VPTransformState::DataState::PerPartScalars
DenseMap< VPValue *, ScalarsPerPartValuesTy > PerPartScalars
Definition: VPlan.h:230
llvm::Instruction
Definition: Instruction.h:42
llvm::InterleavedAccessInfo
Drive the analysis of interleaved memory accesses in the loop.
Definition: VectorUtils.h:759
llvm::VPValue::replaceAllUsesWith
void replaceAllUsesWith(VPValue *New)
Definition: VPlan.cpp:987
llvm::VPBlockBase::printSuccessors
void printSuccessors(raw_ostream &O, const Twine &Indent) const
Print the successors of this block to O, prefixing all lines with Indent.
Definition: VPlan.cpp:481
llvm::raw_ostream
This class implements an extremely fast bulk output stream that can only output to a stream.
Definition: raw_ostream.h:52
llvm::VPValue
Definition: VPlanValue.h:44
llvm::operator<<
raw_ostream & operator<<(raw_ostream &OS, const APFixedPoint &FX)
Definition: APFixedPoint.h:291
llvm::ConstantInt::get
static Constant * get(Type *Ty, uint64_t V, bool IsSigned=false)
If Ty is a vector type, return a Constant with a splat of the given value.
Definition: Constants.cpp:879
llvm::VPlan::getName
const std::string & getName() const
Definition: VPlan.h:2603
llvm::VPTransformState::CFGState::ExitBB
BasicBlock * ExitBB
The last IR BasicBlock in the output IR.
Definition: VPlan.h:339
llvm::DominatorTreeBase::changeImmediateDominator
void changeImmediateDominator(DomTreeNodeBase< NodeT > *N, DomTreeNodeBase< NodeT > *NewIDom)
changeImmediateDominator - This method is used to update the dominator tree information when a node's...
Definition: GenericDomTree.h:655
VPlan.h
llvm::VPBasicBlock::end
iterator end()
Definition: VPlan.h:2044
llvm::Align
This struct is a compact representation of a valid (non-zero power of two) alignment.
Definition: Alignment.h:39
Type.h
llvm::VPBasicBlock::empty
bool empty() const
Definition: VPlan.h:2053
llvm::VPRegionBlock
VPRegionBlock represents a collection of VPBasicBlocks and VPRegionBlocks which form a Single-Entry-S...
Definition: VPlan.h:2137
CFG.h
LoopInfo.h
llvm::LoopInfoBase::AllocateLoop
LoopT * AllocateLoop(ArgsTy &&... Args)
Definition: LoopInfo.h:956
EnableVPlanNativePath
cl::opt< bool > EnableVPlanNativePath
llvm::VPTransformState::setDebugLocFromInst
void setDebugLocFromInst(const Value *V)
Set the debug location in the builder using the debug location in V.
Definition: VPlan.cpp:258
llvm::VPBlockBase::getNumSuccessors
size_t getNumSuccessors() const
Definition: VPlan.h:506
BasicBlock.h
llvm::cl::opt< bool >
llvm::SCEV
This class represents an analyzed expression in the program.
Definition: ScalarEvolution.h:75
llvm::VPBlockBase::getHierarchicalSuccessors
const VPBlocksTy & getHierarchicalSuccessors()
Definition: VPlan.h:526
llvm::RISCVFenceField::O
@ O
Definition: RISCVBaseInfo.h:264
llvm::VPValue::Def
VPDef * Def
Pointer to the VPDef that defines this VPValue.
Definition: VPlanValue.h:65
llvm::VPLane::getAsRuntimeExpr
Value * getAsRuntimeExpr(IRBuilderBase &Builder, const ElementCount &VF) const
Returns an expression describing the lane index that can be used at runtime.
Definition: VPlan.cpp:62
llvm::VPSlotTracker::getSlot
unsigned getSlot(const VPValue *V) const
Definition: VPlanValue.h:465
uint64_t
llvm::VPBlockBase::getSingleHierarchicalSuccessor
VPBlockBase * getSingleHierarchicalSuccessor()
Definition: VPlan.h:532
llvm::VPTransformState::DT
DominatorTree * DT
Hold a pointer to Dominator Tree to register new basic blocks in the loop.
Definition: VPlan.h:356
llvm::RegionBase::getEntry
BlockT * getEntry() const
Get the entry BasicBlock of the Region.
Definition: RegionInfo.h:322
llvm::PHINode::setIncomingBlock
void setIncomingBlock(unsigned i, BasicBlock *BB)
Definition: Instructions.h:2833
llvm::Function::isDebugInfoForProfiling
bool isDebugInfoForProfiling() const
Returns true if we should emit debug info for profiling.
Definition: Metadata.cpp:1630
llvm::BranchInst::Create
static BranchInst * Create(BasicBlock *IfTrue, Instruction *InsertBefore=nullptr)
Definition: Instructions.h:3189
llvm::VPTransformState::UF
unsigned UF
Definition: VPlan.h:214
llvm::DenseMap
Definition: DenseMap.h:714
llvm::StringRef::rtrim
StringRef rtrim(char Char) const
Return string with consecutive Char characters starting from the right removed.
Definition: StringRef.h:795
llvm::LoopInfoBase::getLoopFor
LoopT * getLoopFor(const BlockT *BB) const
Return the inner most loop that BB lives in.
Definition: LoopInfo.h:992
I
#define I(x, y, z)
Definition: MD5.cpp:58
llvm::VPIteration
VPIteration represents a single point in the iteration space of the output (vectorized and/or unrolle...
Definition: VPlan.h:188
llvm::succ_begin
Interval::succ_iterator succ_begin(Interval *I)
succ_begin/succ_end - define methods so that Intervals may be used just like BasicBlocks can with the...
Definition: Interval.h:99
llvm::VPBlockUtils::connectBlocks
static void connectBlocks(VPBlockBase *From, VPBlockBase *To)
Connect VPBlockBases From and To bi-directionally.
Definition: VPlan.h:2847
llvm::VPRegionBlock::getExiting
const VPBlockBase * getExiting() const
Definition: VPlan.h:2194
llvm::make_early_inc_range
iterator_range< early_inc_iterator_impl< detail::IterOfRange< RangeT > > > make_early_inc_range(RangeT &&Range)
Make a range that does early increment to allow mutation of the underlying range without disrupting i...
Definition: STLExtras.h:716
llvm::VPTransformState::CFGState::PrevVPBB
VPBasicBlock * PrevVPBB
The previous VPBasicBlock visited. Initially set to null.
Definition: VPlan.h:331
llvm::IRBuilderBase::CreateUnreachable
UnreachableInst * CreateUnreachable()
Definition: IRBuilder.h:1166
llvm::VPBlockUtils::disconnectBlocks
static void disconnectBlocks(VPBlockBase *From, VPBlockBase *To)
Disconnect VPBlockBases From and To bi-directionally.
Definition: VPlan.h:2858
llvm::ilist_node_with_parent::getPrevNode
NodeTy * getPrevNode()
Definition: ilist_node.h:275
IRBuilder.h
llvm::VPlan::getEntry
VPBlockBase * getEntry()
Definition: VPlan.h:2569
assert
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
getPlanEntry
static T * getPlanEntry(T *Start)
Definition: VPlan.cpp:122
IsCondBranch
static bool IsCondBranch(unsigned BrOpc)
Definition: XCoreInstrInfo.cpp:122
llvm::IRBuilderBase::CreateExtractElement
Value * CreateExtractElement(Value *Vec, Value *Idx, const Twine &Name="")
Definition: IRBuilder.h:2316
llvm::VPLiveOut
A value that is used outside the VPlan.
Definition: VPlan.h:634
llvm::IRBuilderBase::getTrue
ConstantInt * getTrue()
Get the constant value for i1 true.
Definition: IRBuilder.h:449
llvm::VPUser::printOperands
void printOperands(raw_ostream &O, VPSlotTracker &SlotTracker) const
Print the operands to O.
Definition: VPlan.cpp:1018
llvm::LinearPolySize::getKnownMinValue
ScalarTy getKnownMinValue() const
Returns the minimum value this size can represent.
Definition: TypeSize.h:296
llvm::VPTransformState::addNewMetadata
void addNewMetadata(Instruction *To, const Instruction *Orig)
Add additional metadata to To that was not present on Orig.
Definition: VPlan.cpp:238
Builder
assume Assume Builder
Definition: AssumeBundleBuilder.cpp:651
llvm::User::setOperand
void setOperand(unsigned i, Value *Val)
Definition: User.h:174
llvm::SetVector< T, SmallVector< T, N >, SmallDenseSet< T, N > >::insert
bool insert(const value_type &X)
Insert a new element into the SetVector.
Definition: SetVector.h:141
llvm::print
Printable print(const GCNRegPressure &RP, const GCNSubtarget *ST=nullptr)
Definition: GCNRegPressure.cpp:138
llvm::VPlan::getVectorLoopRegion
VPRegionBlock * getVectorLoopRegion()
Returns the VPRegionBlock of the vector loop.
Definition: VPlan.h:2682
llvm::VPTransformState::addMetadata
void addMetadata(Instruction *To, Instruction *From)
Add metadata from one instruction to another.
Definition: VPlan.cpp:246
llvm::getRuntimeVF
Value * getRuntimeVF(IRBuilderBase &B, Type *Ty, ElementCount VF)
Return the runtime value for VF.
Definition: LoopVectorize.cpp:963
llvm::ArrayRef
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
Definition: APInt.h:32
llvm::VPBlockBase::getSingleSuccessor
VPBlockBase * getSingleSuccessor() const
Definition: VPlan.h:496
llvm::StringRef
StringRef - Represent a constant reference to a string, i.e.
Definition: StringRef.h:50
llvm::VPInstruction::BranchOnCount
@ BranchOnCount
Definition: VPlan.h:795
Printer
Memory true print Memory SSA Printer
Definition: MemorySSA.cpp:80
llvm_unreachable
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
Definition: ErrorHandling.h:143
llvm::Value::getType
Type * getType() const
All values are typed, get the type of this value.
Definition: Value.h:255
llvm::Instruction::getFunction
const Function * getFunction() const
Return the function this instruction belongs to.
Definition: Instruction.cpp:73
llvm::BasicBlock::Create
static BasicBlock * Create(LLVMContext &Context, const Twine &Name="", Function *Parent=nullptr, BasicBlock *InsertBefore=nullptr)
Creates a new BasicBlock.
Definition: BasicBlock.h:97
getParent
static const Function * getParent(const Value *V)
Definition: BasicAliasAnalysis.cpp:805
llvm::VPRecipeBase::getParent
VPBasicBlock * getParent()
Definition: VPlan.h:682
llvm::IRBuilderBase
Common base class shared among various IRBuilders.
Definition: IRBuilder.h:93
DL
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
Definition: AArch64SLSHardening.cpp:76
llvm::DominatorTreeBase
Core dominator tree base class.
Definition: LoopInfo.h:66
llvm::VPBasicBlock::splitAt
VPBasicBlock * splitAt(iterator SplitAt)
Split current block at SplitAt by inserting a new block between the current block and its successors ...
Definition: VPlan.cpp:400
llvm::VPlan::execute
void execute(VPTransformState *State)
Generate the IR code for this VPlan.
Definition: VPlan.cpp:674
llvm::VPTransformState::VF
ElementCount VF
The chosen Vectorization and Unroll Factors of the loop being vectorized.
Definition: VPlan.h:213
llvm::Value::getName
StringRef getName() const
Return a constant reference to the value's name.
Definition: Value.cpp:308
llvm::VPlan::addLiveOut
void addLiveOut(PHINode *PN, VPValue *V)
Definition: VPlan.cpp:800
llvm::VPBlockBase::getEntryBasicBlock
const VPBasicBlock * getEntryBasicBlock() const
Definition: VPlan.cpp:147
llvm::vputils::onlyFirstLaneUsed
bool onlyFirstLaneUsed(VPValue *Def)
Returns true if only the first lane of Def is used.
Definition: VPlan.cpp:1101
llvm::DominatorTreeBase::addNewBlock
DomTreeNodeBase< NodeT > * addNewBlock(NodeT *BB, NodeT *DomBB)
Add a new node to the dominator tree information.
Definition: GenericDomTree.h:619
llvm::VPTransformState::Data
struct llvm::VPTransformState::DataState Data
llvm::BasicBlock::getContext
LLVMContext & getContext() const
Get the context in which this basic block lives.
Definition: BasicBlock.cpp:35
llvm::depth_first
iterator_range< df_iterator< T > > depth_first(const T &G)
Definition: DepthFirstIterator.h:230
llvm::VPTransformState::LVer
std::unique_ptr< LoopVersioning > LVer
LoopVersioning.
Definition: VPlan.h:384
llvm::VPDef::dump
void dump() const
Dump the VPDef to stderr (for debugging).
Definition: VPlan.cpp:103
llvm::Twine
Twine - A lightweight data structure for efficiently representing the concatenation of temporary valu...
Definition: Twine.h:81
llvm::VPSlotTracker
This class can be used to assign consecutive numbers to all VPValues in a VPlan and allows querying t...
Definition: VPlanValue.h:452
llvm::GraphProgram::Name
Name
Definition: GraphWriter.h:50
GraphWriter.h
llvm::VPBlockBase::getParent
VPRegionBlock * getParent()
Definition: VPlan.h:461
llvm::VPTransformState::CurrentVectorLoop
Loop * CurrentVectorLoop
The loop object for the current parent region, or nullptr.
Definition: VPlan.h:377
llvm::Successor
@ Successor
Definition: SIMachineScheduler.h:35
llvm::AMDGPU::SendMsg::Op
Op
Definition: SIDefines.h:348
llvm::ilist_iterator
Iterator for intrusive lists based on ilist_node.
Definition: ilist_iterator.h:57
llvm::VPRecipeBase
VPRecipeBase is a base class modeling a sequence of one or more output IR instructions.
Definition: VPlan.h:663
llvm::AArch64CC::LS
@ LS
Definition: AArch64BaseInfo.h:264
Casting.h
llvm::VPRegionBlock::print
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print this VPRegionBlock to O (recursively), prefixing all lines with Indent.
Definition: VPlan.cpp:565
llvm::SlotTracker
This class provides computation of slot numbers for LLVM Assembly writing.
Definition: AsmWriter.cpp:665
GenericDomTreeConstruction.h
llvm::VPTransformState::CFGState
Hold state information used when constructing the CFG of the output IR, traversing the VPBasicBlocks ...
Definition: VPlan.h:329
LoopVersioning.h
llvm::ReversePostOrderTraversal
Definition: PostOrderIterator.h:291
llvm::VPUser
This class augments VPValue with operands which provide the inverse def-use edges from VPValue's user...
Definition: VPlanValue.h:220
llvm::VPlan::prepareToExecute
void prepareToExecute(Value *TripCount, Value *VectorTripCount, Value *CanonicalIVStartValue, VPTransformState &State, bool IsEpilogueVectorization)
Prepare the plan for execution, setting up the required live-in values.
Definition: VPlan.cpp:597
Instructions.h
PostOrderIterator.h
llvm::User::getNumOperands
unsigned getNumOperands() const
Definition: User.h:191
SmallVector.h
llvm::VPTransformState::VPValue2Value
VPValue2ValueTy VPValue2Value
Definition: VPlan.h:361
llvm::Instruction::getDebugLoc
const DebugLoc & getDebugLoc() const
Return the debug location for this node as a DebugLoc.
Definition: Instruction.h:354
llvm::VPBasicBlock::begin
iterator begin()
Recipe iterator methods.
Definition: VPlan.h:2042
llvm::LoopBase::addBasicBlockToLoop
void addBasicBlockToLoop(BlockT *NewBB, LoopInfoBase< BlockT, LoopT > &LI)
This method is used by other analyses to update loop information.
Definition: LoopInfoImpl.h:258
llvm::VPActiveLaneMaskPHIRecipe
A recipe for generating the active lane mask for the vector loop that is used to predicate the vector...
Definition: VPlan.h:1878
llvm::VPUser::getOperand
VPValue * getOperand(unsigned N) const
Definition: VPlanValue.h:270
llvm::VPTransformState::Instance
Optional< VPIteration > Instance
Hold the indices to generate specific scalar instructions.
Definition: VPlan.h:219
llvm::VPlan::printDOT
void printDOT(raw_ostream &O) const
Print this VPlan in DOT format to O.
Definition: VPlan.cpp:791
llvm::CallingConv::Tail
@ Tail
Attemps to make calls as fast as possible while guaranteeing that tail call optimization can always b...
Definition: CallingConv.h:76
llvm::VPBasicBlock::print
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print this VPBsicBlock to O, prefixing all lines with Indent.
Definition: VPlan.cpp:493
llvm::DominatorTreeBase::verify
bool verify(VerificationLevel VL=VerificationLevel::Full) const
verify - checks if the tree is correct.
Definition: GenericDomTree.h:802
Users
iv Induction Variable Users
Definition: IVUsers.cpp:48
llvm::VPExpandSCEVRecipe
Recipe to expand a SCEV expression.
Definition: VPlan.h:1798
llvm::PHINode
Definition: Instructions.h:2698
llvm::BasicBlock::getTerminator
const Instruction * getTerminator() const LLVM_READONLY
Returns the terminator instruction if the block is well formed or null if the block is not well forme...
Definition: BasicBlock.h:119
llvm::VPBlockBase::getExitingBasicBlock
const VPBasicBlock * getExitingBasicBlock() const
Definition: VPlan.cpp:168
IV
static const uint32_t IV[8]
Definition: blake3_impl.h:85
llvm::VPTransformState::CFGState::VPBB2IRBB
SmallDenseMap< VPBasicBlock *, BasicBlock * > VPBB2IRBB
A mapping of each VPBasicBlock to the corresponding BasicBlock.
Definition: VPlan.h:343
llvm::VPTransformState::Plan
VPlan * Plan
Pointer to the VPlan code is generated for.
Definition: VPlan.h:370
llvm::VPLane
In what follows, the term "input IR" refers to code that is fed into the vectorizer whereas the term ...
Definition: VPlan.h:112
llvm::IRBuilderBase::CreateVectorSplat
Value * CreateVectorSplat(unsigned NumElts, Value *V, const Twine &Name="")
Return a vector value that contains.
Definition: IRBuilder.cpp:1246
llvm::SmallSetVector
A SetVector that performs no allocations if smaller than a certain size.
Definition: SetVector.h:307
BB
Common register allocation spilling lr str ldr sxth r3 ldr mla r4 can lr mov lr str ldr sxth r3 mla r4 and then merge mul and lr str ldr sxth r3 mla r4 It also increase the likelihood the store may become dead bb27 Successors according to LLVM BB
Definition: README.txt:39
GEP
Hexagon Common GEP
Definition: HexagonCommonGEP.cpp:171
llvm::UnreachableInst
This function has undefined behavior.
Definition: Instructions.h:4770
llvm::VPBlockBase::deleteCFG
static void deleteCFG(VPBlockBase *Entry)
Delete all blocks reachable from a given VPBlockBase, inclusive.
Definition: VPlan.cpp:198
From
BlockVerifier::State From
Definition: BlockVerifier.cpp:55
llvm::IRBuilderBase::CreateSub
Value * CreateSub(Value *LHS, Value *RHS, const Twine &Name="", bool HasNUW=false, bool HasNSW=false)
Definition: IRBuilder.h:1250
llvm::DebugLoc
A debug info location.
Definition: DebugLoc.h:33
llvm::User::getOperand
Value * getOperand(unsigned i) const
Definition: User.h:169
llvm::Region
Definition: RegionInfo.h:889
llvm::VPValue::getDefiningRecipe
VPRecipeBase * getDefiningRecipe()
Returns the recipe defining this VPValue or nullptr if it is not defined by a recipe,...
Definition: VPlan.cpp:112
llvm::VPValue::~VPValue
virtual ~VPValue()
Definition: VPlan.cpp:81
raw_ostream.h
llvm::VPBasicBlock::getFirstNonPhi
iterator getFirstNonPhi()
Return the position of the first non-phi node recipe in the block.
Definition: VPlan.cpp:205
llvm::VPInterleavedAccessInfo::VPInterleavedAccessInfo
VPInterleavedAccessInfo(VPlan &Plan, InterleavedAccessInfo &IAI)
Definition: VPlan.cpp:1070
llvm::VPBlockRecursiveTraversalWrapper
Helper for GraphTraits specialization that traverses through VPRegionBlocks.
Definition: VPlan.h:2436
BasicBlockUtils.h
llvm::SplitBlock
BasicBlock * SplitBlock(BasicBlock *Old, Instruction *SplitPt, DominatorTree *DT, LoopInfo *LI=nullptr, MemorySSAUpdater *MSSAU=nullptr, const Twine &BBName="", bool Before=false)
Split the specified block at the specified instruction.
Definition: BasicBlockUtils.cpp:918
Value.h
llvm::pdb::PDB_SymType::Block
@ Block
llvm::VPlan
VPlan models a candidate for vectorization, encoding various decisions take to produce efficient outp...
Definition: VPlan.h:2492
llvm::StringRef::split
std::pair< StringRef, StringRef > split(char Separator) const
Split into two substrings around the first occurrence of a separator character.
Definition: StringRef.h:692
llvm::VPBlockBase::setPlan
void setPlan(VPlan *ParentPlan)
Sets the pointer of the plan containing the block.
Definition: VPlan.cpp:161
llvm::MCID::Terminator
@ Terminator
Definition: MCInstrDesc.h:157
llvm::Value
LLVM Value Representation.
Definition: Value.h:74
Debug.h
llvm::VPlan::dump
LLVM_DUMP_METHOD void dump() const
Dump the plan to stderr (for debugging).
Definition: VPlan.cpp:797
llvm::VPValue::printAsOperand
void printAsOperand(raw_ostream &OS, VPSlotTracker &Tracker) const
Definition: VPlan.cpp:1003
llvm::VPlanPrinter
VPlanPrinter prints a given VPlan to a given output stream.
Definition: VPlan.h:2731
llvm::VPTransformState::CFGState::PrevBB
BasicBlock * PrevBB
The previous IR BasicBlock created or used.
Definition: VPlan.h:335
llvm::Instruction::moveBefore
void moveBefore(Instruction *MovePos)
Unlink this instruction from its current basic block and insert it into the basic block that MovePos ...
Definition: Instruction.cpp:100
llvm::VPTransformState::hasVectorValue
bool hasVectorValue(VPValue *Def, unsigned Part)
Definition: VPlan.h:243
llvm::InterleavedAccessInfo::getInterleaveGroup
InterleaveGroup< Instruction > * getInterleaveGroup(const Instruction *Instr) const
Get the interleave group that Instr belongs to.
Definition: VectorUtils.h:804