LLVM  13.0.0git
VPlan.cpp
Go to the documentation of this file.
1 //===- VPlan.cpp - Vectorizer Plan ----------------------------------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 ///
9 /// \file
10 /// This is the LLVM vectorization plan. It represents a candidate for
11 /// vectorization, allowing to plan and optimize how to vectorize a given loop
12 /// before generating LLVM-IR.
13 /// The vectorizer uses vectorization plans to estimate the costs of potential
14 /// candidates and if profitable to execute the desired plan, generating vector
15 /// LLVM-IR code.
16 ///
17 //===----------------------------------------------------------------------===//
18 
19 #include "VPlan.h"
20 #include "VPlanDominatorTree.h"
23 #include "llvm/ADT/STLExtras.h"
24 #include "llvm/ADT/SmallVector.h"
25 #include "llvm/ADT/Twine.h"
27 #include "llvm/Analysis/LoopInfo.h"
28 #include "llvm/IR/BasicBlock.h"
29 #include "llvm/IR/CFG.h"
30 #include "llvm/IR/InstrTypes.h"
31 #include "llvm/IR/Instruction.h"
32 #include "llvm/IR/Instructions.h"
33 #include "llvm/IR/Type.h"
34 #include "llvm/IR/Value.h"
35 #include "llvm/Support/Casting.h"
37 #include "llvm/Support/Debug.h"
43 #include <cassert>
44 #include <iterator>
45 #include <string>
46 #include <vector>
47 
48 using namespace llvm;
50 
51 #define DEBUG_TYPE "vplan"
52 
54  const VPInstruction *Instr = dyn_cast<VPInstruction>(&V);
56  (Instr && Instr->getParent()) ? Instr->getParent()->getPlan() : nullptr);
57  V.print(OS, SlotTracker);
58  return OS;
59 }
60 
61 VPValue::VPValue(const unsigned char SC, Value *UV, VPDef *Def)
62  : SubclassID(SC), UnderlyingVal(UV), Def(Def) {
63  if (Def)
64  Def->addDefinedValue(this);
65 }
66 
68  assert(Users.empty() && "trying to delete a VPValue with remaining users");
69  if (Def)
70  Def->removeDefinedValue(this);
71 }
72 
74  if (const VPRecipeBase *R = dyn_cast_or_null<VPRecipeBase>(Def))
75  R->print(OS, "", SlotTracker);
76  else
78 }
79 
80 void VPValue::dump() const {
81  const VPRecipeBase *Instr = dyn_cast_or_null<VPRecipeBase>(this->Def);
83  (Instr && Instr->getParent()) ? Instr->getParent()->getPlan() : nullptr);
85  dbgs() << "\n";
86 }
87 
88 void VPDef::dump() const {
89  const VPRecipeBase *Instr = dyn_cast_or_null<VPRecipeBase>(this);
91  (Instr && Instr->getParent()) ? Instr->getParent()->getPlan() : nullptr);
92  print(dbgs(), "", SlotTracker);
93  dbgs() << "\n";
94 }
95 
96 // Get the top-most entry block of \p Start. This is the entry block of the
97 // containing VPlan. This function is templated to support both const and non-const blocks
98 template <typename T> static T *getPlanEntry(T *Start) {
99  T *Next = Start;
100  T *Current = Start;
101  while ((Next = Next->getParent()))
102  Current = Next;
103 
104  SmallSetVector<T *, 8> WorkList;
105  WorkList.insert(Current);
106 
107  for (unsigned i = 0; i < WorkList.size(); i++) {
108  T *Current = WorkList[i];
109  if (Current->getNumPredecessors() == 0)
110  return Current;
111  auto &Predecessors = Current->getPredecessors();
112  WorkList.insert(Predecessors.begin(), Predecessors.end());
113  }
114 
115  llvm_unreachable("VPlan without any entry node without predecessors");
116 }
117 
118 VPlan *VPBlockBase::getPlan() { return getPlanEntry(this)->Plan; }
119 
120 const VPlan *VPBlockBase::getPlan() const { return getPlanEntry(this)->Plan; }
121 
122 /// \return the VPBasicBlock that is the entry of Block, possibly indirectly.
124  const VPBlockBase *Block = this;
125  while (const VPRegionBlock *Region = dyn_cast<VPRegionBlock>(Block))
126  Block = Region->getEntry();
127  return cast<VPBasicBlock>(Block);
128 }
129 
131  VPBlockBase *Block = this;
132  while (VPRegionBlock *Region = dyn_cast<VPRegionBlock>(Block))
133  Block = Region->getEntry();
134  return cast<VPBasicBlock>(Block);
135 }
136 
137 void VPBlockBase::setPlan(VPlan *ParentPlan) {
138  assert(ParentPlan->getEntry() == this &&
139  "Can only set plan on its entry block.");
140  Plan = ParentPlan;
141 }
142 
143 /// \return the VPBasicBlock that is the exit of Block, possibly indirectly.
145  const VPBlockBase *Block = this;
146  while (const VPRegionBlock *Region = dyn_cast<VPRegionBlock>(Block))
147  Block = Region->getExit();
148  return cast<VPBasicBlock>(Block);
149 }
150 
152  VPBlockBase *Block = this;
153  while (VPRegionBlock *Region = dyn_cast<VPRegionBlock>(Block))
154  Block = Region->getExit();
155  return cast<VPBasicBlock>(Block);
156 }
157 
159  if (!Successors.empty() || !Parent)
160  return this;
161  assert(Parent->getExit() == this &&
162  "Block w/o successors not the exit of its parent.");
163  return Parent->getEnclosingBlockWithSuccessors();
164 }
165 
167  if (!Predecessors.empty() || !Parent)
168  return this;
169  assert(Parent->getEntry() == this &&
170  "Block w/o predecessors not the entry of its parent.");
171  return Parent->getEnclosingBlockWithPredecessors();
172 }
173 
175  if (U.getNumOperands() == 1)
176  return U.getOperand(0);
177 
178  return nullptr;
179 }
180 
181 static const VPValue *getSingleOperandOrNull(const VPUser &U) {
182  if (U.getNumOperands() == 1)
183  return U.getOperand(0);
184 
185  return nullptr;
186 }
187 
188 static void resetSingleOpUser(VPUser &U, VPValue *NewVal) {
189  assert(U.getNumOperands() <= 1 && "Didn't expect more than one operand!");
190  if (!NewVal) {
191  if (U.getNumOperands() == 1)
192  U.removeLastOperand();
193  return;
194  }
195 
196  if (U.getNumOperands() == 1)
197  U.setOperand(0, NewVal);
198  else
199  U.addOperand(NewVal);
200 }
201 
203  return getSingleOperandOrNull(CondBitUser);
204 }
205 
206 const VPValue *VPBlockBase::getCondBit() const {
207  return getSingleOperandOrNull(CondBitUser);
208 }
209 
211  resetSingleOpUser(CondBitUser, CV);
212 }
213 
215  return getSingleOperandOrNull(PredicateUser);
216 }
217 
218 const VPValue *VPBlockBase::getPredicate() const {
219  return getSingleOperandOrNull(PredicateUser);
220 }
221 
223  resetSingleOpUser(PredicateUser, CV);
224 }
225 
228 
229  for (VPBlockBase *Block : Blocks)
230  delete Block;
231 }
232 
234  iterator It = begin();
235  while (It != end() && (isa<VPWidenPHIRecipe>(&*It) ||
236  isa<VPWidenIntOrFpInductionRecipe>(&*It) ||
237  isa<VPPredInstPHIRecipe>(&*It) ||
238  isa<VPWidenCanonicalIVRecipe>(&*It)))
239  It++;
240  return It;
241 }
242 
244  if (!Def->getDef())
245  return Def->getLiveInIRValue();
246 
248  return Data.PerPartScalars[Def][Instance.Part][Instance.Lane];
249 
251  auto *VecPart = Data.PerPartOutput[Def][Instance.Part];
252  if (!VecPart->getType()->isVectorTy()) {
253  assert(Instance.Lane == 0 && "cannot get lane > 0 for scalar");
254  return VecPart;
255  }
256  // TODO: Cache created scalar values.
257  auto *Extract =
259  // set(Def, Extract, Instance);
260  return Extract;
261 }
262 
263 BasicBlock *
264 VPBasicBlock::createEmptyBasicBlock(VPTransformState::CFGState &CFG) {
265  // BB stands for IR BasicBlocks. VPBB stands for VPlan VPBasicBlocks.
266  // Pred stands for Predessor. Prev stands for Previous - last visited/created.
267  BasicBlock *PrevBB = CFG.PrevBB;
268  BasicBlock *NewBB = BasicBlock::Create(PrevBB->getContext(), getName(),
269  PrevBB->getParent(), CFG.LastBB);
270  LLVM_DEBUG(dbgs() << "LV: created " << NewBB->getName() << '\n');
271 
272  // Hook up the new basic block to its predecessors.
273  for (VPBlockBase *PredVPBlock : getHierarchicalPredecessors()) {
274  VPBasicBlock *PredVPBB = PredVPBlock->getExitBasicBlock();
275  auto &PredVPSuccessors = PredVPBB->getSuccessors();
276  BasicBlock *PredBB = CFG.VPBB2IRBB[PredVPBB];
277 
278  // In outer loop vectorization scenario, the predecessor BBlock may not yet
279  // be visited(backedge). Mark the VPBasicBlock for fixup at the end of
280  // vectorization. We do not encounter this case in inner loop vectorization
281  // as we start out by building a loop skeleton with the vector loop header
282  // and latch blocks. As a result, we never enter this function for the
283  // header block in the non VPlan-native path.
284  if (!PredBB) {
286  "Unexpected null predecessor in non VPlan-native path");
287  CFG.VPBBsToFix.push_back(PredVPBB);
288  continue;
289  }
290 
291  assert(PredBB && "Predecessor basic-block not found building successor.");
292  auto *PredBBTerminator = PredBB->getTerminator();
293  LLVM_DEBUG(dbgs() << "LV: draw edge from" << PredBB->getName() << '\n');
294  if (isa<UnreachableInst>(PredBBTerminator)) {
295  assert(PredVPSuccessors.size() == 1 &&
296  "Predecessor ending w/o branch must have single successor.");
297  PredBBTerminator->eraseFromParent();
298  BranchInst::Create(NewBB, PredBB);
299  } else {
300  assert(PredVPSuccessors.size() == 2 &&
301  "Predecessor ending with branch must have two successors.");
302  unsigned idx = PredVPSuccessors.front() == this ? 0 : 1;
303  assert(!PredBBTerminator->getSuccessor(idx) &&
304  "Trying to reset an existing successor block.");
305  PredBBTerminator->setSuccessor(idx, NewBB);
306  }
307  }
308  return NewBB;
309 }
310 
312  bool Replica = State->Instance && !State->Instance->isFirstIteration();
313  VPBasicBlock *PrevVPBB = State->CFG.PrevVPBB;
314  VPBlockBase *SingleHPred = nullptr;
315  BasicBlock *NewBB = State->CFG.PrevBB; // Reuse it if possible.
316 
317  // 1. Create an IR basic block, or reuse the last one if possible.
318  // The last IR basic block is reused, as an optimization, in three cases:
319  // A. the first VPBB reuses the loop header BB - when PrevVPBB is null;
320  // B. when the current VPBB has a single (hierarchical) predecessor which
321  // is PrevVPBB and the latter has a single (hierarchical) successor; and
322  // C. when the current VPBB is an entry of a region replica - where PrevVPBB
323  // is the exit of this region from a previous instance, or the predecessor
324  // of this region.
325  if (PrevVPBB && /* A */
326  !((SingleHPred = getSingleHierarchicalPredecessor()) &&
327  SingleHPred->getExitBasicBlock() == PrevVPBB &&
328  PrevVPBB->getSingleHierarchicalSuccessor()) && /* B */
329  !(Replica && getPredecessors().empty())) { /* C */
330  NewBB = createEmptyBasicBlock(State->CFG);
331  State->Builder.SetInsertPoint(NewBB);
332  // Temporarily terminate with unreachable until CFG is rewired.
335  // Register NewBB in its loop. In innermost loops its the same for all BB's.
336  Loop *L = State->LI->getLoopFor(State->CFG.LastBB);
337  L->addBasicBlockToLoop(NewBB, *State->LI);
338  State->CFG.PrevBB = NewBB;
339  }
340 
341  // 2. Fill the IR basic block with IR instructions.
342  LLVM_DEBUG(dbgs() << "LV: vectorizing VPBB:" << getName()
343  << " in BB:" << NewBB->getName() << '\n');
344 
345  State->CFG.VPBB2IRBB[this] = NewBB;
346  State->CFG.PrevVPBB = this;
347 
348  for (VPRecipeBase &Recipe : Recipes)
349  Recipe.execute(*State);
350 
351  VPValue *CBV;
352  if (EnableVPlanNativePath && (CBV = getCondBit())) {
353  assert(CBV->getUnderlyingValue() &&
354  "Unexpected null underlying value for condition bit");
355 
356  // Condition bit value in a VPBasicBlock is used as the branch selector. In
357  // the VPlan-native path case, since all branches are uniform we generate a
358  // branch instruction using the condition value from vector lane 0 and dummy
359  // successors. The successors are fixed later when the successor blocks are
360  // visited.
361  Value *NewCond = State->get(CBV, {0, 0});
362 
363  // Replace the temporary unreachable terminator with the new conditional
364  // branch.
365  auto *CurrentTerminator = NewBB->getTerminator();
366  assert(isa<UnreachableInst>(CurrentTerminator) &&
367  "Expected to replace unreachable terminator with conditional "
368  "branch.");
369  auto *CondBr = BranchInst::Create(NewBB, nullptr, NewCond);
370  CondBr->setSuccessor(0, nullptr);
371  ReplaceInstWithInst(CurrentTerminator, CondBr);
372  }
373 
374  LLVM_DEBUG(dbgs() << "LV: filled BB:" << *NewBB);
375 }
376 
378  for (VPRecipeBase &R : Recipes) {
379  for (auto *Def : R.definedValues())
380  Def->replaceAllUsesWith(NewValue);
381 
382  for (unsigned I = 0, E = R.getNumOperands(); I != E; I++)
383  R.setOperand(I, NewValue);
384  }
385 }
386 
388  for (VPBlockBase *Block : depth_first(Entry))
389  // Drop all references in VPBasicBlocks and replace all uses with
390  // DummyValue.
391  Block->dropAllReferences(NewValue);
392 }
393 
396 
397  if (!isReplicator()) {
398  // Visit the VPBlocks connected to "this", starting from it.
399  for (VPBlockBase *Block : RPOT) {
400  if (EnableVPlanNativePath) {
401  // The inner loop vectorization path does not represent loop preheader
402  // and exit blocks as part of the VPlan. In the VPlan-native path, skip
403  // vectorizing loop preheader block. In future, we may replace this
404  // check with the check for loop preheader.
405  if (Block->getNumPredecessors() == 0)
406  continue;
407 
408  // Skip vectorizing loop exit block. In future, we may replace this
409  // check with the check for loop exit.
410  if (Block->getNumSuccessors() == 0)
411  continue;
412  }
413 
414  LLVM_DEBUG(dbgs() << "LV: VPBlock in RPO " << Block->getName() << '\n');
415  Block->execute(State);
416  }
417  return;
418  }
419 
420  assert(!State->Instance && "Replicating a Region with non-null instance.");
421 
422  // Enter replicating mode.
423  State->Instance = VPIteration(0, 0);
424 
425  for (unsigned Part = 0, UF = State->UF; Part < UF; ++Part) {
426  State->Instance->Part = Part;
427  assert(!State->VF.isScalable() && "VF is assumed to be non scalable.");
428  for (unsigned Lane = 0, VF = State->VF.getKnownMinValue(); Lane < VF;
429  ++Lane) {
430  State->Instance->Lane = Lane;
431  // Visit the VPBlocks connected to \p this, starting from it.
432  for (VPBlockBase *Block : RPOT) {
433  LLVM_DEBUG(dbgs() << "LV: VPBlock in RPO " << Block->getName() << '\n');
434  Block->execute(State);
435  }
436  }
437  }
438 
439  // Exit replicating mode.
440  State->Instance.reset();
441 }
442 
444  assert(!Parent && "Recipe already in some VPBasicBlock");
445  assert(InsertPos->getParent() &&
446  "Insertion position not in any VPBasicBlock");
447  Parent = InsertPos->getParent();
448  Parent->getRecipeList().insert(InsertPos->getIterator(), this);
449 }
450 
452  assert(!Parent && "Recipe already in some VPBasicBlock");
453  assert(InsertPos->getParent() &&
454  "Insertion position not in any VPBasicBlock");
455  Parent = InsertPos->getParent();
456  Parent->getRecipeList().insertAfter(InsertPos->getIterator(), this);
457 }
458 
460  assert(getParent() && "Recipe not in any VPBasicBlock");
462  Parent = nullptr;
463 }
464 
466  assert(getParent() && "Recipe not in any VPBasicBlock");
467  return getParent()->getRecipeList().erase(getIterator());
468 }
469 
472  insertAfter(InsertPos);
473 }
474 
477  assert(I == BB.end() || I->getParent() == &BB);
479  Parent = &BB;
480  BB.getRecipeList().insert(I, this);
481 }
482 
483 void VPInstruction::generateInstruction(VPTransformState &State,
484  unsigned Part) {
485  IRBuilder<> &Builder = State.Builder;
486 
488  Value *A = State.get(getOperand(0), Part);
489  Value *B = State.get(getOperand(1), Part);
490  Value *V = Builder.CreateBinOp((Instruction::BinaryOps)getOpcode(), A, B);
491  State.set(this, V, Part);
492  return;
493  }
494 
495  switch (getOpcode()) {
496  case VPInstruction::Not: {
497  Value *A = State.get(getOperand(0), Part);
498  Value *V = Builder.CreateNot(A);
499  State.set(this, V, Part);
500  break;
501  }
502  case VPInstruction::ICmpULE: {
503  Value *IV = State.get(getOperand(0), Part);
504  Value *TC = State.get(getOperand(1), Part);
505  Value *V = Builder.CreateICmpULE(IV, TC);
506  State.set(this, V, Part);
507  break;
508  }
509  case Instruction::Select: {
510  Value *Cond = State.get(getOperand(0), Part);
511  Value *Op1 = State.get(getOperand(1), Part);
512  Value *Op2 = State.get(getOperand(2), Part);
513  Value *V = Builder.CreateSelect(Cond, Op1, Op2);
514  State.set(this, V, Part);
515  break;
516  }
518  // Get first lane of vector induction variable.
519  Value *VIVElem0 = State.get(getOperand(0), VPIteration(Part, 0));
520  // Get the original loop tripcount.
521  Value *ScalarTC = State.TripCount;
522 
523  auto *Int1Ty = Type::getInt1Ty(Builder.getContext());
524  auto *PredTy = FixedVectorType::get(Int1Ty, State.VF.getKnownMinValue());
525  Instruction *Call = Builder.CreateIntrinsic(
526  Intrinsic::get_active_lane_mask, {PredTy, ScalarTC->getType()},
527  {VIVElem0, ScalarTC}, nullptr, "active.lane.mask");
528  State.set(this, Call, Part);
529  break;
530  }
531  default:
532  llvm_unreachable("Unsupported opcode for instruction");
533  }
534 }
535 
537  assert(!State.Instance && "VPInstruction executing an Instance");
538  for (unsigned Part = 0; Part < State.UF; ++Part)
539  generateInstruction(State, Part);
540 }
541 
542 void VPInstruction::dump() const {
543  VPSlotTracker SlotTracker(getParent()->getPlan());
544  print(dbgs(), "", SlotTracker);
545 }
546 
548  VPSlotTracker &SlotTracker) const {
549  O << "EMIT ";
550 
551  if (hasResult()) {
553  O << " = ";
554  }
555 
556  switch (getOpcode()) {
557  case VPInstruction::Not:
558  O << "not";
559  break;
561  O << "icmp ule";
562  break;
564  O << "combined load";
565  break;
567  O << "combined store";
568  break;
570  O << "active lane mask";
571  break;
572 
573  default:
575  }
576 
577  for (const VPValue *Operand : operands()) {
578  O << " ";
579  Operand->printAsOperand(O, SlotTracker);
580  }
581 }
582 
583 /// Generate the code inside the body of the vectorized loop. Assumes a single
584 /// LoopVectorBody basic-block was created for this. Introduce additional
585 /// basic-blocks as needed, and fill them all.
587  // -1. Check if the backedge taken count is needed, and if so build it.
588  if (BackedgeTakenCount && BackedgeTakenCount->getNumUsers()) {
589  Value *TC = State->TripCount;
591  auto *TCMO = Builder.CreateSub(TC, ConstantInt::get(TC->getType(), 1),
592  "trip.count.minus.1");
593  auto VF = State->VF;
594  Value *VTCMO =
595  VF.isScalar() ? TCMO : Builder.CreateVectorSplat(VF, TCMO, "broadcast");
596  for (unsigned Part = 0, UF = State->UF; Part < UF; ++Part)
597  State->set(BackedgeTakenCount, VTCMO, Part);
598  }
599 
600  // 0. Set the reverse mapping from VPValues to Values for code generation.
601  for (auto &Entry : Value2VPValue)
602  State->VPValue2Value[Entry.second] = Entry.first;
603 
604  BasicBlock *VectorPreHeaderBB = State->CFG.PrevBB;
605  BasicBlock *VectorHeaderBB = VectorPreHeaderBB->getSingleSuccessor();
606  assert(VectorHeaderBB && "Loop preheader does not have a single successor.");
607 
608  // 1. Make room to generate basic-blocks inside loop body if needed.
609  BasicBlock *VectorLatchBB = VectorHeaderBB->splitBasicBlock(
610  VectorHeaderBB->getFirstInsertionPt(), "vector.body.latch");
611  Loop *L = State->LI->getLoopFor(VectorHeaderBB);
612  L->addBasicBlockToLoop(VectorLatchBB, *State->LI);
613  // Remove the edge between Header and Latch to allow other connections.
614  // Temporarily terminate with unreachable until CFG is rewired.
615  // Note: this asserts the generated code's assumption that
616  // getFirstInsertionPt() can be dereferenced into an Instruction.
617  VectorHeaderBB->getTerminator()->eraseFromParent();
618  State->Builder.SetInsertPoint(VectorHeaderBB);
621 
622  // 2. Generate code in loop body.
623  State->CFG.PrevVPBB = nullptr;
624  State->CFG.PrevBB = VectorHeaderBB;
625  State->CFG.LastBB = VectorLatchBB;
626 
627  for (VPBlockBase *Block : depth_first(Entry))
628  Block->execute(State);
629 
630  // Setup branch terminator successors for VPBBs in VPBBsToFix based on
631  // VPBB's successors.
632  for (auto VPBB : State->CFG.VPBBsToFix) {
634  "Unexpected VPBBsToFix in non VPlan-native path");
635  BasicBlock *BB = State->CFG.VPBB2IRBB[VPBB];
636  assert(BB && "Unexpected null basic block for VPBB");
637 
638  unsigned Idx = 0;
639  auto *BBTerminator = BB->getTerminator();
640 
641  for (VPBlockBase *SuccVPBlock : VPBB->getHierarchicalSuccessors()) {
642  VPBasicBlock *SuccVPBB = SuccVPBlock->getEntryBasicBlock();
643  BBTerminator->setSuccessor(Idx, State->CFG.VPBB2IRBB[SuccVPBB]);
644  ++Idx;
645  }
646  }
647 
648  // 3. Merge the temporary latch created with the last basic-block filled.
649  BasicBlock *LastBB = State->CFG.PrevBB;
650  // Connect LastBB to VectorLatchBB to facilitate their merge.
652  isa<UnreachableInst>(LastBB->getTerminator())) &&
653  "Expected InnerLoop VPlan CFG to terminate with unreachable");
654  assert((!EnableVPlanNativePath || isa<BranchInst>(LastBB->getTerminator())) &&
655  "Expected VPlan CFG to terminate with branch in NativePath");
656  LastBB->getTerminator()->eraseFromParent();
657  BranchInst::Create(VectorLatchBB, LastBB);
658 
659  // Merge LastBB with Latch.
660  bool Merged = MergeBlockIntoPredecessor(VectorLatchBB, nullptr, State->LI);
661  (void)Merged;
662  assert(Merged && "Could not merge last basic block with latch.");
663  VectorLatchBB = LastBB;
664 
665  // We do not attempt to preserve DT for outer loop vectorization currently.
667  updateDominatorTree(State->DT, VectorPreHeaderBB, VectorLatchBB,
668  L->getExitBlock());
669 }
670 
671 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
673 void VPlan::dump() const { dbgs() << *this << '\n'; }
674 #endif
675 
676 void VPlan::updateDominatorTree(DominatorTree *DT, BasicBlock *LoopPreHeaderBB,
677  BasicBlock *LoopLatchBB,
678  BasicBlock *LoopExitBB) {
679  BasicBlock *LoopHeaderBB = LoopPreHeaderBB->getSingleSuccessor();
680  assert(LoopHeaderBB && "Loop preheader does not have a single successor.");
681  // The vector body may be more than a single basic-block by this point.
682  // Update the dominator tree information inside the vector body by propagating
683  // it from header to latch, expecting only triangular control-flow, if any.
684  BasicBlock *PostDomSucc = nullptr;
685  for (auto *BB = LoopHeaderBB; BB != LoopLatchBB; BB = PostDomSucc) {
686  // Get the list of successors of this block.
687  std::vector<BasicBlock *> Succs(succ_begin(BB), succ_end(BB));
688  assert(Succs.size() <= 2 &&
689  "Basic block in vector loop has more than 2 successors.");
690  PostDomSucc = Succs[0];
691  if (Succs.size() == 1) {
692  assert(PostDomSucc->getSinglePredecessor() &&
693  "PostDom successor has more than one predecessor.");
694  DT->addNewBlock(PostDomSucc, BB);
695  continue;
696  }
697  BasicBlock *InterimSucc = Succs[1];
698  if (PostDomSucc->getSingleSuccessor() == InterimSucc) {
699  PostDomSucc = Succs[1];
700  InterimSucc = Succs[0];
701  }
702  assert(InterimSucc->getSingleSuccessor() == PostDomSucc &&
703  "One successor of a basic block does not lead to the other.");
704  assert(InterimSucc->getSinglePredecessor() &&
705  "Interim successor has more than one predecessor.");
706  assert(PostDomSucc->hasNPredecessors(2) &&
707  "PostDom successor has more than two predecessors.");
708  DT->addNewBlock(InterimSucc, BB);
709  DT->addNewBlock(PostDomSucc, BB);
710  }
711  // Latch block is a new dominator for the loop exit.
712  DT->changeImmediateDominator(LoopExitBB, LoopLatchBB);
714 }
715 
716 const Twine VPlanPrinter::getUID(const VPBlockBase *Block) {
717  return (isa<VPRegionBlock>(Block) ? "cluster_N" : "N") +
718  Twine(getOrCreateBID(Block));
719 }
720 
721 const Twine VPlanPrinter::getOrCreateName(const VPBlockBase *Block) {
722  const std::string &Name = Block->getName();
723  if (!Name.empty())
724  return Name;
725  return "VPB" + Twine(getOrCreateBID(Block));
726 }
727 
728 void VPlanPrinter::dump() {
729  Depth = 1;
730  bumpIndent(0);
731  OS << "digraph VPlan {\n";
732  OS << "graph [labelloc=t, fontsize=30; label=\"Vectorization Plan";
733  if (!Plan.getName().empty())
734  OS << "\\n" << DOT::EscapeString(Plan.getName());
735  if (Plan.BackedgeTakenCount) {
736  OS << ", where:\\n";
737  Plan.BackedgeTakenCount->print(OS, SlotTracker);
738  OS << " := BackedgeTakenCount";
739  }
740  OS << "\"]\n";
741  OS << "node [shape=rect, fontname=Courier, fontsize=30]\n";
742  OS << "edge [fontname=Courier, fontsize=30]\n";
743  OS << "compound=true\n";
744 
745  for (const VPBlockBase *Block : depth_first(Plan.getEntry()))
746  dumpBlock(Block);
747 
748  OS << "}\n";
749 }
750 
751 void VPlanPrinter::dumpBlock(const VPBlockBase *Block) {
752  if (const VPBasicBlock *BasicBlock = dyn_cast<VPBasicBlock>(Block))
753  dumpBasicBlock(BasicBlock);
754  else if (const VPRegionBlock *Region = dyn_cast<VPRegionBlock>(Block))
755  dumpRegion(Region);
756  else
757  llvm_unreachable("Unsupported kind of VPBlock.");
758 }
759 
760 void VPlanPrinter::drawEdge(const VPBlockBase *From, const VPBlockBase *To,
761  bool Hidden, const Twine &Label) {
762  // Due to "dot" we print an edge between two regions as an edge between the
763  // exit basic block and the entry basic of the respective regions.
764  const VPBlockBase *Tail = From->getExitBasicBlock();
765  const VPBlockBase *Head = To->getEntryBasicBlock();
766  OS << Indent << getUID(Tail) << " -> " << getUID(Head);
767  OS << " [ label=\"" << Label << '\"';
768  if (Tail != From)
769  OS << " ltail=" << getUID(From);
770  if (Head != To)
771  OS << " lhead=" << getUID(To);
772  if (Hidden)
773  OS << "; splines=none";
774  OS << "]\n";
775 }
776 
777 void VPlanPrinter::dumpEdges(const VPBlockBase *Block) {
778  auto &Successors = Block->getSuccessors();
779  if (Successors.size() == 1)
780  drawEdge(Block, Successors.front(), false, "");
781  else if (Successors.size() == 2) {
782  drawEdge(Block, Successors.front(), false, "T");
783  drawEdge(Block, Successors.back(), false, "F");
784  } else {
785  unsigned SuccessorNumber = 0;
786  for (auto *Successor : Successors)
787  drawEdge(Block, Successor, false, Twine(SuccessorNumber++));
788  }
789 }
790 
791 void VPlanPrinter::dumpBasicBlock(const VPBasicBlock *BasicBlock) {
792  OS << Indent << getUID(BasicBlock) << " [label =\n";
793  bumpIndent(1);
794  OS << Indent << "\"" << DOT::EscapeString(BasicBlock->getName()) << ":\\n\"";
795  bumpIndent(1);
796 
797  // Dump the block predicate.
798  const VPValue *Pred = BasicBlock->getPredicate();
799  if (Pred) {
800  OS << " +\n" << Indent << " \"BlockPredicate: \"";
801  if (const VPInstruction *PredI = dyn_cast<VPInstruction>(Pred)) {
802  PredI->printAsOperand(OS, SlotTracker);
803  OS << " (" << DOT::EscapeString(PredI->getParent()->getName())
804  << ")\\l\"";
805  } else
806  Pred->printAsOperand(OS, SlotTracker);
807  }
808 
809  for (const VPRecipeBase &Recipe : *BasicBlock) {
810  OS << " +\n" << Indent << "\"";
811  Recipe.print(OS, Indent, SlotTracker);
812  OS << "\\l\"";
813  }
814 
815  // Dump the condition bit.
816  const VPValue *CBV = BasicBlock->getCondBit();
817  if (CBV) {
818  OS << " +\n" << Indent << " \"CondBit: ";
819  if (const VPInstruction *CBI = dyn_cast<VPInstruction>(CBV)) {
820  CBI->printAsOperand(OS, SlotTracker);
821  OS << " (" << DOT::EscapeString(CBI->getParent()->getName()) << ")\\l\"";
822  } else {
823  CBV->printAsOperand(OS, SlotTracker);
824  OS << "\"";
825  }
826  }
827 
828  bumpIndent(-2);
829  OS << "\n" << Indent << "]\n";
830  dumpEdges(BasicBlock);
831 }
832 
833 void VPlanPrinter::dumpRegion(const VPRegionBlock *Region) {
834  OS << Indent << "subgraph " << getUID(Region) << " {\n";
835  bumpIndent(1);
836  OS << Indent << "fontname=Courier\n"
837  << Indent << "label=\""
838  << DOT::EscapeString(Region->isReplicator() ? "<xVFxUF> " : "<x1> ")
839  << DOT::EscapeString(Region->getName()) << "\"\n";
840  // Dump the blocks of the region.
841  assert(Region->getEntry() && "Region contains no inner blocks.");
842  for (const VPBlockBase *Block : depth_first(Region->getEntry()))
843  dumpBlock(Block);
844  bumpIndent(-1);
845  OS << Indent << "}\n";
846  dumpEdges(Region);
847 }
848 
849 void VPlanPrinter::printAsIngredient(raw_ostream &O, const Value *V) {
850  std::string IngredientString;
851  raw_string_ostream RSO(IngredientString);
852  if (auto *Inst = dyn_cast<Instruction>(V)) {
853  if (!Inst->getType()->isVoidTy()) {
854  Inst->printAsOperand(RSO, false);
855  RSO << " = ";
856  }
857  RSO << Inst->getOpcodeName() << " ";
858  unsigned E = Inst->getNumOperands();
859  if (E > 0) {
860  Inst->getOperand(0)->printAsOperand(RSO, false);
861  for (unsigned I = 1; I < E; ++I)
862  Inst->getOperand(I)->printAsOperand(RSO << ", ", false);
863  }
864  } else // !Inst
865  V->printAsOperand(RSO, false);
866  RSO.flush();
867  O << DOT::EscapeString(IngredientString);
868 }
869 
871  VPSlotTracker &SlotTracker) const {
872  O << "WIDEN-CALL ";
873 
874  auto *CI = cast<CallInst>(getUnderlyingInstr());
875  if (CI->getType()->isVoidTy())
876  O << "void ";
877  else {
879  O << " = ";
880  }
881 
882  O << "call @" << CI->getCalledFunction()->getName() << "(";
884  O << ")";
885 }
886 
888  VPSlotTracker &SlotTracker) const {
889  O << "WIDEN-SELECT ";
891  O << " = select ";
893  O << ", ";
895  O << ", ";
897  O << (InvariantCond ? " (condition is loop invariant)" : "");
898 }
899 
901  VPSlotTracker &SlotTracker) const {
902  O << "WIDEN ";
904  O << " = " << getUnderlyingInstr()->getOpcodeName() << " ";
906 }
907 
909  VPSlotTracker &SlotTracker) const {
910  O << "WIDEN-INDUCTION";
911  if (getTruncInst()) {
912  O << "\\l\"";
913  O << " +\n" << Indent << "\" " << VPlanIngredient(IV) << "\\l\"";
914  O << " +\n" << Indent << "\" ";
916  } else
917  O << " " << VPlanIngredient(IV);
918 }
919 
921  VPSlotTracker &SlotTracker) const {
922  O << "WIDEN-GEP ";
923  O << (IsPtrLoopInvariant ? "Inv" : "Var");
924  size_t IndicesNumber = IsIndexLoopInvariant.size();
925  for (size_t I = 0; I < IndicesNumber; ++I)
926  O << "[" << (IsIndexLoopInvariant[I] ? "Inv" : "Var") << "]";
927 
928  O << " ";
930  O << " = getelementptr ";
932 }
933 
935  VPSlotTracker &SlotTracker) const {
936  O << "WIDEN-PHI " << VPlanIngredient(getUnderlyingValue());
937 }
938 
940  VPSlotTracker &SlotTracker) const {
941  O << "BLEND ";
942  Phi->printAsOperand(O, false);
943  O << " =";
944  if (getNumIncomingValues() == 1) {
945  // Not a User of any mask: not really blending, this is a
946  // single-predecessor phi.
947  O << " ";
949  } else {
950  for (unsigned I = 0, E = getNumIncomingValues(); I < E; ++I) {
951  O << " ";
953  O << "/";
955  }
956  }
957 }
958 
960  VPSlotTracker &SlotTracker) const {
961  O << "REDUCE ";
963  O << " = ";
965  O << " + reduce." << Instruction::getOpcodeName(RdxDesc->getOpcode())
966  << " (";
968  if (getCondOp()) {
969  O << ", ";
971  }
972  O << ")";
973 }
974 
976  VPSlotTracker &SlotTracker) const {
977  O << (IsUniform ? "CLONE " : "REPLICATE ");
978 
979  if (!getUnderlyingInstr()->getType()->isVoidTy()) {
981  O << " = ";
982  }
985 
986  if (AlsoPack)
987  O << " (S->V)";
988 }
989 
991  VPSlotTracker &SlotTracker) const {
992  O << "PHI-PREDICATED-INSTRUCTION ";
994  O << " = ";
996 }
997 
999  VPSlotTracker &SlotTracker) const {
1000  O << "WIDEN ";
1001 
1002  if (!isStore()) {
1004  O << " = ";
1005  }
1006  O << Instruction::getOpcodeName(Ingredient.getOpcode()) << " ";
1007 
1009 }
1010 
1012  Value *CanonicalIV = State.CanonicalIV;
1013  Type *STy = CanonicalIV->getType();
1015  ElementCount VF = State.VF;
1016  assert(!VF.isScalable() && "the code following assumes non scalables ECs");
1017  Value *VStart = VF.isScalar()
1018  ? CanonicalIV
1019  : Builder.CreateVectorSplat(VF.getKnownMinValue(),
1020  CanonicalIV, "broadcast");
1021  for (unsigned Part = 0, UF = State.UF; Part < UF; ++Part) {
1023  for (unsigned Lane = 0; Lane < VF.getKnownMinValue(); ++Lane)
1024  Indices.push_back(
1025  ConstantInt::get(STy, Part * VF.getKnownMinValue() + Lane));
1026  // If VF == 1, there is only one iteration in the loop above, thus the
1027  // element pushed back into Indices is ConstantInt::get(STy, Part)
1028  Constant *VStep =
1029  VF.isScalar() ? Indices.back() : ConstantVector::get(Indices);
1030  // Add the consecutive indices to the vector value.
1031  Value *CanonicalVectorIV = Builder.CreateAdd(VStart, VStep, "vec.iv");
1032  State.set(getVPValue(), CanonicalVectorIV, Part);
1033  }
1034 }
1035 
1037  VPSlotTracker &SlotTracker) const {
1038  O << "EMIT ";
1040  O << " = WIDEN-CANONICAL-INDUCTION";
1041 }
1042 
1043 template void DomTreeBuilder::Calculate<VPDominatorTree>(VPDominatorTree &DT);
1044 
1046  for (unsigned J = 0; J < getNumUsers();) {
1047  VPUser *User = Users[J];
1048  unsigned NumUsers = getNumUsers();
1049  for (unsigned I = 0, E = User->getNumOperands(); I < E; ++I)
1050  if (User->getOperand(I) == this)
1051  User->setOperand(I, New);
1052  // If a user got removed after updating the current user, the next user to
1053  // update will be moved to the current position, so we only need to
1054  // increment the index if the number of users did not change.
1055  if (NumUsers == getNumUsers())
1056  J++;
1057  }
1058 }
1059 
1061  if (const Value *UV = getUnderlyingValue()) {
1062  OS << "ir<";
1063  UV->printAsOperand(OS, false);
1064  OS << ">";
1065  return;
1066  }
1067 
1068  unsigned Slot = Tracker.getSlot(this);
1069  if (Slot == unsigned(-1))
1070  OS << "<badref>";
1071  else
1072  OS << "vp<%" << Tracker.getSlot(this) << ">";
1073 }
1074 
1077  Op->printAsOperand(O, SlotTracker);
1078  });
1079 }
1080 
1081 void VPInterleavedAccessInfo::visitRegion(VPRegionBlock *Region,
1082  Old2NewTy &Old2New,
1083  InterleavedAccessInfo &IAI) {
1085  for (VPBlockBase *Base : RPOT) {
1086  visitBlock(Base, Old2New, IAI);
1087  }
1088 }
1089 
1090 void VPInterleavedAccessInfo::visitBlock(VPBlockBase *Block, Old2NewTy &Old2New,
1091  InterleavedAccessInfo &IAI) {
1092  if (VPBasicBlock *VPBB = dyn_cast<VPBasicBlock>(Block)) {
1093  for (VPRecipeBase &VPI : *VPBB) {
1094  if (isa<VPWidenPHIRecipe>(&VPI))
1095  continue;
1096  assert(isa<VPInstruction>(&VPI) && "Can only handle VPInstructions");
1097  auto *VPInst = cast<VPInstruction>(&VPI);
1098  auto *Inst = cast<Instruction>(VPInst->getUnderlyingValue());
1099  auto *IG = IAI.getInterleaveGroup(Inst);
1100  if (!IG)
1101  continue;
1102 
1103  auto NewIGIter = Old2New.find(IG);
1104  if (NewIGIter == Old2New.end())
1105  Old2New[IG] = new InterleaveGroup<VPInstruction>(
1106  IG->getFactor(), IG->isReverse(), IG->getAlign());
1107 
1108  if (Inst == IG->getInsertPos())
1109  Old2New[IG]->setInsertPos(VPInst);
1110 
1111  InterleaveGroupMap[VPInst] = Old2New[IG];
1112  InterleaveGroupMap[VPInst]->insertMember(
1113  VPInst, IG->getIndex(Inst),
1114  Align(IG->isReverse() ? (-1) * int(IG->getFactor())
1115  : IG->getFactor()));
1116  }
1117  } else if (VPRegionBlock *Region = dyn_cast<VPRegionBlock>(Block))
1118  visitRegion(Region, Old2New, IAI);
1119  else
1120  llvm_unreachable("Unsupported kind of VPBlock.");
1121 }
1122 
1124  InterleavedAccessInfo &IAI) {
1125  Old2NewTy Old2New;
1126  visitRegion(cast<VPRegionBlock>(Plan.getEntry()), Old2New, IAI);
1127 }
1128 
1129 void VPSlotTracker::assignSlot(const VPValue *V) {
1130  assert(Slots.find(V) == Slots.end() && "VPValue already has a slot!");
1131  Slots[V] = NextSlot++;
1132 }
1133 
1134 void VPSlotTracker::assignSlots(const VPBlockBase *VPBB) {
1135  if (auto *Region = dyn_cast<VPRegionBlock>(VPBB))
1136  assignSlots(Region);
1137  else
1138  assignSlots(cast<VPBasicBlock>(VPBB));
1139 }
1140 
1141 void VPSlotTracker::assignSlots(const VPRegionBlock *Region) {
1143  for (const VPBlockBase *Block : RPOT)
1144  assignSlots(Block);
1145 }
1146 
1147 void VPSlotTracker::assignSlots(const VPBasicBlock *VPBB) {
1148  for (const VPRecipeBase &Recipe : *VPBB) {
1149  for (VPValue *Def : Recipe.definedValues())
1150  assignSlot(Def);
1151  }
1152 }
1153 
1154 void VPSlotTracker::assignSlots(const VPlan &Plan) {
1155 
1156  for (const VPValue *V : Plan.VPExternalDefs)
1157  assignSlot(V);
1158 
1159  if (Plan.BackedgeTakenCount)
1160  assignSlot(Plan.BackedgeTakenCount);
1161 
1163  for (const VPBlockBase *Block : RPOT)
1164  assignSlots(Block);
1165 }
llvm::VPBasicBlock::getRecipeList
RecipeListTy & getRecipeList()
Returns a reference to the list of recipes.
Definition: VPlan.h:1341
llvm::VPInstruction::getOpcode
unsigned getOpcode() const
Definition: VPlan.h:656
i
i
Definition: README.txt:29
llvm::VPRecipeBase::moveAfter
void moveAfter(VPRecipeBase *MovePos)
Unlink this recipe from its current VPBasicBlock and insert it into the VPBasicBlock that MovePos liv...
Definition: VPlan.cpp:470
llvm::VPReductionRecipe::print
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
Definition: VPlan.cpp:959
llvm::VPBlockBase::getPlan
VPlan * getPlan()
Definition: VPlan.cpp:118
llvm::VPBlendRecipe::getNumIncomingValues
unsigned getNumIncomingValues() const
Return the number of incoming values, taking into account that a single incoming value has no mask.
Definition: VPlan.h:955
llvm::VPlan::execute
void execute(struct VPTransformState *State)
Generate the IR code for this VPlan.
Definition: VPlan.cpp:586
LLVM_DUMP_METHOD
#define LLVM_DUMP_METHOD
Mark debug helper function definitions like dump() that should not be stripped from debug builds.
Definition: Compiler.h:498
llvm::VPBlendRecipe::getIncomingValue
VPValue * getIncomingValue(unsigned Idx) const
Return incoming value number Idx.
Definition: VPlan.h:958
llvm::IRBuilderBase::SetInsertPoint
void SetInsertPoint(BasicBlock *TheBB)
This specifies that created instructions should be appended to the end of the specified block.
Definition: IRBuilder.h:184
llvm
This class represents lattice values for constants.
Definition: AllocatorList.h:23
llvm::tgtok::Def
@ Def
Definition: TGLexer.h:50
llvm::VPRecipeBase::getUnderlyingInstr
Instruction * getUnderlyingInstr()
Returns the underlying instruction, if the recipe is a VPValue or nullptr otherwise.
Definition: VPlan.h:570
llvm::Type::getInt1Ty
static IntegerType * getInt1Ty(LLVMContext &C)
Definition: Type.cpp:194
llvm::VPInstruction::SLPStore
@ SLPStore
Definition: VPlan.h:611
llvm::VPTransformState::CFG
struct llvm::VPTransformState::CFGState CFG
llvm::VPRecipeBase::eraseFromParent
iplist< VPRecipeBase >::iterator eraseFromParent()
This method unlinks 'this' from the containing basic block and deletes it.
Definition: VPlan.cpp:465
llvm::LoopBase::getExitBlock
BlockT * getExitBlock() const
If getExitBlocks would return exactly one block, return that block.
Definition: LoopInfoImpl.h:81
llvm::VPWidenCanonicalIVRecipe::execute
void execute(VPTransformState &State) override
Generate a canonical vector induction variable of the vector loop, with start = {<Part*VF,...
Definition: VPlan.cpp:1011
llvm::VPBlendRecipe::print
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
Definition: VPlan.cpp:939
llvm::VPBlockBase::getEnclosingBlockWithPredecessors
VPBlockBase * getEnclosingBlockWithPredecessors()
Definition: VPlan.cpp:166
llvm::VPInstruction::ActiveLaneMask
@ ActiveLaneMask
Definition: VPlan.h:612
llvm::VPBlockBase::getName
const std::string & getName() const
Definition: VPlan.h:330
llvm::BasicBlock::getParent
const Function * getParent() const
Return the enclosing method, or null if none.
Definition: BasicBlock.h:107
llvm::ElementCount
Definition: TypeSize.h:382
llvm::VPUser::removeLastOperand
void removeLastOperand()
Definition: VPlanValue.h:236
llvm::succ_end
Interval::succ_iterator succ_end(Interval *I)
Definition: Interval.h:102
llvm::VPBasicBlock
VPBasicBlock serves as the leaf of the Hierarchical Control-Flow Graph.
Definition: VPlan.h:1297
llvm::Loop
Represents a single loop in the control flow graph.
Definition: LoopInfo.h:529
llvm::VPBlockBase::getExitBasicBlock
const VPBasicBlock * getExitBasicBlock() const
Definition: VPlan.cpp:144
llvm::raw_string_ostream
A raw_ostream that writes to an std::string.
Definition: raw_ostream.h:614
llvm::VPWidenIntOrFpInductionRecipe::print
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
Definition: VPlan.cpp:908
llvm::SetVector< T, SmallVector< T, N >, SmallDenseSet< T, N > >::size
size_type size() const
Determine the number of elements in the SetVector.
Definition: SetVector.h:77
llvm::SmallVector
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
Definition: SmallVector.h:1168
llvm::VPTransformState::get
Value * get(VPValue *Def, unsigned Part)
Get the generated Value for a given VPValue and a given Part.
Definition: LoopVectorize.cpp:9144
llvm::VPRegionBlock::dropAllReferences
void dropAllReferences(VPValue *NewValue) override
Replace all operands of VPUsers in the block with NewValue and also replaces all uses of VPValues def...
Definition: VPlan.cpp:387
llvm::VPDef::print
virtual void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const =0
Each concrete VPDef prints itself.
llvm::DOT::EscapeString
std::string EscapeString(const std::string &Label)
Definition: GraphWriter.cpp:35
ErrorHandling.h
llvm::IRBuilder<>
llvm::VPTransformState::set
void set(VPValue *Def, Value *V, unsigned Part)
Set the generated Value for a given VPValue and a given Part.
Definition: VPlan.h:166
llvm::DominatorTree
Concrete subclass of DominatorTreeBase that is used to compute a normal dominator tree.
Definition: Dominators.h:151
llvm::RecurrenceDescriptor::getOpcode
static unsigned getOpcode(RecurKind Kind)
Returns the opcode corresponding to the RecurrenceKind.
Definition: IVDescriptors.cpp:805
llvm::interleaveComma
void interleaveComma(const Container &c, StreamT &os, UnaryFunctor each_fn)
Definition: STLExtras.h:1767
llvm::VPBasicBlock::dropAllReferences
void dropAllReferences(VPValue *NewValue) override
Replace all operands of VPUsers in the block with NewValue and also replaces all uses of VPValues def...
Definition: VPlan.cpp:377
llvm::Type
The instances of the Type class are immutable: once they are created, they are never changed.
Definition: Type.h:46
llvm::VPWidenIntOrFpInductionRecipe::getTruncInst
TruncInst * getTruncInst()
Returns the first defined value as TruncInst, if it is one or nullptr otherwise.
Definition: VPlan.h:862
llvm::VPValue::dump
void dump() const
Dump the value to stderr (for debugging).
Definition: VPlan.cpp:80
llvm::BasicBlock::getSingleSuccessor
const BasicBlock * getSingleSuccessor() const
Return the successor of this block if it has a single successor.
Definition: BasicBlock.cpp:294
llvm::VPBlockBase::getHierarchicalPredecessors
const VPBlocksTy & getHierarchicalPredecessors()
Definition: VPlan.h:418
llvm::BasicBlock::splitBasicBlock
BasicBlock * splitBasicBlock(iterator I, const Twine &BBName="", bool Before=false)
Split the basic block into two basic blocks at the specified instruction.
Definition: BasicBlock.cpp:375
T
#define T
Definition: Mips16ISelLowering.cpp:341
llvm::VPWidenMemoryInstructionRecipe::print
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
Definition: VPlan.cpp:998
llvm::VPValue::print
void print(raw_ostream &OS, VPSlotTracker &Tracker) const
Definition: VPlan.cpp:73
llvm::iplist_impl::remove
pointer remove(iterator &IT)
Definition: ilist.h:251
llvm::BasicBlock::getSinglePredecessor
const BasicBlock * getSinglePredecessor() const
Return the predecessor of this block if it has a single predecessor block.
Definition: BasicBlock.cpp:264
STLExtras.h
llvm::VPUser::operands
operand_range operands()
Definition: VPlanValue.h:250
llvm::BasicBlock::hasNPredecessors
bool hasNPredecessors(unsigned N) const
Return true if this block has exactly N predecessors.
Definition: BasicBlock.cpp:286
llvm::VPRegionBlock::getEntry
const VPBlockBase * getEntry() const
Definition: VPlan.h:1425
llvm::VPInstruction::Not
@ Not
Definition: VPlan.h:608
llvm::VPTransformState::hasScalarValue
bool hasScalarValue(VPValue *Def, VPIteration Instance)
Definition: VPlan.h:156
llvm::VPValue::VPValue
VPValue(const unsigned char SC, Value *UV=nullptr, VPDef *Def=nullptr)
Definition: VPlan.cpp:61
llvm::InterleaveGroup
The group of interleaved loads/stores sharing the same stride and close to each other.
Definition: VectorUtils.h:292
llvm::LinearPolySize::isScalable
bool isScalable() const
Returns whether the size is scaled by a runtime quantity (vscale).
Definition: TypeSize.h:295
llvm::iplist
An intrusive list with ownership and callbacks specified/controlled by ilist_traits,...
Definition: ilist.h:390
LLVM_DEBUG
#define LLVM_DEBUG(X)
Definition: Debug.h:122
DepthFirstIterator.h
llvm::VPBlockBase::setPredicate
void setPredicate(VPValue *Pred)
Set the block's predicate.
Definition: VPlan.cpp:222
llvm::iplist_impl::erase
iterator erase(iterator where)
Definition: ilist.h:267
llvm::BasicBlock
LLVM Basic Block Representation.
Definition: BasicBlock.h:58
llvm::VPDef
This class augments a recipe with a set of VPValues defined by the recipe.
Definition: VPlanValue.h:264
llvm::dbgs
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
Definition: Debug.cpp:132
VPlanDominatorTree.h
llvm::VPTransformState::DataState::PerPartOutput
DenseMap< VPValue *, PerPartValuesTy > PerPartOutput
Definition: VPlan.h:130
llvm::ElementCount::isScalar
bool isScalar() const
Counting predicates.
Definition: TypeSize.h:392
llvm::VPPredInstPHIRecipe::print
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
Definition: VPlan.cpp:990
Instruction.h
llvm::ConstantInt::get
static Constant * get(Type *Ty, uint64_t V, bool isSigned=false)
If Ty is a vector type, return a Constant with a splat of the given value.
Definition: Constants.cpp:876
llvm::VPTransformState
VPTransformState holds information passed down when "executing" a VPlan, needed for generating the ou...
Definition: VPlan.h:108
CommandLine.h
llvm::VPBlockBase
VPBlockBase is the building block of the Hierarchical Control-Flow Graph.
Definition: VPlan.h:257
llvm::VPReplicateRecipe::print
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
Definition: VPlan.cpp:975
llvm::Instruction::getOpcode
unsigned getOpcode() const
Returns a member of one of the enums like Instruction::Add.
Definition: Instruction.h:160
llvm::VPInstruction
This is a concrete Recipe that models a single VPlan-level instruction.
Definition: VPlan.h:602
llvm::PPCISD::SC
@ SC
CHAIN = SC CHAIN, Imm128 - System call.
Definition: PPCISelLowering.h:401
E
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
llvm::VPTransformState::LI
LoopInfo * LI
Hold a pointer to LoopInfo to register new basic blocks in the loop.
Definition: VPlan.h:232
llvm::User
Definition: User.h:44
resetSingleOpUser
static void resetSingleOpUser(VPUser &U, VPValue *NewVal)
Definition: VPlan.cpp:188
Twine.h
InstrTypes.h
getOpcode
static Optional< unsigned > getOpcode(ArrayRef< VPValue * > Values)
Returns the opcode of Values or ~0 if they do not all agree.
Definition: VPlanSLP.cpp:199
llvm::VPRecipeBase::moveBefore
void moveBefore(VPBasicBlock &BB, iplist< VPRecipeBase >::iterator I)
Unlink this recipe and insert into BB before I.
Definition: VPlan.cpp:475
llvm::ReplaceInstWithInst
void ReplaceInstWithInst(BasicBlock::InstListType &BIL, BasicBlock::iterator &BI, Instruction *I)
Replace the instruction specified by BI with the instruction specified by I.
Definition: BasicBlockUtils.cpp:471
llvm::BasicBlock::getFirstInsertionPt
const_iterator getFirstInsertionPt() const
Returns an iterator to the first instruction in this block that is suitable for inserting a non-PHI i...
Definition: BasicBlock.cpp:249
llvm::VPBlockBase::getEnclosingBlockWithSuccessors
VPBlockBase * getEnclosingBlockWithSuccessors()
An Enclosing Block of a block B is any block containing B, including B itself.
Definition: VPlan.cpp:158
B
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
llvm::VPWidenCanonicalIVRecipe::print
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
Definition: VPlan.cpp:1036
llvm::Instruction::getOpcodeName
const char * getOpcodeName() const
Definition: Instruction.h:162
llvm::VPTransformState::DataState::PerPartScalars
DenseMap< VPValue *, ScalarsPerPartValuesTy > PerPartScalars
Definition: VPlan.h:133
llvm::Instruction
Definition: Instruction.h:45
llvm::InterleavedAccessInfo
Drive the analysis of interleaved memory accesses in the loop.
Definition: VectorUtils.h:736
llvm::VPValue::replaceAllUsesWith
void replaceAllUsesWith(VPValue *New)
Definition: VPlan.cpp:1045
getSingleOperandOrNull
static VPValue * getSingleOperandOrNull(VPUser &U)
Definition: VPlan.cpp:174
llvm::VPRegionBlock::execute
void execute(struct VPTransformState *State) override
The method which generates the output IR instructions that correspond to this VPRegionBlock,...
Definition: VPlan.cpp:394
llvm::raw_ostream
This class implements an extremely fast bulk output stream that can only output to a stream.
Definition: raw_ostream.h:50
llvm::VPValue
Definition: VPlanValue.h:44
llvm::operator<<
raw_ostream & operator<<(raw_ostream &OS, const APFixedPoint &FX)
Definition: APFixedPoint.h:230
llvm::VPTransformState::CFGState::VPBBsToFix
SmallVector< VPBasicBlock *, 8 > VPBBsToFix
Vector of VPBasicBlocks whose terminator instruction needs to be fixed up at the end of vector code g...
Definition: VPlan.h:226
llvm::VPlan::getName
const std::string & getName() const
Definition: VPlan.h:1673
llvm::DominatorTreeBase::changeImmediateDominator
void changeImmediateDominator(DomTreeNodeBase< NodeT > *N, DomTreeNodeBase< NodeT > *NewIDom)
changeImmediateDominator - This method is used to update the dominator tree information when a node's...
Definition: GenericDomTree.h:655
VPlan.h
llvm::VPBasicBlock::end
iterator end()
Definition: VPlan.h:1325
llvm::FixedVectorType::get
static FixedVectorType * get(Type *ElementType, unsigned NumElts)
Definition: Type.cpp:643
llvm::Align
This struct is a compact representation of a valid (non-zero power of two) alignment.
Definition: Alignment.h:39
llvm::MCID::Call
@ Call
Definition: MCInstrDesc.h:154
llvm::VPTransformState::TripCount
Value * TripCount
Hold the trip count of the scalar loop.
Definition: VPlan.h:246
IVDescriptors.h
llvm::VPUser::getNumOperands
unsigned getNumOperands() const
Definition: VPlanValue.h:224
Type.h
llvm::VPBasicBlock::empty
bool empty() const
Definition: VPlan.h:1334
llvm::VPRegionBlock
VPRegionBlock represents a collection of VPBasicBlocks and VPRegionBlocks which form a Single-Entry-S...
Definition: VPlan.h:1387
llvm::VPWidenMemoryInstructionRecipe::isStore
bool isStore() const
Returns true if this recipe is a store.
Definition: VPlan.h:1254
llvm::VPBlockBase::getPredecessors
const VPBlocksTy & getPredecessors() const
Definition: VPlan.h:367
CFG.h
LoopInfo.h
llvm::VPBlockBase::getCondBit
VPValue * getCondBit()
Definition: VPlan.cpp:202
EnableVPlanNativePath
cl::opt< bool > EnableVPlanNativePath
llvm::VPUser::setOperand
void setOperand(unsigned I, VPValue *New)
Definition: VPlanValue.h:230
BasicBlock.h
llvm::cl::opt< bool >
llvm::CallingConv::Fast
@ Fast
Fast - This calling convention attempts to make calls as fast as possible (e.g.
Definition: CallingConv.h:42
llvm::VPTransformState::CanonicalIV
Value * CanonicalIV
Hold the canonical scalar IV of the vector loop (start=0, step=VF*UF).
Definition: VPlan.h:243
llvm::VPRecipeBase::removeFromParent
void removeFromParent()
This method unlinks 'this' from the containing basic block, but does not delete it.
Definition: VPlan.cpp:459
llvm::VPBlockBase::getHierarchicalSuccessors
const VPBlocksTy & getHierarchicalSuccessors()
Definition: VPlan.h:402
llvm::RISCVFenceField::O
@ O
Definition: RISCVBaseInfo.h:128
llvm::Constant
This is an important base class in LLVM.
Definition: Constant.h:41
llvm::Instruction::eraseFromParent
SymbolTableList< Instruction >::iterator eraseFromParent()
This method unlinks 'this' from the containing basic block and deletes it.
Definition: Instruction.cpp:77
llvm::VPValue::Def
VPDef * Def
Pointer to the VPDef that defines this VPValue.
Definition: VPlanValue.h:65
llvm::VPReductionRecipe::getCondOp
VPValue * getCondOp() const
The VPValue of the condition for the block.
Definition: VPlan.h:1078
llvm::VPSlotTracker::getSlot
unsigned getSlot(const VPValue *V) const
Definition: VPlanValue.h:382
llvm::VPBlockBase::getSingleHierarchicalSuccessor
VPBlockBase * getSingleHierarchicalSuccessor()
Definition: VPlan.h:408
llvm::VPBlockBase::getSingleHierarchicalPredecessor
VPBlockBase * getSingleHierarchicalPredecessor()
Definition: VPlan.h:424
llvm::VPTransformState::DT
DominatorTree * DT
Hold a pointer to Dominator Tree to register new basic blocks in the loop.
Definition: VPlan.h:235
llvm::RegionBase::getEntry
BlockT * getEntry() const
Get the entry BasicBlock of the Region.
Definition: RegionInfo.h:322
llvm::VPInstruction::execute
void execute(VPTransformState &State) override
Generate the instruction.
Definition: VPlan.cpp:536
llvm::IRBuilderBase::getInt32
ConstantInt * getInt32(uint32_t C)
Get a constant 32-bit value.
Definition: IRBuilder.h:478
llvm::BranchInst::Create
static BranchInst * Create(BasicBlock *IfTrue, Instruction *InsertBefore=nullptr)
Definition: Instructions.h:3071
llvm::VPTransformState::UF
unsigned UF
Definition: VPlan.h:117
llvm::DenseMap
Definition: DenseMap.h:714
llvm::VPInstruction::print
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the VPInstruction to O.
Definition: VPlan.cpp:547
llvm::VPValue::getNumUsers
unsigned getNumUsers() const
Definition: VPlanValue.h:125
llvm::LoopInfoBase::getLoopFor
LoopT * getLoopFor(const BlockT *BB) const
Return the inner most loop that BB lives in.
Definition: LoopInfo.h:963
I
#define I(x, y, z)
Definition: MD5.cpp:59
llvm::VPIteration
In what follows, the term "input IR" refers to code that is fed into the vectorizer whereas the term ...
Definition: VPlan.h:94
llvm::succ_begin
Interval::succ_iterator succ_begin(Interval *I)
succ_begin/succ_end - define methods so that Intervals may be used just like BasicBlocks can with the...
Definition: Interval.h:99
llvm::VPBlendRecipe::getMask
VPValue * getMask(unsigned Idx) const
Return mask number Idx.
Definition: VPlan.h:961
llvm::VPTransformState::CFGState::PrevVPBB
VPBasicBlock * PrevVPBB
The previous VPBasicBlock visited. Initially set to null.
Definition: VPlan.h:210
llvm::VPWidenGEPRecipe::print
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
Definition: VPlan.cpp:920
llvm::IRBuilderBase::CreateUnreachable
UnreachableInst * CreateUnreachable()
Definition: IRBuilder.h:1113
llvm::VPlan::getEntry
VPBlockBase * getEntry()
Definition: VPlan.h:1653
assert
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
getPlanEntry
static T * getPlanEntry(T *Start)
Definition: VPlan.cpp:98
llvm::VPBasicBlock::execute
void execute(struct VPTransformState *State) override
The method which generates the output IR instructions that correspond to this VPBasicBlock,...
Definition: VPlan.cpp:311
llvm::IRBuilderBase::CreateExtractElement
Value * CreateExtractElement(Value *Vec, Value *Idx, const Twine &Name="")
Definition: IRBuilder.h:2355
llvm::VPInstruction::hasResult
bool hasResult() const
Definition: VPlan.h:678
llvm::Instruction::isBinaryOp
bool isBinaryOp() const
Definition: Instruction.h:165
llvm::Type::isVoidTy
bool isVoidTy() const
Return true if this is 'void'.
Definition: Type.h:139
llvm::VPUser::printOperands
void printOperands(raw_ostream &O, VPSlotTracker &SlotTracker) const
Print the operands to O.
Definition: VPlan.cpp:1075
llvm::LinearPolySize::getKnownMinValue
ScalarTy getKnownMinValue() const
Returns the minimum value this size can represent.
Definition: TypeSize.h:293
llvm::VPWidenRecipe::print
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
Definition: VPlan.cpp:900
Builder
assume Assume Builder
Definition: AssumeBundleBuilder.cpp:643
llvm::Value::printAsOperand
void printAsOperand(raw_ostream &O, bool PrintType=true, const Module *M=nullptr) const
Print the name of this Value out to the specified raw_ostream.
Definition: AsmWriter.cpp:4668
llvm::User::setOperand
void setOperand(unsigned i, Value *Val)
Definition: User.h:174
llvm::SetVector< T, SmallVector< T, N >, SmallDenseSet< T, N > >::insert
bool insert(const value_type &X)
Insert a new element into the SetVector.
Definition: SetVector.h:141
llvm::VPWidenCallRecipe::print
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
Definition: VPlan.cpp:870
llvm::VPReductionRecipe::getVecOp
VPValue * getVecOp() const
The VPValue of the vector value to be reduced.
Definition: VPlan.h:1076
llvm::MergeBlockIntoPredecessor
bool MergeBlockIntoPredecessor(BasicBlock *BB, DomTreeUpdater *DTU=nullptr, LoopInfo *LI=nullptr, MemorySSAUpdater *MSSAU=nullptr, MemoryDependenceResults *MemDep=nullptr, bool PredecessorWithTwoSuccessors=false)
Attempts to merge a block into its predecessor, if possible.
Definition: BasicBlockUtils.cpp:172
Cond
SmallVector< MachineOperand, 4 > Cond
Definition: BasicBlockSections.cpp:167
llvm::VPUser::addOperand
void addOperand(VPValue *Operand)
Definition: VPlanValue.h:219
llvm_unreachable
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
Definition: ErrorHandling.h:136
llvm::Value::getType
Type * getType() const
All values are typed, get the type of this value.
Definition: Value.h:246
llvm::BasicBlock::Create
static BasicBlock * Create(LLVMContext &Context, const Twine &Name="", Function *Parent=nullptr, BasicBlock *InsertBefore=nullptr)
Creates a new BasicBlock.
Definition: BasicBlock.h:100
llvm::VPRecipeBase::getParent
VPBasicBlock * getParent()
Definition: VPlan.h:535
llvm::ilist_node_impl::getIterator
self_iterator getIterator()
Definition: ilist_node.h:81
llvm::DominatorTreeBase
Core dominator tree base class.
Definition: LoopInfo.h:65
llvm::ConstantVector::get
static Constant * get(ArrayRef< Constant * > V)
Definition: Constants.cpp:1335
llvm::VPTransformState::VF
ElementCount VF
The chosen Vectorization and Unroll Factors of the loop being vectorized.
Definition: VPlan.h:116
llvm::Value::getName
StringRef getName() const
Return a constant reference to the value's name.
Definition: Value.cpp:295
llvm::BasicBlock::getTerminator
const Instruction * getTerminator() const LLVM_READONLY
Returns the terminator instruction if the block is well formed or null if the block is not well forme...
Definition: BasicBlock.cpp:148
llvm::VPBlockBase::getEntryBasicBlock
const VPBasicBlock * getEntryBasicBlock() const
Definition: VPlan.cpp:123
llvm::DominatorTreeBase::addNewBlock
DomTreeNodeBase< NodeT > * addNewBlock(NodeT *BB, NodeT *DomBB)
Add a new node to the dominator tree information.
Definition: GenericDomTree.h:619
llvm::VPTransformState::Data
struct llvm::VPTransformState::DataState Data
llvm::BasicBlock::getContext
LLVMContext & getContext() const
Get the context in which this basic block lives.
Definition: BasicBlock.cpp:32
llvm::depth_first
iterator_range< df_iterator< T > > depth_first(const T &G)
Definition: DepthFirstIterator.h:227
CFG
Common register allocation spilling lr str ldr sxth r3 ldr mla r4 can lr mov lr str ldr sxth r3 mla r4 and then merge mul and lr str ldr sxth r3 mla r4 It also increase the likelihood the store may become dead bb27 Successors according to CFG
Definition: README.txt:39
llvm::VPTransformState::Builder
IRBuilder & Builder
Hold a reference to the IRBuilder used to generate output IR code.
Definition: VPlan.h:238
llvm::VPBlockBase::setCondBit
void setCondBit(VPValue *CV)
Set the condition bit selecting the successor.
Definition: VPlan.cpp:210
llvm::MCID::Select
@ Select
Definition: MCInstrDesc.h:163
llvm::VPDef::dump
void dump() const
Dump the VPDef to stderr (for debugging).
Definition: VPlan.cpp:88
llvm::Twine
Twine - A lightweight data structure for efficiently representing the concatenation of temporary valu...
Definition: Twine.h:80
llvm::VPSlotTracker
This class can be used to assign consecutive numbers to all VPValues in a VPlan and allows querying t...
Definition: VPlanValue.h:365
llvm::CallingConv::Tail
@ Tail
Tail - This calling convention attemps to make calls as fast as possible while guaranteeing that tail...
Definition: CallingConv.h:81
llvm::VPRegionBlock::getExit
const VPBlockBase * getExit() const
Definition: VPlan.h:1443
llvm::pdb::PDB_SymType::Label
@ Label
llvm::GraphProgram::Name
Name
Definition: GraphWriter.h:52
llvm::VPInstruction::ICmpULE
@ ICmpULE
Definition: VPlan.h:609
GraphWriter.h
llvm::Successor
@ Successor
Definition: SIMachineScheduler.h:33
llvm::AMDGPU::SendMsg::Op
Op
Definition: SIDefines.h:302
llvm::ilist_iterator
Iterator for intrusive lists based on ilist_node.
Definition: ilist_iterator.h:57
llvm::VPRecipeBase
VPRecipeBase is a base class modeling a sequence of one or more output IR instructions.
Definition: VPlan.h:515
Casting.h
llvm::VPBlockBase::getPredicate
VPValue * getPredicate()
Definition: VPlan.cpp:214
llvm::SlotTracker
This class provides computation of slot numbers for LLVM Assembly writing.
Definition: AsmWriter.cpp:716
GenericDomTreeConstruction.h
llvm::VPTransformState::CFGState
Hold state information used when constructing the CFG of the output IR, traversing the VPBasicBlocks ...
Definition: VPlan.h:208
llvm::ReversePostOrderTraversal
Definition: PostOrderIterator.h:288
llvm::iplist_impl::insertAfter
iterator insertAfter(iterator where, pointer New)
Definition: ilist.h:237
llvm::iplist_impl::insert
iterator insert(iterator where, pointer New)
Definition: ilist.h:228
llvm::VPUser
This class augments VPValue with operands which provide the inverse def-use edges from VPValue's user...
Definition: VPlanValue.h:191
llvm::Instruction::BinaryOps
BinaryOps
Definition: Instruction.h:761
Instructions.h
PostOrderIterator.h
llvm::User::getNumOperands
unsigned getNumOperands() const
Definition: User.h:191
SmallVector.h
llvm::VPTransformState::VPValue2Value
VPValue2ValueTy VPValue2Value
Definition: VPlan.h:240
llvm::VPBasicBlock::begin
iterator begin()
Recipe iterator methods.
Definition: VPlan.h:1323
llvm::LoopBase::addBasicBlockToLoop
void addBasicBlockToLoop(BlockT *NewBB, LoopInfoBase< BlockT, LoopT > &LI)
This method is used by other analyses to update loop information.
Definition: LoopInfoImpl.h:241
llvm::VPUser::getOperand
VPValue * getOperand(unsigned N) const
Definition: VPlanValue.h:225
llvm::VPTransformState::Instance
Optional< VPIteration > Instance
Hold the indices to generate specific scalar instructions.
Definition: VPlan.h:122
llvm::DominatorTreeBase::verify
bool verify(VerificationLevel VL=VerificationLevel::Full) const
verify - checks if the tree is correct.
Definition: GenericDomTree.h:802
llvm::VPTransformState::CFGState::LastBB
BasicBlock * LastBB
The last IR BasicBlock in the output IR.
Definition: VPlan.h:218
llvm::VPInstruction::SLPLoad
@ SLPLoad
Definition: VPlan.h:610
llvm::VPTransformState::CFGState::VPBB2IRBB
SmallDenseMap< VPBasicBlock *, BasicBlock * > VPBB2IRBB
A mapping of each VPBasicBlock to the corresponding BasicBlock.
Definition: VPlan.h:222
llvm::SmallSetVector
A SetVector that performs no allocations if smaller than a certain size.
Definition: SetVector.h:307
BB
Common register allocation spilling lr str ldr sxth r3 ldr mla r4 can lr mov lr str ldr sxth r3 mla r4 and then merge mul and lr str ldr sxth r3 mla r4 It also increase the likelihood the store may become dead bb27 Successors according to LLVM BB
Definition: README.txt:39
llvm::UnreachableInst
This function has undefined behavior.
Definition: Instructions.h:4670
llvm::VPBlockBase::deleteCFG
static void deleteCFG(VPBlockBase *Entry)
Delete all blocks reachable from a given VPBlockBase, inclusive.
Definition: VPlan.cpp:226
llvm::VPRegionBlock::isReplicator
bool isReplicator() const
An indicator whether this region is to generate multiple replicated instances of output IR correspond...
Definition: VPlan.h:1457
From
BlockVerifier::State From
Definition: BlockVerifier.cpp:55
llvm::VPWidenPHIRecipe::print
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
Definition: VPlan.cpp:934
llvm::User::getOperand
Value * getOperand(unsigned i) const
Definition: User.h:169
llvm::SmallBitVector::size
size_t size() const
Returns the number of bits in this bitvector.
Definition: SmallBitVector.h:192
llvm::Region
Definition: RegionInfo.h:889
llvm::VPValue::~VPValue
virtual ~VPValue()
Definition: VPlan.cpp:67
raw_ostream.h
llvm::VPBasicBlock::getFirstNonPhi
iterator getFirstNonPhi()
Return the position of the first non-phi node recipe in the block.
Definition: VPlan.cpp:233
llvm::VPReductionRecipe::getChainOp
VPValue * getChainOp() const
The VPValue of the scalar Chain being accumulated.
Definition: VPlan.h:1074
llvm::VPInterleavedAccessInfo::VPInterleavedAccessInfo
VPInterleavedAccessInfo(VPlan &Plan, InterleavedAccessInfo &IAI)
Definition: VPlan.cpp:1123
llvm::VPBlockBase::getSuccessors
const VPBlocksTy & getSuccessors() const
Definition: VPlan.h:364
BasicBlockUtils.h
llvm::VPInstruction::dump
void dump() const
Print the VPInstruction to dbgs() (for debugging).
Definition: VPlan.cpp:542
Value.h
llvm::pdb::PDB_SymType::Block
@ Block
llvm::RegionBase::getExit
BlockT * getExit() const
Get the exit BasicBlock of the Region.
Definition: RegionInfo.h:359
llvm::VPDef::getVPValue
VPValue * getVPValue(unsigned I=0)
Returns the VPValue with index I defined by the VPDef.
Definition: VPlanValue.h:328
llvm::VPValue::getUnderlyingValue
Value * getUnderlyingValue()
Return the underlying Value attached to this VPValue.
Definition: VPlanValue.h:84
llvm::VPlan
VPlan models a candidate for vectorization, encoding various decisions take to produce efficient outp...
Definition: VPlan.h:1593
llvm::VPBlockBase::setPlan
void setPlan(VPlan *ParentPlan)
Sets the pointer of the plan containing the block.
Definition: VPlan.cpp:137
llvm::MCID::Terminator
@ Terminator
Definition: MCInstrDesc.h:156
llvm::Value
LLVM Value Representation.
Definition: Value.h:75
llvm::VPRecipeBase::insertBefore
void insertBefore(VPRecipeBase *InsertPos)
Insert an unlinked recipe into a basic block immediately before the specified recipe.
Definition: VPlan.cpp:443
Debug.h
llvm::VPlan::dump
void dump() const
Dump the plan to stderr (for debugging).
Definition: VPlan.cpp:673
llvm::VPlanIngredient
Definition: VPlan.h:1792
llvm::VPWidenSelectRecipe::print
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
Definition: VPlan.cpp:887
llvm::VPValue::printAsOperand
void printAsOperand(raw_ostream &OS, VPSlotTracker &Tracker) const
Definition: VPlan.cpp:1060
llvm::VPTransformState::CFGState::PrevBB
BasicBlock * PrevBB
The previous IR BasicBlock created or used.
Definition: VPlan.h:214
llvm::VPRecipeBase::insertAfter
void insertAfter(VPRecipeBase *InsertPos)
Insert an unlinked Recipe into a basic block immediately after the specified Recipe.
Definition: VPlan.cpp:451
llvm::VPTransformState::hasVectorValue
bool hasVectorValue(VPValue *Def, unsigned Part)
Definition: VPlan.h:146
llvm::InterleavedAccessInfo::getInterleaveGroup
InterleaveGroup< Instruction > * getInterleaveGroup(const Instruction *Instr) const
Get the interleave group that Instr belongs to.
Definition: VectorUtils.h:781