LLVM  13.0.0git
VPlan.h
Go to the documentation of this file.
1 //===- VPlan.h - Represent A Vectorizer Plan --------------------*- C++ -*-===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 /// \file
10 /// This file contains the declarations of the Vectorization Plan base classes:
11 /// 1. VPBasicBlock and VPRegionBlock that inherit from a common pure virtual
12 /// VPBlockBase, together implementing a Hierarchical CFG;
13 /// 2. Specializations of GraphTraits that allow VPBlockBase graphs to be
14 /// treated as proper graphs for generic algorithms;
15 /// 3. Pure virtual VPRecipeBase serving as the base class for recipes contained
16 /// within VPBasicBlocks;
17 /// 4. VPInstruction, a concrete Recipe and VPUser modeling a single planned
18 /// instruction;
19 /// 5. The VPlan class holding a candidate for vectorization;
20 /// 6. The VPlanPrinter class providing a way to print a plan in dot format;
21 /// These are documented in docs/VectorizationPlan.rst.
22 //
23 //===----------------------------------------------------------------------===//
24 
25 #ifndef LLVM_TRANSFORMS_VECTORIZE_VPLAN_H
26 #define LLVM_TRANSFORMS_VECTORIZE_VPLAN_H
27 
28 #include "VPlanLoopInfo.h"
29 #include "VPlanValue.h"
30 #include "llvm/ADT/DenseMap.h"
32 #include "llvm/ADT/GraphTraits.h"
33 #include "llvm/ADT/Optional.h"
35 #include "llvm/ADT/SmallPtrSet.h"
36 #include "llvm/ADT/SmallSet.h"
37 #include "llvm/ADT/SmallVector.h"
38 #include "llvm/ADT/Twine.h"
39 #include "llvm/ADT/ilist.h"
40 #include "llvm/ADT/ilist_node.h"
42 #include "llvm/IR/IRBuilder.h"
43 #include <algorithm>
44 #include <cassert>
45 #include <cstddef>
46 #include <map>
47 #include <string>
48 
49 namespace llvm {
50 
51 class BasicBlock;
52 class DominatorTree;
53 class InnerLoopVectorizer;
54 class LoopInfo;
55 class raw_ostream;
56 class RecurrenceDescriptor;
57 class Value;
58 class VPBasicBlock;
59 class VPRegionBlock;
60 class VPlan;
61 class VPlanSlp;
62 
63 /// A range of powers-of-2 vectorization factors with fixed start and
64 /// adjustable end. The range includes start and excludes end, e.g.,:
65 /// [1, 9) = {1, 2, 4, 8}
66 struct VFRange {
67  // A power of 2.
69 
70  // Need not be a power of 2. If End <= Start range is empty.
72 
73  bool isEmpty() const {
75  }
76 
78  : Start(Start), End(End) {
80  "Both Start and End should have the same scalable flag");
82  "Expected Start to be a power of 2");
83  }
84 };
85 
86 using VPlanPtr = std::unique_ptr<VPlan>;
87 
88 /// In what follows, the term "input IR" refers to code that is fed into the
89 /// vectorizer whereas the term "output IR" refers to code that is generated by
90 /// the vectorizer.
91 
92 /// VPIteration represents a single point in the iteration space of the output
93 /// (vectorized and/or unrolled) IR loop.
94 struct VPIteration {
95  /// in [0..UF)
96  unsigned Part;
97 
98  /// in [0..VF)
99  unsigned Lane;
100 
101  VPIteration(unsigned Part, unsigned Lane) : Part(Part), Lane(Lane) {}
102 
103  bool isFirstIteration() const { return Part == 0 && Lane == 0; }
104 };
105 
106 /// VPTransformState holds information passed down when "executing" a VPlan,
107 /// needed for generating the output IR.
112  : VF(VF), UF(UF), Instance(), LI(LI), DT(DT), Builder(Builder), ILV(ILV),
113  Plan(Plan) {}
114 
115  /// The chosen Vectorization and Unroll Factors of the loop being vectorized.
117  unsigned UF;
118 
119  /// Hold the indices to generate specific scalar instructions. Null indicates
120  /// that all instances are to be generated, using either scalar or vector
121  /// instructions.
123 
124  struct DataState {
125  /// A type for vectorized values in the new loop. Each value from the
126  /// original loop, when vectorized, is represented by UF vector values in
127  /// the new unrolled loop, where UF is the unroll factor.
129 
131 
134  } Data;
135 
136  /// Get the generated Value for a given VPValue and a given Part. Note that
137  /// as some Defs are still created by ILV and managed in its ValueMap, this
138  /// method will delegate the call to ILV in such cases in order to provide
139  /// callers a consistent API.
140  /// \see set.
141  Value *get(VPValue *Def, unsigned Part);
142 
143  /// Get the generated Value for a given VPValue and given Part and Lane.
145 
146  bool hasVectorValue(VPValue *Def, unsigned Part) {
147  auto I = Data.PerPartOutput.find(Def);
148  return I != Data.PerPartOutput.end() && Part < I->second.size() &&
149  I->second[Part];
150  }
151 
153  return Data.PerPartOutput.find(Def) != Data.PerPartOutput.end();
154  }
155 
157  auto I = Data.PerPartScalars.find(Def);
158  if (I == Data.PerPartScalars.end())
159  return false;
160  return Instance.Part < I->second.size() &&
161  Instance.Lane < I->second[Instance.Part].size() &&
162  I->second[Instance.Part][Instance.Lane];
163  }
164 
165  /// Set the generated Value for a given VPValue and a given Part.
166  void set(VPValue *Def, Value *V, unsigned Part) {
167  if (!Data.PerPartOutput.count(Def)) {
169  Data.PerPartOutput[Def] = Entry;
170  }
171  Data.PerPartOutput[Def][Part] = V;
172  }
173  /// Reset an existing vector value for \p Def and a given \p Part.
174  void reset(VPValue *Def, Value *V, unsigned Part) {
175  auto Iter = Data.PerPartOutput.find(Def);
176  assert(Iter != Data.PerPartOutput.end() &&
177  "need to overwrite existing value");
178  Iter->second[Part] = V;
179  }
180 
181  /// Set the generated scalar \p V for \p Def and the given \p Instance.
182  void set(VPValue *Def, Value *V, const VPIteration &Instance) {
183  auto Iter = Data.PerPartScalars.insert({Def, {}});
184  auto &PerPartVec = Iter.first->second;
185  while (PerPartVec.size() <= Instance.Part)
186  PerPartVec.emplace_back();
187  auto &Scalars = PerPartVec[Instance.Part];
188  while (Scalars.size() <= Instance.Lane)
189  Scalars.push_back(nullptr);
190  assert(!Scalars[Instance.Lane] && "should overwrite existing value");
191  Scalars[Instance.Lane] = V;
192  }
193 
194  /// Reset an existing scalar value for \p Def and a given \p Instance.
195  void reset(VPValue *Def, Value *V, const VPIteration &Instance) {
196  auto Iter = Data.PerPartScalars.find(Def);
197  assert(Iter != Data.PerPartScalars.end() &&
198  "need to overwrite existing value");
199  assert(Instance.Part < Iter->second.size() &&
200  "need to overwrite existing value");
201  assert(Instance.Lane < Iter->second[Instance.Part].size() &&
202  "need to overwrite existing value");
203  Iter->second[Instance.Part][Instance.Lane] = V;
204  }
205 
206  /// Hold state information used when constructing the CFG of the output IR,
207  /// traversing the VPBasicBlocks and generating corresponding IR BasicBlocks.
208  struct CFGState {
209  /// The previous VPBasicBlock visited. Initially set to null.
210  VPBasicBlock *PrevVPBB = nullptr;
211 
212  /// The previous IR BasicBlock created or used. Initially set to the new
213  /// header BasicBlock.
214  BasicBlock *PrevBB = nullptr;
215 
216  /// The last IR BasicBlock in the output IR. Set to the new latch
217  /// BasicBlock, used for placing the newly created BasicBlocks.
218  BasicBlock *LastBB = nullptr;
219 
220  /// A mapping of each VPBasicBlock to the corresponding BasicBlock. In case
221  /// of replication, maps the BasicBlock of the last replica created.
223 
224  /// Vector of VPBasicBlocks whose terminator instruction needs to be fixed
225  /// up at the end of vector code generation.
227 
228  CFGState() = default;
229  } CFG;
230 
231  /// Hold a pointer to LoopInfo to register new basic blocks in the loop.
233 
234  /// Hold a pointer to Dominator Tree to register new basic blocks in the loop.
236 
237  /// Hold a reference to the IRBuilder used to generate output IR code.
239 
241 
242  /// Hold the canonical scalar IV of the vector loop (start=0, step=VF*UF).
243  Value *CanonicalIV = nullptr;
244 
245  /// Hold the trip count of the scalar loop.
246  Value *TripCount = nullptr;
247 
248  /// Hold a pointer to InnerLoopVectorizer to reuse its IR generation methods.
250 
251  /// Pointer to the VPlan code is generated for.
253 };
254 
255 /// VPBlockBase is the building block of the Hierarchical Control-Flow Graph.
256 /// A VPBlockBase can be either a VPBasicBlock or a VPRegionBlock.
257 class VPBlockBase {
258  friend class VPBlockUtils;
259 
260  const unsigned char SubclassID; ///< Subclass identifier (for isa/dyn_cast).
261 
262  /// An optional name for the block.
263  std::string Name;
264 
265  /// The immediate VPRegionBlock which this VPBlockBase belongs to, or null if
266  /// it is a topmost VPBlockBase.
267  VPRegionBlock *Parent = nullptr;
268 
269  /// List of predecessor blocks.
270  SmallVector<VPBlockBase *, 1> Predecessors;
271 
272  /// List of successor blocks.
274 
275  /// Successor selector managed by a VPUser. For blocks with zero or one
276  /// successors, there is no operand. Otherwise there is exactly one operand
277  /// which is the branch condition.
278  VPUser CondBitUser;
279 
280  /// If the block is predicated, its predicate is stored as an operand of this
281  /// VPUser to maintain the def-use relations. Otherwise there is no operand
282  /// here.
283  VPUser PredicateUser;
284 
285  /// VPlan containing the block. Can only be set on the entry block of the
286  /// plan.
287  VPlan *Plan = nullptr;
288 
289  /// Add \p Successor as the last successor to this block.
290  void appendSuccessor(VPBlockBase *Successor) {
291  assert(Successor && "Cannot add nullptr successor!");
292  Successors.push_back(Successor);
293  }
294 
295  /// Add \p Predecessor as the last predecessor to this block.
296  void appendPredecessor(VPBlockBase *Predecessor) {
297  assert(Predecessor && "Cannot add nullptr predecessor!");
298  Predecessors.push_back(Predecessor);
299  }
300 
301  /// Remove \p Predecessor from the predecessors of this block.
302  void removePredecessor(VPBlockBase *Predecessor) {
303  auto Pos = find(Predecessors, Predecessor);
304  assert(Pos && "Predecessor does not exist");
305  Predecessors.erase(Pos);
306  }
307 
308  /// Remove \p Successor from the successors of this block.
309  void removeSuccessor(VPBlockBase *Successor) {
310  auto Pos = find(Successors, Successor);
311  assert(Pos && "Successor does not exist");
312  Successors.erase(Pos);
313  }
314 
315 protected:
316  VPBlockBase(const unsigned char SC, const std::string &N)
317  : SubclassID(SC), Name(N) {}
318 
319 public:
320  /// An enumeration for keeping track of the concrete subclass of VPBlockBase
321  /// that are actually instantiated. Values of this enumeration are kept in the
322  /// SubclassID field of the VPBlockBase objects. They are used for concrete
323  /// type identification.
324  using VPBlockTy = enum { VPBasicBlockSC, VPRegionBlockSC };
325 
327 
328  virtual ~VPBlockBase() = default;
329 
330  const std::string &getName() const { return Name; }
331 
332  void setName(const Twine &newName) { Name = newName.str(); }
333 
334  /// \return an ID for the concrete type of this object.
335  /// This is used to implement the classof checks. This should not be used
336  /// for any other purpose, as the values may change as LLVM evolves.
337  unsigned getVPBlockID() const { return SubclassID; }
338 
339  VPRegionBlock *getParent() { return Parent; }
340  const VPRegionBlock *getParent() const { return Parent; }
341 
342  /// \return A pointer to the plan containing the current block.
343  VPlan *getPlan();
344  const VPlan *getPlan() const;
345 
346  /// Sets the pointer of the plan containing the block. The block must be the
347  /// entry block into the VPlan.
348  void setPlan(VPlan *ParentPlan);
349 
350  void setParent(VPRegionBlock *P) { Parent = P; }
351 
352  /// \return the VPBasicBlock that is the entry of this VPBlockBase,
353  /// recursively, if the latter is a VPRegionBlock. Otherwise, if this
354  /// VPBlockBase is a VPBasicBlock, it is returned.
355  const VPBasicBlock *getEntryBasicBlock() const;
357 
358  /// \return the VPBasicBlock that is the exit of this VPBlockBase,
359  /// recursively, if the latter is a VPRegionBlock. Otherwise, if this
360  /// VPBlockBase is a VPBasicBlock, it is returned.
361  const VPBasicBlock *getExitBasicBlock() const;
363 
364  const VPBlocksTy &getSuccessors() const { return Successors; }
365  VPBlocksTy &getSuccessors() { return Successors; }
366 
367  const VPBlocksTy &getPredecessors() const { return Predecessors; }
368  VPBlocksTy &getPredecessors() { return Predecessors; }
369 
370  /// \return the successor of this VPBlockBase if it has a single successor.
371  /// Otherwise return a null pointer.
373  return (Successors.size() == 1 ? *Successors.begin() : nullptr);
374  }
375 
376  /// \return the predecessor of this VPBlockBase if it has a single
377  /// predecessor. Otherwise return a null pointer.
379  return (Predecessors.size() == 1 ? *Predecessors.begin() : nullptr);
380  }
381 
382  size_t getNumSuccessors() const { return Successors.size(); }
383  size_t getNumPredecessors() const { return Predecessors.size(); }
384 
385  /// An Enclosing Block of a block B is any block containing B, including B
386  /// itself. \return the closest enclosing block starting from "this", which
387  /// has successors. \return the root enclosing block if all enclosing blocks
388  /// have no successors.
390 
391  /// \return the closest enclosing block starting from "this", which has
392  /// predecessors. \return the root enclosing block if all enclosing blocks
393  /// have no predecessors.
395 
396  /// \return the successors either attached directly to this VPBlockBase or, if
397  /// this VPBlockBase is the exit block of a VPRegionBlock and has no
398  /// successors of its own, search recursively for the first enclosing
399  /// VPRegionBlock that has successors and return them. If no such
400  /// VPRegionBlock exists, return the (empty) successors of the topmost
401  /// VPBlockBase reached.
404  }
405 
406  /// \return the hierarchical successor of this VPBlockBase if it has a single
407  /// hierarchical successor. Otherwise return a null pointer.
410  }
411 
412  /// \return the predecessors either attached directly to this VPBlockBase or,
413  /// if this VPBlockBase is the entry block of a VPRegionBlock and has no
414  /// predecessors of its own, search recursively for the first enclosing
415  /// VPRegionBlock that has predecessors and return them. If no such
416  /// VPRegionBlock exists, return the (empty) predecessors of the topmost
417  /// VPBlockBase reached.
420  }
421 
422  /// \return the hierarchical predecessor of this VPBlockBase if it has a
423  /// single hierarchical predecessor. Otherwise return a null pointer.
426  }
427 
428  /// \return the condition bit selecting the successor.
429  VPValue *getCondBit();
430  /// \return the condition bit selecting the successor.
431  const VPValue *getCondBit() const;
432  /// Set the condition bit selecting the successor.
433  void setCondBit(VPValue *CV);
434 
435  /// \return the block's predicate.
437  /// \return the block's predicate.
438  const VPValue *getPredicate() const;
439  /// Set the block's predicate.
440  void setPredicate(VPValue *Pred);
441 
442  /// Set a given VPBlockBase \p Successor as the single successor of this
443  /// VPBlockBase. This VPBlockBase is not added as predecessor of \p Successor.
444  /// This VPBlockBase must have no successors.
446  assert(Successors.empty() && "Setting one successor when others exist.");
447  appendSuccessor(Successor);
448  }
449 
450  /// Set two given VPBlockBases \p IfTrue and \p IfFalse to be the two
451  /// successors of this VPBlockBase. \p Condition is set as the successor
452  /// selector. This VPBlockBase is not added as predecessor of \p IfTrue or \p
453  /// IfFalse. This VPBlockBase must have no successors.
454  void setTwoSuccessors(VPBlockBase *IfTrue, VPBlockBase *IfFalse,
455  VPValue *Condition) {
456  assert(Successors.empty() && "Setting two successors when others exist.");
457  assert(Condition && "Setting two successors without condition!");
458  setCondBit(Condition);
459  appendSuccessor(IfTrue);
460  appendSuccessor(IfFalse);
461  }
462 
463  /// Set each VPBasicBlock in \p NewPreds as predecessor of this VPBlockBase.
464  /// This VPBlockBase must have no predecessors. This VPBlockBase is not added
465  /// as successor of any VPBasicBlock in \p NewPreds.
467  assert(Predecessors.empty() && "Block predecessors already set.");
468  for (auto *Pred : NewPreds)
469  appendPredecessor(Pred);
470  }
471 
472  /// Remove all the predecessor of this block.
473  void clearPredecessors() { Predecessors.clear(); }
474 
475  /// Remove all the successors of this block and set to null its condition bit
477  Successors.clear();
478  setCondBit(nullptr);
479  }
480 
481  /// The method which generates the output IR that correspond to this
482  /// VPBlockBase, thereby "executing" the VPlan.
483  virtual void execute(struct VPTransformState *State) = 0;
484 
485  /// Delete all blocks reachable from a given VPBlockBase, inclusive.
486  static void deleteCFG(VPBlockBase *Entry);
487 
488  void printAsOperand(raw_ostream &OS, bool PrintType) const {
489  OS << getName();
490  }
491 
492  void print(raw_ostream &OS) const {
493  // TODO: Only printing VPBB name for now since we only have dot printing
494  // support for VPInstructions/Recipes.
495  printAsOperand(OS, false);
496  }
497 
498  /// Return true if it is legal to hoist instructions into this block.
500  // There are currently no constraints that prevent an instruction to be
501  // hoisted into a VPBlockBase.
502  return true;
503  }
504 
505  /// Replace all operands of VPUsers in the block with \p NewValue and also
506  /// replaces all uses of VPValues defined in the block with NewValue.
507  virtual void dropAllReferences(VPValue *NewValue) = 0;
508 };
509 
510 /// VPRecipeBase is a base class modeling a sequence of one or more output IR
511 /// instructions. VPRecipeBase owns the the VPValues it defines through VPDef
512 /// and is responsible for deleting its defined values. Single-value
513 /// VPRecipeBases that also inherit from VPValue must make sure to inherit from
514 /// VPRecipeBase before VPValue.
515 class VPRecipeBase : public ilist_node_with_parent<VPRecipeBase, VPBasicBlock>,
516  public VPDef,
517  public VPUser {
518  friend VPBasicBlock;
519  friend class VPBlockUtils;
520 
521 
522  /// Each VPRecipe belongs to a single VPBasicBlock.
523  VPBasicBlock *Parent = nullptr;
524 
525 public:
527  : VPDef(SC), VPUser(Operands) {}
528 
529  template <typename IterT>
531  : VPDef(SC), VPUser(Operands) {}
532  virtual ~VPRecipeBase() = default;
533 
534  /// \return the VPBasicBlock which this VPRecipe belongs to.
535  VPBasicBlock *getParent() { return Parent; }
536  const VPBasicBlock *getParent() const { return Parent; }
537 
538  /// The method which generates the output IR instructions that correspond to
539  /// this VPRecipe, thereby "executing" the VPlan.
540  virtual void execute(struct VPTransformState &State) = 0;
541 
542  /// Insert an unlinked recipe into a basic block immediately before
543  /// the specified recipe.
544  void insertBefore(VPRecipeBase *InsertPos);
545 
546  /// Insert an unlinked Recipe into a basic block immediately after
547  /// the specified Recipe.
548  void insertAfter(VPRecipeBase *InsertPos);
549 
550  /// Unlink this recipe from its current VPBasicBlock and insert it into
551  /// the VPBasicBlock that MovePos lives in, right after MovePos.
552  void moveAfter(VPRecipeBase *MovePos);
553 
554  /// Unlink this recipe and insert into BB before I.
555  ///
556  /// \pre I is a valid iterator into BB.
558 
559  /// This method unlinks 'this' from the containing basic block, but does not
560  /// delete it.
561  void removeFromParent();
562 
563  /// This method unlinks 'this' from the containing basic block and deletes it.
564  ///
565  /// \returns an iterator pointing to the element after the erased one
567 
568  /// Returns the underlying instruction, if the recipe is a VPValue or nullptr
569  /// otherwise.
571  return cast<Instruction>(getVPValue()->getUnderlyingValue());
572  }
574  return cast<Instruction>(getVPValue()->getUnderlyingValue());
575  }
576 
577  /// Method to support type inquiry through isa, cast, and dyn_cast.
578  static inline bool classof(const VPDef *D) {
579  // All VPDefs are also VPRecipeBases.
580  return true;
581  }
582 };
583 
584 inline bool VPUser::classof(const VPDef *Def) {
585  return Def->getVPDefID() == VPRecipeBase::VPInstructionSC ||
586  Def->getVPDefID() == VPRecipeBase::VPWidenSC ||
587  Def->getVPDefID() == VPRecipeBase::VPWidenCallSC ||
588  Def->getVPDefID() == VPRecipeBase::VPWidenSelectSC ||
589  Def->getVPDefID() == VPRecipeBase::VPWidenGEPSC ||
590  Def->getVPDefID() == VPRecipeBase::VPBlendSC ||
591  Def->getVPDefID() == VPRecipeBase::VPInterleaveSC ||
592  Def->getVPDefID() == VPRecipeBase::VPReplicateSC ||
593  Def->getVPDefID() == VPRecipeBase::VPReductionSC ||
594  Def->getVPDefID() == VPRecipeBase::VPBranchOnMaskSC ||
595  Def->getVPDefID() == VPRecipeBase::VPWidenMemoryInstructionSC;
596 }
597 
598 /// This is a concrete Recipe that models a single VPlan-level instruction.
599 /// While as any Recipe it may generate a sequence of IR instructions when
600 /// executed, these instructions would always form a single-def expression as
601 /// the VPInstruction is also a single def-use vertex.
602 class VPInstruction : public VPRecipeBase, public VPValue {
603  friend class VPlanSlp;
604 
605 public:
606  /// VPlan opcodes, extending LLVM IR with idiomatics instructions.
607  enum {
608  Not = Instruction::OtherOpsEnd + 1,
613  };
614 
615 private:
616  typedef unsigned char OpcodeTy;
617  OpcodeTy Opcode;
618 
619  /// Utility method serving execute(): generates a single instance of the
620  /// modeled instruction.
621  void generateInstruction(VPTransformState &State, unsigned Part);
622 
623 protected:
625 
626 public:
628  : VPRecipeBase(VPRecipeBase::VPInstructionSC, Operands),
629  VPValue(VPValue::VPVInstructionSC, nullptr, this), Opcode(Opcode) {}
630 
632  : VPRecipeBase(VPRecipeBase::VPInstructionSC, {}),
633  VPValue(VPValue::VPVInstructionSC, nullptr, this), Opcode(Opcode) {
634  for (auto *I : Operands)
635  addOperand(I->getVPValue());
636  }
637 
638  VPInstruction(unsigned Opcode, std::initializer_list<VPValue *> Operands)
639  : VPInstruction(Opcode, ArrayRef<VPValue *>(Operands)) {}
640 
641  /// Method to support type inquiry through isa, cast, and dyn_cast.
642  static inline bool classof(const VPValue *V) {
644  }
645 
646  VPInstruction *clone() const {
648  return new VPInstruction(Opcode, Operands);
649  }
650 
651  /// Method to support type inquiry through isa, cast, and dyn_cast.
652  static inline bool classof(const VPDef *R) {
653  return R->getVPDefID() == VPRecipeBase::VPInstructionSC;
654  }
655 
656  unsigned getOpcode() const { return Opcode; }
657 
658  /// Generate the instruction.
659  /// TODO: We currently execute only per-part unless a specific instance is
660  /// provided.
661  void execute(VPTransformState &State) override;
662 
663  /// Print the VPInstruction to \p O.
664  void print(raw_ostream &O, const Twine &Indent,
665  VPSlotTracker &SlotTracker) const override;
666 
667  /// Print the VPInstruction to dbgs() (for debugging).
668  void dump() const;
669 
670  /// Return true if this instruction may modify memory.
671  bool mayWriteToMemory() const {
672  // TODO: we can use attributes of the called function to rule out memory
673  // modifications.
674  return Opcode == Instruction::Store || Opcode == Instruction::Call ||
675  Opcode == Instruction::Invoke || Opcode == SLPStore;
676  }
677 
678  bool hasResult() const {
679  // CallInst may or may not have a result, depending on the called function.
680  // Conservatively return calls have results for now.
681  switch (getOpcode()) {
682  case Instruction::Ret:
683  case Instruction::Br:
684  case Instruction::Store:
685  case Instruction::Switch:
686  case Instruction::IndirectBr:
687  case Instruction::Resume:
688  case Instruction::CatchRet:
689  case Instruction::Unreachable:
690  case Instruction::Fence:
691  case Instruction::AtomicRMW:
692  return false;
693  default:
694  return true;
695  }
696  }
697 };
698 
699 /// VPWidenRecipe is a recipe for producing a copy of vector type its
700 /// ingredient. This recipe covers most of the traditional vectorization cases
701 /// where each ingredient transforms into a vectorized version of itself.
702 class VPWidenRecipe : public VPRecipeBase, public VPValue {
703 public:
704  template <typename IterT>
706  : VPRecipeBase(VPRecipeBase::VPWidenSC, Operands),
708 
709  ~VPWidenRecipe() override = default;
710 
711  /// Method to support type inquiry through isa, cast, and dyn_cast.
712  static inline bool classof(const VPDef *D) {
713  return D->getVPDefID() == VPRecipeBase::VPWidenSC;
714  }
715  static inline bool classof(const VPValue *V) {
716  return V->getVPValueID() == VPValue::VPVWidenSC;
717  }
718 
719  /// Produce widened copies of all Ingredients.
720  void execute(VPTransformState &State) override;
721 
722  /// Print the recipe.
723  void print(raw_ostream &O, const Twine &Indent,
724  VPSlotTracker &SlotTracker) const override;
725 };
726 
727 /// A recipe for widening Call instructions.
728 class VPWidenCallRecipe : public VPRecipeBase, public VPValue {
729 
730 public:
731  template <typename IterT>
733  : VPRecipeBase(VPRecipeBase::VPWidenCallSC, CallArguments),
735 
736  ~VPWidenCallRecipe() override = default;
737 
738  /// Method to support type inquiry through isa, cast, and dyn_cast.
739  static inline bool classof(const VPDef *D) {
740  return D->getVPDefID() == VPRecipeBase::VPWidenCallSC;
741  }
742 
743  /// Produce a widened version of the call instruction.
744  void execute(VPTransformState &State) override;
745 
746  /// Print the recipe.
747  void print(raw_ostream &O, const Twine &Indent,
748  VPSlotTracker &SlotTracker) const override;
749 };
750 
751 /// A recipe for widening select instructions.
752 class VPWidenSelectRecipe : public VPRecipeBase, public VPValue {
753 
754  /// Is the condition of the select loop invariant?
755  bool InvariantCond;
756 
757 public:
758  template <typename IterT>
760  bool InvariantCond)
761  : VPRecipeBase(VPRecipeBase::VPWidenSelectSC, Operands),
763  InvariantCond(InvariantCond) {}
764 
765  ~VPWidenSelectRecipe() override = default;
766 
767  /// Method to support type inquiry through isa, cast, and dyn_cast.
768  static inline bool classof(const VPDef *D) {
769  return D->getVPDefID() == VPRecipeBase::VPWidenSelectSC;
770  }
771 
772  /// Produce a widened version of the select instruction.
773  void execute(VPTransformState &State) override;
774 
775  /// Print the recipe.
776  void print(raw_ostream &O, const Twine &Indent,
777  VPSlotTracker &SlotTracker) const override;
778 };
779 
780 /// A recipe for handling GEP instructions.
781 class VPWidenGEPRecipe : public VPRecipeBase, public VPValue {
782  bool IsPtrLoopInvariant;
783  SmallBitVector IsIndexLoopInvariant;
784 
785 public:
786  template <typename IterT>
788  : VPRecipeBase(VPRecipeBase::VPWidenGEPSC, Operands),
789  VPValue(VPWidenGEPSC, GEP, this),
790  IsIndexLoopInvariant(GEP->getNumIndices(), false) {}
791 
792  template <typename IterT>
794  Loop *OrigLoop)
795  : VPRecipeBase(VPRecipeBase::VPWidenGEPSC, Operands),
797  IsIndexLoopInvariant(GEP->getNumIndices(), false) {
798  IsPtrLoopInvariant = OrigLoop->isLoopInvariant(GEP->getPointerOperand());
799  for (auto Index : enumerate(GEP->indices()))
800  IsIndexLoopInvariant[Index.index()] =
801  OrigLoop->isLoopInvariant(Index.value().get());
802  }
803  ~VPWidenGEPRecipe() override = default;
804 
805  /// Method to support type inquiry through isa, cast, and dyn_cast.
806  static inline bool classof(const VPDef *D) {
807  return D->getVPDefID() == VPRecipeBase::VPWidenGEPSC;
808  }
809 
810  /// Generate the gep nodes.
811  void execute(VPTransformState &State) override;
812 
813  /// Print the recipe.
814  void print(raw_ostream &O, const Twine &Indent,
815  VPSlotTracker &SlotTracker) const override;
816 };
817 
818 /// A recipe for handling phi nodes of integer and floating-point inductions,
819 /// producing their vector and scalar values.
821  PHINode *IV;
822 
823 public:
825  TruncInst *Trunc = nullptr)
826  : VPRecipeBase(VPWidenIntOrFpInductionSC, {Start}), IV(IV) {
827  if (Trunc)
828  new VPValue(Trunc, this);
829  else
830  new VPValue(IV, this);
831 
832  if (Cast)
833  new VPValue(Cast, this);
834  }
835  ~VPWidenIntOrFpInductionRecipe() override = default;
836 
837  /// Method to support type inquiry through isa, cast, and dyn_cast.
838  static inline bool classof(const VPDef *D) {
839  return D->getVPDefID() == VPRecipeBase::VPWidenIntOrFpInductionSC;
840  }
841 
842  /// Generate the vectorized and scalarized versions of the phi node as
843  /// needed by their users.
844  void execute(VPTransformState &State) override;
845 
846  /// Print the recipe.
847  void print(raw_ostream &O, const Twine &Indent,
848  VPSlotTracker &SlotTracker) const override;
849 
850  /// Returns the start value of the induction.
851  VPValue *getStartValue() { return getOperand(0); }
852 
853  /// Returns the cast VPValue, if one is attached, or nullptr otherwise.
855  if (getNumDefinedValues() != 2)
856  return nullptr;
857  return getVPValue(1);
858  }
859 
860  /// Returns the first defined value as TruncInst, if it is one or nullptr
861  /// otherwise.
863  return dyn_cast_or_null<TruncInst>(getVPValue(0)->getUnderlyingValue());
864  }
865  const TruncInst *getTruncInst() const {
866  return dyn_cast_or_null<TruncInst>(getVPValue(0)->getUnderlyingValue());
867  }
868 };
869 
870 /// A recipe for handling all phi nodes except for integer and FP inductions.
871 /// For reduction PHIs, RdxDesc must point to the corresponding recurrence
872 /// descriptor and the start value is the first operand of the recipe.
873 /// In the VPlan native path, all incoming VPValues & VPBasicBlock pairs are
874 /// managed in the recipe directly.
875 class VPWidenPHIRecipe : public VPRecipeBase, public VPValue {
876  /// Descriptor for a reduction PHI.
877  RecurrenceDescriptor *RdxDesc = nullptr;
878 
879  /// List of incoming blocks. Only used in the VPlan native path.
880  SmallVector<VPBasicBlock *, 2> IncomingBlocks;
881 
882 public:
883  /// Create a new VPWidenPHIRecipe for the reduction \p Phi described by \p
884  /// RdxDesc.
886  : VPWidenPHIRecipe(Phi) {
887  this->RdxDesc = &RdxDesc;
888  addOperand(&Start);
889  }
890 
891  /// Create a VPWidenPHIRecipe for \p Phi
893  : VPRecipeBase(VPWidenPHISC, {}),
894  VPValue(VPValue::VPVWidenPHISC, Phi, this) {}
895  ~VPWidenPHIRecipe() override = default;
896 
897  /// Method to support type inquiry through isa, cast, and dyn_cast.
898  static inline bool classof(const VPDef *D) {
899  return D->getVPDefID() == VPRecipeBase::VPWidenPHISC;
900  }
901  static inline bool classof(const VPValue *V) {
902  return V->getVPValueID() == VPValue::VPVWidenPHISC;
903  }
904 
905  /// Generate the phi/select nodes.
906  void execute(VPTransformState &State) override;
907 
908  /// Print the recipe.
909  void print(raw_ostream &O, const Twine &Indent,
910  VPSlotTracker &SlotTracker) const override;
911 
912  /// Returns the start value of the phi, if it is a reduction.
914  return getNumOperands() == 0 ? nullptr : getOperand(0);
915  }
916 
917  /// Adds a pair (\p IncomingV, \p IncomingBlock) to the phi.
918  void addIncoming(VPValue *IncomingV, VPBasicBlock *IncomingBlock) {
919  addOperand(IncomingV);
920  IncomingBlocks.push_back(IncomingBlock);
921  }
922 
923  /// Returns the \p I th incoming VPValue.
924  VPValue *getIncomingValue(unsigned I) { return getOperand(I); }
925 
926  /// Returns the \p I th incoming VPBasicBlock.
927  VPBasicBlock *getIncomingBlock(unsigned I) { return IncomingBlocks[I]; }
928 };
929 
930 /// A recipe for vectorizing a phi-node as a sequence of mask-based select
931 /// instructions.
932 class VPBlendRecipe : public VPRecipeBase, public VPValue {
933  PHINode *Phi;
934 
935 public:
936  /// The blend operation is a User of the incoming values and of their
937  /// respective masks, ordered [I0, M0, I1, M1, ...]. Note that a single value
938  /// might be incoming with a full mask for which there is no VPValue.
940  : VPRecipeBase(VPBlendSC, Operands),
941  VPValue(VPValue::VPVBlendSC, Phi, this), Phi(Phi) {
942  assert(Operands.size() > 0 &&
943  ((Operands.size() == 1) || (Operands.size() % 2 == 0)) &&
944  "Expected either a single incoming value or a positive even number "
945  "of operands");
946  }
947 
948  /// Method to support type inquiry through isa, cast, and dyn_cast.
949  static inline bool classof(const VPDef *D) {
950  return D->getVPDefID() == VPRecipeBase::VPBlendSC;
951  }
952 
953  /// Return the number of incoming values, taking into account that a single
954  /// incoming value has no mask.
955  unsigned getNumIncomingValues() const { return (getNumOperands() + 1) / 2; }
956 
957  /// Return incoming value number \p Idx.
958  VPValue *getIncomingValue(unsigned Idx) const { return getOperand(Idx * 2); }
959 
960  /// Return mask number \p Idx.
961  VPValue *getMask(unsigned Idx) const { return getOperand(Idx * 2 + 1); }
962 
963  /// Generate the phi/select nodes.
964  void execute(VPTransformState &State) override;
965 
966  /// Print the recipe.
967  void print(raw_ostream &O, const Twine &Indent,
968  VPSlotTracker &SlotTracker) const override;
969 };
970 
971 /// VPInterleaveRecipe is a recipe for transforming an interleave group of load
972 /// or stores into one wide load/store and shuffles. The first operand of a
973 /// VPInterleave recipe is the address, followed by the stored values, followed
974 /// by an optional mask.
977 
978  bool HasMask = false;
979 
980 public:
982  ArrayRef<VPValue *> StoredValues, VPValue *Mask)
983  : VPRecipeBase(VPInterleaveSC, {Addr}), IG(IG) {
984  for (unsigned i = 0; i < IG->getFactor(); ++i)
985  if (Instruction *I = IG->getMember(i)) {
986  if (I->getType()->isVoidTy())
987  continue;
988  new VPValue(I, this);
989  }
990 
991  for (auto *SV : StoredValues)
992  addOperand(SV);
993  if (Mask) {
994  HasMask = true;
995  addOperand(Mask);
996  }
997  }
998  ~VPInterleaveRecipe() override = default;
999 
1000  /// Method to support type inquiry through isa, cast, and dyn_cast.
1001  static inline bool classof(const VPDef *D) {
1002  return D->getVPDefID() == VPRecipeBase::VPInterleaveSC;
1003  }
1004 
1005  /// Return the address accessed by this recipe.
1006  VPValue *getAddr() const {
1007  return getOperand(0); // Address is the 1st, mandatory operand.
1008  }
1009 
1010  /// Return the mask used by this recipe. Note that a full mask is represented
1011  /// by a nullptr.
1012  VPValue *getMask() const {
1013  // Mask is optional and therefore the last, currently 2nd operand.
1014  return HasMask ? getOperand(getNumOperands() - 1) : nullptr;
1015  }
1016 
1017  /// Return the VPValues stored by this interleave group. If it is a load
1018  /// interleave group, return an empty ArrayRef.
1020  // The first operand is the address, followed by the stored values, followed
1021  // by an optional mask.
1023  .slice(1, getNumOperands() - (HasMask ? 2 : 1));
1024  }
1025 
1026  /// Generate the wide load or store, and shuffles.
1027  void execute(VPTransformState &State) override;
1028 
1029  /// Print the recipe.
1030  void print(raw_ostream &O, const Twine &Indent,
1031  VPSlotTracker &SlotTracker) const override;
1032 
1034 };
1035 
1036 /// A recipe to represent inloop reduction operations, performing a reduction on
1037 /// a vector operand into a scalar value, and adding the result to a chain.
1038 /// The Operands are {ChainOp, VecOp, [Condition]}.
1039 class VPReductionRecipe : public VPRecipeBase, public VPValue {
1040  /// The recurrence decriptor for the reduction in question.
1041  RecurrenceDescriptor *RdxDesc;
1042  /// Pointer to the TTI, needed to create the target reduction
1043  const TargetTransformInfo *TTI;
1044 
1045 public:
1047  VPValue *VecOp, VPValue *CondOp,
1048  const TargetTransformInfo *TTI)
1049  : VPRecipeBase(VPRecipeBase::VPReductionSC, {ChainOp, VecOp}),
1050  VPValue(VPValue::VPVReductionSC, I, this), RdxDesc(R), TTI(TTI) {
1051  if (CondOp)
1052  addOperand(CondOp);
1053  }
1054 
1055  ~VPReductionRecipe() override = default;
1056 
1057  /// Method to support type inquiry through isa, cast, and dyn_cast.
1058  static inline bool classof(const VPValue *V) {
1059  return V->getVPValueID() == VPValue::VPVReductionSC;
1060  }
1061 
1062  static inline bool classof(const VPDef *D) {
1063  return D->getVPDefID() == VPRecipeBase::VPReductionSC;
1064  }
1065 
1066  /// Generate the reduction in the loop
1067  void execute(VPTransformState &State) override;
1068 
1069  /// Print the recipe.
1070  void print(raw_ostream &O, const Twine &Indent,
1071  VPSlotTracker &SlotTracker) const override;
1072 
1073  /// The VPValue of the scalar Chain being accumulated.
1074  VPValue *getChainOp() const { return getOperand(0); }
1075  /// The VPValue of the vector value to be reduced.
1076  VPValue *getVecOp() const { return getOperand(1); }
1077  /// The VPValue of the condition for the block.
1078  VPValue *getCondOp() const {
1079  return getNumOperands() > 2 ? getOperand(2) : nullptr;
1080  }
1081 };
1082 
1083 /// VPReplicateRecipe replicates a given instruction producing multiple scalar
1084 /// copies of the original scalar type, one per lane, instead of producing a
1085 /// single copy of widened type for all lanes. If the instruction is known to be
1086 /// uniform only one copy, per lane zero, will be generated.
1087 class VPReplicateRecipe : public VPRecipeBase, public VPValue {
1088  /// Indicator if only a single replica per lane is needed.
1089  bool IsUniform;
1090 
1091  /// Indicator if the replicas are also predicated.
1092  bool IsPredicated;
1093 
1094  /// Indicator if the scalar values should also be packed into a vector.
1095  bool AlsoPack;
1096 
1097 public:
1098  template <typename IterT>
1100  bool IsUniform, bool IsPredicated = false)
1101  : VPRecipeBase(VPReplicateSC, Operands), VPValue(VPVReplicateSC, I, this),
1102  IsUniform(IsUniform), IsPredicated(IsPredicated) {
1103  // Retain the previous behavior of predicateInstructions(), where an
1104  // insert-element of a predicated instruction got hoisted into the
1105  // predicated basic block iff it was its only user. This is achieved by
1106  // having predicated instructions also pack their values into a vector by
1107  // default unless they have a replicated user which uses their scalar value.
1108  AlsoPack = IsPredicated && !I->use_empty();
1109  }
1110 
1111  ~VPReplicateRecipe() override = default;
1112 
1113  /// Method to support type inquiry through isa, cast, and dyn_cast.
1114  static inline bool classof(const VPDef *D) {
1115  return D->getVPDefID() == VPRecipeBase::VPReplicateSC;
1116  }
1117 
1118  static inline bool classof(const VPValue *V) {
1119  return V->getVPValueID() == VPValue::VPVReplicateSC;
1120  }
1121 
1122  /// Generate replicas of the desired Ingredient. Replicas will be generated
1123  /// for all parts and lanes unless a specific part and lane are specified in
1124  /// the \p State.
1125  void execute(VPTransformState &State) override;
1126 
1127  void setAlsoPack(bool Pack) { AlsoPack = Pack; }
1128 
1129  /// Print the recipe.
1130  void print(raw_ostream &O, const Twine &Indent,
1131  VPSlotTracker &SlotTracker) const override;
1132 
1133  bool isUniform() const { return IsUniform; }
1134 
1135  bool isPacked() const { return AlsoPack; }
1136 };
1137 
1138 /// A recipe for generating conditional branches on the bits of a mask.
1140 public:
1142  : VPRecipeBase(VPBranchOnMaskSC, {}) {
1143  if (BlockInMask) // nullptr means all-one mask.
1144  addOperand(BlockInMask);
1145  }
1146 
1147  /// Method to support type inquiry through isa, cast, and dyn_cast.
1148  static inline bool classof(const VPDef *D) {
1149  return D->getVPDefID() == VPRecipeBase::VPBranchOnMaskSC;
1150  }
1151 
1152  /// Generate the extraction of the appropriate bit from the block mask and the
1153  /// conditional branch.
1154  void execute(VPTransformState &State) override;
1155 
1156  /// Print the recipe.
1157  void print(raw_ostream &O, const Twine &Indent,
1158  VPSlotTracker &SlotTracker) const override {
1159  O << " +\n" << Indent << "\"BRANCH-ON-MASK ";
1160  if (VPValue *Mask = getMask())
1161  Mask->printAsOperand(O, SlotTracker);
1162  else
1163  O << " All-One";
1164  O << "\\l\"";
1165  }
1166 
1167  /// Return the mask used by this recipe. Note that a full mask is represented
1168  /// by a nullptr.
1169  VPValue *getMask() const {
1170  assert(getNumOperands() <= 1 && "should have either 0 or 1 operands");
1171  // Mask is optional.
1172  return getNumOperands() == 1 ? getOperand(0) : nullptr;
1173  }
1174 };
1175 
1176 /// VPPredInstPHIRecipe is a recipe for generating the phi nodes needed when
1177 /// control converges back from a Branch-on-Mask. The phi nodes are needed in
1178 /// order to merge values that are set under such a branch and feed their uses.
1179 /// The phi nodes can be scalar or vector depending on the users of the value.
1180 /// This recipe works in concert with VPBranchOnMaskRecipe.
1181 class VPPredInstPHIRecipe : public VPRecipeBase, public VPValue {
1182 public:
1183  /// Construct a VPPredInstPHIRecipe given \p PredInst whose value needs a phi
1184  /// nodes after merging back from a Branch-on-Mask.
1186  : VPRecipeBase(VPPredInstPHISC, PredV),
1187  VPValue(VPValue::VPVPredInstPHI, nullptr, this) {}
1188  ~VPPredInstPHIRecipe() override = default;
1189 
1190  /// Method to support type inquiry through isa, cast, and dyn_cast.
1191  static inline bool classof(const VPDef *D) {
1192  return D->getVPDefID() == VPRecipeBase::VPPredInstPHISC;
1193  }
1194 
1195  /// Generates phi nodes for live-outs as needed to retain SSA form.
1196  void execute(VPTransformState &State) override;
1197 
1198  /// Print the recipe.
1199  void print(raw_ostream &O, const Twine &Indent,
1200  VPSlotTracker &SlotTracker) const override;
1201 };
1202 
1203 /// A Recipe for widening load/store operations.
1204 /// The recipe uses the following VPValues:
1205 /// - For load: Address, optional mask
1206 /// - For store: Address, stored value, optional mask
1207 /// TODO: We currently execute only per-part unless a specific instance is
1208 /// provided.
1210  Instruction &Ingredient;
1211 
1212  void setMask(VPValue *Mask) {
1213  if (!Mask)
1214  return;
1215  addOperand(Mask);
1216  }
1217 
1218  bool isMasked() const {
1219  return isStore() ? getNumOperands() == 3 : getNumOperands() == 2;
1220  }
1221 
1222 public:
1224  : VPRecipeBase(VPWidenMemoryInstructionSC, {Addr}), Ingredient(Load) {
1226  setMask(Mask);
1227  }
1228 
1230  VPValue *StoredValue, VPValue *Mask)
1231  : VPRecipeBase(VPWidenMemoryInstructionSC, {Addr, StoredValue}),
1232  Ingredient(Store) {
1233  setMask(Mask);
1234  }
1235 
1236  /// Method to support type inquiry through isa, cast, and dyn_cast.
1237  static inline bool classof(const VPDef *D) {
1238  return D->getVPDefID() == VPRecipeBase::VPWidenMemoryInstructionSC;
1239  }
1240 
1241  /// Return the address accessed by this recipe.
1242  VPValue *getAddr() const {
1243  return getOperand(0); // Address is the 1st, mandatory operand.
1244  }
1245 
1246  /// Return the mask used by this recipe. Note that a full mask is represented
1247  /// by a nullptr.
1248  VPValue *getMask() const {
1249  // Mask is optional and therefore the last operand.
1250  return isMasked() ? getOperand(getNumOperands() - 1) : nullptr;
1251  }
1252 
1253  /// Returns true if this recipe is a store.
1254  bool isStore() const { return isa<StoreInst>(Ingredient); }
1255 
1256  /// Return the address accessed by this recipe.
1258  assert(isStore() && "Stored value only available for store instructions");
1259  return getOperand(1); // Stored value is the 2nd, mandatory operand.
1260  }
1261 
1262  /// Generate the wide load/store.
1263  void execute(VPTransformState &State) override;
1264 
1265  /// Print the recipe.
1266  void print(raw_ostream &O, const Twine &Indent,
1267  VPSlotTracker &SlotTracker) const override;
1268 };
1269 
1270 /// A Recipe for widening the canonical induction variable of the vector loop.
1272 public:
1273  VPWidenCanonicalIVRecipe() : VPRecipeBase(VPWidenCanonicalIVSC, {}) {
1274  new VPValue(nullptr, this);
1275  }
1276 
1277  ~VPWidenCanonicalIVRecipe() override = default;
1278 
1279  /// Method to support type inquiry through isa, cast, and dyn_cast.
1280  static inline bool classof(const VPDef *D) {
1281  return D->getVPDefID() == VPRecipeBase::VPWidenCanonicalIVSC;
1282  }
1283 
1284  /// Generate a canonical vector induction variable of the vector loop, with
1285  /// start = {<Part*VF, Part*VF+1, ..., Part*VF+VF-1> for 0 <= Part < UF}, and
1286  /// step = <VF*UF, VF*UF, ..., VF*UF>.
1287  void execute(VPTransformState &State) override;
1288 
1289  /// Print the recipe.
1290  void print(raw_ostream &O, const Twine &Indent,
1291  VPSlotTracker &SlotTracker) const override;
1292 };
1293 
1294 /// VPBasicBlock serves as the leaf of the Hierarchical Control-Flow Graph. It
1295 /// holds a sequence of zero or more VPRecipe's each representing a sequence of
1296 /// output IR instructions.
1297 class VPBasicBlock : public VPBlockBase {
1298 public:
1300 
1301 private:
1302  /// The VPRecipes held in the order of output instructions to generate.
1303  RecipeListTy Recipes;
1304 
1305 public:
1306  VPBasicBlock(const Twine &Name = "", VPRecipeBase *Recipe = nullptr)
1307  : VPBlockBase(VPBasicBlockSC, Name.str()) {
1308  if (Recipe)
1309  appendRecipe(Recipe);
1310  }
1311 
1312  ~VPBasicBlock() override { Recipes.clear(); }
1313 
1314  /// Instruction iterators...
1319 
1320  //===--------------------------------------------------------------------===//
1321  /// Recipe iterator methods
1322  ///
1323  inline iterator begin() { return Recipes.begin(); }
1324  inline const_iterator begin() const { return Recipes.begin(); }
1325  inline iterator end() { return Recipes.end(); }
1326  inline const_iterator end() const { return Recipes.end(); }
1327 
1328  inline reverse_iterator rbegin() { return Recipes.rbegin(); }
1329  inline const_reverse_iterator rbegin() const { return Recipes.rbegin(); }
1330  inline reverse_iterator rend() { return Recipes.rend(); }
1331  inline const_reverse_iterator rend() const { return Recipes.rend(); }
1332 
1333  inline size_t size() const { return Recipes.size(); }
1334  inline bool empty() const { return Recipes.empty(); }
1335  inline const VPRecipeBase &front() const { return Recipes.front(); }
1336  inline VPRecipeBase &front() { return Recipes.front(); }
1337  inline const VPRecipeBase &back() const { return Recipes.back(); }
1338  inline VPRecipeBase &back() { return Recipes.back(); }
1339 
1340  /// Returns a reference to the list of recipes.
1341  RecipeListTy &getRecipeList() { return Recipes; }
1342 
1343  /// Returns a pointer to a member of the recipe list.
1345  return &VPBasicBlock::Recipes;
1346  }
1347 
1348  /// Method to support type inquiry through isa, cast, and dyn_cast.
1349  static inline bool classof(const VPBlockBase *V) {
1350  return V->getVPBlockID() == VPBlockBase::VPBasicBlockSC;
1351  }
1352 
1353  void insert(VPRecipeBase *Recipe, iterator InsertPt) {
1354  assert(Recipe && "No recipe to append.");
1355  assert(!Recipe->Parent && "Recipe already in VPlan");
1356  Recipe->Parent = this;
1357  Recipes.insert(InsertPt, Recipe);
1358  }
1359 
1360  /// Augment the existing recipes of a VPBasicBlock with an additional
1361  /// \p Recipe as the last recipe.
1362  void appendRecipe(VPRecipeBase *Recipe) { insert(Recipe, end()); }
1363 
1364  /// The method which generates the output IR instructions that correspond to
1365  /// this VPBasicBlock, thereby "executing" the VPlan.
1366  void execute(struct VPTransformState *State) override;
1367 
1368  /// Return the position of the first non-phi node recipe in the block.
1370 
1371  void dropAllReferences(VPValue *NewValue) override;
1372 
1373 private:
1374  /// Create an IR BasicBlock to hold the output instructions generated by this
1375  /// VPBasicBlock, and return it. Update the CFGState accordingly.
1376  BasicBlock *createEmptyBasicBlock(VPTransformState::CFGState &CFG);
1377 };
1378 
1379 /// VPRegionBlock represents a collection of VPBasicBlocks and VPRegionBlocks
1380 /// which form a Single-Entry-Single-Exit subgraph of the output IR CFG.
1381 /// A VPRegionBlock may indicate that its contents are to be replicated several
1382 /// times. This is designed to support predicated scalarization, in which a
1383 /// scalar if-then code structure needs to be generated VF * UF times. Having
1384 /// this replication indicator helps to keep a single model for multiple
1385 /// candidate VF's. The actual replication takes place only once the desired VF
1386 /// and UF have been determined.
1387 class VPRegionBlock : public VPBlockBase {
1388  /// Hold the Single Entry of the SESE region modelled by the VPRegionBlock.
1389  VPBlockBase *Entry;
1390 
1391  /// Hold the Single Exit of the SESE region modelled by the VPRegionBlock.
1392  VPBlockBase *Exit;
1393 
1394  /// An indicator whether this region is to generate multiple replicated
1395  /// instances of output IR corresponding to its VPBlockBases.
1396  bool IsReplicator;
1397 
1398 public:
1400  const std::string &Name = "", bool IsReplicator = false)
1401  : VPBlockBase(VPRegionBlockSC, Name), Entry(Entry), Exit(Exit),
1402  IsReplicator(IsReplicator) {
1403  assert(Entry->getPredecessors().empty() && "Entry block has predecessors.");
1404  assert(Exit->getSuccessors().empty() && "Exit block has successors.");
1405  Entry->setParent(this);
1406  Exit->setParent(this);
1407  }
1408  VPRegionBlock(const std::string &Name = "", bool IsReplicator = false)
1409  : VPBlockBase(VPRegionBlockSC, Name), Entry(nullptr), Exit(nullptr),
1410  IsReplicator(IsReplicator) {}
1411 
1412  ~VPRegionBlock() override {
1413  if (Entry) {
1414  VPValue DummyValue;
1415  Entry->dropAllReferences(&DummyValue);
1416  deleteCFG(Entry);
1417  }
1418  }
1419 
1420  /// Method to support type inquiry through isa, cast, and dyn_cast.
1421  static inline bool classof(const VPBlockBase *V) {
1422  return V->getVPBlockID() == VPBlockBase::VPRegionBlockSC;
1423  }
1424 
1425  const VPBlockBase *getEntry() const { return Entry; }
1426  VPBlockBase *getEntry() { return Entry; }
1427 
1428  /// Set \p EntryBlock as the entry VPBlockBase of this VPRegionBlock. \p
1429  /// EntryBlock must have no predecessors.
1430  void setEntry(VPBlockBase *EntryBlock) {
1431  assert(EntryBlock->getPredecessors().empty() &&
1432  "Entry block cannot have predecessors.");
1433  Entry = EntryBlock;
1434  EntryBlock->setParent(this);
1435  }
1436 
1437  // FIXME: DominatorTreeBase is doing 'A->getParent()->front()'. 'front' is a
1438  // specific interface of llvm::Function, instead of using
1439  // GraphTraints::getEntryNode. We should add a new template parameter to
1440  // DominatorTreeBase representing the Graph type.
1441  VPBlockBase &front() const { return *Entry; }
1442 
1443  const VPBlockBase *getExit() const { return Exit; }
1444  VPBlockBase *getExit() { return Exit; }
1445 
1446  /// Set \p ExitBlock as the exit VPBlockBase of this VPRegionBlock. \p
1447  /// ExitBlock must have no successors.
1448  void setExit(VPBlockBase *ExitBlock) {
1449  assert(ExitBlock->getSuccessors().empty() &&
1450  "Exit block cannot have successors.");
1451  Exit = ExitBlock;
1452  ExitBlock->setParent(this);
1453  }
1454 
1455  /// An indicator whether this region is to generate multiple replicated
1456  /// instances of output IR corresponding to its VPBlockBases.
1457  bool isReplicator() const { return IsReplicator; }
1458 
1459  /// The method which generates the output IR instructions that correspond to
1460  /// this VPRegionBlock, thereby "executing" the VPlan.
1461  void execute(struct VPTransformState *State) override;
1462 
1463  void dropAllReferences(VPValue *NewValue) override;
1464 };
1465 
1466 //===----------------------------------------------------------------------===//
1467 // GraphTraits specializations for VPlan Hierarchical Control-Flow Graphs //
1468 //===----------------------------------------------------------------------===//
1469 
1470 // The following set of template specializations implement GraphTraits to treat
1471 // any VPBlockBase as a node in a graph of VPBlockBases. It's important to note
1472 // that VPBlockBase traits don't recurse into VPRegioBlocks, i.e., if the
1473 // VPBlockBase is a VPRegionBlock, this specialization provides access to its
1474 // successors/predecessors but not to the blocks inside the region.
1475 
1476 template <> struct GraphTraits<VPBlockBase *> {
1479 
1480  static NodeRef getEntryNode(NodeRef N) { return N; }
1481 
1483  return N->getSuccessors().begin();
1484  }
1485 
1487  return N->getSuccessors().end();
1488  }
1489 };
1490 
1491 template <> struct GraphTraits<const VPBlockBase *> {
1492  using NodeRef = const VPBlockBase *;
1494 
1495  static NodeRef getEntryNode(NodeRef N) { return N; }
1496 
1498  return N->getSuccessors().begin();
1499  }
1500 
1502  return N->getSuccessors().end();
1503  }
1504 };
1505 
1506 // Inverse order specialization for VPBasicBlocks. Predecessors are used instead
1507 // of successors for the inverse traversal.
1508 template <> struct GraphTraits<Inverse<VPBlockBase *>> {
1511 
1512  static NodeRef getEntryNode(Inverse<NodeRef> B) { return B.Graph; }
1513 
1515  return N->getPredecessors().begin();
1516  }
1517 
1519  return N->getPredecessors().end();
1520  }
1521 };
1522 
1523 // The following set of template specializations implement GraphTraits to
1524 // treat VPRegionBlock as a graph and recurse inside its nodes. It's important
1525 // to note that the blocks inside the VPRegionBlock are treated as VPBlockBases
1526 // (i.e., no dyn_cast is performed, VPBlockBases specialization is used), so
1527 // there won't be automatic recursion into other VPBlockBases that turn to be
1528 // VPRegionBlocks.
1529 
1530 template <>
1534 
1535  static NodeRef getEntryNode(GraphRef N) { return N->getEntry(); }
1536 
1538  return nodes_iterator::begin(N->getEntry());
1539  }
1540 
1542  // df_iterator::end() returns an empty iterator so the node used doesn't
1543  // matter.
1544  return nodes_iterator::end(N);
1545  }
1546 };
1547 
1548 template <>
1551  using GraphRef = const VPRegionBlock *;
1553 
1554  static NodeRef getEntryNode(GraphRef N) { return N->getEntry(); }
1555 
1557  return nodes_iterator::begin(N->getEntry());
1558  }
1559 
1561  // df_iterator::end() returns an empty iterator so the node used doesn't
1562  // matter.
1563  return nodes_iterator::end(N);
1564  }
1565 };
1566 
1567 template <>
1572 
1574  return N.Graph->getExit();
1575  }
1576 
1578  return nodes_iterator::begin(N->getExit());
1579  }
1580 
1582  // df_iterator::end() returns an empty iterator so the node used doesn't
1583  // matter.
1584  return nodes_iterator::end(N);
1585  }
1586 };
1587 
1588 /// VPlan models a candidate for vectorization, encoding various decisions take
1589 /// to produce efficient output IR, including which branches, basic-blocks and
1590 /// output IR instructions to generate, and their cost. VPlan holds a
1591 /// Hierarchical-CFG of VPBasicBlocks and VPRegionBlocks rooted at an Entry
1592 /// VPBlock.
1593 class VPlan {
1594  friend class VPlanPrinter;
1595  friend class VPSlotTracker;
1596 
1597  /// Hold the single entry to the Hierarchical CFG of the VPlan.
1598  VPBlockBase *Entry;
1599 
1600  /// Holds the VFs applicable to this VPlan.
1602 
1603  /// Holds the name of the VPlan, for printing.
1604  std::string Name;
1605 
1606  /// Holds all the external definitions created for this VPlan.
1607  // TODO: Introduce a specific representation for external definitions in
1608  // VPlan. External definitions must be immutable and hold a pointer to its
1609  // underlying IR that will be used to implement its structural comparison
1610  // (operators '==' and '<').
1611  SmallPtrSet<VPValue *, 16> VPExternalDefs;
1612 
1613  /// Represents the backedge taken count of the original loop, for folding
1614  /// the tail.
1615  VPValue *BackedgeTakenCount = nullptr;
1616 
1617  /// Holds a mapping between Values and their corresponding VPValue inside
1618  /// VPlan.
1619  Value2VPValueTy Value2VPValue;
1620 
1621  /// Contains all VPValues that been allocated by addVPValue directly and need
1622  /// to be free when the plan's destructor is called.
1623  SmallVector<VPValue *, 16> VPValuesToFree;
1624 
1625  /// Holds the VPLoopInfo analysis for this VPlan.
1626  VPLoopInfo VPLInfo;
1627 
1628 public:
1629  VPlan(VPBlockBase *Entry = nullptr) : Entry(Entry) {
1630  if (Entry)
1631  Entry->setPlan(this);
1632  }
1633 
1635  if (Entry) {
1636  VPValue DummyValue;
1637  for (VPBlockBase *Block : depth_first(Entry))
1638  Block->dropAllReferences(&DummyValue);
1639 
1640  VPBlockBase::deleteCFG(Entry);
1641  }
1642  for (VPValue *VPV : VPValuesToFree)
1643  delete VPV;
1644  if (BackedgeTakenCount)
1645  delete BackedgeTakenCount;
1646  for (VPValue *Def : VPExternalDefs)
1647  delete Def;
1648  }
1649 
1650  /// Generate the IR code for this VPlan.
1651  void execute(struct VPTransformState *State);
1652 
1653  VPBlockBase *getEntry() { return Entry; }
1654  const VPBlockBase *getEntry() const { return Entry; }
1655 
1657  Entry = Block;
1658  Block->setPlan(this);
1659  return Entry;
1660  }
1661 
1662  /// The backedge taken count of the original loop.
1664  if (!BackedgeTakenCount)
1665  BackedgeTakenCount = new VPValue();
1666  return BackedgeTakenCount;
1667  }
1668 
1669  void addVF(ElementCount VF) { VFs.insert(VF); }
1670 
1671  bool hasVF(ElementCount VF) { return VFs.count(VF); }
1672 
1673  const std::string &getName() const { return Name; }
1674 
1675  void setName(const Twine &newName) { Name = newName.str(); }
1676 
1677  /// Add \p VPVal to the pool of external definitions if it's not already
1678  /// in the pool.
1679  void addExternalDef(VPValue *VPVal) {
1680  VPExternalDefs.insert(VPVal);
1681  }
1682 
1683  void addVPValue(Value *V) {
1684  assert(V && "Trying to add a null Value to VPlan");
1685  assert(!Value2VPValue.count(V) && "Value already exists in VPlan");
1686  VPValue *VPV = new VPValue(V);
1687  Value2VPValue[V] = VPV;
1688  VPValuesToFree.push_back(VPV);
1689  }
1690 
1691  void addVPValue(Value *V, VPValue *VPV) {
1692  assert(V && "Trying to add a null Value to VPlan");
1693  assert(!Value2VPValue.count(V) && "Value already exists in VPlan");
1694  Value2VPValue[V] = VPV;
1695  }
1696 
1698  assert(V && "Trying to get the VPValue of a null Value");
1699  assert(Value2VPValue.count(V) && "Value does not exist in VPlan");
1700  return Value2VPValue[V];
1701  }
1702 
1704  assert(V && "Trying to get or add the VPValue of a null Value");
1705  if (!Value2VPValue.count(V))
1706  addVPValue(V);
1707  return getVPValue(V);
1708  }
1709 
1710  void removeVPValueFor(Value *V) { Value2VPValue.erase(V); }
1711 
1712  /// Return the VPLoopInfo analysis for this VPlan.
1713  VPLoopInfo &getVPLoopInfo() { return VPLInfo; }
1714  const VPLoopInfo &getVPLoopInfo() const { return VPLInfo; }
1715 
1716  /// Dump the plan to stderr (for debugging).
1717  void dump() const;
1718 
1719  /// Returns a range mapping the values the range \p Operands to their
1720  /// corresponding VPValues.
1723  std::function<VPValue *(Value *)> Fn = [this](Value *Op) {
1724  return getOrAddVPValue(Op);
1725  };
1726  return map_range(Operands, Fn);
1727  }
1728 
1729 private:
1730  /// Add to the given dominator tree the header block and every new basic block
1731  /// that was created between it and the latch block, inclusive.
1732  static void updateDominatorTree(DominatorTree *DT, BasicBlock *LoopLatchBB,
1733  BasicBlock *LoopPreHeaderBB,
1734  BasicBlock *LoopExitBB);
1735 };
1736 
1737 /// VPlanPrinter prints a given VPlan to a given output stream. The printing is
1738 /// indented and follows the dot format.
1740  friend inline raw_ostream &operator<<(raw_ostream &OS, const VPlan &Plan);
1741  friend inline raw_ostream &operator<<(raw_ostream &OS,
1742  const struct VPlanIngredient &I);
1743 
1744 private:
1745  raw_ostream &OS;
1746  const VPlan &Plan;
1747  unsigned Depth = 0;
1748  unsigned TabWidth = 2;
1749  std::string Indent;
1750  unsigned BID = 0;
1752 
1754 
1755  VPlanPrinter(raw_ostream &O, const VPlan &P)
1756  : OS(O), Plan(P), SlotTracker(&P) {}
1757 
1758  /// Handle indentation.
1759  void bumpIndent(int b) { Indent = std::string((Depth += b) * TabWidth, ' '); }
1760 
1761  /// Print a given \p Block of the Plan.
1762  void dumpBlock(const VPBlockBase *Block);
1763 
1764  /// Print the information related to the CFG edges going out of a given
1765  /// \p Block, followed by printing the successor blocks themselves.
1766  void dumpEdges(const VPBlockBase *Block);
1767 
1768  /// Print a given \p BasicBlock, including its VPRecipes, followed by printing
1769  /// its successor blocks.
1770  void dumpBasicBlock(const VPBasicBlock *BasicBlock);
1771 
1772  /// Print a given \p Region of the Plan.
1773  void dumpRegion(const VPRegionBlock *Region);
1774 
1775  unsigned getOrCreateBID(const VPBlockBase *Block) {
1776  return BlockID.count(Block) ? BlockID[Block] : BlockID[Block] = BID++;
1777  }
1778 
1779  const Twine getOrCreateName(const VPBlockBase *Block);
1780 
1781  const Twine getUID(const VPBlockBase *Block);
1782 
1783  /// Print the information related to a CFG edge between two VPBlockBases.
1784  void drawEdge(const VPBlockBase *From, const VPBlockBase *To, bool Hidden,
1785  const Twine &Label);
1786 
1787  void dump();
1788 
1789  static void printAsIngredient(raw_ostream &O, const Value *V);
1790 };
1791 
1793  const Value *V;
1794 
1795  VPlanIngredient(const Value *V) : V(V) {}
1796 };
1797 
1799  VPlanPrinter::printAsIngredient(OS, I.V);
1800  return OS;
1801 }
1802 
1803 inline raw_ostream &operator<<(raw_ostream &OS, const VPlan &Plan) {
1804  VPlanPrinter Printer(OS, Plan);
1805  Printer.dump();
1806  return OS;
1807 }
1808 
1809 //===----------------------------------------------------------------------===//
1810 // VPlan Utilities
1811 //===----------------------------------------------------------------------===//
1812 
1813 /// Class that provides utilities for VPBlockBases in VPlan.
1815 public:
1816  VPBlockUtils() = delete;
1817 
1818  /// Insert disconnected VPBlockBase \p NewBlock after \p BlockPtr. Add \p
1819  /// NewBlock as successor of \p BlockPtr and \p BlockPtr as predecessor of \p
1820  /// NewBlock, and propagate \p BlockPtr parent to \p NewBlock. If \p BlockPtr
1821  /// has more than one successor, its conditional bit is propagated to \p
1822  /// NewBlock. \p NewBlock must have neither successors nor predecessors.
1823  static void insertBlockAfter(VPBlockBase *NewBlock, VPBlockBase *BlockPtr) {
1824  assert(NewBlock->getSuccessors().empty() &&
1825  "Can't insert new block with successors.");
1826  // TODO: move successors from BlockPtr to NewBlock when this functionality
1827  // is necessary. For now, setBlockSingleSuccessor will assert if BlockPtr
1828  // already has successors.
1829  BlockPtr->setOneSuccessor(NewBlock);
1830  NewBlock->setPredecessors({BlockPtr});
1831  NewBlock->setParent(BlockPtr->getParent());
1832  }
1833 
1834  /// Insert disconnected VPBlockBases \p IfTrue and \p IfFalse after \p
1835  /// BlockPtr. Add \p IfTrue and \p IfFalse as succesors of \p BlockPtr and \p
1836  /// BlockPtr as predecessor of \p IfTrue and \p IfFalse. Propagate \p BlockPtr
1837  /// parent to \p IfTrue and \p IfFalse. \p Condition is set as the successor
1838  /// selector. \p BlockPtr must have no successors and \p IfTrue and \p IfFalse
1839  /// must have neither successors nor predecessors.
1840  static void insertTwoBlocksAfter(VPBlockBase *IfTrue, VPBlockBase *IfFalse,
1841  VPValue *Condition, VPBlockBase *BlockPtr) {
1842  assert(IfTrue->getSuccessors().empty() &&
1843  "Can't insert IfTrue with successors.");
1844  assert(IfFalse->getSuccessors().empty() &&
1845  "Can't insert IfFalse with successors.");
1846  BlockPtr->setTwoSuccessors(IfTrue, IfFalse, Condition);
1847  IfTrue->setPredecessors({BlockPtr});
1848  IfFalse->setPredecessors({BlockPtr});
1849  IfTrue->setParent(BlockPtr->getParent());
1850  IfFalse->setParent(BlockPtr->getParent());
1851  }
1852 
1853  /// Connect VPBlockBases \p From and \p To bi-directionally. Append \p To to
1854  /// the successors of \p From and \p From to the predecessors of \p To. Both
1855  /// VPBlockBases must have the same parent, which can be null. Both
1856  /// VPBlockBases can be already connected to other VPBlockBases.
1858  assert((From->getParent() == To->getParent()) &&
1859  "Can't connect two block with different parents");
1860  assert(From->getNumSuccessors() < 2 &&
1861  "Blocks can't have more than two successors.");
1862  From->appendSuccessor(To);
1863  To->appendPredecessor(From);
1864  }
1865 
1866  /// Disconnect VPBlockBases \p From and \p To bi-directionally. Remove \p To
1867  /// from the successors of \p From and \p From from the predecessors of \p To.
1869  assert(To && "Successor to disconnect is null.");
1870  From->removeSuccessor(To);
1871  To->removePredecessor(From);
1872  }
1873 
1874  /// Returns true if the edge \p FromBlock -> \p ToBlock is a back-edge.
1875  static bool isBackEdge(const VPBlockBase *FromBlock,
1876  const VPBlockBase *ToBlock, const VPLoopInfo *VPLI) {
1877  assert(FromBlock->getParent() == ToBlock->getParent() &&
1878  FromBlock->getParent() && "Must be in same region");
1879  const VPLoop *FromLoop = VPLI->getLoopFor(FromBlock);
1880  const VPLoop *ToLoop = VPLI->getLoopFor(ToBlock);
1881  if (!FromLoop || !ToLoop || FromLoop != ToLoop)
1882  return false;
1883 
1884  // A back-edge is a branch from the loop latch to its header.
1885  return ToLoop->isLoopLatch(FromBlock) && ToBlock == ToLoop->getHeader();
1886  }
1887 
1888  /// Returns true if \p Block is a loop latch
1889  static bool blockIsLoopLatch(const VPBlockBase *Block,
1890  const VPLoopInfo *VPLInfo) {
1891  if (const VPLoop *ParentVPL = VPLInfo->getLoopFor(Block))
1892  return ParentVPL->isLoopLatch(Block);
1893 
1894  return false;
1895  }
1896 
1897  /// Count and return the number of succesors of \p PredBlock excluding any
1898  /// backedges.
1899  static unsigned countSuccessorsNoBE(VPBlockBase *PredBlock,
1900  VPLoopInfo *VPLI) {
1901  unsigned Count = 0;
1902  for (VPBlockBase *SuccBlock : PredBlock->getSuccessors()) {
1903  if (!VPBlockUtils::isBackEdge(PredBlock, SuccBlock, VPLI))
1904  Count++;
1905  }
1906  return Count;
1907  }
1908 };
1909 
1912  InterleaveGroupMap;
1913 
1914  /// Type for mapping of instruction based interleave groups to VPInstruction
1915  /// interleave groups
1918 
1919  /// Recursively \p Region and populate VPlan based interleave groups based on
1920  /// \p IAI.
1921  void visitRegion(VPRegionBlock *Region, Old2NewTy &Old2New,
1922  InterleavedAccessInfo &IAI);
1923  /// Recursively traverse \p Block and populate VPlan based interleave groups
1924  /// based on \p IAI.
1925  void visitBlock(VPBlockBase *Block, Old2NewTy &Old2New,
1926  InterleavedAccessInfo &IAI);
1927 
1928 public:
1930 
1933  // Avoid releasing a pointer twice.
1934  for (auto &I : InterleaveGroupMap)
1935  DelSet.insert(I.second);
1936  for (auto *Ptr : DelSet)
1937  delete Ptr;
1938  }
1939 
1940  /// Get the interleave group that \p Instr belongs to.
1941  ///
1942  /// \returns nullptr if doesn't have such group.
1945  return InterleaveGroupMap.lookup(Instr);
1946  }
1947 };
1948 
1949 /// Class that maps (parts of) an existing VPlan to trees of combined
1950 /// VPInstructions.
1951 class VPlanSlp {
1952  enum class OpMode { Failed, Load, Opcode };
1953 
1954  /// A DenseMapInfo implementation for using SmallVector<VPValue *, 4> as
1955  /// DenseMap keys.
1956  struct BundleDenseMapInfo {
1957  static SmallVector<VPValue *, 4> getEmptyKey() {
1958  return {reinterpret_cast<VPValue *>(-1)};
1959  }
1960 
1961  static SmallVector<VPValue *, 4> getTombstoneKey() {
1962  return {reinterpret_cast<VPValue *>(-2)};
1963  }
1964 
1965  static unsigned getHashValue(const SmallVector<VPValue *, 4> &V) {
1966  return static_cast<unsigned>(hash_combine_range(V.begin(), V.end()));
1967  }
1968 
1969  static bool isEqual(const SmallVector<VPValue *, 4> &LHS,
1970  const SmallVector<VPValue *, 4> &RHS) {
1971  return LHS == RHS;
1972  }
1973  };
1974 
1975  /// Mapping of values in the original VPlan to a combined VPInstruction.
1976  DenseMap<SmallVector<VPValue *, 4>, VPInstruction *, BundleDenseMapInfo>
1977  BundleToCombined;
1978 
1980 
1981  /// Basic block to operate on. For now, only instructions in a single BB are
1982  /// considered.
1983  const VPBasicBlock &BB;
1984 
1985  /// Indicates whether we managed to combine all visited instructions or not.
1986  bool CompletelySLP = true;
1987 
1988  /// Width of the widest combined bundle in bits.
1989  unsigned WidestBundleBits = 0;
1990 
1991  using MultiNodeOpTy =
1992  typename std::pair<VPInstruction *, SmallVector<VPValue *, 4>>;
1993 
1994  // Input operand bundles for the current multi node. Each multi node operand
1995  // bundle contains values not matching the multi node's opcode. They will
1996  // be reordered in reorderMultiNodeOps, once we completed building a
1997  // multi node.
1998  SmallVector<MultiNodeOpTy, 4> MultiNodeOps;
1999 
2000  /// Indicates whether we are building a multi node currently.
2001  bool MultiNodeActive = false;
2002 
2003  /// Check if we can vectorize Operands together.
2004  bool areVectorizable(ArrayRef<VPValue *> Operands) const;
2005 
2006  /// Add combined instruction \p New for the bundle \p Operands.
2007  void addCombined(ArrayRef<VPValue *> Operands, VPInstruction *New);
2008 
2009  /// Indicate we hit a bundle we failed to combine. Returns nullptr for now.
2010  VPInstruction *markFailed();
2011 
2012  /// Reorder operands in the multi node to maximize sequential memory access
2013  /// and commutative operations.
2014  SmallVector<MultiNodeOpTy, 4> reorderMultiNodeOps();
2015 
2016  /// Choose the best candidate to use for the lane after \p Last. The set of
2017  /// candidates to choose from are values with an opcode matching \p Last's
2018  /// or loads consecutive to \p Last.
2019  std::pair<OpMode, VPValue *> getBest(OpMode Mode, VPValue *Last,
2020  SmallPtrSetImpl<VPValue *> &Candidates,
2022 
2023  /// Print bundle \p Values to dbgs().
2024  void dumpBundle(ArrayRef<VPValue *> Values);
2025 
2026 public:
2028 
2029  ~VPlanSlp() = default;
2030 
2031  /// Tries to build an SLP tree rooted at \p Operands and returns a
2032  /// VPInstruction combining \p Operands, if they can be combined.
2034 
2035  /// Return the width of the widest combined bundle in bits.
2036  unsigned getWidestBundleBits() const { return WidestBundleBits; }
2037 
2038  /// Return true if all visited instruction can be combined.
2039  bool isCompletelySLP() const { return CompletelySLP; }
2040 };
2041 } // end namespace llvm
2042 
2043 #endif // LLVM_TRANSFORMS_VECTORIZE_VPLAN_H
llvm::VPBasicBlock::getRecipeList
RecipeListTy & getRecipeList()
Returns a reference to the list of recipes.
Definition: VPlan.h:1341
llvm::VPRegionBlock::front
VPBlockBase & front() const
Definition: VPlan.h:1441
llvm::VPInstruction::getOpcode
unsigned getOpcode() const
Definition: VPlan.h:656
i
i
Definition: README.txt:29
llvm::VPWidenGEPRecipe::~VPWidenGEPRecipe
~VPWidenGEPRecipe() override=default
llvm::VFRange::VFRange
VFRange(const ElementCount &Start, const ElementCount &End)
Definition: VPlan.h:77
llvm::VPRecipeBase::moveAfter
void moveAfter(VPRecipeBase *MovePos)
Unlink this recipe from its current VPBasicBlock and insert it into the VPBasicBlock that MovePos liv...
Definition: VPlan.cpp:470
llvm::VPBlockBase::getSinglePredecessor
VPBlockBase * getSinglePredecessor() const
Definition: VPlan.h:378
llvm::iplist_impl::clear
void clear()
Definition: ilist.h:309
llvm::VPReductionRecipe::print
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
Definition: VPlan.cpp:959
llvm::VPWidenIntOrFpInductionRecipe::getCastValue
VPValue * getCastValue()
Returns the cast VPValue, if one is attached, or nullptr otherwise.
Definition: VPlan.h:854
llvm::VPBlockBase::getPlan
VPlan * getPlan()
Definition: VPlan.cpp:118
llvm::VPWidenMemoryInstructionRecipe::getMask
VPValue * getMask() const
Return the mask used by this recipe.
Definition: VPlan.h:1248
llvm::VPBlendRecipe::getNumIncomingValues
unsigned getNumIncomingValues() const
Return the number of incoming values, taking into account that a single incoming value has no mask.
Definition: VPlan.h:955
llvm::VPlan::execute
void execute(struct VPTransformState *State)
Generate the IR code for this VPlan.
Definition: VPlan.cpp:586
llvm::Loop::isLoopInvariant
bool isLoopInvariant(const Value *V) const
Return true if the specified value is loop invariant.
Definition: LoopInfo.cpp:63
llvm::VPlanSlp
Class that maps (parts of) an existing VPlan to trees of combined VPInstructions.
Definition: VPlan.h:1951
llvm::GraphTraits< const VPBlockBase * >::getEntryNode
static NodeRef getEntryNode(NodeRef N)
Definition: VPlan.h:1495
llvm::VPWidenSelectRecipe::~VPWidenSelectRecipe
~VPWidenSelectRecipe() override=default
llvm::VPWidenSelectRecipe::classof
static bool classof(const VPDef *D)
Method to support type inquiry through isa, cast, and dyn_cast.
Definition: VPlan.h:768
llvm::VPIteration::VPIteration
VPIteration(unsigned Part, unsigned Lane)
Definition: VPlan.h:101
llvm::VPRegionBlock::VPRegionBlock
VPRegionBlock(VPBlockBase *Entry, VPBlockBase *Exit, const std::string &Name="", bool IsReplicator=false)
Definition: VPlan.h:1399
llvm::VPBlendRecipe::getIncomingValue
VPValue * getIncomingValue(unsigned Idx) const
Return incoming value number Idx.
Definition: VPlan.h:958
llvm
This class represents lattice values for constants.
Definition: AllocatorList.h:23
llvm::SmallVectorImpl::erase
iterator erase(const_iterator CI)
Definition: SmallVector.h:705
llvm::tgtok::Def
@ Def
Definition: TGLexer.h:50
llvm::VPRecipeBase::getUnderlyingInstr
Instruction * getUnderlyingInstr()
Returns the underlying instruction, if the recipe is a VPValue or nullptr otherwise.
Definition: VPlan.h:570
llvm::VPWidenCallRecipe::~VPWidenCallRecipe
~VPWidenCallRecipe() override=default
llvm::VPLoop
Hold analysis information for every loop detected by VPLoopInfo.
Definition: VPlanLoopInfo.h:27
llvm::GraphTraits< VPBlockBase * >::getEntryNode
static NodeRef getEntryNode(NodeRef N)
Definition: VPlan.h:1480
llvm::VPlanIngredient::VPlanIngredient
VPlanIngredient(const Value *V)
Definition: VPlan.h:1795
llvm::VPBlockBase::dropAllReferences
virtual void dropAllReferences(VPValue *NewValue)=0
Replace all operands of VPUsers in the block with NewValue and also replaces all uses of VPValues def...
llvm::VPWidenMemoryInstructionRecipe::VPWidenMemoryInstructionRecipe
VPWidenMemoryInstructionRecipe(StoreInst &Store, VPValue *Addr, VPValue *StoredValue, VPValue *Mask)
Definition: VPlan.h:1229
llvm::VPInstruction::SLPStore
@ SLPStore
Definition: VPlan.h:611
llvm::VPTransformState::CFG
struct llvm::VPTransformState::CFGState CFG
llvm::VPRecipeBase::eraseFromParent
iplist< VPRecipeBase >::iterator eraseFromParent()
This method unlinks 'this' from the containing basic block and deletes it.
Definition: VPlan.cpp:465
Optional.h
llvm::VPValue::VPVMemoryInstructionSC
@ VPVMemoryInstructionSC
Definition: VPlanValue.h:95
llvm::VPWidenCanonicalIVRecipe::execute
void execute(VPTransformState &State) override
Generate a canonical vector induction variable of the vector loop, with start = {<Part*VF,...
Definition: VPlan.cpp:1011
llvm::VPPredInstPHIRecipe::execute
void execute(VPTransformState &State) override
Generates phi nodes for live-outs as needed to retain SSA form.
Definition: LoopVectorize.cpp:9041
llvm::VPDef::VPValue
friend class VPValue
Definition: VPlanValue.h:265
llvm::VPBlendRecipe::print
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
Definition: VPlan.cpp:939
llvm::VPBlockBase::getEnclosingBlockWithPredecessors
VPBlockBase * getEnclosingBlockWithPredecessors()
Definition: VPlan.cpp:166
llvm::VPInstruction::ActiveLaneMask
@ ActiveLaneMask
Definition: VPlan.h:612
llvm::VPBasicBlock::back
const VPRecipeBase & back() const
Definition: VPlan.h:1337
llvm::VPBlockBase::isLegalToHoistInto
bool isLegalToHoistInto()
Return true if it is legal to hoist instructions into this block.
Definition: VPlan.h:499
llvm::VPBlockBase::getName
const std::string & getName() const
Definition: VPlan.h:330
llvm::ElementCount
Definition: TypeSize.h:382
llvm::VPBasicBlock
VPBasicBlock serves as the leaf of the Hierarchical Control-Flow Graph.
Definition: VPlan.h:1297
llvm::Loop
Represents a single loop in the control flow graph.
Definition: LoopInfo.h:529
llvm::DenseMapBase< DenseMap< KeyT, ValueT, DenseMapInfo< KeyT >, llvm::detail::DenseMapPair< KeyT, ValueT > >, KeyT, ValueT, DenseMapInfo< KeyT >, llvm::detail::DenseMapPair< KeyT, ValueT > >::lookup
ValueT lookup(const_arg_type_t< KeyT > Val) const
lookup - Return the entry for the specified key, or a default constructed value if no such entry exis...
Definition: DenseMap.h:197
llvm::VPValue::VPVWidenCallSC
@ VPVWidenCallSC
Definition: VPlanValue.h:100
P
This currently compiles esp xmm0 movsd esp eax eax esp ret We should use not the dag combiner This is because dagcombine2 needs to be able to see through the X86ISD::Wrapper which DAGCombine can t really do The code for turning x load into a single vector load is target independent and should be moved to the dag combiner The code for turning x load into a vector load can only handle a direct load from a global or a direct load from the stack It should be generalized to handle any load from P
Definition: README-SSE.txt:411
llvm::VPWidenPHIRecipe
A recipe for handling all phi nodes except for integer and FP inductions.
Definition: VPlan.h:875
llvm::VPBlockBase::getExitBasicBlock
const VPBasicBlock * getExitBasicBlock() const
Definition: VPlan.cpp:144
llvm::VPWidenIntOrFpInductionRecipe::print
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
Definition: VPlan.cpp:908
llvm::VPBlockBase::print
void print(raw_ostream &OS) const
Definition: VPlan.h:492
llvm::VPPredInstPHIRecipe::classof
static bool classof(const VPDef *D)
Method to support type inquiry through isa, cast, and dyn_cast.
Definition: VPlan.h:1191
llvm::SmallVector
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
Definition: SmallVector.h:1168
llvm::VPWidenIntOrFpInductionRecipe::~VPWidenIntOrFpInductionRecipe
~VPWidenIntOrFpInductionRecipe() override=default
llvm::VPInterleaveRecipe::print
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
Definition: LoopVectorize.cpp:8858
ilist.h
llvm::VPBasicBlock::end
const_iterator end() const
Definition: VPlan.h:1326
llvm::VPTransformState::get
Value * get(VPValue *Def, unsigned Part)
Get the generated Value for a given VPValue and a given Part.
Definition: LoopVectorize.cpp:9144
llvm::enumerate
detail::enumerator< R > enumerate(R &&TheRange)
Given an input range, returns a new range whose values are are pair (A,B) such that A is the 0-based ...
Definition: STLExtras.h:1916
llvm::VPWidenPHIRecipe::VPWidenPHIRecipe
VPWidenPHIRecipe(PHINode *Phi, RecurrenceDescriptor &RdxDesc, VPValue &Start)
Create a new VPWidenPHIRecipe for the reduction Phi described by RdxDesc.
Definition: VPlan.h:885
llvm::VPRegionBlock::dropAllReferences
void dropAllReferences(VPValue *NewValue) override
Replace all operands of VPUsers in the block with NewValue and also replaces all uses of VPValues def...
Definition: VPlan.cpp:387
llvm::ilist_node_with_parent
An ilist node that can access its parent list.
Definition: ilist_node.h:256
llvm::VPInterleavedAccessInfo
Definition: VPlan.h:1910
llvm::VPBranchOnMaskRecipe::execute
void execute(VPTransformState &State) override
Generate the extraction of the appropriate bit from the block mask and the conditional branch.
Definition: LoopVectorize.cpp:9015
llvm::VPBlockBase::VPBlockBase
VPBlockBase(const unsigned char SC, const std::string &N)
Definition: VPlan.h:316
llvm::GraphTraits< VPRegionBlock * >::nodes_begin
static nodes_iterator nodes_begin(GraphRef N)
Definition: VPlan.h:1537
llvm::TargetTransformInfo
This pass provides access to the codegen interfaces that are needed for IR-level transformations.
Definition: TargetTransformInfo.h:167
llvm::VPInterleaveRecipe::VPInterleaveRecipe
VPInterleaveRecipe(const InterleaveGroup< Instruction > *IG, VPValue *Addr, ArrayRef< VPValue * > StoredValues, VPValue *Mask)
Definition: VPlan.h:981
llvm::IRBuilder<>
llvm::GraphTraits< const VPBlockBase * >::child_end
static ChildIteratorType child_end(NodeRef N)
Definition: VPlan.h:1501
llvm::SmallDenseMap
Definition: DenseMap.h:880
llvm::SPII::Store
@ Store
Definition: SparcInstrInfo.h:33
llvm::VPTransformState::set
void set(VPValue *Def, Value *V, unsigned Part)
Set the generated Value for a given VPValue and a given Part.
Definition: VPlan.h:166
llvm::DominatorTree
Concrete subclass of DominatorTreeBase that is used to compute a normal dominator tree.
Definition: Dominators.h:151
llvm::VPTransformState::set
void set(VPValue *Def, Value *V, const VPIteration &Instance)
Set the generated scalar V for Def and the given Instance.
Definition: VPlan.h:182
llvm::iplist_impl< simple_ilist< VPRecipeBase, Options... >, ilist_traits< VPRecipeBase > >::const_reverse_iterator
base_list_type::const_reverse_iterator const_reverse_iterator
Definition: ilist.h:181
llvm::VPInterleaveRecipe::~VPInterleaveRecipe
~VPInterleaveRecipe() override=default
llvm::VPRecipeBase::getUnderlyingInstr
const Instruction * getUnderlyingInstr() const
Definition: VPlan.h:573
llvm::DenseMapBase::erase
bool erase(const KeyT &Val)
Definition: DenseMap.h:302
llvm::VPBlendRecipe::execute
void execute(VPTransformState &State) override
Generate the phi/select nodes.
Definition: LoopVectorize.cpp:8906
llvm::Depth
@ Depth
Definition: SIMachineScheduler.h:34
llvm::VPBlockBase::~VPBlockBase
virtual ~VPBlockBase()=default
llvm::VPPredInstPHIRecipe
VPPredInstPHIRecipe is a recipe for generating the phi nodes needed when control converges back from ...
Definition: VPlan.h:1181
llvm::VPBasicBlock::dropAllReferences
void dropAllReferences(VPValue *NewValue) override
Replace all operands of VPUsers in the block with NewValue and also replaces all uses of VPValues def...
Definition: VPlan.cpp:377
DenseMap.h
llvm::VPWidenIntOrFpInductionRecipe::getTruncInst
TruncInst * getTruncInst()
Returns the first defined value as TruncInst, if it is one or nullptr otherwise.
Definition: VPlan.h:862
llvm::VPIteration::isFirstIteration
bool isFirstIteration() const
Definition: VPlan.h:103
llvm::VPlanPtr
std::unique_ptr< VPlan > VPlanPtr
Definition: VPlan.h:86
llvm::sys::path::end
const_iterator end(StringRef path)
Get end iterator over path.
Definition: Path.cpp:233
llvm::sys::path::begin
const_iterator begin(StringRef path, Style style=Style::native)
Get begin iterator over path.
Definition: Path.cpp:224
llvm::GraphTraits< Inverse< VPRegionBlock * > >::getEntryNode
static NodeRef getEntryNode(Inverse< GraphRef > N)
Definition: VPlan.h:1573
llvm::VPBlockBase::getHierarchicalPredecessors
const VPBlocksTy & getHierarchicalPredecessors()
Definition: VPlan.h:418
llvm::VPInterleaveRecipe::classof
static bool classof(const VPDef *D)
Method to support type inquiry through isa, cast, and dyn_cast.
Definition: VPlan.h:1001
llvm::VPBlockUtils
Class that provides utilities for VPBlockBases in VPlan.
Definition: VPlan.h:1814
llvm::Optional
Definition: APInt.h:33
llvm::VPUser::op_begin
operand_iterator op_begin()
Definition: VPlanValue.h:246
llvm::DenseMapBase::count
size_type count(const_arg_type_t< KeyT > Val) const
Return 1 if the specified key is in the map, 0 otherwise.
Definition: DenseMap.h:145
llvm::VPlan::getEntry
const VPBlockBase * getEntry() const
Definition: VPlan.h:1654
llvm::VPValue::VPVPredInstPHI
@ VPVPredInstPHI
Definition: VPlanValue.h:96
llvm::VPWidenMemoryInstructionRecipe::print
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
Definition: VPlan.cpp:998
llvm::VPWidenPHIRecipe::~VPWidenPHIRecipe
~VPWidenPHIRecipe() override=default
llvm::mapped_iterator
Definition: STLExtras.h:286
llvm::SmallPtrSet
SmallPtrSet - This class implements a set which is optimized for holding SmallSize or less elements.
Definition: SmallPtrSet.h:449
llvm::VPReplicateRecipe::classof
static bool classof(const VPDef *D)
Method to support type inquiry through isa, cast, and dyn_cast.
Definition: VPlan.h:1114
llvm::VPWidenCallRecipe
A recipe for widening Call instructions.
Definition: VPlan.h:728
llvm::VPIteration::Lane
unsigned Lane
in [0..VF)
Definition: VPlan.h:99
llvm::MipsISD::Ret
@ Ret
Definition: MipsISelLowering.h:116
llvm::LoopBase::isLoopLatch
bool isLoopLatch(const BlockT *BB) const
Definition: LoopInfo.h:239
llvm::VPUser::operands
operand_range operands()
Definition: VPlanValue.h:250
llvm::isPowerOf2_32
constexpr bool isPowerOf2_32(uint32_t Value)
Return true if the argument is a power of two > 0.
Definition: MathExtras.h:492
llvm::VPRegionBlock::getEntry
const VPBlockBase * getEntry() const
Definition: VPlan.h:1425
llvm::VPInstruction::Not
@ Not
Definition: VPlan.h:608
llvm::VPValue::VPVWidenGEPSC
@ VPVWidenGEPSC
Definition: VPlanValue.h:101
llvm::VPTransformState::hasScalarValue
bool hasScalarValue(VPValue *Def, VPIteration Instance)
Definition: VPlan.h:156
llvm::BitmaskEnumDetail::Mask
std::underlying_type_t< E > Mask()
Get a bitmask with 1s in all places up to the high-order bit of E's largest value.
Definition: BitmaskEnum.h:80
llvm::VPInstruction::mayWriteToMemory
bool mayWriteToMemory() const
Return true if this instruction may modify memory.
Definition: VPlan.h:671
VPlanValue.h
llvm::VPReplicateRecipe::isPacked
bool isPacked() const
Definition: VPlan.h:1135
llvm::VPlanPrinter::operator<<
friend raw_ostream & operator<<(raw_ostream &OS, const VPlan &Plan)
Definition: VPlan.h:1803
llvm::InterleaveGroup
The group of interleaved loads/stores sharing the same stride and close to each other.
Definition: VectorUtils.h:292
llvm::LinearPolySize::isScalable
bool isScalable() const
Returns whether the size is scaled by a runtime quantity (vscale).
Definition: TypeSize.h:295
llvm::iplist
An intrusive list with ownership and callbacks specified/controlled by ilist_traits,...
Definition: ilist.h:390
llvm::VPReductionRecipe
A recipe to represent inloop reduction operations, performing a reduction on a vector operand into a ...
Definition: VPlan.h:1039
llvm::isEqual
bool isEqual(const GCNRPTracker::LiveRegSet &S1, const GCNRPTracker::LiveRegSet &S2)
Definition: GCNRegPressure.cpp:55
DepthFirstIterator.h
llvm::VPWidenPHIRecipe::execute
void execute(VPTransformState &State) override
Generate the phi/select nodes.
Definition: LoopVectorize.cpp:8901
llvm::VPBlockBase::execute
virtual void execute(struct VPTransformState *State)=0
The method which generates the output IR that correspond to this VPBlockBase, thereby "executing" the...
llvm::VPWidenIntOrFpInductionRecipe::VPWidenIntOrFpInductionRecipe
VPWidenIntOrFpInductionRecipe(PHINode *IV, VPValue *Start, Instruction *Cast, TruncInst *Trunc=nullptr)
Definition: VPlan.h:824
llvm::VPBlockBase::setPredicate
void setPredicate(VPValue *Pred)
Set the block's predicate.
Definition: VPlan.cpp:222
llvm::VPInterleaveRecipe::getInterleaveGroup
const InterleaveGroup< Instruction > * getInterleaveGroup()
Definition: VPlan.h:1033
llvm::RISCVFenceField::R
@ R
Definition: RISCVBaseInfo.h:129
llvm::VFRange
A range of powers-of-2 vectorization factors with fixed start and adjustable end.
Definition: VPlan.h:66
llvm::BasicBlock
LLVM Basic Block Representation.
Definition: BasicBlock.h:58
llvm::VPValue::VPVInstructionSC
@ VPVInstructionSC
Definition: VPlanValue.h:94
llvm::VPBranchOnMaskRecipe::classof
static bool classof(const VPDef *D)
Method to support type inquiry through isa, cast, and dyn_cast.
Definition: VPlan.h:1148
llvm::VPDef
This class augments a recipe with a set of VPValues defined by the recipe.
Definition: VPlanValue.h:264
GraphTraits.h
llvm::VPTransformState::DataState::PerPartOutput
DenseMap< VPValue *, PerPartValuesTy > PerPartOutput
Definition: VPlan.h:130
llvm::VPInterleaveRecipe::execute
void execute(VPTransformState &State) override
Generate the wide load or store, and shuffles.
Definition: LoopVectorize.cpp:8945
llvm::SmallBitVector
This is a 'bitvector' (really, a variable-sized bit array), optimized for the case when the array is ...
Definition: SmallBitVector.h:34
llvm::VPPredInstPHIRecipe::print
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
Definition: VPlan.cpp:990
llvm::VPReplicateRecipe
VPReplicateRecipe replicates a given instruction producing multiple scalar copies of the original sca...
Definition: VPlan.h:1087
llvm::VPTransformState
VPTransformState holds information passed down when "executing" a VPlan, needed for generating the ou...
Definition: VPlan.h:108
Printer
print alias Alias Set Printer
Definition: AliasSetTracker.cpp:770
llvm::VPBasicBlock::iterator
RecipeListTy::iterator iterator
Instruction iterators...
Definition: VPlan.h:1315
llvm::VPBlockBase
VPBlockBase is the building block of the Hierarchical Control-Flow Graph.
Definition: VPlan.h:257
llvm::VPReplicateRecipe::print
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
Definition: VPlan.cpp:975
llvm::VPBasicBlock::appendRecipe
void appendRecipe(VPRecipeBase *Recipe)
Augment the existing recipes of a VPBasicBlock with an additional Recipe as the last recipe.
Definition: VPlan.h:1362
llvm::VPlan::hasVF
bool hasVF(ElementCount VF)
Definition: VPlan.h:1671
llvm::VPWidenPHIRecipe::classof
static bool classof(const VPValue *V)
Definition: VPlan.h:901
llvm::VPTransformState::DataState::PerPartValuesTy
SmallVector< Value *, 2 > PerPartValuesTy
A type for vectorized values in the new loop.
Definition: VPlan.h:128
llvm::VPInstruction::classof
static bool classof(const VPValue *V)
Method to support type inquiry through isa, cast, and dyn_cast.
Definition: VPlan.h:642
llvm::VPWidenPHIRecipe::getIncomingBlock
VPBasicBlock * getIncomingBlock(unsigned I)
Returns the I th incoming VPBasicBlock.
Definition: VPlan.h:927
llvm::VPInstruction
This is a concrete Recipe that models a single VPlan-level instruction.
Definition: VPlan.h:602
llvm::VPTransformState::ILV
InnerLoopVectorizer * ILV
Hold a pointer to InnerLoopVectorizer to reuse its IR generation methods.
Definition: VPlan.h:249
llvm::VFRange::isEmpty
bool isEmpty() const
Definition: VPlan.h:73
llvm::PPCISD::SC
@ SC
CHAIN = SC CHAIN, Imm128 - System call.
Definition: PPCISelLowering.h:401
llvm::VPValue::VPVReplicateSC
@ VPVReplicateSC
Definition: VPlanValue.h:98
llvm::Failed
testing::Matcher< const detail::ErrorHolder & > Failed()
Definition: Error.h:177
llvm::VPBlockBase::setPredecessors
void setPredecessors(ArrayRef< VPBlockBase * > NewPreds)
Set each VPBasicBlock in NewPreds as predecessor of this VPBlockBase.
Definition: VPlan.h:466
llvm::VPInterleavedAccessInfo::getInterleaveGroup
InterleaveGroup< VPInstruction > * getInterleaveGroup(VPInstruction *Instr) const
Get the interleave group that Instr belongs to.
Definition: VPlan.h:1944
llvm::VPTransformState::LI
LoopInfo * LI
Hold a pointer to LoopInfo to register new basic blocks in the loop.
Definition: VPlan.h:232
llvm::VPlanSlp::buildGraph
VPInstruction * buildGraph(ArrayRef< VPValue * > Operands)
Tries to build an SLP tree rooted at Operands and returns a VPInstruction combining Operands,...
Definition: VPlanSLP.cpp:365
llvm::VPReplicateRecipe::classof
static bool classof(const VPValue *V)
Definition: VPlan.h:1118
Twine.h
llvm::VPBlockBase::getSuccessors
VPBlocksTy & getSuccessors()
Definition: VPlan.h:365
llvm::VPBlockUtils::insertBlockAfter
static void insertBlockAfter(VPBlockBase *NewBlock, VPBlockBase *BlockPtr)
Insert disconnected VPBlockBase NewBlock after BlockPtr.
Definition: VPlan.h:1823
llvm::VPBasicBlock::getSublistAccess
static RecipeListTy VPBasicBlock::* getSublistAccess(VPRecipeBase *)
Returns a pointer to a member of the recipe list.
Definition: VPlan.h:1344
llvm::VPlan::getVPValue
VPValue * getVPValue(Value *V)
Definition: VPlan.h:1697
llvm::GraphTraits< const VPBlockBase * >::ChildIteratorType
SmallVectorImpl< VPBlockBase * >::const_iterator ChildIteratorType
Definition: VPlan.h:1493
llvm::GraphTraits< Inverse< VPBlockBase * > >::child_end
static ChildIteratorType child_end(NodeRef N)
Definition: VPlan.h:1518
b
the resulting code requires compare and branches when and if the revised code is with conditional branches instead of More there is a byte word extend before each where there should be only and the condition codes are not remembered when the same two values are compared twice More LSR enhancements i8 and i32 load store addressing modes are identical int b
Definition: README.txt:418
llvm::VPRecipeBase::moveBefore
void moveBefore(VPBasicBlock &BB, iplist< VPRecipeBase >::iterator I)
Unlink this recipe and insert into BB before I.
Definition: VPlan.cpp:475
llvm::VPBlockBase::getPredecessors
VPBlocksTy & getPredecessors()
Definition: VPlan.h:368
llvm::VPInterleaveRecipe
VPInterleaveRecipe is a recipe for transforming an interleave group of load or stores into one wide l...
Definition: VPlan.h:975
llvm::VPBranchOnMaskRecipe::print
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
Definition: VPlan.h:1157
llvm::VPBlockBase::VPBlockTy
enum { VPBasicBlockSC, VPRegionBlockSC } VPBlockTy
An enumeration for keeping track of the concrete subclass of VPBlockBase that are actually instantiat...
Definition: VPlan.h:324
false
Definition: StackSlotColoring.cpp:142
llvm::VPValue::VPVReductionSC
@ VPVReductionSC
Definition: VPlanValue.h:97
llvm::VPBlockBase::getEnclosingBlockWithSuccessors
VPBlockBase * getEnclosingBlockWithSuccessors()
An Enclosing Block of a block B is any block containing B, including B itself.
Definition: VPlan.cpp:158
llvm::VPWidenPHIRecipe::getIncomingValue
VPValue * getIncomingValue(unsigned I)
Returns the I th incoming VPValue.
Definition: VPlan.h:924
B
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
llvm::VPWidenCanonicalIVRecipe::print
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
Definition: VPlan.cpp:1036
llvm::GraphTraits< const VPRegionBlock * >::nodes_begin
static nodes_iterator nodes_begin(GraphRef N)
Definition: VPlan.h:1556
llvm::VPTransformState::DataState::PerPartScalars
DenseMap< VPValue *, ScalarsPerPartValuesTy > PerPartScalars
Definition: VPlan.h:133
llvm::Instruction
Definition: Instruction.h:45
llvm::VPReplicateRecipe::~VPReplicateRecipe
~VPReplicateRecipe() override=default
llvm::InterleavedAccessInfo
Drive the analysis of interleaved memory accesses in the loop.
Definition: VectorUtils.h:736
llvm::VPReductionRecipe::execute
void execute(VPTransformState &State) override
Generate the reduction in the loop.
Definition: LoopVectorize.cpp:8951
llvm::VPRegionBlock::execute
void execute(struct VPTransformState *State) override
The method which generates the output IR instructions that correspond to this VPRegionBlock,...
Definition: VPlan.cpp:394
llvm::raw_ostream
This class implements an extremely fast bulk output stream that can only output to a stream.
Definition: raw_ostream.h:50
llvm::VPValue
Definition: VPlanValue.h:44
llvm::VPRegionBlock::~VPRegionBlock
~VPRegionBlock() override
Definition: VPlan.h:1412
llvm::operator<<
raw_ostream & operator<<(raw_ostream &OS, const APFixedPoint &FX)
Definition: APFixedPoint.h:230
llvm::VPWidenCallRecipe::execute
void execute(VPTransformState &State) override
Produce a widened version of the call instruction.
Definition: LoopVectorize.cpp:8874
llvm::SPII::Load
@ Load
Definition: SparcInstrInfo.h:32
llvm::VPTransformState::CFGState::VPBBsToFix
SmallVector< VPBasicBlock *, 8 > VPBBsToFix
Vector of VPBasicBlocks whose terminator instruction needs to be fixed up at the end of vector code g...
Definition: VPlan.h:226
llvm::VPlanSlp::~VPlanSlp
~VPlanSlp()=default
llvm::VPlan::addExternalDef
void addExternalDef(VPValue *VPVal)
Add VPVal to the pool of external definitions if it's not already in the pool.
Definition: VPlan.h:1679
llvm::VPlanSlp::getWidestBundleBits
unsigned getWidestBundleBits() const
Return the width of the widest combined bundle in bits.
Definition: VPlan.h:2036
llvm::VPlan::getName
const std::string & getName() const
Definition: VPlan.h:1673
SmallPtrSet.h
llvm::VPWidenCanonicalIVRecipe
A Recipe for widening the canonical induction variable of the vector loop.
Definition: VPlan.h:1271
llvm::VPDef::getNumDefinedValues
unsigned getNumDefinedValues() const
Returns the number of values defined by the VPDef.
Definition: VPlanValue.h:343
llvm::InnerLoopVectorizer
InnerLoopVectorizer vectorizes loops which contain only one basic block to a specified vectorization ...
Definition: LoopVectorize.cpp:452
llvm::VPBasicBlock::end
iterator end()
Definition: VPlan.h:1325
llvm::MCID::Call
@ Call
Definition: MCInstrDesc.h:154
llvm::InterleaveGroup::getMember
InstTy * getMember(uint32_t Index) const
Get the member with the given index Index.
Definition: VectorUtils.h:662
llvm::VPTransformState::TripCount
Value * TripCount
Hold the trip count of the scalar loop.
Definition: VPlan.h:246
llvm::GraphTraits< const VPBlockBase * >::child_begin
static ChildIteratorType child_begin(NodeRef N)
Definition: VPlan.h:1497
llvm::ArrayRef::slice
ArrayRef< T > slice(size_t N, size_t M) const
slice(n, m) - Chop off the first N elements of the array, and keep M elements in the array.
Definition: ArrayRef.h:193
llvm::VPWidenCanonicalIVRecipe::~VPWidenCanonicalIVRecipe
~VPWidenCanonicalIVRecipe() override=default
llvm::VPUser::getNumOperands
unsigned getNumOperands() const
Definition: VPlanValue.h:224
llvm::iplist_impl< simple_ilist< VPRecipeBase, Options... >, ilist_traits< VPRecipeBase > >::reverse_iterator
base_list_type::reverse_iterator reverse_iterator
Definition: ilist.h:179
llvm::VPBasicBlock::empty
bool empty() const
Definition: VPlan.h:1334
llvm::VPWidenMemoryInstructionRecipe::VPWidenMemoryInstructionRecipe
VPWidenMemoryInstructionRecipe(LoadInst &Load, VPValue *Addr, VPValue *Mask)
Definition: VPlan.h:1223
llvm::VPWidenMemoryInstructionRecipe::getAddr
VPValue * getAddr() const
Return the address accessed by this recipe.
Definition: VPlan.h:1242
llvm::VPRegionBlock
VPRegionBlock represents a collection of VPBasicBlocks and VPRegionBlocks which form a Single-Entry-S...
Definition: VPlan.h:1387
llvm::VPWidenMemoryInstructionRecipe::isStore
bool isStore() const
Returns true if this recipe is a store.
Definition: VPlan.h:1254
llvm::VPBlockBase::getPredecessors
const VPBlocksTy & getPredecessors() const
Definition: VPlan.h:367
llvm::Twine::str
std::string str() const
Return the twine contents as a std::string.
Definition: Twine.cpp:17
Operands
mir Rename Register Operands
Definition: MIRNamerPass.cpp:78
llvm::VPBlockBase::getCondBit
VPValue * getCondBit()
Definition: VPlan.cpp:202
llvm::GraphTraits< Inverse< VPBlockBase * > >::getEntryNode
static NodeRef getEntryNode(Inverse< NodeRef > B)
Definition: VPlan.h:1512
llvm::GraphTraits< VPBlockBase * >::child_begin
static ChildIteratorType child_begin(NodeRef N)
Definition: VPlan.h:1482
llvm::VPInterleaveRecipe::getAddr
VPValue * getAddr() const
Return the address accessed by this recipe.
Definition: VPlan.h:1006
llvm::VPReplicateRecipe::execute
void execute(VPTransformState &State) override
Generate replicas of the desired Ingredient.
Definition: LoopVectorize.cpp:8983
llvm::VPInterleaveRecipe::getStoredValues
ArrayRef< VPValue * > getStoredValues() const
Return the VPValues stored by this interleave group.
Definition: VPlan.h:1019
llvm::VPRecipeBase::VPRecipeBase
VPRecipeBase(const unsigned char SC, ArrayRef< VPValue * > Operands)
Definition: VPlan.h:526
llvm::VPWidenCanonicalIVRecipe::classof
static bool classof(const VPDef *D)
Method to support type inquiry through isa, cast, and dyn_cast.
Definition: VPlan.h:1280
llvm::VPBlockBase::clearPredecessors
void clearPredecessors()
Remove all the predecessor of this block.
Definition: VPlan.h:473
llvm::VPBlockBase::getNumSuccessors
size_t getNumSuccessors() const
Definition: VPlan.h:382
llvm::VPRecipeBase::execute
virtual void execute(struct VPTransformState &State)=0
The method which generates the output IR instructions that correspond to this VPRecipe,...
VectorUtils.h
llvm::VPInterleaveRecipe::getMask
VPValue * getMask() const
Return the mask used by this recipe.
Definition: VPlan.h:1012
llvm::VPTransformState::CanonicalIV
Value * CanonicalIV
Hold the canonical scalar IV of the vector loop (start=0, step=VF*UF).
Definition: VPlan.h:243
llvm::VPRecipeBase::getParent
const VPBasicBlock * getParent() const
Definition: VPlan.h:536
llvm::VPBasicBlock::rbegin
const_reverse_iterator rbegin() const
Definition: VPlan.h:1329
llvm::VPlan::setName
void setName(const Twine &newName)
Definition: VPlan.h:1675
llvm::VPRecipeBase::removeFromParent
void removeFromParent()
This method unlinks 'this' from the containing basic block, but does not delete it.
Definition: VPlan.cpp:459
llvm::VPBlockBase::getHierarchicalSuccessors
const VPBlocksTy & getHierarchicalSuccessors()
Definition: VPlan.h:402
llvm::VPWidenMemoryInstructionRecipe::classof
static bool classof(const VPDef *D)
Method to support type inquiry through isa, cast, and dyn_cast.
Definition: VPlan.h:1237
llvm::VPPredInstPHIRecipe::~VPPredInstPHIRecipe
~VPPredInstPHIRecipe() override=default
llvm::RISCVFenceField::O
@ O
Definition: RISCVBaseInfo.h:128
llvm::VPBranchOnMaskRecipe::VPBranchOnMaskRecipe
VPBranchOnMaskRecipe(VPValue *BlockInMask)
Definition: VPlan.h:1141
llvm::StoreInst
An instruction for storing to memory.
Definition: Instructions.h:303
llvm::VPReductionRecipe::getCondOp
VPValue * getCondOp() const
The VPValue of the condition for the block.
Definition: VPlan.h:1078
llvm::VPTransformState::reset
void reset(VPValue *Def, Value *V, const VPIteration &Instance)
Reset an existing scalar value for Def and a given Instance.
Definition: VPlan.h:195
VPlanLoopInfo.h
Index
uint32_t Index
Definition: ELFObjHandler.cpp:84
llvm::VPBlockBase::clearSuccessors
void clearSuccessors()
Remove all the successors of this block and set to null its condition bit.
Definition: VPlan.h:476
llvm::VPBlockBase::getSingleHierarchicalSuccessor
VPBlockBase * getSingleHierarchicalSuccessor()
Definition: VPlan.h:408
llvm::VPBlockBase::getSingleHierarchicalPredecessor
VPBlockBase * getSingleHierarchicalPredecessor()
Definition: VPlan.h:424
llvm::VPWidenGEPRecipe::execute
void execute(VPTransformState &State) override
Generate the gep nodes.
Definition: LoopVectorize.cpp:8888
D
static GCRegistry::Add< StatepointGC > D("statepoint-example", "an example strategy for statepoint")
llvm::VPRegionBlock::setExit
void setExit(VPBlockBase *ExitBlock)
Set ExitBlock as the exit VPBlockBase of this VPRegionBlock.
Definition: VPlan.h:1448
llvm::VPTransformState::DT
DominatorTree * DT
Hold a pointer to Dominator Tree to register new basic blocks in the loop.
Definition: VPlan.h:235
llvm::VPWidenIntOrFpInductionRecipe::execute
void execute(VPTransformState &State) override
Generate the vectorized and scalarized versions of the phi node as needed by their users.
Definition: LoopVectorize.cpp:8894
llvm::VPWidenRecipe
VPWidenRecipe is a recipe for producing a copy of vector type its ingredient.
Definition: VPlan.h:702
llvm::VPValue::VPVWidenSelectSC
@ VPVWidenSelectSC
Definition: VPlanValue.h:104
llvm::find
auto find(R &&Range, const T &Val)
Provide wrappers to std::find which take ranges instead of having to pass begin/end explicitly.
Definition: STLExtras.h:1518
const
aarch64 promote const
Definition: AArch64PromoteConstant.cpp:232
llvm::VPInstruction::execute
void execute(VPTransformState &State) override
Generate the instruction.
Definition: VPlan.cpp:536
llvm::InterleaveGroup::getFactor
uint32_t getFactor() const
Definition: VectorUtils.h:604
Addr
uint64_t Addr
Definition: ELFObjHandler.cpp:80
llvm::TruncInst
This class represents a truncation of integer types.
Definition: Instructions.h:4711
llvm::iplist_impl< simple_ilist< VPRecipeBase, Options... >, ilist_traits< VPRecipeBase > >::iterator
base_list_type::iterator iterator
Definition: ilist.h:177
llvm::VPReductionRecipe::VPReductionRecipe
VPReductionRecipe(RecurrenceDescriptor *R, Instruction *I, VPValue *ChainOp, VPValue *VecOp, VPValue *CondOp, const TargetTransformInfo *TTI)
Definition: VPlan.h:1046
llvm::VPBlockBase::getVPBlockID
unsigned getVPBlockID() const
Definition: VPlan.h:337
llvm::VPTransformState::UF
unsigned UF
Definition: VPlan.h:117
llvm::DenseMap
Definition: DenseMap.h:714
llvm::VPInstruction::print
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the VPInstruction to O.
Definition: VPlan.cpp:547
llvm::VPTransformState::DataState
Definition: VPlan.h:124
llvm::VPBasicBlock::size
size_t size() const
Definition: VPlan.h:1333
llvm::LoopInfoBase::getLoopFor
LoopT * getLoopFor(const BlockT *BB) const
Return the inner most loop that BB lives in.
Definition: LoopInfo.h:963
I
#define I(x, y, z)
Definition: MD5.cpp:59
llvm::VPIteration
In what follows, the term "input IR" refers to code that is fed into the vectorizer whereas the term ...
Definition: VPlan.h:94
llvm::VPBlockUtils::connectBlocks
static void connectBlocks(VPBlockBase *From, VPBlockBase *To)
Connect VPBlockBases From and To bi-directionally.
Definition: VPlan.h:1857
llvm::GraphTraits< Inverse< VPBlockBase * > >::child_begin
static ChildIteratorType child_begin(NodeRef N)
Definition: VPlan.h:1514
llvm::VPInstruction::VPInstruction
VPInstruction(unsigned Opcode, ArrayRef< VPInstruction * > Operands)
Definition: VPlan.h:631
llvm::GetElementPtrInst
an instruction for type-safe pointer arithmetic to access elements of arrays and structs
Definition: Instructions.h:905
llvm::VPBlendRecipe::getMask
VPValue * getMask(unsigned Idx) const
Return mask number Idx.
Definition: VPlan.h:961
llvm::VPTransformState::CFGState::PrevVPBB
VPBasicBlock * PrevVPBB
The previous VPBasicBlock visited. Initially set to null.
Definition: VPlan.h:210
llvm::VPBlockBase::getNumPredecessors
size_t getNumPredecessors() const
Definition: VPlan.h:383
llvm::VPlan::removeVPValueFor
void removeVPValueFor(Value *V)
Definition: VPlan.h:1710
llvm::VPlan::addVPValue
void addVPValue(Value *V)
Definition: VPlan.h:1683
llvm::VPWidenGEPRecipe::print
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
Definition: VPlan.cpp:920
llvm::VPWidenCanonicalIVRecipe::VPWidenCanonicalIVRecipe
VPWidenCanonicalIVRecipe()
Definition: VPlan.h:1273
llvm::VPWidenMemoryInstructionRecipe::getStoredValue
VPValue * getStoredValue() const
Return the address accessed by this recipe.
Definition: VPlan.h:1257
llvm::VPTransformState::hasAnyVectorValue
bool hasAnyVectorValue(VPValue *Def) const
Definition: VPlan.h:152
llvm::GraphTraits< VPBlockBase * >
Definition: VPlan.h:1476
llvm::VPBasicBlock::VPBasicBlock
VPBasicBlock(const Twine &Name="", VPRecipeBase *Recipe=nullptr)
Definition: VPlan.h:1306
llvm::VPBlockUtils::disconnectBlocks
static void disconnectBlocks(VPBlockBase *From, VPBlockBase *To)
Disconnect VPBlockBases From and To bi-directionally.
Definition: VPlan.h:1868
IRBuilder.h
llvm::VPlan::getEntry
VPBlockBase * getEntry()
Definition: VPlan.h:1653
assert
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
llvm::VPBasicBlock::back
VPRecipeBase & back()
Definition: VPlan.h:1338
llvm::VPWidenIntOrFpInductionRecipe
A recipe for handling phi nodes of integer and floating-point inductions, producing their vector and ...
Definition: VPlan.h:820
llvm::GraphTraits< VPRegionBlock * >::getEntryNode
static NodeRef getEntryNode(GraphRef N)
Definition: VPlan.h:1535
llvm::GraphTraits< Inverse< VPRegionBlock * > >::nodes_end
static nodes_iterator nodes_end(GraphRef N)
Definition: VPlan.h:1581
llvm::VPInstruction::VPInstruction
VPInstruction(unsigned Opcode, ArrayRef< VPValue * > Operands)
Definition: VPlan.h:627
llvm::VPReplicateRecipe::isUniform
bool isUniform() const
Definition: VPlan.h:1133
llvm::VPWidenGEPRecipe
A recipe for handling GEP instructions.
Definition: VPlan.h:781
llvm::VPBasicBlock::execute
void execute(struct VPTransformState *State) override
The method which generates the output IR instructions that correspond to this VPBasicBlock,...
Definition: VPlan.cpp:311
llvm::VPInstruction::hasResult
bool hasResult() const
Definition: VPlan.h:678
llvm::SelectInst
This class represents the LLVM 'select' instruction.
Definition: Instructions.h:1725
llvm::ISD::BasicBlock
@ BasicBlock
Various leaf nodes.
Definition: ISDOpcodes.h:64
Mode
SI Whole Quad Mode
Definition: SIWholeQuadMode.cpp:241
function
print Print MemDeps of function
Definition: MemDepPrinter.cpp:83
llvm::VPReplicateRecipe::VPReplicateRecipe
VPReplicateRecipe(Instruction *I, iterator_range< IterT > Operands, bool IsUniform, bool IsPredicated=false)
Definition: VPlan.h:1099
llvm::VPlan::~VPlan
~VPlan()
Definition: VPlan.h:1634
llvm::VPWidenMemoryInstructionRecipe
A Recipe for widening load/store operations.
Definition: VPlan.h:1209
llvm::VPInstruction::clone
VPInstruction * clone() const
Definition: VPlan.h:646
llvm::VPBasicBlock::~VPBasicBlock
~VPBasicBlock() override
Definition: VPlan.h:1312
llvm::LinearPolySize::getKnownMinValue
ScalarTy getKnownMinValue() const
Returns the minimum value this size can represent.
Definition: TypeSize.h:293
llvm::df_iterator
Definition: DepthFirstIterator.h:85
llvm::VPWidenIntOrFpInductionRecipe::classof
static bool classof(const VPDef *D)
Method to support type inquiry through isa, cast, and dyn_cast.
Definition: VPlan.h:838
llvm::VPWidenRecipe::print
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
Definition: VPlan.cpp:900
llvm::VPWidenSelectRecipe
A recipe for widening select instructions.
Definition: VPlan.h:752
llvm::GraphTraits< const VPRegionBlock * >::nodes_end
static nodes_iterator nodes_end(GraphRef N)
Definition: VPlan.h:1560
llvm::VPInstruction::classof
static bool classof(const VPDef *R)
Method to support type inquiry through isa, cast, and dyn_cast.
Definition: VPlan.h:652
llvm::SetVector< T, SmallVector< T, N >, SmallDenseSet< T, N > >::insert
bool insert(const value_type &X)
Insert a new element into the SetVector.
Definition: SetVector.h:141
llvm::VPWidenCallRecipe::print
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
Definition: VPlan.cpp:870
llvm::VPBlockBase::printAsOperand
void printAsOperand(raw_ostream &OS, bool PrintType) const
Definition: VPlan.h:488
llvm::GraphTraits< const VPBlockBase * >
Definition: VPlan.h:1491
llvm::VPBasicBlock::front
VPRecipeBase & front()
Definition: VPlan.h:1336
llvm::VPTransformState::reset
void reset(VPValue *Def, Value *V, unsigned Part)
Reset an existing vector value for Def and a given Part.
Definition: VPlan.h:174
llvm::VPlanIngredient::V
const Value * V
Definition: VPlan.h:1793
llvm::ArrayRef
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
Definition: APInt.h:32
llvm::LoopInfo
Definition: LoopInfo.h:1079
llvm::VPReductionRecipe::getVecOp
VPValue * getVecOp() const
The VPValue of the vector value to be reduced.
Definition: VPlan.h:1076
llvm::VPBlockBase::getSingleSuccessor
VPBlockBase * getSingleSuccessor() const
Definition: VPlan.h:372
this
Analysis the ScalarEvolution expression for r is this
Definition: README.txt:8
llvm::VPUser::addOperand
void addOperand(VPValue *Operand)
Definition: VPlanValue.h:219
llvm::VPWidenCallRecipe::VPWidenCallRecipe
VPWidenCallRecipe(CallInst &I, iterator_range< IterT > CallArguments)
Definition: VPlan.h:732
llvm::VPWidenCallRecipe::classof
static bool classof(const VPDef *D)
Method to support type inquiry through isa, cast, and dyn_cast.
Definition: VPlan.h:739
llvm::VPTransformState::VPTransformState
VPTransformState(ElementCount VF, unsigned UF, LoopInfo *LI, DominatorTree *DT, IRBuilder<> &Builder, InnerLoopVectorizer *ILV, VPlan *Plan)
Definition: VPlan.h:109
llvm::VPRecipeBase::getParent
VPBasicBlock * getParent()
Definition: VPlan.h:535
llvm::VPIteration::Part
unsigned Part
in [0..UF)
Definition: VPlan.h:96
llvm::VPBlockUtils::countSuccessorsNoBE
static unsigned countSuccessorsNoBE(VPBlockBase *PredBlock, VPLoopInfo *VPLI)
Count and return the number of succesors of PredBlock excluding any backedges.
Definition: VPlan.h:1899
llvm::VPBranchOnMaskRecipe
A recipe for generating conditional branches on the bits of a mask.
Definition: VPlan.h:1139
llvm::VPWidenSelectRecipe::execute
void execute(VPTransformState &State) override
Produce a widened version of the select instruction.
Definition: LoopVectorize.cpp:8879
llvm::VPBlockUtils::insertTwoBlocksAfter
static void insertTwoBlocksAfter(VPBlockBase *IfTrue, VPBlockBase *IfFalse, VPValue *Condition, VPBlockBase *BlockPtr)
Insert disconnected VPBlockBases IfTrue and IfFalse after BlockPtr.
Definition: VPlan.h:1840
llvm::VPBlockBase::setOneSuccessor
void setOneSuccessor(VPBlockBase *Successor)
Set a given VPBlockBase Successor as the single successor of this VPBlockBase.
Definition: VPlan.h:445
llvm::VPRegionBlock::getExit
VPBlockBase * getExit()
Definition: VPlan.h:1444
llvm::VPTransformState::VF
ElementCount VF
The chosen Vectorization and Unroll Factors of the loop being vectorized.
Definition: VPlan.h:116
llvm::LoadInst
An instruction for reading from memory.
Definition: Instructions.h:174
llvm::VPValue::VPInstruction
friend class VPInstruction
Definition: VPlanValue.h:47
llvm::VPBlockBase::getEntryBasicBlock
const VPBasicBlock * getEntryBasicBlock() const
Definition: VPlan.cpp:123
llvm::VPRegionBlock::getEntry
VPBlockBase * getEntry()
Definition: VPlan.h:1426
llvm::VPBasicBlock::classof
static bool classof(const VPBlockBase *V)
Method to support type inquiry through isa, cast, and dyn_cast.
Definition: VPlan.h:1349
llvm::VPTransformState::Data
struct llvm::VPTransformState::DataState Data
llvm::VPValue::setUnderlyingValue
void setUnderlyingValue(Value *Val)
Definition: VPlanValue.h:77
llvm::VPReplicateRecipe::setAlsoPack
void setAlsoPack(bool Pack)
Definition: VPlan.h:1127
llvm::depth_first
iterator_range< df_iterator< T > > depth_first(const T &G)
Definition: DepthFirstIterator.h:227
CFG
Common register allocation spilling lr str ldr sxth r3 ldr mla r4 can lr mov lr str ldr sxth r3 mla r4 and then merge mul and lr str ldr sxth r3 mla r4 It also increase the likelihood the store may become dead bb27 Successors according to CFG
Definition: README.txt:39
llvm::VPTransformState::Builder
IRBuilder & Builder
Hold a reference to the IRBuilder used to generate output IR code.
Definition: VPlan.h:238
llvm::VPBlockBase::setCondBit
void setCondBit(VPValue *CV)
Set the condition bit selecting the successor.
Definition: VPlan.cpp:210
llvm::VPlan::addVPValue
void addVPValue(Value *V, VPValue *VPV)
Definition: VPlan.h:1691
llvm::GraphTraits< VPRegionBlock * >::nodes_end
static nodes_iterator nodes_end(GraphRef N)
Definition: VPlan.h:1541
llvm::VPBlockBase::setParent
void setParent(VPRegionBlock *P)
Definition: VPlan.h:350
llvm::VPWidenPHIRecipe::addIncoming
void addIncoming(VPValue *IncomingV, VPBasicBlock *IncomingBlock)
Adds a pair (IncomingV, IncomingBlock) to the phi.
Definition: VPlan.h:918
llvm::VPRecipeBase::VPRecipeBase
VPRecipeBase(const unsigned char SC, iterator_range< IterT > Operands)
Definition: VPlan.h:530
llvm::VPBasicBlock::rend
const_reverse_iterator rend() const
Definition: VPlan.h:1331
llvm::VPlan::mapToVPValues
iterator_range< mapped_iterator< Use *, std::function< VPValue *(Value *)> > > mapToVPValues(User::op_range Operands)
Returns a range mapping the values the range Operands to their corresponding VPValues.
Definition: VPlan.h:1722
llvm::VPUser::classof
static bool classof(const VPDef *Recipe)
Method to support type inquiry through isa, cast, and dyn_cast.
Definition: VPlan.h:584
llvm::VPBlockUtils::isBackEdge
static bool isBackEdge(const VPBlockBase *FromBlock, const VPBlockBase *ToBlock, const VPLoopInfo *VPLI)
Returns true if the edge FromBlock -> ToBlock is a back-edge.
Definition: VPlan.h:1875
llvm::Twine
Twine - A lightweight data structure for efficiently representing the concatenation of temporary valu...
Definition: Twine.h:80
llvm::VPSlotTracker
This class can be used to assign consecutive numbers to all VPValues in a VPlan and allows querying t...
Definition: VPlanValue.h:365
llvm::map_range
auto map_range(ContainerTy &&C, FuncTy F)
Definition: STLExtras.h:311
llvm::VPBasicBlock::begin
const_iterator begin() const
Definition: VPlan.h:1324
llvm::VPRegionBlock::getExit
const VPBlockBase * getExit() const
Definition: VPlan.h:1443
llvm::GraphProgram::Name
Name
Definition: GraphWriter.h:52
llvm::VPInstruction::ICmpULE
@ ICmpULE
Definition: VPlan.h:609
llvm::VPRecipeBase::~VPRecipeBase
virtual ~VPRecipeBase()=default
llvm::VPWidenGEPRecipe::VPWidenGEPRecipe
VPWidenGEPRecipe(GetElementPtrInst *GEP, iterator_range< IterT > Operands, Loop *OrigLoop)
Definition: VPlan.h:793
llvm::VPlanSlp::VPlanSlp
VPlanSlp(VPInterleavedAccessInfo &IAI, VPBasicBlock &BB)
Definition: VPlan.h:2027
llvm::VPBlockBase::getParent
VPRegionBlock * getParent()
Definition: VPlan.h:339
llvm::VPBasicBlock::rbegin
reverse_iterator rbegin()
Definition: VPlan.h:1328
llvm::VPWidenGEPRecipe::classof
static bool classof(const VPDef *D)
Method to support type inquiry through isa, cast, and dyn_cast.
Definition: VPlan.h:806
llvm::Successor
@ Successor
Definition: SIMachineScheduler.h:33
llvm::AMDGPU::SendMsg::Op
Op
Definition: SIDefines.h:302
llvm::ilist_iterator
Iterator for intrusive lists based on ilist_node.
Definition: ilist_iterator.h:57
llvm::VPRegionBlock::VPRegionBlock
VPRegionBlock(const std::string &Name="", bool IsReplicator=false)
Definition: VPlan.h:1408
llvm::VPRecipeBase::classof
static bool classof(const VPDef *D)
Method to support type inquiry through isa, cast, and dyn_cast.
Definition: VPlan.h:578
llvm::LoopInfoBase< VPBlockBase, VPLoop >
llvm::VPWidenGEPRecipe::VPWidenGEPRecipe
VPWidenGEPRecipe(GetElementPtrInst *GEP, iterator_range< IterT > Operands)
Definition: VPlan.h:787
llvm::VPRecipeBase
VPRecipeBase is a base class modeling a sequence of one or more output IR instructions.
Definition: VPlan.h:515
llvm::VPWidenPHIRecipe::classof
static bool classof(const VPDef *D)
Method to support type inquiry through isa, cast, and dyn_cast.
Definition: VPlan.h:898
llvm::VPlan::setEntry
VPBlockBase * setEntry(VPBlockBase *Block)
Definition: VPlan.h:1656
llvm::LoopBase::getHeader
BlockT * getHeader() const
Definition: LoopInfo.h:104
llvm::VPReductionRecipe::~VPReductionRecipe
~VPReductionRecipe() override=default
llvm::TargetStackID::Value
Value
Definition: TargetFrameLowering.h:27
llvm::VPlan::getOrAddVPValue
VPValue * getOrAddVPValue(Value *V)
Definition: VPlan.h:1703
llvm::SetVector< T, SmallVector< T, N >, SmallDenseSet< T, N > >::count
size_type count(const key_type &key) const
Count the number of elements of a given key in the SetVector.
Definition: SetVector.h:215
llvm::VPBlockBase::getPredicate
VPValue * getPredicate()
Definition: VPlan.cpp:214
llvm::SlotTracker
This class provides computation of slot numbers for LLVM Assembly writing.
Definition: AsmWriter.cpp:716
llvm::SmallVectorImpl::clear
void clear()
Definition: SmallVector.h:585
llvm::VPInstruction::setUnderlyingInstr
void setUnderlyingInstr(Instruction *I)
Definition: VPlan.h:624
llvm::VPTransformState::CFGState
Hold state information used when constructing the CFG of the output IR, traversing the VPBasicBlocks ...
Definition: VPlan.h:208
llvm::VPBlockBase::setName
void setName(const Twine &newName)
Definition: VPlan.h:332
llvm::Inverse
Definition: GraphTraits.h:95
llvm::VPBlockUtils::blockIsLoopLatch
static bool blockIsLoopLatch(const VPBlockBase *Block, const VPLoopInfo *VPLInfo)
Returns true if Block is a loop latch.
Definition: VPlan.h:1889
llvm::VPTransformState::CFGState::CFGState
CFGState()=default
llvm::VPBasicBlock::insert
void insert(VPRecipeBase *Recipe, iterator InsertPt)
Definition: VPlan.h:1353
llvm::VPlan::getVPLoopInfo
VPLoopInfo & getVPLoopInfo()
Return the VPLoopInfo analysis for this VPlan.
Definition: VPlan.h:1713
llvm::VPBlendRecipe
A recipe for vectorizing a phi-node as a sequence of mask-based select instructions.
Definition: VPlan.h:932
llvm::iplist_impl::insert
iterator insert(iterator where, pointer New)
Definition: ilist.h:228
llvm::VPUser
This class augments VPValue with operands which provide the inverse def-use edges from VPValue's user...
Definition: VPlanValue.h:191
llvm::RecurrenceDescriptor
The RecurrenceDescriptor is used to identify recurrences variables in a loop.
Definition: IVDescriptors.h:66
llvm::VPlan::VPlan
VPlan(VPBlockBase *Entry=nullptr)
Definition: VPlan.h:1629
llvm::VPReductionRecipe::classof
static bool classof(const VPValue *V)
Method to support type inquiry through isa, cast, and dyn_cast.
Definition: VPlan.h:1058
llvm::VPValue::VPVWidenSC
@ VPVWidenSC
Definition: VPlanValue.h:99
llvm::VPBlendRecipe::VPBlendRecipe
VPBlendRecipe(PHINode *Phi, ArrayRef< VPValue * > Operands)
The blend operation is a User of the incoming values and of their respective masks,...
Definition: VPlan.h:939
SmallVector.h
llvm::VPBasicBlock::front
const VPRecipeBase & front() const
Definition: VPlan.h:1335
llvm::VPTransformState::VPValue2Value
VPValue2ValueTy VPValue2Value
Definition: VPlan.h:240
llvm::VPBasicBlock::begin
iterator begin()
Recipe iterator methods.
Definition: VPlan.h:1323
llvm::VPlanSlp::isCompletelySLP
bool isCompletelySLP() const
Return true if all visited instruction can be combined.
Definition: VPlan.h:2039
llvm::VPRegionBlock::setEntry
void setEntry(VPBlockBase *EntryBlock)
Set EntryBlock as the entry VPBlockBase of this VPRegionBlock.
Definition: VPlan.h:1430
llvm::VPWidenMemoryInstructionRecipe::execute
void execute(VPTransformState &State) override
Generate the wide load/store.
Definition: LoopVectorize.cpp:9087
llvm::VPBlendRecipe::classof
static bool classof(const VPDef *D)
Method to support type inquiry through isa, cast, and dyn_cast.
Definition: VPlan.h:949
SmallBitVector.h
llvm::VPUser::getOperand
VPValue * getOperand(unsigned N) const
Definition: VPlanValue.h:225
llvm::VPTransformState::Instance
Optional< VPIteration > Instance
Hold the indices to generate specific scalar instructions.
Definition: VPlan.h:122
N
#define N
llvm::iplist_impl< simple_ilist< VPRecipeBase, Options... >, ilist_traits< VPRecipeBase > >::const_iterator
base_list_type::const_iterator const_iterator
Definition: ilist.h:178
llvm::hash_combine_range
hash_code hash_combine_range(InputIteratorT first, InputIteratorT last)
Compute a hash_code for a sequence of values.
Definition: Hashing.h:482
llvm::VPRegionBlock::classof
static bool classof(const VPBlockBase *V)
Method to support type inquiry through isa, cast, and dyn_cast.
Definition: VPlan.h:1421
llvm::VPlan::getOrCreateBackedgeTakenCount
VPValue * getOrCreateBackedgeTakenCount()
The backedge taken count of the original loop.
Definition: VPlan.h:1663
llvm::VPBlockUtils::VPBlockUtils
VPBlockUtils()=delete
llvm::iterator_range
A range adaptor for a pair of iterators.
Definition: iterator_range.h:30
llvm::VPTransformState::CFGState::LastBB
BasicBlock * LastBB
The last IR BasicBlock in the output IR.
Definition: VPlan.h:218
llvm::VPInstruction::VPInstruction
VPInstruction(unsigned Opcode, std::initializer_list< VPValue * > Operands)
Definition: VPlan.h:638
llvm::PHINode
Definition: Instructions.h:2582
llvm::VPWidenRecipe::VPWidenRecipe
VPWidenRecipe(Instruction &I, iterator_range< IterT > Operands)
Definition: VPlan.h:705
ilist_node.h
llvm::VPInstruction::SLPLoad
@ SLPLoad
Definition: VPlan.h:610
llvm::SmallVectorImpl< VPBlockBase * >
llvm::VPTransformState::CFGState::VPBB2IRBB
SmallDenseMap< VPBasicBlock *, BasicBlock * > VPBB2IRBB
A mapping of each VPBasicBlock to the corresponding BasicBlock.
Definition: VPlan.h:222
llvm::VPBlockBase::getParent
const VPRegionBlock * getParent() const
Definition: VPlan.h:340
llvm::VPTransformState::Plan
VPlan * Plan
Pointer to the VPlan code is generated for.
Definition: VPlan.h:252
llvm::SmallPtrSetImpl
A templated base class for SmallPtrSet which provides the typesafe interface that is common across al...
Definition: SmallPtrSet.h:343
llvm::SmallSetVector
A SetVector that performs no allocations if smaller than a certain size.
Definition: SetVector.h:307
llvm::VPReductionRecipe::classof
static bool classof(const VPDef *D)
Definition: VPlan.h:1062
llvm::VPlan::getVPLoopInfo
const VPLoopInfo & getVPLoopInfo() const
Definition: VPlan.h:1714
llvm::VPPredInstPHIRecipe::VPPredInstPHIRecipe
VPPredInstPHIRecipe(VPValue *PredV)
Construct a VPPredInstPHIRecipe given PredInst whose value needs a phi nodes after merging back from ...
Definition: VPlan.h:1185
llvm::CallInst
This class represents a function call, abstracting a target machine's calling convention.
Definition: Instructions.h:1450
llvm::VPValue::getVPValueID
unsigned getVPValueID() const
Definition: VPlanValue.h:117
llvm::VPWidenSelectRecipe::VPWidenSelectRecipe
VPWidenSelectRecipe(SelectInst &I, iterator_range< IterT > Operands, bool InvariantCond)
Definition: VPlan.h:759
llvm::VPBranchOnMaskRecipe::getMask
VPValue * getMask() const
Return the mask used by this recipe.
Definition: VPlan.h:1169
BB
Common register allocation spilling lr str ldr sxth r3 ldr mla r4 can lr mov lr str ldr sxth r3 mla r4 and then merge mul and lr str ldr sxth r3 mla r4 It also increase the likelihood the store may become dead bb27 Successors according to LLVM BB
Definition: README.txt:39
GEP
Hexagon Common GEP
Definition: HexagonCommonGEP.cpp:171
llvm::VPBlockBase::deleteCFG
static void deleteCFG(VPBlockBase *Entry)
Delete all blocks reachable from a given VPBlockBase, inclusive.
Definition: VPlan.cpp:226
llvm::VPRegionBlock::isReplicator
bool isReplicator() const
An indicator whether this region is to generate multiple replicated instances of output IR correspond...
Definition: VPlan.h:1457
llvm::GraphTraits
Definition: GraphTraits.h:35
From
BlockVerifier::State From
Definition: BlockVerifier.cpp:55
llvm::VPWidenPHIRecipe::print
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
Definition: VPlan.cpp:934
llvm::VPWidenIntOrFpInductionRecipe::getTruncInst
const TruncInst * getTruncInst() const
Definition: VPlan.h:865
llvm::Region
Definition: RegionInfo.h:889
llvm::VPValue::VPVBlendSC
@ VPVBlendSC
Definition: VPlanValue.h:93
llvm::VFRange::End
ElementCount End
Definition: VPlan.h:71
llvm::VPBasicBlock::getFirstNonPhi
iterator getFirstNonPhi()
Return the position of the first non-phi node recipe in the block.
Definition: VPlan.cpp:233
llvm::VPReductionRecipe::getChainOp
VPValue * getChainOp() const
The VPValue of the scalar Chain being accumulated.
Definition: VPlan.h:1074
llvm::VPInterleavedAccessInfo::VPInterleavedAccessInfo
VPInterleavedAccessInfo(VPlan &Plan, InterleavedAccessInfo &IAI)
Definition: VPlan.cpp:1123
llvm::VPWidenRecipe::classof
static bool classof(const VPValue *V)
Definition: VPlan.h:715
llvm::VPWidenRecipe::classof
static bool classof(const VPDef *D)
Method to support type inquiry through isa, cast, and dyn_cast.
Definition: VPlan.h:712
llvm::VPBlockBase::getSuccessors
const VPBlocksTy & getSuccessors() const
Definition: VPlan.h:364
llvm::VPValue::VPVWidenPHISC
@ VPVWidenPHISC
Definition: VPlanValue.h:103
llvm::GraphTraits< Inverse< VPBlockBase * > >::ChildIteratorType
SmallVectorImpl< VPBlockBase * >::iterator ChildIteratorType
Definition: VPlan.h:1510
llvm::VPWidenRecipe::~VPWidenRecipe
~VPWidenRecipe() override=default
llvm::VPWidenRecipe::execute
void execute(VPTransformState &State) override
Produce widened copies of all Ingredients.
Definition: LoopVectorize.cpp:8884
llvm::VPInstruction::dump
void dump() const
Print the VPInstruction to dbgs() (for debugging).
Definition: VPlan.cpp:542
llvm::VPBasicBlock::rend
reverse_iterator rend()
Definition: VPlan.h:1330
llvm::VPDef::getVPValue
VPValue * getVPValue(unsigned I=0)
Returns the VPValue with index I defined by the VPDef.
Definition: VPlanValue.h:328
llvm::GraphTraits< VPBlockBase * >::child_end
static ChildIteratorType child_end(NodeRef N)
Definition: VPlan.h:1486
llvm::VPWidenPHIRecipe::getStartValue
VPValue * getStartValue()
Returns the start value of the phi, if it is a reduction.
Definition: VPlan.h:913
llvm::VPlan
VPlan models a candidate for vectorization, encoding various decisions take to produce efficient outp...
Definition: VPlan.h:1593
llvm::GraphTraits< Inverse< VPRegionBlock * > >::nodes_begin
static nodes_iterator nodes_begin(GraphRef N)
Definition: VPlan.h:1577
llvm::VPBlockBase::setTwoSuccessors
void setTwoSuccessors(VPBlockBase *IfTrue, VPBlockBase *IfFalse, VPValue *Condition)
Set two given VPBlockBases IfTrue and IfFalse to be the two successors of this VPBlockBase.
Definition: VPlan.h:454
llvm::VPInterleavedAccessInfo::~VPInterleavedAccessInfo
~VPInterleavedAccessInfo()
Definition: VPlan.h:1931
llvm::VPBlockBase::setPlan
void setPlan(VPlan *ParentPlan)
Sets the pointer of the plan containing the block.
Definition: VPlan.cpp:137
llvm::Value
LLVM Value Representation.
Definition: Value.h:75
llvm::VPRecipeBase::insertBefore
void insertBefore(VPRecipeBase *InsertPos)
Insert an unlinked recipe into a basic block immediately before the specified recipe.
Definition: VPlan.cpp:443
llvm::GraphTraits< const VPRegionBlock * >::getEntryNode
static NodeRef getEntryNode(GraphRef N)
Definition: VPlan.h:1554
llvm::VPWidenPHIRecipe::VPWidenPHIRecipe
VPWidenPHIRecipe(PHINode *Phi)
Create a VPWidenPHIRecipe for Phi.
Definition: VPlan.h:892
llvm::VPlan::dump
void dump() const
Dump the plan to stderr (for debugging).
Definition: VPlan.cpp:673
llvm::VPlanIngredient
Definition: VPlan.h:1792
llvm::VPWidenSelectRecipe::print
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
Definition: VPlan.cpp:887
llvm::VPWidenIntOrFpInductionRecipe::getStartValue
VPValue * getStartValue()
Returns the start value of the induction.
Definition: VPlan.h:851
llvm::VPlanPrinter
VPlanPrinter prints a given VPlan to a given output stream.
Definition: VPlan.h:1739
llvm::VPTransformState::CFGState::PrevBB
BasicBlock * PrevBB
The previous IR BasicBlock created or used.
Definition: VPlan.h:214
llvm::GraphTraits< VPBlockBase * >::ChildIteratorType
SmallVectorImpl< VPBlockBase * >::iterator ChildIteratorType
Definition: VPlan.h:1478
llvm::VPRecipeBase::insertAfter
void insertAfter(VPRecipeBase *InsertPos)
Insert an unlinked Recipe into a basic block immediately after the specified Recipe.
Definition: VPlan.cpp:451
llvm::VFRange::Start
const ElementCount Start
Definition: VPlan.h:68
llvm::VPTransformState::hasVectorValue
bool hasVectorValue(VPValue *Def, unsigned Part)
Definition: VPlan.h:146
llvm::Use
A Use represents the edge between a Value definition and its users.
Definition: Use.h:44
SmallSet.h
llvm::SmallPtrSetImpl::insert
std::pair< iterator, bool > insert(PtrType Ptr)
Inserts Ptr if and only if there is no element in the container equal to Ptr.
Definition: SmallPtrSet.h:364
llvm::VPlan::addVF
void addVF(ElementCount VF)
Definition: VPlan.h:1669