LLVM  14.0.0git
LoopVectorizationLegality.h
Go to the documentation of this file.
1 //===- llvm/Transforms/Vectorize/LoopVectorizationLegality.h ----*- C++ -*-===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 /// \file
10 /// This file defines the LoopVectorizationLegality class. Original code
11 /// in Loop Vectorizer has been moved out to its own file for modularity
12 /// and reusability.
13 ///
14 /// Currently, it works for innermost loop vectorization. Extending this to
15 /// outer loop vectorization is a TODO item.
16 ///
17 /// Also provides:
18 /// 1) LoopVectorizeHints class which keeps a number of loop annotations
19 /// locally for easy look up. It has the ability to write them back as
20 /// loop metadata, upon request.
21 /// 2) LoopVectorizationRequirements class for lazy bail out for the purpose
22 /// of reporting useful failure to vectorize message.
23 //
24 //===----------------------------------------------------------------------===//
25 
26 #ifndef LLVM_TRANSFORMS_VECTORIZE_LOOPVECTORIZATIONLEGALITY_H
27 #define LLVM_TRANSFORMS_VECTORIZE_LOOPVECTORIZATIONLEGALITY_H
28 
29 #include "llvm/ADT/MapVector.h"
33 #include "llvm/Support/TypeSize.h"
35 
36 namespace llvm {
37 
38 /// Utility class for getting and setting loop vectorizer hints in the form
39 /// of loop metadata.
40 /// This class keeps a number of loop annotations locally (as member variables)
41 /// and can, upon request, write them back as metadata on the loop. It will
42 /// initially scan the loop for existing metadata, and will update the local
43 /// values based on information in the loop.
44 /// We cannot write all values to metadata, as the mere presence of some info,
45 /// for example 'force', means a decision has been made. So, we need to be
46 /// careful NOT to add them if the user hasn't specifically asked so.
48  enum HintKind {
49  HK_WIDTH,
50  HK_INTERLEAVE,
51  HK_FORCE,
52  HK_ISVECTORIZED,
53  HK_PREDICATE,
54  HK_SCALABLE
55  };
56 
57  /// Hint - associates name and validation with the hint value.
58  struct Hint {
59  const char *Name;
60  unsigned Value; // This may have to change for non-numeric values.
61  HintKind Kind;
62 
63  Hint(const char *Name, unsigned Value, HintKind Kind)
64  : Name(Name), Value(Value), Kind(Kind) {}
65 
66  bool validate(unsigned Val);
67  };
68 
69  /// Vectorization width.
70  Hint Width;
71 
72  /// Vectorization interleave factor.
73  Hint Interleave;
74 
75  /// Vectorization forced
76  Hint Force;
77 
78  /// Already Vectorized
79  Hint IsVectorized;
80 
81  /// Vector Predicate
82  Hint Predicate;
83 
84  /// Says whether we should use fixed width or scalable vectorization.
85  Hint Scalable;
86 
87  /// Return the loop metadata prefix.
88  static StringRef Prefix() { return "llvm.loop."; }
89 
90  /// True if there is any unsafe math in the loop.
91  bool PotentiallyUnsafe = false;
92 
93 public:
94  enum ForceKind {
95  FK_Undefined = -1, ///< Not selected.
96  FK_Disabled = 0, ///< Forcing disabled.
97  FK_Enabled = 1, ///< Forcing enabled.
98  };
99 
101  /// Not selected.
103  /// Disables vectorization with scalable vectors.
105  /// Vectorize loops using scalable vectors or fixed-width vectors, but favor
106  /// scalable vectors when the cost-model is inconclusive. This is the
107  /// default when the scalable.enable hint is enabled through a pragma.
109  };
110 
111  LoopVectorizeHints(const Loop *L, bool InterleaveOnlyWhenForced,
113  const TargetTransformInfo *TTI = nullptr);
114 
115  /// Mark the loop L as already vectorized by setting the width to 1.
116  void setAlreadyVectorized();
117 
118  bool allowVectorization(Function *F, Loop *L,
119  bool VectorizeOnlyWhenForced) const;
120 
121  /// Dumps all the hint information.
122  void emitRemarkWithHints() const;
123 
125  return ElementCount::get(Width.Value, (ScalableForceKind)Scalable.Value ==
127  }
128 
129  unsigned getInterleave() const {
130  if (Interleave.Value)
131  return Interleave.Value;
132  // If interleaving is not explicitly set, assume that if we do not want
133  // unrolling, we also don't want any interleaving.
135  return 1;
136  return 0;
137  }
138  unsigned getIsVectorized() const { return IsVectorized.Value; }
139  unsigned getPredicate() const { return Predicate.Value; }
140  enum ForceKind getForce() const {
141  if ((ForceKind)Force.Value == FK_Undefined &&
143  return FK_Disabled;
144  return (ForceKind)Force.Value;
145  }
146 
147  /// \return true if scalable vectorization has been explicitly disabled.
149  return (ScalableForceKind)Scalable.Value == SK_FixedWidthOnly;
150  }
151 
152  /// If hints are provided that force vectorization, use the AlwaysPrint
153  /// pass name to force the frontend to print the diagnostic.
154  const char *vectorizeAnalysisPassName() const;
155 
156  /// When enabling loop hints are provided we allow the vectorizer to change
157  /// the order of operations that is given by the scalar loop. This is not
158  /// enabled by default because can be unsafe or inefficient. For example,
159  /// reordering floating-point operations will change the way round-off
160  /// error accumulates in the loop.
161  bool allowReordering() const;
162 
163  bool isPotentiallyUnsafe() const {
164  // Avoid FP vectorization if the target is unsure about proper support.
165  // This may be related to the SIMD unit in the target not handling
166  // IEEE 754 FP ops properly, or bad single-to-double promotions.
167  // Otherwise, a sequence of vectorized loops, even without reduction,
168  // could lead to different end results on the destination vectors.
169  return getForce() != LoopVectorizeHints::FK_Enabled && PotentiallyUnsafe;
170  }
171 
172  void setPotentiallyUnsafe() { PotentiallyUnsafe = true; }
173 
174 private:
175  /// Find hints specified in the loop metadata and update local values.
176  void getHintsFromMetadata();
177 
178  /// Checks string hint with one operand and set value if valid.
179  void setHint(StringRef Name, Metadata *Arg);
180 
181  /// The loop these hints belong to.
182  const Loop *TheLoop;
183 
184  /// Interface to emit optimization remarks.
186 };
187 
188 /// This holds vectorization requirements that must be verified late in
189 /// the process. The requirements are set by legalize and costmodel. Once
190 /// vectorization has been determined to be possible and profitable the
191 /// requirements can be verified by looking for metadata or compiler options.
192 /// For example, some loops require FP commutativity which is only allowed if
193 /// vectorization is explicitly specified or if the fast-math compiler option
194 /// has been provided.
195 /// Late evaluation of these requirements allows helpful diagnostics to be
196 /// composed that tells the user what need to be done to vectorize the loop. For
197 /// example, by specifying #pragma clang loop vectorize or -ffast-math. Late
198 /// evaluation should be used only when diagnostics can generated that can be
199 /// followed by a non-expert user.
201 public:
202  /// Track the 1st floating-point instruction that can not be reassociated.
204  if (I && !ExactFPMathInst)
205  ExactFPMathInst = I;
206  }
207 
208  void addRuntimePointerChecks(unsigned Num) { NumRuntimePointerChecks = Num; }
209 
210 
211  Instruction *getExactFPInst() { return ExactFPMathInst; }
212 
213  unsigned getNumRuntimePointerChecks() const {
214  return NumRuntimePointerChecks;
215  }
216 
217 private:
218  unsigned NumRuntimePointerChecks = 0;
219  Instruction *ExactFPMathInst = nullptr;
220 };
221 
222 /// LoopVectorizationLegality checks if it is legal to vectorize a loop, and
223 /// to what vectorization factor.
224 /// This class does not look at the profitability of vectorization, only the
225 /// legality. This class has two main kinds of checks:
226 /// * Memory checks - The code in canVectorizeMemory checks if vectorization
227 /// will change the order of memory accesses in a way that will change the
228 /// correctness of the program.
229 /// * Scalars checks - The code in canVectorizeInstrs and canVectorizeMemory
230 /// checks for a number of different conditions, such as the availability of a
231 /// single induction variable, that all types are supported and vectorize-able,
232 /// etc. This code reflects the capabilities of InnerLoopVectorizer.
233 /// This class is also used by InnerLoopVectorizer for identifying
234 /// induction variable and the different reduction variables.
236 public:
240  Function *F, std::function<const LoopAccessInfo &(Loop &)> *GetLAA,
244  : TheLoop(L), LI(LI), PSE(PSE), TTI(TTI), TLI(TLI), DT(DT),
245  GetLAA(GetLAA), ORE(ORE), Requirements(R), Hints(H), DB(DB), AC(AC),
246  BFI(BFI), PSI(PSI) {}
247 
248  /// ReductionList contains the reduction descriptors for all
249  /// of the reductions that were found in the loop.
251 
252  /// InductionList saves induction variables and maps them to the
253  /// induction descriptor.
255 
256  /// RecurrenceSet contains the phi nodes that are recurrences other than
257  /// inductions and reductions.
259 
260  /// Returns true if it is legal to vectorize this loop.
261  /// This does not mean that it is profitable to vectorize this
262  /// loop, only that it is legal to do so.
263  /// Temporarily taking UseVPlanNativePath parameter. If true, take
264  /// the new code path being implemented for outer loop vectorization
265  /// (should be functional for inner loop vectorization) based on VPlan.
266  /// If false, good old LV code.
267  bool canVectorize(bool UseVPlanNativePath);
268 
269  /// Returns true if it is legal to vectorize the FP math operations in this
270  /// loop. Vectorizing is legal if we allow reordering of FP operations, or if
271  /// we can use in-order reductions.
272  bool canVectorizeFPMath(bool EnableStrictReductions);
273 
274  /// Return true if we can vectorize this loop while folding its tail by
275  /// masking, and mark all respective loads/stores for masking.
276  /// This object's state is only modified iff this function returns true.
278 
279  /// Returns the primary induction variable.
280  PHINode *getPrimaryInduction() { return PrimaryInduction; }
281 
282  /// Returns the reduction variables found in the loop.
283  const ReductionList &getReductionVars() const { return Reductions; }
284 
285  /// Returns the induction variables found in the loop.
286  const InductionList &getInductionVars() const { return Inductions; }
287 
288  /// Return the first-order recurrences found in the loop.
289  RecurrenceSet &getFirstOrderRecurrences() { return FirstOrderRecurrences; }
290 
291  /// Return the set of instructions to sink to handle first-order recurrences.
293 
294  /// Returns the widest induction type.
295  Type *getWidestInductionType() { return WidestIndTy; }
296 
297  /// Returns True if V is a Phi node of an induction variable in this loop.
298  bool isInductionPhi(const Value *V) const;
299 
300  /// Returns a pointer to the induction descriptor, if \p Phi is an integer or
301  /// floating point induction.
303 
304  /// Returns True if V is a cast that is part of an induction def-use chain,
305  /// and had been proven to be redundant under a runtime guard (in other
306  /// words, the cast has the same SCEV expression as the induction phi).
307  bool isCastedInductionVariable(const Value *V) const;
308 
309  /// Returns True if V can be considered as an induction variable in this
310  /// loop. V can be the induction phi, or some redundant cast in the def-use
311  /// chain of the inducion phi.
312  bool isInductionVariable(const Value *V) const;
313 
314  /// Returns True if PN is a reduction variable in this loop.
315  bool isReductionVariable(PHINode *PN) const { return Reductions.count(PN); }
316 
317  /// Returns True if Phi is a first-order recurrence in this loop.
318  bool isFirstOrderRecurrence(const PHINode *Phi) const;
319 
320  /// Return true if the block BB needs to be predicated in order for the loop
321  /// to be vectorized.
322  bool blockNeedsPredication(BasicBlock *BB) const;
323 
324  /// Check if this pointer is consecutive when vectorizing. This happens
325  /// when the last index of the GEP is the induction variable, or that the
326  /// pointer itself is an induction variable.
327  /// This check allows us to vectorize A[idx] into a wide load/store.
328  /// Returns:
329  /// 0 - Stride is unknown or non-consecutive.
330  /// 1 - Address is consecutive.
331  /// -1 - Address is consecutive, and decreasing.
332  /// NOTE: This method must only be used before modifying the original scalar
333  /// loop. Do not use after invoking 'createVectorizedLoopSkeleton' (PR34965).
334  int isConsecutivePtr(Type *AccessTy, Value *Ptr) const;
335 
336  /// Returns true if the value V is uniform within the loop.
337  bool isUniform(Value *V);
338 
339  /// A uniform memory op is a load or store which accesses the same memory
340  /// location on all lanes.
343  if (!Ptr)
344  return false;
345  // Note: There's nothing inherent which prevents predicated loads and
346  // stores from being uniform. The current lowering simply doesn't handle
347  // it; in particular, the cost model distinguishes scatter/gather from
348  // scalar w/predication, and we currently rely on the scalar path.
349  return isUniform(Ptr) && !blockNeedsPredication(I.getParent());
350  }
351 
352  /// Returns the information that we collected about runtime memory check.
354  return LAI->getRuntimePointerChecking();
355  }
356 
357  const LoopAccessInfo *getLAI() const { return LAI; }
358 
359  bool isSafeForAnyVectorWidth() const {
360  return LAI->getDepChecker().isSafeForAnyVectorWidth();
361  }
362 
363  unsigned getMaxSafeDepDistBytes() { return LAI->getMaxSafeDepDistBytes(); }
364 
367  }
368 
369  bool hasStride(Value *V) { return LAI->hasStride(V); }
370 
371  /// Returns true if vector representation of the instruction \p I
372  /// requires mask.
373  bool isMaskRequired(const Instruction *I) const {
374  return MaskedOp.contains(I);
375  }
376 
377  unsigned getNumStores() const { return LAI->getNumStores(); }
378  unsigned getNumLoads() const { return LAI->getNumLoads(); }
379 
380  /// Returns all assume calls in predicated blocks. They need to be dropped
381  /// when flattening the CFG.
383  return ConditionalAssumes;
384  }
385 
386 private:
387  /// Return true if the pre-header, exiting and latch blocks of \p Lp and all
388  /// its nested loops are considered legal for vectorization. These legal
389  /// checks are common for inner and outer loop vectorization.
390  /// Temporarily taking UseVPlanNativePath parameter. If true, take
391  /// the new code path being implemented for outer loop vectorization
392  /// (should be functional for inner loop vectorization) based on VPlan.
393  /// If false, good old LV code.
394  bool canVectorizeLoopNestCFG(Loop *Lp, bool UseVPlanNativePath);
395 
396  /// Set up outer loop inductions by checking Phis in outer loop header for
397  /// supported inductions (int inductions). Return false if any of these Phis
398  /// is not a supported induction or if we fail to find an induction.
399  bool setupOuterLoopInductions();
400 
401  /// Return true if the pre-header, exiting and latch blocks of \p Lp
402  /// (non-recursive) are considered legal for vectorization.
403  /// Temporarily taking UseVPlanNativePath parameter. If true, take
404  /// the new code path being implemented for outer loop vectorization
405  /// (should be functional for inner loop vectorization) based on VPlan.
406  /// If false, good old LV code.
407  bool canVectorizeLoopCFG(Loop *Lp, bool UseVPlanNativePath);
408 
409  /// Check if a single basic block loop is vectorizable.
410  /// At this point we know that this is a loop with a constant trip count
411  /// and we only need to check individual instructions.
412  bool canVectorizeInstrs();
413 
414  /// When we vectorize loops we may change the order in which
415  /// we read and write from memory. This method checks if it is
416  /// legal to vectorize the code, considering only memory constrains.
417  /// Returns true if the loop is vectorizable
418  bool canVectorizeMemory();
419 
420  /// Return true if we can vectorize this loop using the IF-conversion
421  /// transformation.
422  bool canVectorizeWithIfConvert();
423 
424  /// Return true if we can vectorize this outer loop. The method performs
425  /// specific checks for outer loop vectorization.
426  bool canVectorizeOuterLoop();
427 
428  /// Return true if all of the instructions in the block can be speculatively
429  /// executed, and record the loads/stores that require masking.
430  /// \p SafePtrs is a list of addresses that are known to be legal and we know
431  /// that we can read from them without segfault.
432  /// \p MaskedOp is a list of instructions that have to be transformed into
433  /// calls to the appropriate masked intrinsic when the loop is vectorized.
434  /// \p ConditionalAssumes is a list of assume instructions in predicated
435  /// blocks that must be dropped if the CFG gets flattened.
436  bool blockCanBePredicated(
439  SmallPtrSetImpl<Instruction *> &ConditionalAssumes) const;
440 
441  /// Updates the vectorization state by adding \p Phi to the inductions list.
442  /// This can set \p Phi as the main induction of the loop if \p Phi is a
443  /// better choice for the main induction than the existing one.
444  void addInductionPhi(PHINode *Phi, const InductionDescriptor &ID,
445  SmallPtrSetImpl<Value *> &AllowedExit);
446 
447  /// If an access has a symbolic strides, this maps the pointer value to
448  /// the stride symbol.
449  const ValueToValueMap *getSymbolicStrides() const {
450  // FIXME: Currently, the set of symbolic strides is sometimes queried before
451  // it's collected. This happens from canVectorizeWithIfConvert, when the
452  // pointer is checked to reference consecutive elements suitable for a
453  // masked access.
454  return LAI ? &LAI->getSymbolicStrides() : nullptr;
455  }
456 
457  /// The loop that we evaluate.
458  Loop *TheLoop;
459 
460  /// Loop Info analysis.
461  LoopInfo *LI;
462 
463  /// A wrapper around ScalarEvolution used to add runtime SCEV checks.
464  /// Applies dynamic knowledge to simplify SCEV expressions in the context
465  /// of existing SCEV assumptions. The analysis will also add a minimal set
466  /// of new predicates if this is required to enable vectorization and
467  /// unrolling.
468  PredicatedScalarEvolution &PSE;
469 
470  /// Target Transform Info.
471  TargetTransformInfo *TTI;
472 
473  /// Target Library Info.
474  TargetLibraryInfo *TLI;
475 
476  /// Dominator Tree.
477  DominatorTree *DT;
478 
479  // LoopAccess analysis.
480  std::function<const LoopAccessInfo &(Loop &)> *GetLAA;
481 
482  // And the loop-accesses info corresponding to this loop. This pointer is
483  // null until canVectorizeMemory sets it up.
484  const LoopAccessInfo *LAI = nullptr;
485 
486  /// Interface to emit optimization remarks.
487  OptimizationRemarkEmitter *ORE;
488 
489  // --- vectorization state --- //
490 
491  /// Holds the primary induction variable. This is the counter of the
492  /// loop.
493  PHINode *PrimaryInduction = nullptr;
494 
495  /// Holds the reduction variables.
496  ReductionList Reductions;
497 
498  /// Holds all of the induction variables that we found in the loop.
499  /// Notice that inductions don't need to start at zero and that induction
500  /// variables can be pointers.
501  InductionList Inductions;
502 
503  /// Holds all the casts that participate in the update chain of the induction
504  /// variables, and that have been proven to be redundant (possibly under a
505  /// runtime guard). These casts can be ignored when creating the vectorized
506  /// loop body.
507  SmallPtrSet<Instruction *, 4> InductionCastsToIgnore;
508 
509  /// Holds the phi nodes that are first-order recurrences.
510  RecurrenceSet FirstOrderRecurrences;
511 
512  /// Holds instructions that need to sink past other instructions to handle
513  /// first-order recurrences.
514  MapVector<Instruction *, Instruction *> SinkAfter;
515 
516  /// Holds the widest induction type encountered.
517  Type *WidestIndTy = nullptr;
518 
519  /// Allowed outside users. This holds the variables that can be accessed from
520  /// outside the loop.
521  SmallPtrSet<Value *, 4> AllowedExit;
522 
523  /// Vectorization requirements that will go through late-evaluation.
524  LoopVectorizationRequirements *Requirements;
525 
526  /// Used to emit an analysis of any legality issues.
527  LoopVectorizeHints *Hints;
528 
529  /// The demanded bits analysis is used to compute the minimum type size in
530  /// which a reduction can be computed.
531  DemandedBits *DB;
532 
533  /// The assumption cache analysis is used to compute the minimum type size in
534  /// which a reduction can be computed.
535  AssumptionCache *AC;
536 
537  /// While vectorizing these instructions we have to generate a
538  /// call to the appropriate masked intrinsic
539  SmallPtrSet<const Instruction *, 8> MaskedOp;
540 
541  /// Assume instructions in predicated blocks must be dropped if the CFG gets
542  /// flattened.
543  SmallPtrSet<Instruction *, 8> ConditionalAssumes;
544 
545  /// BFI and PSI are used to check for profile guided size optimizations.
546  BlockFrequencyInfo *BFI;
547  ProfileSummaryInfo *PSI;
548 };
549 
550 } // namespace llvm
551 
552 #endif // LLVM_TRANSFORMS_VECTORIZE_LOOPVECTORIZATIONLEGALITY_H
llvm::LoopVectorizationLegality::RecurrenceSet
SmallPtrSet< const PHINode *, 8 > RecurrenceSet
RecurrenceSet contains the phi nodes that are recurrences other than inductions and reductions.
Definition: LoopVectorizationLegality.h:258
llvm::hasUnrollTransformation
TransformationMode hasUnrollTransformation(const Loop *L)
Definition: LoopUtils.cpp:361
llvm::LoopVectorizeHints::setPotentiallyUnsafe
void setPotentiallyUnsafe()
Definition: LoopVectorizationLegality.h:172
llvm::LoopVectorizationLegality::getReductionVars
const ReductionList & getReductionVars() const
Returns the reduction variables found in the loop.
Definition: LoopVectorizationLegality.h:283
llvm
This is an optimization pass for GlobalISel generic memory operations.
Definition: AllocatorList.h:22
llvm::LoopVectorizationLegality::getSinkAfter
MapVector< Instruction *, Instruction * > & getSinkAfter()
Return the set of instructions to sink to handle first-order recurrences.
Definition: LoopVectorizationLegality.h:292
llvm::LoopVectorizationLegality::getNumLoads
unsigned getNumLoads() const
Definition: LoopVectorizationLegality.h:378
llvm::ElementCount
Definition: TypeSize.h:385
llvm::hasDisableAllTransformsHint
bool hasDisableAllTransformsHint(const Loop *L)
Look for the loop attribute that disables all transformation heuristic.
Definition: LoopUtils.cpp:353
llvm::Function
Definition: Function.h:62
llvm::Loop
Represents a single loop in the control flow graph.
Definition: LoopInfo.h:530
llvm::LoopVectorizationLegality::getLAI
const LoopAccessInfo * getLAI() const
Definition: LoopVectorizationLegality.h:357
llvm::LoopVectorizeHints::getPredicate
unsigned getPredicate() const
Definition: LoopVectorizationLegality.h:139
TypeSize.h
llvm::PredicatedScalarEvolution
An interface layer with SCEV used to manage how we see SCEV expressions for values in the context of ...
Definition: ScalarEvolution.h:2179
llvm::LoopVectorizationRequirements::addRuntimePointerChecks
void addRuntimePointerChecks(unsigned Num)
Definition: LoopVectorizationLegality.h:208
llvm::TargetTransformInfo
This pass provides access to the codegen interfaces that are needed for IR-level transformations.
Definition: TargetTransformInfo.h:168
LoopAccessAnalysis.h
MapVector.h
llvm::LoopVectorizeHints::SK_Unspecified
@ SK_Unspecified
Not selected.
Definition: LoopVectorizationLegality.h:102
llvm::LoopVectorizationLegality
LoopVectorizationLegality checks if it is legal to vectorize a loop, and to what vectorization factor...
Definition: LoopVectorizationLegality.h:235
OptimizationRemarkEmitter.h
llvm::DominatorTree
Concrete subclass of DominatorTreeBase that is used to compute a normal dominator tree.
Definition: Dominators.h:151
llvm::MemoryDepChecker::getMaxSafeVectorWidthInBits
uint64_t getMaxSafeVectorWidthInBits() const
Return the number of elements that are safe to operate on simultaneously, multiplied by the size of t...
Definition: LoopAccessAnalysis.h:206
llvm::LoopVectorizeHints::SK_FixedWidthOnly
@ SK_FixedWidthOnly
Disables vectorization with scalable vectors.
Definition: LoopVectorizationLegality.h:104
llvm::Type
The instances of the Type class are immutable: once they are created, they are never changed.
Definition: Type.h:45
llvm::LoopVectorizeHints::getWidth
ElementCount getWidth() const
Definition: LoopVectorizationLegality.h:124
llvm::LoopVectorizationLegality::canVectorizeFPMath
bool canVectorizeFPMath(bool EnableStrictReductions)
Returns true if it is legal to vectorize the FP math operations in this loop.
Definition: LoopVectorizationLegality.cpp:926
llvm::LoopVectorizeHints::getIsVectorized
unsigned getIsVectorized() const
Definition: LoopVectorizationLegality.h:138
llvm::LoopVectorizeHints::LoopVectorizeHints
LoopVectorizeHints(const Loop *L, bool InterleaveOnlyWhenForced, OptimizationRemarkEmitter &ORE, const TargetTransformInfo *TTI=nullptr)
Definition: LoopVectorizationLegality.cpp:97
llvm::MapVector< PHINode *, RecurrenceDescriptor >
llvm::SmallPtrSet< const PHINode *, 8 >
llvm::RISCVFeatures::validate
void validate(const Triple &TT, const FeatureBitset &FeatureBits)
Definition: RISCVBaseInfo.cpp:100
llvm::LoopVectorizationLegality::prepareToFoldTailByMasking
bool prepareToFoldTailByMasking()
Return true if we can vectorize this loop while folding its tail by masking, and mark all respective ...
Definition: LoopVectorizationLegality.cpp:1286
F
#define F(x, y, z)
Definition: MD5.cpp:55
llvm::LoopVectorizationRequirements::getNumRuntimePointerChecks
unsigned getNumRuntimePointerChecks() const
Definition: LoopVectorizationLegality.h:213
llvm::BasicBlock
LLVM Basic Block Representation.
Definition: BasicBlock.h:58
llvm::LoopVectorizationRequirements
This holds vectorization requirements that must be verified late in the process.
Definition: LoopVectorizationLegality.h:200
llvm::LoopVectorizeHints::FK_Undefined
@ FK_Undefined
Not selected.
Definition: LoopVectorizationLegality.h:95
Arg
amdgpu Simplify well known AMD library false FunctionCallee Value * Arg
Definition: AMDGPULibCalls.cpp:185
llvm::BlockFrequencyInfo
BlockFrequencyInfo pass uses BlockFrequencyInfoImpl implementation to estimate IR basic block frequen...
Definition: BlockFrequencyInfo.h:37
llvm::InductionDescriptor
A struct for saving information about induction variables.
Definition: IVDescriptors.h:306
llvm::LinearPolySize< ElementCount >::get
static ElementCount get(ScalarTy MinVal, bool Scalable)
Definition: TypeSize.h:289
llvm::LoopVectorizationLegality::ReductionList
MapVector< PHINode *, RecurrenceDescriptor > ReductionList
ReductionList contains the reduction descriptors for all of the reductions that were found in the loo...
Definition: LoopVectorizationLegality.h:250
llvm::AAResults
Definition: AliasAnalysis.h:507
llvm::LoopAccessInfo::getNumLoads
unsigned getNumLoads() const
Definition: LoopAccessAnalysis.h:545
llvm::LoopVectorizeHints::vectorizeAnalysisPassName
const char * vectorizeAnalysisPassName() const
If hints are provided that force vectorization, use the AlwaysPrint pass name to force the frontend t...
Definition: LoopVectorizationLegality.cpp:231
llvm::LoopVectorizationRequirements::addExactFPMathInst
void addExactFPMathInst(Instruction *I)
Track the 1st floating-point instruction that can not be reassociated.
Definition: LoopVectorizationLegality.h:203
llvm::LoopVectorizationLegality::blockNeedsPredication
bool blockNeedsPredication(BasicBlock *BB) const
Return true if the block BB needs to be predicated in order for the loop to be vectorized.
Definition: LoopVectorizationLegality.cpp:987
llvm::Instruction
Definition: Instruction.h:45
llvm::LoopVectorizeHints::getForce
enum ForceKind getForce() const
Definition: LoopVectorizationLegality.h:140
llvm::LoopAccessInfo::hasStride
bool hasStride(Value *V) const
Pointer has a symbolic stride.
Definition: LoopAccessAnalysis.h:567
LoopUtils.h
llvm::Metadata
Root of the metadata hierarchy.
Definition: Metadata.h:62
llvm::lltok::Kind
Kind
Definition: LLToken.h:18
llvm::ProfileSummaryInfo
Analysis providing profile information.
Definition: ProfileSummaryInfo.h:39
llvm::LoopVectorizationLegality::hasStride
bool hasStride(Value *V)
Definition: LoopVectorizationLegality.h:369
llvm::LoopVectorizationLegality::getConditionalAssumes
const SmallPtrSetImpl< Instruction * > & getConditionalAssumes() const
Returns all assume calls in predicated blocks.
Definition: LoopVectorizationLegality.h:382
llvm::LoopAccessInfo::getSymbolicStrides
const ValueToValueMap & getSymbolicStrides() const
If an access has a symbolic strides, this maps the pointer value to the stride symbol.
Definition: LoopAccessAnalysis.h:564
llvm::LoopVectorizeHints
Utility class for getting and setting loop vectorizer hints in the form of loop metadata.
Definition: LoopVectorizationLegality.h:47
llvm::LoopVectorizeHints::SK_PreferScalable
@ SK_PreferScalable
Vectorize loops using scalable vectors or fixed-width vectors, but favor scalable vectors when the co...
Definition: LoopVectorizationLegality.h:108
llvm::RuntimePointerChecking
Holds information about the memory runtime legality checks to verify that a group of pointers do not ...
Definition: LoopAccessAnalysis.h:364
llvm::LoopVectorizationLegality::isMaskRequired
bool isMaskRequired(const Instruction *I) const
Returns true if vector representation of the instruction I requires mask.
Definition: LoopVectorizationLegality.h:373
uint64_t
llvm::LoopVectorizeHints::FK_Enabled
@ FK_Enabled
Forcing enabled.
Definition: LoopVectorizationLegality.h:97
llvm::LoopVectorizeHints::getInterleave
unsigned getInterleave() const
Definition: LoopVectorizationLegality.h:129
llvm::LoopVectorizeHints::allowVectorization
bool allowVectorization(Function *F, Loop *L, bool VectorizeOnlyWhenForced) const
Definition: LoopVectorizationLegality.cpp:172
llvm::DenseMap< const Value *, Value * >
llvm::DemandedBits
Definition: DemandedBits.h:40
llvm::LoopAccessInfo::getRuntimePointerChecking
const RuntimePointerChecking * getRuntimePointerChecking() const
Definition: LoopAccessAnalysis.h:525
I
#define I(x, y, z)
Definition: MD5.cpp:58
llvm::LoopVectorizationLegality::getFirstOrderRecurrences
RecurrenceSet & getFirstOrderRecurrences()
Return the first-order recurrences found in the loop.
Definition: LoopVectorizationLegality.h:289
llvm::LoopAccessInfo
Drive the analysis of memory accesses in the loop.
Definition: LoopAccessAnalysis.h:511
llvm::LoopVectorizationLegality::LoopVectorizationLegality
LoopVectorizationLegality(Loop *L, PredicatedScalarEvolution &PSE, DominatorTree *DT, TargetTransformInfo *TTI, TargetLibraryInfo *TLI, AAResults *AA, Function *F, std::function< const LoopAccessInfo &(Loop &)> *GetLAA, LoopInfo *LI, OptimizationRemarkEmitter *ORE, LoopVectorizationRequirements *R, LoopVectorizeHints *H, DemandedBits *DB, AssumptionCache *AC, BlockFrequencyInfo *BFI, ProfileSummaryInfo *PSI)
Definition: LoopVectorizationLegality.h:237
llvm::TM_Disable
@ TM_Disable
The transformation should not be applied.
Definition: LoopUtils.h:274
llvm::LoopVectorizationLegality::isCastedInductionVariable
bool isCastedInductionVariable(const Value *V) const
Returns True if V is a cast that is part of an induction def-use chain, and had been proven to be red...
Definition: LoopVectorizationLegality.cpp:972
llvm::LoopVectorizationLegality::getNumStores
unsigned getNumStores() const
Definition: LoopVectorizationLegality.h:377
llvm::LoopVectorizationLegality::isInductionVariable
bool isInductionVariable(const Value *V) const
Returns True if V can be considered as an induction variable in this loop.
Definition: LoopVectorizationLegality.cpp:978
function
print Print MemDeps of function
Definition: MemDepPrinter.cpp:83
llvm::LoopVectorizationRequirements::getExactFPInst
Instruction * getExactFPInst()
Definition: LoopVectorizationLegality.h:211
llvm::LoopVectorizationLegality::isUniform
bool isUniform(Value *V)
Returns true if the value V is uniform within the loop.
Definition: LoopVectorizationLegality.cpp:459
llvm::LoopAccessInfo::getDepChecker
const MemoryDepChecker & getDepChecker() const
the Memory Dependence Checker which can determine the loop-independent and loop-carried dependences b...
Definition: LoopAccessAnalysis.h:553
llvm::LoopInfo
Definition: LoopInfo.h:1086
llvm::OptimizationRemarkEmitter
The optimization diagnostic interface.
Definition: OptimizationRemarkEmitter.h:33
llvm::LoopVectorizationLegality::isConsecutivePtr
int isConsecutivePtr(Type *AccessTy, Value *Ptr) const
Check if this pointer is consecutive when vectorizing.
Definition: LoopVectorizationLegality.cpp:442
llvm::StringRef
StringRef - Represent a constant reference to a string, i.e.
Definition: StringRef.h:58
llvm::AssumptionCache
A cache of @llvm.assume calls within a function.
Definition: AssumptionCache.h:42
llvm::LoopVectorizeHints::ScalableForceKind
ScalableForceKind
Definition: LoopVectorizationLegality.h:100
llvm::LoopVectorizeHints::emitRemarkWithHints
void emitRemarkWithHints() const
Dumps all the hint information.
Definition: LoopVectorizationLegality.cpp:205
llvm::LoopVectorizationLegality::getRuntimePointerChecking
const RuntimePointerChecking * getRuntimePointerChecking() const
Returns the information that we collected about runtime memory check.
Definition: LoopVectorizationLegality.h:353
llvm::LoopVectorizationLegality::getIntOrFpInductionDescriptor
const InductionDescriptor * getIntOrFpInductionDescriptor(PHINode *Phi) const
Returns a pointer to the induction descriptor, if Phi is an integer or floating point induction.
Definition: LoopVectorizationLegality.cpp:962
llvm::LoopAccessInfo::getMaxSafeDepDistBytes
uint64_t getMaxSafeDepDistBytes() const
Definition: LoopAccessAnalysis.h:543
llvm::LoopVectorizationLegality::isReductionVariable
bool isReductionVariable(PHINode *PN) const
Returns True if PN is a reduction variable in this loop.
Definition: LoopVectorizationLegality.h:315
llvm::LoopVectorizationLegality::isUniformMemOp
bool isUniformMemOp(Instruction &I)
A uniform memory op is a load or store which accesses the same memory location on all lanes.
Definition: LoopVectorizationLegality.h:341
llvm::LoopAccessInfo::getNumStores
unsigned getNumStores() const
Definition: LoopAccessAnalysis.h:544
llvm::MapVector::count
size_type count(const KeyT &Key) const
Definition: MapVector.h:142
llvm::GraphProgram::Name
Name
Definition: GraphWriter.h:50
llvm::LoopVectorizationLegality::isFirstOrderRecurrence
bool isFirstOrderRecurrence(const PHINode *Phi) const
Returns True if Phi is a first-order recurrence in this loop.
Definition: LoopVectorizationLegality.cpp:982
llvm::MemoryDepChecker::isSafeForAnyVectorWidth
bool isSafeForAnyVectorWidth() const
Return true if the number of elements that are safe to operate on simultaneously is not bounded.
Definition: LoopAccessAnalysis.h:196
H
#define H(x, y, z)
Definition: MD5.cpp:57
llvm::LoopVectorizeHints::ForceKind
ForceKind
Definition: LoopVectorizationLegality.h:94
llvm::TargetStackID::Value
Value
Definition: TargetFrameLowering.h:27
llvm::LoopVectorizationLegality::getMaxSafeVectorWidthInBits
uint64_t getMaxSafeVectorWidthInBits() const
Definition: LoopVectorizationLegality.h:365
llvm::TargetLibraryInfo
Provides information about what library functions are available for the current target.
Definition: TargetLibraryInfo.h:221
llvm::LoopVectorizationLegality::isInductionPhi
bool isInductionPhi(const Value *V) const
Returns True if V is a Phi node of an induction variable in this loop.
Definition: LoopVectorizationLegality.cpp:952
llvm::LoopVectorizeHints::setAlreadyVectorized
void setAlreadyVectorized()
Mark the loop L as already vectorized by setting the width to 1.
Definition: LoopVectorizationLegality.cpp:153
llvm::LoopVectorizeHints::isScalableVectorizationDisabled
bool isScalableVectorizationDisabled() const
Definition: LoopVectorizationLegality.h:148
TargetTransformInfo.h
llvm::LoopVectorizationLegality::isSafeForAnyVectorWidth
bool isSafeForAnyVectorWidth() const
Definition: LoopVectorizationLegality.h:359
llvm::PHINode
Definition: Instructions.h:2657
llvm::LoopVectorizeHints::allowReordering
bool allowReordering() const
When enabling loop hints are provided we allow the vectorizer to change the order of operations that ...
Definition: LoopVectorizationLegality.cpp:241
llvm::SmallPtrSetImpl< Instruction * >
llvm::getLoadStorePointerOperand
const Value * getLoadStorePointerOperand(const Value *V)
A helper function that returns the pointer operand of a load or store instruction.
Definition: Instructions.h:5319
llvm::LoopVectorizationLegality::getInductionVars
const InductionList & getInductionVars() const
Returns the induction variables found in the loop.
Definition: LoopVectorizationLegality.h:286
BB
Common register allocation spilling lr str ldr sxth r3 ldr mla r4 can lr mov lr str ldr sxth r3 mla r4 and then merge mul and lr str ldr sxth r3 mla r4 It also increase the likelihood the store may become dead bb27 Successors according to LLVM BB
Definition: README.txt:39
llvm::LoopVectorizationLegality::getPrimaryInduction
PHINode * getPrimaryInduction()
Returns the primary induction variable.
Definition: LoopVectorizationLegality.h:280
llvm::LoopVectorizationLegality::getMaxSafeDepDistBytes
unsigned getMaxSafeDepDistBytes()
Definition: LoopVectorizationLegality.h:363
llvm::LoopVectorizeHints::FK_Disabled
@ FK_Disabled
Forcing disabled.
Definition: LoopVectorizationLegality.h:96
llvm::LoopVectorizationLegality::canVectorize
bool canVectorize(bool UseVPlanNativePath)
Returns true if it is legal to vectorize this loop.
Definition: LoopVectorizationLegality.cpp:1192
llvm::Value
LLVM Value Representation.
Definition: Value.h:74
llvm::LoopVectorizationLegality::getWidestInductionType
Type * getWidestInductionType()
Returns the widest induction type.
Definition: LoopVectorizationLegality.h:295
llvm::LoopVectorizeHints::isPotentiallyUnsafe
bool isPotentiallyUnsafe() const
Definition: LoopVectorizationLegality.h:163
llvm::Intrinsic::ID
unsigned ID
Definition: TargetTransformInfo.h:38
llvm::LoopVectorizationLegality::InductionList
MapVector< PHINode *, InductionDescriptor > InductionList
InductionList saves induction variables and maps them to the induction descriptor.
Definition: LoopVectorizationLegality.h:254