Go to the documentation of this file.
29 using namespace PatternMatch;
31 #define LV_NAME "loop-vectorize"
32 #define DEBUG_TYPE LV_NAME
38 cl::desc(
"Enable if-conversion during vectorization."));
44 cl::desc(
"The maximum number of SCEV checks allowed."));
48 cl::desc(
"The maximum number of SCEV checks allowed with a "
49 "vectorize(enable) pragma"));
67 return (Val == 0 || Val == 1);
73 bool InterleaveOnlyWhenForced,
76 Interleave(
"interleave.count", InterleaveOnlyWhenForced, HK_UNROLL),
77 Force(
"vectorize.enable", FK_Undefined, HK_FORCE),
78 IsVectorized(
"isvectorized", 0, HK_ISVECTORIZED),
79 Predicate(
"vectorize.predicate.enable", FK_Undefined, HK_PREDICATE),
80 Scalable(
"vectorize.scalable.enable",
false, HK_SCALABLE), TheLoop(L),
83 getHintsFromMetadata();
89 if (IsVectorized.Value != 1)
95 LLVM_DEBUG(
if (InterleaveOnlyWhenForced && Interleave.Value == 1)
dbgs()
96 <<
"LV: Interleaving disabled by the pass manager\n");
109 {
Twine(Prefix(),
"vectorize.").
str(),
110 Twine(Prefix(),
"interleave.").
str()},
115 IsVectorized.Value = 1;
121 LLVM_DEBUG(
dbgs() <<
"LV: Not vectorizing: #pragma vectorize disable.\n");
127 LLVM_DEBUG(
dbgs() <<
"LV: Not vectorizing: No #pragma vectorize enable.\n");
133 LLVM_DEBUG(
dbgs() <<
"LV: Not vectorizing: Disabled/already vectorized.\n");
141 <<
"loop not vectorized: vectorization and interleaving are "
142 "explicitly disabled, or the loop has already been "
157 TheLoop->getStartLoc(),
158 TheLoop->getHeader())
159 <<
"loop not vectorized: vectorization is explicitly disabled";
161 OptimizationRemarkMissed R(LV_NAME,
"MissedDetails",
162 TheLoop->getStartLoc(), TheLoop->getHeader());
163 R <<
"loop not vectorized";
164 if (Force.Value == LoopVectorizeHints::FK_Enabled) {
165 R <<
" (Force=" << NV(
"Force", true);
166 if (Width.Value != 0)
167 R <<
", Vector Width=" << NV(
"VectorWidth", getWidth());
168 if (Interleave.Value != 0)
169 R <<
", Interleave Count=" << NV(
"InterleaveCount", Interleave.Value);
187 void LoopVectorizeHints::getHintsFromMetadata() {
203 if (!MD || MD->getNumOperands() == 0)
205 S = dyn_cast<MDString>(MD->getOperand(0));
206 for (
unsigned i = 1, ie = MD->getNumOperands();
i < ie; ++
i)
207 Args.push_back(MD->getOperand(
i));
210 assert(
Args.size() == 0 &&
"too many arguments for MDString");
218 if (
Args.size() == 1)
224 if (!
Name.startswith(Prefix()))
231 unsigned Val =
C->getZExtValue();
233 Hint *Hints[] = {&Width, &Interleave, &Force,
235 for (
auto H : Hints) {
236 if (
Name ==
H->Name) {
237 if (
H->validate(Val))
286 auto *LatchBr = dyn_cast<BranchInst>(Latch->
getTerminator());
287 if (!LatchBr || LatchBr->isUnconditional()) {
293 auto *LatchCmp = dyn_cast<CmpInst>(LatchBr->getCondition());
296 dbgs() <<
"LV: Loop latch condition is not a compare instruction.\n");
300 Value *CondOp0 = LatchCmp->getOperand(0);
301 Value *CondOp1 = LatchCmp->getOperand(1);
305 LLVM_DEBUG(
dbgs() <<
"LV: Loop latch condition is not uniform.\n");
319 for (
Loop *SubLp : *Lp)
332 for (
Value *V : Phi.incoming_values())
333 if (
auto *
C = dyn_cast<Constant>(V))
342 return DL.getIntPtrType(Ty);
366 if (!AllowedExit.
count(Inst))
372 LLVM_DEBUG(
dbgs() <<
"LV: Found an outside user for : " << *UI <<
'\n');
384 bool OptForSize =
F->hasOptSize() ||
387 bool CanAddPredicate = !OptForSize;
388 int Stride =
getPtrStride(PSE, Ptr, TheLoop, Strides, CanAddPredicate,
false);
389 if (Stride == 1 || Stride == -1)
398 bool LoopVectorizationLegality::canVectorizeOuterLoop() {
408 auto *Br = dyn_cast<BranchInst>(
BB->getTerminator());
411 "loop control flow is not understood by vectorizer",
412 "CFGNotUnderstood", ORE, TheLoop);
430 "loop control flow is not understood by vectorizer",
431 "CFGNotUnderstood", ORE, TheLoop);
444 "loop control flow is not understood by vectorizer",
445 "CFGNotUnderstood", ORE, TheLoop);
453 if (!setupOuterLoopInductions()) {
455 "Unsupported outer loop Phi(s)",
456 "UnsupportedPhi", ORE, TheLoop);
466 void LoopVectorizationLegality::addInductionPhi(
469 Inductions[Phi] =
ID;
477 InductionCastsToIgnore.
insert(*Casts.begin());
492 ID.getConstIntStepValue() &&
ID.getConstIntStepValue()->isOne() &&
493 isa<Constant>(
ID.getStartValue()) &&
494 cast<Constant>(
ID.getStartValue())->isNullValue()) {
500 if (!PrimaryInduction || PhiTy == WidestIndTy)
501 PrimaryInduction = Phi;
518 bool LoopVectorizationLegality::setupOuterLoopInductions() {
522 auto isSupportedPhi = [&](
PHINode &Phi) ->
bool {
526 addInductionPhi(&Phi,
ID, AllowedExit);
533 <<
"LV: Found unsupported PHI for outer loop vectorization.\n");
560 TLI.
getWidestVF(ScalarName, WidestFixedVF, WidestScalableVF);
568 "Caller may decide to scalarize a variant using a scalable VF");
573 bool LoopVectorizationLegality::canVectorizeInstrs() {
580 if (
auto *Phi = dyn_cast<PHINode>(&
I)) {
586 "loop control flow is not understood by vectorizer",
587 "CFGNotUnderstood", ORE, TheLoop);
607 "loop control flow is not understood by vectorizer",
608 "CFGNotUnderstood", ORE, TheLoop, Phi);
617 Reductions[Phi] = RedDes;
637 addInductionPhi(Phi,
ID, AllowedExit);
645 FirstOrderRecurrences.
insert(Phi);
652 addInductionPhi(Phi,
ID, AllowedExit);
657 "value that could not be identified as "
658 "reduction is used outside the loop",
659 "NonReductionValueUsedOutsideLoop", ORE, TheLoop, Phi);
667 auto *CI = dyn_cast<CallInst>(&
I);
670 !isa<DbgInfoIntrinsic>(CI) &&
671 !(CI->getCalledFunction() && TLI &&
678 TLI && CI->getCalledFunction() &&
679 CI->getType()->isFloatingPointTy() &&
680 TLI->
getLibFunc(CI->getCalledFunction()->getName(), Func) &&
689 "Found a non-intrinsic callsite",
690 "library call cannot be vectorized. "
691 "Try compiling with -fno-math-errno, -ffast-math, "
693 "CantVectorizeLibcall", ORE, TheLoop, CI);
696 "call instruction cannot be vectorized",
697 "CantVectorizeLibcall", ORE, TheLoop, CI);
705 auto *SE = PSE.
getSE();
707 for (
unsigned i = 0,
e = CI->getNumArgOperands();
i !=
e; ++
i)
709 if (!SE->isLoopInvariant(PSE.
getSCEV(CI->getOperand(
i)), TheLoop)) {
711 "intrinsic instruction cannot be vectorized",
712 "CantVectorizeIntrinsic", ORE, TheLoop, CI);
721 !
I.getType()->isVoidTy()) ||
722 isa<ExtractElementInst>(
I)) {
724 "instruction return type cannot be vectorized",
725 "CantVectorizeInstructionReturnType", ORE, TheLoop, &
I);
730 if (
auto *
ST = dyn_cast<StoreInst>(&
I)) {
731 Type *
T =
ST->getValueOperand()->getType();
734 "store instruction cannot be vectorized",
735 "CantVectorizeStore", ORE, TheLoop,
ST);
741 if (
ST->getMetadata(LLVMContext::MD_nontemporal)) {
744 assert(VecTy &&
"did not find vectorized version of stored type");
747 "nontemporal store instruction cannot be vectorized",
748 "nontemporal store instruction cannot be vectorized",
749 "CantVectorizeNontemporalStore", ORE, TheLoop,
ST);
754 }
else if (
auto *
LD = dyn_cast<LoadInst>(&
I)) {
755 if (
LD->getMetadata(LLVMContext::MD_nontemporal)) {
759 assert(VecTy &&
"did not find vectorized version of load type");
762 "nontemporal load instruction cannot be vectorized",
763 "nontemporal load instruction cannot be vectorized",
764 "CantVectorizeNontemporalLoad", ORE, TheLoop,
LD);
774 }
else if (
I.getType()->isFloatingPointTy() && (CI ||
I.isBinaryOp()) &&
777 Hints->setPotentiallyUnsafe();
792 "value cannot be used outside the loop",
793 "ValueUsedOutsideLoop", ORE, TheLoop, &
I);
799 if (!PrimaryInduction) {
800 if (Inductions.
empty()) {
802 "loop induction variable could not be identified",
803 "NoInductionVariable", ORE, TheLoop);
805 }
else if (!WidestIndTy) {
807 "integer loop induction variable could not be identified",
808 "NoIntegerInductionVariable", ORE, TheLoop);
811 LLVM_DEBUG(
dbgs() <<
"LV: Did not find one integer induction var.\n");
820 if (
any_of(FirstOrderRecurrences, [LoopLatch,
this](
const PHINode *Phi) {
823 return SinkAfter.find(V) != SinkAfter.end();
830 if (PrimaryInduction && WidestIndTy != PrimaryInduction->
getType())
831 PrimaryInduction =
nullptr;
836 bool LoopVectorizationLegality::canVectorizeMemory() {
837 LAI = &(*GetLAA)(*TheLoop);
842 "loop not vectorized: ", *LAR);
850 "write to a loop invariant address could not be vectorized",
851 "CantVectorizeStoreToLoopInvariantAddress", ORE, TheLoop);
862 PHINode *PN = dyn_cast_or_null<PHINode>(In0);
866 return Inductions.
count(PN);
870 auto *Inst = dyn_cast<Instruction>(V);
871 return (Inst && InductionCastsToIgnore.
count(Inst));
879 return FirstOrderRecurrences.
count(Phi);
886 bool LoopVectorizationLegality::blockCanBePredicated(
890 bool PreserveGuards)
const {
891 const bool IsAnnotatedParallel = TheLoop->isAnnotatedParallel();
895 for (
Value *Operand :
I.operands()) {
896 if (
auto *
C = dyn_cast<Constant>(Operand))
903 if (
match(&
I, m_Intrinsic<Intrinsic::assume>())) {
911 if (isa<NoAliasScopeDeclInst>(&
I))
915 if (
I.mayReadFromMemory()) {
916 auto *LI = dyn_cast<LoadInst>(&
I);
919 if (!SafePtrs.
count(LI->getPointerOperand())) {
923 if (!IsAnnotatedParallel || PreserveGuards)
929 if (
I.mayWriteToMemory()) {
930 auto *
SI = dyn_cast<StoreInst>(&
I);
948 bool LoopVectorizationLegality::canVectorizeWithIfConvert() {
951 "if-conversion is disabled",
952 "IfConversionDisabled",
957 assert(TheLoop->getNumBlocks() > 1 &&
"Single block loops are vectorizable");
993 if (!isa<BranchInst>(
BB->getTerminator())) {
995 "loop contains a switch statement",
996 "LoopContainsSwitch", ORE, TheLoop,
997 BB->getTerminator());
1003 if (!blockCanBePredicated(
BB, SafePointers, MaskedOp,
1004 ConditionalAssumes)) {
1006 "Control flow cannot be substituted for a select",
1007 "control flow cannot be substituted for a select",
1008 "NoCFGForSelect", ORE, TheLoop,
1009 BB->getTerminator());
1014 "Control flow cannot be substituted for a select",
1015 "control flow cannot be substituted for a select",
1016 "NoCFGForSelect", ORE, TheLoop,
1017 BB->getTerminator());
1027 bool LoopVectorizationLegality::canVectorizeLoopCFG(
Loop *Lp,
1028 bool UseVPlanNativePath) {
1030 "VPlan-native path is not enabled.");
1046 "loop control flow is not understood by vectorizer",
1047 "CFGNotUnderstood", ORE, TheLoop);
1048 if (DoExtraAnalysis)
1057 "loop control flow is not understood by vectorizer",
1058 "CFGNotUnderstood", ORE, TheLoop);
1059 if (DoExtraAnalysis)
1073 "loop control flow is not understood by vectorizer",
1074 "CFGNotUnderstood", ORE, TheLoop);
1075 if (DoExtraAnalysis)
1083 bool LoopVectorizationLegality::canVectorizeLoopNestCFG(
1084 Loop *Lp,
bool UseVPlanNativePath) {
1089 if (!canVectorizeLoopCFG(Lp, UseVPlanNativePath)) {
1090 if (DoExtraAnalysis)
1098 for (
Loop *SubLp : *Lp)
1099 if (!canVectorizeLoopNestCFG(SubLp, UseVPlanNativePath)) {
1100 if (DoExtraAnalysis)
1117 if (!canVectorizeLoopNestCFG(TheLoop, UseVPlanNativePath)) {
1118 if (DoExtraAnalysis)
1125 LLVM_DEBUG(
dbgs() <<
"LV: Found a loop: " << TheLoop->getHeader()->getName()
1130 if (!TheLoop->isInnermost()) {
1131 assert(UseVPlanNativePath &&
"VPlan-native path is not enabled.");
1133 if (!canVectorizeOuterLoop()) {
1135 "unsupported outer loop",
1136 "UnsupportedOuterLoop",
1147 assert(TheLoop->isInnermost() &&
"Inner loop expected.");
1149 unsigned NumBlocks = TheLoop->getNumBlocks();
1150 if (NumBlocks != 1 && !canVectorizeWithIfConvert()) {
1152 if (DoExtraAnalysis)
1159 if (!canVectorizeInstrs()) {
1160 LLVM_DEBUG(
dbgs() <<
"LV: Can't vectorize the instructions or CFG\n");
1161 if (DoExtraAnalysis)
1168 if (!canVectorizeMemory()) {
1169 LLVM_DEBUG(
dbgs() <<
"LV: Can't vectorize due to memory conflicts\n");
1170 if (DoExtraAnalysis)
1178 ?
" (with a runtime bound check)"
1188 "Too many SCEV assumptions need to be made and checked at runtime",
1189 "TooManySCEVRunTimeChecks", ORE, TheLoop);
1190 if (DoExtraAnalysis)
1205 LLVM_DEBUG(
dbgs() <<
"LV: checking if tail can be folded by masking.\n");
1213 for (
auto *AE : AllowedExit) {
1216 if (ReductionLiveOuts.
count(AE))
1220 if (TheLoop->contains(UI))
1224 <<
"LV: Cannot fold tail by masking, loop has an outside user for "
1239 if (!blockCanBePredicated(
BB, SafePointers, TmpMaskedOp,
1240 TmpConditionalAssumes,
1242 LLVM_DEBUG(
dbgs() <<
"LV: Cannot fold tail by masking as requested.\n");
1250 ConditionalAssumes.
insert(TmpConditionalAssumes.
begin(),
1251 TmpConditionalAssumes.
end());
bool mustSuppressSpeculation(const LoadInst &LI)
Return true if speculation of the given load must be suppressed to avoid ordering or interfering with...
bool isLoopInvariant(const Value *V) const
Return true if the specified value is loop invariant.
int isConsecutivePtr(Value *Ptr)
Check if this pointer is consecutive when vectorizing.
BlockT * getUniqueExitBlock() const
If getUniqueExitBlocks would return exactly one block, return that block.
const Module * getModule() const
Return the module owning the function this instruction belongs to or nullptr it the function does not...
A parsed version of the target data layout string in and methods for querying it.
bool isUniform(Value *V) const
Returns true if the value V is uniform within the loop.
static bool isReductionPHI(PHINode *Phi, Loop *TheLoop, RecurrenceDescriptor &RedDes, DemandedBits *DB=nullptr, AssumptionCache *AC=nullptr, DominatorTree *DT=nullptr)
Returns true if Phi is a reduction in TheLoop.
const Function * getParent() const
Return the enclosing method, or null if none.
bool isPointerTy() const
True if this is an instance of PointerType.
static cl::opt< bool > EnableIfConversion("enable-if-conversion", cl::init(true), cl::Hidden, cl::desc("Enable if-conversion during vectorization."))
Intrinsic::ID getVectorIntrinsicIDForCall(const CallInst *CI, const TargetLibraryInfo *TLI)
Returns intrinsic ID for call.
Represents a single loop in the control flow graph.
bool contains(const LoopT *L) const
Return true if the specified loop is contained within in this loop.
static constexpr size_t npos
static bool isKnownLE(const LinearPolySize &LHS, const LinearPolySize &RHS)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
bool blockNeedsPredication(BasicBlock *BB)
Return true if the block BB needs to be predicated in order for the loop to be vectorized.
void addRuntimePointerChecks(unsigned Num)
DebugLoc getStartLoc() const
Return the debug location of the start of this loop.
bool isFunctionVectorizable(StringRef F, const ElementCount &VF) const
const SCEVUnionPredicate & getUnionPredicate() const
The main scalar evolution driver.
static unsigned VectorizationInterleave
Interleave factor as overridden by the user.
bool isInductionVariable(const Value *V)
Returns True if V can be considered as an induction variable in this loop.
static cl::opt< unsigned > VectorizeSCEVCheckThreshold("vectorize-scev-check-threshold", cl::init(16), cl::Hidden, cl::desc("The maximum number of SCEV checks allowed."))
@ IK_IntInduction
Integer induction variable. Step = C.
The instances of the Type class are immutable: once they are created, they are never changed.
ElementCount getWidth() const
unsigned getIsVectorized() const
SmallPtrSet - This class implements a set which is optimized for holding SmallSize or less elements.
void reportVectorizationFailure(const StringRef DebugMsg, const StringRef OREMsg, const StringRef ORETag, OptimizationRemarkEmitter *ORE, Loop *TheLoop, Instruction *I=nullptr)
Reports a vectorization failure: print DebugMsg for debugging purposes along with the corresponding o...
Value * getPointerOperand()
constexpr bool isPowerOf2_32(uint32_t Value)
Return true if the argument is a power of two > 0.
static bool isUniformLoopNest(Loop *Lp, Loop *OuterLp)
void validate(const Triple &TT, const FeatureBitset &FeatureBits)
bool isInductionPhi(const Value *V)
Returns True if V is a Phi node of an induction variable in this loop.
bool prepareToFoldTailByMasking()
Return true if we can vectorize this loop while folding its tail by masking, and mark all respective ...
bool isFloatingPointTy() const
Return true if this is one of the six floating-point types.
static IntegerType * getInt32Ty(LLVMContext &C)
static MDTuple * get(LLVMContext &Context, ArrayRef< Metadata * > MDs)
LLVM Basic Block Representation.
unsigned getNumOperands() const
Return number of MDNode operands.
llvm::MDNode * makePostTransformationMetadata(llvm::LLVMContext &Context, MDNode *OrigLoopID, llvm::ArrayRef< llvm::StringRef > RemovePrefixes, llvm::ArrayRef< llvm::MDNode * > AddAttrs)
Create a new LoopID after the loop has been transformed.
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
@ FK_Undefined
Not selected.
amdgpu Simplify well known AMD library false FunctionCallee Value * Arg
This is the shared class of boolean and integer constants.
bool all_of(R &&range, UnaryPredicate P)
Provide wrappers to std::all_of which take ranges instead of having to pass begin/end explicitly.
bool shouldOptimizeForSize(const MachineFunction *MF, ProfileSummaryInfo *PSI, const MachineBlockFrequencyInfo *BFI, PGSOQueryType QueryType=PGSOQueryType::Other)
Returns true if machine function MF is suggested to be size-optimized based on the profile.
A struct for saving information about induction variables.
bool hasDependenceInvolvingLoopInvariantAddress() const
If the loop has memory dependence involving an invariant address, i.e.
static const unsigned MaxInterleaveFactor
Maximum vectorization interleave count.
bool match(Val *V, const Pattern &P)
static bool isZero(Value *V, const DataLayout &DL, DominatorTree *DT, AssumptionCache *AC)
static bool blockNeedsPredication(BasicBlock *BB, Loop *TheLoop, DominatorTree *DT)
Return true if the block BB needs to be predicated in order for the loop to be vectorized.
DenseMap< const Value *, Value * > ValueToValueMap
(vector float) vec_cmpeq(*A, *B) C
static bool isUniformLoop(Loop *Lp, Loop *OuterLp)
Instruction * getExactFPMathInst() const
Returns 1st non-reassociative FP instruction in the PHI node's use-chain.
Function * getCalledFunction() const
Returns the function called, or null if this is an indirect function invocation.
const char * vectorizeAnalysisPassName() const
If hints are provided that force vectorization, use the AlwaysPrint pass name to force the frontend t...
int64_t getPtrStride(PredicatedScalarEvolution &PSE, Value *Ptr, const Loop *Lp, const ValueToValueMap &StridesMap=ValueToValueMap(), bool Assume=false, bool ShouldCheckWrap=true)
If the pointer has a constant stride return it in units of its element size.
void addExactFPMathInst(Instruction *I)
Track the 1st floating-point instruction that can not be reassociated.
iterator_range< block_iterator > blocks() const
bool isVectorTy() const
True if this is an instance of VectorType.
cl::opt< bool > EnableVPlanPredication
Value * getIncomingValueForBlock(const BasicBlock *BB) const
bool getLibFunc(StringRef funcName, LibFunc &F) const
Searches for a particular function name.
unsigned getScalarSizeInBits() const LLVM_READONLY
If this is a vector type, return the getPrimitiveSizeInBits value for the element type.
enum ForceKind getForce() const
iterator_range< const_phi_iterator > phis() const
Returns a range that iterates over the phis in the basic block.
static Constant * get(Type *Ty, uint64_t V, bool IsSigned=false)
If Ty is a vector type, return a Constant with a splat of the given value.
bool isFirstOrderRecurrence(const PHINode *Phi)
Returns True if Phi is a first-order recurrence in this loop.
static FixedVectorType * get(Type *ElementType, unsigned NumElts)
unsigned getNumIncomingValues() const
Return the number of incoming edges.
static ElementCount getFixed(ScalarTy MinVal)
std::string str() const
Return the twine contents as a std::string.
bool isIntegerTy() const
True if this is an instance of IntegerType.
const MDOperand & getOperand(unsigned I) const
static bool isValidElementType(Type *ElemTy)
Return true if the specified type is valid as a element type.
static bool canIfConvertPHINodes(BasicBlock *BB)
Check whether it is safe to if-convert this phi node.
unsigned getComplexity() const override
We estimate the complexity of a union predicate as the size number of predicates in the union.
bool Need
This flag indicates if we need to add the runtime check.
static SmallVector< VFInfo, 8 > getMappings(const CallInst &CI)
Retrieve all the VFInfo instances associated to the CallInst CI.
@ FK_Enabled
Forcing enabled.
bool isCastedInductionVariable(const Value *V)
Returns True if V is a cast that is part of an induction def-use chain, and had been proven to be red...
bool allowVectorization(Function *F, Loop *L, bool VectorizeOnlyWhenForced) const
This is an important class for using LLVM in a threaded context.
const RuntimePointerChecking * getRuntimePointerChecking() const
bool hasVectorInstrinsicScalarOpd(Intrinsic::ID ID, unsigned ScalarOpdIdx)
Identifies if the vector form of the intrinsic has a scalar operand.
initializer< Ty > init(const Ty &Val)
BlockT * getLoopPreheader() const
If there is a preheader for this loop, return it.
static const unsigned MaxVectorWidth
Maximum SIMD width.
const OptimizationRemarkAnalysis * getReport() const
The diagnostics report generated for the analysis.
static MDString * get(LLVMContext &Context, StringRef Str)
BlockT * getLoopLatch() const
If there is a single latch block for this loop, return it.
ReductionList & getReductionVars()
Returns the reduction variables found in the loop.
static bool isFirstOrderRecurrence(PHINode *Phi, Loop *TheLoop, DenseMap< Instruction *, Instruction * > &SinkAfter, DominatorTree *DT)
Returns true if Phi is a first-order recurrence.
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
const SCEV * getSCEV(Value *V)
Returns the SCEV expression of V, in the context of the current SCEV predicate.
bool canVectorizeMemory() const
Return true we can analyze the memory accesses in the loop and there are no memory dependence cycles.
static bool isTLIScalarize(const TargetLibraryInfo &TLI, const CallInst &CI)
Checks if a function is scalarizable according to the TLI, in the sense that it should be vectorized ...
bool isDereferenceableAndAlignedInLoop(LoadInst *LI, Loop *L, ScalarEvolution &SE, DominatorTree &DT)
Return true if we can prove that the given load (which is assumed to be within the specified loop) wo...
size_type count(ConstPtrType Ptr) const
count - Return 1 if the specified pointer is in the set, 0 otherwise.
bool isUniform(Value *V)
Returns true if the value V is uniform within the loop.
Class for arbitrary precision integers.
auto size(R &&Range, std::enable_if_t< std::is_base_of< std::random_access_iterator_tag, typename std::iterator_traits< decltype(Range.begin())>::iterator_category >::value, void > *=nullptr)
Get the size of a range.
bool isAlwaysTrue() const override
Implementation of the SCEVPredicate interface.
bool any_of(R &&range, UnaryPredicate P)
Provide wrappers to std::any_of which take ranges instead of having to pass begin/end explicitly.
StringRef - Represent a constant reference to a string, i.e.
PHINode * getCanonicalInductionVariable() const
Check to see if the loop has a canonical induction variable: an integer recurrence that starts at 0 a...
Type * getType() const
All values are typed, get the type of this value.
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
add sub stmia L5 ldr r0 bl L_printf $stub Instead of a and a wouldn t it be better to do three moves *Return an aggregate type is even return S
void emitRemarkWithHints() const
Dumps all the hint information.
static Type * convertPointerToIntegerType(const DataLayout &DL, Type *Ty)
StringRef getName() const
Return a constant reference to the value's name.
LLVMContext & getContext() const
Return the LLVMContext in which this type was uniqued.
An instruction for reading from memory.
const Instruction * getTerminator() const LLVM_READONLY
Returns the terminator instruction if the block is well formed or null if the block is not well forme...
static bool isInterleaveForced()
True if force-vector-interleave was specified by the user.
LLVMContext & getContext() const
Get the context in which this basic block lives.
bool isInnermost() const
Return true if the loop does not contain any (natural) loops.
static bool isInductionPHI(PHINode *Phi, const Loop *L, ScalarEvolution *SE, InductionDescriptor &D, const SCEV *Expr=nullptr, SmallVectorImpl< Instruction * > *CastsToIgnore=nullptr)
Returns true if Phi is an induction in the loop L.
static cl::opt< unsigned > PragmaVectorizeSCEVCheckThreshold("pragma-vectorize-scev-check-threshold", cl::init(128), cl::Hidden, cl::desc("The maximum number of SCEV checks allowed with a " "vectorize(enable) pragma"))
bool isLoopHeader(const BlockT *BB) const
Twine - A lightweight data structure for efficiently representing the concatenation of temporary valu...
size_type count(const KeyT &Key) const
void setLoopID(MDNode *LoopID) const
Set the llvm.loop loop id metadata for this loop.
TODO: The following VectorizationFactor was pulled out of LoopVectorizationCostModel class.
MDNode * getLoopID() const
Return the llvm.loop loop id metadata node for this loop if it is present.
BlockT * getHeader() const
static ElementCount getScalable(ScalarTy MinVal)
Provides information about what library functions are available for the current target.
unsigned getNumRuntimePointerChecks() const
Number of memchecks required to prove independence of otherwise may-alias pointers.
void addPredicate(const SCEVPredicate &Pred)
Adds a new predicate.
unsigned getNumBackEdges() const
Calculate the number of back edges to the loop header.
LoopVectorizeHints(const Loop *L, bool InterleaveOnlyWhenForced, OptimizationRemarkEmitter &ORE)
The RecurrenceDescriptor is used to identify recurrences variables in a loop.
void setAlreadyVectorized()
Mark the loop L as already vectorized by setting the width to 1.
Collection of parameters shared beetween the Loop Vectorizer and the Loop Access Analysis.
void getWidestVF(StringRef ScalarF, ElementCount &FixedVF, ElementCount &ScalableVF) const
Returns the largest vectorization factor used in the list of vector functions.
const DataLayout & getDataLayout() const
Get the data layout for the module's target platform.
const Value * getLoadStorePointerOperand(const Value *V)
A helper function that returns the pointer operand of a load or store instruction.
This class represents a function call, abstracting a target machine's calling convention.
Common register allocation spilling lr str ldr sxth r3 ldr mla r4 can lr mov lr str ldr sxth r3 mla r4 and then merge mul and lr str ldr sxth r3 mla r4 It also increase the likelihood the store may become dead bb27 Successors according to LLVM BB
static Type * getWiderType(const DataLayout &DL, Type *Ty0, Type *Ty1)
bool hasOptimizedCodeGen(LibFunc F) const
Tests if the function is both available and a candidate for optimized code generation.
Instruction * getLoopExitInstr() const
constexpr char Args[]
Key for Kernel::Metadata::mArgs.
static bool hasOutsideLoopUser(const Loop *TheLoop, Instruction *Inst, SmallPtrSetImpl< Value * > &AllowedExit)
Check that the instruction has outside loop users and is not an identified reduction variable.
const PredicatedScalarEvolution & getPSE() const
Used to add runtime SCEV checks.
ScalarEvolution * getSE() const
Returns the ScalarEvolution analysis used.
@ FK_Disabled
Forcing disabled.
loop Loop Strength Reduction
bool canVectorize(bool UseVPlanNativePath)
Returns true if it is legal to vectorize this loop.
LLVM Value Representation.
iterator_range< user_iterator > users()
std::pair< iterator, bool > insert(PtrType Ptr)
Inserts Ptr if and only if there is no element in the container equal to Ptr.