Go to the documentation of this file.
39 #define LV_NAME "loop-vectorize"
40 #define DEBUG_TYPE LV_NAME
44 case VPWidenMemoryInstructionSC: {
45 return cast<VPWidenMemoryInstructionRecipe>(
this)->isStore();
51 case VPBranchOnMaskSC:
53 case VPWidenIntOrFpInductionSC:
54 case VPWidenCanonicalIVSC:
60 case VPWidenSelectSC: {
64 assert((!
I || !
I->mayWriteToMemory()) &&
65 "underlying instruction may write to memory");
75 case VPWidenMemoryInstructionSC: {
76 return !cast<VPWidenMemoryInstructionRecipe>(
this)->isStore();
81 ->mayReadFromMemory();
82 case VPBranchOnMaskSC:
84 case VPWidenIntOrFpInductionSC:
85 case VPWidenCanonicalIVSC:
91 case VPWidenSelectSC: {
95 assert((!
I || !
I->mayReadFromMemory()) &&
96 "underlying instruction may read from memory");
106 case VPPredInstPHISC:
108 case VPWidenIntOrFpInductionSC:
109 case VPWidenPointerInductionSC:
110 case VPWidenCanonicalIVSC:
116 case VPWidenSelectSC:
117 case VPScalarIVStepsSC: {
121 assert((!
I || !
I->mayHaveSideEffects()) &&
122 "underlying instruction has side-effects");
125 case VPReplicateSC: {
126 auto *R = cast<VPReplicateRecipe>(
this);
127 return R->getUnderlyingInstr()->mayHaveSideEffects();
144 assert(!Parent &&
"Recipe already in some VPBasicBlock");
146 "Insertion position not in any VPBasicBlock");
153 assert(!Parent &&
"Recipe already in some VPBasicBlock");
156 BB.getRecipeList().insert(
I,
this);
160 assert(!Parent &&
"Recipe already in some VPBasicBlock");
162 "Insertion position not in any VPBasicBlock");
192 Builder.SetCurrentDebugLocation(DL);
199 State.
set(
this, V, Part);
207 State.
set(
this, V, Part);
214 State.
set(
this, V, Part);
222 State.
set(
this, V, Part);
234 Intrinsic::get_active_lane_mask, {PredTy, ScalarTC->
getType()},
235 {VIVElem0, ScalarTC},
nullptr, Name);
236 State.
set(
this, Call, Part);
256 State.
set(
this, PartMinus1, Part);
259 State.
set(
this,
Builder.CreateVectorSplice(PartMinus1,
V2, -1, Name),
266 Value *Next =
nullptr;
274 Next =
Builder.CreateAdd(Phi, Step, Name, IsNUW,
false);
276 Next = State.
get(
this, 0);
279 State.
set(
this, Next, Part);
288 State.
set(
this,
IV, Part);
296 State.
set(
this, Next, Part);
317 Builder.GetInsertBlock()->getTerminator()->eraseFromParent();
341 Builder.GetInsertBlock()->getTerminator()->eraseFromParent();
353 for (
unsigned Part = 0; Part < State.
UF; ++Part)
354 generateInstruction(State, Part);
357 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
365 O << Indent <<
"EMIT ";
380 O <<
"combined load";
383 O <<
"combined store";
386 O <<
"active lane mask";
389 O <<
"first-order splice";
395 O <<
"VF * UF +(nuw) ";
398 O <<
"branch-on-cond";
404 O <<
"VF * Part +(nuw) ";
407 O <<
"branch-on-count ";
429 assert((Opcode == Instruction::FAdd || Opcode == Instruction::FMul ||
430 Opcode == Instruction::FNeg || Opcode == Instruction::FSub ||
431 Opcode == Instruction::FDiv || Opcode == Instruction::FRem ||
432 Opcode == Instruction::FCmp) &&
433 "this op can't take fast-math flags");
437 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
440 O << Indent <<
"WIDEN-CALL ";
443 if (CI->getType()->isVoidTy())
450 O <<
"call @" << CI->getCalledFunction()->getName() <<
"(";
457 O << Indent <<
"WIDEN-SELECT ";
465 O << (InvariantCond ?
" (condition is loop invariant)" :
"");
480 for (
unsigned Part = 0; Part < State.
UF; ++Part) {
485 State.
set(
this, Sel, Part);
493 switch (
I.getOpcode()) {
495 case Instruction::Br:
496 case Instruction::PHI:
497 case Instruction::GetElementPtr:
500 case Instruction::UDiv:
501 case Instruction::SDiv:
502 case Instruction::SRem:
503 case Instruction::URem:
505 case Instruction::FAdd:
506 case Instruction::Sub:
507 case Instruction::FSub:
508 case Instruction::FNeg:
510 case Instruction::FMul:
511 case Instruction::FDiv:
512 case Instruction::FRem:
513 case Instruction::Shl:
514 case Instruction::LShr:
515 case Instruction::AShr:
516 case Instruction::And:
517 case Instruction::Or:
518 case Instruction::Xor: {
522 for (
unsigned Part = 0; Part < State.
UF; ++Part) {
525 Ops.push_back(State.
get(VPOp, Part));
529 if (
auto *VecOp = dyn_cast<Instruction>(V)) {
530 VecOp->copyIRFlags(&
I);
538 VecOp->dropPoisonGeneratingFlags();
542 State.
set(
this, V, Part);
548 case Instruction::Freeze: {
551 for (
unsigned Part = 0; Part < State.
UF; ++Part) {
555 State.
set(
this, Freeze, Part);
559 case Instruction::ICmp:
560 case Instruction::FCmp: {
562 bool FCmp = (
I.getOpcode() == Instruction::FCmp);
563 auto *Cmp = cast<CmpInst>(&
I);
565 for (
unsigned Part = 0; Part < State.
UF; ++Part) {
572 Builder.setFastMathFlags(Cmp->getFastMathFlags());
573 C =
Builder.CreateFCmp(Cmp->getPredicate(), A,
B);
575 C =
Builder.CreateICmp(Cmp->getPredicate(), A,
B);
577 State.
set(
this,
C, Part);
584 case Instruction::ZExt:
585 case Instruction::SExt:
586 case Instruction::FPToUI:
587 case Instruction::FPToSI:
588 case Instruction::FPExt:
589 case Instruction::PtrToInt:
590 case Instruction::IntToPtr:
591 case Instruction::SIToFP:
592 case Instruction::UIToFP:
593 case Instruction::Trunc:
594 case Instruction::FPTrunc:
595 case Instruction::BitCast: {
596 auto *CI = cast<CastInst>(&
I);
604 for (
unsigned Part = 0; Part < State.
UF; ++Part) {
606 Value *Cast =
Builder.CreateCast(CI->getOpcode(), A, DestTy);
607 State.
set(
this, Cast, Part);
618 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
621 O << Indent <<
"WIDEN ";
629 O << Indent <<
"WIDEN-INDUCTION";
633 O <<
" +\n" << Indent <<
"\" ";
646 return StartC && StartC->isZero() && StepC && StepC->isOne();
650 return cast<VPCanonicalIVPHIRecipe>(
getOperand(0));
660 if (StepVPV->getDef())
662 auto *StepC = dyn_cast_or_null<ConstantInt>(StepVPV->getLiveInIRValue());
663 return StepC && StepC->isOne();
666 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
671 O << Indent <<
"= SCALAR-STEPS ";
684 if (State.
VF.
isVector() && IsPtrLoopInvariant && IsIndexLoopInvariant.
all()) {
698 for (
unsigned Part = 0; Part < State.
UF; ++Part) {
700 State.
set(
this, EntryPart, Part);
711 for (
unsigned Part = 0; Part < State.
UF; ++Part) {
714 auto *
Ptr = IsPtrLoopInvariant
723 if (IsIndexLoopInvariant[
I - 1])
726 Indices.push_back(State.
get(Operand, Part));
740 Indices,
"", IsInBounds);
742 "NewGEP is not a pointer vector");
743 State.
set(
this, NewGEP, Part);
749 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
752 O << Indent <<
"WIDEN-GEP ";
753 O << (IsPtrLoopInvariant ?
"Inv" :
"Var");
754 size_t IndicesNumber = IsIndexLoopInvariant.
size();
755 for (
size_t I = 0;
I < IndicesNumber; ++
I)
756 O <<
"[" << (IsIndexLoopInvariant[
I] ?
"Inv" :
"Var") <<
"]";
760 O <<
" = getelementptr ";
784 for (
unsigned In = 0;
In < NumIncoming; ++
In) {
785 for (
unsigned Part = 0; Part < State.
UF; ++Part) {
800 for (
unsigned Part = 0; Part < State.
UF; ++Part)
801 State.
set(
this, Entry[Part], Part);
804 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
807 O << Indent <<
"BLEND ";
827 O << Indent <<
"REDUCE ";
842 O <<
" (with final reduction value stored in invariant address sank "
848 O << Indent << (IsUniform ?
"CLONE " :
"REPLICATE ");
855 O <<
"call @" << CB->getCalledFunction()->getName() <<
"(";
872 assert(State.
Instance &&
"Branch on Mask works only on single instance.");
874 unsigned Part = State.
Instance->Part;
875 unsigned Lane = State.
Instance->Lane.getKnownLane();
877 Value *ConditionBit =
nullptr;
880 ConditionBit = State.
get(BlockInMask, Part);
890 assert(isa<UnreachableInst>(CurrentTerminator) &&
891 "Expected to replace unreachable terminator with conditional branch.");
898 assert(State.
Instance &&
"Predicated instruction PHI works per instance.");
903 assert(PredicatingBB &&
"Predicated block has no single predecessor.");
905 "operand must be VPReplicateRecipe");
913 unsigned Part = State.
Instance->Part;
921 State.
reset(
this, VPhi, Part);
923 State.
set(
this, VPhi, Part);
943 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
946 O << Indent <<
"PHI-PREDICATED-INSTRUCTION ";
954 O << Indent <<
"WIDEN ";
974 for (
unsigned Part = 0, UF = State.
UF; Part < UF; ++Part)
975 State.
set(
this, EntryPart, Part);
978 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
981 O << Indent <<
"EMIT ";
983 O <<
" = CANONICAL-INDUCTION";
994 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
997 O << Indent <<
"EMIT ";
999 O <<
" = WIDEN-POINTER-INDUCTION ";
1013 for (
unsigned Part = 0, UF = State.
UF; Part < UF; ++Part)
1014 State.
set(
this, Res, Part);
1017 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
1020 O << Indent <<
"EMIT ";
1022 O <<
" = EXPAND SCEV " << *Expr;
1033 :
Builder.CreateVectorSplat(VF, CanonicalIV,
"broadcast");
1034 for (
unsigned Part = 0, UF = State.
UF; Part < UF; ++Part) {
1037 VStep =
Builder.CreateVectorSplat(VF, VStep);
1041 Value *CanonicalVectorIV =
Builder.CreateAdd(VStart, VStep,
"vec.iv");
1042 State.
set(
this, CanonicalVectorIV, Part);
1046 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
1049 O << Indent <<
"EMIT ";
1051 O <<
" = WIDEN-CANONICAL-INDUCTION ";
1062 ? VectorInit->getType()
1067 auto *IdxTy =
Builder.getInt32Ty();
1072 auto *LastIdx =
Builder.CreateSub(RuntimeVF, One);
1073 VectorInit =
Builder.CreateInsertElement(
1081 State.
set(
this, EntryPart, 0);
1084 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
1087 O << Indent <<
"FIRST-ORDER-RECURRENCE-PHI ";
1102 bool ScalarPHI = State.
VF.
isScalar() || IsInLoop;
1108 "recipe must be in the vector loop header");
1109 unsigned LastPartForNewPhi =
isOrdered() ? 1 : State.
UF;
1110 for (
unsigned Part = 0; Part < LastPartForNewPhi; ++Part) {
1113 State.
set(
this, EntryPart, Part);
1123 Value *Iden =
nullptr;
1134 Builder.CreateVectorSplat(State.
VF, StartV,
"minmax.ident");
1141 Iden =
Builder.CreateVectorSplat(State.
VF, Iden);
1145 StartV =
Builder.CreateInsertElement(Iden, StartV, Zero);
1149 for (
unsigned Part = 0; Part < LastPartForNewPhi; ++Part) {
1150 Value *EntryPart = State.
get(
this, Part);
1153 Value *StartVal = (Part == 0) ? StartV : Iden;
1154 cast<PHINode>(EntryPart)->addIncoming(StartVal, VectorPH);
1158 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
1161 O << Indent <<
"WIDEN-REDUCTION-PHI ";
1171 "Non-native vplans are not expected to have VPWidenPHIRecipes.");
1179 unsigned StartIdx = 0;
1192 State.
set(
this, VecPhi, 0);
1195 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
1198 O << Indent <<
"WIDEN-PHI ";
1220 for (
unsigned Part = 0, UF = State.
UF; Part < UF; ++Part) {
1226 State.
set(
this, EntryPart, Part);
1230 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
1233 O << Indent <<
"ACTIVE-LANE-MASK-PHI ";
RecipeListTy & getRecipeList()
Returns a reference to the list of recipes.
unsigned getOpcode() const
void moveAfter(VPRecipeBase *MovePos)
Unlink this recipe from its current VPBasicBlock and insert it into the VPBasicBlock that MovePos liv...
VPBlockBase * getSinglePredecessor() const
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
VPValue * getStepValue()
Returns the step value of the induction.
unsigned getNumIncomingValues() const
Return the number of incoming values, taking into account that a single incoming value has no mask.
VPValue * getIncomingValue(unsigned Idx) const
Return incoming value number Idx.
This is an optimization pass for GlobalISel generic memory operations.
Instruction * getUnderlyingInstr()
Returns the underlying instruction, if the recipe is a VPValue or nullptr otherwise.
static IntegerType * getInt1Ty(LLVMContext &C)
Canonical scalar induction phi of the vector loop.
iterator_range< T > make_range(T x, T y)
Convenience function for iterating over sub-ranges.
iplist< VPRecipeBase >::iterator eraseFromParent()
This method unlinks 'this' from the containing basic block and deletes it.
static VPLane getFirstLane()
void execute(VPTransformState &State) override
Generate a canonical vector induction variable of the vector loop, with start = {<Part*VF,...
void execute(VPTransformState &State) override
Generates phi nodes for live-outs as needed to retain SSA form.
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
A parsed version of the target data layout string in and methods for querying it.
VPBasicBlock serves as the leaf of the Hierarchical Control-Flow Graph.
This class uses information about analyze scalars to rewrite expressions in canonical form.
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
Value * getLiveInIRValue()
Returns the underlying IR value, if this VPValue is defined outside the scope of VPlan.
Type * getScalarType() const
If this is a vector type, return the element type, otherwise return 'this'.
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
VPValue * getVPSingleValue()
Returns the only VPValue defined by the VPDef.
void execute(VPTransformState &State) override
Generate the extraction of the appropriate bit from the block mask and the conditional branch.
void setFastMathFlags(FastMathFlags NewFMF)
Set the fast-math flags to be used with generated fp-math operators.
This provides a uniform API for creating instructions and inserting them into a basic block: either a...
bool onlyScalarsGenerated(ElementCount VF)
Returns true if only scalar values will be generated.
void setFastMathFlags(FastMathFlags FMFNew)
Set the fast-math flags.
static unsigned getOpcode(RecurKind Kind)
Returns the opcode corresponding to the RecurrenceKind.
void execute(VPTransformState &State) override
Generate the phi/select nodes.
void interleaveComma(const Container &c, StreamT &os, UnaryFunctor each_fn)
The instances of the Type class are immutable: once they are created, they are never changed.
TruncInst * getTruncInst()
Returns the first defined value as TruncInst, if it is one or nullptr otherwise.
@ FirstOrderRecurrenceSplice
operand_iterator op_begin()
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
pointer remove(iterator &IT)
const BasicBlock * getSinglePredecessor() const
Return the predecessor of this block if it has a single predecessor block.
const VPBlockBase * getEntry() const
Convenience struct for specifying and reasoning about fast-math flags.
bool isScalable() const
Returns whether the size is scaled by a runtime quantity (vscale).
VPRegionBlock * getEnclosingLoopRegion()
An intrusive list with ownership and callbacks specified/controlled by ilist_traits,...
bool isCanonical() const
Returns true if the induction is canonical, i.e.
void execute(VPTransformState &State) override
Generate the phi/select nodes.
VPCanonicalIVPHIRecipe * getCanonicalIV() const
void setSuccessor(unsigned idx, BasicBlock *NewSucc)
iterator erase(iterator where)
LLVM Basic Block Representation.
void fixPhi(VPlan &Plan, VPTransformState &State)
Fixup the wrapped LCSSA phi node in the unique exit block.
const InductionDescriptor & getInductionDescriptor() const
Returns the induction descriptor for the recipe.
RecurKind
These are the kinds of recurrences that we support.
bool mayWriteToMemory() const
Returns true if the recipe may write to memory.
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
bool isScalar() const
Counting predicates.
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
unsigned getOpcode() const
Returns a member of one of the enums like Instruction::Add.
bool all_of(R &&range, UnaryPredicate P)
Provide wrappers to std::all_of which take ranges instead of having to pass begin/end explicitly.
VPBasicBlock * getIncomingBlock(unsigned I)
Returns the I th incoming VPBasicBlock.
Value * createStepForVF(IRBuilderBase &B, Type *Ty, ElementCount VF, int64_t Step)
Return a value for Step multiplied by VF.
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
Value * CreateGEP(Type *Ty, Value *Ptr, ArrayRef< Value * > IdxList, const Twine &Name="", bool IsInBounds=false)
static bool isSelectCmpRecurrenceKind(RecurKind Kind)
Returns true if the recurrence kind is of the form select(cmp(),x,y) where one of (x,...
(vector float) vec_cmpeq(*A, *B) C
@ CanonicalIVIncrementForPart
static Optional< unsigned > getOpcode(ArrayRef< VPValue * > Values)
Returns the opcode of Values or ~0 if they do not all agree.
void moveBefore(VPBasicBlock &BB, iplist< VPRecipeBase >::iterator I)
Unlink this recipe and insert into BB before I.
void ReplaceInstWithInst(BasicBlock::InstListType &BIL, BasicBlock::iterator &BI, Instruction *I)
Replace the instruction specified by BI with the instruction specified by I.
bool isVectorTy() const
True if this is an instance of VectorType.
This instruction inserts a single (scalar) element into a VectorType value.
const_iterator getFirstInsertionPt() const
Returns an iterator to the first instruction in this block that is suitable for inserting a non-PHI i...
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
bool all() const
Returns true if all bits are set.
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
const char * getOpcodeName() const
Value * getRecurrenceIdentity(RecurKind K, Type *Tp, FastMathFlags FMF) const
Returns identity corresponding to the RecurrenceKind.
cl::opt< bool > EnableVPlanNativePath
RecurKind getRecurrenceKind() const
This class implements an extremely fast bulk output stream that can only output to a stream.
VPValue * getStartValue() const
static Constant * get(Type *Ty, uint64_t V, bool IsSigned=false)
If Ty is a vector type, return a Constant with a splat of the given value.
const Module * getModule() const
Return the module owning the function this basic block belongs to, or nullptr if the function does no...
unsigned getNumOperands() const
VPRegionBlock represents a collection of VPBasicBlocks and VPRegionBlocks which form a Single-Entry-S...
bool isStore() const
Returns true if this recipe is a store.
void execute(VPTransformState &State) override
Generate the active lane mask phi of the vector loop.
void removeFromParent()
This method unlinks 'this' from the containing basic block, but does not delete it.
bool mayHaveSideEffects() const
Returns true if the recipe may have side-effects.
This is an important base class in LLVM.
BasicBlock::iterator GetInsertPoint() const
void execute(VPTransformState &State) override
Generate a canonical vector induction variable of the vector loop, with.
VPValue * getCondOp() const
The VPValue of the condition for the block.
bool isUniformAfterVectorization(VPValue *VPV) const
Returns true if VPV is uniform after vectorization.
void execute(VPTransformState &State) override
Generate the gep nodes.
void execute(VPTransformState &State) override
Generate the instruction.
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
ConstantInt * getInt32(uint32_t C)
Get a constant 32-bit value.
void addIncoming(Value *V, BasicBlock *BB)
Add an incoming value to the end of the PHI list.
static VPLane getLastLaneForVF(const ElementCount &VF)
static BranchInst * Create(BasicBlock *IfTrue, Instruction *InsertBefore=nullptr)
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the VPInstruction to O.
VPIteration represents a single point in the iteration space of the output (vectorized and/or unrolle...
Value * CreateSelect(Value *C, Value *True, Value *False, const Twine &Name="", Instruction *MDFrom=nullptr)
VPValue * getMask(unsigned Idx) const
Return mask number Idx.
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
void setDebugLoc(DebugLoc Loc)
Set the debug location information for this instruction.
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
Value * CreateExtractElement(Value *Vec, Value *Idx, const Twine &Name="")
bool isVoidTy() const
Return true if this is 'void'.
ConstantInt * getTrue()
Get the constant value for i1 true.
void printOperands(raw_ostream &O, VPSlotTracker &SlotTracker) const
Print the operands to O.
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
bool isVector() const
One or more elements.
void printAsOperand(raw_ostream &O, bool PrintType=true, const Module *M=nullptr) const
Print the name of this Value out to the specified raw_ostream.
bool isCanonical() const
Returns true if the induction is canonical, i.e.
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
PHINode * CreatePHI(Type *Ty, unsigned NumReservedValues, const Twine &Name="")
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
Value * getRuntimeVF(IRBuilderBase &B, Type *Ty, ElementCount VF)
Return the runtime value for VF.
VPValue * getVecOp() const
The VPValue of the vector value to be reduced.
FastMathFlags getFastMathFlags() const
Convenience function for getting all the fast-math flags, which must be an operator which supports th...
StoreInst * IntermediateStore
Reductions may store temporary or final result to an invariant address.
SmallVector< MachineOperand, 4 > Cond
FastMathFlags getFastMathFlags() const
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
Type * getType() const
All values are typed, get the type of this value.
const SCEV * getStep() const
VPBasicBlock * getParent()
Common base class shared among various IRBuilders.
self_iterator getIterator()
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
void execute(VPTransformState &State) override
Generate the canonical scalar induction phi of the vector loop.
VPValue * getStepValue() const
void execute(VPTransformState &State) override
Produce a widened version of the select instruction.
const VPBasicBlock * getEntryBasicBlock() const
bool onlyFirstLaneUsed(VPValue *Def)
Returns true if only the first lane of Def is used.
VPValue * getVPValue(unsigned I)
Returns the VPValue with index I defined by the VPDef.
@ CanonicalIVIncrementForPartNUW
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
void execute(VPTransformState &State) override
Generate the phi/select nodes.
Twine - A lightweight data structure for efficiently representing the concatenation of temporary valu...
This class can be used to assign consecutive numbers to all VPValues in a VPlan and allows querying t...
VPRegionBlock * getParent()
static PHINode * Create(Type *Ty, unsigned NumReservedValues, const Twine &NameStr="", Instruction *InsertBefore=nullptr)
Constructors - NumReservedValues is a hint for the number of incoming edges that this phi node will h...
InstTy * Insert(InstTy *I, const Twine &Name="") const
Insert and return the specified instruction.
VPRecipeBase is a base class modeling a sequence of one or more output IR instructions.
BasicBlock * GetInsertBlock() const
BlockT * getHeader() const
void print(raw_ostream &OS) const
prints source location /path/to/file.exe:line:col @[inlined at]
This class provides computation of slot numbers for LLVM Assembly writing.
@ CanonicalIVIncrementNUW
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
iterator insertAfter(iterator where, pointer New)
iterator insert(iterator where, pointer New)
This class augments VPValue with operands which provide the inverse def-use edges from VPValue's user...
size_type size() const
Returns the number of bits in this bitvector.
VectorType * getType() const
Overload to return most specific vector type.
VPValue * getOperand(unsigned N) const
bool mayReadFromMemory() const
Returns true if the recipe may read from memory.
const BasicBlock * getParent() const
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
const Instruction * getTerminator() const LLVM_READONLY
Returns the terminator instruction if the block is well formed or null if the block is not well forme...
const VPBasicBlock * getExitingBasicBlock() const
void execute(VPTransformState &State) override
Generate the phi nodes.
static const uint32_t IV[8]
Value * CreateVectorSplat(unsigned NumElts, Value *V, const Twine &Name="")
Return a vector value that contains.
const DataLayout & getDataLayout() const
Get the data layout for the module's target platform.
Type * getType() const
Return the LLVM type of this SCEV expression.
virtual bool usesScalars(const VPValue *Op) const
Returns true if the VPUser uses scalars of operand Op.
VPValue * getMask() const
Return the mask used by this recipe.
Common register allocation spilling lr str ldr sxth r3 ldr mla r4 can lr mov lr str ldr sxth r3 mla r4 and then merge mul and lr str ldr sxth r3 mla r4 It also increase the likelihood the store may become dead bb27 Successors according to LLVM BB
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
Value * getOperand(unsigned i) const
static bool isMinMaxRecurrenceKind(RecurKind Kind)
Returns true if the recurrence kind is any min/max kind.
Conditional or Unconditional Branch instruction.
VPValue * getChainOp() const
The VPValue of the scalar Chain being accumulated.
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
void execute(VPTransformState &State) override
Produce widened copies of all Ingredients.
unsigned getVPDefID() const
LLVM_DUMP_METHOD void dump() const
Print the VPInstruction to dbgs() (for debugging).
Value * getUnderlyingValue()
Return the underlying Value attached to this VPValue.
VPlan models a candidate for vectorization, encoding various decisions take to produce efficient outp...
LLVM Value Representation.
void insertBefore(VPRecipeBase *InsertPos)
Insert an unlinked recipe into a basic block immediately before the specified recipe.
static VectorType * get(Type *ElementType, ElementCount EC)
This static method is the primary way to construct an VectorType.
bool isOrdered() const
Returns true, if the phi is part of an ordered reduction.
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
void printAsOperand(raw_ostream &OS, VPSlotTracker &Tracker) const
VPValue * getStartValue()
Returns the start value of the induction.
void insertAfter(VPRecipeBase *InsertPos)
Insert an unlinked Recipe into a basic block immediately after the specified Recipe.
static PoisonValue * get(Type *T)
Static factory methods - Return an 'poison' object of the specified type.