41#define LV_NAME "loop-vectorize"
42#define DEBUG_TYPE LV_NAME
46 case VPWidenMemoryInstructionSC: {
47 return cast<VPWidenMemoryInstructionRecipe>(
this)->isStore();
53 case VPBranchOnMaskSC:
54 case VPScalarIVStepsSC:
57 case VPWidenIntOrFpInductionSC:
58 case VPWidenCanonicalIVSC:
64 case VPWidenSelectSC: {
68 assert((!
I || !
I->mayWriteToMemory()) &&
69 "underlying instruction may write to memory");
79 case VPWidenMemoryInstructionSC: {
80 return !cast<VPWidenMemoryInstructionRecipe>(
this)->isStore();
85 ->mayReadFromMemory();
86 case VPBranchOnMaskSC:
87 case VPScalarIVStepsSC:
90 case VPWidenIntOrFpInductionSC:
91 case VPWidenCanonicalIVSC:
97 case VPWidenSelectSC: {
101 assert((!
I || !
I->mayReadFromMemory()) &&
102 "underlying instruction may read from memory");
113 case VPPredInstPHISC:
117 ->mayHaveSideEffects();
118 case VPWidenIntOrFpInductionSC:
119 case VPWidenPointerInductionSC:
120 case VPWidenCanonicalIVSC:
126 case VPWidenSelectSC:
127 case VPScalarIVStepsSC: {
131 assert((!
I || !
I->mayHaveSideEffects()) &&
132 "underlying instruction has side-effects");
135 case VPWidenMemoryInstructionSC:
136 assert(cast<VPWidenMemoryInstructionRecipe>(
this)
139 "mayHaveSideffects result for ingredient differs from this "
142 case VPReplicateSC: {
143 auto *R = cast<VPReplicateRecipe>(
this);
144 return R->getUnderlyingInstr()->mayHaveSideEffects();
161 assert(!Parent &&
"Recipe already in some VPBasicBlock");
163 "Insertion position not in any VPBasicBlock");
170 assert(!Parent &&
"Recipe already in some VPBasicBlock");
177 assert(!Parent &&
"Recipe already in some VPBasicBlock");
179 "Insertion position not in any VPBasicBlock");
209 Builder.SetCurrentDebugLocation(DL);
216 State.
set(
this, V, Part);
224 State.
set(
this, V, Part);
231 State.
set(
this, V, Part);
234 case Instruction::Select: {
239 State.
set(
this, V, Part);
251 Intrinsic::get_active_lane_mask, {PredTy, ScalarTC->
getType()},
252 {VIVElem0, ScalarTC},
nullptr, Name);
253 State.
set(
this, Call, Part);
273 State.
set(
this, PartMinus1, Part);
276 State.
set(
this,
Builder.CreateVectorSplice(PartMinus1, V2, -1, Name),
289 State.
set(
this, Sel, Part);
294 Value *Next =
nullptr;
302 Next =
Builder.CreateAdd(Phi, Step, Name, IsNUW,
false);
304 Next = State.
get(
this, 0);
307 State.
set(
this, Next, Part);
316 State.
set(
this,
IV, Part);
324 State.
set(
this, Next, Part);
345 Builder.GetInsertBlock()->getTerminator()->eraseFromParent();
369 Builder.GetInsertBlock()->getTerminator()->eraseFromParent();
381 for (
unsigned Part = 0; Part < State.
UF; ++Part)
382 generateInstruction(State, Part);
385#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
393 O << Indent <<
"EMIT ";
408 O <<
"combined load";
411 O <<
"combined store";
414 O <<
"active lane mask";
417 O <<
"first-order splice";
423 O <<
"VF * UF +(nuw) ";
426 O <<
"branch-on-cond";
429 O <<
"TC > VF ? TC - VF : 0";
435 O <<
"VF * Part +(nuw) ";
438 O <<
"branch-on-count ";
460 assert((Opcode == Instruction::FAdd || Opcode == Instruction::FMul ||
461 Opcode == Instruction::FNeg || Opcode == Instruction::FSub ||
462 Opcode == Instruction::FDiv || Opcode == Instruction::FRem ||
463 Opcode == Instruction::FCmp) &&
464 "this op can't take fast-math flags");
470 assert(!isa<DbgInfoIntrinsic>(CI) &&
471 "DbgInfoIntrinsic should have been dropped during VPlan construction");
474 for (
unsigned Part = 0; Part < State.
UF; ++Part) {
483 Arg = State.
get(
I.value(), Part);
499 assert(VectorF &&
"Can't retrieve vector intrinsic.");
502 assert(Variant !=
nullptr &&
"Can't create vector function.");
508 CI.getOperandBundlesAsDefs(OpBundles);
511 if (isa<FPMathOperator>(V))
512 V->copyFastMathFlags(&CI);
514 State.
set(
this, V, Part);
519#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
522 O << Indent <<
"WIDEN-CALL ";
525 if (CI->getType()->isVoidTy())
532 O <<
"call @" << CI->getCalledFunction()->getName() <<
"(";
536 if (VectorIntrinsicID)
537 O <<
" (using vector intrinsic)";
539 O <<
" (using library function";
541 O <<
": " << Variant->
getName();
548 O << Indent <<
"WIDEN-SELECT ";
571 for (
unsigned Part = 0; Part < State.
UF; ++Part) {
576 State.
set(
this, Sel, Part);
584 switch (
I.getOpcode()) {
585 case Instruction::Call:
586 case Instruction::Br:
587 case Instruction::PHI:
588 case Instruction::GetElementPtr:
589 case Instruction::Select:
591 case Instruction::UDiv:
592 case Instruction::SDiv:
593 case Instruction::SRem:
594 case Instruction::URem:
595 case Instruction::Add:
596 case Instruction::FAdd:
597 case Instruction::Sub:
598 case Instruction::FSub:
599 case Instruction::FNeg:
600 case Instruction::Mul:
601 case Instruction::FMul:
602 case Instruction::FDiv:
603 case Instruction::FRem:
604 case Instruction::Shl:
605 case Instruction::LShr:
606 case Instruction::AShr:
607 case Instruction::And:
608 case Instruction::Or:
609 case Instruction::Xor: {
613 for (
unsigned Part = 0; Part < State.
UF; ++Part) {
620 if (
auto *VecOp = dyn_cast<Instruction>(V)) {
621 VecOp->copyIRFlags(&
I);
629 VecOp->dropPoisonGeneratingFlags();
633 State.
set(
this, V, Part);
639 case Instruction::Freeze: {
642 for (
unsigned Part = 0; Part < State.
UF; ++Part) {
646 State.
set(
this, Freeze, Part);
650 case Instruction::ICmp:
651 case Instruction::FCmp: {
653 bool FCmp = (
I.getOpcode() == Instruction::FCmp);
654 auto *Cmp = cast<CmpInst>(&
I);
656 for (
unsigned Part = 0; Part < State.
UF; ++Part) {
663 Builder.setFastMathFlags(Cmp->getFastMathFlags());
664 C =
Builder.CreateFCmp(Cmp->getPredicate(),
A,
B);
666 C =
Builder.CreateICmp(Cmp->getPredicate(),
A,
B);
668 State.
set(
this,
C, Part);
675 case Instruction::ZExt:
676 case Instruction::SExt:
677 case Instruction::FPToUI:
678 case Instruction::FPToSI:
679 case Instruction::FPExt:
680 case Instruction::PtrToInt:
681 case Instruction::IntToPtr:
682 case Instruction::SIToFP:
683 case Instruction::UIToFP:
684 case Instruction::Trunc:
685 case Instruction::FPTrunc:
686 case Instruction::BitCast: {
687 auto *CI = cast<CastInst>(&
I);
695 for (
unsigned Part = 0; Part < State.
UF; ++Part) {
697 Value *Cast =
Builder.CreateCast(CI->getOpcode(),
A, DestTy);
698 State.
set(
this, Cast, Part);
709#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
712 O << Indent <<
"WIDEN ";
716 if (
auto *Cmp = dyn_cast<CmpInst>(UI))
717 O << Cmp->getPredicate() <<
" ";
723 O << Indent <<
"WIDEN-INDUCTION";
727 O <<
" +\n" << Indent <<
"\" ";
740 return StartC && StartC->isZero() && StepC && StepC->isOne();
743#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
748 O << Indent <<
"= DERIVED-IV ";
756 O <<
" (truncated to " << *ResultTy <<
")";
760#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
765 O << Indent <<
"= SCALAR-STEPS ";
778 if (State.
VF.
isVector() && areAllOperandsInvariant()) {
792 for (
unsigned Part = 0; Part < State.
UF; ++Part) {
794 State.
set(
this, EntryPart, Part);
805 for (
unsigned Part = 0; Part < State.
UF; ++Part) {
808 auto *
Ptr = isPointerLoopInvariant()
817 if (isIndexLoopInvariant(
I - 1))
834 Indices,
"", IsInBounds);
836 "NewGEP is not a pointer vector");
837 State.
set(
this, NewGEP, Part);
843#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
846 O << Indent <<
"WIDEN-GEP ";
847 O << (isPointerLoopInvariant() ?
"Inv" :
"Var");
849 O <<
"[" << (isIndexLoopInvariant(
I) ?
"Inv" :
"Var") <<
"]";
853 O <<
" = getelementptr ";
877 for (
unsigned In = 0; In < NumIncoming; ++In) {
878 for (
unsigned Part = 0; Part < State.
UF; ++Part) {
893 for (
unsigned Part = 0; Part < State.
UF; ++Part)
894 State.
set(
this, Entry[Part], Part);
897#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
900 O << Indent <<
"BLEND ";
920 O << Indent <<
"REDUCE ";
935 O <<
" (with final reduction value stored in invariant address sank "
944 if (
auto *PredR = dyn_cast<VPPredInstPHIRecipe>(U))
945 return any_of(PredR->users(), [PredR](
const VPUser *U) {
946 return !U->usesScalars(PredR);
952#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
955 O << Indent << (IsUniform ?
"CLONE " :
"REPLICATE ");
962 O <<
"call @" << CB->getCalledFunction()->getName() <<
"(";
979 assert(State.
Instance &&
"Branch on Mask works only on single instance.");
981 unsigned Part = State.
Instance->Part;
982 unsigned Lane = State.
Instance->Lane.getKnownLane();
984 Value *ConditionBit =
nullptr;
987 ConditionBit = State.
get(BlockInMask, Part);
997 assert(isa<UnreachableInst>(CurrentTerminator) &&
998 "Expected to replace unreachable terminator with conditional branch.");
1005 assert(State.
Instance &&
"Predicated instruction PHI works per instance.");
1010 assert(PredicatingBB &&
"Predicated block has no single predecessor.");
1012 "operand must be VPReplicateRecipe");
1020 unsigned Part = State.
Instance->Part;
1028 State.
reset(
this, VPhi, Part);
1030 State.
set(
this, VPhi, Part);
1050#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
1053 O << Indent <<
"PHI-PREDICATED-INSTRUCTION ";
1061 O << Indent <<
"WIDEN ";
1081 for (
unsigned Part = 0, UF = State.
UF; Part < UF; ++Part)
1082 State.
set(
this, EntryPart, Part);
1085#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
1088 O << Indent <<
"EMIT ";
1090 O <<
" = CANONICAL-INDUCTION";
1111 return IsScalarAfterVectorization &&
1115#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
1118 O << Indent <<
"EMIT ";
1120 O <<
" = WIDEN-POINTER-INDUCTION ";
1122 O <<
", " << *IndDesc.
getStep();
1134 for (
unsigned Part = 0, UF = State.
UF; Part < UF; ++Part)
1135 State.
set(
this, Res, Part);
1138#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
1141 O << Indent <<
"EMIT ";
1143 O <<
" = EXPAND SCEV " << *Expr;
1154 :
Builder.CreateVectorSplat(VF, CanonicalIV,
"broadcast");
1155 for (
unsigned Part = 0, UF = State.
UF; Part < UF; ++Part) {
1158 VStep =
Builder.CreateVectorSplat(VF, VStep);
1162 Value *CanonicalVectorIV =
Builder.CreateAdd(VStart, VStep,
"vec.iv");
1163 State.
set(
this, CanonicalVectorIV, Part);
1167#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
1170 O << Indent <<
"EMIT ";
1172 O <<
" = WIDEN-CANONICAL-INDUCTION ";
1183 ? VectorInit->getType()
1188 auto *IdxTy =
Builder.getInt32Ty();
1193 auto *LastIdx =
Builder.CreateSub(RuntimeVF, One);
1194 VectorInit =
Builder.CreateInsertElement(
1202 State.
set(
this, EntryPart, 0);
1205#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
1208 O << Indent <<
"FIRST-ORDER-RECURRENCE-PHI ";
1223 bool ScalarPHI = State.
VF.
isScalar() || IsInLoop;
1229 "recipe must be in the vector loop header");
1230 unsigned LastPartForNewPhi =
isOrdered() ? 1 : State.
UF;
1231 for (
unsigned Part = 0; Part < LastPartForNewPhi; ++Part) {
1234 State.
set(
this, EntryPart, Part);
1244 Value *Iden =
nullptr;
1255 Builder.CreateVectorSplat(State.
VF, StartV,
"minmax.ident");
1262 Iden =
Builder.CreateVectorSplat(State.
VF, Iden);
1266 StartV =
Builder.CreateInsertElement(Iden, StartV, Zero);
1270 for (
unsigned Part = 0; Part < LastPartForNewPhi; ++Part) {
1271 Value *EntryPart = State.
get(
this, Part);
1274 Value *StartVal = (Part == 0) ? StartV : Iden;
1275 cast<PHINode>(EntryPart)->addIncoming(StartVal, VectorPH);
1279#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
1282 O << Indent <<
"WIDEN-REDUCTION-PHI ";
1292 "Non-native vplans are not expected to have VPWidenPHIRecipes.");
1300 unsigned StartIdx = 0;
1313 State.
set(
this, VecPhi, 0);
1316#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
1319 O << Indent <<
"WIDEN-PHI ";
1341 for (
unsigned Part = 0, UF = State.
UF; Part < UF; ++Part) {
1347 State.
set(
this, EntryPart, Part);
1351#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
1354 O << Indent <<
"ACTIVE-LANE-MASK-PHI ";
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
amdgpu Simplify well known AMD library false FunctionCallee Value * Arg
SmallVector< MachineOperand, 4 > Cond
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
This file defines the SmallVector class.
static std::optional< unsigned > getOpcode(ArrayRef< VPValue * > Values)
Returns the opcode of Values or ~0 if they do not all agree.
This file contains the declarations of the Vectorization Plan base classes:
static const uint32_t IV[8]
LLVM Basic Block Representation.
const_iterator getFirstInsertionPt() const
Returns an iterator to the first instruction in this block that is suitable for inserting a non-PHI i...
const BasicBlock * getSinglePredecessor() const
Return the predecessor of this block if it has a single predecessor block.
const Instruction * getTerminator() const LLVM_READONLY
Returns the terminator instruction if the block is well formed or null if the block is not well forme...
const Module * getModule() const
Return the module owning the function this basic block belongs to, or nullptr if the function does no...
Conditional or Unconditional Branch instruction.
static BranchInst * Create(BasicBlock *IfTrue, Instruction *InsertBefore=nullptr)
void setSuccessor(unsigned idx, BasicBlock *NewSucc)
This class represents a function call, abstracting a target machine's calling convention.
@ ICMP_UGT
unsigned greater than
This is the shared class of boolean and integer constants.
bool isOne() const
This is just a convenience method to make client code smaller for a common case.
static Constant * get(Type *Ty, uint64_t V, bool IsSigned=false)
If Ty is a vector type, return a Constant with a splat of the given value.
This is an important base class in LLVM.
A parsed version of the target data layout string in and methods for querying it.
void print(raw_ostream &OS) const
prints source location /path/to/file.exe:line:col @[inlined at]
constexpr bool isVector() const
One or more elements.
constexpr bool isScalar() const
Exactly one element.
Convenience struct for specifying and reasoning about fast-math flags.
Common base class shared among various IRBuilders.
Value * CreateExtractElement(Value *Vec, Value *Idx, const Twine &Name="")
Value * CreateVectorSplat(unsigned NumElts, Value *V, const Twine &Name="")
Return a vector value that contains.
ConstantInt * getTrue()
Get the constant value for i1 true.
Value * CreateSelect(Value *C, Value *True, Value *False, const Twine &Name="", Instruction *MDFrom=nullptr)
BasicBlock::iterator GetInsertPoint() const
BasicBlock * GetInsertBlock() const
void setFastMathFlags(FastMathFlags NewFMF)
Set the fast-math flags to be used with generated fp-math operators.
ConstantInt * getInt32(uint32_t C)
Get a constant 32-bit value.
PHINode * CreatePHI(Type *Ty, unsigned NumReservedValues, const Twine &Name="")
InstTy * Insert(InstTy *I, const Twine &Name="") const
Insert and return the specified instruction.
CallInst * CreateCall(FunctionType *FTy, Value *Callee, ArrayRef< Value * > Args=std::nullopt, const Twine &Name="", MDNode *FPMathTag=nullptr)
Value * CreateGEP(Type *Ty, Value *Ptr, ArrayRef< Value * > IdxList, const Twine &Name="", bool IsInBounds=false)
This provides a uniform API for creating instructions and inserting them into a basic block: either a...
A struct for saving information about induction variables.
const SCEV * getStep() const
@ IK_IntInduction
Integer induction variable. Step = C.
This instruction inserts a single (scalar) element into a VectorType value.
VectorType * getType() const
Overload to return most specific vector type.
const BasicBlock * getParent() const
FastMathFlags getFastMathFlags() const LLVM_READONLY
Convenience function for getting all the fast-math flags, which must be an operator which supports th...
const char * getOpcodeName() const
unsigned getOpcode() const
Returns a member of one of the enums like Instruction::Add.
void setDebugLoc(DebugLoc Loc)
Set the debug location information for this instruction.
BlockT * getHeader() const
A Module instance is used to store all the information related to an LLVM module.
const DataLayout & getDataLayout() const
Get the data layout for the module's target platform.
void addIncoming(Value *V, BasicBlock *BB)
Add an incoming value to the end of the PHI list.
static PHINode * Create(Type *Ty, unsigned NumReservedValues, const Twine &NameStr="", Instruction *InsertBefore=nullptr)
Constructors - NumReservedValues is a hint for the number of incoming edges that this phi node will h...
static PoisonValue * get(Type *T)
Static factory methods - Return an 'poison' object of the specified type.
FastMathFlags getFastMathFlags() const
static unsigned getOpcode(RecurKind Kind)
Returns the opcode corresponding to the RecurrenceKind.
RecurKind getRecurrenceKind() const
Value * getRecurrenceIdentity(RecurKind K, Type *Tp, FastMathFlags FMF) const
Returns identity corresponding to the RecurrenceKind.
StoreInst * IntermediateStore
Reductions may store temporary or final result to an invariant address.
static bool isMinMaxRecurrenceKind(RecurKind Kind)
Returns true if the recurrence kind is any min/max kind.
static bool isSelectCmpRecurrenceKind(RecurKind Kind)
Returns true if the recurrence kind is of the form select(cmp(),x,y) where one of (x,...
This class uses information about analyze scalars to rewrite expressions in canonical form.
Type * getType() const
Return the LLVM type of this SCEV expression.
This class provides computation of slot numbers for LLVM Assembly writing.
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
Twine - A lightweight data structure for efficiently representing the concatenation of temporary valu...
The instances of the Type class are immutable: once they are created, they are never changed.
bool isVectorTy() const
True if this is an instance of VectorType.
static IntegerType * getInt1Ty(LLVMContext &C)
bool isVoidTy() const
Return true if this is 'void'.
Type * getScalarType() const
If this is a vector type, return the element type, otherwise return 'this'.
Value * getOperand(unsigned i) const
void execute(VPTransformState &State) override
Generate the active lane mask phi of the vector loop.
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
VPBasicBlock serves as the leaf of the Hierarchical Control-Flow Graph.
RecipeListTy & getRecipeList()
Returns a reference to the list of recipes.
VPRegionBlock * getEnclosingLoopRegion()
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
VPValue * getIncomingValue(unsigned Idx) const
Return incoming value number Idx.
VPValue * getMask(unsigned Idx) const
Return mask number Idx.
unsigned getNumIncomingValues() const
Return the number of incoming values, taking into account that a single incoming value has no mask.
void execute(VPTransformState &State) override
Generate the phi/select nodes.
VPRegionBlock * getParent()
const VPBasicBlock * getExitingBasicBlock() const
VPBlockBase * getSinglePredecessor() const
const VPBasicBlock * getEntryBasicBlock() const
VPValue * getMask() const
Return the mask used by this recipe.
void execute(VPTransformState &State) override
Generate the extraction of the appropriate bit from the block mask and the conditional branch.
void execute(VPTransformState &State) override
Generate the canonical scalar induction phi of the vector loop.
bool isCanonical(const InductionDescriptor &ID, Type *Ty) const
Check if the induction described by ID is canonical, i.e.
const Type * getScalarType() const
Returns the scalar type of the induction.
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
VPValue * getVPSingleValue()
Returns the only VPValue defined by the VPDef.
VPValue * getVPValue(unsigned I)
Returns the VPValue with index I defined by the VPDef.
unsigned getVPDefID() const
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
VPValue * getStepValue() const
VPValue * getCanonicalIV() const
VPValue * getStartValue() const
void execute(VPTransformState &State) override
Generate a canonical vector induction variable of the vector loop, with.
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
@ CanonicalIVIncrementForPartNUW
@ FirstOrderRecurrenceSplice
@ CanonicalIVIncrementNUW
@ CanonicalIVIncrementForPart
@ CalculateTripCountMinusVF
LLVM_DUMP_METHOD void dump() const
Print the VPInstruction to dbgs() (for debugging).
unsigned getOpcode() const
void setFastMathFlags(FastMathFlags FMFNew)
Set the fast-math flags.
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the VPInstruction to O.
void execute(VPTransformState &State) override
Generate the instruction.
static VPLane getLastLaneForVF(const ElementCount &VF)
static VPLane getFirstLane()
void fixPhi(VPlan &Plan, VPTransformState &State)
Fixup the wrapped LCSSA phi node in the unique exit block.
void execute(VPTransformState &State) override
Generates phi nodes for live-outs as needed to retain SSA form.
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
VPRecipeBase is a base class modeling a sequence of one or more output IR instructions.
bool mayReadFromMemory() const
Returns true if the recipe may read from memory.
bool mayHaveSideEffects() const
Returns true if the recipe may have side-effects.
Instruction * getUnderlyingInstr()
Returns the underlying instruction, if the recipe is a VPValue or nullptr otherwise.
bool mayWriteToMemory() const
Returns true if the recipe may write to memory.
VPBasicBlock * getParent()
void moveBefore(VPBasicBlock &BB, iplist< VPRecipeBase >::iterator I)
Unlink this recipe and insert into BB before I.
void insertBefore(VPRecipeBase *InsertPos)
Insert an unlinked recipe into a basic block immediately before the specified recipe.
void insertAfter(VPRecipeBase *InsertPos)
Insert an unlinked Recipe into a basic block immediately after the specified Recipe.
iplist< VPRecipeBase >::iterator eraseFromParent()
This method unlinks 'this' from the containing basic block and deletes it.
void removeFromParent()
This method unlinks 'this' from the containing basic block, but does not delete it.
void moveAfter(VPRecipeBase *MovePos)
Unlink this recipe from its current VPBasicBlock and insert it into the VPBasicBlock that MovePos liv...
bool isOrdered() const
Returns true, if the phi is part of an ordered reduction.
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
void execute(VPTransformState &State) override
Generate the phi/select nodes.
VPValue * getVecOp() const
The VPValue of the vector value to be reduced.
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
VPValue * getCondOp() const
The VPValue of the condition for the block.
VPValue * getChainOp() const
The VPValue of the scalar Chain being accumulated.
VPRegionBlock represents a collection of VPBasicBlocks and VPRegionBlocks which form a Single-Entry-S...
const VPBlockBase * getEntry() const
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
bool shouldPack() const
Returns true if the recipe is used by a widened recipe via an intervening VPPredInstPHIRecipe.
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
This class can be used to assign consecutive numbers to all VPValues in a VPlan and allows querying t...
This class augments VPValue with operands which provide the inverse def-use edges from VPValue's user...
void printOperands(raw_ostream &O, VPSlotTracker &SlotTracker) const
Print the operands to O.
unsigned getNumOperands() const
operand_iterator op_begin()
VPValue * getOperand(unsigned N) const
Value * getUnderlyingValue()
Return the underlying Value attached to this VPValue.
void printAsOperand(raw_ostream &OS, VPSlotTracker &Tracker) const
Value * getLiveInIRValue()
Returns the underlying IR value, if this VPValue is defined outside the scope of VPlan.
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
void execute(VPTransformState &State) override
Produce a widened version of the call instruction.
void execute(VPTransformState &State) override
Generate a canonical vector induction variable of the vector loop, with start = {<Part*VF,...
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
void execute(VPTransformState &State) override
Generate the gep nodes.
TruncInst * getTruncInst()
Returns the first defined value as TruncInst, if it is one or nullptr otherwise.
VPValue * getStepValue()
Returns the step value of the induction.
bool isCanonical() const
Returns true if the induction is canonical, i.e.
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
const InductionDescriptor & getInductionDescriptor() const
Returns the induction descriptor for the recipe.
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
bool isStore() const
Returns true if this recipe is a store.
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
VPBasicBlock * getIncomingBlock(unsigned I)
Returns the I th incoming VPBasicBlock.
void execute(VPTransformState &State) override
Generate the phi/select nodes.
bool onlyScalarsGenerated(ElementCount VF)
Returns true if only scalar values will be generated.
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
void execute(VPTransformState &State) override
Produce widened copies of all Ingredients.
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
VPlan models a candidate for vectorization, encoding various decisions take to produce efficient outp...
LLVM Value Representation.
Type * getType() const
All values are typed, get the type of this value.
void printAsOperand(raw_ostream &O, bool PrintType=true, const Module *M=nullptr) const
Print the name of this Value out to the specified raw_ostream.
StringRef getName() const
Return a constant reference to the value's name.
static VectorType * get(Type *ElementType, ElementCount EC)
This static method is the primary way to construct an VectorType.
constexpr bool isScalable() const
Returns whether the quantity is scaled by a runtime quantity (vscale).
Iterator for intrusive lists based on ilist_node.
self_iterator getIterator()
iterator erase(iterator where)
pointer remove(iterator &IT)
iterator insertAfter(iterator where, pointer New)
iterator insert(iterator where, pointer New)
This class implements an extremely fast bulk output stream that can only output to a stream.
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
@ C
The default llvm calling convention, compatible with C.
Function * getDeclaration(Module *M, ID id, ArrayRef< Type * > Tys=std::nullopt)
Create or insert an LLVM Function declaration for an intrinsic, and return it.
bool isUniformAfterVectorization(VPValue *VPV)
Returns true if VPV is uniform after vectorization.
bool onlyFirstLaneUsed(VPValue *Def)
Returns true if only the first lane of Def is used.
This is an optimization pass for GlobalISel generic memory operations.
void ReplaceInstWithInst(BasicBlock *BB, BasicBlock::iterator &BI, Instruction *I)
Replace the instruction specified by BI with the instruction specified by I.
bool isVectorIntrinsicWithOverloadTypeAtArg(Intrinsic::ID ID, unsigned OpdIdx)
Identifies if the vector form of the intrinsic has a operand that has an overloaded type.
auto enumerate(FirstRange &&First, RestRanges &&...Rest)
Given two or more input ranges, returns a new range whose values are are tuples (A,...
Value * getRuntimeVF(IRBuilderBase &B, Type *Ty, ElementCount VF)
Return the runtime value for VF.
iterator_range< T > make_range(T x, T y)
Convenience function for iterating over sub-ranges.
void interleaveComma(const Container &c, StreamT &os, UnaryFunctor each_fn)
bool any_of(R &&range, UnaryPredicate P)
Provide wrappers to std::any_of which take ranges instead of having to pass begin/end explicitly.
cl::opt< bool > EnableVPlanNativePath("enable-vplan-native-path", cl::Hidden, cl::desc("Enable VPlan-native vectorization path with " "support for outer loop vectorization."))
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
RecurKind
These are the kinds of recurrences that we support.
Value * createStepForVF(IRBuilderBase &B, Type *Ty, ElementCount VF, int64_t Step)
Return a value for Step multiplied by VF.
bool isVectorIntrinsicWithScalarOpAtArg(Intrinsic::ID ID, unsigned ScalarOpdIdx)
Identifies if the vector form of the intrinsic has a scalar operand.
void execute(VPTransformState &State) override
Generate the phi nodes.
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
VPIteration represents a single point in the iteration space of the output (vectorized and/or unrolle...
bool isInvariantCond() const
VPValue * getCond() const
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
void execute(VPTransformState &State) override
Produce a widened version of the select instruction.