41#define LV_NAME "loop-vectorize"
42#define DEBUG_TYPE LV_NAME
46 case VPWidenMemoryInstructionSC: {
47 return cast<VPWidenMemoryInstructionRecipe>(
this)->isStore();
53 case VPBranchOnMaskSC:
54 case VPScalarIVStepsSC:
59 case VPWidenCanonicalIVSC:
62 case VPWidenIntOrFpInductionSC:
65 case VPWidenSelectSC: {
69 assert((!
I || !
I->mayWriteToMemory()) &&
70 "underlying instruction may write to memory");
80 case VPWidenMemoryInstructionSC: {
81 return !cast<VPWidenMemoryInstructionRecipe>(
this)->isStore();
86 ->mayReadFromMemory();
87 case VPBranchOnMaskSC:
88 case VPScalarIVStepsSC:
93 case VPWidenCanonicalIVSC:
96 case VPWidenIntOrFpInductionSC:
99 case VPWidenSelectSC: {
103 assert((!
I || !
I->mayReadFromMemory()) &&
104 "underlying instruction may read from memory");
115 case VPPredInstPHISC:
117 case VPInstructionSC:
118 switch (cast<VPInstruction>(
this)->
getOpcode()) {
119 case Instruction::ICmp:
130 ->mayHaveSideEffects();
133 case VPScalarIVStepsSC:
134 case VPWidenCanonicalIVSC:
137 case VPWidenIntOrFpInductionSC:
139 case VPWidenPointerInductionSC:
141 case VPWidenSelectSC: {
145 assert((!
I || !
I->mayHaveSideEffects()) &&
146 "underlying instruction has side-effects");
149 case VPWidenMemoryInstructionSC:
150 assert(cast<VPWidenMemoryInstructionRecipe>(
this)
153 "mayHaveSideffects result for ingredient differs from this "
156 case VPReplicateSC: {
157 auto *R = cast<VPReplicateRecipe>(
this);
158 return R->getUnderlyingInstr()->mayHaveSideEffects();
173 "the middle block must not have any successors");
179#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
190 assert(!Parent &&
"Recipe already in some VPBasicBlock");
192 "Insertion position not in any VPBasicBlock");
199 assert(!Parent &&
"Recipe already in some VPBasicBlock");
206 assert(!Parent &&
"Recipe already in some VPBasicBlock");
208 "Insertion position not in any VPBasicBlock");
236 assert(OpType == OperationType::FPMathOp &&
237 "recipe doesn't have fast math flags");
255 assert(Opcode == Instruction::ICmp &&
256 "only ICmp predicates supported at the moment");
260 std::initializer_list<VPValue *>
Operands,
265 assert(isFPMathOp() &&
"this op can't take fast-math flags");
284 case Instruction::ICmp: {
289 case Instruction::Select: {
303 return Builder.CreateIntrinsic(Intrinsic::get_active_lane_mask,
305 {VIVElem0, ScalarTC},
nullptr, Name);
326 return Builder.CreateVectorSplice(PartMinus1, V2, -1, Name);
335 return Builder.CreateSelect(Cmp, Sub, Zero);
347 return State.
get(
this, 0);
379 Builder.GetInsertBlock()->getTerminator()->eraseFromParent();
403 Builder.GetInsertBlock()->getTerminator()->eraseFromParent();
412bool VPInstruction::isFPMathOp()
const {
415 return Opcode == Instruction::FAdd || Opcode == Instruction::FMul ||
416 Opcode == Instruction::FNeg || Opcode == Instruction::FSub ||
417 Opcode == Instruction::FDiv || Opcode == Instruction::FRem ||
418 Opcode == Instruction::FCmp || Opcode == Instruction::Select;
427 "Recipe not a FPMathOp but has fast-math flags?");
430 for (
unsigned Part = 0; Part < State.
UF; ++Part) {
431 Value *GeneratedValue = generateInstruction(State, Part);
434 assert(GeneratedValue &&
"generateInstruction must produce a value");
435 State.
set(
this, GeneratedValue, Part);
439#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
447 O << Indent <<
"EMIT ";
459 O <<
"combined load";
462 O <<
"combined store";
465 O <<
"active lane mask";
468 O <<
"first-order splice";
474 O <<
"branch-on-cond";
477 O <<
"TC > VF ? TC - VF : 0";
483 O <<
"branch-on-count";
502 assert(!isa<DbgInfoIntrinsic>(CI) &&
503 "DbgInfoIntrinsic should have been dropped during VPlan construction");
506 for (
unsigned Part = 0; Part < State.
UF; ++Part) {
520 Arg = State.
get(
I.value(), Part);
533 assert(VectorF &&
"Can't retrieve vector intrinsic.");
536 assert(Variant !=
nullptr &&
"Can't create vector function.");
542 CI.getOperandBundlesAsDefs(OpBundles);
545 if (isa<FPMathOperator>(V))
546 V->copyFastMathFlags(&CI);
548 State.
set(
this, V, Part);
553#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
556 O << Indent <<
"WIDEN-CALL ";
559 if (CI->getType()->isVoidTy())
566 O <<
"call @" << CI->getCalledFunction()->getName() <<
"(";
570 if (VectorIntrinsicID)
571 O <<
" (using vector intrinsic)";
573 O <<
" (using library function";
575 O <<
": " << Variant->
getName();
582 O << Indent <<
"WIDEN-SELECT ";
604 for (
unsigned Part = 0; Part < State.
UF; ++Part) {
609 State.
set(
this, Sel, Part);
614VPRecipeWithIRFlags::FastMathFlagsTy::FastMathFlagsTy(
625#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
628 case OperationType::Cmp:
631 case OperationType::PossiblyExactOp:
635 case OperationType::OverflowingBinOp:
641 case OperationType::FPMathOp:
644 case OperationType::GEPOp:
648 case OperationType::Other:
660 case Instruction::Call:
661 case Instruction::Br:
662 case Instruction::PHI:
663 case Instruction::GetElementPtr:
664 case Instruction::Select:
666 case Instruction::UDiv:
667 case Instruction::SDiv:
668 case Instruction::SRem:
669 case Instruction::URem:
670 case Instruction::Add:
671 case Instruction::FAdd:
672 case Instruction::Sub:
673 case Instruction::FSub:
674 case Instruction::FNeg:
675 case Instruction::Mul:
676 case Instruction::FMul:
677 case Instruction::FDiv:
678 case Instruction::FRem:
679 case Instruction::Shl:
680 case Instruction::LShr:
681 case Instruction::AShr:
682 case Instruction::And:
683 case Instruction::Or:
684 case Instruction::Xor: {
686 for (
unsigned Part = 0; Part < State.
UF; ++Part) {
693 if (
auto *VecOp = dyn_cast<Instruction>(V))
697 State.
set(
this, V, Part);
703 case Instruction::Freeze: {
704 for (
unsigned Part = 0; Part < State.
UF; ++Part) {
708 State.
set(
this, Freeze, Part);
712 case Instruction::ICmp:
713 case Instruction::FCmp: {
715 bool FCmp = Opcode == Instruction::FCmp;
716 for (
unsigned Part = 0; Part < State.
UF; ++Part) {
724 Builder.setFastMathFlags(
I->getFastMathFlags());
729 State.
set(
this,
C, Part);
742#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
745 O << Indent <<
"WIDEN ";
760 for (
unsigned Part = 0; Part < State.
UF; ++Part) {
763 State.
set(
this, Cast, Part);
768#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
771 O << Indent <<
"WIDEN-CAST ";
789 auto *ValVTy = cast<VectorType>(Val->
getType());
794 "Induction Step must be an integer or FP");
802 Type *InitVecValSTy =
806 Value *InitVec =
Builder.CreateStepVector(InitVecValVTy);
809 Value *StartIdxSplat =
Builder.CreateVectorSplat(VLen, StartIdx);
812 InitVec =
Builder.CreateAdd(InitVec, StartIdxSplat);
813 Step =
Builder.CreateVectorSplat(VLen, Step);
817 Step =
Builder.CreateMul(InitVec, Step);
818 return Builder.CreateAdd(Val, Step,
"induction");
822 assert((BinOp == Instruction::FAdd || BinOp == Instruction::FSub) &&
823 "Binary Opcode should be specified for FP induction");
824 InitVec =
Builder.CreateUIToFP(InitVec, ValVTy);
825 InitVec =
Builder.CreateFAdd(InitVec, StartIdxSplat);
827 Step =
Builder.CreateVectorSplat(VLen, Step);
829 return Builder.CreateBinOp(BinOp, Val, MulOp,
"induction");
844 return B.CreateUIToFP(RuntimeVF, FTy);
848 assert(!State.
Instance &&
"Int or FP induction being replicated.");
854 assert(IV->
getType() ==
ID.getStartValue()->getType() &&
"Types must match");
859 Instruction *EntryVal = Trunc ? cast<Instruction>(Trunc) : IV;
863 if (
ID.getInductionBinOp() && isa<FPMathOperator>(
ID.getInductionBinOp()))
864 Builder.setFastMathFlags(
ID.getInductionBinOp()->getFastMathFlags());
869 assert((isa<PHINode>(EntryVal) || isa<TruncInst>(EntryVal)) &&
870 "Expected either an induction phi-node or a truncate of it!");
873 auto CurrIP =
Builder.saveIP();
876 if (isa<TruncInst>(EntryVal)) {
877 assert(Start->getType()->isIntegerTy() &&
878 "Truncation requires an integer type");
879 auto *TruncType = cast<IntegerType>(EntryVal->
getType());
880 Step =
Builder.CreateTrunc(Step, TruncType);
881 Start =
Builder.CreateCast(Instruction::Trunc, Start, TruncType);
887 SplatStart, Zero, Step,
ID.getInductionOpcode(), State.
VF, State.
Builder);
894 AddOp = Instruction::Add;
895 MulOp = Instruction::Mul;
897 AddOp =
ID.getInductionOpcode();
898 MulOp = Instruction::FMul;
927 for (
unsigned Part = 0; Part < State.
UF; ++Part) {
928 State.
set(
this, LastInduction, Part);
930 if (isa<TruncInst>(EntryVal))
933 LastInduction = cast<Instruction>(
934 Builder.CreateBinOp(AddOp, LastInduction, SplatVF,
"step.add"));
938 LastInduction->
setName(
"vec.ind.next");
948#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
951 O << Indent <<
"WIDEN-INDUCTION";
955 O <<
" +\n" << Indent <<
"\" ";
973 return StartC && StartC->isZero() && StepC && StepC->isOne();
976#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
981 O << Indent <<
"= DERIVED-IV ";
989 O <<
" (truncated to " << *TruncResultTy <<
")";
1008 if (BaseIVTy != Step->
getType()) {
1012 "Truncation requires an integer step");
1021 AddOp = Instruction::Add;
1022 MulOp = Instruction::Mul;
1024 AddOp = InductionOpcode;
1025 MulOp = Instruction::FMul;
1034 Type *VecIVTy =
nullptr;
1035 Value *UnitStepVec =
nullptr, *SplatStep =
nullptr, *SplatIV =
nullptr;
1040 SplatStep =
Builder.CreateVectorSplat(State.
VF, Step);
1041 SplatIV =
Builder.CreateVectorSplat(State.
VF, BaseIV);
1044 unsigned StartPart = 0;
1045 unsigned EndPart = State.
UF;
1046 unsigned StartLane = 0;
1050 EndPart = StartPart + 1;
1051 StartLane = State.
Instance->Lane.getKnownLane();
1052 EndLane = StartLane + 1;
1054 for (
unsigned Part = StartPart; Part < EndPart; ++Part) {
1058 auto *SplatStartIdx =
Builder.CreateVectorSplat(State.
VF, StartIdx0);
1059 auto *InitVec =
Builder.CreateAdd(SplatStartIdx, UnitStepVec);
1061 InitVec =
Builder.CreateSIToFP(InitVec, VecIVTy);
1062 auto *
Mul =
Builder.CreateBinOp(MulOp, InitVec, SplatStep);
1064 State.
set(
this,
Add, Part);
1071 StartIdx0 =
Builder.CreateSIToFP(StartIdx0, BaseIVTy);
1073 for (
unsigned Lane = StartLane; Lane < EndLane; ++Lane) {
1079 "Expected StartIdx to be folded to a constant when VF is not "
1081 auto *
Mul =
Builder.CreateBinOp(MulOp, StartIdx, Step);
1088#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
1093 O <<
" = SCALAR-STEPS ";
1107 if (areAllOperandsInvariant()) {
1127 for (
unsigned Part = 0; Part < State.
UF; ++Part) {
1129 State.
set(
this, EntryPart, Part);
1140 for (
unsigned Part = 0; Part < State.
UF; ++Part) {
1143 auto *
Ptr = isPointerLoopInvariant()
1152 if (isIndexLoopInvariant(
I - 1))
1163 "NewGEP is not a pointer vector");
1164 State.
set(
this, NewGEP, Part);
1170#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
1173 O << Indent <<
"WIDEN-GEP ";
1174 O << (isPointerLoopInvariant() ?
"Inv" :
"Var");
1176 O <<
"[" << (isIndexLoopInvariant(
I) ?
"Inv" :
"Var") <<
"]";
1180 O <<
" = getelementptr";
1205 for (
unsigned In = 0; In < NumIncoming; ++In) {
1206 for (
unsigned Part = 0; Part < State.
UF; ++Part) {
1221 for (
unsigned Part = 0; Part < State.
UF; ++Part)
1222 State.
set(
this, Entry[Part], Part);
1225#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
1228 O << Indent <<
"BLEND ";
1248 O << Indent <<
"REDUCE ";
1263 O <<
" (with final reduction value stored in invariant address sank "
1272 if (
auto *PredR = dyn_cast<VPPredInstPHIRecipe>(U))
1273 return any_of(PredR->users(), [PredR](
const VPUser *U) {
1274 return !U->usesScalars(PredR);
1280#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
1283 O << Indent << (IsUniform ?
"CLONE " :
"REPLICATE ");
1292 O <<
"@" << CB->getCalledFunction()->getName() <<
"(";
1310 assert(State.
Instance &&
"Branch on Mask works only on single instance.");
1312 unsigned Part = State.
Instance->Part;
1313 unsigned Lane = State.
Instance->Lane.getKnownLane();
1315 Value *ConditionBit =
nullptr;
1318 ConditionBit = State.
get(BlockInMask, Part);
1328 assert(isa<UnreachableInst>(CurrentTerminator) &&
1329 "Expected to replace unreachable terminator with conditional branch.");
1336 assert(State.
Instance &&
"Predicated instruction PHI works per instance.");
1341 assert(PredicatingBB &&
"Predicated block has no single predecessor.");
1343 "operand must be VPReplicateRecipe");
1351 unsigned Part = State.
Instance->Part;
1359 State.
reset(
this, VPhi, Part);
1361 State.
set(
this, VPhi, Part);
1370 Phi->addIncoming(ScalarPredInst, PredicatedBB);
1381#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
1384 O << Indent <<
"PHI-PREDICATED-INSTRUCTION ";
1392 O << Indent <<
"WIDEN ";
1412 for (
unsigned Part = 0, UF = State.
UF; Part < UF; ++Part)
1413 State.
set(
this, EntryPart, Part);
1416#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
1419 O << Indent <<
"EMIT ";
1421 O <<
" = CANONICAL-INDUCTION";
1440 return StepC && StepC->
isOne();
1444 return IsScalarAfterVectorization &&
1448#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
1451 O << Indent <<
"EMIT ";
1453 O <<
" = WIDEN-POINTER-INDUCTION ";
1455 O <<
", " << *IndDesc.
getStep();
1467 "Same SCEV expanded multiple times");
1469 for (
unsigned Part = 0, UF = State.
UF; Part < UF; ++Part)
1470 State.
set(
this, Res, {Part, 0});
1473#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
1476 O << Indent <<
"EMIT ";
1478 O <<
" = EXPAND SCEV " << *Expr;
1489 :
Builder.CreateVectorSplat(VF, CanonicalIV,
"broadcast");
1490 for (
unsigned Part = 0, UF = State.
UF; Part < UF; ++Part) {
1493 VStep =
Builder.CreateVectorSplat(VF, VStep);
1497 Value *CanonicalVectorIV =
Builder.CreateAdd(VStart, VStep,
"vec.iv");
1498 State.
set(
this, CanonicalVectorIV, Part);
1502#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
1505 O << Indent <<
"EMIT ";
1507 O <<
" = WIDEN-CANONICAL-INDUCTION ";
1518 ? VectorInit->getType()
1523 auto *IdxTy =
Builder.getInt32Ty();
1528 auto *LastIdx =
Builder.CreateSub(RuntimeVF, One);
1529 VectorInit =
Builder.CreateInsertElement(
1537 State.
set(
this, EntryPart, 0);
1540#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
1543 O << Indent <<
"FIRST-ORDER-RECURRENCE-PHI ";
1558 bool ScalarPHI = State.
VF.
isScalar() || IsInLoop;
1564 "recipe must be in the vector loop header");
1565 unsigned LastPartForNewPhi =
isOrdered() ? 1 : State.
UF;
1566 for (
unsigned Part = 0; Part < LastPartForNewPhi; ++Part) {
1569 State.
set(
this, EntryPart, Part);
1579 Value *Iden =
nullptr;
1590 Builder.CreateVectorSplat(State.
VF, StartV,
"minmax.ident");
1597 Iden =
Builder.CreateVectorSplat(State.
VF, Iden);
1601 StartV =
Builder.CreateInsertElement(Iden, StartV, Zero);
1605 for (
unsigned Part = 0; Part < LastPartForNewPhi; ++Part) {
1606 Value *EntryPart = State.
get(
this, Part);
1609 Value *StartVal = (Part == 0) ? StartV : Iden;
1610 cast<PHINode>(EntryPart)->addIncoming(StartVal, VectorPH);
1614#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
1617 O << Indent <<
"WIDEN-REDUCTION-PHI ";
1627 "Non-native vplans are not expected to have VPWidenPHIRecipes.");
1632 State.
set(
this, VecPhi, 0);
1635#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
1638 O << Indent <<
"WIDEN-PHI ";
1660 for (
unsigned Part = 0, UF = State.
UF; Part < UF; ++Part) {
1666 State.
set(
this, EntryPart, Part);
1670#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
1673 O << Indent <<
"ACTIVE-LANE-MASK-PHI ";
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
mir Rename Register Operands
const SmallVectorImpl< MachineOperand > & Cond
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
This file defines the SmallVector class.
static Value * getStepVector(Value *Val, Value *StartIdx, Value *Step, Instruction::BinaryOps BinOp, ElementCount VF, IRBuilderBase &Builder)
This function adds (StartIdx * Step, (StartIdx + 1) * Step, (StartIdx + 2) * Step,...
static Constant * getSignedIntOrFpConstant(Type *Ty, int64_t C)
A helper function that returns an integer or floating-point constant with value C.
static Value * getRuntimeVFAsFloat(IRBuilderBase &B, Type *FTy, ElementCount VF)
static std::optional< unsigned > getOpcode(ArrayRef< VPValue * > Values)
Returns the opcode of Values or ~0 if they do not all agree.
This file contains the declarations of the Vectorization Plan base classes:
static const uint32_t IV[8]
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
LLVM Basic Block Representation.
const_iterator getFirstInsertionPt() const
Returns an iterator to the first instruction in this block that is suitable for inserting a non-PHI i...
const BasicBlock * getSinglePredecessor() const
Return the predecessor of this block if it has a single predecessor block.
const Instruction * getTerminator() const LLVM_READONLY
Returns the terminator instruction if the block is well formed or null if the block is not well forme...
const Module * getModule() const
Return the module owning the function this basic block belongs to, or nullptr if the function does no...
Conditional or Unconditional Branch instruction.
static BranchInst * Create(BasicBlock *IfTrue, Instruction *InsertBefore=nullptr)
void setSuccessor(unsigned idx, BasicBlock *NewSucc)
This class represents a function call, abstracting a target machine's calling convention.
Predicate
This enumeration lists the possible predicates for CmpInst subclasses.
@ ICMP_UGT
unsigned greater than
static StringRef getPredicateName(Predicate P)
static Constant * get(Type *Ty, double V)
This returns a ConstantFP, or a vector containing a splat of a ConstantFP, for the specified value in...
This is the shared class of boolean and integer constants.
bool isOne() const
This is just a convenience method to make client code smaller for a common case.
static Constant * get(Type *Ty, uint64_t V, bool IsSigned=false)
If Ty is a vector type, return a Constant with a splat of the given value.
static ConstantInt * getSigned(IntegerType *Ty, int64_t V)
Return a ConstantInt with the specified value for the specified type.
static Constant * getSplat(ElementCount EC, Constant *Elt)
Return a ConstantVector with the specified constant in each element.
This is an important base class in LLVM.
This class represents an Operation in the Expression.
A parsed version of the target data layout string in and methods for querying it.
constexpr bool isVector() const
One or more elements.
constexpr bool isScalar() const
Exactly one element.
Convenience struct for specifying and reasoning about fast-math flags.
void setAllowContract(bool B=true)
bool noSignedZeros() const
void setAllowReciprocal(bool B=true)
bool allowReciprocal() const
void print(raw_ostream &O) const
Print fast-math flags to O.
void setNoSignedZeros(bool B=true)
bool allowReassoc() const
Flag queries.
void setNoNaNs(bool B=true)
void setAllowReassoc(bool B=true)
Flag setters.
void setApproxFunc(bool B=true)
void setNoInfs(bool B=true)
bool allowContract() const
Common base class shared among various IRBuilders.
Value * CreateTrunc(Value *V, Type *DestTy, const Twine &Name="")
Value * CreateExtractElement(Value *Vec, Value *Idx, const Twine &Name="")
Value * CreateVectorSplat(unsigned NumElts, Value *V, const Twine &Name="")
Return a vector value that contains.
ConstantInt * getTrue()
Get the constant value for i1 true.
Value * CreateSelect(Value *C, Value *True, Value *False, const Twine &Name="", Instruction *MDFrom=nullptr)
BasicBlock::iterator GetInsertPoint() const
BasicBlock * GetInsertBlock() const
void setFastMathFlags(FastMathFlags NewFMF)
Set the fast-math flags to be used with generated fp-math operators.
ConstantInt * getInt32(uint32_t C)
Get a constant 32-bit value.
PHINode * CreatePHI(Type *Ty, unsigned NumReservedValues, const Twine &Name="")
CallInst * CreateCall(FunctionType *FTy, Value *Callee, ArrayRef< Value * > Args=std::nullopt, const Twine &Name="", MDNode *FPMathTag=nullptr)
Value * CreateGEP(Type *Ty, Value *Ptr, ArrayRef< Value * > IdxList, const Twine &Name="", bool IsInBounds=false)
This provides a uniform API for creating instructions and inserting them into a basic block: either a...
A struct for saving information about induction variables.
const SCEV * getStep() const
InductionKind
This enum represents the kinds of inductions that we support.
@ IK_IntInduction
Integer induction variable. Step = C.
This instruction inserts a single (scalar) element into a VectorType value.
VectorType * getType() const
Overload to return most specific vector type.
void insertBefore(Instruction *InsertPos)
Insert an unlinked instruction into a basic block immediately before the specified instruction.
const DebugLoc & getDebugLoc() const
Return the debug location for this node as a DebugLoc.
const BasicBlock * getParent() const
FastMathFlags getFastMathFlags() const LLVM_READONLY
Convenience function for getting all the fast-math flags, which must be an operator which supports th...
const char * getOpcodeName() const
unsigned getOpcode() const
Returns a member of one of the enums like Instruction::Add.
void setDebugLoc(DebugLoc Loc)
Set the debug location information for this instruction.
static IntegerType * get(LLVMContext &C, unsigned NumBits)
This static method is the primary way of constructing an IntegerType.
BlockT * getHeader() const
void print(raw_ostream &OS, const SlotIndexes *=nullptr, bool IsStandalone=true) const
A Module instance is used to store all the information related to an LLVM module.
const DataLayout & getDataLayout() const
Get the data layout for the module's target platform.
void addIncoming(Value *V, BasicBlock *BB)
Add an incoming value to the end of the PHI list.
static PHINode * Create(Type *Ty, unsigned NumReservedValues, const Twine &NameStr="", Instruction *InsertBefore=nullptr)
Constructors - NumReservedValues is a hint for the number of incoming edges that this phi node will h...
static PoisonValue * get(Type *T)
Static factory methods - Return an 'poison' object of the specified type.
FastMathFlags getFastMathFlags() const
static unsigned getOpcode(RecurKind Kind)
Returns the opcode corresponding to the RecurrenceKind.
static bool isAnyOfRecurrenceKind(RecurKind Kind)
Returns true if the recurrence kind is of the form select(cmp(),x,y) where one of (x,...
RecurKind getRecurrenceKind() const
Value * getRecurrenceIdentity(RecurKind K, Type *Tp, FastMathFlags FMF) const
Returns identity corresponding to the RecurrenceKind.
StoreInst * IntermediateStore
Reductions may store temporary or final result to an invariant address.
static bool isMinMaxRecurrenceKind(RecurKind Kind)
Returns true if the recurrence kind is any min/max kind.
This class uses information about analyze scalars to rewrite expressions in canonical form.
Type * getType() const
Return the LLVM type of this SCEV expression.
This class provides computation of slot numbers for LLVM Assembly writing.
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
This class represents a truncation of integer types.
Twine - A lightweight data structure for efficiently representing the concatenation of temporary valu...
The instances of the Type class are immutable: once they are created, they are never changed.
bool isVectorTy() const
True if this is an instance of VectorType.
static IntegerType * getInt1Ty(LLVMContext &C)
unsigned getScalarSizeInBits() const LLVM_READONLY
If this is a vector type, return the getPrimitiveSizeInBits value for the element type.
LLVMContext & getContext() const
Return the LLVMContext in which this type was uniqued.
bool isFloatingPointTy() const
Return true if this is one of the floating-point types.
bool isIntegerTy() const
True if this is an instance of IntegerType.
bool isVoidTy() const
Return true if this is 'void'.
Type * getScalarType() const
If this is a vector type, return the element type, otherwise return 'this'.
Value * getOperand(unsigned i) const
void execute(VPTransformState &State) override
Generate the active lane mask phi of the vector loop.
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
VPBasicBlock serves as the leaf of the Hierarchical Control-Flow Graph.
RecipeListTy & getRecipeList()
Returns a reference to the list of recipes.
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
VPValue * getIncomingValue(unsigned Idx) const
Return incoming value number Idx.
VPValue * getMask(unsigned Idx) const
Return mask number Idx.
unsigned getNumIncomingValues() const
Return the number of incoming values, taking into account that a single incoming value has no mask.
void execute(VPTransformState &State) override
Generate the phi/select nodes.
VPRegionBlock * getParent()
size_t getNumSuccessors() const
const VPBasicBlock * getEntryBasicBlock() const
VPBlockBase * getSingleSuccessor() const
VPValue * getMask() const
Return the mask used by this recipe.
void execute(VPTransformState &State) override
Generate the extraction of the appropriate bit from the block mask and the conditional branch.
bool isCanonical(InductionDescriptor::InductionKind Kind, VPValue *Start, VPValue *Step, Type *Ty) const
Check if the induction described by Kind, /p Start and Step is canonical, i.e.
void execute(VPTransformState &State) override
Generate the canonical scalar induction phi of the vector loop.
const Type * getScalarType() const
Returns the scalar type of the induction.
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
This class augments a recipe with a set of VPValues defined by the recipe.
VPValue * getVPSingleValue()
Returns the only VPValue defined by the VPDef.
VPValue * getVPValue(unsigned I)
Returns the VPValue with index I defined by the VPDef.
unsigned getVPDefID() const
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
VPValue * getStepValue() const
VPValue * getCanonicalIV() const
VPValue * getStartValue() const
void execute(VPTransformState &State) override
Generate a canonical vector induction variable of the vector loop, with.
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
@ FirstOrderRecurrenceSplice
@ CanonicalIVIncrementForPart
@ CalculateTripCountMinusVF
LLVM_DUMP_METHOD void dump() const
Print the VPInstruction to dbgs() (for debugging).
unsigned getOpcode() const
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the VPInstruction to O.
void execute(VPTransformState &State) override
Generate the instruction.
static VPLane getLastLaneForVF(const ElementCount &VF)
static VPLane getFirstLane()
void print(raw_ostream &O, VPSlotTracker &SlotTracker) const
Print the VPLiveOut to O.
void fixPhi(VPlan &Plan, VPTransformState &State)
Fixup the wrapped LCSSA phi node in the unique exit block.
void execute(VPTransformState &State) override
Generates phi nodes for live-outs as needed to retain SSA form.
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
VPRecipeBase is a base class modeling a sequence of one or more output IR instructions.
bool mayReadFromMemory() const
Returns true if the recipe may read from memory.
bool mayHaveSideEffects() const
Returns true if the recipe may have side-effects.
Instruction * getUnderlyingInstr()
Returns the underlying instruction, if the recipe is a VPValue or nullptr otherwise.
bool mayWriteToMemory() const
Returns true if the recipe may write to memory.
VPBasicBlock * getParent()
DebugLoc getDebugLoc() const
Returns the debug location of the recipe.
void moveBefore(VPBasicBlock &BB, iplist< VPRecipeBase >::iterator I)
Unlink this recipe and insert into BB before I.
void insertBefore(VPRecipeBase *InsertPos)
Insert an unlinked recipe into a basic block immediately before the specified recipe.
void insertAfter(VPRecipeBase *InsertPos)
Insert an unlinked Recipe into a basic block immediately after the specified Recipe.
iplist< VPRecipeBase >::iterator eraseFromParent()
This method unlinks 'this' from the containing basic block and deletes it.
void removeFromParent()
This method unlinks 'this' from the containing basic block, but does not delete it.
void moveAfter(VPRecipeBase *MovePos)
Unlink this recipe from its current VPBasicBlock and insert it into the VPBasicBlock that MovePos liv...
Class to record LLVM IR flag for a recipe along with it.
void setFlags(Instruction *I) const
Set the IR flags for I.
bool hasFastMathFlags() const
Returns true if the recipe has fast-math flags.
bool hasNoUnsignedWrap() const
void printFlags(raw_ostream &O) const
CmpInst::Predicate getPredicate() const
bool hasNoSignedWrap() const
FastMathFlags getFastMathFlags() const
bool isOrdered() const
Returns true, if the phi is part of an ordered reduction.
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
void execute(VPTransformState &State) override
Generate the phi/select nodes.
VPValue * getVecOp() const
The VPValue of the vector value to be reduced.
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
VPValue * getCondOp() const
The VPValue of the condition for the block.
VPValue * getChainOp() const
The VPValue of the scalar Chain being accumulated.
VPRegionBlock represents a collection of VPBasicBlocks and VPRegionBlocks which form a Single-Entry-S...
const VPBlockBase * getEntry() const
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
bool shouldPack() const
Returns true if the recipe is used by a widened recipe via an intervening VPPredInstPHIRecipe.
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
VPValue * getStepValue() const
void execute(VPTransformState &State) override
Generate the scalarized versions of the phi node as needed by their users.
This class can be used to assign consecutive numbers to all VPValues in a VPlan and allows querying t...
This class augments VPValue with operands which provide the inverse def-use edges from VPValue's user...
void printOperands(raw_ostream &O, VPSlotTracker &SlotTracker) const
Print the operands to O.
unsigned getNumOperands() const
operand_iterator op_begin()
VPValue * getOperand(unsigned N) const
Value * getUnderlyingValue()
Return the underlying Value attached to this VPValue.
VPRecipeBase * getDefiningRecipe()
Returns the recipe defining this VPValue or nullptr if it is not defined by a recipe,...
void printAsOperand(raw_ostream &OS, VPSlotTracker &Tracker) const
friend class VPInstruction
Value * getLiveInIRValue()
Returns the underlying IR value, if this VPValue is defined outside the scope of VPlan.
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
void execute(VPTransformState &State) override
Produce a widened version of the call instruction.
void execute(VPTransformState &State) override
Generate a canonical vector induction variable of the vector loop, with start = {<Part*VF,...
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
Type * getResultType() const
Returns the result type of the cast.
void execute(VPTransformState &State) override
Produce widened copies of the cast.
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
void execute(VPTransformState &State) override
Generate the gep nodes.
TruncInst * getTruncInst()
Returns the first defined value as TruncInst, if it is one or nullptr otherwise.
void execute(VPTransformState &State) override
Generate the vectorized and scalarized versions of the phi node as needed by their users.
VPValue * getStepValue()
Returns the step value of the induction.
bool isCanonical() const
Returns true if the induction is canonical, i.e.
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
const InductionDescriptor & getInductionDescriptor() const
Returns the induction descriptor for the recipe.
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
bool isStore() const
Returns true if this recipe is a store.
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
void execute(VPTransformState &State) override
Generate the phi/select nodes.
bool onlyScalarsGenerated(ElementCount VF)
Returns true if only scalar values will be generated.
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
void execute(VPTransformState &State) override
Produce widened copies of all Ingredients.
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
VPlan models a candidate for vectorization, encoding various decisions take to produce efficient outp...
VPRegionBlock * getVectorLoopRegion()
Returns the VPRegionBlock of the vector loop.
LLVM Value Representation.
Type * getType() const
All values are typed, get the type of this value.
void setName(const Twine &Name)
Change the name of the value.
void printAsOperand(raw_ostream &O, bool PrintType=true, const Module *M=nullptr) const
Print the name of this Value out to the specified raw_ostream.
StringRef getName() const
Return a constant reference to the value's name.
static VectorType * get(Type *ElementType, ElementCount EC)
This static method is the primary way to construct an VectorType.
constexpr bool isScalable() const
Returns whether the quantity is scaled by a runtime quantity (vscale).
constexpr ScalarTy getKnownMinValue() const
Returns the minimum value this quantity can represent.
Iterator for intrusive lists based on ilist_node.
self_iterator getIterator()
iterator erase(iterator where)
pointer remove(iterator &IT)
iterator insertAfter(iterator where, pointer New)
iterator insert(iterator where, pointer New)
This class implements an extremely fast bulk output stream that can only output to a stream.
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
@ C
The default llvm calling convention, compatible with C.
Function * getDeclaration(Module *M, ID id, ArrayRef< Type * > Tys=std::nullopt)
Create or insert an LLVM Function declaration for an intrinsic, and return it.
NodeAddr< PhiNode * > Phi
bool isUniformAfterVectorization(VPValue *VPV)
Returns true if VPV is uniform after vectorization.
bool onlyFirstLaneUsed(VPValue *Def)
Returns true if only the first lane of Def is used.
This is an optimization pass for GlobalISel generic memory operations.
void ReplaceInstWithInst(BasicBlock *BB, BasicBlock::iterator &BI, Instruction *I)
Replace the instruction specified by BI with the instruction specified by I.
auto enumerate(FirstRange &&First, RestRanges &&...Rest)
Given two or more input ranges, returns a new range whose values are are tuples (A,...
bool isVectorIntrinsicWithOverloadTypeAtArg(Intrinsic::ID ID, int OpdIdx)
Identifies if the vector form of the intrinsic is overloaded on the type of the operand at index OpdI...
Value * getRuntimeVF(IRBuilderBase &B, Type *Ty, ElementCount VF)
Return the runtime value for VF.
iterator_range< T > make_range(T x, T y)
Convenience function for iterating over sub-ranges.
void interleaveComma(const Container &c, StreamT &os, UnaryFunctor each_fn)
bool any_of(R &&range, UnaryPredicate P)
Provide wrappers to std::any_of which take ranges instead of having to pass begin/end explicitly.
cl::opt< bool > EnableVPlanNativePath("enable-vplan-native-path", cl::Hidden, cl::desc("Enable VPlan-native vectorization path with " "support for outer loop vectorization."))
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
RecurKind
These are the kinds of recurrences that we support.
@ Mul
Product of integers.
Value * createStepForVF(IRBuilderBase &B, Type *Ty, ElementCount VF, int64_t Step)
Return a value for Step multiplied by VF.
bool isVectorIntrinsicWithScalarOpAtArg(Intrinsic::ID ID, unsigned ScalarOpdIdx)
Identifies if the vector form of the intrinsic has a scalar operand.
void execute(VPTransformState &State) override
Generate the phi nodes.
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
VPIteration represents a single point in the iteration space of the output (vectorized and/or unrolle...
bool isInvariantCond() const
VPValue * getCond() const
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
void execute(VPTransformState &State) override
Produce a widened version of the select instruction.