43#define LV_NAME "loop-vectorize"
44#define DEBUG_TYPE LV_NAME
49 return cast<VPInterleaveRecipe>(
this)->getNumStoreOperands() > 0;
50 case VPWidenMemoryInstructionSC: {
51 return cast<VPWidenMemoryInstructionRecipe>(
this)->isStore();
57 case VPBranchOnMaskSC:
58 case VPScalarIVStepsSC:
63 case VPWidenCanonicalIVSC:
66 case VPWidenIntOrFpInductionSC:
69 case VPWidenSelectSC: {
73 assert((!
I || !
I->mayWriteToMemory()) &&
74 "underlying instruction may write to memory");
84 case VPWidenMemoryInstructionSC: {
85 return !cast<VPWidenMemoryInstructionRecipe>(
this)->isStore();
90 ->mayReadFromMemory();
91 case VPBranchOnMaskSC:
92 case VPScalarIVStepsSC:
97 case VPWidenCanonicalIVSC:
100 case VPWidenIntOrFpInductionSC:
103 case VPWidenSelectSC: {
107 assert((!
I || !
I->mayReadFromMemory()) &&
108 "underlying instruction may read from memory");
119 case VPPredInstPHISC:
122 case VPInstructionSC:
123 switch (cast<VPInstruction>(
this)->
getOpcode()) {
124 case Instruction::Or:
125 case Instruction::ICmp:
126 case Instruction::Select:
137 ->mayHaveSideEffects();
140 case VPScalarIVStepsSC:
141 case VPWidenCanonicalIVSC:
144 case VPWidenIntOrFpInductionSC:
146 case VPWidenPointerInductionSC:
148 case VPWidenSelectSC: {
152 assert((!
I || !
I->mayHaveSideEffects()) &&
153 "underlying instruction has side-effects");
158 case VPWidenMemoryInstructionSC:
159 assert(cast<VPWidenMemoryInstructionRecipe>(
this)
162 "mayHaveSideffects result for ingredient differs from this "
165 case VPReplicateSC: {
166 auto *R = cast<VPReplicateRecipe>(
this);
167 return R->getUnderlyingInstr()->mayHaveSideEffects();
182 "the middle block must not have any successors");
188#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
199 assert(!Parent &&
"Recipe already in some VPBasicBlock");
201 "Insertion position not in any VPBasicBlock");
207 assert(!Parent &&
"Recipe already in some VPBasicBlock");
213 assert(!Parent &&
"Recipe already in some VPBasicBlock");
215 "Insertion position not in any VPBasicBlock");
242 assert(OpType == OperationType::FPMathOp &&
243 "recipe doesn't have fast math flags");
261 assert(Opcode == Instruction::ICmp &&
262 "only ICmp predicates supported at the moment");
266 std::initializer_list<VPValue *>
Operands,
271 assert(isFPMathOp() &&
"this op can't take fast-math flags");
274bool VPInstruction::doesGeneratePerAllLanes()
const {
278bool VPInstruction::canGenerateScalarForFirstLane()
const {
301 "only PtrAdd opcodes are supported for now");
312 return State.
get(
this, 0, OnlyFirstLaneUsed);
318 if (
auto *
I = dyn_cast<Instruction>(Res))
328 case Instruction::ICmp: {
333 case Instruction::Select: {
355 {VIVElem0, ScalarTC},
nullptr, Name);
380 return State.
get(
this, 0,
true);
393 assert(AVL->getType()->isIntegerTy() &&
394 "Requested vector length should be an integer.");
402 {AVL, VFArg, State.Builder.getTrue()});
407 assert(Part == 0 &&
"No unrolling expected for predicated vectorization.");
412 Value *EVL = GetEVL(State, AVL);
413 assert(!State.
EVL &&
"multiple EVL recipes");
475 return State.
get(
this, 0,
true);
479 auto *PhiR = cast<VPReductionPHIRecipe>(
getOperand(0));
480 auto *OrigPhi = cast<PHINode>(PhiR->getUnderlyingValue());
487 Type *PhiTy = OrigPhi->getType();
489 for (
unsigned Part = 0; Part < State.
UF; ++Part)
490 RdxParts[Part] = State.
get(LoopExitingDef, Part, PhiR->isInLoop());
498 for (
unsigned Part = 0; Part < State.
UF; ++Part)
499 RdxParts[Part] = Builder.
CreateTrunc(RdxParts[Part], RdxVecTy);
502 Value *ReducedPartRdx = RdxParts[0];
505 Op = Instruction::Or;
507 if (PhiR->isOrdered()) {
508 ReducedPartRdx = RdxParts[State.
UF - 1];
513 for (
unsigned Part = 1; Part < State.
UF; ++Part) {
514 Value *RdxPart = RdxParts[Part];
515 if (
Op != Instruction::ICmp &&
Op != Instruction::FCmp)
519 ReducedPartRdx =
createMinMaxOp(Builder, RK, ReducedPartRdx, RdxPart);
542 ReducedPartRdx,
SI->getPointerOperand(),
SI->getAlign());
546 return ReducedPartRdx;
550 "can only generate first lane for PtrAdd");
561bool VPInstruction::isFPMathOp()
const {
564 return Opcode == Instruction::FAdd || Opcode == Instruction::FMul ||
565 Opcode == Instruction::FNeg || Opcode == Instruction::FSub ||
566 Opcode == Instruction::FDiv || Opcode == Instruction::FRem ||
567 Opcode == Instruction::FCmp || Opcode == Instruction::Select;
576 "Recipe not a FPMathOp but has fast-math flags?");
580 bool GeneratesPerFirstLaneOnly =
581 canGenerateScalarForFirstLane() &&
584 bool GeneratesPerAllLanes = doesGeneratePerAllLanes();
585 for (
unsigned Part = 0; Part < State.
UF; ++Part) {
586 if (GeneratesPerAllLanes) {
588 Lane != NumLanes; ++Lane) {
590 assert(GeneratedValue &&
"generatePerLane must produce a value");
596 Value *GeneratedValue = generatePerPart(State, Part);
599 assert(GeneratedValue &&
"generatePerPart must produce a value");
601 !GeneratesPerFirstLaneOnly ||
603 "scalar value but not only first lane defined");
604 State.
set(
this, GeneratedValue, Part,
605 GeneratesPerFirstLaneOnly);
617 case Instruction::ICmp:
631#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
639 O << Indent <<
"EMIT ";
651 O <<
"combined load";
654 O <<
"combined store";
657 O <<
"active lane mask";
660 O <<
"EXPLICIT-VECTOR-LENGTH";
663 O <<
"first-order splice";
666 O <<
"branch-on-cond";
669 O <<
"TC > VF ? TC - VF : 0";
675 O <<
"branch-on-count";
678 O <<
"compute-reduction-result";
700 assert(!isa<DbgInfoIntrinsic>(CI) &&
701 "DbgInfoIntrinsic should have been dropped during VPlan construction");
708 for (
unsigned Part = 0; Part < State.
UF; ++Part) {
729 Arg = State.
get(
I.value(), Part);
741 assert(VectorF &&
"Can't retrieve vector intrinsic.");
744 assert(Variant !=
nullptr &&
"Can't create vector function.");
750 CI.getOperandBundlesAsDefs(OpBundles);
753 if (isa<FPMathOperator>(V))
754 V->copyFastMathFlags(&CI);
756 if (!V->getType()->isVoidTy())
757 State.
set(
this, V, Part);
762#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
765 O << Indent <<
"WIDEN-CALL ";
768 if (CI->getType()->isVoidTy())
775 O <<
"call @" << CI->getCalledFunction()->getName() <<
"(";
779 if (VectorIntrinsicID)
780 O <<
" (using vector intrinsic)";
782 O <<
" (using library function";
784 O <<
": " << Variant->
getName();
791 O << Indent <<
"WIDEN-SELECT ";
813 for (
unsigned Part = 0; Part < State.
UF; ++Part) {
818 State.
set(
this, Sel, Part);
823VPRecipeWithIRFlags::FastMathFlagsTy::FastMathFlagsTy(
834#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
837 case OperationType::Cmp:
840 case OperationType::DisjointOp:
844 case OperationType::PossiblyExactOp:
848 case OperationType::OverflowingBinOp:
854 case OperationType::FPMathOp:
857 case OperationType::GEPOp:
861 case OperationType::NonNegOp:
865 case OperationType::Other:
877 case Instruction::Call:
878 case Instruction::Br:
879 case Instruction::PHI:
880 case Instruction::GetElementPtr:
881 case Instruction::Select:
883 case Instruction::UDiv:
884 case Instruction::SDiv:
885 case Instruction::SRem:
886 case Instruction::URem:
887 case Instruction::Add:
888 case Instruction::FAdd:
889 case Instruction::Sub:
890 case Instruction::FSub:
891 case Instruction::FNeg:
892 case Instruction::Mul:
893 case Instruction::FMul:
894 case Instruction::FDiv:
895 case Instruction::FRem:
896 case Instruction::Shl:
897 case Instruction::LShr:
898 case Instruction::AShr:
899 case Instruction::And:
900 case Instruction::Or:
901 case Instruction::Xor: {
903 for (
unsigned Part = 0; Part < State.
UF; ++Part) {
910 if (
auto *VecOp = dyn_cast<Instruction>(V))
914 State.
set(
this, V, Part);
920 case Instruction::Freeze: {
921 for (
unsigned Part = 0; Part < State.
UF; ++Part) {
925 State.
set(
this, Freeze, Part);
929 case Instruction::ICmp:
930 case Instruction::FCmp: {
932 bool FCmp = Opcode == Instruction::FCmp;
933 for (
unsigned Part = 0; Part < State.
UF; ++Part) {
946 State.
set(
this,
C, Part);
962 for (
unsigned Part = 0; Part < State.
UF; ++Part) {
965 "inferred type and type from generated instructions do not match");
970#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
973 O << Indent <<
"WIDEN ";
988 for (
unsigned Part = 0; Part < State.
UF; ++Part) {
989 if (Part > 0 &&
Op->isLiveIn()) {
991 State.
set(
this, State.
get(
this, 0), Part);
996 State.
set(
this, Cast, Part);
1001#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
1004 O << Indent <<
"WIDEN-CAST ";
1023 auto *ValVTy = cast<VectorType>(Val->
getType());
1028 "Induction Step must be an integer or FP");
1036 Type *InitVecValSTy =
1046 InitVec = Builder.
CreateAdd(InitVec, StartIdxSplat);
1051 Step = Builder.
CreateMul(InitVec, Step);
1052 return Builder.
CreateAdd(Val, Step,
"induction");
1056 assert((BinOp == Instruction::FAdd || BinOp == Instruction::FSub) &&
1057 "Binary Opcode should be specified for FP induction");
1059 InitVec = Builder.
CreateFAdd(InitVec, StartIdxSplat);
1063 return Builder.
CreateBinOp(BinOp, Val, MulOp,
"induction");
1070 : ConstantFP::get(Ty,
C);
1078 return B.CreateUIToFP(RuntimeVF, FTy);
1082 assert(!State.
Instance &&
"Int or FP induction being replicated.");
1088 assert(IV->
getType() ==
ID.getStartValue()->getType() &&
"Types must match");
1093 Instruction *EntryVal = Trunc ? cast<Instruction>(Trunc) : IV;
1097 if (
ID.getInductionBinOp() && isa<FPMathOperator>(
ID.getInductionBinOp()))
1103 assert((isa<PHINode>(EntryVal) || isa<TruncInst>(EntryVal)) &&
1104 "Expected either an induction phi-node or a truncate of it!");
1107 auto CurrIP = Builder.
saveIP();
1110 if (isa<TruncInst>(EntryVal)) {
1111 assert(Start->getType()->isIntegerTy() &&
1112 "Truncation requires an integer type");
1113 auto *TruncType = cast<IntegerType>(EntryVal->
getType());
1115 Start = Builder.
CreateCast(Instruction::Trunc, Start, TruncType);
1121 SplatStart, Zero, Step,
ID.getInductionOpcode(), State.
VF, State.
Builder);
1128 AddOp = Instruction::Add;
1129 MulOp = Instruction::Mul;
1131 AddOp =
ID.getInductionOpcode();
1132 MulOp = Instruction::FMul;
1150 Value *SplatVF = isa<Constant>(
Mul)
1161 for (
unsigned Part = 0; Part < State.
UF; ++Part) {
1162 State.
set(
this, LastInduction, Part);
1164 if (isa<TruncInst>(EntryVal))
1167 LastInduction = cast<Instruction>(
1168 Builder.
CreateBinOp(AddOp, LastInduction, SplatVF,
"step.add"));
1172 LastInduction->
setName(
"vec.ind.next");
1182#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
1185 O << Indent <<
"WIDEN-INDUCTION";
1189 O <<
" +\n" << Indent <<
"\" ";
1207 return StartC && StartC->isZero() && StepC && StepC->isOne();
1210#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
1215 O << Indent <<
"= DERIVED-IV ";
1239 assert(BaseIVTy == Step->
getType() &&
"Types of BaseIV and Step must match!");
1246 AddOp = Instruction::Add;
1247 MulOp = Instruction::Mul;
1249 AddOp = InductionOpcode;
1250 MulOp = Instruction::FMul;
1259 Type *VecIVTy =
nullptr;
1260 Value *UnitStepVec =
nullptr, *SplatStep =
nullptr, *SplatIV =
nullptr;
1269 unsigned StartPart = 0;
1270 unsigned EndPart = State.
UF;
1271 unsigned StartLane = 0;
1275 EndPart = StartPart + 1;
1276 StartLane = State.
Instance->Lane.getKnownLane();
1277 EndLane = StartLane + 1;
1279 for (
unsigned Part = StartPart; Part < EndPart; ++Part) {
1284 auto *InitVec = Builder.
CreateAdd(SplatStartIdx, UnitStepVec);
1289 State.
set(
this,
Add, Part);
1298 for (
unsigned Lane = StartLane; Lane < EndLane; ++Lane) {
1304 "Expected StartIdx to be folded to a constant when VF is not "
1313#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
1318 O <<
" = SCALAR-STEPS ";
1332 if (areAllOperandsInvariant()) {
1352 for (
unsigned Part = 0; Part < State.
UF; ++Part) {
1354 State.
set(
this, EntryPart, Part);
1365 for (
unsigned Part = 0; Part < State.
UF; ++Part) {
1368 auto *
Ptr = isPointerLoopInvariant()
1377 if (isIndexLoopInvariant(
I - 1))
1388 "NewGEP is not a pointer vector");
1389 State.
set(
this, NewGEP, Part);
1395#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
1398 O << Indent <<
"WIDEN-GEP ";
1399 O << (isPointerLoopInvariant() ?
"Inv" :
"Var");
1401 O <<
"[" << (isIndexLoopInvariant(
I) ?
"Inv" :
"Var") <<
"]";
1405 O <<
" = getelementptr";
1412 auto &Builder = State.
Builder;
1414 for (
unsigned Part = 0; Part < State.
UF; ++Part) {
1416 Value *PartPtr =
nullptr;
1422 ?
DL.getIndexType(IndexedTy->getPointerTo())
1425 bool InBounds = isInBounds();
1434 ConstantInt::get(IndexTy, -(int64_t)Part), RunTimeVF);
1437 Builder.
CreateSub(ConstantInt::get(IndexTy, 1), RunTimeVF);
1438 PartPtr = Builder.
CreateGEP(IndexedTy,
Ptr, NumElt,
"", InBounds);
1439 PartPtr = Builder.
CreateGEP(IndexedTy, PartPtr, LastLane,
"", InBounds);
1442 PartPtr = Builder.
CreateGEP(IndexedTy,
Ptr, Increment,
"", InBounds);
1445 State.
set(
this, PartPtr, Part,
true);
1449#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
1454 O <<
" = vector-pointer ";
1481 for (
unsigned In = 0; In < NumIncoming; ++In) {
1482 for (
unsigned Part = 0; Part < State.
UF; ++Part) {
1497 for (
unsigned Part = 0; Part < State.
UF; ++Part)
1498 State.
set(
this, Entry[Part], Part);
1501#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
1504 O << Indent <<
"BLEND ";
1532 for (
unsigned Part = 0; Part < State.
UF; ++Part) {
1557 PrevInChain = NewRed;
1564 NewRed, PrevInChain);
1565 }
else if (IsOrdered)
1566 NextInChain = NewRed;
1570 State.
set(
this, NextInChain, Part,
true);
1574#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
1577 O << Indent <<
"REDUCE ";
1592 O <<
" (with final reduction value stored in invariant address sank "
1601 if (
auto *PredR = dyn_cast<VPPredInstPHIRecipe>(U))
1602 return any_of(PredR->users(), [PredR](
const VPUser *U) {
1603 return !U->usesScalars(PredR);
1609#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
1612 O << Indent << (IsUniform ?
"CLONE " :
"REPLICATE ");
1621 O <<
"@" << CB->getCalledFunction()->getName() <<
"(";
1644 return C->isDefinedOutsideVectorRegions() ||
1645 isa<VPDerivedIVRecipe>(
C->getOperand(0)) ||
1646 isa<VPCanonicalIVPHIRecipe>(
C->getOperand(0));
1651 "Codegen only implemented for first lane.");
1653 case Instruction::SExt:
1654 case Instruction::ZExt:
1655 case Instruction::Trunc: {
1667 for (
unsigned Part = 0; Part != State.
UF; ++Part) {
1671 if (Part > 0 && IsUniformAcrossVFsAndUFs)
1674 Res = generate(State, Part);
1679#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
1682 O << Indent <<
"SCALAR-CAST ";
1686 O <<
" to " << *ResultTy;
1691 assert(State.
Instance &&
"Branch on Mask works only on single instance.");
1693 unsigned Part = State.
Instance->Part;
1694 unsigned Lane = State.
Instance->Lane.getKnownLane();
1696 Value *ConditionBit =
nullptr;
1699 ConditionBit = State.
get(BlockInMask, Part);
1709 assert(isa<UnreachableInst>(CurrentTerminator) &&
1710 "Expected to replace unreachable terminator with conditional branch.");
1717 assert(State.
Instance &&
"Predicated instruction PHI works per instance.");
1722 assert(PredicatingBB &&
"Predicated block has no single predecessor.");
1724 "operand must be VPReplicateRecipe");
1732 unsigned Part = State.
Instance->Part;
1740 State.
reset(
this, VPhi, Part);
1742 State.
set(
this, VPhi, Part);
1751 Phi->addIncoming(ScalarPredInst, PredicatedBB);
1762#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
1765 O << Indent <<
"PHI-PREDICATED-INSTRUCTION ";
1773 O << Indent <<
"WIDEN ";
1793 for (
unsigned Part = 0, UF = State.
UF; Part < UF; ++Part)
1794 State.
set(
this, EntryPart, Part,
true);
1797#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
1800 O << Indent <<
"EMIT ";
1802 O <<
" = CANONICAL-INDUCTION ";
1822 return StepC && StepC->
isOne();
1826 return IsScalarAfterVectorization &&
1830#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
1833 O << Indent <<
"EMIT ";
1835 O <<
" = WIDEN-POINTER-INDUCTION ";
1837 O <<
", " << *IndDesc.
getStep();
1849 "Same SCEV expanded multiple times");
1851 for (
unsigned Part = 0, UF = State.
UF; Part < UF; ++Part)
1852 State.
set(
this, Res, {Part, 0});
1855#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
1858 O << Indent <<
"EMIT ";
1860 O <<
" = EXPAND SCEV " << *Expr;
1872 for (
unsigned Part = 0, UF = State.
UF; Part < UF; ++Part) {
1879 Value *CanonicalVectorIV = Builder.
CreateAdd(VStart, VStep,
"vec.iv");
1880 State.
set(
this, CanonicalVectorIV, Part);
1884#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
1887 O << Indent <<
"EMIT ";
1889 O <<
" = WIDEN-CANONICAL-INDUCTION ";
1895 auto &Builder = State.
Builder;
1900 ? VectorInit->getType()
1906 auto *One = ConstantInt::get(IdxTy, 1);
1910 auto *LastIdx = Builder.
CreateSub(RuntimeVF, One);
1919 State.
set(
this, EntryPart, 0);
1922#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
1925 O << Indent <<
"FIRST-ORDER-RECURRENCE-PHI ";
1933 auto &Builder = State.
Builder;
1944 bool ScalarPHI = State.
VF.
isScalar() || IsInLoop;
1950 "recipe must be in the vector loop header");
1951 unsigned LastPartForNewPhi =
isOrdered() ? 1 : State.
UF;
1952 for (
unsigned Part = 0; Part < LastPartForNewPhi; ++Part) {
1955 State.
set(
this, EntryPart, Part, IsInLoop);
1960 Value *Iden =
nullptr;
1986 for (
unsigned Part = 0; Part < LastPartForNewPhi; ++Part) {
1987 Value *EntryPart = State.
get(
this, Part, IsInLoop);
1990 Value *StartVal = (Part == 0) ? StartV : Iden;
1991 cast<PHINode>(EntryPart)->addIncoming(StartVal, VectorPH);
1995#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
1998 O << Indent <<
"WIDEN-REDUCTION-PHI ";
2008 "Non-native vplans are not expected to have VPWidenPHIRecipes.");
2013 State.
set(
this, VecPhi, 0);
2016#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
2019 O << Indent <<
"WIDEN-PHI ";
2041 for (
unsigned Part = 0, UF = State.
UF; Part < UF; ++Part) {
2047 State.
set(
this, EntryPart, Part);
2051#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
2054 O << Indent <<
"ACTIVE-LANE-MASK-PHI ";
2064 assert(State.
UF == 1 &&
"Expected unroll factor 1 for VP vectorization.");
2070 State.
set(
this, EntryPart, 0,
true);
2073#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
2076 O << Indent <<
"EXPLICIT-VECTOR-LENGTH-BASED-IV-PHI ";
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
amdgpu AMDGPU Register Bank Select
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")
mir Rename Register Operands
static DebugLoc getDebugLoc(MachineBasicBlock::instr_iterator FirstMI, MachineBasicBlock::instr_iterator LastMI)
Return the first found DebugLoc that has a DILocation, given a range of instructions.
const SmallVectorImpl< MachineOperand > & Cond
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
This file defines the SmallVector class.
static Value * getStepVector(Value *Val, Value *StartIdx, Value *Step, Instruction::BinaryOps BinOp, ElementCount VF, IRBuilderBase &Builder)
This function adds (StartIdx * Step, (StartIdx + 1) * Step, (StartIdx + 2) * Step,...
static bool isUniformAcrossVFsAndUFs(VPScalarCastRecipe *C)
Checks if C is uniform across all VFs and UFs.
static Constant * getSignedIntOrFpConstant(Type *Ty, int64_t C)
A helper function that returns an integer or floating-point constant with value C.
static Value * getRuntimeVFAsFloat(IRBuilderBase &B, Type *FTy, ElementCount VF)
static std::optional< unsigned > getOpcode(ArrayRef< VPValue * > Values)
Returns the opcode of Values or ~0 if they do not all agree.
This file contains the declarations of the Vectorization Plan base classes:
static const uint32_t IV[8]
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
LLVM Basic Block Representation.
const_iterator getFirstInsertionPt() const
Returns an iterator to the first instruction in this block that is suitable for inserting a non-PHI i...
const BasicBlock * getSinglePredecessor() const
Return the predecessor of this block if it has a single predecessor block.
const Instruction * getTerminator() const LLVM_READONLY
Returns the terminator instruction if the block is well formed or null if the block is not well forme...
const Module * getModule() const
Return the module owning the function this basic block belongs to, or nullptr if the function does no...
Conditional or Unconditional Branch instruction.
static BranchInst * Create(BasicBlock *IfTrue, BasicBlock::iterator InsertBefore)
void setSuccessor(unsigned idx, BasicBlock *NewSucc)
This class represents a function call, abstracting a target machine's calling convention.
Predicate
This enumeration lists the possible predicates for CmpInst subclasses.
@ ICMP_UGT
unsigned greater than
@ ICMP_ULT
unsigned less than
static StringRef getPredicateName(Predicate P)
This is the shared class of boolean and integer constants.
bool isOne() const
This is just a convenience method to make client code smaller for a common case.
static ConstantInt * getSigned(IntegerType *Ty, int64_t V)
Return a ConstantInt with the specified value for the specified type.
static Constant * getSplat(ElementCount EC, Constant *Elt)
Return a ConstantVector with the specified constant in each element.
This is an important base class in LLVM.
This class represents an Operation in the Expression.
A parsed version of the target data layout string in and methods for querying it.
constexpr bool isVector() const
One or more elements.
constexpr bool isScalar() const
Exactly one element.
Convenience struct for specifying and reasoning about fast-math flags.
void setAllowContract(bool B=true)
bool noSignedZeros() const
void setAllowReciprocal(bool B=true)
bool allowReciprocal() const
void print(raw_ostream &O) const
Print fast-math flags to O.
void setNoSignedZeros(bool B=true)
bool allowReassoc() const
Flag queries.
void setNoNaNs(bool B=true)
void setAllowReassoc(bool B=true)
Flag setters.
void setApproxFunc(bool B=true)
void setNoInfs(bool B=true)
bool allowContract() const
Class to represent function types.
Type * getParamType(unsigned i) const
Parameter type accessors.
FunctionType * getFunctionType() const
Returns the FunctionType for me.
Common base class shared among various IRBuilders.
Value * CreateFCmp(CmpInst::Predicate P, Value *LHS, Value *RHS, const Twine &Name="", MDNode *FPMathTag=nullptr)
Value * CreateInsertElement(Type *VecTy, Value *NewElt, Value *Idx, const Twine &Name="")
Value * CreatePtrAdd(Value *Ptr, Value *Offset, const Twine &Name="", bool IsInBounds=false)
Value * CreateSIToFP(Value *V, Type *DestTy, const Twine &Name="")
Value * CreateExtractElement(Value *Vec, Value *Idx, const Twine &Name="")
Value * CreateFAdd(Value *L, Value *R, const Twine &Name="", MDNode *FPMD=nullptr)
Value * CreateVectorSplice(Value *V1, Value *V2, int64_t Imm, const Twine &Name="")
Return a vector splice intrinsic if using scalable vectors, otherwise return a shufflevector.
Value * CreateVectorSplat(unsigned NumElts, Value *V, const Twine &Name="")
Return a vector value that contains.
ConstantInt * getTrue()
Get the constant value for i1 true.
CallInst * CreateIntrinsic(Intrinsic::ID ID, ArrayRef< Type * > Types, ArrayRef< Value * > Args, Instruction *FMFSource=nullptr, const Twine &Name="")
Create a call to intrinsic ID with Args, mangled using Types.
Value * CreateSelect(Value *C, Value *True, Value *False, const Twine &Name="", Instruction *MDFrom=nullptr)
BasicBlock::iterator GetInsertPoint() const
Value * CreateSExt(Value *V, Type *DestTy, const Twine &Name="")
Value * CreateFreeze(Value *V, const Twine &Name="")
IntegerType * getInt32Ty()
Fetch the type representing a 32-bit integer.
Value * CreateUIToFP(Value *V, Type *DestTy, const Twine &Name="", bool IsNonNeg=false)
BasicBlock * GetInsertBlock() const
void setFastMathFlags(FastMathFlags NewFMF)
Set the fast-math flags to be used with generated fp-math operators.
InsertPoint saveIP() const
Returns the current insert point.
ConstantInt * getInt32(uint32_t C)
Get a constant 32-bit value.
Value * CreateCmp(CmpInst::Predicate Pred, Value *LHS, Value *RHS, const Twine &Name="", MDNode *FPMathTag=nullptr)
PHINode * CreatePHI(Type *Ty, unsigned NumReservedValues, const Twine &Name="")
Value * CreateNot(Value *V, const Twine &Name="")
Value * CreateICmpEQ(Value *LHS, Value *RHS, const Twine &Name="")
Value * CreateSub(Value *LHS, Value *RHS, const Twine &Name="", bool HasNUW=false, bool HasNSW=false)
BranchInst * CreateCondBr(Value *Cond, BasicBlock *True, BasicBlock *False, MDNode *BranchWeights=nullptr, MDNode *Unpredictable=nullptr)
Create a conditional 'br Cond, TrueDest, FalseDest' instruction.
Value * CreateNAryOp(unsigned Opc, ArrayRef< Value * > Ops, const Twine &Name="", MDNode *FPMathTag=nullptr)
Create either a UnaryOperator or BinaryOperator depending on Opc.
Value * CreateZExt(Value *V, Type *DestTy, const Twine &Name="", bool IsNonNeg=false)
LLVMContext & getContext() const
Value * CreateAdd(Value *LHS, Value *RHS, const Twine &Name="", bool HasNUW=false, bool HasNSW=false)
Value * CreateTrunc(Value *V, Type *DestTy, const Twine &Name="", bool IsNUW=false, bool IsNSW=false)
Value * CreateBinOp(Instruction::BinaryOps Opc, Value *LHS, Value *RHS, const Twine &Name="", MDNode *FPMathTag=nullptr)
Value * CreateCast(Instruction::CastOps Op, Value *V, Type *DestTy, const Twine &Name="")
void restoreIP(InsertPoint IP)
Sets the current insert point to a previously-saved location.
void SetInsertPoint(BasicBlock *TheBB)
This specifies that created instructions should be appended to the end of the specified block.
StoreInst * CreateAlignedStore(Value *Val, Value *Ptr, MaybeAlign Align, bool isVolatile=false)
CallInst * CreateCall(FunctionType *FTy, Value *Callee, ArrayRef< Value * > Args=std::nullopt, const Twine &Name="", MDNode *FPMathTag=nullptr)
Value * CreateGEP(Type *Ty, Value *Ptr, ArrayRef< Value * > IdxList, const Twine &Name="", bool IsInBounds=false)
Value * CreateICmp(CmpInst::Predicate P, Value *LHS, Value *RHS, const Twine &Name="")
Value * CreateFMul(Value *L, Value *R, const Twine &Name="", MDNode *FPMD=nullptr)
Value * CreateStepVector(Type *DstType, const Twine &Name="")
Creates a vector of type DstType with the linear sequence <0, 1, ...>
Value * CreateMul(Value *LHS, Value *RHS, const Twine &Name="", bool HasNUW=false, bool HasNSW=false)
This provides a uniform API for creating instructions and inserting them into a basic block: either a...
A struct for saving information about induction variables.
const SCEV * getStep() const
InductionKind
This enum represents the kinds of inductions that we support.
@ IK_IntInduction
Integer induction variable. Step = C.
This instruction inserts a single (scalar) element into a VectorType value.
VectorType * getType() const
Overload to return most specific vector type.
void insertBefore(Instruction *InsertPos)
Insert an unlinked instruction into a basic block immediately before the specified instruction.
const DebugLoc & getDebugLoc() const
Return the debug location for this node as a DebugLoc.
const BasicBlock * getParent() const
InstListType::iterator eraseFromParent()
This method unlinks 'this' from the containing basic block and deletes it.
FastMathFlags getFastMathFlags() const LLVM_READONLY
Convenience function for getting all the fast-math flags, which must be an operator which supports th...
const char * getOpcodeName() const
unsigned getOpcode() const
Returns a member of one of the enums like Instruction::Add.
void setDebugLoc(DebugLoc Loc)
Set the debug location information for this instruction.
static IntegerType * get(LLVMContext &C, unsigned NumBits)
This static method is the primary way of constructing an IntegerType.
BlockT * getHeader() const
void print(raw_ostream &OS, const SlotIndexes *=nullptr, bool IsStandalone=true) const
A Module instance is used to store all the information related to an LLVM module.
const DataLayout & getDataLayout() const
Get the data layout for the module's target platform.
void addIncoming(Value *V, BasicBlock *BB)
Add an incoming value to the end of the PHI list.
static PHINode * Create(Type *Ty, unsigned NumReservedValues, const Twine &NameStr, BasicBlock::iterator InsertBefore)
Constructors - NumReservedValues is a hint for the number of incoming edges that this phi node will h...
static PoisonValue * get(Type *T)
Static factory methods - Return an 'poison' object of the specified type.
The RecurrenceDescriptor is used to identify recurrences variables in a loop.
FastMathFlags getFastMathFlags() const
static unsigned getOpcode(RecurKind Kind)
Returns the opcode corresponding to the RecurrenceKind.
unsigned getOpcode() const
Type * getRecurrenceType() const
Returns the type of the recurrence.
static bool isAnyOfRecurrenceKind(RecurKind Kind)
Returns true if the recurrence kind is of the form select(cmp(),x,y) where one of (x,...
bool isSigned() const
Returns true if all source operands of the recurrence are SExtInsts.
RecurKind getRecurrenceKind() const
Value * getRecurrenceIdentity(RecurKind K, Type *Tp, FastMathFlags FMF) const
Returns identity corresponding to the RecurrenceKind.
StoreInst * IntermediateStore
Reductions may store temporary or final result to an invariant address.
static bool isMinMaxRecurrenceKind(RecurKind Kind)
Returns true if the recurrence kind is any min/max kind.
This class uses information about analyze scalars to rewrite expressions in canonical form.
Type * getType() const
Return the LLVM type of this SCEV expression.
This class provides computation of slot numbers for LLVM Assembly writing.
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
An instruction for storing to memory.
This class represents a truncation of integer types.
Twine - A lightweight data structure for efficiently representing the concatenation of temporary valu...
The instances of the Type class are immutable: once they are created, they are never changed.
bool isVectorTy() const
True if this is an instance of VectorType.
static IntegerType * getInt1Ty(LLVMContext &C)
unsigned getScalarSizeInBits() const LLVM_READONLY
If this is a vector type, return the getPrimitiveSizeInBits value for the element type.
LLVMContext & getContext() const
Return the LLVMContext in which this type was uniqued.
bool isFloatingPointTy() const
Return true if this is one of the floating-point types.
bool isIntegerTy() const
True if this is an instance of IntegerType.
bool isVoidTy() const
Return true if this is 'void'.
Type * getScalarType() const
If this is a vector type, return the element type, otherwise return 'this'.
Value * getOperand(unsigned i) const
void execute(VPTransformState &State) override
Generate the active lane mask phi of the vector loop.
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
VPBasicBlock serves as the leaf of the Hierarchical Control-Flow Graph.
RecipeListTy & getRecipeList()
Returns a reference to the list of recipes.
void insert(VPRecipeBase *Recipe, iterator InsertPt)
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
VPValue * getIncomingValue(unsigned Idx) const
Return incoming value number Idx.
VPValue * getMask(unsigned Idx) const
Return mask number Idx.
unsigned getNumIncomingValues() const
Return the number of incoming values, taking into account that the first incoming value has no mask.
void execute(VPTransformState &State) override
Generate the phi/select nodes.
VPRegionBlock * getParent()
size_t getNumSuccessors() const
const VPBasicBlock * getEntryBasicBlock() const
VPBlockBase * getSingleSuccessor() const
VPValue * getMask() const
Return the mask used by this recipe.
void execute(VPTransformState &State) override
Generate the extraction of the appropriate bit from the block mask and the conditional branch.
void execute(VPTransformState &State) override
Generate the canonical scalar induction phi of the vector loop.
bool isCanonical(InductionDescriptor::InductionKind Kind, VPValue *Start, VPValue *Step) const
Check if the induction described by Kind, /p Start and Step is canonical, i.e.
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
This class augments a recipe with a set of VPValues defined by the recipe.
VPValue * getVPSingleValue()
Returns the only VPValue defined by the VPDef.
VPValue * getVPValue(unsigned I)
Returns the VPValue with index I defined by the VPDef.
unsigned getVPDefID() const
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
VPValue * getStepValue() const
VPValue * getStartValue() const
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
void execute(VPTransformState &State) override
Generate phi for handling IV based on EVL over iterations correctly.
void execute(VPTransformState &State) override
Generate a canonical vector induction variable of the vector loop, with.
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
@ FirstOrderRecurrenceSplice
@ CanonicalIVIncrementForPart
@ CalculateTripCountMinusVF
LLVM_DUMP_METHOD void dump() const
Print the VPInstruction to dbgs() (for debugging).
unsigned getOpcode() const
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the VPInstruction to O.
bool onlyFirstLaneUsed(const VPValue *Op) const override
Returns true if the recipe only uses the first lane of operand Op.
void execute(VPTransformState &State) override
Generate the instruction.
static VPLane getLastLaneForVF(const ElementCount &VF)
static VPLane getFirstLane()
void print(raw_ostream &O, VPSlotTracker &SlotTracker) const
Print the VPLiveOut to O.
void fixPhi(VPlan &Plan, VPTransformState &State)
Fixup the wrapped LCSSA phi node in the unique exit block.
void execute(VPTransformState &State) override
Generates phi nodes for live-outs as needed to retain SSA form.
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
VPRecipeBase is a base class modeling a sequence of one or more output IR instructions.
bool mayReadFromMemory() const
Returns true if the recipe may read from memory.
bool mayHaveSideEffects() const
Returns true if the recipe may have side-effects.
bool mayWriteToMemory() const
Returns true if the recipe may write to memory.
VPBasicBlock * getParent()
DebugLoc getDebugLoc() const
Returns the debug location of the recipe.
void moveBefore(VPBasicBlock &BB, iplist< VPRecipeBase >::iterator I)
Unlink this recipe and insert into BB before I.
void insertBefore(VPRecipeBase *InsertPos)
Insert an unlinked recipe into a basic block immediately before the specified recipe.
void insertAfter(VPRecipeBase *InsertPos)
Insert an unlinked Recipe into a basic block immediately after the specified Recipe.
iplist< VPRecipeBase >::iterator eraseFromParent()
This method unlinks 'this' from the containing basic block and deletes it.
void removeFromParent()
This method unlinks 'this' from the containing basic block, but does not delete it.
void moveAfter(VPRecipeBase *MovePos)
Unlink this recipe from its current VPBasicBlock and insert it into the VPBasicBlock that MovePos liv...
Class to record LLVM IR flag for a recipe along with it.
NonNegFlagsTy NonNegFlags
void setFlags(Instruction *I) const
Set the IR flags for I.
bool hasFastMathFlags() const
Returns true if the recipe has fast-math flags.
DisjointFlagsTy DisjointFlags
bool hasNoUnsignedWrap() const
void printFlags(raw_ostream &O) const
CmpInst::Predicate getPredicate() const
bool hasNoSignedWrap() const
FastMathFlags getFastMathFlags() const
bool isOrdered() const
Returns true, if the phi is part of an ordered reduction.
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
void execute(VPTransformState &State) override
Generate the phi/select nodes.
VPValue * getVecOp() const
The VPValue of the vector value to be reduced.
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
VPValue * getCondOp() const
The VPValue of the condition for the block.
VPValue * getChainOp() const
The VPValue of the scalar Chain being accumulated.
void execute(VPTransformState &State) override
Generate the reduction in the loop.
VPRegionBlock represents a collection of VPBasicBlocks and VPRegionBlocks which form a Single-Entry-S...
const VPBlockBase * getEntry() const
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
unsigned getOpcode() const
bool shouldPack() const
Returns true if the recipe is used by a widened recipe via an intervening VPPredInstPHIRecipe.
VPScalarCastRecipe is a recipe to create scalar cast instructions.
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
VPValue * getStepValue() const
void execute(VPTransformState &State) override
Generate the scalarized versions of the phi node as needed by their users.
Instruction * getUnderlyingInstr()
Returns the underlying instruction.
This class can be used to assign names to VPValues.
Type * inferScalarType(const VPValue *V)
Infer the type of V. Returns the scalar type of V.
This class augments VPValue with operands which provide the inverse def-use edges from VPValue's user...
void printOperands(raw_ostream &O, VPSlotTracker &SlotTracker) const
Print the operands to O.
unsigned getNumOperands() const
operand_iterator op_begin()
VPValue * getOperand(unsigned N) const
Value * getUnderlyingValue()
Return the underlying Value attached to this VPValue.
VPRecipeBase * getDefiningRecipe()
Returns the recipe defining this VPValue or nullptr if it is not defined by a recipe,...
void printAsOperand(raw_ostream &OS, VPSlotTracker &Tracker) const
friend class VPInstruction
Value * getLiveInIRValue()
Returns the underlying IR value, if this VPValue is defined outside the scope of VPlan.
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
void execute(VPTransformState &State) override
Produce a widened version of the call instruction.
void execute(VPTransformState &State) override
Generate a canonical vector induction variable of the vector loop, with start = {<Part*VF,...
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
Type * getResultType() const
Returns the result type of the cast.
void execute(VPTransformState &State) override
Produce widened copies of the cast.
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
void execute(VPTransformState &State) override
Generate the gep nodes.
TruncInst * getTruncInst()
Returns the first defined value as TruncInst, if it is one or nullptr otherwise.
void execute(VPTransformState &State) override
Generate the vectorized and scalarized versions of the phi node as needed by their users.
VPValue * getStepValue()
Returns the step value of the induction.
bool isCanonical() const
Returns true if the induction is canonical, i.e.
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
const InductionDescriptor & getInductionDescriptor() const
Returns the induction descriptor for the recipe.
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
bool isStore() const
Returns true if this recipe is a store.
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
void execute(VPTransformState &State) override
Generate the phi/select nodes.
bool onlyScalarsGenerated(bool IsScalable)
Returns true if only scalar values will be generated.
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
void execute(VPTransformState &State) override
Produce widened copies of all Ingredients.
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
VPlan models a candidate for vectorization, encoding various decisions take to produce efficient outp...
VPRegionBlock * getVectorLoopRegion()
Returns the VPRegionBlock of the vector loop.
LLVM Value Representation.
Type * getType() const
All values are typed, get the type of this value.
void setName(const Twine &Name)
Change the name of the value.
void printAsOperand(raw_ostream &O, bool PrintType=true, const Module *M=nullptr) const
Print the name of this Value out to the specified raw_ostream.
StringRef getName() const
Return a constant reference to the value's name.
Base class of all SIMD vector types.
ElementCount getElementCount() const
Return an ElementCount instance to represent the (possibly scalable) number of elements in the vector...
static VectorType * get(Type *ElementType, ElementCount EC)
This static method is the primary way to construct an VectorType.
Type * getElementType() const
constexpr bool isScalable() const
Returns whether the quantity is scaled by a runtime quantity (vscale).
constexpr ScalarTy getKnownMinValue() const
Returns the minimum value this quantity can represent.
self_iterator getIterator()
base_list_type::iterator iterator
iterator erase(iterator where)
pointer remove(iterator &IT)
This class implements an extremely fast bulk output stream that can only output to a stream.
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
@ C
The default llvm calling convention, compatible with C.
Function * getDeclaration(Module *M, ID id, ArrayRef< Type * > Tys=std::nullopt)
Create or insert an LLVM Function declaration for an intrinsic, and return it.
bool isUniformAfterVectorization(VPValue *VPV)
Returns true if VPV is uniform after vectorization.
bool onlyFirstPartUsed(const VPValue *Def)
Returns true if only the first part of Def is used.
bool onlyFirstLaneUsed(const VPValue *Def)
Returns true if only the first lane of Def is used.
This is an optimization pass for GlobalISel generic memory operations.
void ReplaceInstWithInst(BasicBlock *BB, BasicBlock::iterator &BI, Instruction *I)
Replace the instruction specified by BI with the instruction specified by I.
auto enumerate(FirstRange &&First, RestRanges &&...Rest)
Given two or more input ranges, returns a new range whose values are are tuples (A,...
bool isVectorIntrinsicWithOverloadTypeAtArg(Intrinsic::ID ID, int OpdIdx)
Identifies if the vector form of the intrinsic is overloaded on the type of the operand at index OpdI...
Value * getRuntimeVF(IRBuilderBase &B, Type *Ty, ElementCount VF)
Return the runtime value for VF.
iterator_range< T > make_range(T x, T y)
Convenience function for iterating over sub-ranges.
void interleaveComma(const Container &c, StreamT &os, UnaryFunctor each_fn)
Instruction * propagateMetadata(Instruction *I, ArrayRef< Value * > VL)
Specifically, let Kinds = [MD_tbaa, MD_alias_scope, MD_noalias, MD_fpmath, MD_nontemporal,...
Value * createMinMaxOp(IRBuilderBase &Builder, RecurKind RK, Value *Left, Value *Right)
Returns a Min/Max operation corresponding to MinMaxRecurrenceKind.
bool any_of(R &&range, UnaryPredicate P)
Provide wrappers to std::any_of which take ranges instead of having to pass begin/end explicitly.
cl::opt< bool > EnableVPlanNativePath("enable-vplan-native-path", cl::Hidden, cl::desc("Enable VPlan-native vectorization path with " "support for outer loop vectorization."))
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
Value * createOrderedReduction(IRBuilderBase &B, const RecurrenceDescriptor &Desc, Value *Src, Value *Start)
Create an ordered reduction intrinsic using the given recurrence descriptor Desc.
RecurKind
These are the kinds of recurrences that we support.
@ Mul
Product of integers.
Value * createStepForVF(IRBuilderBase &B, Type *Ty, ElementCount VF, int64_t Step)
Return a value for Step multiplied by VF.
bool is_contained(R &&Range, const E &Element)
Returns true if Element is found in Range.
bool isVectorIntrinsicWithScalarOpAtArg(Intrinsic::ID ID, unsigned ScalarOpdIdx)
Identifies if the vector form of the intrinsic has a scalar operand.
Value * createTargetReduction(IRBuilderBase &B, const RecurrenceDescriptor &Desc, Value *Src, PHINode *OrigPhi=nullptr)
Create a generic target reduction using a recurrence descriptor Desc The target is queried to determi...
void execute(VPTransformState &State) override
Generate the phi nodes.
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
VPIteration represents a single point in the iteration space of the output (vectorized and/or unrolle...
bool isInvariantCond() const
VPValue * getCond() const
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
void execute(VPTransformState &State) override
Produce a widened version of the select instruction.