25#ifndef LLVM_TRANSFORMS_VECTORIZE_VPLAN_H
26#define LLVM_TRANSFORMS_VECTORIZE_VPLAN_H
55class InnerLoopVectorizer;
59class RecurrenceDescriptor;
65class VPReplicateRecipe;
68class LoopVectorizationCostModel;
87 Loop *CurLoop =
nullptr);
114 "Both Start and End should have the same scalable flag");
116 "Expected Start to be a power of 2");
118 "Expected End to be a power of 2");
178 VPLane(
unsigned Lane,
Kind LaneKind) : Lane(Lane), LaneKind(LaneKind) {}
184 "trying to extract with invalid offset");
193 return VPLane(LaneOffset, LaneKind);
299 unsigned CacheIdx =
Instance.Lane.mapToCacheIndex(
VF);
300 return Instance.Part <
I->second.size() &&
301 CacheIdx <
I->second[
Instance.Part].size() &&
313 "scalar values must be stored as (Part, 0)");
325 "need to overwrite existing value");
326 Iter->second[Part] = V;
332 auto &PerPartVec = Iter.first->second;
333 if (PerPartVec.size() <=
Instance.Part)
334 PerPartVec.resize(
Instance.Part + 1);
335 auto &Scalars = PerPartVec[
Instance.Part];
336 unsigned CacheIdx =
Instance.Lane.mapToCacheIndex(
VF);
337 if (Scalars.size() <= CacheIdx)
338 Scalars.resize(CacheIdx + 1);
339 assert(!Scalars[CacheIdx] &&
"should overwrite existing value");
340 Scalars[CacheIdx] = V;
347 "need to overwrite existing value");
349 "need to overwrite existing value");
350 unsigned CacheIdx =
Instance.Lane.mapToCacheIndex(
VF);
352 "need to overwrite existing value");
353 Iter->second[
Instance.Part][CacheIdx] = V;
440 const unsigned char SubclassID;
457 VPlan *Plan =
nullptr;
467 assert(Predecessor &&
"Cannot add nullptr predecessor!");
472 void removePredecessor(VPBlockBase *Predecessor) {
473 auto Pos =
find(Predecessors, Predecessor);
474 assert(Pos &&
"Predecessor does not exist");
475 Predecessors.
erase(Pos);
479 void removeSuccessor(VPBlockBase *
Successor) {
481 assert(Pos &&
"Successor does not exist");
482 Successors.
erase(Pos);
487 : SubclassID(SC),
Name(
N) {}
494 using VPBlockTy =
enum { VPRegionBlockSC, VPBasicBlockSC, VPIRBasicBlockSC };
545 return (Successors.
size() == 1 ? *Successors.
begin() :
nullptr);
551 return (Predecessors.
size() == 1 ? *Predecessors.
begin() :
nullptr);
604 assert(Successors.
empty() &&
"Setting one successor when others exist.");
606 "connected blocks must have the same parent");
615 assert(Successors.
empty() &&
"Setting two successors when others exist.");
616 appendSuccessor(IfTrue);
617 appendSuccessor(IfFalse);
624 assert(Predecessors.
empty() &&
"Block predecessors already set.");
625 for (
auto *Pred : NewPreds)
626 appendPredecessor(Pred);
633 assert(Successors.
empty() &&
"Block successors already set.");
634 for (
auto *Succ : NewSuccs)
635 appendSuccessor(Succ);
665#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
724 "Op must be an operand of the recipe");
730#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
779 template <
typename IterT>
869#define VP_CLASSOF_IMPL(VPDefID) \
870 static inline bool classof(const VPDef *D) { \
871 return D->getVPDefID() == VPDefID; \
873 static inline bool classof(const VPValue *V) { \
874 auto *R = V->getDefiningRecipe(); \
875 return R && R->getVPDefID() == VPDefID; \
877 static inline bool classof(const VPUser *U) { \
878 auto *R = dyn_cast<VPRecipeBase>(U); \
879 return R && R->getVPDefID() == VPDefID; \
881 static inline bool classof(const VPRecipeBase *R) { \
882 return R->getVPDefID() == VPDefID; \
884 static inline bool classof(const VPSingleDefRecipe *R) { \
885 return R->getVPDefID() == VPDefID; \
893 template <
typename IterT>
901 template <
typename IterT>
907 switch (R->getVPDefID()) {
908 case VPRecipeBase::VPDerivedIVSC:
909 case VPRecipeBase::VPEVLBasedIVPHISC:
910 case VPRecipeBase::VPExpandSCEVSC:
911 case VPRecipeBase::VPInstructionSC:
912 case VPRecipeBase::VPReductionEVLSC:
913 case VPRecipeBase::VPReductionSC:
914 case VPRecipeBase::VPReplicateSC:
915 case VPRecipeBase::VPScalarIVStepsSC:
916 case VPRecipeBase::VPVectorPointerSC:
917 case VPRecipeBase::VPWidenCallSC:
918 case VPRecipeBase::VPWidenCanonicalIVSC:
919 case VPRecipeBase::VPWidenCastSC:
920 case VPRecipeBase::VPWidenGEPSC:
921 case VPRecipeBase::VPWidenSC:
922 case VPRecipeBase::VPWidenSelectSC:
923 case VPRecipeBase::VPBlendSC:
924 case VPRecipeBase::VPPredInstPHISC:
925 case VPRecipeBase::VPCanonicalIVPHISC:
926 case VPRecipeBase::VPActiveLaneMaskPHISC:
927 case VPRecipeBase::VPFirstOrderRecurrencePHISC:
928 case VPRecipeBase::VPWidenPHISC:
929 case VPRecipeBase::VPWidenIntOrFpInductionSC:
930 case VPRecipeBase::VPWidenPointerInductionSC:
931 case VPRecipeBase::VPReductionPHISC:
932 case VPRecipeBase::VPScalarCastSC:
934 case VPRecipeBase::VPInterleaveSC:
935 case VPRecipeBase::VPBranchOnMaskSC:
936 case VPRecipeBase::VPWidenLoadEVLSC:
937 case VPRecipeBase::VPWidenLoadSC:
938 case VPRecipeBase::VPWidenStoreEVLSC:
939 case VPRecipeBase::VPWidenStoreSC:
948 auto *R = dyn_cast<VPRecipeBase>(U);
965 enum class OperationType :
unsigned char {
996 struct ExactFlagsTy {
999 struct NonNegFlagsTy {
1002 struct FastMathFlagsTy {
1011 FastMathFlagsTy(
const FastMathFlags &FMF);
1014 OperationType OpType;
1029 OpType =
Other.OpType;
1034 template <
typename IterT>
1037 OpType = OperationType::Other;
1041 template <
typename IterT>
1044 if (
auto *
Op = dyn_cast<CmpInst>(&
I)) {
1045 OpType = OperationType::Cmp;
1047 }
else if (
auto *
Op = dyn_cast<PossiblyDisjointInst>(&
I)) {
1048 OpType = OperationType::DisjointOp;
1050 }
else if (
auto *
Op = dyn_cast<OverflowingBinaryOperator>(&
I)) {
1051 OpType = OperationType::OverflowingBinOp;
1052 WrapFlags = {
Op->hasNoUnsignedWrap(),
Op->hasNoSignedWrap()};
1053 }
else if (
auto *
Op = dyn_cast<PossiblyExactOperator>(&
I)) {
1054 OpType = OperationType::PossiblyExactOp;
1056 }
else if (
auto *
GEP = dyn_cast<GetElementPtrInst>(&
I)) {
1057 OpType = OperationType::GEPOp;
1059 }
else if (
auto *PNNI = dyn_cast<PossiblyNonNegInst>(&
I)) {
1060 OpType = OperationType::NonNegOp;
1062 }
else if (
auto *
Op = dyn_cast<FPMathOperator>(&
I)) {
1063 OpType = OperationType::FPMathOp;
1064 FMFs =
Op->getFastMathFlags();
1066 OpType = OperationType::Other;
1071 template <
typename IterT>
1077 template <
typename IterT>
1083 template <
typename IterT>
1089 template <
typename IterT>
1096 template <
typename IterT>
1104 return R->getVPDefID() == VPRecipeBase::VPInstructionSC ||
1105 R->getVPDefID() == VPRecipeBase::VPWidenSC ||
1106 R->getVPDefID() == VPRecipeBase::VPWidenGEPSC ||
1107 R->getVPDefID() == VPRecipeBase::VPWidenCastSC ||
1108 R->getVPDefID() == VPRecipeBase::VPReplicateSC ||
1109 R->getVPDefID() == VPRecipeBase::VPVectorPointerSC;
1113 auto *R = dyn_cast<VPRecipeBase>(U);
1122 case OperationType::OverflowingBinOp:
1126 case OperationType::DisjointOp:
1129 case OperationType::PossiblyExactOp:
1132 case OperationType::GEPOp:
1135 case OperationType::FPMathOp:
1136 FMFs.NoNaNs =
false;
1137 FMFs.NoInfs =
false;
1139 case OperationType::NonNegOp:
1142 case OperationType::Cmp:
1143 case OperationType::Other:
1151 case OperationType::OverflowingBinOp:
1155 case OperationType::DisjointOp:
1158 case OperationType::PossiblyExactOp:
1161 case OperationType::GEPOp:
1163 cast<GetElementPtrInst>(
I)->setNoWrapFlags(
1167 case OperationType::FPMathOp:
1168 I->setHasAllowReassoc(
FMFs.AllowReassoc);
1169 I->setHasNoNaNs(
FMFs.NoNaNs);
1170 I->setHasNoInfs(
FMFs.NoInfs);
1171 I->setHasNoSignedZeros(
FMFs.NoSignedZeros);
1172 I->setHasAllowReciprocal(
FMFs.AllowReciprocal);
1173 I->setHasAllowContract(
FMFs.AllowContract);
1174 I->setHasApproxFunc(
FMFs.ApproxFunc);
1176 case OperationType::NonNegOp:
1179 case OperationType::Cmp:
1180 case OperationType::Other:
1186 assert(OpType == OperationType::Cmp &&
1187 "recipe doesn't have a compare predicate");
1192 assert(OpType == OperationType::GEPOp &&
1193 "recipe doesn't have inbounds flag");
1203 assert(OpType == OperationType::OverflowingBinOp &&
1204 "recipe doesn't have a NUW flag");
1209 assert(OpType == OperationType::OverflowingBinOp &&
1210 "recipe doesn't have a NSW flag");
1215 assert(OpType == OperationType::DisjointOp &&
1216 "recipe cannot have a disjoing flag");
1220#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
1236 Instruction::OtherOpsEnd + 1,
1268 typedef unsigned char OpcodeTy;
1272 const std::string
Name;
1280 bool doesGeneratePerAllLanes()
const;
1284 bool canGenerateScalarForFirstLane()
const;
1300 bool isFPMathOp()
const;
1307 Opcode(Opcode),
Name(
Name.str()) {}
1314 VPValue *
B, DebugLoc
DL = {},
const Twine &
Name =
"");
1319 Opcode(Opcode),
Name(
Name.str()) {}
1326 assert(Opcode == Instruction::Or &&
"only OR opcodes can be disjoint");
1329 VPInstruction(
unsigned Opcode, std::initializer_list<VPValue *>
Operands,
1330 FastMathFlags
FMFs, DebugLoc
DL = {},
const Twine &
Name =
"");
1337 New->transferFlags(*
this);
1348#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
1361 return Opcode == Instruction::Store || Opcode == Instruction::Call ||
1362 Opcode == Instruction::Invoke || Opcode ==
SLPStore;
1369 case Instruction::Ret:
1370 case Instruction::Br:
1371 case Instruction::Store:
1372 case Instruction::Switch:
1373 case Instruction::IndirectBr:
1374 case Instruction::Resume:
1375 case Instruction::CatchRet:
1376 case Instruction::Unreachable:
1377 case Instruction::Fence:
1378 case Instruction::AtomicRMW:
1410 template <
typename IterT>
1419 R->transferFlags(*
this);
1431#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
1450 ResultTy(ResultTy) {
1452 "opcode of underlying cast doesn't match");
1457 ResultTy(ResultTy) {}
1464 *cast<CastInst>(UV));
1474#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
1497 ResultTy(ResultTy) {}
1509#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
1520 "Op must be an operand of the recipe");
1537 template <
typename IterT>
1542 VectorIntrinsicID(VectorIntrinsicID), Variant(Variant) {
1545 "last operand must be the called function");
1571#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
1580 template <
typename IterT>
1597#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
1614 bool isPointerLoopInvariant()
const {
1618 bool isIndexLoopInvariant(
unsigned I)
const {
1622 bool areAllOperandsInvariant()
const {
1624 return Op->isDefinedOutsideVectorRegions();
1629 template <
typename IterT>
1645#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
1664 IndexedTy(IndexedTy), IsReverse(IsReverse) {}
1672 "Op must be an operand of the recipe");
1681#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
1725 return B->getVPDefID() >= VPDef::VPFirstHeaderPHISC &&
1726 B->getVPDefID() <= VPDef::VPLastHeaderPHISC;
1729 auto *
B = V->getDefiningRecipe();
1730 return B &&
B->getVPDefID() >= VPRecipeBase::VPFirstHeaderPHISC &&
1731 B->getVPDefID() <= VPRecipeBase::VPLastHeaderPHISC;
1737#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
1777 Trunc(nullptr), IndDesc(IndDesc) {
1785 IV(IV), Trunc(Trunc), IndDesc(IndDesc) {
1802#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
1812 "VPWidenIntOrFpInductionRecipe generates its own backedge value");
1819 "VPWidenIntOrFpInductionRecipe generates its own backedge value");
1850 bool IsScalarAfterVectorization;
1857 bool IsScalarAfterVectorization)
1860 IsScalarAfterVectorization(IsScalarAfterVectorization) {
1870 IndDesc, IsScalarAfterVectorization);
1884#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
1917#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
1926 IncomingBlocks.
push_back(IncomingBlock);
1946 return R->getVPDefID() == VPDef::VPFirstOrderRecurrencePHISC;
1956#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
1980 VPValue &Start,
bool IsInLoop =
false,
1981 bool IsOrdered =
false)
1983 RdxDesc(RdxDesc), IsInLoop(IsInLoop), IsOrdered(IsOrdered) {
1984 assert((!IsOrdered || IsInLoop) &&
"IsOrdered requires IsInLoop");
2000 return R->getVPDefID() == VPDef::VPReductionPHISC;
2006#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
2033 "Expected an odd number of operands");
2054 assert(
Idx > 0 &&
"First index has no mask associated.");
2061#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
2070 "Op must be an operand of the recipe");
2074 [
this](
VPUser *U) {
return U->onlyFirstLaneUsed(
this); });
2087 bool HasMask =
false;
2091 bool NeedsMaskForGaps =
false;
2096 bool NeedsMaskForGaps)
2098 NeedsMaskForGaps(NeedsMaskForGaps) {
2099 for (
unsigned i = 0; i < IG->
getFactor(); ++i)
2101 if (
I->getType()->isVoidTy())
2106 for (
auto *SV : StoredValues)
2146#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
2163 "Op must be an operand of the recipe");
2178 bool IsConditional =
false;
2183 VPValue *CondOp,
bool IsOrdered)
2186 IsConditional =
true;
2207 return R->getVPDefID() == VPRecipeBase::VPReductionSC ||
2208 R->getVPDefID() == VPRecipeBase::VPReductionEVLSC;
2212 auto *R = dyn_cast<VPRecipeBase>(U);
2219#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
2267#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
2279 "Op must be an operand of the recipe");
2296 template <
typename IterT>
2298 bool IsUniform,
VPValue *Mask =
nullptr)
2300 IsUniform(IsUniform), IsPredicated(Mask) {
2311 Copy->transferFlags(*
this);
2322#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
2335 "Op must be an operand of the recipe");
2342 "Op must be an operand of the recipe");
2379#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
2383 O << Indent <<
"BRANCH-ON-MASK ";
2402 "Op must be an operand of the recipe");
2429#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
2438 "Op must be an operand of the recipe");
2467 std::initializer_list<VPValue *> Operands,
2480 return R->getVPDefID() == VPRecipeBase::VPWidenLoadSC ||
2481 R->getVPDefID() == VPRecipeBase::VPWidenStoreSC ||
2482 R->getVPDefID() == VPRecipeBase::VPWidenLoadEVLSC ||
2483 R->getVPDefID() == VPRecipeBase::VPWidenStoreEVLSC;
2487 auto *R = dyn_cast<VPRecipeBase>(U);
2541#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
2550 "Op must be an operand of the recipe");
2563 {L->getAddr(), EVL}, L->isConsecutive(),
2564 L->isReverse(), L->getDebugLoc()),
2577#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
2586 "Op must be an operand of the recipe");
2617#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
2626 "Op must be an operand of the recipe");
2640 S->isConsecutive(), S->isReverse(),
2656#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
2665 "Op must be an operand of the recipe");
2697#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
2726 return D->getVPDefID() == VPDef::VPCanonicalIVPHISC;
2732#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
2746 "Op must be an operand of the recipe");
2753 "Op must be an operand of the recipe");
2782 return D->getVPDefID() == VPDef::VPActiveLaneMaskPHISC;
2788#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
2813 return D->getVPDefID() == VPDef::VPEVLBasedIVPHISC;
2823 "Op must be an operand of the recipe");
2827#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
2844 cast<VPCanonicalIVPHIRecipe>(
getOperand(0)));
2854#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
2877 Start, CanonicalIV, Step) {}
2898#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
2914 "Op must be an operand of the recipe");
2929 InductionOpcode(Opcode) {}
2934 IV, Step, IndDesc.getInductionOpcode(),
2952#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
2963 "Op must be an operand of the recipe");
3030 return V->getVPBlockID() == VPBlockBase::VPBasicBlockSC ||
3031 V->getVPBlockID() == VPBlockBase::VPIRBasicBlockSC;
3035 assert(Recipe &&
"No recipe to append.");
3036 assert(!Recipe->Parent &&
"Recipe already in VPlan");
3037 Recipe->Parent =
this;
3069#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
3093 NewBlock->appendRecipe(R.clone());
3124 return V->getVPBlockID() == VPBlockBase::VPIRBasicBlockSC;
3134 NewBlock->appendRecipe(R.clone());
3163 const std::string &Name =
"",
bool IsReplicator =
false)
3164 :
VPBlockBase(VPRegionBlockSC, Name), Entry(Entry), Exiting(Exiting),
3165 IsReplicator(IsReplicator) {
3166 assert(Entry->getPredecessors().empty() &&
"Entry block has predecessors.");
3168 Entry->setParent(
this);
3172 :
VPBlockBase(VPRegionBlockSC, Name), Entry(nullptr), Exiting(nullptr),
3173 IsReplicator(IsReplicator) {}
3178 Entry->dropAllReferences(&DummyValue);
3185 return V->getVPBlockID() == VPBlockBase::VPRegionBlockSC;
3195 "Entry block cannot have predecessors.");
3207 "Exit block cannot have successors.");
3208 Exiting = ExitingBlock;
3231#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
3282 VPValue *BackedgeTakenCount =
nullptr;
3314 :
VPlan(Preheader, Entry) {
3323 : Entry(Entry), Preheader(Preheader) {
3324 Entry->setPlan(
this);
3328 "preheader must be disconnected");
3342 bool RequiresScalarEpilogueCheck,
3343 bool TailFolded,
Loop *TheLoop);
3360 assert(TripCount &&
"trip count needs to be set before accessing it");
3368 "TripCount always must be set");
3369 TripCount = NewTripCount;
3374 if (!BackedgeTakenCount)
3375 BackedgeTakenCount =
new VPValue();
3376 return BackedgeTakenCount;
3388 assert(
hasVF(VF) &&
"Cannot set VF not already in plan");
3409 assert(
hasUF(UF) &&
"Cannot set the UF not already in plan");
3422 assert(V &&
"Trying to get or add the VPValue of a null Value");
3423 if (!Value2VPValue.
count(V)) {
3427 assert(!Value2VPValue.
count(V) &&
"Value already exists in VPlan");
3428 Value2VPValue[V] = VPV;
3431 assert(Value2VPValue.
count(V) &&
"Value does not exist in VPlan");
3432 assert(Value2VPValue[V]->isLiveIn() &&
3433 "Only live-ins should be in mapping");
3434 return Value2VPValue[V];
3440#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
3456 return cast<VPRegionBlock>(
getEntry()->getSingleSuccessor());
3459 return cast<VPRegionBlock>(
getEntry()->getSingleSuccessor());
3465 if (EntryVPBB->
empty()) {
3469 return cast<VPCanonicalIVPHIRecipe>(&*EntryVPBB->
begin());
3475 delete LiveOuts[PN];
3484 return SCEVToExpansion.
lookup(S);
3489 SCEVToExpansion[S] = V;
3501#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
3508 unsigned TabWidth = 2;
3516 void bumpIndent(
int b) { Indent = std::string((
Depth += b) * TabWidth,
' '); }
3542 const Twine &Label);
3587 "Can't insert new block with predecessors or successors.");
3606 "Can't insert IfTrue with successors.");
3608 "Can't insert IfFalse with successors.");
3622 "Can't connect two block with different parents");
3624 "Blocks can't have more than two successors.");
3625 From->appendSuccessor(To);
3626 To->appendPredecessor(
From);
3632 assert(To &&
"Successor to disconnect is null.");
3633 From->removeSuccessor(To);
3634 To->removePredecessor(
From);
3639 template <
typename BlockTy,
typename T>
3642 using BaseTy = std::conditional_t<std::is_const<BlockTy>::value,
3652 return cast<BlockTy>(&
Block);
3681 for (
auto &
I : InterleaveGroupMap)
3683 for (
auto *
Ptr : DelSet)
3692 return InterleaveGroupMap.
lookup(Instr);
3699 enum class OpMode {
Failed, Load, Opcode };
3703 struct BundleDenseMapInfo {
3705 return {
reinterpret_cast<VPValue *
>(-1)};
3709 return {
reinterpret_cast<VPValue *
>(-2)};
3733 bool CompletelySLP =
true;
3736 unsigned WidestBundleBits = 0;
3738 using MultiNodeOpTy =
3739 typename std::pair<VPInstruction *, SmallVector<VPValue *, 4>>;
3748 bool MultiNodeActive =
false;
3770#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
3814 assert(Def &&
"Must have definition for value defined inside vector region");
3815 if (
auto Rep = dyn_cast<VPReplicateRecipe>(Def))
3816 return Rep->isUniform();
3817 if (
auto *
GEP = dyn_cast<VPWidenGEPRecipe>(Def))
3819 if (
auto *VPI = dyn_cast<VPInstruction>(Def))
3820 return VPI->isSingleScalar() || VPI->isVectorToScalar();
static MCDisassembler::DecodeStatus addOperand(MCInst &Inst, const MCOperand &Opnd)
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
BlockVerifier::State From
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")
static GCRegistry::Add< StatepointGC > D("statepoint-example", "an example strategy for statepoint")
#define LLVM_DUMP_METHOD
Mark debug helper function definitions like dump() that should not be stripped from debug builds.
Returns the sub type a function will return at a given Idx Should correspond to the result type of an ExtractValue instruction executed with just that one unsigned Idx
This file defines the DenseMap class.
std::optional< std::vector< StOtherPiece > > Other
std::pair< BasicBlock *, unsigned > BlockTy
A pair of (basic block, score).
This file defines an InstructionCost class that is used when calculating the cost of an instruction,...
mir Rename Register Operands
This file implements a map that provides insertion order iteration.
ConstantRange Range(APInt(BitWidth, Low), APInt(BitWidth, High))
static cl::opt< RegAllocEvictionAdvisorAnalysis::AdvisorMode > Mode("regalloc-enable-advisor", cl::Hidden, cl::init(RegAllocEvictionAdvisorAnalysis::AdvisorMode::Default), cl::desc("Enable regalloc advisor mode"), cl::values(clEnumValN(RegAllocEvictionAdvisorAnalysis::AdvisorMode::Default, "default", "Default"), clEnumValN(RegAllocEvictionAdvisorAnalysis::AdvisorMode::Release, "release", "precompiled"), clEnumValN(RegAllocEvictionAdvisorAnalysis::AdvisorMode::Development, "development", "for training")))
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
This file implements the SmallBitVector class.
This file defines the SmallPtrSet class.
This file defines the SmallVector class.
This file contains the declarations of the entities induced by Vectorization Plans,...
#define VP_CLASSOF_IMPL(VPDefID)
static const uint32_t IV[8]
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
ArrayRef< T > slice(size_t N, size_t M) const
slice(n, m) - Chop off the first N elements of the array, and keep M elements in the array.
LLVM Basic Block Representation.
This is the base class for all instructions that perform data casts.
Instruction::CastOps getOpcode() const
Return the opcode of this CastInst.
Predicate
This enumeration lists the possible predicates for CmpInst subclasses.
This class represents an Operation in the Expression.
ValueT lookup(const_arg_type_t< KeyT > Val) const
lookup - Return the entry for the specified key, or a default constructed value if no such entry exis...
size_type count(const_arg_type_t< KeyT > Val) const
Return 1 if the specified key is in the map, 0 otherwise.
bool contains(const_arg_type_t< KeyT > Val) const
Return true if the specified key is in the map, false otherwise.
Concrete subclass of DominatorTreeBase that is used to compute a normal dominator tree.
constexpr bool isScalar() const
Exactly one element.
Utility class for floating point operations which can have information about relaxed accuracy require...
Convenience struct for specifying and reasoning about fast-math flags.
static GEPNoWrapFlags inBounds()
static GEPNoWrapFlags none()
an instruction for type-safe pointer arithmetic to access elements of arrays and structs
Common base class shared among various IRBuilders.
A struct for saving information about induction variables.
InductionKind
This enum represents the kinds of inductions that we support.
InnerLoopVectorizer vectorizes loops which contain only one basic block to a specified vectorization ...
unsigned getOpcode() const
Returns a member of one of the enums like Instruction::Add.
The group of interleaved loads/stores sharing the same stride and close to each other.
uint32_t getFactor() const
InstTy * getMember(uint32_t Index) const
Get the member with the given index Index.
InstTy * getInsertPos() const
Drive the analysis of interleaved memory accesses in the loop.
This is an important class for using LLVM in a threaded context.
An instruction for reading from memory.
LoopVectorizationCostModel - estimates the expected speedups due to vectorization.
This class emits a version of the loop where run-time checks ensure that may-alias pointers can't ove...
Represents a single loop in the control flow graph.
This class implements a map that also provides access to all stored values in a deterministic order.
VectorType::iterator erase(typename VectorType::iterator Iterator)
Remove the element given by Iterator.
The RecurrenceDescriptor is used to identify recurrences variables in a loop.
This class represents an analyzed expression in the program.
The main scalar evolution driver.
This class represents the LLVM 'select' instruction.
size_type size() const
Determine the number of elements in the SetVector.
iterator end()
Get an iterator to the end of the SetVector.
void clear()
Completely clear the SetVector.
size_type count(const key_type &key) const
Count the number of elements of a given key in the SetVector.
bool empty() const
Determine if the SetVector is empty or not.
iterator begin()
Get an iterator to the beginning of the SetVector.
bool insert(const value_type &X)
Insert a new element into the SetVector.
bool contains(const key_type &key) const
Check if the SetVector contains the given key.
This class provides computation of slot numbers for LLVM Assembly writing.
A templated base class for SmallPtrSet which provides the typesafe interface that is common across al...
std::pair< iterator, bool > insert(PtrType Ptr)
Inserts Ptr if and only if there is no element in the container equal to Ptr.
SmallPtrSet - This class implements a set which is optimized for holding SmallSize or less elements.
A SetVector that performs no allocations if smaller than a certain size.
iterator erase(const_iterator CI)
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
An instruction for storing to memory.
This class represents a truncation of integer types.
Twine - A lightweight data structure for efficiently representing the concatenation of temporary valu...
std::string str() const
Return the twine contents as a std::string.
The instances of the Type class are immutable: once they are created, they are never changed.
Iterator to iterate over vectorization factors in a VFRange.
ElementCount operator*() const
iterator(ElementCount VF)
bool operator==(const iterator &Other) const
A recipe for generating the active lane mask for the vector loop that is used to predicate the vector...
void execute(VPTransformState &State) override
Generate the active lane mask phi of the vector loop.
VPActiveLaneMaskPHIRecipe * clone() override
Clone the current recipe.
static bool classof(const VPHeaderPHIRecipe *D)
VPActiveLaneMaskPHIRecipe(VPValue *StartMask, DebugLoc DL)
~VPActiveLaneMaskPHIRecipe() override=default
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
VPBasicBlock serves as the leaf of the Hierarchical Control-Flow Graph.
RecipeListTy::const_iterator const_iterator
void appendRecipe(VPRecipeBase *Recipe)
Augment the existing recipes of a VPBasicBlock with an additional Recipe as the last recipe.
VPBasicBlock * clone() override
Clone the current block and it's recipes, without updating the operands of the cloned recipes.
RecipeListTy::const_reverse_iterator const_reverse_iterator
RecipeListTy::iterator iterator
Instruction iterators...
void execute(VPTransformState *State) override
The method which generates the output IR instructions that correspond to this VPBasicBlock,...
RecipeListTy & getRecipeList()
Returns a reference to the list of recipes.
VPBasicBlock(const unsigned char BlockSC, const Twine &Name="")
VPBasicBlock(const Twine &Name="", VPRecipeBase *Recipe=nullptr)
iterator begin()
Recipe iterator methods.
RecipeListTy::reverse_iterator reverse_iterator
iterator_range< iterator > phis()
Returns an iterator range over the PHI-like recipes in the block.
InstructionCost cost(ElementCount VF, VPCostContext &Ctx) override
Return the cost of this VPBasicBlock.
iterator getFirstNonPhi()
Return the position of the first non-phi node recipe in the block.
VPRegionBlock * getEnclosingLoopRegion()
void dropAllReferences(VPValue *NewValue) override
Replace all operands of VPUsers in the block with NewValue and also replaces all uses of VPValues def...
const_reverse_iterator rbegin() const
VPBasicBlock * splitAt(iterator SplitAt)
Split current block at SplitAt by inserting a new block between the current block and its successors ...
RecipeListTy Recipes
The VPRecipes held in the order of output instructions to generate.
void executeRecipes(VPTransformState *State, BasicBlock *BB)
Execute the recipes in the IR basic block BB.
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print this VPBsicBlock to O, prefixing all lines with Indent.
const VPRecipeBase & front() const
const_iterator begin() const
bool isExiting() const
Returns true if the block is exiting it's parent region.
VPRecipeBase * getTerminator()
If the block has multiple successors, return the branch recipe terminating the block.
const VPRecipeBase & back() const
void insert(VPRecipeBase *Recipe, iterator InsertPt)
const_iterator end() const
static bool classof(const VPBlockBase *V)
Method to support type inquiry through isa, cast, and dyn_cast.
static RecipeListTy VPBasicBlock::* getSublistAccess(VPRecipeBase *)
Returns a pointer to a member of the recipe list.
reverse_iterator rbegin()
const_reverse_iterator rend() const
A recipe for vectorizing a phi-node as a sequence of mask-based select instructions.
VPBlendRecipe(PHINode *Phi, ArrayRef< VPValue * > Operands)
The blend operation is a User of the incoming values and of their respective masks,...
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
bool onlyFirstLaneUsed(const VPValue *Op) const override
Returns true if the recipe only uses the first lane of operand Op.
VPValue * getIncomingValue(unsigned Idx) const
Return incoming value number Idx.
VPValue * getMask(unsigned Idx) const
Return mask number Idx.
unsigned getNumIncomingValues() const
Return the number of incoming values, taking into account that the first incoming value has no mask.
VPBlendRecipe * clone() override
Clone the current recipe.
void execute(VPTransformState &State) override
Generate the phi/select nodes.
VPBlockBase is the building block of the Hierarchical Control-Flow Graph.
void setSuccessors(ArrayRef< VPBlockBase * > NewSuccs)
Set each VPBasicBlock in NewSuccss as successor of this VPBlockBase.
VPRegionBlock * getParent()
VPBlocksTy & getPredecessors()
const VPBasicBlock * getExitingBasicBlock() const
LLVM_DUMP_METHOD void dump() const
Dump this VPBlockBase to dbgs().
void setName(const Twine &newName)
size_t getNumSuccessors() const
iterator_range< VPBlockBase ** > successors()
virtual void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const =0
Print plain-text dump of this VPBlockBase to O, prefixing all lines with Indent.
void printSuccessors(raw_ostream &O, const Twine &Indent) const
Print the successors of this block to O, prefixing all lines with Indent.
bool isLegalToHoistInto()
Return true if it is legal to hoist instructions into this block.
virtual ~VPBlockBase()=default
void print(raw_ostream &O) const
Print plain-text dump of this VPlan to O.
const VPBlocksTy & getHierarchicalPredecessors()
size_t getNumPredecessors() const
void setPredecessors(ArrayRef< VPBlockBase * > NewPreds)
Set each VPBasicBlock in NewPreds as predecessor of this VPBlockBase.
VPBlockBase * getEnclosingBlockWithPredecessors()
const VPBlocksTy & getPredecessors() const
virtual VPBlockBase * clone()=0
Clone the current block and it's recipes without updating the operands of the cloned recipes,...
static void deleteCFG(VPBlockBase *Entry)
Delete all blocks reachable from a given VPBlockBase, inclusive.
enum { VPRegionBlockSC, VPBasicBlockSC, VPIRBasicBlockSC } VPBlockTy
An enumeration for keeping track of the concrete subclass of VPBlockBase that are actually instantiat...
virtual InstructionCost cost(ElementCount VF, VPCostContext &Ctx)=0
Return the cost of the block.
void setPlan(VPlan *ParentPlan)
Sets the pointer of the plan containing the block.
const VPRegionBlock * getParent() const
void printAsOperand(raw_ostream &OS, bool PrintType) const
const std::string & getName() const
void clearSuccessors()
Remove all the successors of this block.
VPBlockBase * getSingleHierarchicalSuccessor()
void setTwoSuccessors(VPBlockBase *IfTrue, VPBlockBase *IfFalse)
Set two given VPBlockBases IfTrue and IfFalse to be the two successors of this VPBlockBase.
VPBlockBase * getSinglePredecessor() const
virtual void execute(VPTransformState *State)=0
The method which generates the output IR that correspond to this VPBlockBase, thereby "executing" the...
const VPBlocksTy & getHierarchicalSuccessors()
void clearPredecessors()
Remove all the predecessor of this block.
unsigned getVPBlockID() const
VPBlockBase(const unsigned char SC, const std::string &N)
VPBlocksTy & getSuccessors()
VPBlockBase * getEnclosingBlockWithSuccessors()
An Enclosing Block of a block B is any block containing B, including B itself.
const VPBasicBlock * getEntryBasicBlock() const
void setOneSuccessor(VPBlockBase *Successor)
Set a given VPBlockBase Successor as the single successor of this VPBlockBase.
void setParent(VPRegionBlock *P)
virtual void dropAllReferences(VPValue *NewValue)=0
Replace all operands of VPUsers in the block with NewValue and also replaces all uses of VPValues def...
VPBlockBase * getSingleHierarchicalPredecessor()
VPBlockBase * getSingleSuccessor() const
const VPBlocksTy & getSuccessors() const
Class that provides utilities for VPBlockBases in VPlan.
static auto blocksOnly(const T &Range)
Return an iterator range over Range which only includes BlockTy blocks.
static void insertBlockAfter(VPBlockBase *NewBlock, VPBlockBase *BlockPtr)
Insert disconnected VPBlockBase NewBlock after BlockPtr.
static void insertTwoBlocksAfter(VPBlockBase *IfTrue, VPBlockBase *IfFalse, VPBlockBase *BlockPtr)
Insert disconnected VPBlockBases IfTrue and IfFalse after BlockPtr.
static void disconnectBlocks(VPBlockBase *From, VPBlockBase *To)
Disconnect VPBlockBases From and To bi-directionally.
static void connectBlocks(VPBlockBase *From, VPBlockBase *To)
Connect VPBlockBases From and To bi-directionally.
A recipe for generating conditional branches on the bits of a mask.
VPValue * getMask() const
Return the mask used by this recipe.
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
VPBranchOnMaskRecipe(VPValue *BlockInMask)
VPBranchOnMaskRecipe * clone() override
Clone the current recipe.
bool usesScalars(const VPValue *Op) const override
Returns true if the recipe uses scalars of operand Op.
void execute(VPTransformState &State) override
Generate the extraction of the appropriate bit from the block mask and the conditional branch.
Canonical scalar induction phi of the vector loop.
bool onlyFirstPartUsed(const VPValue *Op) const override
Returns true if the recipe only uses the first part of operand Op.
~VPCanonicalIVPHIRecipe() override=default
static bool classof(const VPHeaderPHIRecipe *D)
VPCanonicalIVPHIRecipe * clone() override
Clone the current recipe.
VPCanonicalIVPHIRecipe(VPValue *StartV, DebugLoc DL)
bool onlyFirstLaneUsed(const VPValue *Op) const override
Returns true if the recipe only uses the first lane of operand Op.
void execute(VPTransformState &State) override
Generate the canonical scalar induction phi of the vector loop.
Type * getScalarType() const
Returns the scalar type of the induction.
bool isCanonical(InductionDescriptor::InductionKind Kind, VPValue *Start, VPValue *Step) const
Check if the induction described by Kind, /p Start and Step is canonical, i.e.
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
This class augments a recipe with a set of VPValues defined by the recipe.
unsigned getVPDefID() const
A recipe for converting the input value IV value to the corresponding value of an IV with different s...
void execute(VPTransformState &State) override
Generate the transformed value of the induction at offset StartValue (1.
VPDerivedIVRecipe(InductionDescriptor::InductionKind Kind, const FPMathOperator *FPBinOp, VPValue *Start, VPValue *IV, VPValue *Step)
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
VPValue * getStepValue() const
VPDerivedIVRecipe(const InductionDescriptor &IndDesc, VPValue *Start, VPCanonicalIVPHIRecipe *CanonicalIV, VPValue *Step)
Type * getScalarType() const
VPDerivedIVRecipe * clone() override
Clone the current recipe.
~VPDerivedIVRecipe() override=default
bool onlyFirstLaneUsed(const VPValue *Op) const override
Returns true if the recipe only uses the first lane of operand Op.
VPValue * getStartValue() const
A recipe for generating the phi node for the current index of elements, adjusted in accordance with E...
static bool classof(const VPHeaderPHIRecipe *D)
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
VPEVLBasedIVPHIRecipe * clone() override
Clone the current recipe.
~VPEVLBasedIVPHIRecipe() override=default
void execute(VPTransformState &State) override
Generate phi for handling IV based on EVL over iterations correctly.
VPEVLBasedIVPHIRecipe(VPValue *StartIV, DebugLoc DL)
bool onlyFirstLaneUsed(const VPValue *Op) const override
Returns true if the recipe only uses the first lane of operand Op.
Recipe to expand a SCEV expression.
VPExpandSCEVRecipe(const SCEV *Expr, ScalarEvolution &SE)
const SCEV * getSCEV() const
void execute(VPTransformState &State) override
Generate a canonical vector induction variable of the vector loop, with.
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
VPExpandSCEVRecipe * clone() override
Clone the current recipe.
~VPExpandSCEVRecipe() override=default
A special type of VPBasicBlock that wraps an existing IR basic block.
void execute(VPTransformState *State) override
The method which generates the output IR instructions that correspond to this VPBasicBlock,...
VPIRBasicBlock(BasicBlock *IRBB)
BasicBlock * getIRBasicBlock() const
~VPIRBasicBlock() override
static bool classof(const VPBlockBase *V)
VPIRBasicBlock * clone() override
Clone the current block and it's recipes, without updating the operands of the cloned recipes.
This is a concrete Recipe that models a single VPlan-level instruction.
@ ResumePhi
Creates a scalar phi in a leaf VPBB with a single predecessor in VPlan.
@ FirstOrderRecurrenceSplice
@ CanonicalIVIncrementForPart
@ CalculateTripCountMinusVF
VPInstruction(unsigned Opcode, ArrayRef< VPValue * > Operands, DebugLoc DL, const Twine &Name="")
VPInstruction * clone() override
Clone the current recipe.
LLVM_DUMP_METHOD void dump() const
Print the VPInstruction to dbgs() (for debugging).
unsigned getOpcode() const
VPInstruction(unsigned Opcode, std::initializer_list< VPValue * > Operands, WrapFlagsTy WrapFlags, DebugLoc DL={}, const Twine &Name="")
bool onlyFirstPartUsed(const VPValue *Op) const override
Returns true if the recipe only uses the first part of operand Op.
VPInstruction(unsigned Opcode, std::initializer_list< VPValue * > Operands, DebugLoc DL={}, const Twine &Name="")
VPInstruction(unsigned Opcode, std::initializer_list< VPValue * > Operands, DisjointFlagsTy DisjointFlag, DebugLoc DL={}, const Twine &Name="")
bool isVectorToScalar() const
Returns true if this VPInstruction produces a scalar value from a vector, e.g.
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the VPInstruction to O.
bool onlyFirstLaneUsed(const VPValue *Op) const override
Returns true if the recipe only uses the first lane of operand Op.
bool isSingleScalar() const
Returns true if this VPInstruction's operands are single scalars and the result is also a single scal...
bool mayWriteToMemory() const
Return true if this instruction may modify memory.
void execute(VPTransformState &State) override
Generate the instruction.
VPInterleaveRecipe is a recipe for transforming an interleave group of load or stores into one wide l...
bool onlyFirstLaneUsed(const VPValue *Op) const override
The recipe only uses the first lane of the address.
~VPInterleaveRecipe() override=default
VPValue * getAddr() const
Return the address accessed by this recipe.
VPInterleaveRecipe(const InterleaveGroup< Instruction > *IG, VPValue *Addr, ArrayRef< VPValue * > StoredValues, VPValue *Mask, bool NeedsMaskForGaps)
VPValue * getMask() const
Return the mask used by this recipe.
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
VPInterleaveRecipe * clone() override
Clone the current recipe.
void execute(VPTransformState &State) override
Generate the wide load or store, and shuffles.
ArrayRef< VPValue * > getStoredValues() const
Return the VPValues stored by this interleave group.
Instruction * getInsertPos() const
const InterleaveGroup< Instruction > * getInterleaveGroup()
unsigned getNumStoreOperands() const
Returns the number of stored operands of this interleave group.
~VPInterleavedAccessInfo()
InterleaveGroup< VPInstruction > * getInterleaveGroup(VPInstruction *Instr) const
Get the interleave group that Instr belongs to.
In what follows, the term "input IR" refers to code that is fed into the vectorizer whereas the term ...
static VPLane getLastLaneForVF(const ElementCount &VF)
static unsigned getNumCachedLanes(const ElementCount &VF)
Returns the maxmimum number of lanes that we are able to consider caching for VF.
Value * getAsRuntimeExpr(IRBuilderBase &Builder, const ElementCount &VF) const
Returns an expression describing the lane index that can be used at runtime.
VPLane(unsigned Lane, Kind LaneKind)
Kind getKind() const
Returns the Kind of lane offset.
static VPLane getLaneFromEnd(const ElementCount &VF, unsigned Offset)
bool isFirstLane() const
Returns true if this is the first lane of the whole vector.
unsigned getKnownLane() const
Returns a compile-time known value for the lane index and asserts if the lane can only be calculated ...
static VPLane getFirstLane()
Kind
Kind describes how to interpret Lane.
@ ScalableLast
For ScalableLast, Lane is the offset from the start of the last N-element subvector in a scalable vec...
@ First
For First, Lane is the index into the first N elements of a fixed-vector <N x <ElTy>> or a scalable v...
unsigned mapToCacheIndex(const ElementCount &VF) const
Maps the lane to a cache index based on VF.
A value that is used outside the VPlan.
VPLiveOut(PHINode *Phi, VPValue *Op)
static bool classof(const VPUser *U)
bool usesScalars(const VPValue *Op) const override
Returns true if the VPLiveOut uses scalars of operand Op.
void print(raw_ostream &O, VPSlotTracker &SlotTracker) const
Print the VPLiveOut to O.
void fixPhi(VPlan &Plan, VPTransformState &State)
Fix the wrapped phi node.
VPPredInstPHIRecipe is a recipe for generating the phi nodes needed when control converges back from ...
~VPPredInstPHIRecipe() override=default
bool usesScalars(const VPValue *Op) const override
Returns true if the recipe uses scalars of operand Op.
VPPredInstPHIRecipe(VPValue *PredV)
Construct a VPPredInstPHIRecipe given PredInst whose value needs a phi nodes after merging back from ...
void execute(VPTransformState &State) override
Generates phi nodes for live-outs as needed to retain SSA form.
VPPredInstPHIRecipe * clone() override
Clone the current recipe.
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
VPRecipeBase is a base class modeling a sequence of one or more output IR instructions.
bool mayReadFromMemory() const
Returns true if the recipe may read from memory.
bool mayReadOrWriteMemory() const
Returns true if the recipe may read from or write to memory.
bool mayHaveSideEffects() const
Returns true if the recipe may have side-effects.
bool mayWriteToMemory() const
Returns true if the recipe may write to memory.
InstructionCost computeCost(ElementCount VF, VPCostContext &Ctx) const
Compute the cost of this recipe using the legacy cost model and the underlying instructions.
virtual ~VPRecipeBase()=default
VPBasicBlock * getParent()
DebugLoc getDebugLoc() const
Returns the debug location of the recipe.
virtual void execute(VPTransformState &State)=0
The method which generates the output IR instructions that correspond to this VPRecipe,...
void moveBefore(VPBasicBlock &BB, iplist< VPRecipeBase >::iterator I)
Unlink this recipe and insert into BB before I.
void insertBefore(VPRecipeBase *InsertPos)
Insert an unlinked recipe into a basic block immediately before the specified recipe.
void insertAfter(VPRecipeBase *InsertPos)
Insert an unlinked Recipe into a basic block immediately after the specified Recipe.
static bool classof(const VPDef *D)
Method to support type inquiry through isa, cast, and dyn_cast.
iplist< VPRecipeBase >::iterator eraseFromParent()
This method unlinks 'this' from the containing basic block and deletes it.
VPRecipeBase(const unsigned char SC, ArrayRef< VPValue * > Operands, DebugLoc DL={})
virtual VPRecipeBase * clone()=0
Clone the current recipe.
const VPBasicBlock * getParent() const
virtual InstructionCost cost(ElementCount VF, VPCostContext &Ctx)
Return the cost of this recipe, taking into account if the cost computation should be skipped and the...
static bool classof(const VPUser *U)
VPRecipeBase(const unsigned char SC, iterator_range< IterT > Operands, DebugLoc DL={})
void removeFromParent()
This method unlinks 'this' from the containing basic block, but does not delete it.
bool isPhi() const
Returns true for PHI-like recipes.
void moveAfter(VPRecipeBase *MovePos)
Unlink this recipe from its current VPBasicBlock and insert it into the VPBasicBlock that MovePos liv...
Class to record LLVM IR flag for a recipe along with it.
VPRecipeWithIRFlags(const unsigned char SC, IterT Operands, GEPFlagsTy GEPFlags, DebugLoc DL={})
NonNegFlagsTy NonNegFlags
CmpInst::Predicate CmpPredicate
VPRecipeWithIRFlags(const unsigned char SC, IterT Operands, CmpInst::Predicate Pred, DebugLoc DL={})
void setFlags(Instruction *I) const
Set the IR flags for I.
static bool classof(const VPRecipeBase *R)
VPRecipeWithIRFlags(const unsigned char SC, IterT Operands, FastMathFlags FMFs, DebugLoc DL={})
void dropPoisonGeneratingFlags()
Drop all poison-generating flags.
bool hasFastMathFlags() const
Returns true if the recipe has fast-math flags.
VPRecipeWithIRFlags(const unsigned char SC, IterT Operands, Instruction &I)
DisjointFlagsTy DisjointFlags
VPRecipeWithIRFlags(const unsigned char SC, IterT Operands, WrapFlagsTy WrapFlags, DebugLoc DL={})
VPRecipeWithIRFlags(const unsigned char SC, IterT Operands, DisjointFlagsTy DisjointFlags, DebugLoc DL={})
void transferFlags(VPRecipeWithIRFlags &Other)
bool hasNoUnsignedWrap() const
void printFlags(raw_ostream &O) const
CmpInst::Predicate getPredicate() const
bool hasNoSignedWrap() const
static bool classof(const VPUser *U)
FastMathFlags getFastMathFlags() const
VPRecipeWithIRFlags(const unsigned char SC, IterT Operands, DebugLoc DL={})
A recipe to represent inloop reduction operations with vector-predication intrinsics,...
void execute(VPTransformState &State) override
Generate the reduction in the loop.
bool onlyFirstLaneUsed(const VPValue *Op) const override
Returns true if the recipe only uses the first lane of operand Op.
VPValue * getEVL() const
The VPValue of the explicit vector length.
VPReductionEVLRecipe * clone() override
Clone the current recipe.
VPReductionEVLRecipe(VPReductionRecipe *R, VPValue *EVL, VPValue *CondOp)
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
~VPReductionEVLRecipe() override=default
A recipe for handling reduction phis.
VPReductionPHIRecipe(PHINode *Phi, const RecurrenceDescriptor &RdxDesc, VPValue &Start, bool IsInLoop=false, bool IsOrdered=false)
Create a new VPReductionPHIRecipe for the reduction Phi described by RdxDesc.
bool isOrdered() const
Returns true, if the phi is part of an ordered reduction.
VPReductionPHIRecipe * clone() override
Clone the current recipe.
~VPReductionPHIRecipe() override=default
bool isInLoop() const
Returns true, if the phi is part of an in-loop reduction.
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
void execute(VPTransformState &State) override
Generate the phi/select nodes.
static bool classof(const VPHeaderPHIRecipe *R)
const RecurrenceDescriptor & getRecurrenceDescriptor() const
A recipe to represent inloop reduction operations, performing a reduction on a vector operand into a ...
bool isConditional() const
Return true if the in-loop reduction is conditional.
static bool classof(const VPRecipeBase *R)
VPValue * getVecOp() const
The VPValue of the vector value to be reduced.
const RecurrenceDescriptor & getRecurrenceDescriptor() const
Return the recurrence decriptor for the in-loop reduction.
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
VPReductionRecipe(const RecurrenceDescriptor &R, Instruction *I, VPValue *ChainOp, VPValue *VecOp, VPValue *CondOp, bool IsOrdered)
VPValue * getCondOp() const
The VPValue of the condition for the block.
VPReductionRecipe(const unsigned char SC, const RecurrenceDescriptor &R, Instruction *I, ArrayRef< VPValue * > Operands, VPValue *CondOp, bool IsOrdered)
bool isOrdered() const
Return true if the in-loop reduction is ordered.
~VPReductionRecipe() override=default
VPValue * getChainOp() const
The VPValue of the scalar Chain being accumulated.
VPReductionRecipe * clone() override
Clone the current recipe.
void execute(VPTransformState &State) override
Generate the reduction in the loop.
static bool classof(const VPUser *U)
VPRegionBlock represents a collection of VPBasicBlocks and VPRegionBlocks which form a Single-Entry-S...
VPRegionBlock * clone() override
Clone all blocks in the single-entry single-exit region of the block and their recipes without updati...
const VPBlockBase * getEntry() const
bool isReplicator() const
An indicator whether this region is to generate multiple replicated instances of output IR correspond...
void dropAllReferences(VPValue *NewValue) override
Replace all operands of VPUsers in the block with NewValue and also replaces all uses of VPValues def...
void setExiting(VPBlockBase *ExitingBlock)
Set ExitingBlock as the exiting VPBlockBase of this VPRegionBlock.
VPBlockBase * getExiting()
void setEntry(VPBlockBase *EntryBlock)
Set EntryBlock as the entry VPBlockBase of this VPRegionBlock.
InstructionCost cost(ElementCount VF, VPCostContext &Ctx) override
Return the cost of the block.
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print this VPRegionBlock to O (recursively), prefixing all lines with Indent.
VPRegionBlock(const std::string &Name="", bool IsReplicator=false)
VPRegionBlock(VPBlockBase *Entry, VPBlockBase *Exiting, const std::string &Name="", bool IsReplicator=false)
void execute(VPTransformState *State) override
The method which generates the output IR instructions that correspond to this VPRegionBlock,...
const VPBlockBase * getExiting() const
VPBasicBlock * getPreheaderVPBB()
Returns the pre-header VPBasicBlock of the loop region.
~VPRegionBlock() override
static bool classof(const VPBlockBase *V)
Method to support type inquiry through isa, cast, and dyn_cast.
VPReplicateRecipe replicates a given instruction producing multiple scalar copies of the original sca...
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
void execute(VPTransformState &State) override
Generate replicas of the desired Ingredient.
~VPReplicateRecipe() override=default
bool onlyFirstLaneUsed(const VPValue *Op) const override
Returns true if the recipe only uses the first lane of operand Op.
bool usesScalars(const VPValue *Op) const override
Returns true if the recipe uses scalars of operand Op.
bool isPredicated() const
VPReplicateRecipe * clone() override
Clone the current recipe.
VPReplicateRecipe(Instruction *I, iterator_range< IterT > Operands, bool IsUniform, VPValue *Mask=nullptr)
unsigned getOpcode() const
VPValue * getMask()
Return the mask of a predicated VPReplicateRecipe.
bool shouldPack() const
Returns true if the recipe is used by a widened recipe via an intervening VPPredInstPHIRecipe.
VPScalarCastRecipe is a recipe to create scalar cast instructions.
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Each concrete VPDef prints itself.
~VPScalarCastRecipe() override=default
VPScalarCastRecipe * clone() override
Clone the current recipe.
bool onlyFirstLaneUsed(const VPValue *Op) const override
Returns true if the VPUser only uses the first lane of operand Op.
void execute(VPTransformState &State) override
The method which generates the output IR instructions that correspond to this VPRecipe,...
Type * getResultType() const
Returns the result type of the cast.
VPScalarCastRecipe(Instruction::CastOps Opcode, VPValue *Op, Type *ResultTy)
A recipe for handling phi nodes of integer and floating-point inductions, producing their scalar valu...
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
bool onlyFirstLaneUsed(const VPValue *Op) const override
Returns true if the recipe only uses the first lane of operand Op.
VPValue * getStepValue() const
VPScalarIVStepsRecipe(const InductionDescriptor &IndDesc, VPValue *IV, VPValue *Step)
VPScalarIVStepsRecipe * clone() override
Clone the current recipe.
VPScalarIVStepsRecipe(VPValue *IV, VPValue *Step, Instruction::BinaryOps Opcode, FastMathFlags FMFs)
~VPScalarIVStepsRecipe() override=default
void execute(VPTransformState &State) override
Generate the scalarized versions of the phi node as needed by their users.
VPSingleDef is a base class for recipes for modeling a sequence of one or more output IR that define ...
VPSingleDefRecipe(const unsigned char SC, ArrayRef< VPValue * > Operands, DebugLoc DL={})
Instruction * getUnderlyingInstr()
Returns the underlying instruction.
static bool classof(const VPRecipeBase *R)
const Instruction * getUnderlyingInstr() const
VPSingleDefRecipe(const unsigned char SC, IterT Operands, DebugLoc DL={})
static bool classof(const VPUser *U)
VPSingleDefRecipe(const unsigned char SC, IterT Operands, Value *UV, DebugLoc DL={})
virtual VPSingleDefRecipe * clone() override=0
Clone the current recipe.
This class can be used to assign names to VPValues.
An analysis for type-inference for VPValues.
This class augments VPValue with operands which provide the inverse def-use edges from VPValue's user...
void setOperand(unsigned I, VPValue *New)
unsigned getNumOperands() const
operand_iterator op_begin()
VPValue * getOperand(unsigned N) const
void addOperand(VPValue *Operand)
VPRecipeBase * getDefiningRecipe()
Returns the recipe defining this VPValue or nullptr if it is not defined by a recipe,...
Value * getUnderlyingValue() const
Return the underlying Value attached to this VPValue.
unsigned getNumUsers() const
Value * getLiveInIRValue()
Returns the underlying IR value, if this VPValue is defined outside the scope of VPlan.
bool isLiveIn() const
Returns true if this VPValue is a live-in, i.e. defined outside the VPlan.
friend class VPRecipeBase
bool isDefinedOutsideVectorRegions() const
Returns true if the VPValue is defined outside any vector regions, i.e.
A recipe to compute the pointers for widened memory accesses of IndexTy for all parts.
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
void execute(VPTransformState &State) override
The method which generates the output IR instructions that correspond to this VPRecipe,...
VPVectorPointerRecipe(VPValue *Ptr, Type *IndexedTy, bool IsReverse, bool IsInBounds, DebugLoc DL)
bool onlyFirstLaneUsed(const VPValue *Op) const override
Returns true if the VPUser only uses the first lane of operand Op.
VPVectorPointerRecipe * clone() override
Clone the current recipe.
A recipe for widening Call instructions.
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
const_operand_range arg_operands() const
VPWidenCallRecipe * clone() override
Clone the current recipe.
VPWidenCallRecipe(Value *UV, iterator_range< IterT > CallArguments, Intrinsic::ID VectorIntrinsicID, DebugLoc DL={}, Function *Variant=nullptr)
Function * getCalledScalarFunction() const
void execute(VPTransformState &State) override
Produce a widened version of the call instruction.
operand_range arg_operands()
~VPWidenCallRecipe() override=default
A Recipe for widening the canonical induction variable of the vector loop.
void execute(VPTransformState &State) override
Generate a canonical vector induction variable of the vector loop, with start = {<Part*VF,...
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
~VPWidenCanonicalIVRecipe() override=default
VPWidenCanonicalIVRecipe * clone() override
Clone the current recipe.
VPWidenCanonicalIVRecipe(VPCanonicalIVPHIRecipe *CanonicalIV)
VPWidenCastRecipe is a recipe to create vector cast instructions.
VPWidenCastRecipe(Instruction::CastOps Opcode, VPValue *Op, Type *ResultTy, CastInst &UI)
Instruction::CastOps getOpcode() const
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
Type * getResultType() const
Returns the result type of the cast.
VPWidenCastRecipe(Instruction::CastOps Opcode, VPValue *Op, Type *ResultTy)
void execute(VPTransformState &State) override
Produce widened copies of the cast.
~VPWidenCastRecipe() override=default
VPWidenCastRecipe * clone() override
Clone the current recipe.
A recipe for handling GEP instructions.
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
void execute(VPTransformState &State) override
Generate the gep nodes.
VPWidenGEPRecipe * clone() override
Clone the current recipe.
~VPWidenGEPRecipe() override=default
VPWidenGEPRecipe(GetElementPtrInst *GEP, iterator_range< IterT > Operands)
A recipe for handling phi nodes of integer and floating-point inductions, producing their vector valu...
VPWidenIntOrFpInductionRecipe(PHINode *IV, VPValue *Start, VPValue *Step, const InductionDescriptor &IndDesc, TruncInst *Trunc)
const TruncInst * getTruncInst() const
VPRecipeBase & getBackedgeRecipe() override
Returns the backedge value as a recipe.
~VPWidenIntOrFpInductionRecipe() override=default
VPWidenIntOrFpInductionRecipe * clone() override
Clone the current recipe.
TruncInst * getTruncInst()
Returns the first defined value as TruncInst, if it is one or nullptr otherwise.
void execute(VPTransformState &State) override
Generate the vectorized and scalarized versions of the phi node as needed by their users.
VPValue * getStepValue()
Returns the step value of the induction.
VPWidenIntOrFpInductionRecipe(PHINode *IV, VPValue *Start, VPValue *Step, const InductionDescriptor &IndDesc)
const VPValue * getStepValue() const
Type * getScalarType() const
Returns the scalar type of the induction.
VPValue * getBackedgeValue() override
Returns the incoming value from the loop backedge.
bool isCanonical() const
Returns true if the induction is canonical, i.e.
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
const InductionDescriptor & getInductionDescriptor() const
Returns the induction descriptor for the recipe.
A common base class for widening memory operations.
bool IsMasked
Whether the memory access is masked.
bool Reverse
Whether the consecutive accessed addresses are in reverse order.
bool isConsecutive() const
Return whether the loaded-from / stored-to addresses are consecutive.
static bool classof(const VPUser *U)
void execute(VPTransformState &State) override
Generate the wide load/store.
VPWidenMemoryRecipe * clone() override
Clone the current recipe.
Instruction & getIngredient() const
bool Consecutive
Whether the accessed addresses are consecutive.
static bool classof(const VPRecipeBase *R)
VPWidenMemoryRecipe(const char unsigned SC, Instruction &I, std::initializer_list< VPValue * > Operands, bool Consecutive, bool Reverse, DebugLoc DL)
VPValue * getMask() const
Return the mask used by this recipe.
bool isMasked() const
Returns true if the recipe is masked.
void setMask(VPValue *Mask)
VPValue * getAddr() const
Return the address accessed by this recipe.
bool isReverse() const
Return whether the consecutive loaded/stored addresses are in reverse order.
A recipe for handling phis that are widened in the vector loop.
void addIncoming(VPValue *IncomingV, VPBasicBlock *IncomingBlock)
Adds a pair (IncomingV, IncomingBlock) to the phi.
VPValue * getIncomingValue(unsigned I)
Returns the I th incoming VPValue.
VPWidenPHIRecipe(PHINode *Phi, VPValue *Start=nullptr)
Create a new VPWidenPHIRecipe for Phi with start value Start.
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
VPWidenPHIRecipe * clone() override
Clone the current recipe.
~VPWidenPHIRecipe() override=default
VPBasicBlock * getIncomingBlock(unsigned I)
Returns the I th incoming VPBasicBlock.
void execute(VPTransformState &State) override
Generate the phi/select nodes.
VPWidenPointerInductionRecipe * clone() override
Clone the current recipe.
const InductionDescriptor & getInductionDescriptor() const
Returns the induction descriptor for the recipe.
~VPWidenPointerInductionRecipe() override=default
bool onlyScalarsGenerated(bool IsScalable)
Returns true if only scalar values will be generated.
void execute(VPTransformState &State) override
Generate vector values for the pointer induction.
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
VPWidenPointerInductionRecipe(PHINode *Phi, VPValue *Start, VPValue *Step, const InductionDescriptor &IndDesc, bool IsScalarAfterVectorization)
Create a new VPWidenPointerInductionRecipe for Phi with start value Start.
VPWidenRecipe is a recipe for producing a widened instruction using the opcode and operands of the re...
void execute(VPTransformState &State) override
Produce a widened instruction using the opcode and operands of the recipe, processing State....
VPWidenRecipe * clone() override
Clone the current recipe.
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
~VPWidenRecipe() override=default
VPWidenRecipe(Instruction &I, iterator_range< IterT > Operands)
unsigned getOpcode() const
VPlanPrinter prints a given VPlan to a given output stream.
VPlanPrinter(raw_ostream &O, const VPlan &P)
LLVM_DUMP_METHOD void dump()
Class that maps (parts of) an existing VPlan to trees of combined VPInstructions.
VPInstruction * buildGraph(ArrayRef< VPValue * > Operands)
Tries to build an SLP tree rooted at Operands and returns a VPInstruction combining Operands,...
bool isCompletelySLP() const
Return true if all visited instruction can be combined.
VPlanSlp(VPInterleavedAccessInfo &IAI, VPBasicBlock &BB)
unsigned getWidestBundleBits() const
Return the width of the widest combined bundle in bits.
VPlan models a candidate for vectorization, encoding various decisions take to produce efficient outp...
void printDOT(raw_ostream &O) const
Print this VPlan in DOT format to O.
std::string getName() const
Return a string with the name of the plan and the applicable VFs and UFs.
void prepareToExecute(Value *TripCount, Value *VectorTripCount, Value *CanonicalIVStartValue, VPTransformState &State)
Prepare the plan for execution, setting up the required live-in values.
VPBasicBlock * getEntry()
VPValue & getVectorTripCount()
The vector trip count.
void setName(const Twine &newName)
VPValue & getVFxUF()
Returns VF * UF of the vector loop region.
VPValue * getTripCount() const
The trip count of the original loop.
VPValue * getOrCreateBackedgeTakenCount()
The backedge taken count of the original loop.
void removeLiveOut(PHINode *PN)
iterator_range< SmallSetVector< ElementCount, 2 >::iterator > vectorFactors() const
Returns an iterator range over all VFs of the plan.
void addLiveOut(PHINode *PN, VPValue *V)
const VPBasicBlock * getEntry() const
VPlan(VPBasicBlock *Preheader, VPValue *TC, VPBasicBlock *Entry)
Construct a VPlan with original preheader Preheader, trip count TC and Entry to the plan.
VPBasicBlock * getPreheader()
VPRegionBlock * getVectorLoopRegion()
Returns the VPRegionBlock of the vector loop.
const VPRegionBlock * getVectorLoopRegion() const
bool hasVF(ElementCount VF)
void addSCEVExpansion(const SCEV *S, VPValue *V)
bool hasUF(unsigned UF) const
void setVF(ElementCount VF)
InstructionCost cost(ElementCount VF, VPCostContext &Ctx)
Return the cost of this plan.
void resetTripCount(VPValue *NewTripCount)
Resets the trip count for the VPlan.
static VPlanPtr createInitialVPlan(const SCEV *TripCount, ScalarEvolution &PSE, bool RequiresScalarEpilogueCheck, bool TailFolded, Loop *TheLoop)
Create initial VPlan, having an "entry" VPBasicBlock (wrapping original scalar pre-header ) which con...
VPlan(VPBasicBlock *Preheader, VPBasicBlock *Entry)
Construct a VPlan with original preheader Preheader and Entry to the plan.
const VPBasicBlock * getPreheader() const
VPValue * getOrAddLiveIn(Value *V)
Gets the live-in VPValue for V or adds a new live-in (if none exists yet) for V.
LLVM_DUMP_METHOD void dump() const
Dump the plan to stderr (for debugging).
bool hasScalarVFOnly() const
void execute(VPTransformState *State)
Generate the IR code for this VPlan.
VPCanonicalIVPHIRecipe * getCanonicalIV()
Returns the canonical induction recipe of the vector loop.
const MapVector< PHINode *, VPLiveOut * > & getLiveOuts() const
void print(raw_ostream &O) const
Print this VPlan to O.
void addVF(ElementCount VF)
VPValue * getLiveIn(Value *V) const
Return the live-in VPValue for V, if there is one or nullptr otherwise.
VPValue * getSCEVExpansion(const SCEV *S) const
void printLiveIns(raw_ostream &O) const
Print the live-ins of this VPlan to O.
VPlan * duplicate()
Clone the current VPlan, update all VPValues of the new VPlan and cloned recipes to refer to the clon...
LLVM Value Representation.
Type * getType() const
All values are typed, get the type of this value.
constexpr bool isScalable() const
Returns whether the quantity is scaled by a runtime quantity (vscale).
constexpr ScalarTy getKnownMinValue() const
Returns the minimum value this quantity can represent.
An ilist node that can access its parent list.
base_list_type::const_reverse_iterator const_reverse_iterator
base_list_type::reverse_iterator reverse_iterator
base_list_type::const_iterator const_iterator
base_list_type::iterator iterator
iterator insert(iterator where, pointer New)
CRTP base class which implements the entire standard iterator facade in terms of a minimal subset of ...
A range adaptor for a pair of iterators.
This class implements an extremely fast bulk output stream that can only output to a stream.
This file defines classes to implement an intrusive doubly linked list class (i.e.
This file defines the ilist_node class template, which is a convenient base class for creating classe...
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
@ BasicBlock
Various leaf nodes.
VPValue * getOrCreateVPValueForSCEVExpr(VPlan &Plan, const SCEV *Expr, ScalarEvolution &SE)
Get or create a VPValue that corresponds to the expansion of Expr.
bool isUniformAfterVectorization(VPValue *VPV)
Returns true if VPV is uniform after vectorization.
bool onlyFirstPartUsed(const VPValue *Def)
Returns true if only the first part of Def is used.
bool onlyFirstLaneUsed(const VPValue *Def)
Returns true if only the first lane of Def is used.
bool isHeaderMask(VPValue *V, VPlan &Plan)
Return true if V is a header mask in Plan.
This is an optimization pass for GlobalISel generic memory operations.
auto find(R &&Range, const T &Val)
Provide wrappers to std::find which take ranges instead of having to pass begin/end explicitly.
bool all_of(R &&range, UnaryPredicate P)
Provide wrappers to std::all_of which take ranges instead of having to pass begin/end explicitly.
bool isEqual(const GCNRPTracker::LiveRegSet &S1, const GCNRPTracker::LiveRegSet &S2)
const SCEV * createTripCountSCEV(Type *IdxTy, PredicatedScalarEvolution &PSE, Loop *OrigLoop)
testing::Matcher< const detail::ErrorHolder & > Failed()
Value * getRuntimeVF(IRBuilderBase &B, Type *Ty, ElementCount VF)
Return the runtime value for VF.
iterator_range< T > make_range(T x, T y)
Convenience function for iterating over sub-ranges.
auto cast_or_null(const Y &Val)
auto map_range(ContainerTy &&C, FuncTy F)
auto dyn_cast_or_null(const Y &Val)
bool any_of(R &&range, UnaryPredicate P)
Provide wrappers to std::any_of which take ranges instead of having to pass begin/end explicitly.
constexpr bool isPowerOf2_32(uint32_t Value)
Return true if the argument is a power of two > 0.
std::unique_ptr< VPlan > VPlanPtr
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
iterator_range< filter_iterator< detail::IterOfRange< RangeT >, PredicateT > > make_filter_range(RangeT &&Range, PredicateT Pred)
Convenience function that takes a range of elements and a predicate, and return a new filter_iterator...
raw_ostream & operator<<(raw_ostream &OS, const APFixedPoint &FX)
Value * createStepForVF(IRBuilderBase &B, Type *Ty, ElementCount VF, int64_t Step)
Return a value for Step multiplied by VF.
bool is_contained(R &&Range, const E &Element)
Returns true if Element is found in Range.
unsigned getReciprocalPredBlockProb()
A helper function that returns the reciprocal of the block probability of predicated blocks.
hash_code hash_combine_range(InputIteratorT first, InputIteratorT last)
Compute a hash_code for a sequence of values.
A range of powers-of-2 vectorization factors with fixed start and adjustable end.
VFRange(const ElementCount &Start, const ElementCount &End)
Struct to hold various analysis needed for cost computations.
LoopVectorizationCostModel & CM
bool skipCostComputation(Instruction *UI, bool IsVector) const
Return true if the cost for UI shouldn't be computed, e.g.
InstructionCost getLegacyCost(Instruction *UI, ElementCount VF) const
Return the cost for UI with VF using the legacy cost model as fallback until computing the cost of al...
VPCostContext(const TargetTransformInfo &TTI, Type *CanIVTy, LLVMContext &LLVMCtx, LoopVectorizationCostModel &CM)
const TargetTransformInfo & TTI
SmallPtrSet< Instruction *, 8 > SkipCostComputation
A recipe for handling first-order recurrence phis.
void execute(VPTransformState &State) override
Generate the phi nodes.
VPFirstOrderRecurrencePHIRecipe * clone() override
Clone the current recipe.
VPFirstOrderRecurrencePHIRecipe(PHINode *Phi, VPValue &Start)
static bool classof(const VPHeaderPHIRecipe *R)
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
VPIteration represents a single point in the iteration space of the output (vectorized and/or unrolle...
VPIteration(unsigned Part, const VPLane &Lane)
VPIteration(unsigned Part, unsigned Lane, VPLane::Kind Kind=VPLane::Kind::First)
bool isFirstIteration() const
DisjointFlagsTy(bool IsDisjoint)
GEPFlagsTy(bool IsInBounds)
WrapFlagsTy(bool HasNUW, bool HasNSW)
A recipe for widening load operations with vector-predication intrinsics, using the address to load f...
void execute(VPTransformState &State) override
Generate the wide load or gather.
VPValue * getEVL() const
Return the EVL operand.
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
VPWidenLoadEVLRecipe(VPWidenLoadRecipe *L, VPValue *EVL, VPValue *Mask)
bool onlyFirstLaneUsed(const VPValue *Op) const override
Returns true if the recipe only uses the first lane of operand Op.
A recipe for widening load operations, using the address to load from and an optional mask.
VP_CLASSOF_IMPL(VPDef::VPWidenLoadSC)
VPWidenLoadRecipe(LoadInst &Load, VPValue *Addr, VPValue *Mask, bool Consecutive, bool Reverse, DebugLoc DL)
bool onlyFirstLaneUsed(const VPValue *Op) const override
Returns true if the recipe only uses the first lane of operand Op.
void execute(VPTransformState &State) override
Generate a wide load or gather.
VPWidenLoadRecipe * clone() override
Clone the current recipe.
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
A recipe for widening select instructions.
bool isInvariantCond() const
VPWidenSelectRecipe * clone() override
Clone the current recipe.
VPWidenSelectRecipe(SelectInst &I, iterator_range< IterT > Operands)
VPValue * getCond() const
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
void execute(VPTransformState &State) override
Produce a widened version of the select instruction.
~VPWidenSelectRecipe() override=default
A recipe for widening store operations with vector-predication intrinsics, using the value to store,...
VPValue * getStoredValue() const
Return the address accessed by this recipe.
void execute(VPTransformState &State) override
Generate the wide store or scatter.
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
VPWidenStoreEVLRecipe(VPWidenStoreRecipe *S, VPValue *EVL, VPValue *Mask)
bool onlyFirstLaneUsed(const VPValue *Op) const override
Returns true if the recipe only uses the first lane of operand Op.
VPValue * getEVL() const
Return the EVL operand.
A recipe for widening store operations, using the stored value, the address to store to and an option...
void execute(VPTransformState &State) override
Generate a wide store or scatter.
bool onlyFirstLaneUsed(const VPValue *Op) const override
Returns true if the recipe only uses the first lane of operand Op.
VPWidenStoreRecipe(StoreInst &Store, VPValue *Addr, VPValue *StoredVal, VPValue *Mask, bool Consecutive, bool Reverse, DebugLoc DL)
VP_CLASSOF_IMPL(VPDef::VPWidenStoreSC)
VPValue * getStoredValue() const
Return the value stored by this recipe.
VPWidenStoreRecipe * clone() override
Clone the current recipe.
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
VPlanIngredient(const Value *V)
void print(raw_ostream &O) const