30#define DEBUG_TYPE "vectorutils"
38 cl::desc(
"Maximum factor for an interleaved access group (default = 8)"),
48 case Intrinsic::bswap:
49 case Intrinsic::bitreverse:
50 case Intrinsic::ctpop:
59 case Intrinsic::sadd_sat:
60 case Intrinsic::ssub_sat:
61 case Intrinsic::uadd_sat:
62 case Intrinsic::usub_sat:
63 case Intrinsic::smul_fix:
64 case Intrinsic::smul_fix_sat:
65 case Intrinsic::umul_fix:
66 case Intrinsic::umul_fix_sat:
73 case Intrinsic::log10:
76 case Intrinsic::minnum:
77 case Intrinsic::maxnum:
78 case Intrinsic::minimum:
79 case Intrinsic::maximum:
80 case Intrinsic::copysign:
81 case Intrinsic::floor:
83 case Intrinsic::trunc:
85 case Intrinsic::nearbyint:
86 case Intrinsic::round:
87 case Intrinsic::roundeven:
90 case Intrinsic::fmuladd:
91 case Intrinsic::is_fpclass:
93 case Intrinsic::canonicalize:
94 case Intrinsic::fptosi_sat:
95 case Intrinsic::fptoui_sat:
96 case Intrinsic::lrint:
97 case Intrinsic::llrint:
106 unsigned ScalarOpdIdx) {
109 case Intrinsic::ctlz:
110 case Intrinsic::cttz:
111 case Intrinsic::is_fpclass:
112 case Intrinsic::powi:
113 return (ScalarOpdIdx == 1);
114 case Intrinsic::smul_fix:
115 case Intrinsic::smul_fix_sat:
116 case Intrinsic::umul_fix:
117 case Intrinsic::umul_fix_sat:
118 return (ScalarOpdIdx == 2);
129 case Intrinsic::fptosi_sat:
130 case Intrinsic::fptoui_sat:
131 case Intrinsic::lrint:
132 case Intrinsic::llrint:
133 return OpdIdx == -1 || OpdIdx == 0;
134 case Intrinsic::is_fpclass:
136 case Intrinsic::powi:
137 return OpdIdx == -1 || OpdIdx == 1;
153 ID == Intrinsic::lifetime_end ||
ID == Intrinsic::assume ||
154 ID == Intrinsic::experimental_noalias_scope_decl ||
155 ID == Intrinsic::sideeffect ||
ID == Intrinsic::pseudoprobe)
164 assert(V->getType()->isVectorTy() &&
"Not looking at a vector?");
165 VectorType *VTy = cast<VectorType>(V->getType());
167 if (
auto *FVTy = dyn_cast<FixedVectorType>(VTy)) {
168 unsigned Width = FVTy->getNumElements();
174 return C->getAggregateElement(EltNo);
178 if (!isa<ConstantInt>(III->getOperand(2)))
180 unsigned IIElt = cast<ConstantInt>(III->getOperand(2))->getZExtValue();
185 return III->getOperand(1);
188 if (III == III->getOperand(0))
198 if (SVI && isa<FixedVectorType>(SVI->
getType())) {
204 if (InEl < (
int)LHSWidth)
213 if (
Constant *Elt =
C->getAggregateElement(EltNo))
214 if (Elt->isNullValue())
218 if (isa<ScalableVectorType>(VTy))
220 if (EltNo < VTy->getElementCount().getKnownMinValue())
235 if (SplatIndex != -1 && SplatIndex != M)
241 assert((SplatIndex == -1 || SplatIndex >= 0) &&
"Negative index?");
250 if (isa<VectorType>(V->getType()))
251 if (
auto *
C = dyn_cast<Constant>(V))
252 return C->getSplatValue();
267 if (isa<VectorType>(V->getType())) {
268 if (isa<UndefValue>(V))
272 if (
auto *
C = dyn_cast<Constant>(V))
273 return C->getSplatValue() !=
nullptr;
276 if (
auto *Shuf = dyn_cast<ShuffleVectorInst>(V)) {
311 const APInt &DemandedElts,
APInt &DemandedLHS,
312 APInt &DemandedRHS,
bool AllowUndefElts) {
316 if (DemandedElts.
isZero())
320 if (
all_of(Mask, [](
int Elt) {
return Elt == 0; })) {
325 for (
unsigned I = 0, E = Mask.size();
I != E; ++
I) {
327 assert((-1 <= M) && (M < (SrcWidth * 2)) &&
328 "Invalid shuffle mask constant");
330 if (!DemandedElts[
I] || (AllowUndefElts && (M < 0)))
341 DemandedRHS.
setBit(M - SrcWidth);
349 assert(Scale > 0 &&
"Unexpected scaling factor");
353 ScaledMask.
assign(Mask.begin(), Mask.end());
358 for (
int MaskElt : Mask) {
361 "Overflowed 32-bits");
363 for (
int SliceElt = 0; SliceElt != Scale; ++SliceElt)
364 ScaledMask.
push_back(MaskElt < 0 ? MaskElt : Scale * MaskElt + SliceElt);
370 assert(Scale > 0 &&
"Unexpected scaling factor");
374 ScaledMask.
assign(Mask.begin(), Mask.end());
379 int NumElts = Mask.size();
380 if (NumElts % Scale != 0)
384 ScaledMask.
reserve(NumElts / Scale);
389 assert((
int)MaskSlice.
size() == Scale &&
"Expected Scale-sized slice.");
392 int SliceFront = MaskSlice.
front();
393 if (SliceFront < 0) {
401 if (SliceFront % Scale != 0)
404 for (
int i = 1; i < Scale; ++i)
405 if (MaskSlice[i] != SliceFront + i)
407 ScaledMask.
push_back(SliceFront / Scale);
409 Mask = Mask.drop_front(Scale);
410 }
while (!Mask.empty());
412 assert((
int)ScaledMask.
size() * Scale == NumElts &&
"Unexpected scaled mask");
421 std::array<SmallVector<int, 16>, 2> TmpMasks;
424 for (
unsigned Scale = 2; Scale <= InputMask.
size(); ++Scale) {
434 ArrayRef<int> Mask,
unsigned NumOfSrcRegs,
unsigned NumOfDestRegs,
435 unsigned NumOfUsedRegs,
function_ref<
void()> NoInputAction,
443 int Sz = Mask.size();
444 unsigned SzDest = Sz / NumOfDestRegs;
445 unsigned SzSrc = Sz / NumOfSrcRegs;
446 for (
unsigned I = 0;
I < NumOfDestRegs; ++
I) {
447 auto &RegMasks = Res[
I];
448 RegMasks.
assign(NumOfSrcRegs, {});
451 for (
unsigned K = 0; K < SzDest; ++K) {
452 int Idx =
I * SzDest + K;
457 int SrcRegIdx = Mask[
Idx] / SzSrc;
460 if (RegMasks[SrcRegIdx].empty())
462 RegMasks[SrcRegIdx][K] = Mask[
Idx] % SzSrc;
466 for (
unsigned I = 0;
I < NumOfUsedRegs; ++
I) {
470 switch (NumSrcRegs) {
479 unsigned SrcReg = std::distance(Dest.begin(), It);
480 SingleInputAction(*It, SrcReg,
I);
495 "Expected undefined mask element.");
496 FirstMask[
Idx] = SecondMask[
Idx] + VF;
501 for (
int Idx = 0, VF = Mask.size();
Idx < VF; ++
Idx) {
511 for (
unsigned I = 0;
I < NumOfDestRegs; ++
I) {
516 if (FirstIdx == SecondIdx) {
522 SecondMask = RegMask;
523 CombineMasks(FirstMask, SecondMask);
524 ManyInputsAction(FirstMask, FirstIdx, SecondIdx);
525 NormalizeMask(FirstMask);
527 SecondMask = FirstMask;
528 SecondIdx = FirstIdx;
530 if (FirstIdx != SecondIdx && SecondIdx >= 0) {
531 CombineMasks(SecondMask, FirstMask);
532 ManyInputsAction(SecondMask, SecondIdx, FirstIdx);
533 Dest[FirstIdx].clear();
534 NormalizeMask(SecondMask);
536 }
while (SecondIdx >= 0);
559 bool SeenExtFromIllegalType =
false;
561 for (
auto &
I : *BB) {
564 if (
TTI && (isa<ZExtInst>(&
I) || isa<SExtInst>(&
I)) &&
566 SeenExtFromIllegalType =
true;
569 if ((isa<TruncInst>(&
I) || isa<ICmpInst>(&
I)) &&
570 !
I.getType()->isVectorTy() &&
571 I.getOperand(0)->getType()->getScalarSizeInBits() <= 64) {
582 if (Worklist.
empty() || (
TTI && !SeenExtFromIllegalType))
586 while (!Worklist.
empty()) {
590 if (!Visited.
insert(Val).second)
594 if (!isa<Instruction>(Val))
600 if (DB.getDemandedBits(
I).getBitWidth() > 64)
603 uint64_t V = DB.getDemandedBits(
I).getZExtValue();
609 if (isa<SExtInst>(
I) || isa<ZExtInst>(
I) || isa<LoadInst>(
I) ||
616 if (isa<BitCastInst>(
I) || isa<PtrToIntInst>(
I) || isa<IntToPtrInst>(
I) ||
617 !
I->getType()->isIntegerTy()) {
618 DBits[Leader] |= ~0ULL;
628 if (DBits[Leader] == ~0ULL)
632 for (
Value *O : cast<User>(
I)->operands()) {
641 for (
auto &
I : DBits)
642 for (
auto *U :
I.first->users())
643 if (U->getType()->isIntegerTy() && DBits.
count(U) == 0)
646 for (
auto I = ECs.
begin(), E = ECs.
end();
I != E; ++
I) {
649 LeaderDemandedBits |= DBits[M];
669 auto *
MI = dyn_cast<Instruction>(M);
672 Type *Ty = M->getType();
674 Ty =
MI->getOperand(0)->getType();
682 auto *CI = dyn_cast<ConstantInt>(U);
686 isa<ShlOperator, LShrOperator, AShrOperator>(U.getUser()) &&
687 U.getOperandNo() == 1)
688 return CI->uge(MinBW);
689 uint64_t BW = bit_width(DB.getDemandedBits(&U).getZExtValue());
690 return bit_ceil(BW) > MinBW;
702template <
typename ListT>
707 List.insert(AccGroups);
711 for (
const auto &AccGroupListOp : AccGroups->
operands()) {
712 auto *Item = cast<MDNode>(AccGroupListOp.get());
723 if (AccGroups1 == AccGroups2)
730 if (Union.size() == 0)
732 if (Union.size() == 1)
733 return cast<MDNode>(Union.front());
744 if (!MayAccessMem1 && !MayAccessMem2)
747 return Inst2->
getMetadata(LLVMContext::MD_access_group);
749 return Inst1->
getMetadata(LLVMContext::MD_access_group);
765 if (AccGroupSet2.
count(MD1))
769 auto *Item = cast<MDNode>(Node.get());
771 if (AccGroupSet2.
count(Item))
776 if (Intersection.
size() == 0)
778 if (Intersection.
size() == 1)
779 return cast<MDNode>(Intersection.
front());
793 for (
auto Kind : {LLVMContext::MD_tbaa, LLVMContext::MD_alias_scope,
794 LLVMContext::MD_noalias, LLVMContext::MD_fpmath,
795 LLVMContext::MD_nontemporal, LLVMContext::MD_invariant_load,
796 LLVMContext::MD_access_group}) {
799 for (
int J = 1, E = VL.
size(); MD && J != E; ++J) {
803 case LLVMContext::MD_tbaa:
806 case LLVMContext::MD_alias_scope:
809 case LLVMContext::MD_fpmath:
812 case LLVMContext::MD_noalias:
813 case LLVMContext::MD_nontemporal:
814 case LLVMContext::MD_invariant_load:
817 case LLVMContext::MD_access_group:
842 for (
unsigned i = 0; i < VF; i++)
843 for (
unsigned j = 0; j < Group.
getFactor(); ++j) {
844 unsigned HasMember = Group.
getMember(j) ? 1 : 0;
845 Mask.push_back(Builder.
getInt1(HasMember));
854 for (
unsigned i = 0; i < VF; i++)
855 for (
unsigned j = 0; j < ReplicationFactor; j++)
864 for (
unsigned i = 0; i < VF; i++)
865 for (
unsigned j = 0; j < NumVecs; j++)
866 Mask.push_back(j * VF + i);
874 for (
unsigned i = 0; i < VF; i++)
875 Mask.push_back(Start + i * Stride);
882 unsigned NumUndefs) {
884 for (
unsigned i = 0; i < NumInts; i++)
885 Mask.push_back(Start + i);
887 for (
unsigned i = 0; i < NumUndefs; i++)
896 int NumEltsSigned = NumElts;
897 assert(NumEltsSigned > 0 &&
"Expected smaller or non-zero element count");
902 for (
int MaskElt : Mask) {
903 assert((MaskElt < NumEltsSigned * 2) &&
"Expected valid shuffle mask");
904 int UnaryElt = MaskElt >= NumEltsSigned ? MaskElt - NumEltsSigned : MaskElt;
916 VectorType *VecTy2 = dyn_cast<VectorType>(V2->getType());
917 assert(VecTy1 && VecTy2 &&
918 VecTy1->getScalarType() == VecTy2->getScalarType() &&
919 "Expect two vectors with the same element type");
921 unsigned NumElts1 = cast<FixedVectorType>(VecTy1)->getNumElements();
922 unsigned NumElts2 = cast<FixedVectorType>(VecTy2)->getNumElements();
923 assert(NumElts1 >= NumElts2 &&
"Unexpect the first vector has less elements");
925 if (NumElts1 > NumElts2) {
937 unsigned NumVecs = Vecs.
size();
938 assert(NumVecs > 1 &&
"Should be at least two vectors");
944 for (
unsigned i = 0; i < NumVecs - 1; i += 2) {
945 Value *V0 = ResList[i], *V1 = ResList[i + 1];
946 assert((V0->
getType() == V1->getType() || i == NumVecs - 2) &&
947 "Only the last vector may have a different type");
953 if (NumVecs % 2 != 0)
957 NumVecs = ResList.
size();
958 }
while (NumVecs > 1);
964 assert(isa<VectorType>(Mask->getType()) &&
965 isa<IntegerType>(Mask->getType()->getScalarType()) &&
966 cast<IntegerType>(Mask->getType()->getScalarType())->getBitWidth() ==
968 "Mask must be a vector of i1");
970 auto *ConstMask = dyn_cast<Constant>(Mask);
973 if (ConstMask->isNullValue() || isa<UndefValue>(ConstMask))
975 if (isa<ScalableVectorType>(ConstMask->getType()))
979 E = cast<FixedVectorType>(ConstMask->getType())->getNumElements();
981 if (
auto *MaskElt = ConstMask->getAggregateElement(
I))
982 if (MaskElt->isNullValue() || isa<UndefValue>(MaskElt))
990 assert(isa<VectorType>(Mask->getType()) &&
991 isa<IntegerType>(Mask->getType()->getScalarType()) &&
992 cast<IntegerType>(Mask->getType()->getScalarType())->getBitWidth() ==
994 "Mask must be a vector of i1");
996 auto *ConstMask = dyn_cast<Constant>(Mask);
999 if (ConstMask->isAllOnesValue() || isa<UndefValue>(ConstMask))
1001 if (isa<ScalableVectorType>(ConstMask->getType()))
1005 E = cast<FixedVectorType>(ConstMask->getType())->getNumElements();
1007 if (
auto *MaskElt = ConstMask->getAggregateElement(
I))
1008 if (MaskElt->isAllOnesValue() || isa<UndefValue>(MaskElt))
1016 assert(isa<VectorType>(Mask->getType()) &&
1017 isa<IntegerType>(Mask->getType()->getScalarType()) &&
1018 cast<IntegerType>(Mask->getType()->getScalarType())->getBitWidth() ==
1020 "Mask must be a vector of i1");
1022 auto *ConstMask = dyn_cast<Constant>(Mask);
1025 if (ConstMask->isAllOnesValue() || isa<UndefValue>(ConstMask))
1027 if (isa<ScalableVectorType>(ConstMask->getType()))
1031 E = cast<FixedVectorType>(ConstMask->getType())->getNumElements();
1033 if (
auto *MaskElt = ConstMask->getAggregateElement(
I))
1034 if (MaskElt->isAllOnesValue() || isa<UndefValue>(MaskElt))
1043 assert(isa<FixedVectorType>(Mask->getType()) &&
1044 isa<IntegerType>(Mask->getType()->getScalarType()) &&
1045 cast<IntegerType>(Mask->getType()->getScalarType())->getBitWidth() ==
1047 "Mask must be a fixed width vector of i1");
1049 const unsigned VWidth =
1050 cast<FixedVectorType>(Mask->getType())->getNumElements();
1052 if (
auto *CV = dyn_cast<ConstantVector>(Mask))
1053 for (
unsigned i = 0; i < VWidth; i++)
1054 if (CV->getAggregateElement(i)->isNullValue())
1056 return DemandedElts;
1059bool InterleavedAccessInfo::isStrided(
int Stride) {
1060 unsigned Factor = std::abs(Stride);
1064void InterleavedAccessInfo::collectConstStrideAccesses(
1078 for (
auto &
I : *BB) {
1087 if (
Size * 8 !=
DL.getTypeSizeInBits(ElementTy))
1099 true,
false).value_or(0);
1102 AccessStrideInfo[&
I] = StrideDescriptor(Stride, Scev,
Size,
1144 bool EnablePredicatedInterleavedMemAccesses) {
1150 collectConstStrideAccesses(AccessStrideInfo, Strides);
1152 if (AccessStrideInfo.
empty())
1156 collectDependences();
1177 for (
auto BI = AccessStrideInfo.
rbegin(), E = AccessStrideInfo.
rend();
1180 StrideDescriptor DesB = BI->second;
1186 if (isStrided(DesB.Stride) &&
1187 (!isPredicated(
B->getParent()) || EnablePredicatedInterleavedMemAccesses)) {
1192 GroupB = createInterleaveGroup(
B, DesB.Stride, DesB.Alignment);
1193 if (
B->mayWriteToMemory())
1194 StoreGroups.
insert(GroupB);
1196 LoadGroups.
insert(GroupB);
1200 for (
auto AI = std::next(BI); AI != E; ++AI) {
1202 StrideDescriptor DesA = AI->second;
1227 if (MemberOfGroupB && !canReorderMemAccessesForInterleavedGroups(
1228 A, &*AccessStrideInfo.
find(MemberOfGroupB)))
1229 return MemberOfGroupB;
1239 if (
A->mayWriteToMemory() && GroupA != GroupB) {
1247 if (GroupB && LoadGroups.
contains(GroupB))
1248 DependentInst = DependentMember(GroupB, &*AI);
1249 else if (!canReorderMemAccessesForInterleavedGroups(&*AI, &*BI))
1252 if (DependentInst) {
1257 if (GroupA && StoreGroups.
contains(GroupA)) {
1259 "dependence between "
1260 << *
A <<
" and " << *DependentInst <<
'\n');
1261 StoreGroups.
remove(GroupA);
1262 releaseGroup(GroupA);
1268 if (GroupB && LoadGroups.
contains(GroupB)) {
1270 <<
" as complete.\n");
1271 CompletedLoadGroups.
insert(GroupB);
1275 if (CompletedLoadGroups.
contains(GroupB)) {
1283 if (!isStrided(DesA.Stride) || !isStrided(DesB.Stride))
1293 (
A->mayReadFromMemory() !=
B->mayReadFromMemory()) ||
1294 (
A->mayWriteToMemory() !=
B->mayWriteToMemory()))
1299 if (DesA.Stride != DesB.Stride || DesA.Size != DesB.Size)
1316 if (DistanceToB %
static_cast<int64_t
>(DesB.Size))
1323 if ((isPredicated(BlockA) || isPredicated(BlockB)) &&
1324 (!EnablePredicatedInterleavedMemAccesses || BlockA != BlockB))
1330 GroupB->
getIndex(
B) + DistanceToB /
static_cast<int64_t
>(DesB.Size);
1335 <<
" into the interleave group with" << *
B
1337 InterleaveGroupMap[
A] = GroupB;
1340 if (
A->mayReadFromMemory())
1348 std::string FirstOrLast) ->
bool {
1350 assert(Member &&
"Group member does not exist");
1353 if (
getPtrStride(PSE, AccessTy, MemberPtr, TheLoop, Strides,
1354 false,
true).value_or(0))
1356 LLVM_DEBUG(
dbgs() <<
"LV: Invalidate candidate interleaved group due to "
1358 <<
" group member potentially pointer-wrapping.\n");
1359 releaseGroup(Group);
1377 for (
auto *Group : LoadGroups) {
1389 if (InvalidateGroupIfMemberMayWrap(Group, 0, std::string(
"first")))
1392 InvalidateGroupIfMemberMayWrap(Group, Group->
getFactor() - 1,
1393 std::string(
"last"));
1402 dbgs() <<
"LV: Invalidate candidate interleaved group due to "
1403 "a reverse access with gaps.\n");
1404 releaseGroup(Group);
1408 dbgs() <<
"LV: Interleaved group requires epilogue iteration.\n");
1409 RequiresScalarEpilogue =
true;
1413 for (
auto *Group : StoreGroups) {
1423 if (!EnablePredicatedInterleavedMemAccesses) {
1425 dbgs() <<
"LV: Invalidate candidate interleaved store group due "
1427 releaseGroup(Group);
1437 if (InvalidateGroupIfMemberMayWrap(Group, 0, std::string(
"first")))
1441 InvalidateGroupIfMemberMayWrap(Group,
Index, std::string(
"last"));
1453 bool ReleasedGroup =
false;
1457 if (!Group->requiresScalarEpilogue())
1461 <<
"LV: Invalidate candidate interleaved group due to gaps that "
1462 "require a scalar epilogue (not allowed under optsize) and cannot "
1463 "be masked (not enabled). \n");
1464 releaseGroup(Group);
1465 ReleasedGroup =
true;
1467 assert(ReleasedGroup &&
"At least one group must be invalidated, as a "
1468 "scalar epilogue was required");
1469 (void)ReleasedGroup;
1470 RequiresScalarEpilogue =
false;
1473template <
typename InstT>
1482 std::transform(Members.begin(), Members.end(), std::back_inserter(VL),
1483 [](std::pair<int, Instruction *> p) { return p.second; });
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")
This file contains the declarations for the subclasses of Constant, which represent the different fla...
Returns the sub type a function will return at a given Idx Should correspond to the result type of an ExtractValue instruction executed with just that one unsigned Idx
DenseMap< Block *, BlockRelaxAux > Blocks
Generic implementation of equivalence classes through the use Tarjan's efficient union-find algorithm...
static GCMetadataPrinterRegistry::Add< ErlangGCPrinter > X("erlang", "erlang-compatible garbage collector")
static GCMetadataPrinterRegistry::Add< OcamlGCMetadataPrinter > Y("ocaml", "ocaml 3.10-compatible collector")
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
This file defines the SmallVector class.
static SymbolRef::Type getType(const Symbol *Sym)
static Value * concatenateTwoVectors(IRBuilderBase &Builder, Value *V1, Value *V2)
A helper function for concatenating vectors.
static cl::opt< unsigned > MaxInterleaveGroupFactor("max-interleave-group-factor", cl::Hidden, cl::desc("Maximum factor for an interleaved access group (default = 8)"), cl::init(8))
Maximum factor for an interleaved memory access.
static void addToAccessGroupList(ListT &List, MDNode *AccGroups)
Add all access groups in AccGroups to List.
Class for arbitrary precision integers.
static APInt getAllOnes(unsigned numBits)
Return an APInt of a specified width with all bits set.
void clearBit(unsigned BitPosition)
Set a given bit to 0.
void setBit(unsigned BitPosition)
Set the given bit to 1 whose position is given as "bitPosition".
bool isZero() const
Determine if this value is zero, i.e. all bits are clear.
static APInt getZero(unsigned numBits)
Get the '0' value for the specified bit-width.
int64_t getSExtValue() const
Get sign extended value.
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
const T & front() const
front - Get the first element.
size_t size() const
size - Get the array size.
bool empty() const
empty - Check if the array is empty.
LLVM Basic Block Representation.
const Module * getModule() const
Return the module owning the function this basic block belongs to, or nullptr if the function does no...
This class represents a function call, abstracting a target machine's calling convention.
static Constant * get(ArrayRef< Constant * > V)
This is an important base class in LLVM.
size_type count(const_arg_type_t< KeyT > Val) const
Return 1 if the specified key is in the map, 0 otherwise.
EquivalenceClasses - This represents a collection of equivalence classes and supports three efficient...
const ElemTy & getOrInsertLeaderValue(const ElemTy &V)
getOrInsertLeaderValue - Return the leader for the specified value that is in the set.
member_iterator member_end() const
member_iterator member_begin(iterator I) const
member_iterator unionSets(const ElemTy &V1, const ElemTy &V2)
union - Merge the two equivalence sets for the specified values, inserting them if they do not alread...
Common base class shared among various IRBuilders.
ConstantInt * getInt1(bool V)
Get a constant value representing either true or false.
Value * CreateShuffleVector(Value *V1, Value *V2, Value *Mask, const Twine &Name="")
This instruction inserts a single (scalar) element into a VectorType value.
bool mayReadOrWriteMemory() const
Return true if this instruction may read or write memory.
MDNode * getMetadata(unsigned KindID) const
Get the metadata of given kind attached to this Instruction.
void setMetadata(unsigned KindID, MDNode *Node)
Set the metadata of the specified kind to the specified node.
void getAllMetadataOtherThanDebugLoc(SmallVectorImpl< std::pair< unsigned, MDNode * > > &MDs) const
This does the same thing as getAllMetadata, except that it filters out the debug location.
The group of interleaved loads/stores sharing the same stride and close to each other.
uint32_t getFactor() const
InstTy * getMember(uint32_t Index) const
Get the member with the given index Index.
uint32_t getIndex(const InstTy *Instr) const
Get the index for the given member.
void setInsertPos(InstTy *Inst)
void addMetadata(InstTy *NewInst) const
Add metadata (e.g.
bool insertMember(InstTy *Instr, int32_t Index, Align NewAlign)
Try to insert a new member Instr with index Index and alignment NewAlign.
uint32_t getNumMembers() const
InterleaveGroup< Instruction > * getInterleaveGroup(const Instruction *Instr) const
Get the interleave group that Instr belongs to.
bool requiresScalarEpilogue() const
Returns true if an interleaved group that may access memory out-of-bounds requires a scalar epilogue ...
bool isInterleaved(Instruction *Instr) const
Check if Instr belongs to any interleave group.
void analyzeInterleaving(bool EnableMaskedInterleavedGroup)
Analyze the interleaved accesses and collect them in interleave groups.
void invalidateGroupsRequiringScalarEpilogue()
Invalidate groups that require a scalar epilogue (due to gaps).
This is an important class for using LLVM in a threaded context.
const DenseMap< Value *, const SCEV * > & getSymbolicStrides() const
If an access has a symbolic strides, this maps the pointer value to the stride symbol.
BlockT * getHeader() const
Store the result of a depth first search within basic blocks contained by a single loop.
static MDNode * getMostGenericAliasScope(MDNode *A, MDNode *B)
static MDNode * getMostGenericTBAA(MDNode *A, MDNode *B)
ArrayRef< MDOperand > operands() const
static MDTuple * get(LLVMContext &Context, ArrayRef< Metadata * > MDs)
static MDNode * getMostGenericFPMath(MDNode *A, MDNode *B)
unsigned getNumOperands() const
Return number of MDNode operands.
static MDNode * intersect(MDNode *A, MDNode *B)
LLVMContext & getContext() const
Tracking metadata reference owned by Metadata.
This class implements a map that also provides access to all stored values in a deterministic order.
iterator find(const KeyT &Key)
reverse_iterator rbegin()
const DataLayout & getDataLayout() const
Get the data layout for the module's target platform.
MutableArrayRef - Represent a mutable reference to an array (0 or more elements consecutively in memo...
ScalarEvolution * getSE() const
Returns the ScalarEvolution analysis used.
This class represents a constant integer value.
const APInt & getAPInt() const
This class represents an analyzed expression in the program.
const SCEV * getMinusSCEV(const SCEV *LHS, const SCEV *RHS, SCEV::NoWrapFlags Flags=SCEV::FlagAnyWrap, unsigned Depth=0)
Return LHS-RHS.
bool remove(const value_type &X)
Remove an item from the set vector.
bool insert(const value_type &X)
Insert a new element into the SetVector.
bool contains(const key_type &key) const
Check if the SetVector contains the given key.
This instruction constructs a fixed permutation of two input vectors.
int getMaskValue(unsigned Elt) const
Return the shuffle mask value of this instruction for the given element index.
VectorType * getType() const
Overload to return most specific vector type.
size_type count(ConstPtrType Ptr) const
count - Return 1 if the specified pointer is in the set, 0 otherwise.
std::pair< iterator, bool > insert(PtrType Ptr)
Inserts Ptr if and only if there is no element in the container equal to Ptr.
bool contains(ConstPtrType Ptr) const
SmallPtrSet - This class implements a set which is optimized for holding SmallSize or less elements.
A SetVector that performs no allocations if smaller than a certain size.
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
void assign(size_type NumElts, ValueParamT Elt)
void reserve(size_type N)
void append(ItTy in_start, ItTy in_end)
Add the specified range to the end of the SmallVector.
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
Provides information about what library functions are available for the current target.
The instances of the Type class are immutable: once they are created, they are never changed.
unsigned getScalarSizeInBits() const LLVM_READONLY
If this is a vector type, return the getPrimitiveSizeInBits value for the element type.
static UndefValue * get(Type *T)
Static factory methods - Return an 'undef' object of the specified type.
A Use represents the edge between a Value definition and its users.
Value * getOperand(unsigned i) const
LLVM Value Representation.
Type * getType() const
All values are typed, get the type of this value.
LLVMContext & getContext() const
All values hold a context through their type.
Base class of all SIMD vector types.
Type * getElementType() const
An efficient, type-erasing, non-owning reference to a callable.
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
unsigned ID
LLVM IR allows to use arbitrary numbers as calling convention identifiers.
@ C
The default llvm calling convention, compatible with C.
BinaryOp_match< LHS, RHS, Instruction::Add > m_Add(const LHS &L, const RHS &R)
class_match< BinaryOperator > m_BinOp()
Match an arbitrary binary operation and ignore it.
class_match< Constant > m_Constant()
Match an arbitrary Constant and ignore it.
bool match(Val *V, const Pattern &P)
ThreeOps_match< Cond, LHS, RHS, Instruction::Select > m_Select(const Cond &C, const LHS &L, const RHS &R)
Matches SelectInst.
cst_pred_ty< is_zero_int > m_ZeroInt()
Match an integer 0 or a vector with all elements equal to 0.
TwoOps_match< V1_t, V2_t, Instruction::ShuffleVector > m_Shuffle(const V1_t &v1, const V2_t &v2)
Matches ShuffleVectorInst independently of mask value.
class_match< Value > m_Value()
Match an arbitrary value and ignore it.
ThreeOps_match< Val_t, Elt_t, Idx_t, Instruction::InsertElement > m_InsertElt(const Val_t &Val, const Elt_t &Elt, const Idx_t &Idx)
Matches InsertElementInst.
initializer< Ty > init(const Ty &Val)
This is an optimization pass for GlobalISel generic memory operations.
bool all_of(R &&range, UnaryPredicate P)
Provide wrappers to std::all_of which take ranges instead of having to pass begin/end explicitly.
Intrinsic::ID getVectorIntrinsicIDForCall(const CallInst *CI, const TargetLibraryInfo *TLI)
Returns intrinsic ID for call.
APInt possiblyDemandedEltsInMask(Value *Mask)
Given a mask vector of the form <Y x i1>, return an APInt (of bitwidth Y) for each lane which may be ...
bool isVectorIntrinsicWithOverloadTypeAtArg(Intrinsic::ID ID, int OpdIdx)
Identifies if the vector form of the intrinsic is overloaded on the type of the operand at index OpdI...
unsigned getLoadStoreAddressSpace(Value *I)
A helper function that returns the address space of the pointer operand of load or store instruction.
const Value * getLoadStorePointerOperand(const Value *V)
A helper function that returns the pointer operand of a load or store instruction.
llvm::SmallVector< int, 16 > createUnaryMask(ArrayRef< int > Mask, unsigned NumElts)
Given a shuffle mask for a binary shuffle, create the equivalent shuffle mask assuming both operands ...
iterator_range< T > make_range(T x, T y)
Convenience function for iterating over sub-ranges.
int bit_width(T Value)
Returns the number of bits needed to represent Value if Value is nonzero.
iterator_range< early_inc_iterator_impl< detail::IterOfRange< RangeT > > > make_early_inc_range(RangeT &&Range)
Make a range that does early increment to allow mutation of the underlying range without disrupting i...
Value * concatenateVectors(IRBuilderBase &Builder, ArrayRef< Value * > Vecs)
Concatenate a list of vectors.
bool widenShuffleMaskElts(int Scale, ArrayRef< int > Mask, SmallVectorImpl< int > &ScaledMask)
Try to transform a shuffle mask by replacing elements with the scaled index for an equivalent mask of...
Instruction * propagateMetadata(Instruction *I, ArrayRef< Value * > VL)
Specifically, let Kinds = [MD_tbaa, MD_alias_scope, MD_noalias, MD_fpmath, MD_nontemporal,...
Value * getSplatValue(const Value *V)
Get splat value if the input is a splat vector or return nullptr.
T bit_ceil(T Value)
Returns the smallest integral power of two no smaller than Value if Value is nonzero.
MDNode * intersectAccessGroups(const Instruction *Inst1, const Instruction *Inst2)
Compute the access-group list of access groups that Inst1 and Inst2 are both in.
bool any_of(R &&range, UnaryPredicate P)
Provide wrappers to std::any_of which take ranges instead of having to pass begin/end explicitly.
bool getShuffleDemandedElts(int SrcWidth, ArrayRef< int > Mask, const APInt &DemandedElts, APInt &DemandedLHS, APInt &DemandedRHS, bool AllowUndefElts=false)
Transform a shuffle mask's output demanded element mask into demanded element masks for the 2 operand...
bool isSplatValue(const Value *V, int Index=-1, unsigned Depth=0)
Return true if each element of the vector value V is poisoned or equal to every other non-poisoned el...
Constant * createBitMaskForGaps(IRBuilderBase &Builder, unsigned VF, const InterleaveGroup< Instruction > &Group)
Create a mask that filters the members of an interleave group where there are gaps.
constexpr unsigned MaxAnalysisRecursionDepth
llvm::SmallVector< int, 16 > createStrideMask(unsigned Start, unsigned Stride, unsigned VF)
Create a stride shuffle mask.
llvm::SmallVector< int, 16 > createReplicatedMask(unsigned ReplicationFactor, unsigned VF)
Create a mask with replicated elements.
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
std::optional< int64_t > getPtrStride(PredicatedScalarEvolution &PSE, Type *AccessTy, Value *Ptr, const Loop *Lp, const DenseMap< Value *, const SCEV * > &StridesMap=DenseMap< Value *, const SCEV * >(), bool Assume=false, bool ShouldCheckWrap=true)
If the pointer has a constant stride return it in units of the access type size.
Align getLoadStoreAlignment(Value *I)
A helper function that returns the alignment of load or store instruction.
bool maskIsAllOneOrUndef(Value *Mask)
Given a mask vector of i1, Return true if all of the elements of this predicate mask are known to be ...
constexpr int PoisonMaskElem
bool isValidAsAccessGroup(MDNode *AccGroup)
Return whether an MDNode might represent an access group.
Intrinsic::ID getIntrinsicForCallSite(const CallBase &CB, const TargetLibraryInfo *TLI)
Map a call instruction to an intrinsic ID.
void processShuffleMasks(ArrayRef< int > Mask, unsigned NumOfSrcRegs, unsigned NumOfDestRegs, unsigned NumOfUsedRegs, function_ref< void()> NoInputAction, function_ref< void(ArrayRef< int >, unsigned, unsigned)> SingleInputAction, function_ref< void(ArrayRef< int >, unsigned, unsigned)> ManyInputsAction)
Splits and processes shuffle mask depending on the number of input and output registers.
void narrowShuffleMaskElts(int Scale, ArrayRef< int > Mask, SmallVectorImpl< int > &ScaledMask)
Replace each shuffle mask index with the scaled sequential indices for an equivalent mask of narrowed...
llvm::SmallVector< int, 16 > createInterleaveMask(unsigned VF, unsigned NumVecs)
Create an interleave shuffle mask.
const SCEV * replaceSymbolicStrideSCEV(PredicatedScalarEvolution &PSE, const DenseMap< Value *, const SCEV * > &PtrToStride, Value *Ptr)
Return the SCEV corresponding to a pointer with the symbolic stride replaced with constant one,...
Value * findScalarElement(Value *V, unsigned EltNo)
Given a vector and an element number, see if the scalar value is already around as a register,...
MDNode * uniteAccessGroups(MDNode *AccGroups1, MDNode *AccGroups2)
Compute the union of two access-group lists.
auto count_if(R &&Range, UnaryPredicate P)
Wrapper function around std::count_if to count the number of times an element satisfying a given pred...
bool maskIsAllZeroOrUndef(Value *Mask)
Given a mask vector of i1, Return true if all of the elements of this predicate mask are known to be ...
auto find_if(R &&Range, UnaryPredicate P)
Provide wrappers to std::find_if which take ranges instead of having to pass begin/end explicitly.
void getShuffleMaskWithWidestElts(ArrayRef< int > Mask, SmallVectorImpl< int > &ScaledMask)
Repetitively apply widenShuffleMaskElts() for as long as it succeeds, to get the shuffle mask with wi...
bool isVectorIntrinsicWithScalarOpAtArg(Intrinsic::ID ID, unsigned ScalarOpdIdx)
Identifies if the vector form of the intrinsic has a scalar operand.
bool all_equal(std::initializer_list< T > Values)
Returns true if all Values in the initializer lists are equal or the list.
bool maskContainsAllOneOrUndef(Value *Mask)
Given a mask vector of i1, Return true if any of the elements of this predicate mask are known to be ...
bool isTriviallyVectorizable(Intrinsic::ID ID)
Identify if the intrinsic is trivially vectorizable.
llvm::SmallVector< int, 16 > createSequentialMask(unsigned Start, unsigned NumInts, unsigned NumUndefs)
Create a sequential shuffle mask.
Type * getLoadStoreType(Value *I)
A helper function that returns the type of a load or store instruction.
MapVector< Instruction *, uint64_t > computeMinimumValueSizes(ArrayRef< BasicBlock * > Blocks, DemandedBits &DB, const TargetTransformInfo *TTI=nullptr)
Compute a map of integer instructions to their minimum legal type size.
int getSplatIndex(ArrayRef< int > Mask)
If all non-negative Mask elements are the same value, return that value.
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.