LLVM 24.0.0git
VecUtils.h
Go to the documentation of this file.
1//===- VecUtils.h -----------------------------------------------*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// Collector for SandboxVectorizer related convenience functions that don't
10// belong in other classes.
11
12#ifndef LLVM_TRANSFORMS_VECTORIZE_SANDBOXVECTORIZER_VECUTILS_H
13#define LLVM_TRANSFORMS_VECTORIZE_SANDBOXVECTORIZER_VECUTILS_H
14
15#include "llvm/ADT/DenseSet.h"
17#include "llvm/IR/DataLayout.h"
18#include "llvm/SandboxIR/Type.h"
21#include <iterator>
22
23namespace llvm {
24/// Traits for DenseMap.
25template <> struct DenseMapInfo<SmallVector<sandboxir::Value *>> {
26 static unsigned getHashValue(const SmallVector<sandboxir::Value *> &Vec) {
27 return hash_combine_range(Vec);
28 }
31 return Vec1 == Vec2;
32 }
33};
34
35namespace sandboxir {
36
37/// An ArrayRef of Values or Instructions that we can print/dump for debugging.
38/// It is mainly used for the vectorizer's instr/value bundles.
39template <typename T> class BndlRef : public ArrayRef<T> {
40public:
41 // Inherit constructors.
42 using ArrayRef<T>::ArrayRef;
43
44#ifndef NDEBUG
45 /// Helper dump function for debugging.
46 void print(raw_ostream &OS) const {
47 for (const auto &[Idx, Val] : enumerate(*this))
48 OS << Idx << ". " << *Val << "\n";
49 }
50 LLVM_DUMP_METHOD void dump() const;
51#endif // NDEBUG
52};
53
54/// @name BndlRef Deduction guides
55/// @{
56/// Deduction guide to construct a BndlRef from a single element.
57template <typename T> BndlRef(const T &OneElt) -> BndlRef<T>;
58/// Deduction guide to construct a BndlRef from a pointer and length
59template <typename T> BndlRef(const T *data, size_t length) -> BndlRef<T>;
60/// Deduction guide to construct a BndlRef from a range
61template <typename T> BndlRef(const T *data, const T *end) -> BndlRef<T>;
62/// Deduction guide to construct a BndlRef from a SmallVector
63template <typename T> BndlRef(const SmallVectorImpl<T> &Vec) -> BndlRef<T>;
64/// Deduction guide to construct a BndlRef from a SmallVector
65template <typename T, unsigned N>
67/// Deduction guide to construct a BndlRef from a std::vector
68template <typename T> BndlRef(const std::vector<T> &Vec) -> BndlRef<T>;
69/// Deduction guide to construct a BndlRef from a std::array
70template <typename T, std::size_t N>
71BndlRef(const std::array<T, N> &Vec) -> BndlRef<T>;
72/// Deduction guide to construct a BndlRef from an BndlRef (const)
73template <typename T> BndlRef(const BndlRef<T> &Vec) -> BndlRef<T>;
74/// Deduction guide to construct a BndlRef from an BndlRef
75template <typename T> BndlRef(BndlRef<T> &Vec) -> BndlRef<T>;
76/// Deduction guide to construct a BndlRef from a C array.
77template <typename T, size_t N> BndlRef(const T (&Arr)[N]) -> BndlRef<T>;
78/// @}
79
80class InstrMaps;
81
83
84class VecUtils {
85public:
86 /// \Returns the number of elements in \p Ty. That is the number of lanes if a
87 /// fixed vector or 1 if scalar. ScalableVectors have unknown size and
88 /// therefore are unsupported.
89 static int getNumElements(Type *Ty) {
91 return Ty->isVectorTy() ? cast<FixedVectorType>(Ty)->getNumElements() : 1;
92 }
93 /// Returns \p Ty if scalar or its element type if vector.
94 static Type *getElementType(Type *Ty) {
95 return Ty->isVectorTy() ? cast<FixedVectorType>(Ty)->getElementType() : Ty;
96 }
97
98 /// \Returns true if \p I1 and \p I2 are load/stores accessing consecutive
99 /// memory addresses.
100 template <typename LoadOrStoreT>
101 static bool areConsecutive(LoadOrStoreT *I1, LoadOrStoreT *I2,
102 ScalarEvolution &SE, const DataLayout &DL) {
103 static_assert(std::is_same<LoadOrStoreT, LoadInst>::value ||
104 std::is_same<LoadOrStoreT, StoreInst>::value,
105 "Expected Load or Store!");
106 auto Diff = Utils::getPointerDiffInBytes(I1, I2, SE);
107 if (!Diff)
108 return false;
109 int ElmBytes = Utils::getNumBits(I1) / 8;
110 return *Diff == ElmBytes;
111 }
112
113 template <typename LoadOrStoreT, typename ValT>
115 const DataLayout &DL) {
116 static_assert(std::is_same<LoadOrStoreT, LoadInst>::value ||
117 std::is_same<LoadOrStoreT, StoreInst>::value,
118 "Expected Load or Store!");
119 assert(isa<LoadOrStoreT>(Bndl[0]) && "Expected Load or Store!");
120 auto *LastLS = cast<LoadOrStoreT>(Bndl[0]);
121 for (Value *V : drop_begin(Bndl)) {
123 "Unimplemented: we only support StoreInst!");
124 auto *LS = cast<LoadOrStoreT>(V);
125 if (!VecUtils::areConsecutive(LastLS, LS, SE, DL))
126 return false;
127 LastLS = LS;
128 }
129 return true;
130 }
131
132 /// \Returns the number of vector lanes of \p Ty or 1 if not a vector.
133 /// NOTE: It asserts that \p Ty is a fixed vector type.
134 static unsigned getNumLanes(Type *Ty) {
135 assert(!isa<ScalableVectorType>(Ty) && "Expect scalar or fixed vector");
136 if (auto *FixedVecTy = dyn_cast<FixedVectorType>(Ty))
137 return FixedVecTy->getNumElements();
138 return 1u;
139 }
140
141 /// \Returns the expected vector lanes of \p V or 1 if not a vector.
142 /// NOTE: It asserts that \p V is a fixed vector.
143 static unsigned getNumLanes(Value *V) {
145 }
146
147 /// \Returns the total number of lanes across all values in \p Bndl.
148 static unsigned getNumLanes(ArrayRef<Value *> Bndl) {
149 unsigned Lanes = 0;
150 for (Value *V : Bndl)
151 Lanes += getNumLanes(V);
152 return Lanes;
153 }
154
155 /// \Returns <NumElts x ElemTy>.
156 /// It works for both scalar and vector \p ElemTy.
157 static Type *getWideType(Type *ElemTy, unsigned NumElts) {
158 if (ElemTy->isVectorTy()) {
159 auto *VecTy = cast<FixedVectorType>(ElemTy);
160 ElemTy = VecTy->getElementType();
161 NumElts = VecTy->getNumElements() * NumElts;
162 }
163 return FixedVectorType::get(ElemTy, NumElts);
164 }
165 /// Classifies whether all bundle elements are integers, halves, floats,
166 /// doubles, or pointers.
168 unsigned NumIntElements = 0;
169 unsigned NumHalfElements = 0;
170 unsigned NumFloatElements = 0;
171 unsigned NumDoubleElements = 0;
172 unsigned NumPtrElements = 0;
173 unsigned NumElements = 0;
174
175 public:
176 template <typename T>
177 BndlTypeClassifier(BndlRef<T *> Bndl) : NumElements(Bndl.size()) {
178 for (T *V : Bndl) {
180 if (ElmTy->isIntegerTy())
181 ++NumIntElements;
182 else if (ElmTy->isHalfTy())
183 ++NumHalfElements;
184 else if (ElmTy->isFloatTy())
185 ++NumFloatElements;
186 else if (ElmTy->isDoubleTy())
187 ++NumDoubleElements;
188 else if (ElmTy->isPointerTy())
189 ++NumPtrElements;
190 }
191 }
192
193 bool areAllIntegers() const { return NumIntElements == NumElements; }
194 bool areAllHalves() const { return NumHalfElements == NumElements; }
195 bool areAllFloats() const { return NumFloatElements == NumElements; }
196 bool areAllDoubles() const { return NumDoubleElements == NumElements; }
197 bool areAllPointers() const { return NumPtrElements == NumElements; }
198 };
199
200 /// \Returns the combined vector type for \p Bndl, even when the element types
201 /// differ. For example: i8,i8,i16 will return <4 x i8>. Homogeneous half,
202 /// float, and double bundles keep their element type; other mixed-type
203 /// bundles are combined into an integer vector.
204 template <typename T>
206 const DataLayout &DL) {
207 assert(!Bndl.empty() && "Expected non-empty Bndl!");
208 unsigned TotalBits = 0;
209 unsigned GCDBits = 0;
210 BndlTypeClassifier Classifier(Bndl);
211 for (auto *V : Bndl) {
213 unsigned ElmBits = Utils::getNumBits(ElmTy, DL);
214 TotalBits += ElmBits * getNumLanes(V);
215 GCDBits = std::gcd(GCDBits, ElmBits);
216 }
217
218 auto &Ctx = Bndl[0]->getContext();
219 // Keep a half, float, or double lane type when every element has that type.
220 // Otherwise use an integer lane type, so an integer bundle such as
221 // {i32, i16, i16} is not rewritten as a float vector such as <4 x half>.
222 Type *VecElmTy = nullptr;
223 if (Classifier.areAllHalves())
224 VecElmTy = Type::getHalfTy(Ctx);
225 else if (Classifier.areAllFloats())
226 VecElmTy = Type::getFloatTy(Ctx);
227 else if (Classifier.areAllDoubles())
228 VecElmTy = Type::getDoubleTy(Ctx);
229 else
230 VecElmTy = IntegerType::get(Ctx, GCDBits);
231 return FixedVectorType::get(VecElmTy, TotalBits / GCDBits);
232 }
233
234 static Type *getCombinedVectorTypeFor(std::initializer_list<Value *> Bndl,
235 const DataLayout &DL) {
237 }
238 /// \Returns the instruction in \p Instrs that is lowest in the BB. Expects
239 /// that all instructions are in the same BB.
241 Instruction *LowestI = Instrs.front();
242 for (auto *I : drop_begin(Instrs)) {
243 if (LowestI->comesBefore(I))
244 LowestI = I;
245 }
246 return LowestI;
247 }
248 /// \Returns the instruction in \p Instrs that is highest in the BB. Expects
249 /// that all instructions are in the same BB.
251 Instruction *HighestI = Instrs.front();
252 for (auto *I : drop_begin(Instrs)) {
253 if (I->comesBefore(HighestI))
254 HighestI = I;
255 }
256 return HighestI;
257 }
258 /// \Returns the lowest instruction in \p Vals, or nullptr if no instructions
259 /// are found. Skips instructions not in \p BB.
261 // Find the first Instruction in Vals that is also in `BB`.
262 auto It = find_if(Vals, [BB](Value *V) {
263 return isa<Instruction>(V) && cast<Instruction>(V)->getParent() == BB;
264 });
265 // If we couldn't find an instruction return nullptr.
266 if (It == Vals.end())
267 return nullptr;
268 Instruction *FirstI = cast<Instruction>(*It);
269 // Now look for the lowest instruction in Vals starting from one position
270 // after FirstI.
271 Instruction *LowestI = FirstI;
272 for (auto *V : make_range(std::next(It), Vals.end())) {
273 auto *I = dyn_cast<Instruction>(V);
274 // Skip non-instructions.
275 if (I == nullptr)
276 continue;
277 // Skips instructions not in \p BB.
278 if (I->getParent() != BB)
279 continue;
280 // If `LowestI` comes before `I` then `I` is the new lowest.
281 if (LowestI->comesBefore(I))
282 LowestI = I;
283 }
284 return LowestI;
285 }
286
287 /// If \p I is not a PHI it returns it. Else it walks down the instruction
288 /// chain looking for the last PHI and returns it. \Returns nullptr if \p I is
289 /// nullptr.
291 Instruction *LastI = I;
292 while (I != nullptr && isa<PHINode>(I)) {
293 LastI = I;
294 I = I->getNextNode();
295 }
296 return LastI;
297 }
298
299 /// \Returns the BB iterator after the lowest instruction in \p Vals
300 /// (skipping instructions not in \p BB), or the top of BB if no
301 /// instruction found in \p Vals.
303 BasicBlock *BB) {
304 auto *BotI = getLastPHIOrSelf(getLowest(Vals, BB));
305 if (BotI == nullptr)
306 // We are using BB->begin() (or after PHIs) as the fallback insert point.
307 return BB->empty()
308 ? BB->begin()
309 : std::next(getLastPHIOrSelf(&*BB->begin())->getIterator());
310 return std::next(BotI->getIterator());
311 }
312
313 /// If all values in \p Bndl are of the same scalar type then return it,
314 /// otherwise return nullptr.
316 Value *V0 = Bndl[0];
317 Type *Ty0 = Utils::getExpectedType(V0);
318 Type *ScalarTy = VecUtils::getElementType(Ty0);
319 for (auto *V : drop_begin(Bndl)) {
321 Type *NScalarTy = VecUtils::getElementType(NTy);
322 if (NScalarTy != ScalarTy)
323 return nullptr;
324 }
325 return ScalarTy;
326 }
327
328 /// Similar to tryGetCommonScalarType() but will assert that there is a common
329 /// type. So this is faster in release builds as it won't iterate through the
330 /// values.
332 Value *V0 = Bndl[0];
333 Type *Ty0 = Utils::getExpectedType(V0);
334 Type *ScalarTy = VecUtils::getElementType(Ty0);
335 assert(tryGetCommonScalarType(Bndl) && "Expected common scalar type!");
336 return ScalarTy;
337 }
338 /// \Returns the first integer power of 2 that is <= Num.
339 LLVM_ABI static unsigned getFloorPowerOf2(unsigned Num);
340
341 /// For each user of lane 0 in \p Bndl, try to form a bundle of matching
342 /// users for all lanes. Returns all complete user bundles found.
343 /// \p Claimed contains instructions that have already been claimed by a
344 /// bundle.
348
349 /// Helper struct for `matchPack()`. Describes the instructions and operands
350 /// of a pack pattern.
351 struct PackPattern {
352 /// The insertelement instructions that form the pack pattern in bottom-up
353 /// order, i.e., the first instruction in `Instrs` is the bottom-most
354 /// InsertElement instruction of the pack pattern.
355 /// For example in this simple pack pattern:
356 /// %Pack0 = insertelement <2 x i8> poison, i8 %v0, i64 0
357 /// %Pack1 = insertelement <2 x i8> %Pack0, i8 %v1, i64 1
358 /// this is [ %Pack1, %Pack0 ].
360 /// The "external" operands of the pack pattern, i.e., the values that get
361 /// packed into a vector, skipping the ones in `Instrs`. The operands are in
362 /// bottom-up order, starting from the operands of the bottom-most insert.
363 /// So in our example this would be [ %v1, %v0 ].
365 };
366
367 /// If \p I is the last instruction of a pack pattern (i.e., an InsertElement
368 /// into a vector), then this function returns the instructions in the pack
369 /// and the operands in the pack, else returns nullopt.
370 /// Here is an example of a matched pattern:
371 /// %PackA0 = insertelement <2 x i8> poison, i8 %v0, i64 0
372 /// %PackA1 = insertelement <2 x i8> %PackA0, i8 %v1, i64 1
373 /// TODO: this currently detects only simple canonicalized patterns.
374 static std::optional<PackPattern> matchPack(Instruction *I) {
375 // TODO: Support vector pack patterns.
376 // TODO: Support out-of-order inserts.
377
378 // Early return if `I` is not an Insert.
380 return std::nullopt;
381 auto *BB0 = I->getParent();
382 // The pack contains as many instrs as the lanes of the bottom-most Insert
383 unsigned ExpectedNumInserts = VecUtils::getNumLanes(I);
384 assert(ExpectedNumInserts >= 2 && "Expected at least 2 inserts!");
386 Pack.Operands.resize(ExpectedNumInserts);
387 // Collect the inserts by walking up the use-def chain.
388 Instruction *InsertI = I;
389 for (auto ExpectedLane : reverse(seq<unsigned>(ExpectedNumInserts))) {
390 if (InsertI == nullptr)
391 return std::nullopt;
392 if (InsertI->getParent() != BB0)
393 return std::nullopt;
394 // Check the lane.
395 auto *LaneC = dyn_cast<ConstantInt>(InsertI->getOperand(2));
396 if (LaneC == nullptr || LaneC->getSExtValue() != ExpectedLane)
397 return std::nullopt;
398 Pack.Instrs.push_back(InsertI);
399 Pack.Operands[ExpectedLane] = InsertI->getOperand(1);
400
401 Value *Op = InsertI->getOperand(0);
402 if (ExpectedLane == 0) {
403 // Check the topmost insert. The operand should be a Poison.
404 if (!isa<PoisonValue>(Op))
405 return std::nullopt;
406 } else {
408 }
409 }
410 return Pack;
411 }
412
413 /// Emits the necessary instruction sequence to extract element of type \p
414 /// ExtrTy at \p Lane from \p FromVec. Emits instructions before \p WhereIt.
415 /// Returns the extracted value.
416 /// Note: This handles both vectors and scalars. In the vector case it
417 /// extracts an N-wide element (with N dictated by \p ExtrTy).
418 static Value *unpack(Value *FromVec, Type *ExtrTy, unsigned Lane,
419 BasicBlock::iterator WhereIt) {
420 assert(isa<FixedVectorType>(FromVec->getType()) && "Expected vector!");
421 auto &Ctx = FromVec->getContext();
422 if (!ExtrTy->isVectorTy()) {
423 // For scalar elements we emit a single ExtractElementInst.
424 assert(Lane <
425 cast<FixedVectorType>(FromVec->getType())->getNumElements() &&
426 "Out of bounds!");
427 assert(ExtrTy ==
428 cast<FixedVectorType>(FromVec->getType())->getElementType() &&
429 "Expected same element type!");
430 Constant *ExtractLaneC =
432 // Note: This may be folded into a Constant if FromVec is a Constant.
433 return ExtractElementInst::create(FromVec, ExtractLaneC, WhereIt, Ctx,
434 "Unpack");
435 }
436 // For vector elements we emit a shuffle.
437 // For example, extracting lanes 2 and 3 of a <4 x i32> vector %vec:
438 // shufflevector <4 x i32> %vec, <4 x i32> poison, <2 x i32> <i32 2, i32 3>
439 auto *VecTy = cast<FixedVectorType>(FromVec->getType());
440 auto *ExtrVecTy = cast<FixedVectorType>(ExtrTy);
441 assert(ExtrVecTy->getElementType() == VecTy->getElementType() &&
442 "Expected same element type!");
444 for (unsigned Idx = 0, E = ExtrVecTy->getNumElements(); Idx != E; ++Idx) {
445 int MaskLane = Lane + Idx;
446 assert((unsigned)MaskLane <
447 cast<FixedVectorType>(FromVec->getType())->getNumElements() &&
448 "Out of bounds!");
449 Mask.push_back(MaskLane);
450 }
451 return ShuffleVectorInst::create(FromVec, PoisonValue::get(VecTy), Mask,
452 WhereIt, Ctx, "Unpack");
453 }
454
455 /// Iterate over all lanes and Value pairs.
456 // For example, given a range: {i32 %v0, <2 x i32> %v1, i32 %v2} we get:
457 // Lane Elm
458 // 0 %v0
459 // 1 %v1
460 // 3 %v2
461 template <typename RangeIteratorT> class LaneValueEnumerator {
462 /// Points to current element.
463 RangeIteratorT It;
464 RangeIteratorT ItE;
465 /// Accumulator of lanes.
466 unsigned Lane;
467
468 public:
469 // Note that We can start counting from a non-zero BeginLane, though the
470 // user must make sure it corresponds to the correct lane matching Begin.
471 LaneValueEnumerator(RangeIteratorT Begin, RangeIteratorT End,
472 unsigned BeginLane)
473 : It(Begin), ItE(End), Lane(BeginLane) {}
474 using iterator_catecotry = std::input_iterator_tag;
475 // NOTE: dereference returns by value instead of by reference.
476 using value_type = std::pair<unsigned, Value *>;
477 using difference_type = std::ptrdiff_t;
478 using pointer = std::pair<unsigned, Value *> *;
479 using reference = std::pair<unsigned, Value *> &;
481 assert(It != ItE && "Already at end!");
482 auto *Ty = Utils::getExpectedType(*It);
483 if (auto *VecTy = dyn_cast<FixedVectorType>(Ty)) {
484 Lane += VecTy->getNumElements();
485 } else {
486 assert(!isa<VectorType>(Ty) && "Expected scalar type!");
487 Lane += 1;
488 }
489 ++It;
490 return *this;
491 }
492 value_type operator*() const { return {Lane, *It}; }
494 return It == Other.It;
495 }
497 return !(*this == Other);
498 }
499 };
500
501 /// Utility class to collect and erase dead instructions.
503 public:
506
507 /// Record instructions in \p Bndl that may be dead after vectorization.
508 /// For load/store bundles, also record non-first-lane pointer operands;
509 /// the first lane's pointer is skipped because the vector load/store
510 /// reuses it. Erased later by \c tryEraseDeadInstrs().
511 template <typename T> void collectPotentiallyDeadInstrs(BndlRef<T *> Bndl);
512
513 /// Erase candidates recorded by \c collectPotentiallyDeadInstrs() that
514 /// now have no uses, then clear the candidate set.
516
517#ifndef NDEBUG
518 void print(raw_ostream &OS) const {
519 OS << "DeadInstrCandidates:\n";
520 for (auto *I : DeadInstrCandidates)
521 OS << *I << '\n';
522 }
523 LLVM_DUMP_METHOD void debug() const {
524 print(dbgs());
525 dbgs() << '\n';
526 }
527#endif /* NDEBUG */
528
529 private:
530 DenseSet<Instruction *> DeadInstrCandidates;
531 };
532
533 /// Helper for creating LaneValueEnumerator ranges. Can be used in for loops
534 /// like: `for (auto [Lane, V] : enumerateLanes(Range))`
535 template <typename ValueContainerT>
536 static auto enumerateLanes(const ValueContainerT &Range) {
537 auto Begin = LaneValueEnumerator<decltype(Range.begin())>(Range.begin(),
538 Range.end(), 0);
539 auto End = LaneValueEnumerator<decltype(Range.begin())>(Range.end(),
540 Range.end(), 0);
541 return make_range(Begin, End);
542 }
543
544 /// Given the alignments of the instructions before vectorization, this
545 /// returns the alignment of the vectorized instruction.
546 template <typename AlignRangeT, typename TypeRangeT>
547 static Align getVecAlignment(AlignRangeT AlignRange, TypeRangeT TypeRange) {
548 assert(AlignRange.begin() != AlignRange.end() && "Expected non empty!");
549 assert(TypeRange.begin() != TypeRange.end() && "Expected non empty!");
550 assert(AlignRange.end() - AlignRange.begin() ==
551 TypeRange.end() - TypeRange.begin() &&
552 "Expected same size!");
553 // TODO: Implement smarter aligment deduction based on types.
554 // For now we return the alignment of the first element.
555 return *AlignRange.begin();
556 }
557
558 /// Helper for getVecAlignment(AlignRange, TypeRange).
559 template <typename InstrRangeT>
560 static Align getVecAlignment(InstrRangeT InstrRange) {
561 auto AlignRange = map_range(
562 InstrRange, [](auto *I) -> Align { return getLoadStoreAlignment(I); });
563 auto TypeRange = map_range(InstrRange, [](auto *I) -> Type * {
565 });
566 return getVecAlignment(AlignRange, TypeRange);
567 }
568
569#ifndef NDEBUG
570 /// Helper dump function for debugging.
571 LLVM_DUMP_METHOD static void dump(ArrayRef<Value *> Bndl);
573#endif // NDEBUG
574};
575
576extern template LLVM_TEMPLATE_ABI void
579
580} // namespace sandboxir
581
582} // namespace llvm
583
584#endif // LLVM_TRANSFORMS_VECTORIZE_SANDBOXVECTORIZER_VECUTILS_H
assert(UImm &&(UImm !=~static_cast< T >(0)) &&"Invalid immediate!")
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
#define LLVM_ABI
Definition Compiler.h:211
#define LLVM_TEMPLATE_ABI
Definition Compiler.h:212
#define LLVM_DUMP_METHOD
Mark debug helper function definitions like dump() that should not be stripped from debug builds.
Definition Compiler.h:682
This file defines the DenseSet and SmallDenseSet classes.
#define I(x, y, z)
Definition MD5.cpp:57
#define T
ConstantRange Range(APInt(BitWidth, Low), APInt(BitWidth, High))
static Split data
Represent a constant reference to an array (0 or more elements consecutively in memory),...
Definition ArrayRef.h:40
const T & front() const
Get the first element.
Definition ArrayRef.h:144
iterator end() const
Definition ArrayRef.h:130
ArrayRef()=default
Construct an empty ArrayRef.
bool empty() const
Check if the array is empty.
Definition ArrayRef.h:136
iterator begin()
Instruction iterator methods.
Definition BasicBlock.h:440
bool empty() const
Definition BasicBlock.h:462
InstListType::iterator iterator
Instruction iterators...
Definition BasicBlock.h:170
A parsed version of the target data layout string in and methods for querying it.
Definition DataLayout.h:64
Implements a dense probed hash-table based set.
Definition DenseSet.h:281
The main scalar evolution driver.
SmallPtrSet - This class implements a set which is optimized for holding SmallSize or less elements.
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
LLVM Value Representation.
Definition Value.h:75
This class implements an extremely fast bulk output stream that can only output to a stream.
Definition raw_ostream.h:53
An ArrayRef of Values or Instructions that we can print/dump for debugging.
Definition VecUtils.h:39
void print(raw_ostream &OS) const
Helper dump function for debugging.
Definition VecUtils.h:46
LLVM_DUMP_METHOD void dump() const
Definition VecUtils.cpp:169
static LLVM_ABI ConstantInt * getSigned(IntegerType *Ty, int64_t V)
Return a ConstantInt with the specified value for the specified type.
Definition Constant.cpp:56
static LLVM_ABI Value * create(Value *Vec, Value *Idx, InsertPosition Pos, Context &Ctx, const Twine &Name="")
static LLVM_ABI FixedVectorType * get(Type *ElementType, unsigned NumElts)
Maps the original instructions to the vectorized instrs and the reverse.
Definition InstrMaps.h:50
A sandboxir::User with operands, opcode and linked with previous/next instructions in an instruction ...
Definition Instruction.h:43
LLVM_ABI BBIterator getIterator() const
\Returns a BasicBlock::iterator for this Instruction.
bool comesBefore(const Instruction *Other) const
Given an instruction Other in the same basic block as this instruction, return true if this instructi...
LLVM_ABI BasicBlock * getParent() const
\Returns the BasicBlock containing this Instruction, or null if it is detached.
static LLVM_ABI IntegerType * get(Context &C, unsigned NumBits)
static LLVM_ABI PoisonValue * get(Type *T)
Static factory methods - Return an 'poison' object of the specified type.
Definition Constant.cpp:263
static LLVM_ABI Value * create(Value *V1, Value *V2, Value *Mask, InsertPosition Pos, Context &Ctx, const Twine &Name="")
Just like llvm::Type these are immutable, unique, never get freed and can only be created via static ...
Definition Type.h:49
static LLVM_ABI Type * getHalfTy(Context &Ctx)
Definition Type.cpp:66
static LLVM_ABI Type * getDoubleTy(Context &Ctx)
Definition Type.cpp:60
bool isIntegerTy() const
True if this is an instance of IntegerType.
Definition Type.h:173
bool isHalfTy() const
Return true if this is 'half', a 16-bit IEEE fp type.
Definition Type.h:107
bool isDoubleTy() const
Return true if this is 'double', a 64-bit IEEE fp type.
Definition Type.h:119
static LLVM_ABI IntegerType * getInt32Ty(Context &Ctx)
Definition Type.cpp:21
bool isVectorTy() const
True if this is an instance of VectorType.
Definition Type.h:214
bool isPointerTy() const
True if this is an instance of PointerType.
Definition Type.h:208
bool isFloatTy() const
Return true if this is 'float', a 32-bit IEEE fp type.
Definition Type.h:116
static LLVM_ABI Type * getFloatTy(Context &Ctx)
Definition Type.cpp:63
Value * getOperand(unsigned OpIdx) const
Definition User.h:123
static std::optional< int > getPointerDiffInBytes(LoadOrStoreT *I0, LoadOrStoreT *I1, ScalarEvolution &SE)
\Returns the gap between the memory locations accessed by I0 and I1 in bytes.
Definition Utils.h:93
static unsigned getNumBits(Type *Ty, const DataLayout &DL)
\Returns the number of bits of Ty.
Definition Utils.h:67
static Type * getExpectedType(const Value *V)
\Returns the expected type of Value V.
Definition Utils.h:33
A SandboxIR Value has users. This is the base class.
Definition Value.h:72
LLVM_ABI Type * getType() const
Definition Value.cpp:46
Context & getContext() const
Definition Value.h:285
Classifies whether all bundle elements are integers, halves, floats, doubles, or pointers.
Definition VecUtils.h:167
DeadInstructionMorgue(const DeadInstructionMorgue &)=delete
LLVM_DUMP_METHOD void debug() const
Definition VecUtils.h:523
LLVM_ABI void tryEraseDeadInstrs()
Erase candidates recorded by collectPotentiallyDeadInstrs() that now have no uses,...
Definition VecUtils.cpp:139
void collectPotentiallyDeadInstrs(BndlRef< T * > Bndl)
Record instructions in Bndl that may be dead after vectorization.
Definition VecUtils.cpp:104
Iterate over all lanes and Value pairs.
Definition VecUtils.h:461
bool operator==(const LaneValueEnumerator &Other) const
Definition VecUtils.h:493
bool operator!=(const LaneValueEnumerator &Other) const
Definition VecUtils.h:496
std::pair< unsigned, Value * > value_type
Definition VecUtils.h:476
std::pair< unsigned, Value * > & reference
Definition VecUtils.h:479
LaneValueEnumerator(RangeIteratorT Begin, RangeIteratorT End, unsigned BeginLane)
Definition VecUtils.h:471
std::pair< unsigned, Value * > * pointer
Definition VecUtils.h:478
static Type * tryGetCommonScalarType(ArrayRef< Value * > Bndl)
If all values in Bndl are of the same scalar type then return it, otherwise return nullptr.
Definition VecUtils.h:315
static Instruction * getLowest(ArrayRef< Instruction * > Instrs)
\Returns the instruction in Instrs that is lowest in the BB.
Definition VecUtils.h:240
static Type * getCommonScalarType(ArrayRef< Value * > Bndl)
Similar to tryGetCommonScalarType() but will assert that there is a common type.
Definition VecUtils.h:331
static int getNumElements(Type *Ty)
\Returns the number of elements in Ty.
Definition VecUtils.h:89
static std::optional< PackPattern > matchPack(Instruction *I)
If I is the last instruction of a pack pattern (i.e., an InsertElement into a vector),...
Definition VecUtils.h:374
static Instruction * getLastPHIOrSelf(Instruction *I)
If I is not a PHI it returns it.
Definition VecUtils.h:290
static unsigned getNumLanes(Type *Ty)
\Returns the number of vector lanes of Ty or 1 if not a vector.
Definition VecUtils.h:134
static Instruction * getLowest(ArrayRef< Value * > Vals, BasicBlock *BB)
\Returns the lowest instruction in Vals, or nullptr if no instructions are found.
Definition VecUtils.h:260
static Value * unpack(Value *FromVec, Type *ExtrTy, unsigned Lane, BasicBlock::iterator WhereIt)
Emits the necessary instruction sequence to extract element of type ExtrTy at Lane from FromVec.
Definition VecUtils.h:418
static LLVM_DUMP_METHOD void dump(ArrayRef< Value * > Bndl)
Helper dump function for debugging.
Definition VecUtils.cpp:166
static Type * getWideType(Type *ElemTy, unsigned NumElts)
\Returns <NumElts x ElemTy>.
Definition VecUtils.h:157
static Instruction * getHighest(ArrayRef< Instruction * > Instrs)
\Returns the instruction in Instrs that is highest in the BB.
Definition VecUtils.h:250
static auto enumerateLanes(const ValueContainerT &Range)
Helper for creating LaneValueEnumerator ranges.
Definition VecUtils.h:536
static bool areConsecutive(LoadOrStoreT *I1, LoadOrStoreT *I2, ScalarEvolution &SE, const DataLayout &DL)
\Returns true if I1 and I2 are load/stores accessing consecutive memory addresses.
Definition VecUtils.h:101
static Type * getCombinedVectorTypeFor(std::initializer_list< Value * > Bndl, const DataLayout &DL)
Definition VecUtils.h:234
static bool areConsecutive(ArrayRef< ValT * > Bndl, ScalarEvolution &SE, const DataLayout &DL)
Definition VecUtils.h:114
static Type * getElementType(Type *Ty)
Returns Ty if scalar or its element type if vector.
Definition VecUtils.h:94
static unsigned getNumLanes(Value *V)
\Returns the expected vector lanes of V or 1 if not a vector.
Definition VecUtils.h:143
static Align getVecAlignment(AlignRangeT AlignRange, TypeRangeT TypeRange)
Given the alignments of the instructions before vectorization, this returns the alignment of the vect...
Definition VecUtils.h:547
static Align getVecAlignment(InstrRangeT InstrRange)
Helper for getVecAlignment(AlignRange, TypeRange).
Definition VecUtils.h:560
static Type * getCombinedVectorTypeFor(BndlRef< T * > Bndl, const DataLayout &DL)
\Returns the combined vector type for Bndl, even when the element types differ.
Definition VecUtils.h:205
static unsigned getNumLanes(ArrayRef< Value * > Bndl)
\Returns the total number of lanes across all values in Bndl.
Definition VecUtils.h:148
static BasicBlock::iterator getInsertPointAfterInstrs(ArrayRef< Value * > Vals, BasicBlock *BB)
\Returns the BB iterator after the lowest instruction in Vals (skipping instructions not in BB),...
Definition VecUtils.h:302
static LLVM_ABI unsigned getFloorPowerOf2(unsigned Num)
\Returns the first integer power of 2 that is <= Num.
Definition VecUtils.cpp:93
static LLVM_ABI SmallVector< BundleTy > getNextUserBundles(ArrayRef< Value * > Bndl, const InstrMaps &IMaps, SmallPtrSet< Instruction *, 4 > &Claimed)
For each user of lane 0 in Bndl, try to form a bundle of matching users for all lanes.
Definition VecUtils.cpp:64
BndlRef(const T &OneElt) -> BndlRef< T >
BasicBlock(llvm::BasicBlock *BB, Context &SBCtx)
Definition BasicBlock.h:75
iterator end() const
Definition BasicBlock.h:89
SmallVector< Value *, 4 > BundleTy
Definition VecUtils.h:82
Align getLoadStoreAlignment(const Value *I)
A helper function that returns the alignment of load or store instruction.
This is an optimization pass for GlobalISel generic memory operations.
auto drop_begin(T &&RangeOrContainer, size_t N=1)
Return a range covering RangeOrContainer with the first N elements excluded.
Definition STLExtras.h:316
auto size(R &&Range, std::enable_if_t< std::is_base_of< std::random_access_iterator_tag, typename std::iterator_traits< decltype(Range.begin())>::iterator_category >::value, void > *=nullptr)
Get the size of a range.
Definition STLExtras.h:1685
auto enumerate(FirstRange &&First, RestRanges &&...Rest)
Given two or more input ranges, returns a new range whose values are tuples (A, B,...
Definition STLExtras.h:2570
decltype(auto) dyn_cast(const From &Val)
dyn_cast<X> - Return the argument parameter cast to the specified type.
Definition Casting.h:643
iterator_range< T > make_range(T x, T y)
Convenience function for iterating over sub-ranges.
auto map_range(ContainerTy &&C, FuncTy F)
Return a range that applies F to the elements of C.
Definition STLExtras.h:366
auto reverse(ContainerTy &&C)
Definition STLExtras.h:408
LLVM_ABI raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
Definition Debug.cpp:209
bool isa(const From &Val)
isa<X> - Return true if the parameter to the template is an instance of one of the template type argu...
Definition Casting.h:547
@ Other
Any other memory.
Definition ModRef.h:68
DWARFExpression::Operation Op
decltype(auto) cast(const From &Val)
cast<X> - Return the argument parameter cast to the specified type.
Definition Casting.h:559
auto find_if(R &&Range, UnaryPredicate P)
Provide wrappers to std::find_if which take ranges instead of having to pass begin/end explicitly.
Definition STLExtras.h:1788
constexpr auto seq(T Begin, T End)
Iterate over an integral type from Begin up to - but not including - End.
Definition Sequence.h:341
@ End
Run the machine verifier once, at the end of the pipeline.
Definition CodeGen.h:267
hash_code hash_combine_range(InputIteratorT first, InputIteratorT last)
Compute a hash_code for a sequence of values.
Definition Hashing.h:287
#define N
This struct is a compact representation of a valid (non-zero power of two) alignment.
Definition Alignment.h:39
static bool isEqual(const SmallVector< sandboxir::Value * > &Vec1, const SmallVector< sandboxir::Value * > &Vec2)
Definition VecUtils.h:29
static unsigned getHashValue(const SmallVector< sandboxir::Value * > &Vec)
Definition VecUtils.h:26
An information struct used to provide DenseMap with the various necessary components for a given valu...
Helper struct for matchPack().
Definition VecUtils.h:351
SmallVector< Value * > Operands
The "external" operands of the pack pattern, i.e., the values that get packed into a vector,...
Definition VecUtils.h:364
SmallVector< Instruction * > Instrs
The insertelement instructions that form the pack pattern in bottom-up order, i.e....
Definition VecUtils.h:359