12#ifndef LLVM_TRANSFORMS_VECTORIZE_SANDBOXVECTORIZER_VECUTILS_H
13#define LLVM_TRANSFORMS_VECTORIZE_SANDBOXVECTORIZER_VECUTILS_H
47 for (
const auto &[Idx, Val] :
enumerate(*
this))
48 OS << Idx <<
". " << *Val <<
"\n";
65template <
typename T,
unsigned N>
70template <
typename T, std::
size_t N>
100 template <
typename LoadOrStoreT>
103 static_assert(std::is_same<LoadOrStoreT, LoadInst>::value ||
104 std::is_same<LoadOrStoreT, StoreInst>::value,
105 "Expected Load or Store!");
110 return *Diff == ElmBytes;
113 template <
typename LoadOrStoreT,
typename ValT>
116 static_assert(std::is_same<LoadOrStoreT, LoadInst>::value ||
117 std::is_same<LoadOrStoreT, StoreInst>::value,
118 "Expected Load or Store!");
123 "Unimplemented: we only support StoreInst!");
137 return FixedVecTy->getNumElements();
150 for (
Value *V : Bndl)
158 if (
ElemTy->isVectorTy()) {
160 ElemTy = VecTy->getElementType();
161 NumElts = VecTy->getNumElements() * NumElts;
168 unsigned NumIntElements = 0;
169 unsigned NumHalfElements = 0;
170 unsigned NumFloatElements = 0;
171 unsigned NumDoubleElements = 0;
172 unsigned NumPtrElements = 0;
173 unsigned NumElements = 0;
176 template <
typename T>
204 template <
typename T>
207 assert(!Bndl.
empty() &&
"Expected non-empty Bndl!");
208 unsigned TotalBits = 0;
209 unsigned GCDBits = 0;
211 for (
auto *V : Bndl) {
215 GCDBits = std::gcd(GCDBits, ElmBits);
218 auto &Ctx = Bndl[0]->getContext();
222 Type *VecElmTy =
nullptr;
253 if (
I->comesBefore(HighestI))
266 if (It == Vals.
end())
278 if (
I->getParent() != BB)
294 I =
I->getNextNode();
310 return std::next(BotI->getIterator());
322 if (NScalarTy != ScalarTy)
381 auto *BB0 =
I->getParent();
384 assert(ExpectedNumInserts >= 2 &&
"Expected at least 2 inserts!");
386 Pack.Operands.resize(ExpectedNumInserts);
390 if (InsertI ==
nullptr)
396 if (LaneC ==
nullptr || LaneC->getSExtValue() != ExpectedLane)
398 Pack.Instrs.push_back(InsertI);
402 if (ExpectedLane == 0) {
429 "Expected same element type!");
441 assert(ExtrVecTy->getElementType() == VecTy->getElementType() &&
442 "Expected same element type!");
444 for (
unsigned Idx = 0,
E = ExtrVecTy->getNumElements(); Idx !=
E; ++Idx) {
445 int MaskLane = Lane + Idx;
446 assert((
unsigned)MaskLane <
449 Mask.push_back(MaskLane);
452 WhereIt, Ctx,
"Unpack");
473 : It(Begin), ItE(
End), Lane(BeginLane) {}
478 using pointer = std::pair<unsigned, Value *> *;
481 assert(It != ItE &&
"Already at end!");
484 Lane += VecTy->getNumElements();
494 return It ==
Other.It;
497 return !(*
this ==
Other);
519 OS <<
"DeadInstrCandidates:\n";
520 for (
auto *
I : DeadInstrCandidates)
535 template <
typename ValueContainerT>
546 template <
typename AlignRangeT,
typename TypeRangeT>
548 assert(AlignRange.begin() != AlignRange.end() &&
"Expected non empty!");
549 assert(TypeRange.begin() != TypeRange.end() &&
"Expected non empty!");
550 assert(AlignRange.end() - AlignRange.begin() ==
551 TypeRange.end() - TypeRange.begin() &&
552 "Expected same size!");
555 return *AlignRange.begin();
559 template <
typename InstrRangeT>
assert(UImm &&(UImm !=~static_cast< T >(0)) &&"Invalid immediate!")
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
#define LLVM_TEMPLATE_ABI
#define LLVM_DUMP_METHOD
Mark debug helper function definitions like dump() that should not be stripped from debug builds.
This file defines the DenseSet and SmallDenseSet classes.
ConstantRange Range(APInt(BitWidth, Low), APInt(BitWidth, High))
Represent a constant reference to an array (0 or more elements consecutively in memory),...
const T & front() const
Get the first element.
ArrayRef()=default
Construct an empty ArrayRef.
bool empty() const
Check if the array is empty.
iterator begin()
Instruction iterator methods.
InstListType::iterator iterator
Instruction iterators...
A parsed version of the target data layout string in and methods for querying it.
Implements a dense probed hash-table based set.
The main scalar evolution driver.
SmallPtrSet - This class implements a set which is optimized for holding SmallSize or less elements.
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
LLVM Value Representation.
This class implements an extremely fast bulk output stream that can only output to a stream.
An ArrayRef of Values or Instructions that we can print/dump for debugging.
void print(raw_ostream &OS) const
Helper dump function for debugging.
LLVM_DUMP_METHOD void dump() const
static LLVM_ABI ConstantInt * getSigned(IntegerType *Ty, int64_t V)
Return a ConstantInt with the specified value for the specified type.
static LLVM_ABI Value * create(Value *Vec, Value *Idx, InsertPosition Pos, Context &Ctx, const Twine &Name="")
static LLVM_ABI FixedVectorType * get(Type *ElementType, unsigned NumElts)
Maps the original instructions to the vectorized instrs and the reverse.
A sandboxir::User with operands, opcode and linked with previous/next instructions in an instruction ...
LLVM_ABI BBIterator getIterator() const
\Returns a BasicBlock::iterator for this Instruction.
bool comesBefore(const Instruction *Other) const
Given an instruction Other in the same basic block as this instruction, return true if this instructi...
LLVM_ABI BasicBlock * getParent() const
\Returns the BasicBlock containing this Instruction, or null if it is detached.
static LLVM_ABI IntegerType * get(Context &C, unsigned NumBits)
static LLVM_ABI PoisonValue * get(Type *T)
Static factory methods - Return an 'poison' object of the specified type.
static LLVM_ABI Value * create(Value *V1, Value *V2, Value *Mask, InsertPosition Pos, Context &Ctx, const Twine &Name="")
Just like llvm::Type these are immutable, unique, never get freed and can only be created via static ...
static LLVM_ABI Type * getHalfTy(Context &Ctx)
static LLVM_ABI Type * getDoubleTy(Context &Ctx)
bool isIntegerTy() const
True if this is an instance of IntegerType.
bool isHalfTy() const
Return true if this is 'half', a 16-bit IEEE fp type.
bool isDoubleTy() const
Return true if this is 'double', a 64-bit IEEE fp type.
static LLVM_ABI IntegerType * getInt32Ty(Context &Ctx)
bool isVectorTy() const
True if this is an instance of VectorType.
bool isPointerTy() const
True if this is an instance of PointerType.
bool isFloatTy() const
Return true if this is 'float', a 32-bit IEEE fp type.
static LLVM_ABI Type * getFloatTy(Context &Ctx)
Value * getOperand(unsigned OpIdx) const
static std::optional< int > getPointerDiffInBytes(LoadOrStoreT *I0, LoadOrStoreT *I1, ScalarEvolution &SE)
\Returns the gap between the memory locations accessed by I0 and I1 in bytes.
static unsigned getNumBits(Type *Ty, const DataLayout &DL)
\Returns the number of bits of Ty.
static Type * getExpectedType(const Value *V)
\Returns the expected type of Value V.
A SandboxIR Value has users. This is the base class.
LLVM_ABI Type * getType() const
Context & getContext() const
Classifies whether all bundle elements are integers, halves, floats, doubles, or pointers.
bool areAllPointers() const
bool areAllDoubles() const
BndlTypeClassifier(BndlRef< T * > Bndl)
bool areAllHalves() const
bool areAllFloats() const
bool areAllIntegers() const
DeadInstructionMorgue(const DeadInstructionMorgue &)=delete
LLVM_DUMP_METHOD void debug() const
LLVM_ABI void tryEraseDeadInstrs()
Erase candidates recorded by collectPotentiallyDeadInstrs() that now have no uses,...
void print(raw_ostream &OS) const
void collectPotentiallyDeadInstrs(BndlRef< T * > Bndl)
Record instructions in Bndl that may be dead after vectorization.
DeadInstructionMorgue()=default
Iterate over all lanes and Value pairs.
std::input_iterator_tag iterator_catecotry
bool operator==(const LaneValueEnumerator &Other) const
bool operator!=(const LaneValueEnumerator &Other) const
std::pair< unsigned, Value * > value_type
std::pair< unsigned, Value * > & reference
LaneValueEnumerator(RangeIteratorT Begin, RangeIteratorT End, unsigned BeginLane)
std::pair< unsigned, Value * > * pointer
LaneValueEnumerator operator++()
value_type operator*() const
std::ptrdiff_t difference_type
static Type * tryGetCommonScalarType(ArrayRef< Value * > Bndl)
If all values in Bndl are of the same scalar type then return it, otherwise return nullptr.
static Instruction * getLowest(ArrayRef< Instruction * > Instrs)
\Returns the instruction in Instrs that is lowest in the BB.
static Type * getCommonScalarType(ArrayRef< Value * > Bndl)
Similar to tryGetCommonScalarType() but will assert that there is a common type.
static int getNumElements(Type *Ty)
\Returns the number of elements in Ty.
static std::optional< PackPattern > matchPack(Instruction *I)
If I is the last instruction of a pack pattern (i.e., an InsertElement into a vector),...
static Instruction * getLastPHIOrSelf(Instruction *I)
If I is not a PHI it returns it.
static unsigned getNumLanes(Type *Ty)
\Returns the number of vector lanes of Ty or 1 if not a vector.
static Instruction * getLowest(ArrayRef< Value * > Vals, BasicBlock *BB)
\Returns the lowest instruction in Vals, or nullptr if no instructions are found.
static Value * unpack(Value *FromVec, Type *ExtrTy, unsigned Lane, BasicBlock::iterator WhereIt)
Emits the necessary instruction sequence to extract element of type ExtrTy at Lane from FromVec.
static LLVM_DUMP_METHOD void dump(ArrayRef< Value * > Bndl)
Helper dump function for debugging.
static Type * getWideType(Type *ElemTy, unsigned NumElts)
\Returns <NumElts x ElemTy>.
static Instruction * getHighest(ArrayRef< Instruction * > Instrs)
\Returns the instruction in Instrs that is highest in the BB.
static auto enumerateLanes(const ValueContainerT &Range)
Helper for creating LaneValueEnumerator ranges.
static bool areConsecutive(LoadOrStoreT *I1, LoadOrStoreT *I2, ScalarEvolution &SE, const DataLayout &DL)
\Returns true if I1 and I2 are load/stores accessing consecutive memory addresses.
static Type * getCombinedVectorTypeFor(std::initializer_list< Value * > Bndl, const DataLayout &DL)
static bool areConsecutive(ArrayRef< ValT * > Bndl, ScalarEvolution &SE, const DataLayout &DL)
static Type * getElementType(Type *Ty)
Returns Ty if scalar or its element type if vector.
static unsigned getNumLanes(Value *V)
\Returns the expected vector lanes of V or 1 if not a vector.
static Align getVecAlignment(AlignRangeT AlignRange, TypeRangeT TypeRange)
Given the alignments of the instructions before vectorization, this returns the alignment of the vect...
static Align getVecAlignment(InstrRangeT InstrRange)
Helper for getVecAlignment(AlignRange, TypeRange).
static Type * getCombinedVectorTypeFor(BndlRef< T * > Bndl, const DataLayout &DL)
\Returns the combined vector type for Bndl, even when the element types differ.
static unsigned getNumLanes(ArrayRef< Value * > Bndl)
\Returns the total number of lanes across all values in Bndl.
static BasicBlock::iterator getInsertPointAfterInstrs(ArrayRef< Value * > Vals, BasicBlock *BB)
\Returns the BB iterator after the lowest instruction in Vals (skipping instructions not in BB),...
static LLVM_ABI unsigned getFloorPowerOf2(unsigned Num)
\Returns the first integer power of 2 that is <= Num.
static LLVM_ABI SmallVector< BundleTy > getNextUserBundles(ArrayRef< Value * > Bndl, const InstrMaps &IMaps, SmallPtrSet< Instruction *, 4 > &Claimed)
For each user of lane 0 in Bndl, try to form a bundle of matching users for all lanes.
BndlRef(const T &OneElt) -> BndlRef< T >
BasicBlock(llvm::BasicBlock *BB, Context &SBCtx)
SmallVector< Value *, 4 > BundleTy
Align getLoadStoreAlignment(const Value *I)
A helper function that returns the alignment of load or store instruction.
This is an optimization pass for GlobalISel generic memory operations.
auto drop_begin(T &&RangeOrContainer, size_t N=1)
Return a range covering RangeOrContainer with the first N elements excluded.
auto size(R &&Range, std::enable_if_t< std::is_base_of< std::random_access_iterator_tag, typename std::iterator_traits< decltype(Range.begin())>::iterator_category >::value, void > *=nullptr)
Get the size of a range.
auto enumerate(FirstRange &&First, RestRanges &&...Rest)
Given two or more input ranges, returns a new range whose values are tuples (A, B,...
decltype(auto) dyn_cast(const From &Val)
dyn_cast<X> - Return the argument parameter cast to the specified type.
iterator_range< T > make_range(T x, T y)
Convenience function for iterating over sub-ranges.
auto map_range(ContainerTy &&C, FuncTy F)
Return a range that applies F to the elements of C.
auto reverse(ContainerTy &&C)
LLVM_ABI raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
bool isa(const From &Val)
isa<X> - Return true if the parameter to the template is an instance of one of the template type argu...
DWARFExpression::Operation Op
decltype(auto) cast(const From &Val)
cast<X> - Return the argument parameter cast to the specified type.
auto find_if(R &&Range, UnaryPredicate P)
Provide wrappers to std::find_if which take ranges instead of having to pass begin/end explicitly.
constexpr auto seq(T Begin, T End)
Iterate over an integral type from Begin up to - but not including - End.
@ End
Run the machine verifier once, at the end of the pipeline.
hash_code hash_combine_range(InputIteratorT first, InputIteratorT last)
Compute a hash_code for a sequence of values.
This struct is a compact representation of a valid (non-zero power of two) alignment.
static bool isEqual(const SmallVector< sandboxir::Value * > &Vec1, const SmallVector< sandboxir::Value * > &Vec2)
static unsigned getHashValue(const SmallVector< sandboxir::Value * > &Vec)
An information struct used to provide DenseMap with the various necessary components for a given valu...
Helper struct for matchPack().
SmallVector< Value * > Operands
The "external" operands of the pack pattern, i.e., the values that get packed into a vector,...
SmallVector< Instruction * > Instrs
The insertelement instructions that form the pack pattern in bottom-up order, i.e....