LLVM  16.0.0git
InterleavedAccessPass.cpp
Go to the documentation of this file.
1 //===- InterleavedAccessPass.cpp ------------------------------------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file implements the Interleaved Access pass, which identifies
10 // interleaved memory accesses and transforms them into target specific
11 // intrinsics.
12 //
13 // An interleaved load reads data from memory into several vectors, with
14 // DE-interleaving the data on a factor. An interleaved store writes several
15 // vectors to memory with RE-interleaving the data on a factor.
16 //
17 // As interleaved accesses are difficult to identified in CodeGen (mainly
18 // because the VECTOR_SHUFFLE DAG node is quite different from the shufflevector
19 // IR), we identify and transform them to intrinsics in this pass so the
20 // intrinsics can be easily matched into target specific instructions later in
21 // CodeGen.
22 //
23 // E.g. An interleaved load (Factor = 2):
24 // %wide.vec = load <8 x i32>, <8 x i32>* %ptr
25 // %v0 = shuffle <8 x i32> %wide.vec, <8 x i32> poison, <0, 2, 4, 6>
26 // %v1 = shuffle <8 x i32> %wide.vec, <8 x i32> poison, <1, 3, 5, 7>
27 //
28 // It could be transformed into a ld2 intrinsic in AArch64 backend or a vld2
29 // intrinsic in ARM backend.
30 //
31 // In X86, this can be further optimized into a set of target
32 // specific loads followed by an optimized sequence of shuffles.
33 //
34 // E.g. An interleaved store (Factor = 3):
35 // %i.vec = shuffle <8 x i32> %v0, <8 x i32> %v1,
36 // <0, 4, 8, 1, 5, 9, 2, 6, 10, 3, 7, 11>
37 // store <12 x i32> %i.vec, <12 x i32>* %ptr
38 //
39 // It could be transformed into a st3 intrinsic in AArch64 backend or a vst3
40 // intrinsic in ARM backend.
41 //
42 // Similarly, a set of interleaved stores can be transformed into an optimized
43 // sequence of shuffles followed by a set of target specific stores for X86.
44 //
45 //===----------------------------------------------------------------------===//
46 
47 #include "llvm/ADT/ArrayRef.h"
48 #include "llvm/ADT/DenseMap.h"
49 #include "llvm/ADT/SetVector.h"
50 #include "llvm/ADT/SmallVector.h"
54 #include "llvm/IR/Constants.h"
55 #include "llvm/IR/Dominators.h"
56 #include "llvm/IR/Function.h"
57 #include "llvm/IR/IRBuilder.h"
58 #include "llvm/IR/InstIterator.h"
59 #include "llvm/IR/Instruction.h"
60 #include "llvm/IR/Instructions.h"
61 #include "llvm/InitializePasses.h"
62 #include "llvm/Pass.h"
63 #include "llvm/Support/Casting.h"
65 #include "llvm/Support/Debug.h"
70 #include <cassert>
71 #include <utility>
72 
73 using namespace llvm;
74 
75 #define DEBUG_TYPE "interleaved-access"
76 
78  "lower-interleaved-accesses",
79  cl::desc("Enable lowering interleaved accesses to intrinsics"),
80  cl::init(true), cl::Hidden);
81 
82 namespace {
83 
84 class InterleavedAccess : public FunctionPass {
85 public:
86  static char ID;
87 
88  InterleavedAccess() : FunctionPass(ID) {
90  }
91 
92  StringRef getPassName() const override { return "Interleaved Access Pass"; }
93 
94  bool runOnFunction(Function &F) override;
95 
96  void getAnalysisUsage(AnalysisUsage &AU) const override {
98  AU.setPreservesCFG();
99  }
100 
101 private:
102  DominatorTree *DT = nullptr;
103  const TargetLowering *TLI = nullptr;
104 
105  /// The maximum supported interleave factor.
106  unsigned MaxFactor;
107 
108  /// Transform an interleaved load into target specific intrinsics.
109  bool lowerInterleavedLoad(LoadInst *LI,
110  SmallVector<Instruction *, 32> &DeadInsts);
111 
112  /// Transform an interleaved store into target specific intrinsics.
113  bool lowerInterleavedStore(StoreInst *SI,
114  SmallVector<Instruction *, 32> &DeadInsts);
115 
116  /// Returns true if the uses of an interleaved load by the
117  /// extractelement instructions in \p Extracts can be replaced by uses of the
118  /// shufflevector instructions in \p Shuffles instead. If so, the necessary
119  /// replacements are also performed.
120  bool tryReplaceExtracts(ArrayRef<ExtractElementInst *> Extracts,
122 
123  /// Given a number of shuffles of the form shuffle(binop(x,y)), convert them
124  /// to binop(shuffle(x), shuffle(y)) to allow the formation of an
125  /// interleaving load. Any newly created shuffles that operate on \p LI will
126  /// be added to \p Shuffles. Returns true, if any changes to the IR have been
127  /// made.
128  bool replaceBinOpShuffles(ArrayRef<ShuffleVectorInst *> BinOpShuffles,
130  LoadInst *LI);
131 };
132 
133 } // end anonymous namespace.
134 
135 char InterleavedAccess::ID = 0;
136 
137 INITIALIZE_PASS_BEGIN(InterleavedAccess, DEBUG_TYPE,
138  "Lower interleaved memory accesses to target specific intrinsics", false,
139  false)
141 INITIALIZE_PASS_END(InterleavedAccess, DEBUG_TYPE,
142  "Lower interleaved memory accesses to target specific intrinsics", false,
143  false)
144 
146  return new InterleavedAccess();
147 }
148 
149 /// Check if the mask is a DE-interleave mask of the given factor
150 /// \p Factor like:
151 /// <Index, Index+Factor, ..., Index+(NumElts-1)*Factor>
152 static bool isDeInterleaveMaskOfFactor(ArrayRef<int> Mask, unsigned Factor,
153  unsigned &Index) {
154  // Check all potential start indices from 0 to (Factor - 1).
155  for (Index = 0; Index < Factor; Index++) {
156  unsigned i = 0;
157 
158  // Check that elements are in ascending order by Factor. Ignore undef
159  // elements.
160  for (; i < Mask.size(); i++)
161  if (Mask[i] >= 0 && static_cast<unsigned>(Mask[i]) != Index + i * Factor)
162  break;
163 
164  if (i == Mask.size())
165  return true;
166  }
167 
168  return false;
169 }
170 
171 /// Check if the mask is a DE-interleave mask for an interleaved load.
172 ///
173 /// E.g. DE-interleave masks (Factor = 2) could be:
174 /// <0, 2, 4, 6> (mask of index 0 to extract even elements)
175 /// <1, 3, 5, 7> (mask of index 1 to extract odd elements)
176 static bool isDeInterleaveMask(ArrayRef<int> Mask, unsigned &Factor,
177  unsigned &Index, unsigned MaxFactor,
178  unsigned NumLoadElements) {
179  if (Mask.size() < 2)
180  return false;
181 
182  // Check potential Factors.
183  for (Factor = 2; Factor <= MaxFactor; Factor++) {
184  // Make sure we don't produce a load wider than the input load.
185  if (Mask.size() * Factor > NumLoadElements)
186  return false;
187  if (isDeInterleaveMaskOfFactor(Mask, Factor, Index))
188  return true;
189  }
190 
191  return false;
192 }
193 
194 /// Check if the mask can be used in an interleaved store.
195 //
196 /// It checks for a more general pattern than the RE-interleave mask.
197 /// I.e. <x, y, ... z, x+1, y+1, ...z+1, x+2, y+2, ...z+2, ...>
198 /// E.g. For a Factor of 2 (LaneLen=4): <4, 32, 5, 33, 6, 34, 7, 35>
199 /// E.g. For a Factor of 3 (LaneLen=4): <4, 32, 16, 5, 33, 17, 6, 34, 18, 7, 35, 19>
200 /// E.g. For a Factor of 4 (LaneLen=2): <8, 2, 12, 4, 9, 3, 13, 5>
201 ///
202 /// The particular case of an RE-interleave mask is:
203 /// I.e. <0, LaneLen, ... , LaneLen*(Factor - 1), 1, LaneLen + 1, ...>
204 /// E.g. For a Factor of 2 (LaneLen=4): <0, 4, 1, 5, 2, 6, 3, 7>
205 static bool isReInterleaveMask(ArrayRef<int> Mask, unsigned &Factor,
206  unsigned MaxFactor, unsigned OpNumElts) {
207  unsigned NumElts = Mask.size();
208  if (NumElts < 4)
209  return false;
210 
211  // Check potential Factors.
212  for (Factor = 2; Factor <= MaxFactor; Factor++) {
213  if (NumElts % Factor)
214  continue;
215 
216  unsigned LaneLen = NumElts / Factor;
217  if (!isPowerOf2_32(LaneLen))
218  continue;
219 
220  // Check whether each element matches the general interleaved rule.
221  // Ignore undef elements, as long as the defined elements match the rule.
222  // Outer loop processes all factors (x, y, z in the above example)
223  unsigned I = 0, J;
224  for (; I < Factor; I++) {
225  unsigned SavedLaneValue;
226  unsigned SavedNoUndefs = 0;
227 
228  // Inner loop processes consecutive accesses (x, x+1... in the example)
229  for (J = 0; J < LaneLen - 1; J++) {
230  // Lane computes x's position in the Mask
231  unsigned Lane = J * Factor + I;
232  unsigned NextLane = Lane + Factor;
233  int LaneValue = Mask[Lane];
234  int NextLaneValue = Mask[NextLane];
235 
236  // If both are defined, values must be sequential
237  if (LaneValue >= 0 && NextLaneValue >= 0 &&
238  LaneValue + 1 != NextLaneValue)
239  break;
240 
241  // If the next value is undef, save the current one as reference
242  if (LaneValue >= 0 && NextLaneValue < 0) {
243  SavedLaneValue = LaneValue;
244  SavedNoUndefs = 1;
245  }
246 
247  // Undefs are allowed, but defined elements must still be consecutive:
248  // i.e.: x,..., undef,..., x + 2,..., undef,..., undef,..., x + 5, ....
249  // Verify this by storing the last non-undef followed by an undef
250  // Check that following non-undef masks are incremented with the
251  // corresponding distance.
252  if (SavedNoUndefs > 0 && LaneValue < 0) {
253  SavedNoUndefs++;
254  if (NextLaneValue >= 0 &&
255  SavedLaneValue + SavedNoUndefs != (unsigned)NextLaneValue)
256  break;
257  }
258  }
259 
260  if (J < LaneLen - 1)
261  break;
262 
263  int StartMask = 0;
264  if (Mask[I] >= 0) {
265  // Check that the start of the I range (J=0) is greater than 0
266  StartMask = Mask[I];
267  } else if (Mask[(LaneLen - 1) * Factor + I] >= 0) {
268  // StartMask defined by the last value in lane
269  StartMask = Mask[(LaneLen - 1) * Factor + I] - J;
270  } else if (SavedNoUndefs > 0) {
271  // StartMask defined by some non-zero value in the j loop
272  StartMask = SavedLaneValue - (LaneLen - 1 - SavedNoUndefs);
273  }
274  // else StartMask remains set to 0, i.e. all elements are undefs
275 
276  if (StartMask < 0)
277  break;
278  // We must stay within the vectors; This case can happen with undefs.
279  if (StartMask + LaneLen > OpNumElts*2)
280  break;
281  }
282 
283  // Found an interleaved mask of current factor.
284  if (I == Factor)
285  return true;
286  }
287 
288  return false;
289 }
290 
291 bool InterleavedAccess::lowerInterleavedLoad(
292  LoadInst *LI, SmallVector<Instruction *, 32> &DeadInsts) {
293  if (!LI->isSimple() || isa<ScalableVectorType>(LI->getType()))
294  return false;
295 
296  // Check if all users of this load are shufflevectors. If we encounter any
297  // users that are extractelement instructions or binary operators, we save
298  // them to later check if they can be modified to extract from one of the
299  // shufflevectors instead of the load.
300 
303  // BinOpShuffles need to be handled a single time in case both operands of the
304  // binop are the same load.
306 
307  for (auto *User : LI->users()) {
308  auto *Extract = dyn_cast<ExtractElementInst>(User);
309  if (Extract && isa<ConstantInt>(Extract->getIndexOperand())) {
310  Extracts.push_back(Extract);
311  continue;
312  }
313  if (auto *BI = dyn_cast<BinaryOperator>(User)) {
314  if (all_of(BI->users(),
315  [](auto *U) { return isa<ShuffleVectorInst>(U); })) {
316  for (auto *SVI : BI->users())
317  BinOpShuffles.insert(cast<ShuffleVectorInst>(SVI));
318  continue;
319  }
320  }
321  auto *SVI = dyn_cast<ShuffleVectorInst>(User);
322  if (!SVI || !isa<UndefValue>(SVI->getOperand(1)))
323  return false;
324 
325  Shuffles.push_back(SVI);
326  }
327 
328  if (Shuffles.empty() && BinOpShuffles.empty())
329  return false;
330 
331  unsigned Factor, Index;
332 
333  unsigned NumLoadElements =
334  cast<FixedVectorType>(LI->getType())->getNumElements();
335  auto *FirstSVI = Shuffles.size() > 0 ? Shuffles[0] : BinOpShuffles[0];
336  // Check if the first shufflevector is DE-interleave shuffle.
337  if (!isDeInterleaveMask(FirstSVI->getShuffleMask(), Factor, Index, MaxFactor,
338  NumLoadElements))
339  return false;
340 
341  // Holds the corresponding index for each DE-interleave shuffle.
342  SmallVector<unsigned, 4> Indices;
343 
344  Type *VecTy = FirstSVI->getType();
345 
346  // Check if other shufflevectors are also DE-interleaved of the same type
347  // and factor as the first shufflevector.
348  for (auto *Shuffle : Shuffles) {
349  if (Shuffle->getType() != VecTy)
350  return false;
351  if (!isDeInterleaveMaskOfFactor(Shuffle->getShuffleMask(), Factor,
352  Index))
353  return false;
354 
355  assert(Shuffle->getShuffleMask().size() <= NumLoadElements);
356  Indices.push_back(Index);
357  }
358  for (auto *Shuffle : BinOpShuffles) {
359  if (Shuffle->getType() != VecTy)
360  return false;
361  if (!isDeInterleaveMaskOfFactor(Shuffle->getShuffleMask(), Factor,
362  Index))
363  return false;
364 
365  assert(Shuffle->getShuffleMask().size() <= NumLoadElements);
366 
367  if (cast<Instruction>(Shuffle->getOperand(0))->getOperand(0) == LI)
368  Indices.push_back(Index);
369  if (cast<Instruction>(Shuffle->getOperand(0))->getOperand(1) == LI)
370  Indices.push_back(Index);
371  }
372 
373  // Try and modify users of the load that are extractelement instructions to
374  // use the shufflevector instructions instead of the load.
375  if (!tryReplaceExtracts(Extracts, Shuffles))
376  return false;
377 
378  bool BinOpShuffleChanged =
379  replaceBinOpShuffles(BinOpShuffles.getArrayRef(), Shuffles, LI);
380 
381  LLVM_DEBUG(dbgs() << "IA: Found an interleaved load: " << *LI << "\n");
382 
383  // Try to create target specific intrinsics to replace the load and shuffles.
384  if (!TLI->lowerInterleavedLoad(LI, Shuffles, Indices, Factor)) {
385  // If Extracts is not empty, tryReplaceExtracts made changes earlier.
386  return !Extracts.empty() || BinOpShuffleChanged;
387  }
388 
389  append_range(DeadInsts, Shuffles);
390 
391  DeadInsts.push_back(LI);
392  return true;
393 }
394 
395 bool InterleavedAccess::replaceBinOpShuffles(
396  ArrayRef<ShuffleVectorInst *> BinOpShuffles,
398  for (auto *SVI : BinOpShuffles) {
399  BinaryOperator *BI = cast<BinaryOperator>(SVI->getOperand(0));
400  Type *BIOp0Ty = BI->getOperand(0)->getType();
401  ArrayRef<int> Mask = SVI->getShuffleMask();
402  assert(all_of(Mask, [&](int Idx) {
403  return Idx < (int)cast<FixedVectorType>(BIOp0Ty)->getNumElements();
404  }));
405 
406  auto *NewSVI1 =
407  new ShuffleVectorInst(BI->getOperand(0), PoisonValue::get(BIOp0Ty),
408  Mask, SVI->getName(), SVI);
409  auto *NewSVI2 = new ShuffleVectorInst(
410  BI->getOperand(1), PoisonValue::get(BI->getOperand(1)->getType()), Mask,
411  SVI->getName(), SVI);
413  BI->getOpcode(), NewSVI1, NewSVI2, BI, BI->getName(), SVI);
414  SVI->replaceAllUsesWith(NewBI);
415  LLVM_DEBUG(dbgs() << " Replaced: " << *BI << "\n And : " << *SVI
416  << "\n With : " << *NewSVI1 << "\n And : "
417  << *NewSVI2 << "\n And : " << *NewBI << "\n");
419  if (NewSVI1->getOperand(0) == LI)
420  Shuffles.push_back(NewSVI1);
421  if (NewSVI2->getOperand(0) == LI)
422  Shuffles.push_back(NewSVI2);
423  }
424 
425  return !BinOpShuffles.empty();
426 }
427 
428 bool InterleavedAccess::tryReplaceExtracts(
431  // If there aren't any extractelement instructions to modify, there's nothing
432  // to do.
433  if (Extracts.empty())
434  return true;
435 
436  // Maps extractelement instructions to vector-index pairs. The extractlement
437  // instructions will be modified to use the new vector and index operands.
439 
440  for (auto *Extract : Extracts) {
441  // The vector index that is extracted.
442  auto *IndexOperand = cast<ConstantInt>(Extract->getIndexOperand());
443  auto Index = IndexOperand->getSExtValue();
444 
445  // Look for a suitable shufflevector instruction. The goal is to modify the
446  // extractelement instruction (which uses an interleaved load) to use one
447  // of the shufflevector instructions instead of the load.
448  for (auto *Shuffle : Shuffles) {
449  // If the shufflevector instruction doesn't dominate the extract, we
450  // can't create a use of it.
451  if (!DT->dominates(Shuffle, Extract))
452  continue;
453 
454  // Inspect the indices of the shufflevector instruction. If the shuffle
455  // selects the same index that is extracted, we can modify the
456  // extractelement instruction.
457  SmallVector<int, 4> Indices;
458  Shuffle->getShuffleMask(Indices);
459  for (unsigned I = 0; I < Indices.size(); ++I)
460  if (Indices[I] == Index) {
461  assert(Extract->getOperand(0) == Shuffle->getOperand(0) &&
462  "Vector operations do not match");
463  ReplacementMap[Extract] = std::make_pair(Shuffle, I);
464  break;
465  }
466 
467  // If we found a suitable shufflevector instruction, stop looking.
468  if (ReplacementMap.count(Extract))
469  break;
470  }
471 
472  // If we did not find a suitable shufflevector instruction, the
473  // extractelement instruction cannot be modified, so we must give up.
474  if (!ReplacementMap.count(Extract))
475  return false;
476  }
477 
478  // Finally, perform the replacements.
479  IRBuilder<> Builder(Extracts[0]->getContext());
480  for (auto &Replacement : ReplacementMap) {
481  auto *Extract = Replacement.first;
482  auto *Vector = Replacement.second.first;
483  auto Index = Replacement.second.second;
484  Builder.SetInsertPoint(Extract);
485  Extract->replaceAllUsesWith(Builder.CreateExtractElement(Vector, Index));
486  Extract->eraseFromParent();
487  }
488 
489  return true;
490 }
491 
492 bool InterleavedAccess::lowerInterleavedStore(
494  if (!SI->isSimple())
495  return false;
496 
497  auto *SVI = dyn_cast<ShuffleVectorInst>(SI->getValueOperand());
498  if (!SVI || !SVI->hasOneUse() || isa<ScalableVectorType>(SVI->getType()))
499  return false;
500 
501  // Check if the shufflevector is RE-interleave shuffle.
502  unsigned Factor;
503  unsigned OpNumElts =
504  cast<FixedVectorType>(SVI->getOperand(0)->getType())->getNumElements();
505  if (!isReInterleaveMask(SVI->getShuffleMask(), Factor, MaxFactor, OpNumElts))
506  return false;
507 
508  LLVM_DEBUG(dbgs() << "IA: Found an interleaved store: " << *SI << "\n");
509 
510  // Try to create target specific intrinsics to replace the store and shuffle.
511  if (!TLI->lowerInterleavedStore(SI, SVI, Factor))
512  return false;
513 
514  // Already have a new target specific interleaved store. Erase the old store.
515  DeadInsts.push_back(SI);
516  DeadInsts.push_back(SVI);
517  return true;
518 }
519 
521  auto *TPC = getAnalysisIfAvailable<TargetPassConfig>();
522  if (!TPC || !LowerInterleavedAccesses)
523  return false;
524 
525  LLVM_DEBUG(dbgs() << "*** " << getPassName() << ": " << F.getName() << "\n");
526 
527  DT = &getAnalysis<DominatorTreeWrapperPass>().getDomTree();
528  auto &TM = TPC->getTM<TargetMachine>();
529  TLI = TM.getSubtargetImpl(F)->getTargetLowering();
530  MaxFactor = TLI->getMaxSupportedInterleaveFactor();
531 
532  // Holds dead instructions that will be erased later.
534  bool Changed = false;
535 
536  for (auto &I : instructions(F)) {
537  if (auto *LI = dyn_cast<LoadInst>(&I))
538  Changed |= lowerInterleavedLoad(LI, DeadInsts);
539 
540  if (auto *SI = dyn_cast<StoreInst>(&I))
541  Changed |= lowerInterleavedStore(SI, DeadInsts);
542  }
543 
544  for (auto *I : DeadInsts)
545  I->eraseFromParent();
546 
547  return Changed;
548 }
i
i
Definition: README.txt:29
llvm::RecursivelyDeleteTriviallyDeadInstructions
bool RecursivelyDeleteTriviallyDeadInstructions(Value *V, const TargetLibraryInfo *TLI=nullptr, MemorySSAUpdater *MSSAU=nullptr, std::function< void(Value *)> AboutToDeleteCallback=std::function< void(Value *)>())
If the specified value is a trivially dead instruction, delete it.
Definition: Local.cpp:518
MathExtras.h
llvm
This is an optimization pass for GlobalISel generic memory operations.
Definition: AddressRanges.h:18
isDeInterleaveMask
static bool isDeInterleaveMask(ArrayRef< int > Mask, unsigned &Factor, unsigned &Index, unsigned MaxFactor, unsigned NumLoadElements)
Check if the mask is a DE-interleave mask for an interleaved load.
Definition: InterleavedAccessPass.cpp:176
InstIterator.h
llvm::Function
Definition: Function.h:60
Pass.h
llvm::BinaryOperator::CreateWithCopiedFlags
static BinaryOperator * CreateWithCopiedFlags(BinaryOps Opc, Value *V1, Value *V2, Instruction *CopyO, const Twine &Name="", Instruction *InsertBefore=nullptr)
Definition: InstrTypes.h:249
llvm::SmallVector
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
Definition: SmallVector.h:1183
llvm::IRBuilder<>
Local.h
llvm::DominatorTree
Concrete subclass of DominatorTreeBase that is used to compute a normal dominator tree.
Definition: Dominators.h:166
llvm::cl::Hidden
@ Hidden
Definition: CommandLine.h:139
to
Should compile to
Definition: README.txt:449
llvm::Type
The instances of the Type class are immutable: once they are created, they are never changed.
Definition: Type.h:45
DenseMap.h
llvm::DenseMapBase< DenseMap< KeyT, ValueT, DenseMapInfo< KeyT >, llvm::detail::DenseMapPair< KeyT, ValueT > >, KeyT, ValueT, DenseMapInfo< KeyT >, llvm::detail::DenseMapPair< KeyT, ValueT > >::count
size_type count(const_arg_type_t< KeyT > Val) const
Return 1 if the specified key is in the map, 0 otherwise.
Definition: DenseMap.h:145
Vector
So we should use XX3Form_Rcr to implement intrinsic Convert DP outs ins xscvdpsp No builtin are required Round &Convert QP DP(dword[1] is set to zero) No builtin are required Round to Quad Precision because you need to assign rounding mode in instruction Provide builtin(set f128:$vT,(int_ppc_vsx_xsrqpi f128:$vB))(set f128 yields< n x< ty > >< result > yields< ty >< result > No builtin are required Load Store Vector
Definition: README_P9.txt:497
isReInterleaveMask
static bool isReInterleaveMask(ArrayRef< int > Mask, unsigned &Factor, unsigned MaxFactor, unsigned OpNumElts)
Check if the mask can be used in an interleaved store.
Definition: InterleavedAccessPass.cpp:205
llvm::isPowerOf2_32
constexpr bool isPowerOf2_32(uint32_t Value)
Return true if the argument is a power of two > 0.
Definition: MathExtras.h:487
DEBUG_TYPE
#define DEBUG_TYPE
Definition: InterleavedAccessPass.cpp:75
llvm::ArrayRef::empty
bool empty() const
empty - Check if the array is empty.
Definition: ArrayRef.h:159
LLVM_DEBUG
#define LLVM_DEBUG(X)
Definition: Debug.h:101
F
#define F(x, y, z)
Definition: MD5.cpp:55
llvm::dbgs
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
Definition: Debug.cpp:163
llvm::BitmaskEnumDetail::Mask
constexpr std::underlying_type_t< E > Mask()
Get a bitmask with 1s in all places up to the high-order bit of E's largest value.
Definition: BitmaskEnum.h:80
Instruction.h
CommandLine.h
TargetLowering.h
llvm::all_of
bool all_of(R &&range, UnaryPredicate P)
Provide wrappers to std::all_of which take ranges instead of having to pass begin/end explicitly.
Definition: STLExtras.h:1618
TargetMachine.h
llvm::PassRegistry::getPassRegistry
static PassRegistry * getPassRegistry()
getPassRegistry - Access the global registry object, which is automatically initialized at applicatio...
Definition: PassRegistry.cpp:24
Constants.h
llvm::User
Definition: User.h:44
llvm::TargetLowering
This class defines information used to lower LLVM code to legal SelectionDAG operators that the targe...
Definition: TargetLowering.h:3417
llvm::AnalysisUsage
Represent the analysis usage information of a pass.
Definition: PassAnalysisSupport.h:47
false
Definition: StackSlotColoring.cpp:141
llvm::dwarf::Index
Index
Definition: Dwarf.h:472
llvm::BinaryOperator::getOpcode
BinaryOps getOpcode() const
Definition: InstrTypes.h:392
llvm::DominatorTreeWrapperPass
Legacy analysis pass which computes a DominatorTree.
Definition: Dominators.h:302
llvm::SetVector< T, SmallVector< T, N >, SmallDenseSet< T, N > >::empty
bool empty() const
Determine if the SetVector is empty or not.
Definition: SetVector.h:72
llvm::CallingConv::ID
unsigned ID
LLVM IR allows to use arbitrary numbers as calling convention identifiers.
Definition: CallingConv.h:24
INITIALIZE_PASS_END
#define INITIALIZE_PASS_END(passName, arg, name, cfg, analysis)
Definition: PassSupport.h:58
llvm::cl::opt< bool >
llvm::instructions
inst_range instructions(Function *F)
Definition: InstIterator.h:133
llvm::StoreInst
An instruction for storing to memory.
Definition: Instructions.h:298
Index
uint32_t Index
Definition: ELFObjHandler.cpp:82
INITIALIZE_PASS_DEPENDENCY
INITIALIZE_PASS_DEPENDENCY(DominatorTreeWrapperPass)
llvm::DenseMap
Definition: DenseMap.h:714
I
#define I(x, y, z)
Definition: MD5.cpp:58
llvm::cl::init
initializer< Ty > init(const Ty &Val)
Definition: CommandLine.h:439
ArrayRef.h
TargetPassConfig.h
IRBuilder.h
assert
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
llvm::TargetMachine
Primary interface to the complete machine description for the target machine.
Definition: TargetMachine.h:77
SI
StandardInstrumentations SI(Debug, VerifyEach)
INITIALIZE_PASS_BEGIN
INITIALIZE_PASS_BEGIN(InterleavedAccess, DEBUG_TYPE, "Lower interleaved memory accesses to target specific intrinsics", false, false) INITIALIZE_PASS_END(InterleavedAccess
Builder
assume Assume Builder
Definition: AssumeBundleBuilder.cpp:651
llvm::SetVector< T, SmallVector< T, N >, SmallDenseSet< T, N > >::insert
bool insert(const value_type &X)
Insert a new element into the SetVector.
Definition: SetVector.h:141
llvm::ArrayRef
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
Definition: APInt.h:32
llvm::BinaryOperator
Definition: InstrTypes.h:188
isDeInterleaveMaskOfFactor
static bool isDeInterleaveMaskOfFactor(ArrayRef< int > Mask, unsigned Factor, unsigned &Index)
Check if the mask is a DE-interleave mask of the given factor Factor like: <Index,...
Definition: InterleavedAccessPass.cpp:152
llvm::AnalysisUsage::setPreservesCFG
void setPreservesCFG()
This function should be called by the pass, iff they do not:
Definition: Pass.cpp:263
llvm::StringRef
StringRef - Represent a constant reference to a string, i.e.
Definition: StringRef.h:50
llvm::LoadInst::isSimple
bool isSimple() const
Definition: Instructions.h:253
llvm::Value::getType
Type * getType() const
All values are typed, get the type of this value.
Definition: Value.h:255
llvm::append_range
void append_range(Container &C, Range &&R)
Wrapper function to append a range to a container.
Definition: STLExtras.h:1829
TargetSubtargetInfo.h
llvm::Value::getName
StringRef getName() const
Return a constant reference to the value's name.
Definition: Value.cpp:308
llvm::LoadInst
An instruction for reading from memory.
Definition: Instructions.h:174
llvm::createInterleavedAccessPass
FunctionPass * createInterleavedAccessPass()
InterleavedAccess Pass - This pass identifies and matches interleaved memory accesses to target speci...
Definition: InterleavedAccessPass.cpp:145
runOnFunction
static bool runOnFunction(Function &F, bool PostInlining)
Definition: EntryExitInstrumenter.cpp:69
Casting.h
Function.h
intrinsics
Lower interleaved memory accesses to target specific intrinsics
Definition: InterleavedAccessPass.cpp:142
llvm::ShuffleVectorInst
This instruction constructs a fixed permutation of two input vectors.
Definition: Instructions.h:2008
Instructions.h
SmallVector.h
Dominators.h
LowerInterleavedAccesses
static cl::opt< bool > LowerInterleavedAccesses("lower-interleaved-accesses", cl::desc("Enable lowering interleaved accesses to intrinsics"), cl::init(true), cl::Hidden)
llvm::SmallVectorImpl
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
Definition: APFloat.h:42
llvm::SmallSetVector
A SetVector that performs no allocations if smaller than a certain size.
Definition: SetVector.h:307
TM
const char LLVMTargetMachineRef TM
Definition: PassBuilderBindings.cpp:47
llvm::FunctionPass
FunctionPass class - This class is used to implement most global optimizations.
Definition: Pass.h:308
llvm::AnalysisUsage::addRequired
AnalysisUsage & addRequired()
Definition: PassAnalysisSupport.h:75
llvm::User::getOperand
Value * getOperand(unsigned i) const
Definition: User.h:169
llvm::initializeInterleavedAccessPass
void initializeInterleavedAccessPass(PassRegistry &)
llvm::cl::desc
Definition: CommandLine.h:412
raw_ostream.h
InitializePasses.h
Debug.h
llvm::Value::users
iterator_range< user_iterator > users()
Definition: Value.h:421
SetVector.h
llvm::Intrinsic::ID
unsigned ID
Definition: TargetTransformInfo.h:38
llvm::PoisonValue::get
static PoisonValue * get(Type *T)
Static factory methods - Return an 'poison' object of the specified type.
Definition: Constants.cpp:1729