LLVM  13.0.0git
Scalarizer.cpp
Go to the documentation of this file.
1 //===- Scalarizer.cpp - Scalarize vector operations -----------------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This pass converts vector operations into scalar operations, in order
10 // to expose optimization opportunities on the individual scalar operations.
11 // It is mainly intended for targets that do not have vector units, but it
12 // may also be useful for revectorizing code to different vector widths.
13 //
14 //===----------------------------------------------------------------------===//
15 
18 #include "llvm/ADT/SmallVector.h"
19 #include "llvm/ADT/Twine.h"
21 #include "llvm/IR/Argument.h"
22 #include "llvm/IR/BasicBlock.h"
23 #include "llvm/IR/Constants.h"
24 #include "llvm/IR/DataLayout.h"
25 #include "llvm/IR/DerivedTypes.h"
26 #include "llvm/IR/Dominators.h"
27 #include "llvm/IR/Function.h"
28 #include "llvm/IR/IRBuilder.h"
29 #include "llvm/IR/InstVisitor.h"
30 #include "llvm/IR/InstrTypes.h"
31 #include "llvm/IR/Instruction.h"
32 #include "llvm/IR/Instructions.h"
33 #include "llvm/IR/Intrinsics.h"
34 #include "llvm/IR/LLVMContext.h"
35 #include "llvm/IR/Module.h"
36 #include "llvm/IR/Type.h"
37 #include "llvm/IR/Value.h"
38 #include "llvm/InitializePasses.h"
39 #include "llvm/Pass.h"
40 #include "llvm/Support/Casting.h"
43 #include "llvm/Transforms/Scalar.h"
45 #include <cassert>
46 #include <cstdint>
47 #include <iterator>
48 #include <map>
49 #include <utility>
50 
51 using namespace llvm;
52 
53 #define DEBUG_TYPE "scalarizer"
54 
56  "scalarize-variable-insert-extract", cl::init(true), cl::Hidden,
57  cl::desc("Allow the scalarizer pass to scalarize "
58  "insertelement/extractelement with variable index"));
59 
60 // This is disabled by default because having separate loads and stores
61 // makes it more likely that the -combiner-alias-analysis limits will be
62 // reached.
63 static cl::opt<bool>
64  ScalarizeLoadStore("scalarize-load-store", cl::init(false), cl::Hidden,
65  cl::desc("Allow the scalarizer pass to scalarize loads and store"));
66 
67 namespace {
68 
69 // Used to store the scattered form of a vector.
70 using ValueVector = SmallVector<Value *, 8>;
71 
72 // Used to map a vector Value to its scattered form. We use std::map
73 // because we want iterators to persist across insertion and because the
74 // values are relatively large.
75 using ScatterMap = std::map<Value *, ValueVector>;
76 
77 // Lists Instructions that have been replaced with scalar implementations,
78 // along with a pointer to their scattered forms.
80 
81 // Provides a very limited vector-like interface for lazily accessing one
82 // component of a scattered vector or vector pointer.
83 class Scatterer {
84 public:
85  Scatterer() = default;
86 
87  // Scatter V into Size components. If new instructions are needed,
88  // insert them before BBI in BB. If Cache is nonnull, use it to cache
89  // the results.
90  Scatterer(BasicBlock *bb, BasicBlock::iterator bbi, Value *v,
91  ValueVector *cachePtr = nullptr);
92 
93  // Return component I, creating a new Value for it if necessary.
94  Value *operator[](unsigned I);
95 
96  // Return the number of components.
97  unsigned size() const { return Size; }
98 
99 private:
100  BasicBlock *BB;
102  Value *V;
103  ValueVector *CachePtr;
104  PointerType *PtrTy;
105  ValueVector Tmp;
106  unsigned Size;
107 };
108 
109 // FCmpSpliiter(FCI)(Builder, X, Y, Name) uses Builder to create an FCmp
110 // called Name that compares X and Y in the same way as FCI.
111 struct FCmpSplitter {
112  FCmpSplitter(FCmpInst &fci) : FCI(fci) {}
113 
114  Value *operator()(IRBuilder<> &Builder, Value *Op0, Value *Op1,
115  const Twine &Name) const {
116  return Builder.CreateFCmp(FCI.getPredicate(), Op0, Op1, Name);
117  }
118 
119  FCmpInst &FCI;
120 };
121 
122 // ICmpSpliiter(ICI)(Builder, X, Y, Name) uses Builder to create an ICmp
123 // called Name that compares X and Y in the same way as ICI.
124 struct ICmpSplitter {
125  ICmpSplitter(ICmpInst &ici) : ICI(ici) {}
126 
127  Value *operator()(IRBuilder<> &Builder, Value *Op0, Value *Op1,
128  const Twine &Name) const {
129  return Builder.CreateICmp(ICI.getPredicate(), Op0, Op1, Name);
130  }
131 
132  ICmpInst &ICI;
133 };
134 
135 // UnarySpliiter(UO)(Builder, X, Name) uses Builder to create
136 // a unary operator like UO called Name with operand X.
137 struct UnarySplitter {
138  UnarySplitter(UnaryOperator &uo) : UO(uo) {}
139 
140  Value *operator()(IRBuilder<> &Builder, Value *Op, const Twine &Name) const {
141  return Builder.CreateUnOp(UO.getOpcode(), Op, Name);
142  }
143 
144  UnaryOperator &UO;
145 };
146 
147 // BinarySpliiter(BO)(Builder, X, Y, Name) uses Builder to create
148 // a binary operator like BO called Name with operands X and Y.
149 struct BinarySplitter {
150  BinarySplitter(BinaryOperator &bo) : BO(bo) {}
151 
152  Value *operator()(IRBuilder<> &Builder, Value *Op0, Value *Op1,
153  const Twine &Name) const {
154  return Builder.CreateBinOp(BO.getOpcode(), Op0, Op1, Name);
155  }
156 
157  BinaryOperator &BO;
158 };
159 
160 // Information about a load or store that we're scalarizing.
161 struct VectorLayout {
162  VectorLayout() = default;
163 
164  // Return the alignment of element I.
165  Align getElemAlign(unsigned I) {
166  return commonAlignment(VecAlign, I * ElemSize);
167  }
168 
169  // The type of the vector.
170  VectorType *VecTy = nullptr;
171 
172  // The type of each element.
173  Type *ElemTy = nullptr;
174 
175  // The alignment of the vector.
176  Align VecAlign;
177 
178  // The size of each element.
179  uint64_t ElemSize = 0;
180 };
181 
182 class ScalarizerVisitor : public InstVisitor<ScalarizerVisitor, bool> {
183 public:
184  ScalarizerVisitor(unsigned ParallelLoopAccessMDKind, DominatorTree *DT)
185  : ParallelLoopAccessMDKind(ParallelLoopAccessMDKind), DT(DT) {
186  }
187 
188  bool visit(Function &F);
189 
190  // InstVisitor methods. They return true if the instruction was scalarized,
191  // false if nothing changed.
192  bool visitInstruction(Instruction &I) { return false; }
193  bool visitSelectInst(SelectInst &SI);
194  bool visitICmpInst(ICmpInst &ICI);
195  bool visitFCmpInst(FCmpInst &FCI);
196  bool visitUnaryOperator(UnaryOperator &UO);
197  bool visitBinaryOperator(BinaryOperator &BO);
198  bool visitGetElementPtrInst(GetElementPtrInst &GEPI);
199  bool visitCastInst(CastInst &CI);
200  bool visitBitCastInst(BitCastInst &BCI);
201  bool visitInsertElementInst(InsertElementInst &IEI);
202  bool visitExtractElementInst(ExtractElementInst &EEI);
203  bool visitShuffleVectorInst(ShuffleVectorInst &SVI);
204  bool visitPHINode(PHINode &PHI);
205  bool visitLoadInst(LoadInst &LI);
206  bool visitStoreInst(StoreInst &SI);
207  bool visitCallInst(CallInst &ICI);
208 
209 private:
210  Scatterer scatter(Instruction *Point, Value *V);
211  void gather(Instruction *Op, const ValueVector &CV);
212  bool canTransferMetadata(unsigned Kind);
213  void transferMetadataAndIRFlags(Instruction *Op, const ValueVector &CV);
214  Optional<VectorLayout> getVectorLayout(Type *Ty, Align Alignment,
215  const DataLayout &DL);
216  bool finish();
217 
218  template<typename T> bool splitUnary(Instruction &, const T &);
219  template<typename T> bool splitBinary(Instruction &, const T &);
220 
221  bool splitCall(CallInst &CI);
222 
223  ScatterMap Scattered;
224  GatherList Gathered;
225 
226  SmallVector<WeakTrackingVH, 32> PotentiallyDeadInstrs;
227 
228  unsigned ParallelLoopAccessMDKind;
229 
230  DominatorTree *DT;
231 };
232 
233 class ScalarizerLegacyPass : public FunctionPass {
234 public:
235  static char ID;
236 
237  ScalarizerLegacyPass() : FunctionPass(ID) {
239  }
240 
241  bool runOnFunction(Function &F) override;
242 
243  void getAnalysisUsage(AnalysisUsage& AU) const override {
246  }
247 };
248 
249 } // end anonymous namespace
250 
251 char ScalarizerLegacyPass::ID = 0;
252 INITIALIZE_PASS_BEGIN(ScalarizerLegacyPass, "scalarizer",
253  "Scalarize vector operations", false, false)
255 INITIALIZE_PASS_END(ScalarizerLegacyPass, "scalarizer",
256  "Scalarize vector operations", false, false)
257 
258 Scatterer::Scatterer(BasicBlock *bb, BasicBlock::iterator bbi, Value *v,
259  ValueVector *cachePtr)
260  : BB(bb), BBI(bbi), V(v), CachePtr(cachePtr) {
261  Type *Ty = V->getType();
262  PtrTy = dyn_cast<PointerType>(Ty);
263  if (PtrTy)
264  Ty = PtrTy->getElementType();
265  Size = cast<FixedVectorType>(Ty)->getNumElements();
266  if (!CachePtr)
267  Tmp.resize(Size, nullptr);
268  else if (CachePtr->empty())
269  CachePtr->resize(Size, nullptr);
270  else
271  assert(Size == CachePtr->size() && "Inconsistent vector sizes");
272 }
273 
274 // Return component I, creating a new Value for it if necessary.
275 Value *Scatterer::operator[](unsigned I) {
276  ValueVector &CV = (CachePtr ? *CachePtr : Tmp);
277  // Try to reuse a previous value.
278  if (CV[I])
279  return CV[I];
280  IRBuilder<> Builder(BB, BBI);
281  if (PtrTy) {
282  Type *ElTy = cast<VectorType>(PtrTy->getElementType())->getElementType();
283  if (!CV[0]) {
284  Type *NewPtrTy = PointerType::get(ElTy, PtrTy->getAddressSpace());
285  CV[0] = Builder.CreateBitCast(V, NewPtrTy, V->getName() + ".i0");
286  }
287  if (I != 0)
288  CV[I] = Builder.CreateConstGEP1_32(ElTy, CV[0], I,
289  V->getName() + ".i" + Twine(I));
290  } else {
291  // Search through a chain of InsertElementInsts looking for element I.
292  // Record other elements in the cache. The new V is still suitable
293  // for all uncached indices.
294  while (true) {
295  InsertElementInst *Insert = dyn_cast<InsertElementInst>(V);
296  if (!Insert)
297  break;
298  ConstantInt *Idx = dyn_cast<ConstantInt>(Insert->getOperand(2));
299  if (!Idx)
300  break;
301  unsigned J = Idx->getZExtValue();
302  V = Insert->getOperand(0);
303  if (I == J) {
304  CV[J] = Insert->getOperand(1);
305  return CV[J];
306  } else if (!CV[J]) {
307  // Only cache the first entry we find for each index we're not actively
308  // searching for. This prevents us from going too far up the chain and
309  // caching incorrect entries.
310  CV[J] = Insert->getOperand(1);
311  }
312  }
313  CV[I] = Builder.CreateExtractElement(V, Builder.getInt32(I),
314  V->getName() + ".i" + Twine(I));
315  }
316  return CV[I];
317 }
318 
320  if (skipFunction(F))
321  return false;
322 
323  Module &M = *F.getParent();
324  unsigned ParallelLoopAccessMDKind =
325  M.getContext().getMDKindID("llvm.mem.parallel_loop_access");
326  DominatorTree *DT = &getAnalysis<DominatorTreeWrapperPass>().getDomTree();
327  ScalarizerVisitor Impl(ParallelLoopAccessMDKind, DT);
328  return Impl.visit(F);
329 }
330 
332  return new ScalarizerLegacyPass();
333 }
334 
335 bool ScalarizerVisitor::visit(Function &F) {
336  assert(Gathered.empty() && Scattered.empty());
337 
338  // To ensure we replace gathered components correctly we need to do an ordered
339  // traversal of the basic blocks in the function.
340  ReversePostOrderTraversal<BasicBlock *> RPOT(&F.getEntryBlock());
341  for (BasicBlock *BB : RPOT) {
342  for (BasicBlock::iterator II = BB->begin(), IE = BB->end(); II != IE;) {
343  Instruction *I = &*II;
344  bool Done = InstVisitor::visit(I);
345  ++II;
346  if (Done && I->getType()->isVoidTy())
347  I->eraseFromParent();
348  }
349  }
350  return finish();
351 }
352 
353 // Return a scattered form of V that can be accessed by Point. V must be a
354 // vector or a pointer to a vector.
355 Scatterer ScalarizerVisitor::scatter(Instruction *Point, Value *V) {
356  if (Argument *VArg = dyn_cast<Argument>(V)) {
357  // Put the scattered form of arguments in the entry block,
358  // so that it can be used everywhere.
359  Function *F = VArg->getParent();
360  BasicBlock *BB = &F->getEntryBlock();
361  return Scatterer(BB, BB->begin(), V, &Scattered[V]);
362  }
363  if (Instruction *VOp = dyn_cast<Instruction>(V)) {
364  // When scalarizing PHI nodes we might try to examine/rewrite InsertElement
365  // nodes in predecessors. If those predecessors are unreachable from entry,
366  // then the IR in those blocks could have unexpected properties resulting in
367  // infinite loops in Scatterer::operator[]. By simply treating values
368  // originating from instructions in unreachable blocks as undef we do not
369  // need to analyse them further.
370  if (!DT->isReachableFromEntry(VOp->getParent()))
371  return Scatterer(Point->getParent(), Point->getIterator(),
372  UndefValue::get(V->getType()));
373  // Put the scattered form of an instruction directly after the
374  // instruction.
375  BasicBlock *BB = VOp->getParent();
376  return Scatterer(BB, std::next(BasicBlock::iterator(VOp)),
377  V, &Scattered[V]);
378  }
379  // In the fallback case, just put the scattered before Point and
380  // keep the result local to Point.
381  return Scatterer(Point->getParent(), Point->getIterator(), V);
382 }
383 
384 // Replace Op with the gathered form of the components in CV. Defer the
385 // deletion of Op and creation of the gathered form to the end of the pass,
386 // so that we can avoid creating the gathered form if all uses of Op are
387 // replaced with uses of CV.
388 void ScalarizerVisitor::gather(Instruction *Op, const ValueVector &CV) {
389  transferMetadataAndIRFlags(Op, CV);
390 
391  // If we already have a scattered form of Op (created from ExtractElements
392  // of Op itself), replace them with the new form.
393  ValueVector &SV = Scattered[Op];
394  if (!SV.empty()) {
395  for (unsigned I = 0, E = SV.size(); I != E; ++I) {
396  Value *V = SV[I];
397  if (V == nullptr || SV[I] == CV[I])
398  continue;
399 
400  Instruction *Old = cast<Instruction>(V);
401  if (isa<Instruction>(CV[I]))
402  CV[I]->takeName(Old);
403  Old->replaceAllUsesWith(CV[I]);
404  PotentiallyDeadInstrs.emplace_back(Old);
405  }
406  }
407  SV = CV;
408  Gathered.push_back(GatherList::value_type(Op, &SV));
409 }
410 
411 // Return true if it is safe to transfer the given metadata tag from
412 // vector to scalar instructions.
413 bool ScalarizerVisitor::canTransferMetadata(unsigned Tag) {
414  return (Tag == LLVMContext::MD_tbaa
415  || Tag == LLVMContext::MD_fpmath
416  || Tag == LLVMContext::MD_tbaa_struct
417  || Tag == LLVMContext::MD_invariant_load
418  || Tag == LLVMContext::MD_alias_scope
419  || Tag == LLVMContext::MD_noalias
420  || Tag == ParallelLoopAccessMDKind
421  || Tag == LLVMContext::MD_access_group);
422 }
423 
424 // Transfer metadata from Op to the instructions in CV if it is known
425 // to be safe to do so.
426 void ScalarizerVisitor::transferMetadataAndIRFlags(Instruction *Op,
427  const ValueVector &CV) {
429  Op->getAllMetadataOtherThanDebugLoc(MDs);
430  for (unsigned I = 0, E = CV.size(); I != E; ++I) {
431  if (Instruction *New = dyn_cast<Instruction>(CV[I])) {
432  for (const auto &MD : MDs)
433  if (canTransferMetadata(MD.first))
434  New->setMetadata(MD.first, MD.second);
435  New->copyIRFlags(Op);
436  if (Op->getDebugLoc() && !New->getDebugLoc())
437  New->setDebugLoc(Op->getDebugLoc());
438  }
439  }
440 }
441 
442 // Try to fill in Layout from Ty, returning true on success. Alignment is
443 // the alignment of the vector, or None if the ABI default should be used.
445 ScalarizerVisitor::getVectorLayout(Type *Ty, Align Alignment,
446  const DataLayout &DL) {
447  VectorLayout Layout;
448  // Make sure we're dealing with a vector.
449  Layout.VecTy = dyn_cast<VectorType>(Ty);
450  if (!Layout.VecTy)
451  return None;
452  // Check that we're dealing with full-byte elements.
453  Layout.ElemTy = Layout.VecTy->getElementType();
454  if (!DL.typeSizeEqualsStoreSize(Layout.ElemTy))
455  return None;
456  Layout.VecAlign = Alignment;
457  Layout.ElemSize = DL.getTypeStoreSize(Layout.ElemTy);
458  return Layout;
459 }
460 
461 // Scalarize one-operand instruction I, using Split(Builder, X, Name)
462 // to create an instruction like I with operand X and name Name.
463 template<typename Splitter>
464 bool ScalarizerVisitor::splitUnary(Instruction &I, const Splitter &Split) {
465  VectorType *VT = dyn_cast<VectorType>(I.getType());
466  if (!VT)
467  return false;
468 
469  unsigned NumElems = cast<FixedVectorType>(VT)->getNumElements();
471  Scatterer Op = scatter(&I, I.getOperand(0));
472  assert(Op.size() == NumElems && "Mismatched unary operation");
473  ValueVector Res;
474  Res.resize(NumElems);
475  for (unsigned Elem = 0; Elem < NumElems; ++Elem)
476  Res[Elem] = Split(Builder, Op[Elem], I.getName() + ".i" + Twine(Elem));
477  gather(&I, Res);
478  return true;
479 }
480 
481 // Scalarize two-operand instruction I, using Split(Builder, X, Y, Name)
482 // to create an instruction like I with operands X and Y and name Name.
483 template<typename Splitter>
484 bool ScalarizerVisitor::splitBinary(Instruction &I, const Splitter &Split) {
485  VectorType *VT = dyn_cast<VectorType>(I.getType());
486  if (!VT)
487  return false;
488 
489  unsigned NumElems = cast<FixedVectorType>(VT)->getNumElements();
491  Scatterer VOp0 = scatter(&I, I.getOperand(0));
492  Scatterer VOp1 = scatter(&I, I.getOperand(1));
493  assert(VOp0.size() == NumElems && "Mismatched binary operation");
494  assert(VOp1.size() == NumElems && "Mismatched binary operation");
495  ValueVector Res;
496  Res.resize(NumElems);
497  for (unsigned Elem = 0; Elem < NumElems; ++Elem) {
498  Value *Op0 = VOp0[Elem];
499  Value *Op1 = VOp1[Elem];
500  Res[Elem] = Split(Builder, Op0, Op1, I.getName() + ".i" + Twine(Elem));
501  }
502  gather(&I, Res);
503  return true;
504 }
505 
507  return isTriviallyVectorizable(ID);
508 }
509 
510 // All of the current scalarizable intrinsics only have one mangled type.
513  VectorType *Ty) {
514  return Intrinsic::getDeclaration(M, ID, { Ty->getScalarType() });
515 }
516 
517 /// If a call to a vector typed intrinsic function, split into a scalar call per
518 /// element if possible for the intrinsic.
519 bool ScalarizerVisitor::splitCall(CallInst &CI) {
520  VectorType *VT = dyn_cast<VectorType>(CI.getType());
521  if (!VT)
522  return false;
523 
524  Function *F = CI.getCalledFunction();
525  if (!F)
526  return false;
527 
528  Intrinsic::ID ID = F->getIntrinsicID();
530  return false;
531 
532  unsigned NumElems = cast<FixedVectorType>(VT)->getNumElements();
533  unsigned NumArgs = CI.getNumArgOperands();
534 
535  ValueVector ScalarOperands(NumArgs);
536  SmallVector<Scatterer, 8> Scattered(NumArgs);
537 
538  Scattered.resize(NumArgs);
539 
540  // Assumes that any vector type has the same number of elements as the return
541  // vector type, which is true for all current intrinsics.
542  for (unsigned I = 0; I != NumArgs; ++I) {
543  Value *OpI = CI.getOperand(I);
544  if (OpI->getType()->isVectorTy()) {
545  Scattered[I] = scatter(&CI, OpI);
546  assert(Scattered[I].size() == NumElems && "mismatched call operands");
547  } else {
548  ScalarOperands[I] = OpI;
549  }
550  }
551 
552  ValueVector Res(NumElems);
553  ValueVector ScalarCallOps(NumArgs);
554 
555  Function *NewIntrin = getScalarIntrinsicDeclaration(F->getParent(), ID, VT);
556  IRBuilder<> Builder(&CI);
557 
558  // Perform actual scalarization, taking care to preserve any scalar operands.
559  for (unsigned Elem = 0; Elem < NumElems; ++Elem) {
560  ScalarCallOps.clear();
561 
562  for (unsigned J = 0; J != NumArgs; ++J) {
564  ScalarCallOps.push_back(ScalarOperands[J]);
565  else
566  ScalarCallOps.push_back(Scattered[J][Elem]);
567  }
568 
569  Res[Elem] = Builder.CreateCall(NewIntrin, ScalarCallOps,
570  CI.getName() + ".i" + Twine(Elem));
571  }
572 
573  gather(&CI, Res);
574  return true;
575 }
576 
577 bool ScalarizerVisitor::visitSelectInst(SelectInst &SI) {
578  VectorType *VT = dyn_cast<VectorType>(SI.getType());
579  if (!VT)
580  return false;
581 
582  unsigned NumElems = cast<FixedVectorType>(VT)->getNumElements();
584  Scatterer VOp1 = scatter(&SI, SI.getOperand(1));
585  Scatterer VOp2 = scatter(&SI, SI.getOperand(2));
586  assert(VOp1.size() == NumElems && "Mismatched select");
587  assert(VOp2.size() == NumElems && "Mismatched select");
588  ValueVector Res;
589  Res.resize(NumElems);
590 
591  if (SI.getOperand(0)->getType()->isVectorTy()) {
592  Scatterer VOp0 = scatter(&SI, SI.getOperand(0));
593  assert(VOp0.size() == NumElems && "Mismatched select");
594  for (unsigned I = 0; I < NumElems; ++I) {
595  Value *Op0 = VOp0[I];
596  Value *Op1 = VOp1[I];
597  Value *Op2 = VOp2[I];
598  Res[I] = Builder.CreateSelect(Op0, Op1, Op2,
599  SI.getName() + ".i" + Twine(I));
600  }
601  } else {
602  Value *Op0 = SI.getOperand(0);
603  for (unsigned I = 0; I < NumElems; ++I) {
604  Value *Op1 = VOp1[I];
605  Value *Op2 = VOp2[I];
606  Res[I] = Builder.CreateSelect(Op0, Op1, Op2,
607  SI.getName() + ".i" + Twine(I));
608  }
609  }
610  gather(&SI, Res);
611  return true;
612 }
613 
614 bool ScalarizerVisitor::visitICmpInst(ICmpInst &ICI) {
615  return splitBinary(ICI, ICmpSplitter(ICI));
616 }
617 
618 bool ScalarizerVisitor::visitFCmpInst(FCmpInst &FCI) {
619  return splitBinary(FCI, FCmpSplitter(FCI));
620 }
621 
622 bool ScalarizerVisitor::visitUnaryOperator(UnaryOperator &UO) {
623  return splitUnary(UO, UnarySplitter(UO));
624 }
625 
626 bool ScalarizerVisitor::visitBinaryOperator(BinaryOperator &BO) {
627  return splitBinary(BO, BinarySplitter(BO));
628 }
629 
630 bool ScalarizerVisitor::visitGetElementPtrInst(GetElementPtrInst &GEPI) {
631  VectorType *VT = dyn_cast<VectorType>(GEPI.getType());
632  if (!VT)
633  return false;
634 
635  IRBuilder<> Builder(&GEPI);
636  unsigned NumElems = cast<FixedVectorType>(VT)->getNumElements();
637  unsigned NumIndices = GEPI.getNumIndices();
638 
639  // The base pointer might be scalar even if it's a vector GEP. In those cases,
640  // splat the pointer into a vector value, and scatter that vector.
641  Value *Op0 = GEPI.getOperand(0);
642  if (!Op0->getType()->isVectorTy())
643  Op0 = Builder.CreateVectorSplat(NumElems, Op0);
644  Scatterer Base = scatter(&GEPI, Op0);
645 
647  Ops.resize(NumIndices);
648  for (unsigned I = 0; I < NumIndices; ++I) {
649  Value *Op = GEPI.getOperand(I + 1);
650 
651  // The indices might be scalars even if it's a vector GEP. In those cases,
652  // splat the scalar into a vector value, and scatter that vector.
653  if (!Op->getType()->isVectorTy())
654  Op = Builder.CreateVectorSplat(NumElems, Op);
655 
656  Ops[I] = scatter(&GEPI, Op);
657  }
658 
659  ValueVector Res;
660  Res.resize(NumElems);
661  for (unsigned I = 0; I < NumElems; ++I) {
662  SmallVector<Value *, 8> Indices;
663  Indices.resize(NumIndices);
664  for (unsigned J = 0; J < NumIndices; ++J)
665  Indices[J] = Ops[J][I];
666  Res[I] = Builder.CreateGEP(GEPI.getSourceElementType(), Base[I], Indices,
667  GEPI.getName() + ".i" + Twine(I));
668  if (GEPI.isInBounds())
669  if (GetElementPtrInst *NewGEPI = dyn_cast<GetElementPtrInst>(Res[I]))
670  NewGEPI->setIsInBounds();
671  }
672  gather(&GEPI, Res);
673  return true;
674 }
675 
676 bool ScalarizerVisitor::visitCastInst(CastInst &CI) {
677  VectorType *VT = dyn_cast<VectorType>(CI.getDestTy());
678  if (!VT)
679  return false;
680 
681  unsigned NumElems = cast<FixedVectorType>(VT)->getNumElements();
682  IRBuilder<> Builder(&CI);
683  Scatterer Op0 = scatter(&CI, CI.getOperand(0));
684  assert(Op0.size() == NumElems && "Mismatched cast");
685  ValueVector Res;
686  Res.resize(NumElems);
687  for (unsigned I = 0; I < NumElems; ++I)
688  Res[I] = Builder.CreateCast(CI.getOpcode(), Op0[I], VT->getElementType(),
689  CI.getName() + ".i" + Twine(I));
690  gather(&CI, Res);
691  return true;
692 }
693 
694 bool ScalarizerVisitor::visitBitCastInst(BitCastInst &BCI) {
695  VectorType *DstVT = dyn_cast<VectorType>(BCI.getDestTy());
696  VectorType *SrcVT = dyn_cast<VectorType>(BCI.getSrcTy());
697  if (!DstVT || !SrcVT)
698  return false;
699 
700  unsigned DstNumElems = cast<FixedVectorType>(DstVT)->getNumElements();
701  unsigned SrcNumElems = cast<FixedVectorType>(SrcVT)->getNumElements();
702  IRBuilder<> Builder(&BCI);
703  Scatterer Op0 = scatter(&BCI, BCI.getOperand(0));
704  ValueVector Res;
705  Res.resize(DstNumElems);
706 
707  if (DstNumElems == SrcNumElems) {
708  for (unsigned I = 0; I < DstNumElems; ++I)
709  Res[I] = Builder.CreateBitCast(Op0[I], DstVT->getElementType(),
710  BCI.getName() + ".i" + Twine(I));
711  } else if (DstNumElems > SrcNumElems) {
712  // <M x t1> -> <N*M x t2>. Convert each t1 to <N x t2> and copy the
713  // individual elements to the destination.
714  unsigned FanOut = DstNumElems / SrcNumElems;
715  auto *MidTy = FixedVectorType::get(DstVT->getElementType(), FanOut);
716  unsigned ResI = 0;
717  for (unsigned Op0I = 0; Op0I < SrcNumElems; ++Op0I) {
718  Value *V = Op0[Op0I];
719  Instruction *VI;
720  // Look through any existing bitcasts before converting to <N x t2>.
721  // In the best case, the resulting conversion might be a no-op.
722  while ((VI = dyn_cast<Instruction>(V)) &&
723  VI->getOpcode() == Instruction::BitCast)
724  V = VI->getOperand(0);
725  V = Builder.CreateBitCast(V, MidTy, V->getName() + ".cast");
726  Scatterer Mid = scatter(&BCI, V);
727  for (unsigned MidI = 0; MidI < FanOut; ++MidI)
728  Res[ResI++] = Mid[MidI];
729  }
730  } else {
731  // <N*M x t1> -> <M x t2>. Convert each group of <N x t1> into a t2.
732  unsigned FanIn = SrcNumElems / DstNumElems;
733  auto *MidTy = FixedVectorType::get(SrcVT->getElementType(), FanIn);
734  unsigned Op0I = 0;
735  for (unsigned ResI = 0; ResI < DstNumElems; ++ResI) {
736  Value *V = PoisonValue::get(MidTy);
737  for (unsigned MidI = 0; MidI < FanIn; ++MidI)
738  V = Builder.CreateInsertElement(V, Op0[Op0I++], Builder.getInt32(MidI),
739  BCI.getName() + ".i" + Twine(ResI)
740  + ".upto" + Twine(MidI));
741  Res[ResI] = Builder.CreateBitCast(V, DstVT->getElementType(),
742  BCI.getName() + ".i" + Twine(ResI));
743  }
744  }
745  gather(&BCI, Res);
746  return true;
747 }
748 
749 bool ScalarizerVisitor::visitInsertElementInst(InsertElementInst &IEI) {
750  VectorType *VT = dyn_cast<VectorType>(IEI.getType());
751  if (!VT)
752  return false;
753 
754  unsigned NumElems = cast<FixedVectorType>(VT)->getNumElements();
755  IRBuilder<> Builder(&IEI);
756  Scatterer Op0 = scatter(&IEI, IEI.getOperand(0));
757  Value *NewElt = IEI.getOperand(1);
758  Value *InsIdx = IEI.getOperand(2);
759 
760  ValueVector Res;
761  Res.resize(NumElems);
762 
763  if (auto *CI = dyn_cast<ConstantInt>(InsIdx)) {
764  for (unsigned I = 0; I < NumElems; ++I)
765  Res[I] = CI->getValue().getZExtValue() == I ? NewElt : Op0[I];
766  } else {
768  return false;
769 
770  for (unsigned I = 0; I < NumElems; ++I) {
771  Value *ShouldReplace =
772  Builder.CreateICmpEQ(InsIdx, ConstantInt::get(InsIdx->getType(), I),
773  InsIdx->getName() + ".is." + Twine(I));
774  Value *OldElt = Op0[I];
775  Res[I] = Builder.CreateSelect(ShouldReplace, NewElt, OldElt,
776  IEI.getName() + ".i" + Twine(I));
777  }
778  }
779 
780  gather(&IEI, Res);
781  return true;
782 }
783 
784 bool ScalarizerVisitor::visitExtractElementInst(ExtractElementInst &EEI) {
785  VectorType *VT = dyn_cast<VectorType>(EEI.getOperand(0)->getType());
786  if (!VT)
787  return false;
788 
789  unsigned NumSrcElems = cast<FixedVectorType>(VT)->getNumElements();
790  IRBuilder<> Builder(&EEI);
791  Scatterer Op0 = scatter(&EEI, EEI.getOperand(0));
792  Value *ExtIdx = EEI.getOperand(1);
793 
794  if (auto *CI = dyn_cast<ConstantInt>(ExtIdx)) {
795  Value *Res = Op0[CI->getValue().getZExtValue()];
796  gather(&EEI, {Res});
797  return true;
798  }
799 
801  return false;
802 
803  Value *Res = UndefValue::get(VT->getElementType());
804  for (unsigned I = 0; I < NumSrcElems; ++I) {
805  Value *ShouldExtract =
806  Builder.CreateICmpEQ(ExtIdx, ConstantInt::get(ExtIdx->getType(), I),
807  ExtIdx->getName() + ".is." + Twine(I));
808  Value *Elt = Op0[I];
809  Res = Builder.CreateSelect(ShouldExtract, Elt, Res,
810  EEI.getName() + ".upto" + Twine(I));
811  }
812  gather(&EEI, {Res});
813  return true;
814 }
815 
816 bool ScalarizerVisitor::visitShuffleVectorInst(ShuffleVectorInst &SVI) {
817  VectorType *VT = dyn_cast<VectorType>(SVI.getType());
818  if (!VT)
819  return false;
820 
821  unsigned NumElems = cast<FixedVectorType>(VT)->getNumElements();
822  Scatterer Op0 = scatter(&SVI, SVI.getOperand(0));
823  Scatterer Op1 = scatter(&SVI, SVI.getOperand(1));
824  ValueVector Res;
825  Res.resize(NumElems);
826 
827  for (unsigned I = 0; I < NumElems; ++I) {
828  int Selector = SVI.getMaskValue(I);
829  if (Selector < 0)
830  Res[I] = UndefValue::get(VT->getElementType());
831  else if (unsigned(Selector) < Op0.size())
832  Res[I] = Op0[Selector];
833  else
834  Res[I] = Op1[Selector - Op0.size()];
835  }
836  gather(&SVI, Res);
837  return true;
838 }
839 
840 bool ScalarizerVisitor::visitPHINode(PHINode &PHI) {
841  VectorType *VT = dyn_cast<VectorType>(PHI.getType());
842  if (!VT)
843  return false;
844 
845  unsigned NumElems = cast<FixedVectorType>(VT)->getNumElements();
846  IRBuilder<> Builder(&PHI);
847  ValueVector Res;
848  Res.resize(NumElems);
849 
850  unsigned NumOps = PHI.getNumOperands();
851  for (unsigned I = 0; I < NumElems; ++I)
852  Res[I] = Builder.CreatePHI(VT->getElementType(), NumOps,
853  PHI.getName() + ".i" + Twine(I));
854 
855  for (unsigned I = 0; I < NumOps; ++I) {
856  Scatterer Op = scatter(&PHI, PHI.getIncomingValue(I));
857  BasicBlock *IncomingBlock = PHI.getIncomingBlock(I);
858  for (unsigned J = 0; J < NumElems; ++J)
859  cast<PHINode>(Res[J])->addIncoming(Op[J], IncomingBlock);
860  }
861  gather(&PHI, Res);
862  return true;
863 }
864 
865 bool ScalarizerVisitor::visitLoadInst(LoadInst &LI) {
866  if (!ScalarizeLoadStore)
867  return false;
868  if (!LI.isSimple())
869  return false;
870 
871  Optional<VectorLayout> Layout = getVectorLayout(
872  LI.getType(), LI.getAlign(), LI.getModule()->getDataLayout());
873  if (!Layout)
874  return false;
875 
876  unsigned NumElems = cast<FixedVectorType>(Layout->VecTy)->getNumElements();
877  IRBuilder<> Builder(&LI);
878  Scatterer Ptr = scatter(&LI, LI.getPointerOperand());
879  ValueVector Res;
880  Res.resize(NumElems);
881 
882  for (unsigned I = 0; I < NumElems; ++I)
883  Res[I] = Builder.CreateAlignedLoad(Layout->VecTy->getElementType(), Ptr[I],
884  Align(Layout->getElemAlign(I)),
885  LI.getName() + ".i" + Twine(I));
886  gather(&LI, Res);
887  return true;
888 }
889 
890 bool ScalarizerVisitor::visitStoreInst(StoreInst &SI) {
891  if (!ScalarizeLoadStore)
892  return false;
893  if (!SI.isSimple())
894  return false;
895 
896  Value *FullValue = SI.getValueOperand();
897  Optional<VectorLayout> Layout = getVectorLayout(
898  FullValue->getType(), SI.getAlign(), SI.getModule()->getDataLayout());
899  if (!Layout)
900  return false;
901 
902  unsigned NumElems = cast<FixedVectorType>(Layout->VecTy)->getNumElements();
904  Scatterer VPtr = scatter(&SI, SI.getPointerOperand());
905  Scatterer VVal = scatter(&SI, FullValue);
906 
907  ValueVector Stores;
908  Stores.resize(NumElems);
909  for (unsigned I = 0; I < NumElems; ++I) {
910  Value *Val = VVal[I];
911  Value *Ptr = VPtr[I];
912  Stores[I] = Builder.CreateAlignedStore(Val, Ptr, Layout->getElemAlign(I));
913  }
914  transferMetadataAndIRFlags(&SI, Stores);
915  return true;
916 }
917 
918 bool ScalarizerVisitor::visitCallInst(CallInst &CI) {
919  return splitCall(CI);
920 }
921 
922 // Delete the instructions that we scalarized. If a full vector result
923 // is still needed, recreate it using InsertElements.
924 bool ScalarizerVisitor::finish() {
925  // The presence of data in Gathered or Scattered indicates changes
926  // made to the Function.
927  if (Gathered.empty() && Scattered.empty())
928  return false;
929  for (const auto &GMI : Gathered) {
930  Instruction *Op = GMI.first;
931  ValueVector &CV = *GMI.second;
932  if (!Op->use_empty()) {
933  // The value is still needed, so recreate it using a series of
934  // InsertElements.
935  Value *Res = PoisonValue::get(Op->getType());
936  if (auto *Ty = dyn_cast<VectorType>(Op->getType())) {
937  BasicBlock *BB = Op->getParent();
938  unsigned Count = cast<FixedVectorType>(Ty)->getNumElements();
940  if (isa<PHINode>(Op))
941  Builder.SetInsertPoint(BB, BB->getFirstInsertionPt());
942  for (unsigned I = 0; I < Count; ++I)
943  Res = Builder.CreateInsertElement(Res, CV[I], Builder.getInt32(I),
944  Op->getName() + ".upto" + Twine(I));
945  Res->takeName(Op);
946  } else {
947  assert(CV.size() == 1 && Op->getType() == CV[0]->getType());
948  Res = CV[0];
949  if (Op == Res)
950  continue;
951  }
952  Op->replaceAllUsesWith(Res);
953  }
954  PotentiallyDeadInstrs.emplace_back(Op);
955  }
956  Gathered.clear();
957  Scattered.clear();
958 
960 
961  return true;
962 }
963 
965  Module &M = *F.getParent();
966  unsigned ParallelLoopAccessMDKind =
967  M.getContext().getMDKindID("llvm.mem.parallel_loop_access");
969  ScalarizerVisitor Impl(ParallelLoopAccessMDKind, DT);
970  bool Changed = Impl.visit(F);
973  return Changed ? PA : PreservedAnalyses::all();
974 }
llvm::Check::Size
@ Size
Definition: FileCheck.h:73
llvm::PreservedAnalyses
A set of analyses that are preserved following a run of a transformation pass.
Definition: PassManager.h:155
llvm::EngineKind::Kind
Kind
Definition: ExecutionEngine.h:524
llvm::Argument
This class represents an incoming formal argument to a Function.
Definition: Argument.h:29
MathExtras.h
llvm
Definition: AllocatorList.h:23
M
We currently emits eax Perhaps this is what we really should generate is Is imull three or four cycles eax eax The current instruction priority is based on pattern complexity The former is more complex because it folds a load so the latter will not be emitted Perhaps we should use AddedComplexity to give LEA32r a higher priority We should always try to match LEA first since the LEA matching code does some estimate to determine whether the match is profitable if we care more about code then imull is better It s two bytes shorter than movl leal On a Pentium M
Definition: README.txt:252
llvm::Instruction::getModule
const Module * getModule() const
Return the module owning the function this instruction belongs to or nullptr it the function does not...
Definition: Instruction.cpp:66
llvm::DataLayout
A parsed version of the target data layout string in and methods for querying it.
Definition: DataLayout.h:112
llvm::Intrinsic::getDeclaration
Function * getDeclaration(Module *M, ID id, ArrayRef< Type * > Tys=None)
Create or insert an LLVM Function declaration for an intrinsic, and return it.
Definition: Function.cpp:1295
Insert
Vector Rotate Left Mask Mask Insert
Definition: README_P9.txt:112
llvm::BasicBlock::iterator
InstListType::iterator iterator
Instruction iterators...
Definition: BasicBlock.h:90
llvm::AnalysisManager::getResult
PassT::Result & getResult(IRUnitT &IR, ExtraArgTs... ExtraArgs)
Get the result of an analysis pass for a given IR unit.
Definition: PassManager.h:785
Scalar.h
llvm::ExtractElementInst
This instruction extracts a single (scalar) element from a VectorType value.
Definition: Instructions.h:1850
llvm::Function
Definition: Function.h:61
Pass.h
llvm::PointerType::get
static PointerType * get(Type *ElementType, unsigned AddressSpace)
This constructs a pointer to an object of the specified type in a numbered address space.
Definition: Type.cpp:693
llvm::BitCastInst
This class represents a no-op cast from one type to another.
Definition: Instructions.h:5136
llvm::Type::getScalarType
Type * getScalarType() const
If this is a vector type, return the element type, otherwise return 'this'.
Definition: Type.h:317
llvm::SmallVector< Value *, 8 >
llvm::IRBuilder<>
Local.h
llvm::DominatorTree
Concrete subclass of DominatorTreeBase that is used to compute a normal dominator tree.
Definition: Dominators.h:151
llvm::cl::Hidden
@ Hidden
Definition: CommandLine.h:140
llvm::Type
The instances of the Type class are immutable: once they are created, they are never changed.
Definition: Type.h:46
Module.h
llvm::Optional
Definition: APInt.h:34
llvm::GetElementPtrInst::getNumIndices
unsigned getNumIndices() const
Definition: Instructions.h:1091
llvm::VectorType::getElementType
Type * getElementType() const
Definition: DerivedTypes.h:424
llvm::Intrinsic::not_intrinsic
@ not_intrinsic
Definition: Intrinsics.h:45
llvm::CallBase::getNumArgOperands
unsigned getNumArgOperands() const
Definition: InstrTypes.h:1339
llvm::LoadInst::getPointerOperand
Value * getPointerOperand()
Definition: Instructions.h:266
llvm::UnaryOperator
Definition: InstrTypes.h:103
llvm::LoadInst::getAlign
Align getAlign() const
Return the alignment of the access that is being performed.
Definition: Instructions.h:222
INITIALIZE_PASS_END
INITIALIZE_PASS_END(RegBankSelect, DEBUG_TYPE, "Assign register bank of generic virtual registers", false, false) RegBankSelect
Definition: RegBankSelect.cpp:69
llvm::CastInst::getDestTy
Type * getDestTy() const
Return the destination type, as a convenience.
Definition: InstrTypes.h:686
llvm::GetElementPtrInst::getSourceElementType
Type * getSourceElementType() const
Definition: Instructions.h:1002
F
#define F(x, y, z)
Definition: MD5.cpp:56
llvm::BasicBlock
LLVM Basic Block Representation.
Definition: BasicBlock.h:58
Scalarizer.h
llvm::GetElementPtrInst::isInBounds
bool isInBounds() const
Determine whether the GEP has the inbounds flag.
Definition: Instructions.cpp:1799
Instruction.h
CommandLine.h
operations
Scalarize vector operations
Definition: Scalarizer.cpp:256
llvm::ConstantInt
This is the shared class of boolean and integer constants.
Definition: Constants.h:77
bb
< i1 > br i1 label label bb bb
Definition: README.txt:978
llvm::AArch64Layout::VectorLayout
VectorLayout
Definition: AArch64BaseInfo.h:486
llvm::PassRegistry::getPassRegistry
static PassRegistry * getPassRegistry()
getPassRegistry - Access the global registry object, which is automatically initialized at applicatio...
Definition: PassRegistry.cpp:31
Constants.h
llvm::PHINode::getIncomingValue
Value * getIncomingValue(unsigned i) const
Return incoming value number x.
Definition: Instructions.h:2666
E
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
llvm::isTriviallyVectorizable
bool isTriviallyVectorizable(Intrinsic::ID ID)
Identify if the intrinsic is trivially vectorizable.
Definition: VectorUtils.cpp:44
Intrinsics.h
Twine.h
InstrTypes.h
llvm::CallBase::getCalledFunction
Function * getCalledFunction() const
Returns the function called, or null if this is an indirect function invocation.
Definition: InstrTypes.h:1396
llvm::FCmpInst
This instruction compares its operands according to the predicate given to the constructor.
Definition: Instructions.h:1344
SI
@ SI
Definition: SIInstrInfo.cpp:7342
llvm::AnalysisUsage
Represent the analysis usage information of a pass.
Definition: PassAnalysisSupport.h:47
llvm::Type::isVectorTy
bool isVectorTy() const
True if this is an instance of VectorType.
Definition: Type.h:235
llvm::InsertElementInst
This instruction inserts a single (scalar) element into a VectorType value.
Definition: Instructions.h:1914
false
Definition: StackSlotColoring.cpp:142
llvm::ShuffleVectorInst::getMaskValue
int getMaskValue(unsigned Elt) const
Return the shuffle mask value of this instruction for the given element index.
Definition: Instructions.h:2032
llvm::Instruction
Definition: Instruction.h:45
llvm::DominatorTreeWrapperPass
Legacy analysis pass which computes a DominatorTree.
Definition: Dominators.h:281
llvm::SmallVectorImpl::resize
void resize(size_type N)
Definition: SmallVector.h:606
llvm::UndefValue::get
static UndefValue * get(Type *T)
Static factory methods - Return an 'undef' object of the specified type.
Definition: Constants.cpp:1770
llvm::ConstantInt::get
static Constant * get(Type *Ty, uint64_t V, bool IsSigned=false)
If Ty is a vector type, return a Constant with a splat of the given value.
Definition: Constants.cpp:885
llvm::ScalarizerPass::run
PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM)
Definition: Scalarizer.cpp:964
llvm::FixedVectorType::get
static FixedVectorType * get(Type *ElementType, unsigned NumElts)
Definition: Type.cpp:650
llvm::Align
This struct is a compact representation of a valid (non-zero power of two) alignment.
Definition: Alignment.h:39
llvm::CastInst::getSrcTy
Type * getSrcTy() const
Return the source type, as a convenience.
Definition: InstrTypes.h:684
llvm::None
const NoneType None
Definition: None.h:23
llvm::ShuffleVectorInst::getType
VectorType * getType() const
Overload to return most specific vector type.
Definition: Instructions.h:2023
Type.h
llvm::ARM_PROC::IE
@ IE
Definition: ARMBaseInfo.h:27
llvm::VectorType
Base class of all SIMD vector types.
Definition: DerivedTypes.h:391
VectorUtils.h
BasicBlock.h
llvm::cl::opt< bool >
llvm::StoreInst
An instruction for storing to memory.
Definition: Instructions.h:303
VI
@ VI
Definition: SIInstrInfo.cpp:7343
ScalarizeLoadStore
static cl::opt< bool > ScalarizeLoadStore("scalarize-load-store", cl::init(false), cl::Hidden, cl::desc("Allow the scalarizer pass to scalarize loads and store"))
llvm::ICmpInst
This instruction compares its operands according to the predicate given to the constructor.
Definition: Instructions.h:1178
llvm::PreservedAnalyses::preserve
void preserve()
Mark an analysis as preserved.
Definition: PassManager.h:176
INITIALIZE_PASS_DEPENDENCY
INITIALIZE_PASS_DEPENDENCY(DominatorTreeWrapperPass)
I
#define I(x, y, z)
Definition: MD5.cpp:59
llvm::GetElementPtrInst
an instruction for type-safe pointer arithmetic to access elements of arrays and structs
Definition: Instructions.h:905
llvm::hasVectorInstrinsicScalarOpd
bool hasVectorInstrinsicScalarOpd(Intrinsic::ID ID, unsigned ScalarOpdIdx)
Identifies if the vector form of the intrinsic has a scalar operand.
Definition: VectorUtils.cpp:99
llvm::cl::init
initializer< Ty > init(const Ty &Val)
Definition: CommandLine.h:440
llvm::PointerType
Class to represent pointers.
Definition: DerivedTypes.h:634
llvm::InstVisitor::visit
void visit(Iterator Start, Iterator End)
Definition: InstVisitor.h:88
IRBuilder.h
assert
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
llvm::createScalarizerPass
FunctionPass * createScalarizerPass()
Create a legacy pass manager instance of the Scalarizer pass.
Definition: Scalarizer.cpp:331
llvm::DominatorTree::isReachableFromEntry
bool isReachableFromEntry(const Use &U) const
Provide an overload for a Use.
Definition: Dominators.cpp:328
llvm::SelectInst
This class represents the LLVM 'select' instruction.
Definition: Instructions.h:1715
llvm::Module
A Module instance is used to store all the information related to an LLVM module.
Definition: Module.h:67
getScalarIntrinsicDeclaration
static Function * getScalarIntrinsicDeclaration(Module *M, Intrinsic::ID ID, VectorType *Ty)
Definition: Scalarizer.cpp:511
Builder
assume Assume Builder
Definition: AssumeBundleBuilder.cpp:649
llvm::size
auto size(R &&Range, std::enable_if_t< std::is_base_of< std::random_access_iterator_tag, typename std::iterator_traits< decltype(Range.begin())>::iterator_category >::value, void > *=nullptr)
Get the size of a range.
Definition: STLExtras.h:1486
llvm::BinaryOperator
Definition: InstrTypes.h:190
DataLayout.h
InstVisitor.h
llvm::LoadInst::isSimple
bool isSimple() const
Definition: Instructions.h:258
llvm::Value::getType
Type * getType() const
All values are typed, get the type of this value.
Definition: Value.h:256
llvm::AnalysisUsage::addPreserved
AnalysisUsage & addPreserved()
Add the specified Pass class to the set of analyses preserved by this pass.
Definition: PassAnalysisSupport.h:98
llvm::Value::replaceAllUsesWith
void replaceAllUsesWith(Value *V)
Change all uses of this to point to a new Value.
Definition: Value.cpp:527
llvm::ms_demangle::IntrinsicFunctionKind::New
@ New
llvm::ilist_node_impl::getIterator
self_iterator getIterator()
Definition: ilist_node.h:81
DL
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
Definition: AArch64SLSHardening.cpp:76
llvm::initializeScalarizerLegacyPassPass
void initializeScalarizerLegacyPassPass(PassRegistry &)
llvm::InstVisitor
Base class for instruction visitors.
Definition: InstVisitor.h:79
llvm::CastInst
This is the base class for all instructions that perform data casts.
Definition: InstrTypes.h:432
llvm::Value::getName
StringRef getName() const
Return a constant reference to the value's name.
Definition: Value.cpp:299
llvm::LoadInst
An instruction for reading from memory.
Definition: Instructions.h:174
Argument.h
llvm::ConstantInt::getZExtValue
uint64_t getZExtValue() const
Return the constant as a 64-bit unsigned integer value after it has been zero extended as appropriate...
Definition: Constants.h:140
runOnFunction
static bool runOnFunction(Function &F, bool PostInlining)
Definition: EntryExitInstrumenter.cpp:69
llvm::Twine
Twine - A lightweight data structure for efficiently representing the concatenation of temporary valu...
Definition: Twine.h:80
llvm::commonAlignment
Align commonAlignment(Align A, Align B)
Returns the alignment that satisfies both alignments.
Definition: Alignment.h:221
llvm::GraphProgram::Name
Name
Definition: GraphWriter.h:52
llvm::AMDGPU::SendMsg::Op
Op
Definition: SIDefines.h:314
llvm::PreservedAnalyses::all
static PreservedAnalyses all()
Construct a special preserved set that preserves all passes.
Definition: PassManager.h:161
Casting.h
isTriviallyScalariable
static bool isTriviallyScalariable(Intrinsic::ID ID)
Definition: Scalarizer.cpp:506
Function.h
INITIALIZE_PASS_BEGIN
INITIALIZE_PASS_BEGIN(ScalarizerLegacyPass, "scalarizer", "Scalarize vector operations", false, false) INITIALIZE_PASS_END(ScalarizerLegacyPass
llvm::ReversePostOrderTraversal
Definition: PostOrderIterator.h:290
scalarizer
scalarizer
Definition: Scalarizer.cpp:255
llvm::DominatorTreeAnalysis
Analysis pass which computes a DominatorTree.
Definition: Dominators.h:252
llvm::InsertElementInst::getType
VectorType * getType() const
Overload to return most specific vector type.
Definition: Instructions.h:1947
llvm::ShuffleVectorInst
This instruction constructs a fixed permutation of two input vectors.
Definition: Instructions.h:1986
Instructions.h
PostOrderIterator.h
llvm::User::getNumOperands
unsigned getNumOperands() const
Definition: User.h:191
SmallVector.h
Dominators.h
llvm::CastInst::getOpcode
Instruction::CastOps getOpcode() const
Return the opcode of this CastInst.
Definition: InstrTypes.h:679
llvm::Instruction::getParent
const BasicBlock * getParent() const
Definition: Instruction.h:94
llvm::PHINode::getIncomingBlock
BasicBlock * getIncomingBlock(unsigned i) const
Return incoming basic block number i.
Definition: Instructions.h:2686
llvm::PHINode
Definition: Instructions.h:2572
llvm::Module::getDataLayout
const DataLayout & getDataLayout() const
Get the data layout for the module's target platform.
Definition: Module.cpp:397
DerivedTypes.h
llvm::AnalysisManager
A container for analyses that lazily runs them and caches their results.
Definition: InstructionSimplify.h:44
llvm::FunctionPass
FunctionPass class - This class is used to implement most global optimizations.
Definition: Pass.h:298
llvm::CallInst
This class represents a function call, abstracting a target machine's calling convention.
Definition: Instructions.h:1450
BB
Common register allocation spilling lr str ldr sxth r3 ldr mla r4 can lr mov lr str ldr sxth r3 mla r4 and then merge mul and lr str ldr sxth r3 mla r4 It also increase the likelihood the store may become dead bb27 Successors according to LLVM BB
Definition: README.txt:39
llvm::AnalysisUsage::addRequired
AnalysisUsage & addRequired()
Definition: PassAnalysisSupport.h:75
LLVMContext.h
llvm::Value::takeName
void takeName(Value *V)
Transfer the name from V to this value.
Definition: Value.cpp:377
llvm::User::getOperand
Value * getOperand(unsigned i) const
Definition: User.h:169
llvm::cl::desc
Definition: CommandLine.h:411
Value.h
InitializePasses.h
llvm::RecursivelyDeleteTriviallyDeadInstructionsPermissive
bool RecursivelyDeleteTriviallyDeadInstructionsPermissive(SmallVectorImpl< WeakTrackingVH > &DeadInsts, const TargetLibraryInfo *TLI=nullptr, MemorySSAUpdater *MSSAU=nullptr, std::function< void(Value *)> AboutToDeleteCallback=std::function< void(Value *)>())
Same functionality as RecursivelyDeleteTriviallyDeadInstructions, but allow instructions that are not...
Definition: Local.cpp:501
llvm::Value
LLVM Value Representation.
Definition: Value.h:75
ScalarizeVariableInsertExtract
static cl::opt< bool > ScalarizeVariableInsertExtract("scalarize-variable-insert-extract", cl::init(true), cl::Hidden, cl::desc("Allow the scalarizer pass to scalarize " "insertelement/extractelement with variable index"))
llvm::Intrinsic::ID
unsigned ID
Definition: TargetTransformInfo.h:38
llvm::PoisonValue::get
static PoisonValue * get(Type *T)
Static factory methods - Return an 'poison' object of the specified type.
Definition: Constants.cpp:1789