LLVM  14.0.0git
Scalarizer.cpp
Go to the documentation of this file.
1 //===- Scalarizer.cpp - Scalarize vector operations -----------------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This pass converts vector operations into scalar operations, in order
10 // to expose optimization opportunities on the individual scalar operations.
11 // It is mainly intended for targets that do not have vector units, but it
12 // may also be useful for revectorizing code to different vector widths.
13 //
14 //===----------------------------------------------------------------------===//
15 
18 #include "llvm/ADT/SmallVector.h"
19 #include "llvm/ADT/Twine.h"
21 #include "llvm/IR/Argument.h"
22 #include "llvm/IR/BasicBlock.h"
23 #include "llvm/IR/Constants.h"
24 #include "llvm/IR/DataLayout.h"
25 #include "llvm/IR/DerivedTypes.h"
26 #include "llvm/IR/Dominators.h"
27 #include "llvm/IR/Function.h"
28 #include "llvm/IR/IRBuilder.h"
29 #include "llvm/IR/InstVisitor.h"
30 #include "llvm/IR/InstrTypes.h"
31 #include "llvm/IR/Instruction.h"
32 #include "llvm/IR/Instructions.h"
33 #include "llvm/IR/Intrinsics.h"
34 #include "llvm/IR/LLVMContext.h"
35 #include "llvm/IR/Module.h"
36 #include "llvm/IR/Type.h"
37 #include "llvm/IR/Value.h"
38 #include "llvm/InitializePasses.h"
39 #include "llvm/Pass.h"
40 #include "llvm/Support/Casting.h"
43 #include "llvm/Transforms/Scalar.h"
45 #include <cassert>
46 #include <cstdint>
47 #include <iterator>
48 #include <map>
49 #include <utility>
50 
51 using namespace llvm;
52 
53 #define DEBUG_TYPE "scalarizer"
54 
56  "scalarize-variable-insert-extract", cl::init(true), cl::Hidden,
57  cl::desc("Allow the scalarizer pass to scalarize "
58  "insertelement/extractelement with variable index"));
59 
60 // This is disabled by default because having separate loads and stores
61 // makes it more likely that the -combiner-alias-analysis limits will be
62 // reached.
63 static cl::opt<bool>
64  ScalarizeLoadStore("scalarize-load-store", cl::init(false), cl::Hidden,
65  cl::desc("Allow the scalarizer pass to scalarize loads and store"));
66 
67 namespace {
68 
69 // Used to store the scattered form of a vector.
70 using ValueVector = SmallVector<Value *, 8>;
71 
72 // Used to map a vector Value to its scattered form. We use std::map
73 // because we want iterators to persist across insertion and because the
74 // values are relatively large.
75 using ScatterMap = std::map<Value *, ValueVector>;
76 
77 // Lists Instructions that have been replaced with scalar implementations,
78 // along with a pointer to their scattered forms.
80 
81 // Provides a very limited vector-like interface for lazily accessing one
82 // component of a scattered vector or vector pointer.
83 class Scatterer {
84 public:
85  Scatterer() = default;
86 
87  // Scatter V into Size components. If new instructions are needed,
88  // insert them before BBI in BB. If Cache is nonnull, use it to cache
89  // the results.
90  Scatterer(BasicBlock *bb, BasicBlock::iterator bbi, Value *v,
91  ValueVector *cachePtr = nullptr);
92 
93  // Return component I, creating a new Value for it if necessary.
94  Value *operator[](unsigned I);
95 
96  // Return the number of components.
97  unsigned size() const { return Size; }
98 
99 private:
100  BasicBlock *BB;
102  Value *V;
103  ValueVector *CachePtr;
104  PointerType *PtrTy;
105  ValueVector Tmp;
106  unsigned Size;
107 };
108 
109 // FCmpSpliiter(FCI)(Builder, X, Y, Name) uses Builder to create an FCmp
110 // called Name that compares X and Y in the same way as FCI.
111 struct FCmpSplitter {
112  FCmpSplitter(FCmpInst &fci) : FCI(fci) {}
113 
114  Value *operator()(IRBuilder<> &Builder, Value *Op0, Value *Op1,
115  const Twine &Name) const {
116  return Builder.CreateFCmp(FCI.getPredicate(), Op0, Op1, Name);
117  }
118 
119  FCmpInst &FCI;
120 };
121 
122 // ICmpSpliiter(ICI)(Builder, X, Y, Name) uses Builder to create an ICmp
123 // called Name that compares X and Y in the same way as ICI.
124 struct ICmpSplitter {
125  ICmpSplitter(ICmpInst &ici) : ICI(ici) {}
126 
127  Value *operator()(IRBuilder<> &Builder, Value *Op0, Value *Op1,
128  const Twine &Name) const {
129  return Builder.CreateICmp(ICI.getPredicate(), Op0, Op1, Name);
130  }
131 
132  ICmpInst &ICI;
133 };
134 
135 // UnarySpliiter(UO)(Builder, X, Name) uses Builder to create
136 // a unary operator like UO called Name with operand X.
137 struct UnarySplitter {
138  UnarySplitter(UnaryOperator &uo) : UO(uo) {}
139 
140  Value *operator()(IRBuilder<> &Builder, Value *Op, const Twine &Name) const {
141  return Builder.CreateUnOp(UO.getOpcode(), Op, Name);
142  }
143 
144  UnaryOperator &UO;
145 };
146 
147 // BinarySpliiter(BO)(Builder, X, Y, Name) uses Builder to create
148 // a binary operator like BO called Name with operands X and Y.
149 struct BinarySplitter {
150  BinarySplitter(BinaryOperator &bo) : BO(bo) {}
151 
152  Value *operator()(IRBuilder<> &Builder, Value *Op0, Value *Op1,
153  const Twine &Name) const {
154  return Builder.CreateBinOp(BO.getOpcode(), Op0, Op1, Name);
155  }
156 
157  BinaryOperator &BO;
158 };
159 
160 // Information about a load or store that we're scalarizing.
161 struct VectorLayout {
162  VectorLayout() = default;
163 
164  // Return the alignment of element I.
165  Align getElemAlign(unsigned I) {
166  return commonAlignment(VecAlign, I * ElemSize);
167  }
168 
169  // The type of the vector.
170  VectorType *VecTy = nullptr;
171 
172  // The type of each element.
173  Type *ElemTy = nullptr;
174 
175  // The alignment of the vector.
176  Align VecAlign;
177 
178  // The size of each element.
179  uint64_t ElemSize = 0;
180 };
181 
182 class ScalarizerVisitor : public InstVisitor<ScalarizerVisitor, bool> {
183 public:
184  ScalarizerVisitor(unsigned ParallelLoopAccessMDKind, DominatorTree *DT)
185  : ParallelLoopAccessMDKind(ParallelLoopAccessMDKind), DT(DT) {
186  }
187 
188  bool visit(Function &F);
189 
190  // InstVisitor methods. They return true if the instruction was scalarized,
191  // false if nothing changed.
192  bool visitInstruction(Instruction &I) { return false; }
193  bool visitSelectInst(SelectInst &SI);
194  bool visitICmpInst(ICmpInst &ICI);
195  bool visitFCmpInst(FCmpInst &FCI);
196  bool visitUnaryOperator(UnaryOperator &UO);
197  bool visitBinaryOperator(BinaryOperator &BO);
198  bool visitGetElementPtrInst(GetElementPtrInst &GEPI);
199  bool visitCastInst(CastInst &CI);
200  bool visitBitCastInst(BitCastInst &BCI);
201  bool visitInsertElementInst(InsertElementInst &IEI);
202  bool visitExtractElementInst(ExtractElementInst &EEI);
203  bool visitShuffleVectorInst(ShuffleVectorInst &SVI);
204  bool visitPHINode(PHINode &PHI);
205  bool visitLoadInst(LoadInst &LI);
206  bool visitStoreInst(StoreInst &SI);
207  bool visitCallInst(CallInst &ICI);
208 
209 private:
210  Scatterer scatter(Instruction *Point, Value *V);
211  void gather(Instruction *Op, const ValueVector &CV);
212  bool canTransferMetadata(unsigned Kind);
213  void transferMetadataAndIRFlags(Instruction *Op, const ValueVector &CV);
214  Optional<VectorLayout> getVectorLayout(Type *Ty, Align Alignment,
215  const DataLayout &DL);
216  bool finish();
217 
218  template<typename T> bool splitUnary(Instruction &, const T &);
219  template<typename T> bool splitBinary(Instruction &, const T &);
220 
221  bool splitCall(CallInst &CI);
222 
223  ScatterMap Scattered;
224  GatherList Gathered;
225 
226  SmallVector<WeakTrackingVH, 32> PotentiallyDeadInstrs;
227 
228  unsigned ParallelLoopAccessMDKind;
229 
230  DominatorTree *DT;
231 };
232 
233 class ScalarizerLegacyPass : public FunctionPass {
234 public:
235  static char ID;
236 
237  ScalarizerLegacyPass() : FunctionPass(ID) {
239  }
240 
241  bool runOnFunction(Function &F) override;
242 
243  void getAnalysisUsage(AnalysisUsage& AU) const override {
246  }
247 };
248 
249 } // end anonymous namespace
250 
251 char ScalarizerLegacyPass::ID = 0;
252 INITIALIZE_PASS_BEGIN(ScalarizerLegacyPass, "scalarizer",
253  "Scalarize vector operations", false, false)
255 INITIALIZE_PASS_END(ScalarizerLegacyPass, "scalarizer",
256  "Scalarize vector operations", false, false)
257 
258 Scatterer::Scatterer(BasicBlock *bb, BasicBlock::iterator bbi, Value *v,
259  ValueVector *cachePtr)
260  : BB(bb), BBI(bbi), V(v), CachePtr(cachePtr) {
261  Type *Ty = V->getType();
262  PtrTy = dyn_cast<PointerType>(Ty);
263  if (PtrTy)
264  Ty = PtrTy->getElementType();
265  Size = cast<FixedVectorType>(Ty)->getNumElements();
266  if (!CachePtr)
267  Tmp.resize(Size, nullptr);
268  else if (CachePtr->empty())
269  CachePtr->resize(Size, nullptr);
270  else
271  assert(Size == CachePtr->size() && "Inconsistent vector sizes");
272 }
273 
274 // Return component I, creating a new Value for it if necessary.
275 Value *Scatterer::operator[](unsigned I) {
276  ValueVector &CV = (CachePtr ? *CachePtr : Tmp);
277  // Try to reuse a previous value.
278  if (CV[I])
279  return CV[I];
280  IRBuilder<> Builder(BB, BBI);
281  if (PtrTy) {
282  Type *ElTy = cast<VectorType>(PtrTy->getElementType())->getElementType();
283  if (!CV[0]) {
284  Type *NewPtrTy = PointerType::get(ElTy, PtrTy->getAddressSpace());
285  CV[0] = Builder.CreateBitCast(V, NewPtrTy, V->getName() + ".i0");
286  }
287  if (I != 0)
288  CV[I] = Builder.CreateConstGEP1_32(ElTy, CV[0], I,
289  V->getName() + ".i" + Twine(I));
290  } else {
291  // Search through a chain of InsertElementInsts looking for element I.
292  // Record other elements in the cache. The new V is still suitable
293  // for all uncached indices.
294  while (true) {
295  InsertElementInst *Insert = dyn_cast<InsertElementInst>(V);
296  if (!Insert)
297  break;
298  ConstantInt *Idx = dyn_cast<ConstantInt>(Insert->getOperand(2));
299  if (!Idx)
300  break;
301  unsigned J = Idx->getZExtValue();
302  V = Insert->getOperand(0);
303  if (I == J) {
304  CV[J] = Insert->getOperand(1);
305  return CV[J];
306  } else if (!CV[J]) {
307  // Only cache the first entry we find for each index we're not actively
308  // searching for. This prevents us from going too far up the chain and
309  // caching incorrect entries.
310  CV[J] = Insert->getOperand(1);
311  }
312  }
313  CV[I] = Builder.CreateExtractElement(V, Builder.getInt32(I),
314  V->getName() + ".i" + Twine(I));
315  }
316  return CV[I];
317 }
318 
320  if (skipFunction(F))
321  return false;
322 
323  Module &M = *F.getParent();
324  unsigned ParallelLoopAccessMDKind =
325  M.getContext().getMDKindID("llvm.mem.parallel_loop_access");
326  DominatorTree *DT = &getAnalysis<DominatorTreeWrapperPass>().getDomTree();
327  ScalarizerVisitor Impl(ParallelLoopAccessMDKind, DT);
328  return Impl.visit(F);
329 }
330 
332  return new ScalarizerLegacyPass();
333 }
334 
335 bool ScalarizerVisitor::visit(Function &F) {
336  assert(Gathered.empty() && Scattered.empty());
337 
338  // To ensure we replace gathered components correctly we need to do an ordered
339  // traversal of the basic blocks in the function.
340  ReversePostOrderTraversal<BasicBlock *> RPOT(&F.getEntryBlock());
341  for (BasicBlock *BB : RPOT) {
342  for (BasicBlock::iterator II = BB->begin(), IE = BB->end(); II != IE;) {
343  Instruction *I = &*II;
344  bool Done = InstVisitor::visit(I);
345  ++II;
346  if (Done && I->getType()->isVoidTy())
347  I->eraseFromParent();
348  }
349  }
350  return finish();
351 }
352 
353 // Return a scattered form of V that can be accessed by Point. V must be a
354 // vector or a pointer to a vector.
355 Scatterer ScalarizerVisitor::scatter(Instruction *Point, Value *V) {
356  if (Argument *VArg = dyn_cast<Argument>(V)) {
357  // Put the scattered form of arguments in the entry block,
358  // so that it can be used everywhere.
359  Function *F = VArg->getParent();
360  BasicBlock *BB = &F->getEntryBlock();
361  return Scatterer(BB, BB->begin(), V, &Scattered[V]);
362  }
363  if (Instruction *VOp = dyn_cast<Instruction>(V)) {
364  // When scalarizing PHI nodes we might try to examine/rewrite InsertElement
365  // nodes in predecessors. If those predecessors are unreachable from entry,
366  // then the IR in those blocks could have unexpected properties resulting in
367  // infinite loops in Scatterer::operator[]. By simply treating values
368  // originating from instructions in unreachable blocks as undef we do not
369  // need to analyse them further.
370  if (!DT->isReachableFromEntry(VOp->getParent()))
371  return Scatterer(Point->getParent(), Point->getIterator(),
372  UndefValue::get(V->getType()));
373  // Put the scattered form of an instruction directly after the
374  // instruction.
375  BasicBlock *BB = VOp->getParent();
376  return Scatterer(BB, std::next(BasicBlock::iterator(VOp)),
377  V, &Scattered[V]);
378  }
379  // In the fallback case, just put the scattered before Point and
380  // keep the result local to Point.
381  return Scatterer(Point->getParent(), Point->getIterator(), V);
382 }
383 
384 // Replace Op with the gathered form of the components in CV. Defer the
385 // deletion of Op and creation of the gathered form to the end of the pass,
386 // so that we can avoid creating the gathered form if all uses of Op are
387 // replaced with uses of CV.
388 void ScalarizerVisitor::gather(Instruction *Op, const ValueVector &CV) {
389  transferMetadataAndIRFlags(Op, CV);
390 
391  // If we already have a scattered form of Op (created from ExtractElements
392  // of Op itself), replace them with the new form.
393  ValueVector &SV = Scattered[Op];
394  if (!SV.empty()) {
395  for (unsigned I = 0, E = SV.size(); I != E; ++I) {
396  Value *V = SV[I];
397  if (V == nullptr || SV[I] == CV[I])
398  continue;
399 
400  Instruction *Old = cast<Instruction>(V);
401  if (isa<Instruction>(CV[I]))
402  CV[I]->takeName(Old);
403  Old->replaceAllUsesWith(CV[I]);
404  PotentiallyDeadInstrs.emplace_back(Old);
405  }
406  }
407  SV = CV;
408  Gathered.push_back(GatherList::value_type(Op, &SV));
409 }
410 
411 // Return true if it is safe to transfer the given metadata tag from
412 // vector to scalar instructions.
413 bool ScalarizerVisitor::canTransferMetadata(unsigned Tag) {
414  return (Tag == LLVMContext::MD_tbaa
415  || Tag == LLVMContext::MD_fpmath
416  || Tag == LLVMContext::MD_tbaa_struct
417  || Tag == LLVMContext::MD_invariant_load
418  || Tag == LLVMContext::MD_alias_scope
419  || Tag == LLVMContext::MD_noalias
420  || Tag == ParallelLoopAccessMDKind
421  || Tag == LLVMContext::MD_access_group);
422 }
423 
424 // Transfer metadata from Op to the instructions in CV if it is known
425 // to be safe to do so.
426 void ScalarizerVisitor::transferMetadataAndIRFlags(Instruction *Op,
427  const ValueVector &CV) {
429  Op->getAllMetadataOtherThanDebugLoc(MDs);
430  for (unsigned I = 0, E = CV.size(); I != E; ++I) {
431  if (Instruction *New = dyn_cast<Instruction>(CV[I])) {
432  for (const auto &MD : MDs)
433  if (canTransferMetadata(MD.first))
434  New->setMetadata(MD.first, MD.second);
435  New->copyIRFlags(Op);
436  if (Op->getDebugLoc() && !New->getDebugLoc())
437  New->setDebugLoc(Op->getDebugLoc());
438  }
439  }
440 }
441 
442 // Try to fill in Layout from Ty, returning true on success. Alignment is
443 // the alignment of the vector, or None if the ABI default should be used.
445 ScalarizerVisitor::getVectorLayout(Type *Ty, Align Alignment,
446  const DataLayout &DL) {
447  VectorLayout Layout;
448  // Make sure we're dealing with a vector.
449  Layout.VecTy = dyn_cast<VectorType>(Ty);
450  if (!Layout.VecTy)
451  return None;
452  // Check that we're dealing with full-byte elements.
453  Layout.ElemTy = Layout.VecTy->getElementType();
454  if (!DL.typeSizeEqualsStoreSize(Layout.ElemTy))
455  return None;
456  Layout.VecAlign = Alignment;
457  Layout.ElemSize = DL.getTypeStoreSize(Layout.ElemTy);
458  return Layout;
459 }
460 
461 // Scalarize one-operand instruction I, using Split(Builder, X, Name)
462 // to create an instruction like I with operand X and name Name.
463 template<typename Splitter>
464 bool ScalarizerVisitor::splitUnary(Instruction &I, const Splitter &Split) {
465  VectorType *VT = dyn_cast<VectorType>(I.getType());
466  if (!VT)
467  return false;
468 
469  unsigned NumElems = cast<FixedVectorType>(VT)->getNumElements();
471  Scatterer Op = scatter(&I, I.getOperand(0));
472  assert(Op.size() == NumElems && "Mismatched unary operation");
473  ValueVector Res;
474  Res.resize(NumElems);
475  for (unsigned Elem = 0; Elem < NumElems; ++Elem)
476  Res[Elem] = Split(Builder, Op[Elem], I.getName() + ".i" + Twine(Elem));
477  gather(&I, Res);
478  return true;
479 }
480 
481 // Scalarize two-operand instruction I, using Split(Builder, X, Y, Name)
482 // to create an instruction like I with operands X and Y and name Name.
483 template<typename Splitter>
484 bool ScalarizerVisitor::splitBinary(Instruction &I, const Splitter &Split) {
485  VectorType *VT = dyn_cast<VectorType>(I.getType());
486  if (!VT)
487  return false;
488 
489  unsigned NumElems = cast<FixedVectorType>(VT)->getNumElements();
491  Scatterer VOp0 = scatter(&I, I.getOperand(0));
492  Scatterer VOp1 = scatter(&I, I.getOperand(1));
493  assert(VOp0.size() == NumElems && "Mismatched binary operation");
494  assert(VOp1.size() == NumElems && "Mismatched binary operation");
495  ValueVector Res;
496  Res.resize(NumElems);
497  for (unsigned Elem = 0; Elem < NumElems; ++Elem) {
498  Value *Op0 = VOp0[Elem];
499  Value *Op1 = VOp1[Elem];
500  Res[Elem] = Split(Builder, Op0, Op1, I.getName() + ".i" + Twine(Elem));
501  }
502  gather(&I, Res);
503  return true;
504 }
505 
507  return isTriviallyVectorizable(ID);
508 }
509 
510 // All of the current scalarizable intrinsics only have one mangled type.
513  ArrayRef<Type*> Tys) {
514  return Intrinsic::getDeclaration(M, ID, Tys);
515 }
516 
517 /// If a call to a vector typed intrinsic function, split into a scalar call per
518 /// element if possible for the intrinsic.
519 bool ScalarizerVisitor::splitCall(CallInst &CI) {
520  VectorType *VT = dyn_cast<VectorType>(CI.getType());
521  if (!VT)
522  return false;
523 
524  Function *F = CI.getCalledFunction();
525  if (!F)
526  return false;
527 
528  Intrinsic::ID ID = F->getIntrinsicID();
530  return false;
531 
532  unsigned NumElems = cast<FixedVectorType>(VT)->getNumElements();
533  unsigned NumArgs = CI.getNumArgOperands();
534 
535  ValueVector ScalarOperands(NumArgs);
536  SmallVector<Scatterer, 8> Scattered(NumArgs);
537 
538  Scattered.resize(NumArgs);
539 
541  Tys.push_back(VT->getScalarType());
542 
543  // Assumes that any vector type has the same number of elements as the return
544  // vector type, which is true for all current intrinsics.
545  for (unsigned I = 0; I != NumArgs; ++I) {
546  Value *OpI = CI.getOperand(I);
547  if (OpI->getType()->isVectorTy()) {
548  Scattered[I] = scatter(&CI, OpI);
549  assert(Scattered[I].size() == NumElems && "mismatched call operands");
550  } else {
551  ScalarOperands[I] = OpI;
553  Tys.push_back(OpI->getType());
554  }
555  }
556 
557  ValueVector Res(NumElems);
558  ValueVector ScalarCallOps(NumArgs);
559 
560  Function *NewIntrin = getScalarIntrinsicDeclaration(F->getParent(), ID, Tys);
561  IRBuilder<> Builder(&CI);
562 
563  // Perform actual scalarization, taking care to preserve any scalar operands.
564  for (unsigned Elem = 0; Elem < NumElems; ++Elem) {
565  ScalarCallOps.clear();
566 
567  for (unsigned J = 0; J != NumArgs; ++J) {
569  ScalarCallOps.push_back(ScalarOperands[J]);
570  else
571  ScalarCallOps.push_back(Scattered[J][Elem]);
572  }
573 
574  Res[Elem] = Builder.CreateCall(NewIntrin, ScalarCallOps,
575  CI.getName() + ".i" + Twine(Elem));
576  }
577 
578  gather(&CI, Res);
579  return true;
580 }
581 
582 bool ScalarizerVisitor::visitSelectInst(SelectInst &SI) {
583  VectorType *VT = dyn_cast<VectorType>(SI.getType());
584  if (!VT)
585  return false;
586 
587  unsigned NumElems = cast<FixedVectorType>(VT)->getNumElements();
589  Scatterer VOp1 = scatter(&SI, SI.getOperand(1));
590  Scatterer VOp2 = scatter(&SI, SI.getOperand(2));
591  assert(VOp1.size() == NumElems && "Mismatched select");
592  assert(VOp2.size() == NumElems && "Mismatched select");
593  ValueVector Res;
594  Res.resize(NumElems);
595 
596  if (SI.getOperand(0)->getType()->isVectorTy()) {
597  Scatterer VOp0 = scatter(&SI, SI.getOperand(0));
598  assert(VOp0.size() == NumElems && "Mismatched select");
599  for (unsigned I = 0; I < NumElems; ++I) {
600  Value *Op0 = VOp0[I];
601  Value *Op1 = VOp1[I];
602  Value *Op2 = VOp2[I];
603  Res[I] = Builder.CreateSelect(Op0, Op1, Op2,
604  SI.getName() + ".i" + Twine(I));
605  }
606  } else {
607  Value *Op0 = SI.getOperand(0);
608  for (unsigned I = 0; I < NumElems; ++I) {
609  Value *Op1 = VOp1[I];
610  Value *Op2 = VOp2[I];
611  Res[I] = Builder.CreateSelect(Op0, Op1, Op2,
612  SI.getName() + ".i" + Twine(I));
613  }
614  }
615  gather(&SI, Res);
616  return true;
617 }
618 
619 bool ScalarizerVisitor::visitICmpInst(ICmpInst &ICI) {
620  return splitBinary(ICI, ICmpSplitter(ICI));
621 }
622 
623 bool ScalarizerVisitor::visitFCmpInst(FCmpInst &FCI) {
624  return splitBinary(FCI, FCmpSplitter(FCI));
625 }
626 
627 bool ScalarizerVisitor::visitUnaryOperator(UnaryOperator &UO) {
628  return splitUnary(UO, UnarySplitter(UO));
629 }
630 
631 bool ScalarizerVisitor::visitBinaryOperator(BinaryOperator &BO) {
632  return splitBinary(BO, BinarySplitter(BO));
633 }
634 
635 bool ScalarizerVisitor::visitGetElementPtrInst(GetElementPtrInst &GEPI) {
636  VectorType *VT = dyn_cast<VectorType>(GEPI.getType());
637  if (!VT)
638  return false;
639 
640  IRBuilder<> Builder(&GEPI);
641  unsigned NumElems = cast<FixedVectorType>(VT)->getNumElements();
642  unsigned NumIndices = GEPI.getNumIndices();
643 
644  // The base pointer might be scalar even if it's a vector GEP. In those cases,
645  // splat the pointer into a vector value, and scatter that vector.
646  Value *Op0 = GEPI.getOperand(0);
647  if (!Op0->getType()->isVectorTy())
648  Op0 = Builder.CreateVectorSplat(NumElems, Op0);
649  Scatterer Base = scatter(&GEPI, Op0);
650 
652  Ops.resize(NumIndices);
653  for (unsigned I = 0; I < NumIndices; ++I) {
654  Value *Op = GEPI.getOperand(I + 1);
655 
656  // The indices might be scalars even if it's a vector GEP. In those cases,
657  // splat the scalar into a vector value, and scatter that vector.
658  if (!Op->getType()->isVectorTy())
659  Op = Builder.CreateVectorSplat(NumElems, Op);
660 
661  Ops[I] = scatter(&GEPI, Op);
662  }
663 
664  ValueVector Res;
665  Res.resize(NumElems);
666  for (unsigned I = 0; I < NumElems; ++I) {
667  SmallVector<Value *, 8> Indices;
668  Indices.resize(NumIndices);
669  for (unsigned J = 0; J < NumIndices; ++J)
670  Indices[J] = Ops[J][I];
671  Res[I] = Builder.CreateGEP(GEPI.getSourceElementType(), Base[I], Indices,
672  GEPI.getName() + ".i" + Twine(I));
673  if (GEPI.isInBounds())
674  if (GetElementPtrInst *NewGEPI = dyn_cast<GetElementPtrInst>(Res[I]))
675  NewGEPI->setIsInBounds();
676  }
677  gather(&GEPI, Res);
678  return true;
679 }
680 
681 bool ScalarizerVisitor::visitCastInst(CastInst &CI) {
682  VectorType *VT = dyn_cast<VectorType>(CI.getDestTy());
683  if (!VT)
684  return false;
685 
686  unsigned NumElems = cast<FixedVectorType>(VT)->getNumElements();
687  IRBuilder<> Builder(&CI);
688  Scatterer Op0 = scatter(&CI, CI.getOperand(0));
689  assert(Op0.size() == NumElems && "Mismatched cast");
690  ValueVector Res;
691  Res.resize(NumElems);
692  for (unsigned I = 0; I < NumElems; ++I)
693  Res[I] = Builder.CreateCast(CI.getOpcode(), Op0[I], VT->getElementType(),
694  CI.getName() + ".i" + Twine(I));
695  gather(&CI, Res);
696  return true;
697 }
698 
699 bool ScalarizerVisitor::visitBitCastInst(BitCastInst &BCI) {
700  VectorType *DstVT = dyn_cast<VectorType>(BCI.getDestTy());
701  VectorType *SrcVT = dyn_cast<VectorType>(BCI.getSrcTy());
702  if (!DstVT || !SrcVT)
703  return false;
704 
705  unsigned DstNumElems = cast<FixedVectorType>(DstVT)->getNumElements();
706  unsigned SrcNumElems = cast<FixedVectorType>(SrcVT)->getNumElements();
707  IRBuilder<> Builder(&BCI);
708  Scatterer Op0 = scatter(&BCI, BCI.getOperand(0));
709  ValueVector Res;
710  Res.resize(DstNumElems);
711 
712  if (DstNumElems == SrcNumElems) {
713  for (unsigned I = 0; I < DstNumElems; ++I)
714  Res[I] = Builder.CreateBitCast(Op0[I], DstVT->getElementType(),
715  BCI.getName() + ".i" + Twine(I));
716  } else if (DstNumElems > SrcNumElems) {
717  // <M x t1> -> <N*M x t2>. Convert each t1 to <N x t2> and copy the
718  // individual elements to the destination.
719  unsigned FanOut = DstNumElems / SrcNumElems;
720  auto *MidTy = FixedVectorType::get(DstVT->getElementType(), FanOut);
721  unsigned ResI = 0;
722  for (unsigned Op0I = 0; Op0I < SrcNumElems; ++Op0I) {
723  Value *V = Op0[Op0I];
724  Instruction *VI;
725  // Look through any existing bitcasts before converting to <N x t2>.
726  // In the best case, the resulting conversion might be a no-op.
727  while ((VI = dyn_cast<Instruction>(V)) &&
728  VI->getOpcode() == Instruction::BitCast)
729  V = VI->getOperand(0);
730  V = Builder.CreateBitCast(V, MidTy, V->getName() + ".cast");
731  Scatterer Mid = scatter(&BCI, V);
732  for (unsigned MidI = 0; MidI < FanOut; ++MidI)
733  Res[ResI++] = Mid[MidI];
734  }
735  } else {
736  // <N*M x t1> -> <M x t2>. Convert each group of <N x t1> into a t2.
737  unsigned FanIn = SrcNumElems / DstNumElems;
738  auto *MidTy = FixedVectorType::get(SrcVT->getElementType(), FanIn);
739  unsigned Op0I = 0;
740  for (unsigned ResI = 0; ResI < DstNumElems; ++ResI) {
741  Value *V = PoisonValue::get(MidTy);
742  for (unsigned MidI = 0; MidI < FanIn; ++MidI)
743  V = Builder.CreateInsertElement(V, Op0[Op0I++], Builder.getInt32(MidI),
744  BCI.getName() + ".i" + Twine(ResI)
745  + ".upto" + Twine(MidI));
746  Res[ResI] = Builder.CreateBitCast(V, DstVT->getElementType(),
747  BCI.getName() + ".i" + Twine(ResI));
748  }
749  }
750  gather(&BCI, Res);
751  return true;
752 }
753 
754 bool ScalarizerVisitor::visitInsertElementInst(InsertElementInst &IEI) {
755  VectorType *VT = dyn_cast<VectorType>(IEI.getType());
756  if (!VT)
757  return false;
758 
759  unsigned NumElems = cast<FixedVectorType>(VT)->getNumElements();
760  IRBuilder<> Builder(&IEI);
761  Scatterer Op0 = scatter(&IEI, IEI.getOperand(0));
762  Value *NewElt = IEI.getOperand(1);
763  Value *InsIdx = IEI.getOperand(2);
764 
765  ValueVector Res;
766  Res.resize(NumElems);
767 
768  if (auto *CI = dyn_cast<ConstantInt>(InsIdx)) {
769  for (unsigned I = 0; I < NumElems; ++I)
770  Res[I] = CI->getValue().getZExtValue() == I ? NewElt : Op0[I];
771  } else {
773  return false;
774 
775  for (unsigned I = 0; I < NumElems; ++I) {
776  Value *ShouldReplace =
777  Builder.CreateICmpEQ(InsIdx, ConstantInt::get(InsIdx->getType(), I),
778  InsIdx->getName() + ".is." + Twine(I));
779  Value *OldElt = Op0[I];
780  Res[I] = Builder.CreateSelect(ShouldReplace, NewElt, OldElt,
781  IEI.getName() + ".i" + Twine(I));
782  }
783  }
784 
785  gather(&IEI, Res);
786  return true;
787 }
788 
789 bool ScalarizerVisitor::visitExtractElementInst(ExtractElementInst &EEI) {
790  VectorType *VT = dyn_cast<VectorType>(EEI.getOperand(0)->getType());
791  if (!VT)
792  return false;
793 
794  unsigned NumSrcElems = cast<FixedVectorType>(VT)->getNumElements();
795  IRBuilder<> Builder(&EEI);
796  Scatterer Op0 = scatter(&EEI, EEI.getOperand(0));
797  Value *ExtIdx = EEI.getOperand(1);
798 
799  if (auto *CI = dyn_cast<ConstantInt>(ExtIdx)) {
800  Value *Res = Op0[CI->getValue().getZExtValue()];
801  gather(&EEI, {Res});
802  return true;
803  }
804 
806  return false;
807 
808  Value *Res = UndefValue::get(VT->getElementType());
809  for (unsigned I = 0; I < NumSrcElems; ++I) {
810  Value *ShouldExtract =
811  Builder.CreateICmpEQ(ExtIdx, ConstantInt::get(ExtIdx->getType(), I),
812  ExtIdx->getName() + ".is." + Twine(I));
813  Value *Elt = Op0[I];
814  Res = Builder.CreateSelect(ShouldExtract, Elt, Res,
815  EEI.getName() + ".upto" + Twine(I));
816  }
817  gather(&EEI, {Res});
818  return true;
819 }
820 
821 bool ScalarizerVisitor::visitShuffleVectorInst(ShuffleVectorInst &SVI) {
822  VectorType *VT = dyn_cast<VectorType>(SVI.getType());
823  if (!VT)
824  return false;
825 
826  unsigned NumElems = cast<FixedVectorType>(VT)->getNumElements();
827  Scatterer Op0 = scatter(&SVI, SVI.getOperand(0));
828  Scatterer Op1 = scatter(&SVI, SVI.getOperand(1));
829  ValueVector Res;
830  Res.resize(NumElems);
831 
832  for (unsigned I = 0; I < NumElems; ++I) {
833  int Selector = SVI.getMaskValue(I);
834  if (Selector < 0)
835  Res[I] = UndefValue::get(VT->getElementType());
836  else if (unsigned(Selector) < Op0.size())
837  Res[I] = Op0[Selector];
838  else
839  Res[I] = Op1[Selector - Op0.size()];
840  }
841  gather(&SVI, Res);
842  return true;
843 }
844 
845 bool ScalarizerVisitor::visitPHINode(PHINode &PHI) {
846  VectorType *VT = dyn_cast<VectorType>(PHI.getType());
847  if (!VT)
848  return false;
849 
850  unsigned NumElems = cast<FixedVectorType>(VT)->getNumElements();
851  IRBuilder<> Builder(&PHI);
852  ValueVector Res;
853  Res.resize(NumElems);
854 
855  unsigned NumOps = PHI.getNumOperands();
856  for (unsigned I = 0; I < NumElems; ++I)
857  Res[I] = Builder.CreatePHI(VT->getElementType(), NumOps,
858  PHI.getName() + ".i" + Twine(I));
859 
860  for (unsigned I = 0; I < NumOps; ++I) {
861  Scatterer Op = scatter(&PHI, PHI.getIncomingValue(I));
862  BasicBlock *IncomingBlock = PHI.getIncomingBlock(I);
863  for (unsigned J = 0; J < NumElems; ++J)
864  cast<PHINode>(Res[J])->addIncoming(Op[J], IncomingBlock);
865  }
866  gather(&PHI, Res);
867  return true;
868 }
869 
870 bool ScalarizerVisitor::visitLoadInst(LoadInst &LI) {
871  if (!ScalarizeLoadStore)
872  return false;
873  if (!LI.isSimple())
874  return false;
875 
876  Optional<VectorLayout> Layout = getVectorLayout(
877  LI.getType(), LI.getAlign(), LI.getModule()->getDataLayout());
878  if (!Layout)
879  return false;
880 
881  unsigned NumElems = cast<FixedVectorType>(Layout->VecTy)->getNumElements();
882  IRBuilder<> Builder(&LI);
883  Scatterer Ptr = scatter(&LI, LI.getPointerOperand());
884  ValueVector Res;
885  Res.resize(NumElems);
886 
887  for (unsigned I = 0; I < NumElems; ++I)
888  Res[I] = Builder.CreateAlignedLoad(Layout->VecTy->getElementType(), Ptr[I],
889  Align(Layout->getElemAlign(I)),
890  LI.getName() + ".i" + Twine(I));
891  gather(&LI, Res);
892  return true;
893 }
894 
895 bool ScalarizerVisitor::visitStoreInst(StoreInst &SI) {
896  if (!ScalarizeLoadStore)
897  return false;
898  if (!SI.isSimple())
899  return false;
900 
901  Value *FullValue = SI.getValueOperand();
902  Optional<VectorLayout> Layout = getVectorLayout(
903  FullValue->getType(), SI.getAlign(), SI.getModule()->getDataLayout());
904  if (!Layout)
905  return false;
906 
907  unsigned NumElems = cast<FixedVectorType>(Layout->VecTy)->getNumElements();
909  Scatterer VPtr = scatter(&SI, SI.getPointerOperand());
910  Scatterer VVal = scatter(&SI, FullValue);
911 
912  ValueVector Stores;
913  Stores.resize(NumElems);
914  for (unsigned I = 0; I < NumElems; ++I) {
915  Value *Val = VVal[I];
916  Value *Ptr = VPtr[I];
917  Stores[I] = Builder.CreateAlignedStore(Val, Ptr, Layout->getElemAlign(I));
918  }
919  transferMetadataAndIRFlags(&SI, Stores);
920  return true;
921 }
922 
923 bool ScalarizerVisitor::visitCallInst(CallInst &CI) {
924  return splitCall(CI);
925 }
926 
927 // Delete the instructions that we scalarized. If a full vector result
928 // is still needed, recreate it using InsertElements.
929 bool ScalarizerVisitor::finish() {
930  // The presence of data in Gathered or Scattered indicates changes
931  // made to the Function.
932  if (Gathered.empty() && Scattered.empty())
933  return false;
934  for (const auto &GMI : Gathered) {
935  Instruction *Op = GMI.first;
936  ValueVector &CV = *GMI.second;
937  if (!Op->use_empty()) {
938  // The value is still needed, so recreate it using a series of
939  // InsertElements.
940  Value *Res = PoisonValue::get(Op->getType());
941  if (auto *Ty = dyn_cast<VectorType>(Op->getType())) {
942  BasicBlock *BB = Op->getParent();
943  unsigned Count = cast<FixedVectorType>(Ty)->getNumElements();
945  if (isa<PHINode>(Op))
946  Builder.SetInsertPoint(BB, BB->getFirstInsertionPt());
947  for (unsigned I = 0; I < Count; ++I)
948  Res = Builder.CreateInsertElement(Res, CV[I], Builder.getInt32(I),
949  Op->getName() + ".upto" + Twine(I));
950  Res->takeName(Op);
951  } else {
952  assert(CV.size() == 1 && Op->getType() == CV[0]->getType());
953  Res = CV[0];
954  if (Op == Res)
955  continue;
956  }
957  Op->replaceAllUsesWith(Res);
958  }
959  PotentiallyDeadInstrs.emplace_back(Op);
960  }
961  Gathered.clear();
962  Scattered.clear();
963 
965 
966  return true;
967 }
968 
970  Module &M = *F.getParent();
971  unsigned ParallelLoopAccessMDKind =
972  M.getContext().getMDKindID("llvm.mem.parallel_loop_access");
974  ScalarizerVisitor Impl(ParallelLoopAccessMDKind, DT);
975  bool Changed = Impl.visit(F);
978  return Changed ? PA : PreservedAnalyses::all();
979 }
llvm::Check::Size
@ Size
Definition: FileCheck.h:73
llvm::PreservedAnalyses
A set of analyses that are preserved following a run of a transformation pass.
Definition: PassManager.h:155
llvm::Argument
This class represents an incoming formal argument to a Function.
Definition: Argument.h:29
MathExtras.h
llvm
---------------------— PointerInfo ------------------------------------—
Definition: AllocatorList.h:23
M
We currently emits eax Perhaps this is what we really should generate is Is imull three or four cycles eax eax The current instruction priority is based on pattern complexity The former is more complex because it folds a load so the latter will not be emitted Perhaps we should use AddedComplexity to give LEA32r a higher priority We should always try to match LEA first since the LEA matching code does some estimate to determine whether the match is profitable if we care more about code then imull is better It s two bytes shorter than movl leal On a Pentium M
Definition: README.txt:252
llvm::Instruction::getModule
const Module * getModule() const
Return the module owning the function this instruction belongs to or nullptr it the function does not...
Definition: Instruction.cpp:66
llvm::DataLayout
A parsed version of the target data layout string in and methods for querying it.
Definition: DataLayout.h:112
llvm::Intrinsic::getDeclaration
Function * getDeclaration(Module *M, ID id, ArrayRef< Type * > Tys=None)
Create or insert an LLVM Function declaration for an intrinsic, and return it.
Definition: Function.cpp:1379
Insert
Vector Rotate Left Mask Mask Insert
Definition: README_P9.txt:112
llvm::BasicBlock::iterator
InstListType::iterator iterator
Instruction iterators...
Definition: BasicBlock.h:90
llvm::AnalysisManager::getResult
PassT::Result & getResult(IRUnitT &IR, ExtraArgTs... ExtraArgs)
Get the result of an analysis pass for a given IR unit.
Definition: PassManager.h:779
Scalar.h
llvm::ExtractElementInst
This instruction extracts a single (scalar) element from a VectorType value.
Definition: Instructions.h:1873
llvm::Function
Definition: Function.h:61
Pass.h
llvm::PointerType::get
static PointerType * get(Type *ElementType, unsigned AddressSpace)
This constructs a pointer to an object of the specified type in a numbered address space.
Definition: Type.cpp:691
llvm::BitCastInst
This class represents a no-op cast from one type to another.
Definition: Instructions.h:5192
llvm::Type::getScalarType
Type * getScalarType() const
If this is a vector type, return the element type, otherwise return 'this'.
Definition: Type.h:319
llvm::SmallVector< Value *, 8 >
llvm::IRBuilder<>
Local.h
llvm::DominatorTree
Concrete subclass of DominatorTreeBase that is used to compute a normal dominator tree.
Definition: Dominators.h:151
llvm::cl::Hidden
@ Hidden
Definition: CommandLine.h:143
llvm::Type
The instances of the Type class are immutable: once they are created, they are never changed.
Definition: Type.h:45
Module.h
llvm::Optional
Definition: APInt.h:33
llvm::GetElementPtrInst::getNumIndices
unsigned getNumIndices() const
Definition: Instructions.h:1114
llvm::VectorType::getElementType
Type * getElementType() const
Definition: DerivedTypes.h:422
llvm::Intrinsic::not_intrinsic
@ not_intrinsic
Definition: Intrinsics.h:45
llvm::CallBase::getNumArgOperands
unsigned getNumArgOperands() const
Definition: InstrTypes.h:1336
llvm::LoadInst::getPointerOperand
Value * getPointerOperand()
Definition: Instructions.h:267
llvm::UnaryOperator
Definition: InstrTypes.h:102
getScalarIntrinsicDeclaration
static Function * getScalarIntrinsicDeclaration(Module *M, Intrinsic::ID ID, ArrayRef< Type * > Tys)
Definition: Scalarizer.cpp:511
llvm::LoadInst::getAlign
Align getAlign() const
Return the alignment of the access that is being performed.
Definition: Instructions.h:223
llvm::CastInst::getDestTy
Type * getDestTy() const
Return the destination type, as a convenience.
Definition: InstrTypes.h:684
llvm::GetElementPtrInst::getSourceElementType
Type * getSourceElementType() const
Definition: Instructions.h:1021
F
#define F(x, y, z)
Definition: MD5.cpp:56
llvm::BasicBlock
LLVM Basic Block Representation.
Definition: BasicBlock.h:58
Scalarizer.h
llvm::GetElementPtrInst::isInBounds
bool isInBounds() const
Determine whether the GEP has the inbounds flag.
Definition: Instructions.cpp:1802
Instruction.h
CommandLine.h
operations
Scalarize vector operations
Definition: Scalarizer.cpp:256
llvm::ConstantInt
This is the shared class of boolean and integer constants.
Definition: Constants.h:79
bb
< i1 > br i1 label label bb bb
Definition: README.txt:978
llvm::AArch64Layout::VectorLayout
VectorLayout
Definition: AArch64BaseInfo.h:567
llvm::PassRegistry::getPassRegistry
static PassRegistry * getPassRegistry()
getPassRegistry - Access the global registry object, which is automatically initialized at applicatio...
Definition: PassRegistry.cpp:31
Constants.h
llvm::PHINode::getIncomingValue
Value * getIncomingValue(unsigned i) const
Return incoming value number x.
Definition: Instructions.h:2721
E
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
llvm::isTriviallyVectorizable
bool isTriviallyVectorizable(Intrinsic::ID ID)
Identify if the intrinsic is trivially vectorizable.
Definition: VectorUtils.cpp:44
Intrinsics.h
Twine.h
InstrTypes.h
llvm::CallBase::getCalledFunction
Function * getCalledFunction() const
Returns the function called, or null if this is an indirect function invocation.
Definition: InstrTypes.h:1393
llvm::FCmpInst
This instruction compares its operands according to the predicate given to the constructor.
Definition: Instructions.h:1369
llvm::AnalysisUsage
Represent the analysis usage information of a pass.
Definition: PassAnalysisSupport.h:47
llvm::Type::isVectorTy
bool isVectorTy() const
True if this is an instance of VectorType.
Definition: Type.h:237
llvm::InsertElementInst
This instruction inserts a single (scalar) element into a VectorType value.
Definition: Instructions.h:1937
false
Definition: StackSlotColoring.cpp:142
llvm::ShuffleVectorInst::getMaskValue
int getMaskValue(unsigned Elt) const
Return the shuffle mask value of this instruction for the given element index.
Definition: Instructions.h:2056
llvm::Instruction
Definition: Instruction.h:45
llvm::DominatorTreeWrapperPass
Legacy analysis pass which computes a DominatorTree.
Definition: Dominators.h:287
llvm::SmallVectorImpl::resize
void resize(size_type N)
Definition: SmallVector.h:606
llvm::UndefValue::get
static UndefValue * get(Type *T)
Static factory methods - Return an 'undef' object of the specified type.
Definition: Constants.cpp:1771
llvm::ConstantInt::get
static Constant * get(Type *Ty, uint64_t V, bool IsSigned=false)
If Ty is a vector type, return a Constant with a splat of the given value.
Definition: Constants.cpp:900
llvm::ScalarizerPass::run
PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM)
Definition: Scalarizer.cpp:969
llvm::FixedVectorType::get
static FixedVectorType * get(Type *ElementType, unsigned NumElts)
Definition: Type.cpp:648
llvm::Align
This struct is a compact representation of a valid (non-zero power of two) alignment.
Definition: Alignment.h:39
llvm::CastInst::getSrcTy
Type * getSrcTy() const
Return the source type, as a convenience.
Definition: InstrTypes.h:682
llvm::None
const NoneType None
Definition: None.h:23
llvm::lltok::Kind
Kind
Definition: LLToken.h:18
llvm::ShuffleVectorInst::getType
VectorType * getType() const
Overload to return most specific vector type.
Definition: Instructions.h:2047
Type.h
INITIALIZE_PASS_END
#define INITIALIZE_PASS_END(passName, arg, name, cfg, analysis)
Definition: PassSupport.h:58
llvm::ARM_PROC::IE
@ IE
Definition: ARMBaseInfo.h:27
llvm::VectorType
Base class of all SIMD vector types.
Definition: DerivedTypes.h:389
VectorUtils.h
BasicBlock.h
llvm::cl::opt< bool >
llvm::StoreInst
An instruction for storing to memory.
Definition: Instructions.h:304
VI
@ VI
Definition: SIInstrInfo.cpp:7679
ScalarizeLoadStore
static cl::opt< bool > ScalarizeLoadStore("scalarize-load-store", cl::init(false), cl::Hidden, cl::desc("Allow the scalarizer pass to scalarize loads and store"))
llvm::hasVectorInstrinsicOverloadedScalarOpd
bool hasVectorInstrinsicOverloadedScalarOpd(Intrinsic::ID ID, unsigned ScalarOpdIdx)
Identifies if the vector form of the intrinsic has a scalar operand that has an overloaded type.
Definition: VectorUtils.cpp:117
llvm::ICmpInst
This instruction compares its operands according to the predicate given to the constructor.
Definition: Instructions.h:1203
uint64_t
llvm::PreservedAnalyses::preserve
void preserve()
Mark an analysis as preserved.
Definition: PassManager.h:176
INITIALIZE_PASS_DEPENDENCY
INITIALIZE_PASS_DEPENDENCY(DominatorTreeWrapperPass)
I
#define I(x, y, z)
Definition: MD5.cpp:59
llvm::GetElementPtrInst
an instruction for type-safe pointer arithmetic to access elements of arrays and structs
Definition: Instructions.h:928
llvm::hasVectorInstrinsicScalarOpd
bool hasVectorInstrinsicScalarOpd(Intrinsic::ID ID, unsigned ScalarOpdIdx)
Identifies if the vector form of the intrinsic has a scalar operand.
Definition: VectorUtils.cpp:99
llvm::cl::init
initializer< Ty > init(const Ty &Val)
Definition: CommandLine.h:443
llvm::PointerType
Class to represent pointers.
Definition: DerivedTypes.h:632
llvm::InstVisitor::visit
void visit(Iterator Start, Iterator End)
Definition: InstVisitor.h:88
IRBuilder.h
assert
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
llvm::createScalarizerPass
FunctionPass * createScalarizerPass()
Create a legacy pass manager instance of the Scalarizer pass.
Definition: Scalarizer.cpp:331
SI
StandardInstrumentations SI(Debug, VerifyEach)
llvm::DominatorTree::isReachableFromEntry
bool isReachableFromEntry(const Use &U) const
Provide an overload for a Use.
Definition: Dominators.cpp:328
llvm::SelectInst
This class represents the LLVM 'select' instruction.
Definition: Instructions.h:1738
llvm::Module
A Module instance is used to store all the information related to an LLVM module.
Definition: Module.h:67
Builder
assume Assume Builder
Definition: AssumeBundleBuilder.cpp:650
llvm::size
auto size(R &&Range, std::enable_if_t< std::is_base_of< std::random_access_iterator_tag, typename std::iterator_traits< decltype(Range.begin())>::iterator_category >::value, void > *=nullptr)
Get the size of a range.
Definition: STLExtras.h:1528
llvm::ArrayRef
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
Definition: APInt.h:32
llvm::BinaryOperator
Definition: InstrTypes.h:189
DataLayout.h
InstVisitor.h
llvm::LoadInst::isSimple
bool isSimple() const
Definition: Instructions.h:259
llvm::Value::getType
Type * getType() const
All values are typed, get the type of this value.
Definition: Value.h:256
llvm::AnalysisUsage::addPreserved
AnalysisUsage & addPreserved()
Add the specified Pass class to the set of analyses preserved by this pass.
Definition: PassAnalysisSupport.h:98
llvm::Value::replaceAllUsesWith
void replaceAllUsesWith(Value *V)
Change all uses of this to point to a new Value.
Definition: Value.cpp:520
llvm::ms_demangle::IntrinsicFunctionKind::New
@ New
llvm::ilist_node_impl::getIterator
self_iterator getIterator()
Definition: ilist_node.h:81
DL
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
Definition: AArch64SLSHardening.cpp:76
llvm::initializeScalarizerLegacyPassPass
void initializeScalarizerLegacyPassPass(PassRegistry &)
llvm::InstVisitor
Base class for instruction visitors.
Definition: InstVisitor.h:79
llvm::CastInst
This is the base class for all instructions that perform data casts.
Definition: InstrTypes.h:430
llvm::Value::getName
StringRef getName() const
Return a constant reference to the value's name.
Definition: Value.cpp:297
llvm::LoadInst
An instruction for reading from memory.
Definition: Instructions.h:175
Argument.h
llvm::ConstantInt::getZExtValue
uint64_t getZExtValue() const
Return the constant as a 64-bit unsigned integer value after it has been zero extended as appropriate...
Definition: Constants.h:142
runOnFunction
static bool runOnFunction(Function &F, bool PostInlining)
Definition: EntryExitInstrumenter.cpp:69
llvm::Twine
Twine - A lightweight data structure for efficiently representing the concatenation of temporary valu...
Definition: Twine.h:83
llvm::commonAlignment
Align commonAlignment(Align A, Align B)
Returns the alignment that satisfies both alignments.
Definition: Alignment.h:211
llvm::GraphProgram::Name
Name
Definition: GraphWriter.h:52
llvm::AMDGPU::SendMsg::Op
Op
Definition: SIDefines.h:321
llvm::PreservedAnalyses::all
static PreservedAnalyses all()
Construct a special preserved set that preserves all passes.
Definition: PassManager.h:161
Casting.h
isTriviallyScalariable
static bool isTriviallyScalariable(Intrinsic::ID ID)
Definition: Scalarizer.cpp:506
Function.h
INITIALIZE_PASS_BEGIN
INITIALIZE_PASS_BEGIN(ScalarizerLegacyPass, "scalarizer", "Scalarize vector operations", false, false) INITIALIZE_PASS_END(ScalarizerLegacyPass
llvm::ReversePostOrderTraversal
Definition: PostOrderIterator.h:290
scalarizer
scalarizer
Definition: Scalarizer.cpp:255
llvm::DominatorTreeAnalysis
Analysis pass which computes a DominatorTree.
Definition: Dominators.h:252
llvm::InsertElementInst::getType
VectorType * getType() const
Overload to return most specific vector type.
Definition: Instructions.h:1970
llvm::ShuffleVectorInst
This instruction constructs a fixed permutation of two input vectors.
Definition: Instructions.h:2009
Instructions.h
PostOrderIterator.h
llvm::User::getNumOperands
unsigned getNumOperands() const
Definition: User.h:191
SmallVector.h
Dominators.h
llvm::CastInst::getOpcode
Instruction::CastOps getOpcode() const
Return the opcode of this CastInst.
Definition: InstrTypes.h:677
llvm::Instruction::getParent
const BasicBlock * getParent() const
Definition: Instruction.h:94
llvm::PHINode::getIncomingBlock
BasicBlock * getIncomingBlock(unsigned i) const
Return incoming basic block number i.
Definition: Instructions.h:2741
llvm::PHINode
Definition: Instructions.h:2625
llvm::Module::getDataLayout
const DataLayout & getDataLayout() const
Get the data layout for the module's target platform.
Definition: Module.cpp:401
DerivedTypes.h
llvm::AnalysisManager
A container for analyses that lazily runs them and caches their results.
Definition: InstructionSimplify.h:44
llvm::FunctionPass
FunctionPass class - This class is used to implement most global optimizations.
Definition: Pass.h:298
llvm::CallInst
This class represents a function call, abstracting a target machine's calling convention.
Definition: Instructions.h:1475
BB
Common register allocation spilling lr str ldr sxth r3 ldr mla r4 can lr mov lr str ldr sxth r3 mla r4 and then merge mul and lr str ldr sxth r3 mla r4 It also increase the likelihood the store may become dead bb27 Successors according to LLVM BB
Definition: README.txt:39
llvm::AnalysisUsage::addRequired
AnalysisUsage & addRequired()
Definition: PassAnalysisSupport.h:75
LLVMContext.h
llvm::Value::takeName
void takeName(Value *V)
Transfer the name from V to this value.
Definition: Value.cpp:370
llvm::User::getOperand
Value * getOperand(unsigned i) const
Definition: User.h:169
llvm::cl::desc
Definition: CommandLine.h:414
Value.h
InitializePasses.h
llvm::RecursivelyDeleteTriviallyDeadInstructionsPermissive
bool RecursivelyDeleteTriviallyDeadInstructionsPermissive(SmallVectorImpl< WeakTrackingVH > &DeadInsts, const TargetLibraryInfo *TLI=nullptr, MemorySSAUpdater *MSSAU=nullptr, std::function< void(Value *)> AboutToDeleteCallback=std::function< void(Value *)>())
Same functionality as RecursivelyDeleteTriviallyDeadInstructions, but allow instructions that are not...
Definition: Local.cpp:526
llvm::Value
LLVM Value Representation.
Definition: Value.h:75
ScalarizeVariableInsertExtract
static cl::opt< bool > ScalarizeVariableInsertExtract("scalarize-variable-insert-extract", cl::init(true), cl::Hidden, cl::desc("Allow the scalarizer pass to scalarize " "insertelement/extractelement with variable index"))
llvm::sampleprof::Base
@ Base
Definition: Discriminator.h:58
llvm::Intrinsic::ID
unsigned ID
Definition: TargetTransformInfo.h:37
llvm::PoisonValue::get
static PoisonValue * get(Type *T)
Static factory methods - Return an 'poison' object of the specified type.
Definition: Constants.cpp:1790