LLVM  14.0.0git
SLPVectorizer.cpp
Go to the documentation of this file.
1 //===- SLPVectorizer.cpp - A bottom up SLP Vectorizer ---------------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This pass implements the Bottom Up SLP vectorizer. It detects consecutive
10 // stores that can be put together into vector-stores. Next, it attempts to
11 // construct vectorizable tree using the use-def chains. If a profitable tree
12 // was found, the SLP vectorizer performs vectorization on the tree.
13 //
14 // The pass is inspired by the work described in the paper:
15 // "Loop-Aware SLP in GCC" by Ira Rosen, Dorit Nuzman, Ayal Zaks.
16 //
17 //===----------------------------------------------------------------------===//
18 
20 #include "llvm/ADT/DenseMap.h"
21 #include "llvm/ADT/DenseSet.h"
22 #include "llvm/ADT/Optional.h"
24 #include "llvm/ADT/PriorityQueue.h"
25 #include "llvm/ADT/STLExtras.h"
26 #include "llvm/ADT/SetOperations.h"
27 #include "llvm/ADT/SetVector.h"
29 #include "llvm/ADT/SmallPtrSet.h"
30 #include "llvm/ADT/SmallSet.h"
31 #include "llvm/ADT/SmallString.h"
32 #include "llvm/ADT/Statistic.h"
33 #include "llvm/ADT/iterator.h"
42 #include "llvm/Analysis/LoopInfo.h"
51 #include "llvm/IR/Attributes.h"
52 #include "llvm/IR/BasicBlock.h"
53 #include "llvm/IR/Constant.h"
54 #include "llvm/IR/Constants.h"
55 #include "llvm/IR/DataLayout.h"
56 #include "llvm/IR/DebugLoc.h"
57 #include "llvm/IR/DerivedTypes.h"
58 #include "llvm/IR/Dominators.h"
59 #include "llvm/IR/Function.h"
60 #include "llvm/IR/IRBuilder.h"
61 #include "llvm/IR/InstrTypes.h"
62 #include "llvm/IR/Instruction.h"
63 #include "llvm/IR/Instructions.h"
64 #include "llvm/IR/IntrinsicInst.h"
65 #include "llvm/IR/Intrinsics.h"
66 #include "llvm/IR/Module.h"
67 #include "llvm/IR/NoFolder.h"
68 #include "llvm/IR/Operator.h"
69 #include "llvm/IR/PatternMatch.h"
70 #include "llvm/IR/Type.h"
71 #include "llvm/IR/Use.h"
72 #include "llvm/IR/User.h"
73 #include "llvm/IR/Value.h"
74 #include "llvm/IR/ValueHandle.h"
75 #include "llvm/IR/Verifier.h"
76 #include "llvm/InitializePasses.h"
77 #include "llvm/Pass.h"
78 #include "llvm/Support/Casting.h"
80 #include "llvm/Support/Compiler.h"
82 #include "llvm/Support/Debug.h"
86 #include "llvm/Support/KnownBits.h"
92 #include <algorithm>
93 #include <cassert>
94 #include <cstdint>
95 #include <iterator>
96 #include <memory>
97 #include <set>
98 #include <string>
99 #include <tuple>
100 #include <utility>
101 #include <vector>
102 
103 using namespace llvm;
104 using namespace llvm::PatternMatch;
105 using namespace slpvectorizer;
106 
107 #define SV_NAME "slp-vectorizer"
108 #define DEBUG_TYPE "SLP"
109 
110 STATISTIC(NumVectorInstructions, "Number of vector instructions generated");
111 
112 cl::opt<bool> RunSLPVectorization("vectorize-slp", cl::init(true), cl::Hidden,
113  cl::desc("Run the SLP vectorization passes"));
114 
115 static cl::opt<int>
116  SLPCostThreshold("slp-threshold", cl::init(0), cl::Hidden,
117  cl::desc("Only vectorize if you gain more than this "
118  "number "));
119 
120 static cl::opt<bool>
121 ShouldVectorizeHor("slp-vectorize-hor", cl::init(true), cl::Hidden,
122  cl::desc("Attempt to vectorize horizontal reductions"));
123 
125  "slp-vectorize-hor-store", cl::init(false), cl::Hidden,
126  cl::desc(
127  "Attempt to vectorize horizontal reductions feeding into a store"));
128 
129 static cl::opt<int>
130 MaxVectorRegSizeOption("slp-max-reg-size", cl::init(128), cl::Hidden,
131  cl::desc("Attempt to vectorize for this register size in bits"));
132 
133 static cl::opt<unsigned>
134 MaxVFOption("slp-max-vf", cl::init(0), cl::Hidden,
135  cl::desc("Maximum SLP vectorization factor (0=unlimited)"));
136 
137 static cl::opt<int>
138 MaxStoreLookup("slp-max-store-lookup", cl::init(32), cl::Hidden,
139  cl::desc("Maximum depth of the lookup for consecutive stores."));
140 
141 /// Limits the size of scheduling regions in a block.
142 /// It avoid long compile times for _very_ large blocks where vector
143 /// instructions are spread over a wide range.
144 /// This limit is way higher than needed by real-world functions.
145 static cl::opt<int>
146 ScheduleRegionSizeBudget("slp-schedule-budget", cl::init(100000), cl::Hidden,
147  cl::desc("Limit the size of the SLP scheduling region per block"));
148 
150  "slp-min-reg-size", cl::init(128), cl::Hidden,
151  cl::desc("Attempt to vectorize for this register size in bits"));
152 
154  "slp-recursion-max-depth", cl::init(12), cl::Hidden,
155  cl::desc("Limit the recursion depth when building a vectorizable tree"));
156 
158  "slp-min-tree-size", cl::init(3), cl::Hidden,
159  cl::desc("Only vectorize small trees if they are fully vectorizable"));
160 
161 // The maximum depth that the look-ahead score heuristic will explore.
162 // The higher this value, the higher the compilation time overhead.
164  "slp-max-look-ahead-depth", cl::init(2), cl::Hidden,
165  cl::desc("The maximum look-ahead depth for operand reordering scores"));
166 
167 // The Look-ahead heuristic goes through the users of the bundle to calculate
168 // the users cost in getExternalUsesCost(). To avoid compilation time increase
169 // we limit the number of users visited to this value.
171  "slp-look-ahead-users-budget", cl::init(2), cl::Hidden,
172  cl::desc("The maximum number of users to visit while visiting the "
173  "predecessors. This prevents compilation time increase."));
174 
175 static cl::opt<bool>
176  ViewSLPTree("view-slp-tree", cl::Hidden,
177  cl::desc("Display the SLP trees with Graphviz"));
178 
179 // Limit the number of alias checks. The limit is chosen so that
180 // it has no negative effect on the llvm benchmarks.
181 static const unsigned AliasedCheckLimit = 10;
182 
183 // Another limit for the alias checks: The maximum distance between load/store
184 // instructions where alias checks are done.
185 // This limit is useful for very large basic blocks.
186 static const unsigned MaxMemDepDistance = 160;
187 
188 /// If the ScheduleRegionSizeBudget is exhausted, we allow small scheduling
189 /// regions to be handled.
190 static const int MinScheduleRegionSize = 16;
191 
192 /// Predicate for the element types that the SLP vectorizer supports.
193 ///
194 /// The most important thing to filter here are types which are invalid in LLVM
195 /// vectors. We also filter target specific types which have absolutely no
196 /// meaningful vectorization path such as x86_fp80 and ppc_f128. This just
197 /// avoids spending time checking the cost model and realizing that they will
198 /// be inevitably scalarized.
199 static bool isValidElementType(Type *Ty) {
200  return VectorType::isValidElementType(Ty) && !Ty->isX86_FP80Ty() &&
201  !Ty->isPPC_FP128Ty();
202 }
203 
204 /// \returns true if all of the instructions in \p VL are in the same block or
205 /// false otherwise.
207  Instruction *I0 = dyn_cast<Instruction>(VL[0]);
208  if (!I0)
209  return false;
210  BasicBlock *BB = I0->getParent();
211  for (int I = 1, E = VL.size(); I < E; I++) {
212  auto *II = dyn_cast<Instruction>(VL[I]);
213  if (!II)
214  return false;
215 
216  if (BB != II->getParent())
217  return false;
218  }
219  return true;
220 }
221 
222 /// \returns True if the value is a constant (but not globals/constant
223 /// expressions).
224 static bool isConstant(Value *V) {
225  return isa<Constant>(V) && !isa<ConstantExpr>(V) && !isa<GlobalValue>(V);
226 }
227 
228 /// \returns True if all of the values in \p VL are constants (but not
229 /// globals/constant expressions).
231  // Constant expressions and globals can't be vectorized like normal integer/FP
232  // constants.
233  return all_of(VL, isConstant);
234 }
235 
236 /// \returns True if all of the values in \p VL are identical.
237 static bool isSplat(ArrayRef<Value *> VL) {
238  for (unsigned i = 1, e = VL.size(); i < e; ++i)
239  if (VL[i] != VL[0])
240  return false;
241  return true;
242 }
243 
244 /// \returns True if \p I is commutative, handles CmpInst and BinaryOperator.
245 static bool isCommutative(Instruction *I) {
246  if (auto *Cmp = dyn_cast<CmpInst>(I))
247  return Cmp->isCommutative();
248  if (auto *BO = dyn_cast<BinaryOperator>(I))
249  return BO->isCommutative();
250  // TODO: This should check for generic Instruction::isCommutative(), but
251  // we need to confirm that the caller code correctly handles Intrinsics
252  // for example (does not have 2 operands).
253  return false;
254 }
255 
256 /// Checks if the vector of instructions can be represented as a shuffle, like:
257 /// %x0 = extractelement <4 x i8> %x, i32 0
258 /// %x3 = extractelement <4 x i8> %x, i32 3
259 /// %y1 = extractelement <4 x i8> %y, i32 1
260 /// %y2 = extractelement <4 x i8> %y, i32 2
261 /// %x0x0 = mul i8 %x0, %x0
262 /// %x3x3 = mul i8 %x3, %x3
263 /// %y1y1 = mul i8 %y1, %y1
264 /// %y2y2 = mul i8 %y2, %y2
265 /// %ins1 = insertelement <4 x i8> poison, i8 %x0x0, i32 0
266 /// %ins2 = insertelement <4 x i8> %ins1, i8 %x3x3, i32 1
267 /// %ins3 = insertelement <4 x i8> %ins2, i8 %y1y1, i32 2
268 /// %ins4 = insertelement <4 x i8> %ins3, i8 %y2y2, i32 3
269 /// ret <4 x i8> %ins4
270 /// can be transformed into:
271 /// %1 = shufflevector <4 x i8> %x, <4 x i8> %y, <4 x i32> <i32 0, i32 3, i32 5,
272 /// i32 6>
273 /// %2 = mul <4 x i8> %1, %1
274 /// ret <4 x i8> %2
275 /// We convert this initially to something like:
276 /// %x0 = extractelement <4 x i8> %x, i32 0
277 /// %x3 = extractelement <4 x i8> %x, i32 3
278 /// %y1 = extractelement <4 x i8> %y, i32 1
279 /// %y2 = extractelement <4 x i8> %y, i32 2
280 /// %1 = insertelement <4 x i8> poison, i8 %x0, i32 0
281 /// %2 = insertelement <4 x i8> %1, i8 %x3, i32 1
282 /// %3 = insertelement <4 x i8> %2, i8 %y1, i32 2
283 /// %4 = insertelement <4 x i8> %3, i8 %y2, i32 3
284 /// %5 = mul <4 x i8> %4, %4
285 /// %6 = extractelement <4 x i8> %5, i32 0
286 /// %ins1 = insertelement <4 x i8> poison, i8 %6, i32 0
287 /// %7 = extractelement <4 x i8> %5, i32 1
288 /// %ins2 = insertelement <4 x i8> %ins1, i8 %7, i32 1
289 /// %8 = extractelement <4 x i8> %5, i32 2
290 /// %ins3 = insertelement <4 x i8> %ins2, i8 %8, i32 2
291 /// %9 = extractelement <4 x i8> %5, i32 3
292 /// %ins4 = insertelement <4 x i8> %ins3, i8 %9, i32 3
293 /// ret <4 x i8> %ins4
294 /// InstCombiner transforms this into a shuffle and vector mul
295 /// Mask will return the Shuffle Mask equivalent to the extracted elements.
296 /// TODO: Can we split off and reuse the shuffle mask detection from
297 /// TargetTransformInfo::getInstructionThroughput?
300  auto *EI0 = cast<ExtractElementInst>(VL[0]);
301  unsigned Size =
302  cast<FixedVectorType>(EI0->getVectorOperandType())->getNumElements();
303  Value *Vec1 = nullptr;
304  Value *Vec2 = nullptr;
305  enum ShuffleMode { Unknown, Select, Permute };
306  ShuffleMode CommonShuffleMode = Unknown;
307  for (unsigned I = 0, E = VL.size(); I < E; ++I) {
308  auto *EI = cast<ExtractElementInst>(VL[I]);
309  auto *Vec = EI->getVectorOperand();
310  // All vector operands must have the same number of vector elements.
311  if (cast<FixedVectorType>(Vec->getType())->getNumElements() != Size)
312  return None;
313  auto *Idx = dyn_cast<ConstantInt>(EI->getIndexOperand());
314  if (!Idx)
315  return None;
316  // Undefined behavior if Idx is negative or >= Size.
317  if (Idx->getValue().uge(Size)) {
318  Mask.push_back(UndefMaskElem);
319  continue;
320  }
321  unsigned IntIdx = Idx->getValue().getZExtValue();
322  Mask.push_back(IntIdx);
323  // We can extractelement from undef or poison vector.
324  if (isa<UndefValue>(Vec))
325  continue;
326  // For correct shuffling we have to have at most 2 different vector operands
327  // in all extractelement instructions.
328  if (!Vec1 || Vec1 == Vec)
329  Vec1 = Vec;
330  else if (!Vec2 || Vec2 == Vec)
331  Vec2 = Vec;
332  else
333  return None;
334  if (CommonShuffleMode == Permute)
335  continue;
336  // If the extract index is not the same as the operation number, it is a
337  // permutation.
338  if (IntIdx != I) {
339  CommonShuffleMode = Permute;
340  continue;
341  }
342  CommonShuffleMode = Select;
343  }
344  // If we're not crossing lanes in different vectors, consider it as blending.
345  if (CommonShuffleMode == Select && Vec2)
347  // If Vec2 was never used, we have a permutation of a single vector, otherwise
348  // we have permutation of 2 vectors.
351 }
352 
353 namespace {
354 
355 /// Main data required for vectorization of instructions.
356 struct InstructionsState {
357  /// The very first instruction in the list with the main opcode.
358  Value *OpValue = nullptr;
359 
360  /// The main/alternate instruction.
361  Instruction *MainOp = nullptr;
362  Instruction *AltOp = nullptr;
363 
364  /// The main/alternate opcodes for the list of instructions.
365  unsigned getOpcode() const {
366  return MainOp ? MainOp->getOpcode() : 0;
367  }
368 
369  unsigned getAltOpcode() const {
370  return AltOp ? AltOp->getOpcode() : 0;
371  }
372 
373  /// Some of the instructions in the list have alternate opcodes.
374  bool isAltShuffle() const { return getOpcode() != getAltOpcode(); }
375 
376  bool isOpcodeOrAlt(Instruction *I) const {
377  unsigned CheckedOpcode = I->getOpcode();
378  return getOpcode() == CheckedOpcode || getAltOpcode() == CheckedOpcode;
379  }
380 
381  InstructionsState() = delete;
382  InstructionsState(Value *OpValue, Instruction *MainOp, Instruction *AltOp)
383  : OpValue(OpValue), MainOp(MainOp), AltOp(AltOp) {}
384 };
385 
386 } // end anonymous namespace
387 
388 /// Chooses the correct key for scheduling data. If \p Op has the same (or
389 /// alternate) opcode as \p OpValue, the key is \p Op. Otherwise the key is \p
390 /// OpValue.
391 static Value *isOneOf(const InstructionsState &S, Value *Op) {
392  auto *I = dyn_cast<Instruction>(Op);
393  if (I && S.isOpcodeOrAlt(I))
394  return Op;
395  return S.OpValue;
396 }
397 
398 /// \returns true if \p Opcode is allowed as part of of the main/alternate
399 /// instruction for SLP vectorization.
400 ///
401 /// Example of unsupported opcode is SDIV that can potentially cause UB if the
402 /// "shuffled out" lane would result in division by zero.
403 static bool isValidForAlternation(unsigned Opcode) {
404  if (Instruction::isIntDivRem(Opcode))
405  return false;
406 
407  return true;
408 }
409 
410 /// \returns analysis of the Instructions in \p VL described in
411 /// InstructionsState, the Opcode that we suppose the whole list
412 /// could be vectorized even if its structure is diverse.
413 static InstructionsState getSameOpcode(ArrayRef<Value *> VL,
414  unsigned BaseIndex = 0) {
415  // Make sure these are all Instructions.
416  if (llvm::any_of(VL, [](Value *V) { return !isa<Instruction>(V); }))
417  return InstructionsState(VL[BaseIndex], nullptr, nullptr);
418 
419  bool IsCastOp = isa<CastInst>(VL[BaseIndex]);
420  bool IsBinOp = isa<BinaryOperator>(VL[BaseIndex]);
421  unsigned Opcode = cast<Instruction>(VL[BaseIndex])->getOpcode();
422  unsigned AltOpcode = Opcode;
423  unsigned AltIndex = BaseIndex;
424 
425  // Check for one alternate opcode from another BinaryOperator.
426  // TODO - generalize to support all operators (types, calls etc.).
427  for (int Cnt = 0, E = VL.size(); Cnt < E; Cnt++) {
428  unsigned InstOpcode = cast<Instruction>(VL[Cnt])->getOpcode();
429  if (IsBinOp && isa<BinaryOperator>(VL[Cnt])) {
430  if (InstOpcode == Opcode || InstOpcode == AltOpcode)
431  continue;
432  if (Opcode == AltOpcode && isValidForAlternation(InstOpcode) &&
433  isValidForAlternation(Opcode)) {
434  AltOpcode = InstOpcode;
435  AltIndex = Cnt;
436  continue;
437  }
438  } else if (IsCastOp && isa<CastInst>(VL[Cnt])) {
439  Type *Ty0 = cast<Instruction>(VL[BaseIndex])->getOperand(0)->getType();
440  Type *Ty1 = cast<Instruction>(VL[Cnt])->getOperand(0)->getType();
441  if (Ty0 == Ty1) {
442  if (InstOpcode == Opcode || InstOpcode == AltOpcode)
443  continue;
444  if (Opcode == AltOpcode) {
445  assert(isValidForAlternation(Opcode) &&
446  isValidForAlternation(InstOpcode) &&
447  "Cast isn't safe for alternation, logic needs to be updated!");
448  AltOpcode = InstOpcode;
449  AltIndex = Cnt;
450  continue;
451  }
452  }
453  } else if (InstOpcode == Opcode || InstOpcode == AltOpcode)
454  continue;
455  return InstructionsState(VL[BaseIndex], nullptr, nullptr);
456  }
457 
458  return InstructionsState(VL[BaseIndex], cast<Instruction>(VL[BaseIndex]),
459  cast<Instruction>(VL[AltIndex]));
460 }
461 
462 /// \returns true if all of the values in \p VL have the same type or false
463 /// otherwise.
465  Type *Ty = VL[0]->getType();
466  for (int i = 1, e = VL.size(); i < e; i++)
467  if (VL[i]->getType() != Ty)
468  return false;
469 
470  return true;
471 }
472 
473 /// \returns True if Extract{Value,Element} instruction extracts element Idx.
475  unsigned Opcode = E->getOpcode();
476  assert((Opcode == Instruction::ExtractElement ||
477  Opcode == Instruction::ExtractValue) &&
478  "Expected extractelement or extractvalue instruction.");
479  if (Opcode == Instruction::ExtractElement) {
480  auto *CI = dyn_cast<ConstantInt>(E->getOperand(1));
481  if (!CI)
482  return None;
483  return CI->getZExtValue();
484  }
485  ExtractValueInst *EI = cast<ExtractValueInst>(E);
486  if (EI->getNumIndices() != 1)
487  return None;
488  return *EI->idx_begin();
489 }
490 
491 /// \returns True if in-tree use also needs extract. This refers to
492 /// possible scalar operand in vectorized instruction.
493 static bool InTreeUserNeedToExtract(Value *Scalar, Instruction *UserInst,
494  TargetLibraryInfo *TLI) {
495  unsigned Opcode = UserInst->getOpcode();
496  switch (Opcode) {
497  case Instruction::Load: {
498  LoadInst *LI = cast<LoadInst>(UserInst);
499  return (LI->getPointerOperand() == Scalar);
500  }
501  case Instruction::Store: {
502  StoreInst *SI = cast<StoreInst>(UserInst);
503  return (SI->getPointerOperand() == Scalar);
504  }
505  case Instruction::Call: {
506  CallInst *CI = cast<CallInst>(UserInst);
508  for (unsigned i = 0, e = CI->getNumArgOperands(); i != e; ++i) {
510  return (CI->getArgOperand(i) == Scalar);
511  }
513  }
514  default:
515  return false;
516  }
517 }
518 
519 /// \returns the AA location that is being access by the instruction.
521  if (StoreInst *SI = dyn_cast<StoreInst>(I))
522  return MemoryLocation::get(SI);
523  if (LoadInst *LI = dyn_cast<LoadInst>(I))
524  return MemoryLocation::get(LI);
525  return MemoryLocation();
526 }
527 
528 /// \returns True if the instruction is not a volatile or atomic load/store.
529 static bool isSimple(Instruction *I) {
530  if (LoadInst *LI = dyn_cast<LoadInst>(I))
531  return LI->isSimple();
532  if (StoreInst *SI = dyn_cast<StoreInst>(I))
533  return SI->isSimple();
534  if (MemIntrinsic *MI = dyn_cast<MemIntrinsic>(I))
535  return !MI->isVolatile();
536  return true;
537 }
538 
539 /// Shuffles \p Mask in accordance with the given \p SubMask.
541  if (SubMask.empty())
542  return;
543  if (Mask.empty()) {
544  Mask.append(SubMask.begin(), SubMask.end());
545  return;
546  }
547  SmallVector<int> NewMask(SubMask.size(), UndefMaskElem);
548  int TermValue = std::min(Mask.size(), SubMask.size());
549  for (int I = 0, E = SubMask.size(); I < E; ++I) {
550  if (SubMask[I] >= TermValue || SubMask[I] == UndefMaskElem ||
551  Mask[SubMask[I]] >= TermValue)
552  continue;
553  NewMask[I] = Mask[SubMask[I]];
554  }
555  Mask.swap(NewMask);
556 }
557 
558 /// Order may have elements assigned special value (size) which is out of
559 /// bounds. Such indices only appear on places which correspond to undef values
560 /// (see canReuseExtract for details) and used in order to avoid undef values
561 /// have effect on operands ordering.
562 /// The first loop below simply finds all unused indices and then the next loop
563 /// nest assigns these indices for undef values positions.
564 /// As an example below Order has two undef positions and they have assigned
565 /// values 3 and 7 respectively:
566 /// before: 6 9 5 4 9 2 1 0
567 /// after: 6 3 5 4 7 2 1 0
568 /// \returns Fixed ordering.
570  const unsigned Sz = Order.size();
571  SmallBitVector UsedIndices(Sz);
572  SmallVector<int> MaskedIndices;
573  for (unsigned I = 0; I < Sz; ++I) {
574  if (Order[I] < Sz)
575  UsedIndices.set(Order[I]);
576  else
577  MaskedIndices.push_back(I);
578  }
579  if (MaskedIndices.empty())
580  return;
581  SmallVector<int> AvailableIndices(MaskedIndices.size());
582  unsigned Cnt = 0;
583  int Idx = UsedIndices.find_first();
584  do {
585  AvailableIndices[Cnt] = Idx;
586  Idx = UsedIndices.find_next(Idx);
587  ++Cnt;
588  } while (Idx > 0);
589  assert(Cnt == MaskedIndices.size() && "Non-synced masked/available indices.");
590  for (int I = 0, E = MaskedIndices.size(); I < E; ++I)
591  Order[MaskedIndices[I]] = AvailableIndices[I];
592 }
593 
594 namespace llvm {
595 
598  Mask.clear();
599  const unsigned E = Indices.size();
600  Mask.resize(E, UndefMaskElem);
601  for (unsigned I = 0; I < E; ++I)
602  Mask[Indices[I]] = I;
603 }
604 
605 /// \returns inserting index of InsertElement or InsertValue instruction,
606 /// using Offset as base offset for index.
607 static Optional<int> getInsertIndex(Value *InsertInst, unsigned Offset) {
608  int Index = Offset;
609  if (auto *IE = dyn_cast<InsertElementInst>(InsertInst)) {
610  if (auto *CI = dyn_cast<ConstantInt>(IE->getOperand(2))) {
611  auto *VT = cast<FixedVectorType>(IE->getType());
612  if (CI->getValue().uge(VT->getNumElements()))
613  return UndefMaskElem;
614  Index *= VT->getNumElements();
615  Index += CI->getZExtValue();
616  return Index;
617  }
618  if (isa<UndefValue>(IE->getOperand(2)))
619  return UndefMaskElem;
620  return None;
621  }
622 
623  auto *IV = cast<InsertValueInst>(InsertInst);
624  Type *CurrentType = IV->getType();
625  for (unsigned I : IV->indices()) {
626  if (auto *ST = dyn_cast<StructType>(CurrentType)) {
627  Index *= ST->getNumElements();
628  CurrentType = ST->getElementType(I);
629  } else if (auto *AT = dyn_cast<ArrayType>(CurrentType)) {
630  Index *= AT->getNumElements();
631  CurrentType = AT->getElementType();
632  } else {
633  return None;
634  }
635  Index += I;
636  }
637  return Index;
638 }
639 
640 /// Reorders the list of scalars in accordance with the given \p Order and then
641 /// the \p Mask. \p Order - is the original order of the scalars, need to
642 /// reorder scalars into an unordered state at first according to the given
643 /// order. Then the ordered scalars are shuffled once again in accordance with
644 /// the provided mask.
647  assert(!Mask.empty() && "Expected non-empty mask.");
648  SmallVector<Value *> Prev(Scalars.size(),
649  UndefValue::get(Scalars.front()->getType()));
650  Prev.swap(Scalars);
651  for (unsigned I = 0, E = Prev.size(); I < E; ++I)
652  if (Mask[I] != UndefMaskElem)
653  Scalars[Mask[I]] = Prev[I];
654 }
655 
656 namespace slpvectorizer {
657 
658 /// Bottom Up SLP Vectorizer.
659 class BoUpSLP {
660  struct TreeEntry;
661  struct ScheduleData;
662 
663 public:
671 
673  TargetLibraryInfo *TLi, AAResults *Aa, LoopInfo *Li,
676  : F(Func), SE(Se), TTI(Tti), TLI(TLi), AA(Aa), LI(Li), DT(Dt), AC(AC),
677  DB(DB), DL(DL), ORE(ORE), Builder(Se->getContext()) {
678  CodeMetrics::collectEphemeralValues(F, AC, EphValues);
679  // Use the vector register size specified by the target unless overridden
680  // by a command-line option.
681  // TODO: It would be better to limit the vectorization factor based on
682  // data type rather than just register size. For example, x86 AVX has
683  // 256-bit registers, but it does not support integer operations
684  // at that width (that requires AVX2).
685  if (MaxVectorRegSizeOption.getNumOccurrences())
686  MaxVecRegSize = MaxVectorRegSizeOption;
687  else
688  MaxVecRegSize =
690  .getFixedSize();
691 
692  if (MinVectorRegSizeOption.getNumOccurrences())
693  MinVecRegSize = MinVectorRegSizeOption;
694  else
695  MinVecRegSize = TTI->getMinVectorRegisterBitWidth();
696  }
697 
698  /// Vectorize the tree that starts with the elements in \p VL.
699  /// Returns the vectorized root.
700  Value *vectorizeTree();
701 
702  /// Vectorize the tree but with the list of externally used values \p
703  /// ExternallyUsedValues. Values in this MapVector can be replaced but the
704  /// generated extractvalue instructions.
705  Value *vectorizeTree(ExtraValueToDebugLocsMap &ExternallyUsedValues);
706 
707  /// \returns the cost incurred by unwanted spills and fills, caused by
708  /// holding live values over call sites.
709  InstructionCost getSpillCost() const;
710 
711  /// \returns the vectorization cost of the subtree that starts at \p VL.
712  /// A negative number means that this is profitable.
713  InstructionCost getTreeCost(ArrayRef<Value *> VectorizedVals = None);
714 
715  /// Construct a vectorizable tree that starts at \p Roots, ignoring users for
716  /// the purpose of scheduling and extraction in the \p UserIgnoreLst.
717  void buildTree(ArrayRef<Value *> Roots,
718  ArrayRef<Value *> UserIgnoreLst = None);
719 
720  /// Builds external uses of the vectorized scalars, i.e. the list of
721  /// vectorized scalars to be extracted, their lanes and their scalar users. \p
722  /// ExternallyUsedValues contains additional list of external uses to handle
723  /// vectorization of reductions.
724  void
725  buildExternalUses(const ExtraValueToDebugLocsMap &ExternallyUsedValues = {});
726 
727  /// Clear the internal data structures that are created by 'buildTree'.
728  void deleteTree() {
729  VectorizableTree.clear();
730  ScalarToTreeEntry.clear();
731  MustGather.clear();
732  ExternalUses.clear();
733  for (auto &Iter : BlocksSchedules) {
734  BlockScheduling *BS = Iter.second.get();
735  BS->clear();
736  }
737  MinBWs.clear();
738  InstrElementSize.clear();
739  }
740 
741  unsigned getTreeSize() const { return VectorizableTree.size(); }
742 
743  /// Perform LICM and CSE on the newly generated gather sequences.
744  void optimizeGatherSequence();
745 
746  /// Reorders the current graph to the most profitable order starting from the
747  /// root node to the leaf nodes. The best order is chosen only from the nodes
748  /// of the same size (vectorization factor). Smaller nodes are considered
749  /// parts of subgraph with smaller VF and they are reordered independently. We
750  /// can make it because we still need to extend smaller nodes to the wider VF
751  /// and we can merge reordering shuffles with the widening shuffles.
752  void reorderTopToBottom();
753 
754  /// Reorders the current graph to the most profitable order starting from
755  /// leaves to the root. It allows to rotate small subgraphs and reduce the
756  /// number of reshuffles if the leaf nodes use the same order. In this case we
757  /// can merge the orders and just shuffle user node instead of shuffling its
758  /// operands. Plus, even the leaf nodes have different orders, it allows to
759  /// sink reordering in the graph closer to the root node and merge it later
760  /// during analysis.
761  void reorderBottomToTop();
762 
763  /// \return The vector element size in bits to use when vectorizing the
764  /// expression tree ending at \p V. If V is a store, the size is the width of
765  /// the stored value. Otherwise, the size is the width of the largest loaded
766  /// value reaching V. This method is used by the vectorizer to calculate
767  /// vectorization factors.
768  unsigned getVectorElementSize(Value *V);
769 
770  /// Compute the minimum type sizes required to represent the entries in a
771  /// vectorizable tree.
773 
774  // \returns maximum vector register size as set by TTI or overridden by cl::opt.
775  unsigned getMaxVecRegSize() const {
776  return MaxVecRegSize;
777  }
778 
779  // \returns minimum vector register size as set by cl::opt.
780  unsigned getMinVecRegSize() const {
781  return MinVecRegSize;
782  }
783 
784  unsigned getMinVF(unsigned Sz) const {
785  return std::max(2U, getMinVecRegSize() / Sz);
786  }
787 
788  unsigned getMaximumVF(unsigned ElemWidth, unsigned Opcode) const {
789  unsigned MaxVF = MaxVFOption.getNumOccurrences() ?
790  MaxVFOption : TTI->getMaximumVF(ElemWidth, Opcode);
791  return MaxVF ? MaxVF : UINT_MAX;
792  }
793 
794  /// Check if homogeneous aggregate is isomorphic to some VectorType.
795  /// Accepts homogeneous multidimensional aggregate of scalars/vectors like
796  /// {[4 x i16], [4 x i16]}, { <2 x float>, <2 x float> },
797  /// {{{i16, i16}, {i16, i16}}, {{i16, i16}, {i16, i16}}} and so on.
798  ///
799  /// \returns number of elements in vector if isomorphism exists, 0 otherwise.
800  unsigned canMapToVector(Type *T, const DataLayout &DL) const;
801 
802  /// \returns True if the VectorizableTree is both tiny and not fully
803  /// vectorizable. We do not vectorize such trees.
804  bool isTreeTinyAndNotFullyVectorizable() const;
805 
806  /// Assume that a legal-sized 'or'-reduction of shifted/zexted loaded values
807  /// can be load combined in the backend. Load combining may not be allowed in
808  /// the IR optimizer, so we do not want to alter the pattern. For example,
809  /// partially transforming a scalar bswap() pattern into vector code is
810  /// effectively impossible for the backend to undo.
811  /// TODO: If load combining is allowed in the IR optimizer, this analysis
812  /// may not be necessary.
813  bool isLoadCombineReductionCandidate(RecurKind RdxKind) const;
814 
815  /// Assume that a vector of stores of bitwise-or/shifted/zexted loaded values
816  /// can be load combined in the backend. Load combining may not be allowed in
817  /// the IR optimizer, so we do not want to alter the pattern. For example,
818  /// partially transforming a scalar bswap() pattern into vector code is
819  /// effectively impossible for the backend to undo.
820  /// TODO: If load combining is allowed in the IR optimizer, this analysis
821  /// may not be necessary.
822  bool isLoadCombineCandidate() const;
823 
825 
826  /// This structure holds any data we need about the edges being traversed
827  /// during buildTree_rec(). We keep track of:
828  /// (i) the user TreeEntry index, and
829  /// (ii) the index of the edge.
830  struct EdgeInfo {
831  EdgeInfo() = default;
832  EdgeInfo(TreeEntry *UserTE, unsigned EdgeIdx)
833  : UserTE(UserTE), EdgeIdx(EdgeIdx) {}
834  /// The user TreeEntry.
835  TreeEntry *UserTE = nullptr;
836  /// The operand index of the use.
837  unsigned EdgeIdx = UINT_MAX;
838 #ifndef NDEBUG
839  friend inline raw_ostream &operator<<(raw_ostream &OS,
840  const BoUpSLP::EdgeInfo &EI) {
841  EI.dump(OS);
842  return OS;
843  }
844  /// Debug print.
845  void dump(raw_ostream &OS) const {
846  OS << "{User:" << (UserTE ? std::to_string(UserTE->Idx) : "null")
847  << " EdgeIdx:" << EdgeIdx << "}";
848  }
849  LLVM_DUMP_METHOD void dump() const { dump(dbgs()); }
850 #endif
851  };
852 
853  /// A helper data structure to hold the operands of a vector of instructions.
854  /// This supports a fixed vector length for all operand vectors.
855  class VLOperands {
856  /// For each operand we need (i) the value, and (ii) the opcode that it
857  /// would be attached to if the expression was in a left-linearized form.
858  /// This is required to avoid illegal operand reordering.
859  /// For example:
860  /// \verbatim
861  /// 0 Op1
862  /// |/
863  /// Op1 Op2 Linearized + Op2
864  /// \ / ----------> |/
865  /// - -
866  ///
867  /// Op1 - Op2 (0 + Op1) - Op2
868  /// \endverbatim
869  ///
870  /// Value Op1 is attached to a '+' operation, and Op2 to a '-'.
871  ///
872  /// Another way to think of this is to track all the operations across the
873  /// path from the operand all the way to the root of the tree and to
874  /// calculate the operation that corresponds to this path. For example, the
875  /// path from Op2 to the root crosses the RHS of the '-', therefore the
876  /// corresponding operation is a '-' (which matches the one in the
877  /// linearized tree, as shown above).
878  ///
879  /// For lack of a better term, we refer to this operation as Accumulated
880  /// Path Operation (APO).
881  struct OperandData {
882  OperandData() = default;
883  OperandData(Value *V, bool APO, bool IsUsed)
884  : V(V), APO(APO), IsUsed(IsUsed) {}
885  /// The operand value.
886  Value *V = nullptr;
887  /// TreeEntries only allow a single opcode, or an alternate sequence of
888  /// them (e.g, +, -). Therefore, we can safely use a boolean value for the
889  /// APO. It is set to 'true' if 'V' is attached to an inverse operation
890  /// in the left-linearized form (e.g., Sub/Div), and 'false' otherwise
891  /// (e.g., Add/Mul)
892  bool APO = false;
893  /// Helper data for the reordering function.
894  bool IsUsed = false;
895  };
896 
897  /// During operand reordering, we are trying to select the operand at lane
898  /// that matches best with the operand at the neighboring lane. Our
899  /// selection is based on the type of value we are looking for. For example,
900  /// if the neighboring lane has a load, we need to look for a load that is
901  /// accessing a consecutive address. These strategies are summarized in the
902  /// 'ReorderingMode' enumerator.
903  enum class ReorderingMode {
904  Load, ///< Matching loads to consecutive memory addresses
905  Opcode, ///< Matching instructions based on opcode (same or alternate)
906  Constant, ///< Matching constants
907  Splat, ///< Matching the same instruction multiple times (broadcast)
908  Failed, ///< We failed to create a vectorizable group
909  };
910 
912 
913  /// A vector of operand vectors.
915 
916  const DataLayout &DL;
917  ScalarEvolution &SE;
918  const BoUpSLP &R;
919 
920  /// \returns the operand data at \p OpIdx and \p Lane.
921  OperandData &getData(unsigned OpIdx, unsigned Lane) {
922  return OpsVec[OpIdx][Lane];
923  }
924 
925  /// \returns the operand data at \p OpIdx and \p Lane. Const version.
926  const OperandData &getData(unsigned OpIdx, unsigned Lane) const {
927  return OpsVec[OpIdx][Lane];
928  }
929 
930  /// Clears the used flag for all entries.
931  void clearUsed() {
932  for (unsigned OpIdx = 0, NumOperands = getNumOperands();
933  OpIdx != NumOperands; ++OpIdx)
934  for (unsigned Lane = 0, NumLanes = getNumLanes(); Lane != NumLanes;
935  ++Lane)
936  OpsVec[OpIdx][Lane].IsUsed = false;
937  }
938 
939  /// Swap the operand at \p OpIdx1 with that one at \p OpIdx2.
940  void swap(unsigned OpIdx1, unsigned OpIdx2, unsigned Lane) {
941  std::swap(OpsVec[OpIdx1][Lane], OpsVec[OpIdx2][Lane]);
942  }
943 
944  // The hard-coded scores listed here are not very important. When computing
945  // the scores of matching one sub-tree with another, we are basically
946  // counting the number of values that are matching. So even if all scores
947  // are set to 1, we would still get a decent matching result.
948  // However, sometimes we have to break ties. For example we may have to
949  // choose between matching loads vs matching opcodes. This is what these
950  // scores are helping us with: they provide the order of preference.
951 
952  /// Loads from consecutive memory addresses, e.g. load(A[i]), load(A[i+1]).
953  static const int ScoreConsecutiveLoads = 3;
954  /// ExtractElementInst from same vector and consecutive indexes.
955  static const int ScoreConsecutiveExtracts = 3;
956  /// Constants.
957  static const int ScoreConstants = 2;
958  /// Instructions with the same opcode.
959  static const int ScoreSameOpcode = 2;
960  /// Instructions with alt opcodes (e.g, add + sub).
961  static const int ScoreAltOpcodes = 1;
962  /// Identical instructions (a.k.a. splat or broadcast).
963  static const int ScoreSplat = 1;
964  /// Matching with an undef is preferable to failing.
965  static const int ScoreUndef = 1;
966  /// Score for failing to find a decent match.
967  static const int ScoreFail = 0;
968  /// User exteranl to the vectorized code.
969  static const int ExternalUseCost = 1;
970  /// The user is internal but in a different lane.
971  static const int UserInDiffLaneCost = ExternalUseCost;
972 
973  /// \returns the score of placing \p V1 and \p V2 in consecutive lanes.
974  static int getShallowScore(Value *V1, Value *V2, const DataLayout &DL,
975  ScalarEvolution &SE) {
976  auto *LI1 = dyn_cast<LoadInst>(V1);
977  auto *LI2 = dyn_cast<LoadInst>(V2);
978  if (LI1 && LI2) {
979  if (LI1->getParent() != LI2->getParent())
980  return VLOperands::ScoreFail;
981 
983  LI1->getType(), LI1->getPointerOperand(), LI2->getType(),
984  LI2->getPointerOperand(), DL, SE, /*StrictCheck=*/true);
985  return (Dist && *Dist == 1) ? VLOperands::ScoreConsecutiveLoads
986  : VLOperands::ScoreFail;
987  }
988 
989  auto *C1 = dyn_cast<Constant>(V1);
990  auto *C2 = dyn_cast<Constant>(V2);
991  if (C1 && C2)
992  return VLOperands::ScoreConstants;
993 
994  // Extracts from consecutive indexes of the same vector better score as
995  // the extracts could be optimized away.
996  Value *EV;
997  ConstantInt *Ex1Idx, *Ex2Idx;
998  if (match(V1, m_ExtractElt(m_Value(EV), m_ConstantInt(Ex1Idx))) &&
999  match(V2, m_ExtractElt(m_Deferred(EV), m_ConstantInt(Ex2Idx))) &&
1000  Ex1Idx->getZExtValue() + 1 == Ex2Idx->getZExtValue())
1001  return VLOperands::ScoreConsecutiveExtracts;
1002 
1003  auto *I1 = dyn_cast<Instruction>(V1);
1004  auto *I2 = dyn_cast<Instruction>(V2);
1005  if (I1 && I2) {
1006  if (I1 == I2)
1007  return VLOperands::ScoreSplat;
1008  InstructionsState S = getSameOpcode({I1, I2});
1009  // Note: Only consider instructions with <= 2 operands to avoid
1010  // complexity explosion.
1011  if (S.getOpcode() && S.MainOp->getNumOperands() <= 2)
1012  return S.isAltShuffle() ? VLOperands::ScoreAltOpcodes
1013  : VLOperands::ScoreSameOpcode;
1014  }
1015 
1016  if (isa<UndefValue>(V2))
1017  return VLOperands::ScoreUndef;
1018 
1019  return VLOperands::ScoreFail;
1020  }
1021 
1022  /// Holds the values and their lane that are taking part in the look-ahead
1023  /// score calculation. This is used in the external uses cost calculation.
1024  SmallDenseMap<Value *, int> InLookAheadValues;
1025 
1026  /// \Returns the additinal cost due to uses of \p LHS and \p RHS that are
1027  /// either external to the vectorized code, or require shuffling.
1028  int getExternalUsesCost(const std::pair<Value *, int> &LHS,
1029  const std::pair<Value *, int> &RHS) {
1030  int Cost = 0;
1031  std::array<std::pair<Value *, int>, 2> Values = {{LHS, RHS}};
1032  for (int Idx = 0, IdxE = Values.size(); Idx != IdxE; ++Idx) {
1033  Value *V = Values[Idx].first;
1034  if (isa<Constant>(V)) {
1035  // Since this is a function pass, it doesn't make semantic sense to
1036  // walk the users of a subclass of Constant. The users could be in
1037  // another function, or even another module that happens to be in
1038  // the same LLVMContext.
1039  continue;
1040  }
1041 
1042  // Calculate the absolute lane, using the minimum relative lane of LHS
1043  // and RHS as base and Idx as the offset.
1044  int Ln = std::min(LHS.second, RHS.second) + Idx;
1045  assert(Ln >= 0 && "Bad lane calculation");
1046  unsigned UsersBudget = LookAheadUsersBudget;
1047  for (User *U : V->users()) {
1048  if (const TreeEntry *UserTE = R.getTreeEntry(U)) {
1049  // The user is in the VectorizableTree. Check if we need to insert.
1050  auto It = llvm::find(UserTE->Scalars, U);
1051  assert(It != UserTE->Scalars.end() && "U is in UserTE");
1052  int UserLn = std::distance(UserTE->Scalars.begin(), It);
1053  assert(UserLn >= 0 && "Bad lane");
1054  if (UserLn != Ln)
1055  Cost += UserInDiffLaneCost;
1056  } else {
1057  // Check if the user is in the look-ahead code.
1058  auto It2 = InLookAheadValues.find(U);
1059  if (It2 != InLookAheadValues.end()) {
1060  // The user is in the look-ahead code. Check the lane.
1061  if (It2->second != Ln)
1062  Cost += UserInDiffLaneCost;
1063  } else {
1064  // The user is neither in SLP tree nor in the look-ahead code.
1065  Cost += ExternalUseCost;
1066  }
1067  }
1068  // Limit the number of visited uses to cap compilation time.
1069  if (--UsersBudget == 0)
1070  break;
1071  }
1072  }
1073  return Cost;
1074  }
1075 
1076  /// Go through the operands of \p LHS and \p RHS recursively until \p
1077  /// MaxLevel, and return the cummulative score. For example:
1078  /// \verbatim
1079  /// A[0] B[0] A[1] B[1] C[0] D[0] B[1] A[1]
1080  /// \ / \ / \ / \ /
1081  /// + + + +
1082  /// G1 G2 G3 G4
1083  /// \endverbatim
1084  /// The getScoreAtLevelRec(G1, G2) function will try to match the nodes at
1085  /// each level recursively, accumulating the score. It starts from matching
1086  /// the additions at level 0, then moves on to the loads (level 1). The
1087  /// score of G1 and G2 is higher than G1 and G3, because {A[0],A[1]} and
1088  /// {B[0],B[1]} match with VLOperands::ScoreConsecutiveLoads, while
1089  /// {A[0],C[0]} has a score of VLOperands::ScoreFail.
1090  /// Please note that the order of the operands does not matter, as we
1091  /// evaluate the score of all profitable combinations of operands. In
1092  /// other words the score of G1 and G4 is the same as G1 and G2. This
1093  /// heuristic is based on ideas described in:
1094  /// Look-ahead SLP: Auto-vectorization in the presence of commutative
1095  /// operations, CGO 2018 by Vasileios Porpodas, Rodrigo C. O. Rocha,
1096  /// Luís F. W. Góes
1097  int getScoreAtLevelRec(const std::pair<Value *, int> &LHS,
1098  const std::pair<Value *, int> &RHS, int CurrLevel,
1099  int MaxLevel) {
1100 
1101  Value *V1 = LHS.first;
1102  Value *V2 = RHS.first;
1103  // Get the shallow score of V1 and V2.
1104  int ShallowScoreAtThisLevel =
1105  std::max((int)ScoreFail, getShallowScore(V1, V2, DL, SE) -
1106  getExternalUsesCost(LHS, RHS));
1107  int Lane1 = LHS.second;
1108  int Lane2 = RHS.second;
1109 
1110  // If reached MaxLevel,
1111  // or if V1 and V2 are not instructions,
1112  // or if they are SPLAT,
1113  // or if they are not consecutive, early return the current cost.
1114  auto *I1 = dyn_cast<Instruction>(V1);
1115  auto *I2 = dyn_cast<Instruction>(V2);
1116  if (CurrLevel == MaxLevel || !(I1 && I2) || I1 == I2 ||
1117  ShallowScoreAtThisLevel == VLOperands::ScoreFail ||
1118  (isa<LoadInst>(I1) && isa<LoadInst>(I2) && ShallowScoreAtThisLevel))
1119  return ShallowScoreAtThisLevel;
1120  assert(I1 && I2 && "Should have early exited.");
1121 
1122  // Keep track of in-tree values for determining the external-use cost.
1123  InLookAheadValues[V1] = Lane1;
1124  InLookAheadValues[V2] = Lane2;
1125 
1126  // Contains the I2 operand indexes that got matched with I1 operands.
1127  SmallSet<unsigned, 4> Op2Used;
1128 
1129  // Recursion towards the operands of I1 and I2. We are trying all possbile
1130  // operand pairs, and keeping track of the best score.
1131  for (unsigned OpIdx1 = 0, NumOperands1 = I1->getNumOperands();
1132  OpIdx1 != NumOperands1; ++OpIdx1) {
1133  // Try to pair op1I with the best operand of I2.
1134  int MaxTmpScore = 0;
1135  unsigned MaxOpIdx2 = 0;
1136  bool FoundBest = false;
1137  // If I2 is commutative try all combinations.
1138  unsigned FromIdx = isCommutative(I2) ? 0 : OpIdx1;
1139  unsigned ToIdx = isCommutative(I2)
1140  ? I2->getNumOperands()
1141  : std::min(I2->getNumOperands(), OpIdx1 + 1);
1142  assert(FromIdx <= ToIdx && "Bad index");
1143  for (unsigned OpIdx2 = FromIdx; OpIdx2 != ToIdx; ++OpIdx2) {
1144  // Skip operands already paired with OpIdx1.
1145  if (Op2Used.count(OpIdx2))
1146  continue;
1147  // Recursively calculate the cost at each level
1148  int TmpScore = getScoreAtLevelRec({I1->getOperand(OpIdx1), Lane1},
1149  {I2->getOperand(OpIdx2), Lane2},
1150  CurrLevel + 1, MaxLevel);
1151  // Look for the best score.
1152  if (TmpScore > VLOperands::ScoreFail && TmpScore > MaxTmpScore) {
1153  MaxTmpScore = TmpScore;
1154  MaxOpIdx2 = OpIdx2;
1155  FoundBest = true;
1156  }
1157  }
1158  if (FoundBest) {
1159  // Pair {OpIdx1, MaxOpIdx2} was found to be best. Never revisit it.
1160  Op2Used.insert(MaxOpIdx2);
1161  ShallowScoreAtThisLevel += MaxTmpScore;
1162  }
1163  }
1164  return ShallowScoreAtThisLevel;
1165  }
1166 
1167  /// \Returns the look-ahead score, which tells us how much the sub-trees
1168  /// rooted at \p LHS and \p RHS match, the more they match the higher the
1169  /// score. This helps break ties in an informed way when we cannot decide on
1170  /// the order of the operands by just considering the immediate
1171  /// predecessors.
1172  int getLookAheadScore(const std::pair<Value *, int> &LHS,
1173  const std::pair<Value *, int> &RHS) {
1174  InLookAheadValues.clear();
1175  return getScoreAtLevelRec(LHS, RHS, 1, LookAheadMaxDepth);
1176  }
1177 
1178  // Search all operands in Ops[*][Lane] for the one that matches best
1179  // Ops[OpIdx][LastLane] and return its opreand index.
1180  // If no good match can be found, return None.
1182  getBestOperand(unsigned OpIdx, int Lane, int LastLane,
1183  ArrayRef<ReorderingMode> ReorderingModes) {
1184  unsigned NumOperands = getNumOperands();
1185 
1186  // The operand of the previous lane at OpIdx.
1187  Value *OpLastLane = getData(OpIdx, LastLane).V;
1188 
1189  // Our strategy mode for OpIdx.
1190  ReorderingMode RMode = ReorderingModes[OpIdx];
1191 
1192  // The linearized opcode of the operand at OpIdx, Lane.
1193  bool OpIdxAPO = getData(OpIdx, Lane).APO;
1194 
1195  // The best operand index and its score.
1196  // Sometimes we have more than one option (e.g., Opcode and Undefs), so we
1197  // are using the score to differentiate between the two.
1198  struct BestOpData {
1199  Optional<unsigned> Idx = None;
1200  unsigned Score = 0;
1201  } BestOp;
1202 
1203  // Iterate through all unused operands and look for the best.
1204  for (unsigned Idx = 0; Idx != NumOperands; ++Idx) {
1205  // Get the operand at Idx and Lane.
1206  OperandData &OpData = getData(Idx, Lane);
1207  Value *Op = OpData.V;
1208  bool OpAPO = OpData.APO;
1209 
1210  // Skip already selected operands.
1211  if (OpData.IsUsed)
1212  continue;
1213 
1214  // Skip if we are trying to move the operand to a position with a
1215  // different opcode in the linearized tree form. This would break the
1216  // semantics.
1217  if (OpAPO != OpIdxAPO)
1218  continue;
1219 
1220  // Look for an operand that matches the current mode.
1221  switch (RMode) {
1222  case ReorderingMode::Load:
1224  case ReorderingMode::Opcode: {
1225  bool LeftToRight = Lane > LastLane;
1226  Value *OpLeft = (LeftToRight) ? OpLastLane : Op;
1227  Value *OpRight = (LeftToRight) ? Op : OpLastLane;
1228  unsigned Score =
1229  getLookAheadScore({OpLeft, LastLane}, {OpRight, Lane});
1230  if (Score > BestOp.Score) {
1231  BestOp.Idx = Idx;
1232  BestOp.Score = Score;
1233  }
1234  break;
1235  }
1236  case ReorderingMode::Splat:
1237  if (Op == OpLastLane)
1238  BestOp.Idx = Idx;
1239  break;
1241  return None;
1242  }
1243  }
1244 
1245  if (BestOp.Idx) {
1246  getData(BestOp.Idx.getValue(), Lane).IsUsed = true;
1247  return BestOp.Idx;
1248  }
1249  // If we could not find a good match return None.
1250  return None;
1251  }
1252 
1253  /// Helper for reorderOperandVecs. \Returns the lane that we should start
1254  /// reordering from. This is the one which has the least number of operands
1255  /// that can freely move about.
1256  unsigned getBestLaneToStartReordering() const {
1257  unsigned BestLane = 0;
1258  unsigned Min = UINT_MAX;
1259  for (unsigned Lane = 0, NumLanes = getNumLanes(); Lane != NumLanes;
1260  ++Lane) {
1261  unsigned NumFreeOps = getMaxNumOperandsThatCanBeReordered(Lane);
1262  if (NumFreeOps < Min) {
1263  Min = NumFreeOps;
1264  BestLane = Lane;
1265  }
1266  }
1267  return BestLane;
1268  }
1269 
1270  /// \Returns the maximum number of operands that are allowed to be reordered
1271  /// for \p Lane. This is used as a heuristic for selecting the first lane to
1272  /// start operand reordering.
1273  unsigned getMaxNumOperandsThatCanBeReordered(unsigned Lane) const {
1274  unsigned CntTrue = 0;
1275  unsigned NumOperands = getNumOperands();
1276  // Operands with the same APO can be reordered. We therefore need to count
1277  // how many of them we have for each APO, like this: Cnt[APO] = x.
1278  // Since we only have two APOs, namely true and false, we can avoid using
1279  // a map. Instead we can simply count the number of operands that
1280  // correspond to one of them (in this case the 'true' APO), and calculate
1281  // the other by subtracting it from the total number of operands.
1282  for (unsigned OpIdx = 0; OpIdx != NumOperands; ++OpIdx)
1283  if (getData(OpIdx, Lane).APO)
1284  ++CntTrue;
1285  unsigned CntFalse = NumOperands - CntTrue;
1286  return std::max(CntTrue, CntFalse);
1287  }
1288 
1289  /// Go through the instructions in VL and append their operands.
1290  void appendOperandsOfVL(ArrayRef<Value *> VL) {
1291  assert(!VL.empty() && "Bad VL");
1292  assert((empty() || VL.size() == getNumLanes()) &&
1293  "Expected same number of lanes");
1294  assert(isa<Instruction>(VL[0]) && "Expected instruction");
1295  unsigned NumOperands = cast<Instruction>(VL[0])->getNumOperands();
1296  OpsVec.resize(NumOperands);
1297  unsigned NumLanes = VL.size();
1298  for (unsigned OpIdx = 0; OpIdx != NumOperands; ++OpIdx) {
1299  OpsVec[OpIdx].resize(NumLanes);
1300  for (unsigned Lane = 0; Lane != NumLanes; ++Lane) {
1301  assert(isa<Instruction>(VL[Lane]) && "Expected instruction");
1302  // Our tree has just 3 nodes: the root and two operands.
1303  // It is therefore trivial to get the APO. We only need to check the
1304  // opcode of VL[Lane] and whether the operand at OpIdx is the LHS or
1305  // RHS operand. The LHS operand of both add and sub is never attached
1306  // to an inversese operation in the linearized form, therefore its APO
1307  // is false. The RHS is true only if VL[Lane] is an inverse operation.
1308 
1309  // Since operand reordering is performed on groups of commutative
1310  // operations or alternating sequences (e.g., +, -), we can safely
1311  // tell the inverse operations by checking commutativity.
1312  bool IsInverseOperation = !isCommutative(cast<Instruction>(VL[Lane]));
1313  bool APO = (OpIdx == 0) ? false : IsInverseOperation;
1314  OpsVec[OpIdx][Lane] = {cast<Instruction>(VL[Lane])->getOperand(OpIdx),
1315  APO, false};
1316  }
1317  }
1318  }
1319 
1320  /// \returns the number of operands.
1321  unsigned getNumOperands() const { return OpsVec.size(); }
1322 
1323  /// \returns the number of lanes.
1324  unsigned getNumLanes() const { return OpsVec[0].size(); }
1325 
1326  /// \returns the operand value at \p OpIdx and \p Lane.
1327  Value *getValue(unsigned OpIdx, unsigned Lane) const {
1328  return getData(OpIdx, Lane).V;
1329  }
1330 
1331  /// \returns true if the data structure is empty.
1332  bool empty() const { return OpsVec.empty(); }
1333 
1334  /// Clears the data.
1335  void clear() { OpsVec.clear(); }
1336 
1337  /// \Returns true if there are enough operands identical to \p Op to fill
1338  /// the whole vector.
1339  /// Note: This modifies the 'IsUsed' flag, so a cleanUsed() must follow.
1340  bool shouldBroadcast(Value *Op, unsigned OpIdx, unsigned Lane) {
1341  bool OpAPO = getData(OpIdx, Lane).APO;
1342  for (unsigned Ln = 0, Lns = getNumLanes(); Ln != Lns; ++Ln) {
1343  if (Ln == Lane)
1344  continue;
1345  // This is set to true if we found a candidate for broadcast at Lane.
1346  bool FoundCandidate = false;
1347  for (unsigned OpI = 0, OpE = getNumOperands(); OpI != OpE; ++OpI) {
1348  OperandData &Data = getData(OpI, Ln);
1349  if (Data.APO != OpAPO || Data.IsUsed)
1350  continue;
1351  if (Data.V == Op) {
1352  FoundCandidate = true;
1353  Data.IsUsed = true;
1354  break;
1355  }
1356  }
1357  if (!FoundCandidate)
1358  return false;
1359  }
1360  return true;
1361  }
1362 
1363  public:
1364  /// Initialize with all the operands of the instruction vector \p RootVL.
1366  ScalarEvolution &SE, const BoUpSLP &R)
1367  : DL(DL), SE(SE), R(R) {
1368  // Append all the operands of RootVL.
1369  appendOperandsOfVL(RootVL);
1370  }
1371 
1372  /// \Returns a value vector with the operands across all lanes for the
1373  /// opearnd at \p OpIdx.
1374  ValueList getVL(unsigned OpIdx) const {
1375  ValueList OpVL(OpsVec[OpIdx].size());
1376  assert(OpsVec[OpIdx].size() == getNumLanes() &&
1377  "Expected same num of lanes across all operands");
1378  for (unsigned Lane = 0, Lanes = getNumLanes(); Lane != Lanes; ++Lane)
1379  OpVL[Lane] = OpsVec[OpIdx][Lane].V;
1380  return OpVL;
1381  }
1382 
1383  // Performs operand reordering for 2 or more operands.
1384  // The original operands are in OrigOps[OpIdx][Lane].
1385  // The reordered operands are returned in 'SortedOps[OpIdx][Lane]'.
1386  void reorder() {
1387  unsigned NumOperands = getNumOperands();
1388  unsigned NumLanes = getNumLanes();
1389  // Each operand has its own mode. We are using this mode to help us select
1390  // the instructions for each lane, so that they match best with the ones
1391  // we have selected so far.
1392  SmallVector<ReorderingMode, 2> ReorderingModes(NumOperands);
1393 
1394  // This is a greedy single-pass algorithm. We are going over each lane
1395  // once and deciding on the best order right away with no back-tracking.
1396  // However, in order to increase its effectiveness, we start with the lane
1397  // that has operands that can move the least. For example, given the
1398  // following lanes:
1399  // Lane 0 : A[0] = B[0] + C[0] // Visited 3rd
1400  // Lane 1 : A[1] = C[1] - B[1] // Visited 1st
1401  // Lane 2 : A[2] = B[2] + C[2] // Visited 2nd
1402  // Lane 3 : A[3] = C[3] - B[3] // Visited 4th
1403  // we will start at Lane 1, since the operands of the subtraction cannot
1404  // be reordered. Then we will visit the rest of the lanes in a circular
1405  // fashion. That is, Lanes 2, then Lane 0, and finally Lane 3.
1406 
1407  // Find the first lane that we will start our search from.
1408  unsigned FirstLane = getBestLaneToStartReordering();
1409 
1410  // Initialize the modes.
1411  for (unsigned OpIdx = 0; OpIdx != NumOperands; ++OpIdx) {
1412  Value *OpLane0 = getValue(OpIdx, FirstLane);
1413  // Keep track if we have instructions with all the same opcode on one
1414  // side.
1415  if (isa<LoadInst>(OpLane0))
1416  ReorderingModes[OpIdx] = ReorderingMode::Load;
1417  else if (isa<Instruction>(OpLane0)) {
1418  // Check if OpLane0 should be broadcast.
1419  if (shouldBroadcast(OpLane0, OpIdx, FirstLane))
1420  ReorderingModes[OpIdx] = ReorderingMode::Splat;
1421  else
1422  ReorderingModes[OpIdx] = ReorderingMode::Opcode;
1423  }
1424  else if (isa<Constant>(OpLane0))
1425  ReorderingModes[OpIdx] = ReorderingMode::Constant;
1426  else if (isa<Argument>(OpLane0))
1427  // Our best hope is a Splat. It may save some cost in some cases.
1428  ReorderingModes[OpIdx] = ReorderingMode::Splat;
1429  else
1430  // NOTE: This should be unreachable.
1431  ReorderingModes[OpIdx] = ReorderingMode::Failed;
1432  }
1433 
1434  // If the initial strategy fails for any of the operand indexes, then we
1435  // perform reordering again in a second pass. This helps avoid assigning
1436  // high priority to the failed strategy, and should improve reordering for
1437  // the non-failed operand indexes.
1438  for (int Pass = 0; Pass != 2; ++Pass) {
1439  // Skip the second pass if the first pass did not fail.
1440  bool StrategyFailed = false;
1441  // Mark all operand data as free to use.
1442  clearUsed();
1443  // We keep the original operand order for the FirstLane, so reorder the
1444  // rest of the lanes. We are visiting the nodes in a circular fashion,
1445  // using FirstLane as the center point and increasing the radius
1446  // distance.
1447  for (unsigned Distance = 1; Distance != NumLanes; ++Distance) {
1448  // Visit the lane on the right and then the lane on the left.
1449  for (int Direction : {+1, -1}) {
1450  int Lane = FirstLane + Direction * Distance;
1451  if (Lane < 0 || Lane >= (int)NumLanes)
1452  continue;
1453  int LastLane = Lane - Direction;
1454  assert(LastLane >= 0 && LastLane < (int)NumLanes &&
1455  "Out of bounds");
1456  // Look for a good match for each operand.
1457  for (unsigned OpIdx = 0; OpIdx != NumOperands; ++OpIdx) {
1458  // Search for the operand that matches SortedOps[OpIdx][Lane-1].
1459  Optional<unsigned> BestIdx =
1460  getBestOperand(OpIdx, Lane, LastLane, ReorderingModes);
1461  // By not selecting a value, we allow the operands that follow to
1462  // select a better matching value. We will get a non-null value in
1463  // the next run of getBestOperand().
1464  if (BestIdx) {
1465  // Swap the current operand with the one returned by
1466  // getBestOperand().
1467  swap(OpIdx, BestIdx.getValue(), Lane);
1468  } else {
1469  // We failed to find a best operand, set mode to 'Failed'.
1470  ReorderingModes[OpIdx] = ReorderingMode::Failed;
1471  // Enable the second pass.
1472  StrategyFailed = true;
1473  }
1474  }
1475  }
1476  }
1477  // Skip second pass if the strategy did not fail.
1478  if (!StrategyFailed)
1479  break;
1480  }
1481  }
1482 
1483 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
1484  LLVM_DUMP_METHOD static StringRef getModeStr(ReorderingMode RMode) {
1485  switch (RMode) {
1486  case ReorderingMode::Load:
1487  return "Load";
1488  case ReorderingMode::Opcode:
1489  return "Opcode";
1491  return "Constant";
1492  case ReorderingMode::Splat:
1493  return "Splat";
1495  return "Failed";
1496  }
1497  llvm_unreachable("Unimplemented Reordering Type");
1498  }
1499 
1500  LLVM_DUMP_METHOD static raw_ostream &printMode(ReorderingMode RMode,
1501  raw_ostream &OS) {
1502  return OS << getModeStr(RMode);
1503  }
1504 
1505  /// Debug print.
1506  LLVM_DUMP_METHOD static void dumpMode(ReorderingMode RMode) {
1507  printMode(RMode, dbgs());
1508  }
1509 
1510  friend raw_ostream &operator<<(raw_ostream &OS, ReorderingMode RMode) {
1511  return printMode(RMode, OS);
1512  }
1513 
1515  const unsigned Indent = 2;
1516  unsigned Cnt = 0;
1517  for (const OperandDataVec &OpDataVec : OpsVec) {
1518  OS << "Operand " << Cnt++ << "\n";
1519  for (const OperandData &OpData : OpDataVec) {
1520  OS.indent(Indent) << "{";
1521  if (Value *V = OpData.V)
1522  OS << *V;
1523  else
1524  OS << "null";
1525  OS << ", APO:" << OpData.APO << "}\n";
1526  }
1527  OS << "\n";
1528  }
1529  return OS;
1530  }
1531 
1532  /// Debug print.
1533  LLVM_DUMP_METHOD void dump() const { print(dbgs()); }
1534 #endif
1535  };
1536 
1537  /// Checks if the instruction is marked for deletion.
1538  bool isDeleted(Instruction *I) const { return DeletedInstructions.count(I); }
1539 
1540  /// Marks values operands for later deletion by replacing them with Undefs.
1541  void eraseInstructions(ArrayRef<Value *> AV);
1542 
1543  ~BoUpSLP();
1544 
1545 private:
1546  /// Checks if all users of \p I are the part of the vectorization tree.
1547  bool areAllUsersVectorized(Instruction *I,
1548  ArrayRef<Value *> VectorizedVals) const;
1549 
1550  /// \returns the cost of the vectorizable entry.
1551  InstructionCost getEntryCost(const TreeEntry *E,
1552  ArrayRef<Value *> VectorizedVals);
1553 
1554  /// This is the recursive part of buildTree.
1555  void buildTree_rec(ArrayRef<Value *> Roots, unsigned Depth,
1556  const EdgeInfo &EI);
1557 
1558  /// \returns true if the ExtractElement/ExtractValue instructions in \p VL can
1559  /// be vectorized to use the original vector (or aggregate "bitcast" to a
1560  /// vector) and sets \p CurrentOrder to the identity permutation; otherwise
1561  /// returns false, setting \p CurrentOrder to either an empty vector or a
1562  /// non-identity permutation that allows to reuse extract instructions.
1563  bool canReuseExtract(ArrayRef<Value *> VL, Value *OpValue,
1564  SmallVectorImpl<unsigned> &CurrentOrder) const;
1565 
1566  /// Vectorize a single entry in the tree.
1567  Value *vectorizeTree(TreeEntry *E);
1568 
1569  /// Vectorize a single entry in the tree, starting in \p VL.
1570  Value *vectorizeTree(ArrayRef<Value *> VL);
1571 
1572  /// \returns the scalarization cost for this type. Scalarization in this
1573  /// context means the creation of vectors from a group of scalars.
1575  getGatherCost(FixedVectorType *Ty,
1576  const DenseSet<unsigned> &ShuffledIndices) const;
1577 
1578  /// Checks if the gathered \p VL can be represented as shuffle(s) of previous
1579  /// tree entries.
1580  /// \returns ShuffleKind, if gathered values can be represented as shuffles of
1581  /// previous tree entries. \p Mask is filled with the shuffle mask.
1583  isGatherShuffledEntry(const TreeEntry *TE, SmallVectorImpl<int> &Mask,
1585 
1586  /// \returns the scalarization cost for this list of values. Assuming that
1587  /// this subtree gets vectorized, we may need to extract the values from the
1588  /// roots. This method calculates the cost of extracting the values.
1589  InstructionCost getGatherCost(ArrayRef<Value *> VL) const;
1590 
1591  /// Set the Builder insert point to one after the last instruction in
1592  /// the bundle
1593  void setInsertPointAfterBundle(const TreeEntry *E);
1594 
1595  /// \returns a vector from a collection of scalars in \p VL.
1596  Value *gather(ArrayRef<Value *> VL);
1597 
1598  /// \returns whether the VectorizableTree is fully vectorizable and will
1599  /// be beneficial even the tree height is tiny.
1600  bool isFullyVectorizableTinyTree() const;
1601 
1602  /// Reorder commutative or alt operands to get better probability of
1603  /// generating vectorized code.
1604  static void reorderInputsAccordingToOpcode(ArrayRef<Value *> VL,
1606  SmallVectorImpl<Value *> &Right,
1607  const DataLayout &DL,
1608  ScalarEvolution &SE,
1609  const BoUpSLP &R);
1610  struct TreeEntry {
1611  using VecTreeTy = SmallVector<std::unique_ptr<TreeEntry>, 8>;
1612  TreeEntry(VecTreeTy &Container) : Container(Container) {}
1613 
1614  /// \returns true if the scalars in VL are equal to this entry.
1615  bool isSame(ArrayRef<Value *> VL) const {
1616  auto &&IsSame = [VL](ArrayRef<Value *> Scalars, ArrayRef<int> Mask) {
1617  if (Mask.size() != VL.size() && VL.size() == Scalars.size())
1618  return std::equal(VL.begin(), VL.end(), Scalars.begin());
1619  return VL.size() == Mask.size() &&
1620  std::equal(
1621  VL.begin(), VL.end(), Mask.begin(),
1622  [Scalars](Value *V, int Idx) { return V == Scalars[Idx]; });
1623  };
1624  if (!ReorderIndices.empty()) {
1625  // TODO: implement matching if the nodes are just reordered, still can
1626  // treat the vector as the same if the list of scalars matches VL
1627  // directly, without reordering.
1629  inversePermutation(ReorderIndices, Mask);
1630  if (VL.size() == Scalars.size())
1631  return IsSame(Scalars, Mask);
1632  if (VL.size() == ReuseShuffleIndices.size()) {
1633  ::addMask(Mask, ReuseShuffleIndices);
1634  return IsSame(Scalars, Mask);
1635  }
1636  return false;
1637  }
1638  return IsSame(Scalars, ReuseShuffleIndices);
1639  }
1640 
1641  /// A vector of scalars.
1642  ValueList Scalars;
1643 
1644  /// The Scalars are vectorized into this value. It is initialized to Null.
1645  Value *VectorizedValue = nullptr;
1646 
1647  /// Do we need to gather this sequence or vectorize it
1648  /// (either with vector instruction or with scatter/gather
1649  /// intrinsics for store/load)?
1650  enum EntryState { Vectorize, ScatterVectorize, NeedToGather };
1651  EntryState State;
1652 
1653  /// Does this sequence require some shuffling?
1654  SmallVector<int, 4> ReuseShuffleIndices;
1655 
1656  /// Does this entry require reordering?
1657  SmallVector<unsigned, 4> ReorderIndices;
1658 
1659  /// Points back to the VectorizableTree.
1660  ///
1661  /// Only used for Graphviz right now. Unfortunately GraphTrait::NodeRef has
1662  /// to be a pointer and needs to be able to initialize the child iterator.
1663  /// Thus we need a reference back to the container to translate the indices
1664  /// to entries.
1665  VecTreeTy &Container;
1666 
1667  /// The TreeEntry index containing the user of this entry. We can actually
1668  /// have multiple users so the data structure is not truly a tree.
1669  SmallVector<EdgeInfo, 1> UserTreeIndices;
1670 
1671  /// The index of this treeEntry in VectorizableTree.
1672  int Idx = -1;
1673 
1674  private:
1675  /// The operands of each instruction in each lane Operands[op_index][lane].
1676  /// Note: This helps avoid the replication of the code that performs the
1677  /// reordering of operands during buildTree_rec() and vectorizeTree().
1679 
1680  /// The main/alternate instruction.
1681  Instruction *MainOp = nullptr;
1682  Instruction *AltOp = nullptr;
1683 
1684  public:
1685  /// Set this bundle's \p OpIdx'th operand to \p OpVL.
1686  void setOperand(unsigned OpIdx, ArrayRef<Value *> OpVL) {
1687  if (Operands.size() < OpIdx + 1)
1688  Operands.resize(OpIdx + 1);
1689  assert(Operands[OpIdx].empty() && "Already resized?");
1690  Operands[OpIdx].resize(Scalars.size());
1691  for (unsigned Lane = 0, E = Scalars.size(); Lane != E; ++Lane)
1692  Operands[OpIdx][Lane] = OpVL[Lane];
1693  }
1694 
1695  /// Set the operands of this bundle in their original order.
1696  void setOperandsInOrder() {
1697  assert(Operands.empty() && "Already initialized?");
1698  auto *I0 = cast<Instruction>(Scalars[0]);
1699  Operands.resize(I0->getNumOperands());
1700  unsigned NumLanes = Scalars.size();
1701  for (unsigned OpIdx = 0, NumOperands = I0->getNumOperands();
1702  OpIdx != NumOperands; ++OpIdx) {
1703  Operands[OpIdx].resize(NumLanes);
1704  for (unsigned Lane = 0; Lane != NumLanes; ++Lane) {
1705  auto *I = cast<Instruction>(Scalars[Lane]);
1706  assert(I->getNumOperands() == NumOperands &&
1707  "Expected same number of operands");
1708  Operands[OpIdx][Lane] = I->getOperand(OpIdx);
1709  }
1710  }
1711  }
1712 
1713  /// Reorders operands of the node to the given mask \p Mask.
1714  void reorderOperands(ArrayRef<int> Mask) {
1715  for (ValueList &Operand : Operands)
1716  reorderScalars(Operand, Mask);
1717  }
1718 
1719  /// \returns the \p OpIdx operand of this TreeEntry.
1720  ValueList &getOperand(unsigned OpIdx) {
1721  assert(OpIdx < Operands.size() && "Off bounds");
1722  return Operands[OpIdx];
1723  }
1724 
1725  /// \returns the number of operands.
1726  unsigned getNumOperands() const { return Operands.size(); }
1727 
1728  /// \return the single \p OpIdx operand.
1729  Value *getSingleOperand(unsigned OpIdx) const {
1730  assert(OpIdx < Operands.size() && "Off bounds");
1731  assert(!Operands[OpIdx].empty() && "No operand available");
1732  return Operands[OpIdx][0];
1733  }
1734 
1735  /// Some of the instructions in the list have alternate opcodes.
1736  bool isAltShuffle() const {
1737  return getOpcode() != getAltOpcode();
1738  }
1739 
1740  bool isOpcodeOrAlt(Instruction *I) const {
1741  unsigned CheckedOpcode = I->getOpcode();
1742  return (getOpcode() == CheckedOpcode ||
1743  getAltOpcode() == CheckedOpcode);
1744  }
1745 
1746  /// Chooses the correct key for scheduling data. If \p Op has the same (or
1747  /// alternate) opcode as \p OpValue, the key is \p Op. Otherwise the key is
1748  /// \p OpValue.
1749  Value *isOneOf(Value *Op) const {
1750  auto *I = dyn_cast<Instruction>(Op);
1751  if (I && isOpcodeOrAlt(I))
1752  return Op;
1753  return MainOp;
1754  }
1755 
1756  void setOperations(const InstructionsState &S) {
1757  MainOp = S.MainOp;
1758  AltOp = S.AltOp;
1759  }
1760 
1761  Instruction *getMainOp() const {
1762  return MainOp;
1763  }
1764 
1765  Instruction *getAltOp() const {
1766  return AltOp;
1767  }
1768 
1769  /// The main/alternate opcodes for the list of instructions.
1770  unsigned getOpcode() const {
1771  return MainOp ? MainOp->getOpcode() : 0;
1772  }
1773 
1774  unsigned getAltOpcode() const {
1775  return AltOp ? AltOp->getOpcode() : 0;
1776  }
1777 
1778  /// When ReuseReorderShuffleIndices is empty it just returns position of \p
1779  /// V within vector of Scalars. Otherwise, try to remap on its reuse index.
1780  int findLaneForValue(Value *V) const {
1781  unsigned FoundLane = std::distance(Scalars.begin(), find(Scalars, V));
1782  assert(FoundLane < Scalars.size() && "Couldn't find extract lane");
1783  if (!ReorderIndices.empty())
1784  FoundLane = ReorderIndices[FoundLane];
1785  assert(FoundLane < Scalars.size() && "Couldn't find extract lane");
1786  if (!ReuseShuffleIndices.empty()) {
1787  FoundLane = std::distance(ReuseShuffleIndices.begin(),
1788  find(ReuseShuffleIndices, FoundLane));
1789  }
1790  return FoundLane;
1791  }
1792 
1793 #ifndef NDEBUG
1794  /// Debug printer.
1795  LLVM_DUMP_METHOD void dump() const {
1796  dbgs() << Idx << ".\n";
1797  for (unsigned OpI = 0, OpE = Operands.size(); OpI != OpE; ++OpI) {
1798  dbgs() << "Operand " << OpI << ":\n";
1799  for (const Value *V : Operands[OpI])
1800  dbgs().indent(2) << *V << "\n";
1801  }
1802  dbgs() << "Scalars: \n";
1803  for (Value *V : Scalars)
1804  dbgs().indent(2) << *V << "\n";
1805  dbgs() << "State: ";
1806  switch (State) {
1807  case Vectorize:
1808  dbgs() << "Vectorize\n";
1809  break;
1810  case ScatterVectorize:
1811  dbgs() << "ScatterVectorize\n";
1812  break;
1813  case NeedToGather:
1814  dbgs() << "NeedToGather\n";
1815  break;
1816  }
1817  dbgs() << "MainOp: ";
1818  if (MainOp)
1819  dbgs() << *MainOp << "\n";
1820  else
1821  dbgs() << "NULL\n";
1822  dbgs() << "AltOp: ";
1823  if (AltOp)
1824  dbgs() << *AltOp << "\n";
1825  else
1826  dbgs() << "NULL\n";
1827  dbgs() << "VectorizedValue: ";
1828  if (VectorizedValue)
1829  dbgs() << *VectorizedValue << "\n";
1830  else
1831  dbgs() << "NULL\n";
1832  dbgs() << "ReuseShuffleIndices: ";
1833  if (ReuseShuffleIndices.empty())
1834  dbgs() << "Empty";
1835  else
1836  for (unsigned ReuseIdx : ReuseShuffleIndices)
1837  dbgs() << ReuseIdx << ", ";
1838  dbgs() << "\n";
1839  dbgs() << "ReorderIndices: ";
1840  for (unsigned ReorderIdx : ReorderIndices)
1841  dbgs() << ReorderIdx << ", ";
1842  dbgs() << "\n";
1843  dbgs() << "UserTreeIndices: ";
1844  for (const auto &EInfo : UserTreeIndices)
1845  dbgs() << EInfo << ", ";
1846  dbgs() << "\n";
1847  }
1848 #endif
1849  };
1850 
1851 #ifndef NDEBUG
1852  void dumpTreeCosts(const TreeEntry *E, InstructionCost ReuseShuffleCost,
1853  InstructionCost VecCost,
1854  InstructionCost ScalarCost) const {
1855  dbgs() << "SLP: Calculated costs for Tree:\n"; E->dump();
1856  dbgs() << "SLP: Costs:\n";
1857  dbgs() << "SLP: ReuseShuffleCost = " << ReuseShuffleCost << "\n";
1858  dbgs() << "SLP: VectorCost = " << VecCost << "\n";
1859  dbgs() << "SLP: ScalarCost = " << ScalarCost << "\n";
1860  dbgs() << "SLP: ReuseShuffleCost + VecCost - ScalarCost = " <<
1861  ReuseShuffleCost + VecCost - ScalarCost << "\n";
1862  }
1863 #endif
1864 
1865  /// Create a new VectorizableTree entry.
1866  TreeEntry *newTreeEntry(ArrayRef<Value *> VL, Optional<ScheduleData *> Bundle,
1867  const InstructionsState &S,
1868  const EdgeInfo &UserTreeIdx,
1869  ArrayRef<int> ReuseShuffleIndices = None,
1870  ArrayRef<unsigned> ReorderIndices = None) {
1871  TreeEntry::EntryState EntryState =
1872  Bundle ? TreeEntry::Vectorize : TreeEntry::NeedToGather;
1873  return newTreeEntry(VL, EntryState, Bundle, S, UserTreeIdx,
1874  ReuseShuffleIndices, ReorderIndices);
1875  }
1876 
1877  TreeEntry *newTreeEntry(ArrayRef<Value *> VL,
1878  TreeEntry::EntryState EntryState,
1879  Optional<ScheduleData *> Bundle,
1880  const InstructionsState &S,
1881  const EdgeInfo &UserTreeIdx,
1882  ArrayRef<int> ReuseShuffleIndices = None,
1883  ArrayRef<unsigned> ReorderIndices = None) {
1884  assert(((!Bundle && EntryState == TreeEntry::NeedToGather) ||
1885  (Bundle && EntryState != TreeEntry::NeedToGather)) &&
1886  "Need to vectorize gather entry?");
1887  VectorizableTree.push_back(std::make_unique<TreeEntry>(VectorizableTree));
1888  TreeEntry *Last = VectorizableTree.back().get();
1889  Last->Idx = VectorizableTree.size() - 1;
1890  Last->State = EntryState;
1891  Last->ReuseShuffleIndices.append(ReuseShuffleIndices.begin(),
1892  ReuseShuffleIndices.end());
1893  if (ReorderIndices.empty()) {
1894  Last->Scalars.assign(VL.begin(), VL.end());
1895  Last->setOperations(S);
1896  } else {
1897  // Reorder scalars and build final mask.
1898  Last->Scalars.assign(VL.size(), nullptr);
1899  transform(ReorderIndices, Last->Scalars.begin(),
1900  [VL](unsigned Idx) -> Value * {
1901  if (Idx >= VL.size())
1902  return UndefValue::get(VL.front()->getType());
1903  return VL[Idx];
1904  });
1905  InstructionsState S = getSameOpcode(Last->Scalars);
1906  Last->setOperations(S);
1907  Last->ReorderIndices.append(ReorderIndices.begin(), ReorderIndices.end());
1908  }
1909  if (Last->State != TreeEntry::NeedToGather) {
1910  for (Value *V : VL) {
1911  assert(!getTreeEntry(V) && "Scalar already in tree!");
1912  ScalarToTreeEntry[V] = Last;
1913  }
1914  // Update the scheduler bundle to point to this TreeEntry.
1915  unsigned Lane = 0;
1916  for (ScheduleData *BundleMember = Bundle.getValue(); BundleMember;
1917  BundleMember = BundleMember->NextInBundle) {
1918  BundleMember->TE = Last;
1919  BundleMember->Lane = Lane;
1920  ++Lane;
1921  }
1922  assert((!Bundle.getValue() || Lane == VL.size()) &&
1923  "Bundle and VL out of sync");
1924  } else {
1925  MustGather.insert(VL.begin(), VL.end());
1926  }
1927 
1928  if (UserTreeIdx.UserTE)
1929  Last->UserTreeIndices.push_back(UserTreeIdx);
1930 
1931  return Last;
1932  }
1933 
1934  /// -- Vectorization State --
1935  /// Holds all of the tree entries.
1936  TreeEntry::VecTreeTy VectorizableTree;
1937 
1938 #ifndef NDEBUG
1939  /// Debug printer.
1940  LLVM_DUMP_METHOD void dumpVectorizableTree() const {
1941  for (unsigned Id = 0, IdE = VectorizableTree.size(); Id != IdE; ++Id) {
1942  VectorizableTree[Id]->dump();
1943  dbgs() << "\n";
1944  }
1945  }
1946 #endif
1947 
1948  TreeEntry *getTreeEntry(Value *V) { return ScalarToTreeEntry.lookup(V); }
1949 
1950  const TreeEntry *getTreeEntry(Value *V) const {
1951  return ScalarToTreeEntry.lookup(V);
1952  }
1953 
1954  /// Maps a specific scalar to its tree entry.
1955  SmallDenseMap<Value*, TreeEntry *> ScalarToTreeEntry;
1956 
1957  /// Maps a value to the proposed vectorizable size.
1958  SmallDenseMap<Value *, unsigned> InstrElementSize;
1959 
1960  /// A list of scalars that we found that we need to keep as scalars.
1961  ValueSet MustGather;
1962 
1963  /// This POD struct describes one external user in the vectorized tree.
1964  struct ExternalUser {
1965  ExternalUser(Value *S, llvm::User *U, int L)
1966  : Scalar(S), User(U), Lane(L) {}
1967 
1968  // Which scalar in our function.
1969  Value *Scalar;
1970 
1971  // Which user that uses the scalar.
1972  llvm::User *User;
1973 
1974  // Which lane does the scalar belong to.
1975  int Lane;
1976  };
1977  using UserList = SmallVector<ExternalUser, 16>;
1978 
1979  /// Checks if two instructions may access the same memory.
1980  ///
1981  /// \p Loc1 is the location of \p Inst1. It is passed explicitly because it
1982  /// is invariant in the calling loop.
1983  bool isAliased(const MemoryLocation &Loc1, Instruction *Inst1,
1984  Instruction *Inst2) {
1985  // First check if the result is already in the cache.
1986  AliasCacheKey key = std::make_pair(Inst1, Inst2);
1987  Optional<bool> &result = AliasCache[key];
1988  if (result.hasValue()) {
1989  return result.getValue();
1990  }
1991  bool aliased = true;
1992  if (Loc1.Ptr && isSimple(Inst1))
1993  aliased = isModOrRefSet(AA->getModRefInfo(Inst2, Loc1));
1994  // Store the result in the cache.
1995  result = aliased;
1996  return aliased;
1997  }
1998 
1999  using AliasCacheKey = std::pair<Instruction *, Instruction *>;
2000 
2001  /// Cache for alias results.
2002  /// TODO: consider moving this to the AliasAnalysis itself.
2004 
2005  /// Removes an instruction from its block and eventually deletes it.
2006  /// It's like Instruction::eraseFromParent() except that the actual deletion
2007  /// is delayed until BoUpSLP is destructed.
2008  /// This is required to ensure that there are no incorrect collisions in the
2009  /// AliasCache, which can happen if a new instruction is allocated at the
2010  /// same address as a previously deleted instruction.
2011  void eraseInstruction(Instruction *I, bool ReplaceOpsWithUndef = false) {
2012  auto It = DeletedInstructions.try_emplace(I, ReplaceOpsWithUndef).first;
2013  It->getSecond() = It->getSecond() && ReplaceOpsWithUndef;
2014  }
2015 
2016  /// Temporary store for deleted instructions. Instructions will be deleted
2017  /// eventually when the BoUpSLP is destructed.
2018  DenseMap<Instruction *, bool> DeletedInstructions;
2019 
2020  /// A list of values that need to extracted out of the tree.
2021  /// This list holds pairs of (Internal Scalar : External User). External User
2022  /// can be nullptr, it means that this Internal Scalar will be used later,
2023  /// after vectorization.
2024  UserList ExternalUses;
2025 
2026  /// Values used only by @llvm.assume calls.
2028 
2029  /// Holds all of the instructions that we gathered.
2030  SetVector<Instruction *> GatherSeq;
2031 
2032  /// A list of blocks that we are going to CSE.
2033  SetVector<BasicBlock *> CSEBlocks;
2034 
2035  /// Contains all scheduling relevant data for an instruction.
2036  /// A ScheduleData either represents a single instruction or a member of an
2037  /// instruction bundle (= a group of instructions which is combined into a
2038  /// vector instruction).
2039  struct ScheduleData {
2040  // The initial value for the dependency counters. It means that the
2041  // dependencies are not calculated yet.
2042  enum { InvalidDeps = -1 };
2043 
2044  ScheduleData() = default;
2045 
2046  void init(int BlockSchedulingRegionID, Value *OpVal) {
2047  FirstInBundle = this;
2048  NextInBundle = nullptr;
2049  NextLoadStore = nullptr;
2050  IsScheduled = false;
2051  SchedulingRegionID = BlockSchedulingRegionID;
2052  UnscheduledDepsInBundle = UnscheduledDeps;
2053  clearDependencies();
2054  OpValue = OpVal;
2055  TE = nullptr;
2056  Lane = -1;
2057  }
2058 
2059  /// Returns true if the dependency information has been calculated.
2060  bool hasValidDependencies() const { return Dependencies != InvalidDeps; }
2061 
2062  /// Returns true for single instructions and for bundle representatives
2063  /// (= the head of a bundle).
2064  bool isSchedulingEntity() const { return FirstInBundle == this; }
2065 
2066  /// Returns true if it represents an instruction bundle and not only a
2067  /// single instruction.
2068  bool isPartOfBundle() const {
2069  return NextInBundle != nullptr || FirstInBundle != this;
2070  }
2071 
2072  /// Returns true if it is ready for scheduling, i.e. it has no more
2073  /// unscheduled depending instructions/bundles.
2074  bool isReady() const {
2075  assert(isSchedulingEntity() &&
2076  "can't consider non-scheduling entity for ready list");
2077  return UnscheduledDepsInBundle == 0 && !IsScheduled;
2078  }
2079 
2080  /// Modifies the number of unscheduled dependencies, also updating it for
2081  /// the whole bundle.
2082  int incrementUnscheduledDeps(int Incr) {
2083  UnscheduledDeps += Incr;
2084  return FirstInBundle->UnscheduledDepsInBundle += Incr;
2085  }
2086 
2087  /// Sets the number of unscheduled dependencies to the number of
2088  /// dependencies.
2089  void resetUnscheduledDeps() {
2090  incrementUnscheduledDeps(Dependencies - UnscheduledDeps);
2091  }
2092 
2093  /// Clears all dependency information.
2094  void clearDependencies() {
2095  Dependencies = InvalidDeps;
2096  resetUnscheduledDeps();
2097  MemoryDependencies.clear();
2098  }
2099 
2100  void dump(raw_ostream &os) const {
2101  if (!isSchedulingEntity()) {
2102  os << "/ " << *Inst;
2103  } else if (NextInBundle) {
2104  os << '[' << *Inst;
2105  ScheduleData *SD = NextInBundle;
2106  while (SD) {
2107  os << ';' << *SD->Inst;
2108  SD = SD->NextInBundle;
2109  }
2110  os << ']';
2111  } else {
2112  os << *Inst;
2113  }
2114  }
2115 
2116  Instruction *Inst = nullptr;
2117 
2118  /// Points to the head in an instruction bundle (and always to this for
2119  /// single instructions).
2120  ScheduleData *FirstInBundle = nullptr;
2121 
2122  /// Single linked list of all instructions in a bundle. Null if it is a
2123  /// single instruction.
2124  ScheduleData *NextInBundle = nullptr;
2125 
2126  /// Single linked list of all memory instructions (e.g. load, store, call)
2127  /// in the block - until the end of the scheduling region.
2128  ScheduleData *NextLoadStore = nullptr;
2129 
2130  /// The dependent memory instructions.
2131  /// This list is derived on demand in calculateDependencies().
2132  SmallVector<ScheduleData *, 4> MemoryDependencies;
2133 
2134  /// This ScheduleData is in the current scheduling region if this matches
2135  /// the current SchedulingRegionID of BlockScheduling.
2136  int SchedulingRegionID = 0;
2137 
2138  /// Used for getting a "good" final ordering of instructions.
2139  int SchedulingPriority = 0;
2140 
2141  /// The number of dependencies. Constitutes of the number of users of the
2142  /// instruction plus the number of dependent memory instructions (if any).
2143  /// This value is calculated on demand.
2144  /// If InvalidDeps, the number of dependencies is not calculated yet.
2145  int Dependencies = InvalidDeps;
2146 
2147  /// The number of dependencies minus the number of dependencies of scheduled
2148  /// instructions. As soon as this is zero, the instruction/bundle gets ready
2149  /// for scheduling.
2150  /// Note that this is negative as long as Dependencies is not calculated.
2151  int UnscheduledDeps = InvalidDeps;
2152 
2153  /// The sum of UnscheduledDeps in a bundle. Equals to UnscheduledDeps for
2154  /// single instructions.
2155  int UnscheduledDepsInBundle = InvalidDeps;
2156 
2157  /// True if this instruction is scheduled (or considered as scheduled in the
2158  /// dry-run).
2159  bool IsScheduled = false;
2160 
2161  /// Opcode of the current instruction in the schedule data.
2162  Value *OpValue = nullptr;
2163 
2164  /// The TreeEntry that this instruction corresponds to.
2165  TreeEntry *TE = nullptr;
2166 
2167  /// The lane of this node in the TreeEntry.
2168  int Lane = -1;
2169  };
2170 
2171 #ifndef NDEBUG
2172  friend inline raw_ostream &operator<<(raw_ostream &os,
2173  const BoUpSLP::ScheduleData &SD) {
2174  SD.dump(os);
2175  return os;
2176  }
2177 #endif
2178 
2179  friend struct GraphTraits<BoUpSLP *>;
2180  friend struct DOTGraphTraits<BoUpSLP *>;
2181 
2182  /// Contains all scheduling data for a basic block.
2183  struct BlockScheduling {
2184  BlockScheduling(BasicBlock *BB)
2185  : BB(BB), ChunkSize(BB->size()), ChunkPos(ChunkSize) {}
2186 
2187  void clear() {
2188  ReadyInsts.clear();
2189  ScheduleStart = nullptr;
2190  ScheduleEnd = nullptr;
2191  FirstLoadStoreInRegion = nullptr;
2192  LastLoadStoreInRegion = nullptr;
2193 
2194  // Reduce the maximum schedule region size by the size of the
2195  // previous scheduling run.
2196  ScheduleRegionSizeLimit -= ScheduleRegionSize;
2197  if (ScheduleRegionSizeLimit < MinScheduleRegionSize)
2198  ScheduleRegionSizeLimit = MinScheduleRegionSize;
2199  ScheduleRegionSize = 0;
2200 
2201  // Make a new scheduling region, i.e. all existing ScheduleData is not
2202  // in the new region yet.
2203  ++SchedulingRegionID;
2204  }
2205 
2206  ScheduleData *getScheduleData(Value *V) {
2207  ScheduleData *SD = ScheduleDataMap[V];
2208  if (SD && SD->SchedulingRegionID == SchedulingRegionID)
2209  return SD;
2210  return nullptr;
2211  }
2212 
2213  ScheduleData *getScheduleData(Value *V, Value *Key) {
2214  if (V == Key)
2215  return getScheduleData(V);
2216  auto I = ExtraScheduleDataMap.find(V);
2217  if (I != ExtraScheduleDataMap.end()) {
2218  ScheduleData *SD = I->second[Key];
2219  if (SD && SD->SchedulingRegionID == SchedulingRegionID)
2220  return SD;
2221  }
2222  return nullptr;
2223  }
2224 
2225  bool isInSchedulingRegion(ScheduleData *SD) const {
2226  return SD->SchedulingRegionID == SchedulingRegionID;
2227  }
2228 
2229  /// Marks an instruction as scheduled and puts all dependent ready
2230  /// instructions into the ready-list.
2231  template <typename ReadyListType>
2232  void schedule(ScheduleData *SD, ReadyListType &ReadyList) {
2233  SD->IsScheduled = true;
2234  LLVM_DEBUG(dbgs() << "SLP: schedule " << *SD << "\n");
2235 
2236  ScheduleData *BundleMember = SD;
2237  while (BundleMember) {
2238  if (BundleMember->Inst != BundleMember->OpValue) {
2239  BundleMember = BundleMember->NextInBundle;
2240  continue;
2241  }
2242  // Handle the def-use chain dependencies.
2243 
2244  // Decrement the unscheduled counter and insert to ready list if ready.
2245  auto &&DecrUnsched = [this, &ReadyList](Instruction *I) {
2246  doForAllOpcodes(I, [&ReadyList](ScheduleData *OpDef) {
2247  if (OpDef && OpDef->hasValidDependencies() &&
2248  OpDef->incrementUnscheduledDeps(-1) == 0) {
2249  // There are no more unscheduled dependencies after
2250  // decrementing, so we can put the dependent instruction
2251  // into the ready list.
2252  ScheduleData *DepBundle = OpDef->FirstInBundle;
2253  assert(!DepBundle->IsScheduled &&
2254  "already scheduled bundle gets ready");
2255  ReadyList.insert(DepBundle);
2256  LLVM_DEBUG(dbgs()
2257  << "SLP: gets ready (def): " << *DepBundle << "\n");
2258  }
2259  });
2260  };
2261 
2262  // If BundleMember is a vector bundle, its operands may have been
2263  // reordered duiring buildTree(). We therefore need to get its operands
2264  // through the TreeEntry.
2265  if (TreeEntry *TE = BundleMember->TE) {
2266  int Lane = BundleMember->Lane;
2267  assert(Lane >= 0 && "Lane not set");
2268 
2269  // Since vectorization tree is being built recursively this assertion
2270  // ensures that the tree entry has all operands set before reaching
2271  // this code. Couple of exceptions known at the moment are extracts
2272  // where their second (immediate) operand is not added. Since
2273  // immediates do not affect scheduler behavior this is considered
2274  // okay.
2275  auto *In = TE->getMainOp();
2276  assert(In &&
2277  (isa<ExtractValueInst>(In) || isa<ExtractElementInst>(In) ||
2278  In->getNumOperands() == TE->getNumOperands()) &&
2279  "Missed TreeEntry operands?");
2280  (void)In; // fake use to avoid build failure when assertions disabled
2281 
2282  for (unsigned OpIdx = 0, NumOperands = TE->getNumOperands();
2283  OpIdx != NumOperands; ++OpIdx)
2284  if (auto *I = dyn_cast<Instruction>(TE->getOperand(OpIdx)[Lane]))
2285  DecrUnsched(I);
2286  } else {
2287  // If BundleMember is a stand-alone instruction, no operand reordering
2288  // has taken place, so we directly access its operands.
2289  for (Use &U : BundleMember->Inst->operands())
2290  if (auto *I = dyn_cast<Instruction>(U.get()))
2291  DecrUnsched(I);
2292  }
2293  // Handle the memory dependencies.
2294  for (ScheduleData *MemoryDepSD : BundleMember->MemoryDependencies) {
2295  if (MemoryDepSD->incrementUnscheduledDeps(-1) == 0) {
2296  // There are no more unscheduled dependencies after decrementing,
2297  // so we can put the dependent instruction into the ready list.
2298  ScheduleData *DepBundle = MemoryDepSD->FirstInBundle;
2299  assert(!DepBundle->IsScheduled &&
2300  "already scheduled bundle gets ready");
2301  ReadyList.insert(DepBundle);
2302  LLVM_DEBUG(dbgs()
2303  << "SLP: gets ready (mem): " << *DepBundle << "\n");
2304  }
2305  }
2306  BundleMember = BundleMember->NextInBundle;
2307  }
2308  }
2309 
2310  void doForAllOpcodes(Value *V,
2311  function_ref<void(ScheduleData *SD)> Action) {
2312  if (ScheduleData *SD = getScheduleData(V))
2313  Action(SD);
2314  auto I = ExtraScheduleDataMap.find(V);
2315  if (I != ExtraScheduleDataMap.end())
2316  for (auto &P : I->second)
2317  if (P.second->SchedulingRegionID == SchedulingRegionID)
2318  Action(P.second);
2319  }
2320 
2321  /// Put all instructions into the ReadyList which are ready for scheduling.
2322  template <typename ReadyListType>
2323  void initialFillReadyList(ReadyListType &ReadyList) {
2324  for (auto *I = ScheduleStart; I != ScheduleEnd; I = I->getNextNode()) {
2325  doForAllOpcodes(I, [&](ScheduleData *SD) {
2326  if (SD->isSchedulingEntity() && SD->isReady()) {
2327  ReadyList.insert(SD);
2328  LLVM_DEBUG(dbgs()
2329  << "SLP: initially in ready list: " << *I << "\n");
2330  }
2331  });
2332  }
2333  }
2334 
2335  /// Checks if a bundle of instructions can be scheduled, i.e. has no
2336  /// cyclic dependencies. This is only a dry-run, no instructions are
2337  /// actually moved at this stage.
2338  /// \returns the scheduling bundle. The returned Optional value is non-None
2339  /// if \p VL is allowed to be scheduled.
2341  tryScheduleBundle(ArrayRef<Value *> VL, BoUpSLP *SLP,
2342  const InstructionsState &S);
2343 
2344  /// Un-bundles a group of instructions.
2345  void cancelScheduling(ArrayRef<Value *> VL, Value *OpValue);
2346 
2347  /// Allocates schedule data chunk.
2348  ScheduleData *allocateScheduleDataChunks();
2349 
2350  /// Extends the scheduling region so that V is inside the region.
2351  /// \returns true if the region size is within the limit.
2352  bool extendSchedulingRegion(Value *V, const InstructionsState &S);
2353 
2354  /// Initialize the ScheduleData structures for new instructions in the
2355  /// scheduling region.
2356  void initScheduleData(Instruction *FromI, Instruction *ToI,
2357  ScheduleData *PrevLoadStore,
2358  ScheduleData *NextLoadStore);
2359 
2360  /// Updates the dependency information of a bundle and of all instructions/
2361  /// bundles which depend on the original bundle.
2362  void calculateDependencies(ScheduleData *SD, bool InsertInReadyList,
2363  BoUpSLP *SLP);
2364 
2365  /// Sets all instruction in the scheduling region to un-scheduled.
2366  void resetSchedule();
2367 
2368  BasicBlock *BB;
2369 
2370  /// Simple memory allocation for ScheduleData.
2371  std::vector<std::unique_ptr<ScheduleData[]>> ScheduleDataChunks;
2372 
2373  /// The size of a ScheduleData array in ScheduleDataChunks.
2374  int ChunkSize;
2375 
2376  /// The allocator position in the current chunk, which is the last entry
2377  /// of ScheduleDataChunks.
2378  int ChunkPos;
2379 
2380  /// Attaches ScheduleData to Instruction.
2381  /// Note that the mapping survives during all vectorization iterations, i.e.
2382  /// ScheduleData structures are recycled.
2383  DenseMap<Value *, ScheduleData *> ScheduleDataMap;
2384 
2385  /// Attaches ScheduleData to Instruction with the leading key.
2387  ExtraScheduleDataMap;
2388 
2389  struct ReadyList : SmallVector<ScheduleData *, 8> {
2390  void insert(ScheduleData *SD) { push_back(SD); }
2391  };
2392 
2393  /// The ready-list for scheduling (only used for the dry-run).
2394  ReadyList ReadyInsts;
2395 
2396  /// The first instruction of the scheduling region.
2397  Instruction *ScheduleStart = nullptr;
2398 
2399  /// The first instruction _after_ the scheduling region.
2400  Instruction *ScheduleEnd = nullptr;
2401 
2402  /// The first memory accessing instruction in the scheduling region
2403  /// (can be null).
2404  ScheduleData *FirstLoadStoreInRegion = nullptr;
2405 
2406  /// The last memory accessing instruction in the scheduling region
2407  /// (can be null).
2408  ScheduleData *LastLoadStoreInRegion = nullptr;
2409 
2410  /// The current size of the scheduling region.
2411  int ScheduleRegionSize = 0;
2412 
2413  /// The maximum size allowed for the scheduling region.
2414  int ScheduleRegionSizeLimit = ScheduleRegionSizeBudget;
2415 
2416  /// The ID of the scheduling region. For a new vectorization iteration this
2417  /// is incremented which "removes" all ScheduleData from the region.
2418  // Make sure that the initial SchedulingRegionID is greater than the
2419  // initial SchedulingRegionID in ScheduleData (which is 0).
2420  int SchedulingRegionID = 1;
2421  };
2422 
2423  /// Attaches the BlockScheduling structures to basic blocks.
2425 
2426  /// Performs the "real" scheduling. Done before vectorization is actually
2427  /// performed in a basic block.
2428  void scheduleBlock(BlockScheduling *BS);
2429 
2430  /// List of users to ignore during scheduling and that don't need extracting.
2431  ArrayRef<Value *> UserIgnoreList;
2432 
2433  /// A DenseMapInfo implementation for holding DenseMaps and DenseSets of
2434  /// sorted SmallVectors of unsigned.
2435  struct OrdersTypeDenseMapInfo {
2436  static OrdersType getEmptyKey() {
2437  OrdersType V;
2438  V.push_back(~1U);
2439  return V;
2440  }
2441 
2442  static OrdersType getTombstoneKey() {
2443  OrdersType V;
2444  V.push_back(~2U);
2445  return V;
2446  }
2447 
2448  static unsigned getHashValue(const OrdersType &V) {
2449  return static_cast<unsigned>(hash_combine_range(V.begin(), V.end()));
2450  }
2451 
2452  static bool isEqual(const OrdersType &LHS, const OrdersType &RHS) {
2453  return LHS == RHS;
2454  }
2455  };
2456 
2457  // Analysis and block reference.
2458  Function *F;
2459  ScalarEvolution *SE;
2461  TargetLibraryInfo *TLI;
2462  AAResults *AA;
2463  LoopInfo *LI;
2464  DominatorTree *DT;
2465  AssumptionCache *AC;
2466  DemandedBits *DB;
2467  const DataLayout *DL;
2469 
2470  unsigned MaxVecRegSize; // This is set by TTI or overridden by cl::opt.
2471  unsigned MinVecRegSize; // Set by cl::opt (default: 128).
2472 
2473  /// Instruction builder to construct the vectorized tree.
2475 
2476  /// A map of scalar integer values to the smallest bit width with which they
2477  /// can legally be represented. The values map to (width, signed) pairs,
2478  /// where "width" indicates the minimum bit width and "signed" is True if the
2479  /// value must be signed-extended, rather than zero-extended, back to its
2480  /// original width.
2482 };
2483 
2484 } // end namespace slpvectorizer
2485 
2486 template <> struct GraphTraits<BoUpSLP *> {
2487  using TreeEntry = BoUpSLP::TreeEntry;
2488 
2489  /// NodeRef has to be a pointer per the GraphWriter.
2490  using NodeRef = TreeEntry *;
2491 
2492  using ContainerTy = BoUpSLP::TreeEntry::VecTreeTy;
2493 
2494  /// Add the VectorizableTree to the index iterator to be able to return
2495  /// TreeEntry pointers.
2496  struct ChildIteratorType
2497  : public iterator_adaptor_base<
2498  ChildIteratorType, SmallVector<BoUpSLP::EdgeInfo, 1>::iterator> {
2500 
2502  ContainerTy &VT)
2503  : ChildIteratorType::iterator_adaptor_base(W), VectorizableTree(VT) {}
2504 
2505  NodeRef operator*() { return I->UserTE; }
2506  };
2507 
2508  static NodeRef getEntryNode(BoUpSLP &R) {
2509  return R.VectorizableTree[0].get();
2510  }
2511 
2512  static ChildIteratorType child_begin(NodeRef N) {
2513  return {N->UserTreeIndices.begin(), N->Container};
2514  }
2515 
2516  static ChildIteratorType child_end(NodeRef N) {
2517  return {N->UserTreeIndices.end(), N->Container};
2518  }
2519 
2520  /// For the node iterator we just need to turn the TreeEntry iterator into a
2521  /// TreeEntry* iterator so that it dereferences to NodeRef.
2522  class nodes_iterator {
2523  using ItTy = ContainerTy::iterator;
2524  ItTy It;
2525 
2526  public:
2527  nodes_iterator(const ItTy &It2) : It(It2) {}
2528  NodeRef operator*() { return It->get(); }
2529  nodes_iterator operator++() {
2530  ++It;
2531  return *this;
2532  }
2533  bool operator!=(const nodes_iterator &N2) const { return N2.It != It; }
2534  };
2535 
2536  static nodes_iterator nodes_begin(BoUpSLP *R) {
2537  return nodes_iterator(R->VectorizableTree.begin());
2538  }
2539 
2540  static nodes_iterator nodes_end(BoUpSLP *R) {
2541  return nodes_iterator(R->VectorizableTree.end());
2542  }
2543 
2544  static unsigned size(BoUpSLP *R) { return R->VectorizableTree.size(); }
2545 };
2546 
2547 template <> struct DOTGraphTraits<BoUpSLP *> : public DefaultDOTGraphTraits {
2548  using TreeEntry = BoUpSLP::TreeEntry;
2549 
2551 
2552  std::string getNodeLabel(const TreeEntry *Entry, const BoUpSLP *R) {
2553  std::string Str;
2554  raw_string_ostream OS(Str);
2555  if (isSplat(Entry->Scalars)) {
2556  OS << "<splat> " << *Entry->Scalars[0];
2557  return Str;
2558  }
2559  for (auto V : Entry->Scalars) {
2560  OS << *V;
2561  if (llvm::any_of(R->ExternalUses, [&](const BoUpSLP::ExternalUser &EU) {
2562  return EU.Scalar == V;
2563  }))
2564  OS << " <extract>";
2565  OS << "\n";
2566  }
2567  return Str;
2568  }
2569 
2570  static std::string getNodeAttributes(const TreeEntry *Entry,
2571  const BoUpSLP *) {
2572  if (Entry->State == TreeEntry::NeedToGather)
2573  return "color=red";
2574  return "";
2575  }
2576 };
2577 
2578 } // end namespace llvm
2579 
2580 BoUpSLP::~BoUpSLP() {
2581  for (const auto &Pair : DeletedInstructions) {
2582  // Replace operands of ignored instructions with Undefs in case if they were
2583  // marked for deletion.
2584  if (Pair.getSecond()) {
2585  Value *Undef = UndefValue::get(Pair.getFirst()->getType());
2586  Pair.getFirst()->replaceAllUsesWith(Undef);
2587  }
2588  Pair.getFirst()->dropAllReferences();
2589  }
2590  for (const auto &Pair : DeletedInstructions) {
2591  assert(Pair.getFirst()->use_empty() &&
2592  "trying to erase instruction with users.");
2593  Pair.getFirst()->eraseFromParent();
2594  }
2595 #ifdef EXPENSIVE_CHECKS
2596  // If we could guarantee that this call is not extremely slow, we could
2597  // remove the ifdef limitation (see PR47712).
2598  assert(!verifyFunction(*F, &dbgs()));
2599 #endif
2600 }
2601 
2602 void BoUpSLP::eraseInstructions(ArrayRef<Value *> AV) {
2603  for (auto *V : AV) {
2604  if (auto *I = dyn_cast<Instruction>(V))
2605  eraseInstruction(I, /*ReplaceOpsWithUndef=*/true);
2606  };
2607 }
2608 
2609 /// Reorders the given \p Reuses mask according to the given \p Mask. \p Reuses
2610 /// contains original mask for the scalars reused in the node. Procedure
2611 /// transform this mask in accordance with the given \p Mask.
2613  assert(!Mask.empty() && Reuses.size() == Mask.size() &&
2614  "Expected non-empty mask.");
2615  SmallVector<int> Prev(Reuses.begin(), Reuses.end());
2616  Prev.swap(Reuses);
2617  for (unsigned I = 0, E = Prev.size(); I < E; ++I)
2618  if (Mask[I] != UndefMaskElem)
2619  Reuses[Mask[I]] = Prev[I];
2620 }
2621 
2622 /// Reorders the given \p Order according to the given \p Mask. \p Order - is
2623 /// the original order of the scalars. Procedure transforms the provided order
2624 /// in accordance with the given \p Mask. If the resulting \p Order is just an
2625 /// identity order, \p Order is cleared.
2627  assert(!Mask.empty() && "Expected non-empty mask.");
2628  SmallVector<int> MaskOrder;
2629  if (Order.empty()) {
2630  MaskOrder.resize(Mask.size());
2631  std::iota(MaskOrder.begin(), MaskOrder.end(), 0);
2632  } else {
2633  inversePermutation(Order, MaskOrder);
2634  }
2635  reorderReuses(MaskOrder, Mask);
2636  if (ShuffleVectorInst::isIdentityMask(MaskOrder)) {
2637  Order.clear();
2638  return;
2639  }
2640  Order.assign(Mask.size(), Mask.size());
2641  for (unsigned I = 0, E = Mask.size(); I < E; ++I)
2642  if (MaskOrder[I] != UndefMaskElem)
2643  Order[MaskOrder[I]] = I;
2644  fixupOrderingIndices(Order);
2645 }
2646 
2647 void BoUpSLP::reorderTopToBottom() {
2648  // Maps VF to the graph nodes.
2650  // ExtractElement gather nodes which can be vectorized and need to handle
2651  // their ordering.
2653  // Find all reorderable nodes with the given VF.
2654  // Currently the are vectorized loads,extracts + some gathering of extracts.
2655  for_each(VectorizableTree, [this, &VFToOrderedEntries, &GathersToOrders](
2656  const std::unique_ptr<TreeEntry> &TE) {
2657  // No need to reorder if need to shuffle reuses, still need to shuffle the
2658  // node.
2659  if (!TE->ReuseShuffleIndices.empty())
2660  return;
2661  if (TE->State == TreeEntry::Vectorize &&
2663  InsertElementInst>(TE->getMainOp()) &&
2664  !TE->isAltShuffle()) {
2665  VFToOrderedEntries[TE->Scalars.size()].insert(TE.get());
2666  } else if (TE->State == TreeEntry::NeedToGather &&
2667  TE->getOpcode() == Instruction::ExtractElement &&
2668  !TE->isAltShuffle() &&
2669  isa<FixedVectorType>(cast<ExtractElementInst>(TE->getMainOp())
2670  ->getVectorOperandType()) &&
2671  allSameType(TE->Scalars) && allSameBlock(TE->Scalars)) {
2672  // Check that gather of extractelements can be represented as
2673  // just a shuffle of a single vector.
2674  OrdersType CurrentOrder;
2675  bool Reuse = canReuseExtract(TE->Scalars, TE->getMainOp(), CurrentOrder);
2676  if (Reuse || !CurrentOrder.empty()) {
2677  VFToOrderedEntries[TE->Scalars.size()].insert(TE.get());
2678  GathersToOrders.try_emplace(TE.get(), CurrentOrder);
2679  }
2680  }
2681  });
2682 
2683  // Reorder the graph nodes according to their vectorization factor.
2684  for (unsigned VF = VectorizableTree.front()->Scalars.size(); VF > 1;
2685  VF /= 2) {
2686  auto It = VFToOrderedEntries.find(VF);
2687  if (It == VFToOrderedEntries.end())
2688  continue;
2689  // Try to find the most profitable order. We just are looking for the most
2690  // used order and reorder scalar elements in the nodes according to this
2691  // mostly used order.
2692  const SmallPtrSetImpl<TreeEntry *> &OrderedEntries = It->getSecond();
2693  // All operands are reordered and used only in this node - propagate the
2694  // most used order to the user node.
2697  for (const TreeEntry *OpTE : OrderedEntries) {
2698  // No need to reorder this nodes, still need to extend and to use shuffle,
2699  // just need to merge reordering shuffle and the reuse shuffle.
2700  if (!OpTE->ReuseShuffleIndices.empty())
2701  continue;
2702  // Count number of orders uses.
2703  const auto &Order = [OpTE, &GathersToOrders]() -> const OrdersType & {
2704  if (OpTE->State == TreeEntry::NeedToGather)
2705  return GathersToOrders.find(OpTE)->second;
2706  return OpTE->ReorderIndices;
2707  }();
2708  // Stores actually store the mask, not the order, need to invert.
2709  if (OpTE->State == TreeEntry::Vectorize && !OpTE->isAltShuffle() &&
2710  OpTE->getOpcode() == Instruction::Store && !Order.empty()) {
2712  inversePermutation(Order, Mask);
2713  unsigned E = Order.size();
2714  OrdersType CurrentOrder(E, E);
2715  transform(Mask, CurrentOrder.begin(), [E](int Idx) {
2716  return Idx == UndefMaskElem ? E : static_cast<unsigned>(Idx);
2717  });
2718  fixupOrderingIndices(CurrentOrder);
2719  ++OrdersUses.try_emplace(CurrentOrder).first->getSecond();
2720  } else {
2721  ++OrdersUses.try_emplace(Order).first->getSecond();
2722  }
2723  }
2724  // Set order of the user node.
2725  if (OrdersUses.empty())
2726  continue;
2727  // Choose the most used order.
2728  ArrayRef<unsigned> BestOrder = OrdersUses.begin()->first;
2729  unsigned Cnt = OrdersUses.begin()->second;
2730  for (const auto &Pair : llvm::drop_begin(OrdersUses)) {
2731  if (Cnt < Pair.second || (Cnt == Pair.second && Pair.first.empty())) {
2732  BestOrder = Pair.first;
2733  Cnt = Pair.second;
2734  }
2735  }
2736  // Set order of the user node.
2737  if (BestOrder.empty())
2738  continue;
2740  inversePermutation(BestOrder, Mask);
2741  SmallVector<int> MaskOrder(BestOrder.size(), UndefMaskElem);
2742  unsigned E = BestOrder.size();
2743  transform(BestOrder, MaskOrder.begin(), [E](unsigned I) {
2744  return I < E ? static_cast<int>(I) : UndefMaskElem;
2745  });
2746  // Do an actual reordering, if profitable.
2747  for (std::unique_ptr<TreeEntry> &TE : VectorizableTree) {
2748  // Just do the reordering for the nodes with the given VF.
2749  if (TE->Scalars.size() != VF) {
2750  if (TE->ReuseShuffleIndices.size() == VF) {
2751  // Need to reorder the reuses masks of the operands with smaller VF to
2752  // be able to find the match between the graph nodes and scalar
2753  // operands of the given node during vectorization/cost estimation.
2754  assert(all_of(TE->UserTreeIndices,
2755  [VF, &TE](const EdgeInfo &EI) {
2756  return EI.UserTE->Scalars.size() == VF ||
2757  EI.UserTE->Scalars.size() ==
2758  TE->Scalars.size();
2759  }) &&
2760  "All users must be of VF size.");
2761  // Update ordering of the operands with the smaller VF than the given
2762  // one.
2763  reorderReuses(TE->ReuseShuffleIndices, Mask);
2764  }
2765  continue;
2766  }
2767  if (TE->State == TreeEntry::Vectorize &&
2769  InsertElementInst>(TE->getMainOp()) &&
2770  !TE->isAltShuffle()) {
2771  // Build correct orders for extract{element,value}, loads and
2772  // stores.
2773  reorderOrder(TE->ReorderIndices, Mask);
2774  if (isa<InsertElementInst, StoreInst>(TE->getMainOp()))
2775  TE->reorderOperands(Mask);
2776  } else {
2777  // Reorder the node and its operands.
2778  TE->reorderOperands(Mask);
2779  assert(TE->ReorderIndices.empty() &&
2780  "Expected empty reorder sequence.");
2781  reorderScalars(TE->Scalars, Mask);
2782  }
2783  if (!TE->ReuseShuffleIndices.empty()) {
2784  // Apply reversed order to keep the original ordering of the reused
2785  // elements to avoid extra reorder indices shuffling.
2786  OrdersType CurrentOrder;
2787  reorderOrder(CurrentOrder, MaskOrder);
2788  SmallVector<int> NewReuses;
2789  inversePermutation(CurrentOrder, NewReuses);
2790  addMask(NewReuses, TE->ReuseShuffleIndices);
2791  TE->ReuseShuffleIndices.swap(NewReuses);
2792  }
2793  }
2794  }
2795 }
2796 
2797 void BoUpSLP::reorderBottomToTop() {
2798  SetVector<TreeEntry *> OrderedEntries;
2800  // Find all reorderable leaf nodes with the given VF.
2801  // Currently the are vectorized loads,extracts without alternate operands +
2802  // some gathering of extracts.
2803  SmallVector<TreeEntry *> NonVectorized;
2804  for_each(VectorizableTree, [this, &OrderedEntries, &GathersToOrders,
2805  &NonVectorized](
2806  const std::unique_ptr<TreeEntry> &TE) {
2807  // No need to reorder if need to shuffle reuses, still need to shuffle the
2808  // node.
2809  if (!TE->ReuseShuffleIndices.empty())
2810  return;
2811  if (TE->State == TreeEntry::Vectorize &&
2812  isa<LoadInst, ExtractElementInst, ExtractValueInst>(TE->getMainOp()) &&
2813  !TE->isAltShuffle()) {
2814  OrderedEntries.insert(TE.get());
2815  } else if (TE->State == TreeEntry::NeedToGather &&
2816  TE->getOpcode() == Instruction::ExtractElement &&
2817  !TE->isAltShuffle() &&
2818  isa<FixedVectorType>(cast<ExtractElementInst>(TE->getMainOp())
2819  ->getVectorOperandType()) &&
2820  allSameType(TE->Scalars) && allSameBlock(TE->Scalars)) {
2821  // Check that gather of extractelements can be represented as
2822  // just a shuffle of a single vector with a single user only.
2823  OrdersType CurrentOrder;
2824  bool Reuse = canReuseExtract(TE->Scalars, TE->getMainOp(), CurrentOrder);
2825  if ((Reuse || !CurrentOrder.empty()) &&
2826  !any_of(
2827  VectorizableTree, [&TE](const std::unique_ptr<TreeEntry> &Entry) {
2828  return Entry->State == TreeEntry::NeedToGather &&
2829  Entry.get() != TE.get() && Entry->isSame(TE->Scalars);
2830  })) {
2831  OrderedEntries.insert(TE.get());
2832  GathersToOrders.try_emplace(TE.get(), CurrentOrder);
2833  }
2834  }
2835  if (TE->State != TreeEntry::Vectorize)
2836  NonVectorized.push_back(TE.get());
2837  });
2838 
2839  // Checks if the operands of the users are reordarable and have only single
2840  // use.
2841  auto &&CheckOperands =
2842  [this, &NonVectorized](const auto &Data,
2843  SmallVectorImpl<TreeEntry *> &GatherOps) {
2844  for (unsigned I = 0, E = Data.first->getNumOperands(); I < E; ++I) {
2845  if (any_of(Data.second,
2846  [I](const std::pair<unsigned, TreeEntry *> &OpData) {
2847  return OpData.first == I &&
2848  OpData.second->State == TreeEntry::Vectorize;
2849  }))
2850  continue;
2851  ArrayRef<Value *> VL = Data.first->getOperand(I);
2852  const TreeEntry *TE = nullptr;
2853  const auto *It = find_if(VL, [this, &TE](Value *V) {
2854  TE = getTreeEntry(V);
2855  return TE;
2856  });
2857  if (It != VL.end() && TE->isSame(VL))
2858  return false;
2859  TreeEntry *Gather = nullptr;
2860  if (count_if(NonVectorized, [VL, &Gather](TreeEntry *TE) {
2861  assert(TE->State != TreeEntry::Vectorize &&
2862  "Only non-vectorized nodes are expected.");
2863  if (TE->isSame(VL)) {
2864  Gather = TE;
2865  return true;
2866  }
2867  return false;
2868  }) > 1)
2869  return false;
2870  if (Gather)
2871  GatherOps.push_back(Gather);
2872  }
2873  return true;
2874  };
2875  // 1. Propagate order to the graph nodes, which use only reordered nodes.
2876  // I.e., if the node has operands, that are reordered, try to make at least
2877  // one operand order in the natural order and reorder others + reorder the
2878  // user node itself.
2880  while (!OrderedEntries.empty()) {
2881  // 1. Filter out only reordered nodes.
2882  // 2. If the entry has multiple uses - skip it and jump to the next node.
2884  SmallVector<TreeEntry *> Filtered;
2885  for (TreeEntry *TE : OrderedEntries) {
2886  if (!(TE->State == TreeEntry::Vectorize ||
2887  (TE->State == TreeEntry::NeedToGather &&
2888  TE->getOpcode() == Instruction::ExtractElement)) ||
2889  TE->UserTreeIndices.empty() || !TE->ReuseShuffleIndices.empty() ||
2890  !all_of(drop_begin(TE->UserTreeIndices),
2891  [TE](const EdgeInfo &EI) {
2892  return EI.UserTE == TE->UserTreeIndices.front().UserTE;
2893  }) ||
2894  !Visited.insert(TE).second) {
2895  Filtered.push_back(TE);
2896  continue;
2897  }
2898  // Build a map between user nodes and their operands order to speedup
2899  // search. The graph currently does not provide this dependency directly.
2900  for (EdgeInfo &EI : TE->UserTreeIndices) {
2901  TreeEntry *UserTE = EI.UserTE;
2902  auto It = Users.find(UserTE);
2903  if (It == Users.end())
2904  It = Users.insert({UserTE, {}}).first;
2905  It->second.emplace_back(EI.EdgeIdx, TE);
2906  }
2907  }
2908  // Erase filtered entries.
2909  for_each(Filtered,
2910  [&OrderedEntries](TreeEntry *TE) { OrderedEntries.remove(TE); });
2911  for (const auto &Data : Users) {
2912  // Check that operands are used only in the User node.
2913  SmallVector<TreeEntry *> GatherOps;
2914  if (!CheckOperands(Data, GatherOps)) {
2915  for_each(Data.second,
2916  [&OrderedEntries](const std::pair<unsigned, TreeEntry *> &Op) {
2917  OrderedEntries.remove(Op.second);
2918  });
2919  continue;
2920  }
2921  // All operands are reordered and used only in this node - propagate the
2922  // most used order to the user node.
2925  for (const auto &Op : Data.second) {
2926  TreeEntry *OpTE = Op.second;
2927  if (!OpTE->ReuseShuffleIndices.empty())
2928  continue;
2929  const auto &Order = [OpTE, &GathersToOrders]() -> const OrdersType & {
2930  if (OpTE->State == TreeEntry::NeedToGather)
2931  return GathersToOrders.find(OpTE)->second;
2932  return OpTE->ReorderIndices;
2933  }();
2934  // Stores actually store the mask, not the order, need to invert.
2935  if (OpTE->State == TreeEntry::Vectorize && !OpTE->isAltShuffle() &&
2936  OpTE->getOpcode() == Instruction::Store && !Order.empty()) {
2938  inversePermutation(Order, Mask);
2939  unsigned E = Order.size();
2940  OrdersType CurrentOrder(E, E);
2941  transform(Mask, CurrentOrder.begin(), [E](int Idx) {
2942  return Idx == UndefMaskElem ? E : static_cast<unsigned>(Idx);
2943  });
2944  fixupOrderingIndices(CurrentOrder);
2945  ++OrdersUses.try_emplace(CurrentOrder).first->getSecond();
2946  } else {
2947  ++OrdersUses.try_emplace(Order).first->getSecond();
2948  }
2949  if (VisitedOps.insert(OpTE).second)
2950  OrdersUses.try_emplace({}, 0).first->getSecond() +=
2951  OpTE->UserTreeIndices.size();
2952  --OrdersUses[{}];
2953  }
2954  // If no orders - skip current nodes and jump to the next one, if any.
2955  if (OrdersUses.empty()) {
2956  for_each(Data.second,
2957  [&OrderedEntries](const std::pair<unsigned, TreeEntry *> &Op) {
2958  OrderedEntries.remove(Op.second);
2959  });
2960  continue;
2961  }
2962  // Choose the best order.
2963  ArrayRef<unsigned> BestOrder = OrdersUses.begin()->first;
2964  unsigned Cnt = OrdersUses.begin()->second;
2965  for (const auto &Pair : llvm::drop_begin(OrdersUses)) {
2966  if (Cnt < Pair.second || (Cnt == Pair.second && Pair.first.empty())) {
2967  BestOrder = Pair.first;
2968  Cnt = Pair.second;
2969  }
2970  }
2971  // Set order of the user node (reordering of operands and user nodes).
2972  if (BestOrder.empty()) {
2973  for_each(Data.second,
2974  [&OrderedEntries](const std::pair<unsigned, TreeEntry *> &Op) {
2975  OrderedEntries.remove(Op.second);
2976  });
2977  continue;
2978  }
2979  // Erase operands from OrderedEntries list and adjust their orders.
2980  VisitedOps.clear();
2982  inversePermutation(BestOrder, Mask);
2983  SmallVector<int> MaskOrder(BestOrder.size(), UndefMaskElem);
2984  unsigned E = BestOrder.size();
2985  transform(BestOrder, MaskOrder.begin(), [E](unsigned I) {
2986  return I < E ? static_cast<int>(I) : UndefMaskElem;
2987  });
2988  for (const std::pair<unsigned, TreeEntry *> &Op : Data.second) {
2989  TreeEntry *TE = Op.second;
2990  OrderedEntries.remove(TE);
2991  if (!VisitedOps.insert(TE).second)
2992  continue;
2993  if (!TE->ReuseShuffleIndices.empty() && TE->ReorderIndices.empty()) {
2994  // Just reorder reuses indices.
2995  reorderReuses(TE->ReuseShuffleIndices, Mask);
2996  continue;
2997  }
2998  // Gathers are processed separately.
2999  if (TE->State != TreeEntry::Vectorize)
3000  continue;
3001  assert((BestOrder.size() == TE->ReorderIndices.size() ||
3002  TE->ReorderIndices.empty()) &&
3003  "Non-matching sizes of user/operand entries.");
3004  reorderOrder(TE->ReorderIndices, Mask);
3005  }
3006  // For gathers just need to reorder its scalars.
3007  for (TreeEntry *Gather : GatherOps) {
3008  if (!Gather->ReuseShuffleIndices.empty())
3009  continue;
3010  assert(Gather->ReorderIndices.empty() &&
3011  "Unexpected reordering of gathers.");
3012  reorderScalars(Gather->Scalars, Mask);
3013  OrderedEntries.remove(Gather);
3014  }
3015  // Reorder operands of the user node and set the ordering for the user
3016  // node itself.
3017  if (Data.first->State != TreeEntry::Vectorize ||
3018  !isa<ExtractElementInst, ExtractValueInst, LoadInst>(
3019  Data.first->getMainOp()) ||
3020  Data.first->isAltShuffle())
3021  Data.first->reorderOperands(Mask);
3022  if (!isa<InsertElementInst, StoreInst>(Data.first->getMainOp()) ||
3023  Data.first->isAltShuffle()) {
3024  reorderScalars(Data.first->Scalars, Mask);
3025  reorderOrder(Data.first->ReorderIndices, MaskOrder);
3026  if (Data.first->ReuseShuffleIndices.empty() &&
3027  !Data.first->ReorderIndices.empty() &&
3028  !Data.first->isAltShuffle()) {
3029  // Insert user node to the list to try to sink reordering deeper in
3030  // the graph.
3031  OrderedEntries.insert(Data.first);
3032  }
3033  } else {
3034  reorderOrder(Data.first->ReorderIndices, Mask);
3035  }
3036  }
3037  }
3038 }
3039 
3040 void BoUpSLP::buildExternalUses(
3041  const ExtraValueToDebugLocsMap &ExternallyUsedValues) {
3042  // Collect the values that we need to extract from the tree.
3043  for (auto &TEPtr : VectorizableTree) {
3044  TreeEntry *Entry = TEPtr.get();
3045 
3046  // No need to handle users of gathered values.
3047  if (Entry->State == TreeEntry::NeedToGather)
3048  continue;
3049 
3050  // For each lane:
3051  for (int Lane = 0, LE = Entry->Scalars.size(); Lane != LE; ++Lane) {
3052  Value *Scalar = Entry->Scalars[Lane];
3053  int FoundLane = Entry->findLaneForValue(Scalar);
3054 
3055  // Check if the scalar is externally used as an extra arg.
3056  auto ExtI = ExternallyUsedValues.find(Scalar);
3057  if (ExtI != ExternallyUsedValues.end()) {
3058  LLVM_DEBUG(dbgs() << "SLP: Need to extract: Extra arg from lane "
3059  << Lane << " from " << *Scalar << ".\n");
3060  ExternalUses.emplace_back(Scalar, nullptr, FoundLane);
3061  }
3062  for (User *U : Scalar->users()) {
3063  LLVM_DEBUG(dbgs() << "SLP: Checking user:" << *U << ".\n");
3064 
3065  Instruction *UserInst = dyn_cast<Instruction>(U);
3066  if (!UserInst)
3067  continue;
3068 
3069  if (isDeleted(UserInst))
3070  continue;
3071 
3072  // Skip in-tree scalars that become vectors
3073  if (TreeEntry *UseEntry = getTreeEntry(U)) {
3074  Value *UseScalar = UseEntry->Scalars[0];
3075  // Some in-tree scalars will remain as scalar in vectorized
3076  // instructions. If that is the case, the one in Lane 0 will
3077  // be used.
3078  if (UseScalar != U ||
3079  UseEntry->State == TreeEntry::ScatterVectorize ||
3080  !InTreeUserNeedToExtract(Scalar, UserInst, TLI)) {
3081  LLVM_DEBUG(dbgs() << "SLP: \tInternal user will be removed:" << *U
3082  << ".\n");
3083  assert(UseEntry->State != TreeEntry::NeedToGather && "Bad state");
3084  continue;
3085  }
3086  }
3087 
3088  // Ignore users in the user ignore list.
3089  if (is_contained(UserIgnoreList, UserInst))
3090  continue;
3091 
3092  LLVM_DEBUG(dbgs() << "SLP: Need to extract:" << *U << " from lane "
3093  << Lane << " from " << *Scalar << ".\n");
3094  ExternalUses.push_back(ExternalUser(Scalar, U, FoundLane));
3095  }
3096  }
3097  }
3098 }
3099 
3100 void BoUpSLP::buildTree(ArrayRef<Value *> Roots,
3101  ArrayRef<Value *> UserIgnoreLst) {
3102  deleteTree();
3103  UserIgnoreList = UserIgnoreLst;
3104  if (!allSameType(Roots))
3105  return;
3106  buildTree_rec(Roots, 0, EdgeInfo());
3107 }
3108 
3109 namespace {
3110 /// Tracks the state we can represent the loads in the given sequence.
3111 enum class LoadsState { Gather, Vectorize, ScatterVectorize };
3112 } // anonymous namespace
3113 
3114 /// Checks if the given array of loads can be represented as a vectorized,
3115 /// scatter or just simple gather.
3116 static LoadsState canVectorizeLoads(ArrayRef<Value *> VL, const Value *VL0,
3117  const TargetTransformInfo &TTI,
3118  const DataLayout &DL, ScalarEvolution &SE,
3120  SmallVectorImpl<Value *> &PointerOps) {
3121  // Check that a vectorized load would load the same memory as a scalar
3122  // load. For example, we don't want to vectorize loads that are smaller
3123  // than 8-bit. Even though we have a packed struct {<i2, i2, i2, i2>} LLVM
3124  // treats loading/storing it as an i8 struct. If we vectorize loads/stores
3125  // from such a struct, we read/write packed bits disagreeing with the
3126  // unvectorized version.
3127  Type *ScalarTy = VL0->getType();
3128 
3129  if (DL.getTypeSizeInBits(ScalarTy) != DL.getTypeAllocSizeInBits(ScalarTy))
3130  return LoadsState::Gather;
3131 
3132  // Make sure all loads in the bundle are simple - we can't vectorize
3133  // atomic or volatile loads.
3134  PointerOps.clear();
3135  PointerOps.resize(VL.size());
3136  auto *POIter = PointerOps.begin();
3137  for (Value *V : VL) {
3138  auto *L = cast<LoadInst>(V);
3139  if (!L->isSimple())
3140  return LoadsState::Gather;
3141  *POIter = L->getPointerOperand();
3142  ++POIter;
3143  }
3144 
3145  Order.clear();
3146  // Check the order of pointer operands.
3147  if (llvm::sortPtrAccesses(PointerOps, ScalarTy, DL, SE, Order)) {
3148  Value *Ptr0;
3149  Value *PtrN;
3150  if (Order.empty()) {
3151  Ptr0 = PointerOps.front();
3152  PtrN = PointerOps.back();
3153  } else {
3154  Ptr0 = PointerOps[Order.front()];
3155  PtrN = PointerOps[Order.back()];
3156  }
3157  Optional<int> Diff =
3158  getPointersDiff(ScalarTy, Ptr0, ScalarTy, PtrN, DL, SE);
3159  // Check that the sorted loads are consecutive.
3160  if (static_cast<unsigned>(*Diff) == VL.size() - 1)
3161  return LoadsState::Vectorize;
3162  Align CommonAlignment = cast<LoadInst>(VL0)->getAlign();
3163  for (Value *V : VL)
3164  CommonAlignment =
3165  commonAlignment(CommonAlignment, cast<LoadInst>(V)->getAlign());
3166  if (TTI.isLegalMaskedGather(FixedVectorType::get(ScalarTy, VL.size()),
3167  CommonAlignment))
3168  return LoadsState::ScatterVectorize;
3169  }
3170 
3171  return LoadsState::Gather;
3172 }
3173 
3174 void BoUpSLP::buildTree_rec(ArrayRef<Value *> VL, unsigned Depth,
3175  const EdgeInfo &UserTreeIdx) {
3176  assert((allConstant(VL) || allSameType(VL)) && "Invalid types!");
3177 
3178  InstructionsState S = getSameOpcode(VL);
3179  if (Depth == RecursionMaxDepth) {
3180  LLVM_DEBUG(dbgs() << "SLP: Gathering due to max recursion depth.\n");
3181  newTreeEntry(VL, None /*not vectorized*/, S, UserTreeIdx);
3182  return;
3183  }
3184 
3185  // Don't handle scalable vectors
3186  if (S.getOpcode() == Instruction::ExtractElement &&
3187  isa<ScalableVectorType>(
3188  cast<ExtractElementInst>(S.OpValue)->getVectorOperandType())) {
3189  LLVM_DEBUG(dbgs() << "SLP: Gathering due to scalable vector type.\n");
3190  newTreeEntry(VL, None /*not vectorized*/, S, UserTreeIdx);
3191  return;
3192  }
3193 
3194  // Don't handle vectors.
3195  if (S.OpValue->getType()->isVectorTy() &&
3196  !isa<InsertElementInst>(S.OpValue)) {
3197  LLVM_DEBUG(dbgs() << "SLP: Gathering due to vector type.\n");
3198  newTreeEntry(VL, None /*not vectorized*/, S, UserTreeIdx);
3199  return;
3200  }
3201 
3202  if (StoreInst *SI = dyn_cast<StoreInst>(S.OpValue))
3203  if (SI->getValueOperand()->getType()->isVectorTy()) {
3204  LLVM_DEBUG(dbgs() << "SLP: Gathering due to store vector type.\n");
3205  newTreeEntry(VL, None /*not vectorized*/, S, UserTreeIdx);
3206  return;
3207  }
3208 
3209  // If all of the operands are identical or constant we have a simple solution.
3210  if (allConstant(VL) || isSplat(VL) || !allSameBlock(VL) || !S.getOpcode()) {
3211  LLVM_DEBUG(dbgs() << "SLP: Gathering due to C,S,B,O. \n");
3212  newTreeEntry(VL, None /*not vectorized*/, S, UserTreeIdx);
3213  return;
3214  }
3215 
3216  // We now know that this is a vector of instructions of the same type from
3217  // the same block.
3218 
3219  // Don't vectorize ephemeral values.
3220  for (Value *V : VL) {
3221  if (EphValues.count(V)) {
3222  LLVM_DEBUG(dbgs() << "SLP: The instruction (" << *V
3223  << ") is ephemeral.\n");
3224  newTreeEntry(VL, None /*not vectorized*/, S, UserTreeIdx);
3225  return;
3226  }
3227  }
3228 
3229  // Check if this is a duplicate of another entry.
3230  if (TreeEntry *E = getTreeEntry(S.OpValue)) {
3231  LLVM_DEBUG(dbgs() << "SLP: \tChecking bundle: " << *S.OpValue << ".\n");
3232  if (!E->isSame(VL)) {
3233  LLVM_DEBUG(dbgs() << "SLP: Gathering due to partial overlap.\n");
3234  newTreeEntry(VL, None /*not vectorized*/, S, UserTreeIdx);
3235  return;
3236  }
3237  // Record the reuse of the tree node. FIXME, currently this is only used to
3238  // properly draw the graph rather than for the actual vectorization.
3239  E->UserTreeIndices.push_back(UserTreeIdx);
3240  LLVM_DEBUG(dbgs() << "SLP: Perfect diamond merge at " << *S.OpValue
3241  << ".\n");
3242  return;
3243  }
3244 
3245  // Check that none of the instructions in the bundle are already in the tree.
3246  for (Value *V : VL) {
3247  auto *I = dyn_cast<Instruction>(V);
3248  if (!I)
3249  continue;
3250  if (getTreeEntry(I)) {
3251  LLVM_DEBUG(dbgs() << "SLP: The instruction (" << *V
3252  << ") is already in tree.\n");
3253  newTreeEntry(VL, None /*not vectorized*/, S, UserTreeIdx);
3254  return;
3255  }
3256  }
3257 
3258  // If any of the scalars is marked as a value that needs to stay scalar, then
3259  // we need to gather the scalars.
3260  // The reduction nodes (stored in UserIgnoreList) also should stay scalar.
3261  for (Value *V : VL) {
3262  if (MustGather.count(V) || is_contained(UserIgnoreList, V)) {
3263  LLVM_DEBUG(dbgs() << "SLP: Gathering due to gathered scalar.\n");
3264  newTreeEntry(VL, None /*not vectorized*/, S, UserTreeIdx);
3265  return;
3266  }
3267  }
3268 
3269  // Check that all of the users of the scalars that we want to vectorize are
3270  // schedulable.
3271  auto *VL0 = cast<Instruction>(S.OpValue);
3272  BasicBlock *BB = VL0->getParent();
3273 
3274  if (!DT->isReachableFromEntry(BB)) {
3275  // Don't go into unreachable blocks. They may contain instructions with
3276  // dependency cycles which confuse the final scheduling.
3277  LLVM_DEBUG(dbgs() << "SLP: bundle in unreachable block.\n");
3278  newTreeEntry(VL, None /*not vectorized*/, S, UserTreeIdx);
3279  return;
3280  }
3281 
3282  // Check that every instruction appears once in this bundle.
3283  SmallVector<int> ReuseShuffleIndicies;
3284  SmallVector<Value *, 4> UniqueValues;
3285  DenseMap<Value *, unsigned> UniquePositions;
3286  for (Value *V : VL) {
3287  auto Res = UniquePositions.try_emplace(V, UniqueValues.size());
3288  ReuseShuffleIndicies.emplace_back(Res.first->second);
3289  if (Res.second)
3290  UniqueValues.emplace_back(V);
3291  }
3292  size_t NumUniqueScalarValues = UniqueValues.size();
3293  if (NumUniqueScalarValues == VL.size()) {
3294  ReuseShuffleIndicies.clear();
3295  } else {
3296  LLVM_DEBUG(dbgs() << "SLP: Shuffle for reused scalars.\n");
3297  if (NumUniqueScalarValues <= 1 ||
3298  !llvm::isPowerOf2_32(NumUniqueScalarValues)) {
3299  LLVM_DEBUG(dbgs() << "SLP: Scalar used twice in bundle.\n");
3300  newTreeEntry(VL, None /*not vectorized*/, S, UserTreeIdx);
3301  return;
3302  }
3303  VL = UniqueValues;
3304  }
3305 
3306  auto &BSRef = BlocksSchedules[BB];
3307  if (!BSRef)
3308  BSRef = std::make_unique<BlockScheduling>(BB);
3309 
3310  BlockScheduling &BS = *BSRef.get();
3311 
3312  Optional<ScheduleData *> Bundle = BS.tryScheduleBundle(VL, this, S);
3313  if (!Bundle) {
3314  LLVM_DEBUG(dbgs() << "SLP: We are not able to schedule this bundle!\n");
3315  assert((!BS.getScheduleData(VL0) ||
3316  !BS.getScheduleData(VL0)->isPartOfBundle()) &&
3317  "tryScheduleBundle should cancelScheduling on failure");
3318  newTreeEntry(VL, None /*not vectorized*/, S, UserTreeIdx,
3319  ReuseShuffleIndicies);
3320  return;
3321  }
3322  LLVM_DEBUG(dbgs() << "SLP: We are able to schedule this bundle.\n");
3323 
3324  unsigned ShuffleOrOp = S.isAltShuffle() ?
3325  (unsigned) Instruction::ShuffleVector : S.getOpcode();
3326  switch (ShuffleOrOp) {
3327  case Instruction::PHI: {
3328  auto *PH = cast<PHINode>(VL0);
3329 
3330  // Check for terminator values (e.g. invoke).
3331  for (Value *V : VL)
3332  for (unsigned I = 0, E = PH->getNumIncomingValues(); I < E; ++I) {
3333  Instruction *Term = dyn_cast<Instruction>(
3334  cast<PHINode>(V)->getIncomingValueForBlock(
3335  PH->getIncomingBlock(I)));
3336  if (Term && Term->isTerminator()) {
3337  LLVM_DEBUG(dbgs()
3338  << "SLP: Need to swizzle PHINodes (terminator use).\n");
3339  BS.cancelScheduling(VL, VL0);
3340  newTreeEntry(VL, None /*not vectorized*/, S, UserTreeIdx,
3341  ReuseShuffleIndicies);
3342  return;
3343  }
3344  }
3345 
3346  TreeEntry *TE =
3347  newTreeEntry(VL, Bundle, S, UserTreeIdx, ReuseShuffleIndicies);
3348  LLVM_DEBUG(dbgs() << "SLP: added a vector of PHINodes.\n");
3349 
3350  // Keeps the reordered operands to avoid code duplication.
3351  SmallVector<ValueList, 2> OperandsVec;
3352  for (unsigned I = 0, E = PH->getNumIncomingValues(); I < E; ++I) {
3353  if (!DT->isReachableFromEntry(PH->getIncomingBlock(I))) {
3354  ValueList Operands(VL.size(), PoisonValue::get(PH->getType()));
3355  TE->setOperand(I, Operands);
3356  OperandsVec.push_back(Operands);
3357  continue;
3358  }
3359  ValueList Operands;
3360  // Prepare the operand vector.
3361  for (Value *V : VL)
3362  Operands.push_back(cast<PHINode>(V)->getIncomingValueForBlock(
3363  PH->getIncomingBlock(I)));
3364  TE->setOperand(I, Operands);
3365  OperandsVec.push_back(Operands);
3366  }
3367  for (unsigned OpIdx = 0, OpE = OperandsVec.size(); OpIdx != OpE; ++OpIdx)
3368  buildTree_rec(OperandsVec[OpIdx], Depth + 1, {TE, OpIdx});
3369  return;
3370  }
3371  case Instruction::ExtractValue:
3372  case Instruction::ExtractElement: {
3373  OrdersType CurrentOrder;
3374  bool Reuse = canReuseExtract(VL, VL0, CurrentOrder);
3375  if (Reuse) {
3376  LLVM_DEBUG(dbgs() << "SLP: Reusing or shuffling extract sequence.\n");
3377  newTreeEntry(VL, Bundle /*vectorized*/, S, UserTreeIdx,
3378  ReuseShuffleIndicies);
3379  // This is a special case, as it does not gather, but at the same time
3380  // we are not extending buildTree_rec() towards the operands.
3381  ValueList Op0;
3382  Op0.assign(VL.size(), VL0->getOperand(0));
3383  VectorizableTree.back()->setOperand(0, Op0);
3384  return;
3385  }
3386  if (!CurrentOrder.empty()) {
3387  LLVM_DEBUG({
3388  dbgs() << "SLP: Reusing or shuffling of reordered extract sequence "
3389  "with order";
3390  for (unsigned Idx : CurrentOrder)
3391  dbgs() << " " << Idx;
3392  dbgs() << "\n";
3393  });
3394  fixupOrderingIndices(CurrentOrder);
3395  // Insert new order with initial value 0, if it does not exist,
3396  // otherwise return the iterator to the existing one.
3397  newTreeEntry(VL, Bundle /*vectorized*/, S, UserTreeIdx,
3398  ReuseShuffleIndicies, CurrentOrder);
3399  // This is a special case, as it does not gather, but at the same time
3400  // we are not extending buildTree_rec() towards the operands.
3401  ValueList Op0;
3402  Op0.assign(VL.size(), VL0->getOperand(0));
3403  VectorizableTree.back()->setOperand(0, Op0);
3404  return;
3405  }
3406  LLVM_DEBUG(dbgs() << "SLP: Gather extract sequence.\n");
3407  newTreeEntry(VL, None /*not vectorized*/, S, UserTreeIdx,
3408  ReuseShuffleIndicies);
3409  BS.cancelScheduling(VL, VL0);
3410  return;
3411  }
3412  case Instruction::InsertElement: {
3413  assert(ReuseShuffleIndicies.empty() && "All inserts should be unique");
3414 
3415  // Check that we have a buildvector and not a shuffle of 2 or more
3416  // different vectors.
3417  ValueSet SourceVectors;
3418  int MinIdx = std::numeric_limits<int>::max();
3419  for (Value *V : VL) {
3420  SourceVectors.insert(cast<Instruction>(V)->getOperand(0));
3421  Optional<int> Idx = *getInsertIndex(V, 0);
3422  if (!Idx || *Idx == UndefMaskElem)
3423  continue;
3424  MinIdx = std::min(MinIdx, *Idx);
3425  }
3426 
3427  if (count_if(VL, [&SourceVectors](Value *V) {
3428  return !SourceVectors.contains(V);
3429  }) >= 2) {
3430  // Found 2nd source vector - cancel.
3431  LLVM_DEBUG(dbgs() << "SLP: Gather of insertelement vectors with "
3432  "different source vectors.\n");
3433  newTreeEntry(VL, None /*not vectorized*/, S, UserTreeIdx);
3434  BS.cancelScheduling(VL, VL0);
3435  return;
3436  }
3437 
3438  auto OrdCompare = [](const std::pair<int, int> &P1,
3439  const std::pair<int, int> &P2) {
3440  return P1.first > P2.first;
3441  };
3443  decltype(OrdCompare)>
3444  Indices(OrdCompare);
3445  for (int I = 0, E = VL.size(); I < E; ++I) {
3446  Optional<int> Idx = *getInsertIndex(VL[I], 0);
3447  if (!Idx || *Idx == UndefMaskElem)
3448  continue;
3449  Indices.emplace(*Idx, I);
3450  }
3451  OrdersType CurrentOrder(VL.size(), VL.size());
3452  bool IsIdentity = true;
3453  for (int I = 0, E = VL.size(); I < E; ++I) {
3454  CurrentOrder[Indices.top().second] = I;
3455  IsIdentity &= Indices.top().second == I;
3456  Indices.pop();
3457  }
3458  if (IsIdentity)
3459  CurrentOrder.clear();
3460  TreeEntry *TE = newTreeEntry(VL, Bundle /*vectorized*/, S, UserTreeIdx,
3461  None, CurrentOrder);
3462  LLVM_DEBUG(dbgs() << "SLP: added inserts bundle.\n");
3463 
3464  constexpr int NumOps = 2;
3465  ValueList VectorOperands[NumOps];
3466  for (int I = 0; I < NumOps; ++I) {
3467  for (Value *V : VL)
3468  VectorOperands[I].push_back(cast<Instruction>(V)->getOperand(I));
3469 
3470  TE->setOperand(I, VectorOperands[I]);
3471  }
3472  buildTree_rec(VectorOperands[NumOps - 1], Depth + 1, {TE, NumOps - 1});
3473  return;
3474  }
3475  case Instruction::Load: {
3476  // Check that a vectorized load would load the same memory as a scalar
3477  // load. For example, we don't want to vectorize loads that are smaller
3478  // than 8-bit. Even though we have a packed struct {<i2, i2, i2, i2>} LLVM
3479  // treats loading/storing it as an i8 struct. If we vectorize loads/stores
3480  // from such a struct, we read/write packed bits disagreeing with the
3481  // unvectorized version.
3482  SmallVector<Value *> PointerOps;
3483  OrdersType CurrentOrder;
3484  TreeEntry *TE = nullptr;
3485  switch (canVectorizeLoads(VL, VL0, *TTI, *DL, *SE, CurrentOrder,
3486  PointerOps)) {
3487  case LoadsState::Vectorize:
3488  if (CurrentOrder.empty()) {
3489  // Original loads are consecutive and does not require reordering.
3490  TE = newTreeEntry(VL, Bundle /*vectorized*/, S, UserTreeIdx,
3491  ReuseShuffleIndicies);
3492  LLVM_DEBUG(dbgs() << "SLP: added a vector of loads.\n");
3493  } else {
3494  fixupOrderingIndices(CurrentOrder);
3495  // Need to reorder.
3496  TE = newTreeEntry(VL, Bundle /*vectorized*/, S, UserTreeIdx,
3497  ReuseShuffleIndicies, CurrentOrder);
3498  LLVM_DEBUG(dbgs() << "SLP: added a vector of jumbled loads.\n");
3499  }
3500  TE->setOperandsInOrder();
3501  break;
3502  case LoadsState::ScatterVectorize:
3503  // Vectorizing non-consecutive loads with `llvm.masked.gather`.
3504  TE = newTreeEntry(VL, TreeEntry::ScatterVectorize, Bundle, S,
3505  UserTreeIdx, ReuseShuffleIndicies);
3506  TE->setOperandsInOrder();
3507  buildTree_rec(PointerOps, Depth + 1, {TE, 0});
3508  LLVM_DEBUG(dbgs() << "SLP: added a vector of non-consecutive loads.\n");
3509  break;
3510  case LoadsState::Gather:
3511  BS.cancelScheduling(VL, VL0);
3512  newTreeEntry(VL, None /*not vectorized*/, S, UserTreeIdx,
3513  ReuseShuffleIndicies);
3514 #ifndef NDEBUG
3515  Type *ScalarTy = VL0->getType();
3516  if (DL->getTypeSizeInBits(ScalarTy) !=
3517  DL->getTypeAllocSizeInBits(ScalarTy))
3518  LLVM_DEBUG(dbgs() << "SLP: Gathering loads of non-packed type.\n");
3519  else if (any_of(VL, [](Value *V) {
3520  return !cast<LoadInst>(V)->isSimple();
3521  }))
3522  LLVM_DEBUG(dbgs() << "SLP: Gathering non-simple loads.\n");
3523  else
3524  LLVM_DEBUG(dbgs() << "SLP: Gathering non-consecutive loads.\n");
3525 #endif // NDEBUG
3526  break;
3527  }
3528  return;
3529  }
3530  case Instruction::ZExt:
3531  case Instruction::SExt:
3532  case Instruction::FPToUI:
3533  case Instruction::FPToSI:
3534  case Instruction::FPExt:
3535  case Instruction::PtrToInt:
3536  case Instruction::IntToPtr:
3537  case Instruction::SIToFP:
3538  case Instruction::UIToFP:
3539  case Instruction::Trunc:
3540  case Instruction::FPTrunc:
3541  case Instruction::BitCast: {
3542  Type *SrcTy = VL0->getOperand(0)->getType();
3543  for (Value *V : VL) {
3544  Type *Ty = cast<Instruction>(V)->getOperand(0)->getType();
3545  if (Ty != SrcTy || !isValidElementType(Ty)) {
3546  BS.cancelScheduling(VL, VL0);
3547  newTreeEntry(VL, None /*not vectorized*/, S, UserTreeIdx,
3548  ReuseShuffleIndicies);
3549  LLVM_DEBUG(dbgs()
3550  << "SLP: Gathering casts with different src types.\n");
3551  return;
3552  }
3553  }
3554  TreeEntry *TE = newTreeEntry(VL, Bundle /*vectorized*/, S, UserTreeIdx,
3555  ReuseShuffleIndicies);
3556  LLVM_DEBUG(dbgs() << "SLP: added a vector of casts.\n");
3557 
3558  TE->setOperandsInOrder();
3559  for (unsigned i = 0, e = VL0->getNumOperands(); i < e; ++i) {
3560  ValueList Operands;
3561  // Prepare the operand vector.
3562  for (Value *V : VL)
3563  Operands.push_back(cast<Instruction>(V)->getOperand(i));
3564 
3565  buildTree_rec(Operands, Depth + 1, {TE, i});
3566  }
3567  return;
3568  }
3569  case Instruction::ICmp:
3570  case Instruction::FCmp: {
3571  // Check that all of the compares have the same predicate.
3572  CmpInst::Predicate P0 = cast<CmpInst>(VL0)->getPredicate();
3574  Type *ComparedTy = VL0->getOperand(0)->getType();
3575  for (Value *V : VL) {
3576  CmpInst *Cmp = cast<CmpInst>(V);
3577  if ((Cmp->getPredicate() != P0 && Cmp->getPredicate() != SwapP0) ||
3578  Cmp->getOperand(0)->getType() != ComparedTy) {
3579  BS.cancelScheduling(VL, VL0);
3580  newTreeEntry(VL, None /*not vectorized*/, S, UserTreeIdx,
3581  ReuseShuffleIndicies);
3582  LLVM_DEBUG(dbgs()
3583  << "SLP: Gathering cmp with different predicate.\n");
3584  return;
3585  }
3586  }
3587 
3588  TreeEntry *TE = newTreeEntry(VL, Bundle /*vectorized*/, S, UserTreeIdx,
3589  ReuseShuffleIndicies);
3590  LLVM_DEBUG(dbgs() << "SLP: added a vector of compares.\n");
3591 
3592  ValueList Left, Right;
3593  if (cast<CmpInst>(VL0)->isCommutative()) {
3594  // Commutative predicate - collect + sort operands of the instructions
3595  // so that each side is more likely to have the same opcode.
3596  assert(P0 == SwapP0 && "Commutative Predicate mismatch");
3597  reorderInputsAccordingToOpcode(VL, Left, Right, *DL, *SE, *this);
3598  } else {
3599  // Collect operands - commute if it uses the swapped predicate.
3600  for (Value *V : VL) {
3601  auto *Cmp = cast<CmpInst>(V);
3602  Value *LHS = Cmp->getOperand(0);
3603  Value *RHS = Cmp->getOperand(1);
3604  if (Cmp->getPredicate() != P0)
3605  std::swap(LHS, RHS);
3606  Left.push_back(LHS);
3607  Right.push_back(RHS);
3608  }
3609  }
3610  TE->setOperand(0, Left);
3611  TE->setOperand(1, Right);
3612  buildTree_rec(Left, Depth + 1, {TE, 0});
3613  buildTree_rec(Right, Depth + 1, {TE, 1});
3614  return;
3615  }
3616  case Instruction::Select:
3617  case Instruction::FNeg:
3618  case Instruction::Add:
3619  case Instruction::FAdd:
3620  case Instruction::Sub:
3621  case Instruction::FSub:
3622  case Instruction::Mul:
3623  case Instruction::FMul:
3624  case Instruction::UDiv:
3625  case Instruction::SDiv:
3626  case Instruction::FDiv:
3627  case Instruction::URem:
3628  case Instruction::SRem:
3629  case Instruction::FRem:
3630  case Instruction::Shl:
3631  case Instruction::LShr:
3632  case Instruction::AShr:
3633  case Instruction::And:
3634  case Instruction::Or:
3635  case Instruction::Xor: {
3636  TreeEntry *TE = newTreeEntry(VL, Bundle /*vectorized*/, S, UserTreeIdx,
3637  ReuseShuffleIndicies);
3638  LLVM_DEBUG(dbgs() << "SLP: added a vector of un/bin op.\n");
3639 
3640  // Sort operands of the instructions so that each side is more likely to
3641  // have the same opcode.
3642  if (isa<BinaryOperator>(VL0) && VL0->isCommutative()) {
3643  ValueList Left, Right;
3644  reorderInputsAccordingToOpcode(VL, Left, Right, *DL, *SE, *this);
3645  TE->setOperand(0, Left);
3646  TE->setOperand(1, Right);
3647  buildTree_rec(Left, Depth + 1, {TE, 0});
3648  buildTree_rec(Right, Depth + 1, {TE, 1});
3649  return;
3650  }
3651 
3652  TE->setOperandsInOrder();
3653  for (unsigned i = 0, e = VL0->getNumOperands(); i < e; ++i) {
3654  ValueList Operands;
3655  // Prepare the operand vector.
3656  for (Value *V : VL)
3657  Operands.push_back(cast<Instruction>(V)->getOperand(i));
3658 
3659  buildTree_rec(Operands, Depth + 1, {TE, i});
3660  }
3661  return;
3662  }
3663  case Instruction::GetElementPtr: {
3664  // We don't combine GEPs with complicated (nested) indexing.
3665  for (Value *V : VL) {
3666  if (cast<Instruction>(V)->getNumOperands() != 2) {
3667  LLVM_DEBUG(dbgs() << "SLP: not-vectorizable GEP (nested indexes).\n");
3668  BS.cancelScheduling(VL, VL0);
3669  newTreeEntry(VL, None /*not vectorized*/, S, UserTreeIdx,
3670  ReuseShuffleIndicies);
3671  return;
3672  }
3673  }
3674 
3675  // We can't combine several GEPs into one vector if they operate on
3676  // different types.
3677  Type *Ty0 = VL0->getOperand(0)->getType();
3678  for (Value *V : VL) {
3679  Type *CurTy = cast<Instruction>(V)->getOperand(0)->getType();
3680  if (Ty0 != CurTy) {
3681  LLVM_DEBUG(dbgs()
3682  << "SLP: not-vectorizable GEP (different types).\n");
3683  BS.cancelScheduling(VL, VL0);
3684  newTreeEntry(VL, None /*not vectorized*/, S, UserTreeIdx,
3685  ReuseShuffleIndicies);
3686  return;
3687  }
3688  }
3689 
3690  // We don't combine GEPs with non-constant indexes.
3691  Type *Ty1 = VL0->getOperand(1)->getType();
3692  for (Value *V : VL) {
3693  auto Op = cast<Instruction>(V)->getOperand(1);
3694  if (!isa<ConstantInt>(Op) ||
3695  (Op->getType() != Ty1 &&
3696  Op->getType()->getScalarSizeInBits() >
3697  DL->getIndexSizeInBits(
3698  V->getType()->getPointerAddressSpace()))) {
3699  LLVM_DEBUG(dbgs()
3700  << "SLP: not-vectorizable GEP (non-constant indexes).\n");
3701  BS.cancelScheduling(VL, VL0);
3702  newTreeEntry(VL, None /*not vectorized*/, S, UserTreeIdx,
3703  ReuseShuffleIndicies);
3704  return;
3705  }
3706  }
3707 
3708  TreeEntry *TE = newTreeEntry(VL, Bundle /*vectorized*/, S, UserTreeIdx,
3709  ReuseShuffleIndicies);
3710  LLVM_DEBUG(dbgs() << "SLP: added a vector of GEPs.\n");
3711  TE->setOperandsInOrder();
3712  for (unsigned i = 0, e = 2; i < e; ++i) {
3713  ValueList Operands;
3714  // Prepare the operand vector.
3715  for (Value *V : VL)
3716  Operands.push_back(cast<Instruction>(V)->getOperand(i));
3717 
3718  buildTree_rec(Operands, Depth + 1, {TE, i});
3719  }
3720  return;
3721  }
3722  case Instruction::Store: {
3723  // Check if the stores are consecutive or if we need to swizzle them.
3724  llvm::Type *ScalarTy = cast<StoreInst>(VL0)->getValueOperand()->getType();
3725  // Avoid types that are padded when being allocated as scalars, while
3726  // being packed together in a vector (such as i1).
3727  if (DL->getTypeSizeInBits(ScalarTy) !=
3728  DL->getTypeAllocSizeInBits(ScalarTy)) {
3729  BS.cancelScheduling(VL, VL0);
3730  newTreeEntry(VL, None /*not vectorized*/, S, UserTreeIdx,
3731  ReuseShuffleIndicies);
3732  LLVM_DEBUG(dbgs() << "SLP: Gathering stores of non-packed type.\n");
3733  return;
3734  }
3735  // Make sure all stores in the bundle are simple - we can't vectorize
3736  // atomic or volatile stores.
3737  SmallVector<Value *, 4> PointerOps(VL.size());
3738  ValueList Operands(VL.size());
3739  auto POIter = PointerOps.begin();
3740  auto OIter = Operands.begin();
3741  for (Value *V : VL) {
3742  auto *SI = cast<StoreInst>(V);
3743  if (!SI->isSimple()) {
3744  BS.cancelScheduling(VL, VL0);
3745  newTreeEntry(VL, None /*not vectorized*/, S, UserTreeIdx,
3746  ReuseShuffleIndicies);
3747  LLVM_DEBUG(dbgs() << "SLP: Gathering non-simple stores.\n");
3748  return;
3749  }
3750  *POIter = SI->getPointerOperand();
3751  *OIter = SI->getValueOperand();
3752  ++POIter;
3753  ++OIter;
3754  }
3755 
3756  OrdersType CurrentOrder;
3757  // Check the order of pointer operands.
3758  if (llvm::sortPtrAccesses(PointerOps, ScalarTy, *DL, *SE, CurrentOrder)) {
3759  Value *Ptr0;
3760  Value *PtrN;
3761  if (CurrentOrder.empty()) {
3762  Ptr0 = PointerOps.front();
3763  PtrN = PointerOps.back();
3764  } else {
3765  Ptr0 = PointerOps[CurrentOrder.front()];
3766  PtrN = PointerOps[CurrentOrder.back()];
3767  }
3768  Optional<int> Dist =
3769  getPointersDiff(ScalarTy, Ptr0, ScalarTy, PtrN, *DL, *SE);
3770  // Check that the sorted pointer operands are consecutive.
3771  if (static_cast<unsigned>(*Dist) == VL.size() - 1) {
3772  if (CurrentOrder.empty()) {
3773  // Original stores are consecutive and does not require reordering.
3774  TreeEntry *TE = newTreeEntry(VL, Bundle /*vectorized*/, S,
3775  UserTreeIdx, ReuseShuffleIndicies);
3776  TE->setOperandsInOrder();
3777  buildTree_rec(Operands, Depth + 1, {TE, 0});
3778  LLVM_DEBUG(dbgs() << "SLP: added a vector of stores.\n");
3779  } else {
3780  fixupOrderingIndices(CurrentOrder);
3781  TreeEntry *TE =
3782  newTreeEntry(VL, Bundle /*vectorized*/, S, UserTreeIdx,
3783  ReuseShuffleIndicies, CurrentOrder);
3784  TE->setOperandsInOrder();
3785  buildTree_rec(Operands, Depth + 1, {TE, 0});
3786  LLVM_DEBUG(dbgs() << "SLP: added a vector of jumbled stores.\n");
3787  }
3788  return;
3789  }
3790  }
3791 
3792  BS.cancelScheduling(VL, VL0);
3793  newTreeEntry(VL, None /*not vectorized*/, S, UserTreeIdx,
3794  ReuseShuffleIndicies);
3795  LLVM_DEBUG(dbgs() << "SLP: Non-consecutive store.\n");
3796  return;
3797  }
3798  case Instruction::Call: {
3799  // Check if the calls are all to the same vectorizable intrinsic or
3800  // library function.
3801  CallInst *CI = cast<CallInst>(VL0);
3803 
3804  VFShape Shape = VFShape::get(
3805  *CI, ElementCount::getFixed(static_cast<unsigned int>(VL.size())),
3806  false /*HasGlobalPred*/);
3807  Function *VecFunc = VFDatabase(*CI).getVectorizedFunction(Shape);
3808 
3809  if (!VecFunc && !isTriviallyVectorizable(ID)) {
3810  BS.cancelScheduling(VL, VL0);
3811  newTreeEntry(VL, None /*not vectorized*/, S, UserTreeIdx,
3812  ReuseShuffleIndicies);
3813  LLVM_DEBUG(dbgs() << "SLP: Non-vectorizable call.\n");
3814  return;
3815  }
3816  Function *F = CI->getCalledFunction();
3817  unsigned NumArgs = CI->getNumArgOperands();
3818  SmallVector<Value*, 4> ScalarArgs(NumArgs, nullptr);
3819  for (unsigned j = 0; j != NumArgs; ++j)
3821  ScalarArgs[j] = CI->getArgOperand(j);
3822  for (Value *V : VL) {
3823  CallInst *CI2 = dyn_cast<CallInst>(V);
3824  if (!CI2 || CI2->getCalledFunction() != F ||
3825  getVectorIntrinsicIDForCall(CI2, TLI) != ID ||
3826  (VecFunc &&
3827  VecFunc != VFDatabase(*CI2).getVectorizedFunction(Shape)) ||
3828  !CI->hasIdenticalOperandBundleSchema(*CI2)) {
3829  BS.cancelScheduling(VL, VL0);
3830  newTreeEntry(VL, None /*not vectorized*/, S, UserTreeIdx,
3831  ReuseShuffleIndicies);
3832  LLVM_DEBUG(dbgs() << "SLP: mismatched calls:" << *CI << "!=" << *V
3833  << "\n");
3834  return;
3835  }
3836  // Some intrinsics have scalar arguments and should be same in order for
3837  // them to be vectorized.
3838  for (unsigned j = 0; j != NumArgs; ++j) {
3840  Value *A1J = CI2->getArgOperand(j);
3841  if (ScalarArgs[j] != A1J) {
3842  BS.cancelScheduling(VL, VL0);
3843  newTreeEntry(VL, None /*not vectorized*/, S, UserTreeIdx,
3844  ReuseShuffleIndicies);
3845  LLVM_DEBUG(dbgs() << "SLP: mismatched arguments in call:" << *CI
3846  << " argument " << ScalarArgs[j] << "!=" << A1J
3847  << "\n");
3848  return;
3849  }
3850  }
3851  }
3852  // Verify that the bundle operands are identical between the two calls.
3853  if (CI->hasOperandBundles() &&
3855  CI->op_begin() + CI->getBundleOperandsEndIndex(),
3856  CI2->op_begin() + CI2->getBundleOperandsStartIndex())) {
3857  BS.cancelScheduling(VL, VL0);
3858  newTreeEntry(VL, None /*not vectorized*/, S, UserTreeIdx,
3859  ReuseShuffleIndicies);
3860  LLVM_DEBUG(dbgs() << "SLP: mismatched bundle operands in calls:"
3861  << *CI << "!=" << *V << '\n');
3862  return;
3863  }
3864  }
3865 
3866  TreeEntry *TE = newTreeEntry(VL, Bundle /*vectorized*/, S, UserTreeIdx,
3867  ReuseShuffleIndicies);
3868  TE->setOperandsInOrder();
3869  for (unsigned i = 0, e = CI->getNumArgOperands(); i != e; ++i) {
3870  ValueList Operands;
3871  // Prepare the operand vector.
3872  for (Value *V : VL) {
3873  auto *CI2 = cast<CallInst>(V);
3874  Operands.push_back(CI2->getArgOperand(i));
3875  }
3876  buildTree_rec(Operands, Depth + 1, {TE, i});
3877  }
3878  return;
3879  }
3880  case Instruction::ShuffleVector: {
3881  // If this is not an alternate sequence of opcode like add-sub
3882  // then do not vectorize this instruction.
3883  if (!S.isAltShuffle()) {
3884  BS.cancelScheduling(VL, VL0);
3885  newTreeEntry(VL, None /*not vectorized*/, S, UserTreeIdx,
3886  ReuseShuffleIndicies);
3887  LLVM_DEBUG(dbgs() << "SLP: ShuffleVector are not vectorized.\n");
3888  return;
3889  }
3890  TreeEntry *TE = newTreeEntry(VL, Bundle /*vectorized*/, S, UserTreeIdx,
3891  ReuseShuffleIndicies);
3892  LLVM_DEBUG(dbgs() << "SLP: added a ShuffleVector op.\n");
3893 
3894  // Reorder operands if reordering would enable vectorization.
3895  if (isa<BinaryOperator>(VL0)) {
3896  ValueList Left, Right;
3897  reorderInputsAccordingToOpcode(VL, Left, Right, *DL, *SE, *this);
3898  TE->setOperand(0, Left);
3899  TE->setOperand(1, Right);
3900  buildTree_rec(Left, Depth + 1, {TE, 0});
3901  buildTree_rec(Right, Depth + 1, {TE, 1});
3902  return;
3903  }
3904 
3905  TE->setOperandsInOrder();
3906  for (unsigned i = 0, e = VL0->getNumOperands(); i < e; ++i) {
3907  ValueList Operands;
3908  // Prepare the operand vector.
3909  for (Value *V : VL)
3910  Operands.push_back(cast<Instruction>(V)->getOperand(i));
3911 
3912  buildTree_rec(Operands, Depth + 1, {TE, i});
3913  }
3914  return;
3915  }
3916  default:
3917  BS.cancelScheduling(VL, VL0);
3918  newTreeEntry(VL, None /*not vectorized*/, S, UserTreeIdx,
3919  ReuseShuffleIndicies);
3920  LLVM_DEBUG(dbgs() << "SLP: Gathering unknown instruction.\n");
3921  return;
3922  }
3923 }
3924 
3925 unsigned BoUpSLP::canMapToVector(Type *T, const DataLayout &DL) const {
3926  unsigned N = 1;
3927  Type *EltTy = T;
3928 
3929  while (isa<StructType>(EltTy) || isa<ArrayType>(EltTy) ||
3930  isa<VectorType>(EltTy)) {
3931  if (auto *ST = dyn_cast<StructType>(EltTy)) {
3932  // Check that struct is homogeneous.
3933  for (const auto *Ty : ST->elements())
3934  if (Ty != *ST->element_begin())
3935  return 0;
3936  N *= ST->getNumElements();
3937  EltTy = *ST->element_begin();
3938  } else if (auto *AT = dyn_cast<ArrayType>(EltTy)) {
3939  N *= AT->getNumElements();
3940  EltTy = AT->getElementType();
3941  } else {
3942  auto *VT = cast<FixedVectorType>(EltTy);
3943  N *= VT->getNumElements();
3944  EltTy = VT->getElementType();
3945  }
3946  }
3947 
3948  if (!isValidElementType(EltTy))
3949  return 0;
3950  uint64_t VTSize = DL.getTypeStoreSizeInBits(FixedVectorType::get(EltTy, N));
3951  if (VTSize < MinVecRegSize || VTSize > MaxVecRegSize || VTSize != DL.getTypeStoreSizeInBits(T))
3952  return 0;
3953  return N;
3954 }
3955 
3956 bool BoUpSLP::canReuseExtract(ArrayRef<Value *> VL, Value *OpValue,
3957  SmallVectorImpl<unsigned> &CurrentOrder) const {
3958  Instruction *E0 = cast<Instruction>(OpValue);
3959  assert(E0->getOpcode() == Instruction::ExtractElement ||
3960  E0->getOpcode() == Instruction::ExtractValue);
3961  assert(E0->getOpcode() == getSameOpcode(VL).getOpcode() && "Invalid opcode");
3962  // Check if all of the extracts come from the same vector and from the
3963  // correct offset.
3964  Value *Vec = E0->getOperand(0);
3965 
3966  CurrentOrder.clear();
3967 
3968  // We have to extract from a vector/aggregate with the same number of elements.
3969  unsigned NElts;
3970  if (E0->getOpcode() == Instruction::ExtractValue) {
3971  const DataLayout &DL = E0->getModule()->getDataLayout();
3972  NElts = canMapToVector(Vec->getType(), DL);
3973  if (!NElts)
3974  return false;
3975  // Check if load can be rewritten as load of vector.
3976  LoadInst *LI = dyn_cast<LoadInst>(Vec);
3977  if (!LI || !LI->isSimple() || !LI->hasNUses(VL.size()))
3978  return false;
3979  } else {
3980  NElts = cast<FixedVectorType>(Vec->getType())->getNumElements();
3981  }
3982 
3983  if (NElts != VL.size())
3984  return false;
3985 
3986  // Check that all of the indices extract from the correct offset.
3987  bool ShouldKeepOrder = true;
3988  unsigned E = VL.size();
3989  // Assign to all items the initial value E + 1 so we can check if the extract
3990  // instruction index was used already.
3991  // Also, later we can check that all the indices are used and we have a
3992  // consecutive access in the extract instructions, by checking that no
3993  // element of CurrentOrder still has value E + 1.
3994  CurrentOrder.assign(E, E + 1);
3995  unsigned I = 0;
3996  for (; I < E; ++I) {
3997  auto *Inst = cast<Instruction>(VL[I]);
3998  if (Inst->getOperand(0) != Vec)
3999  break;
4000  Optional<unsigned> Idx = getExtractIndex(Inst);
4001  if (!Idx)
4002  break;
4003  const unsigned ExtIdx = *Idx;
4004  if (ExtIdx != I) {
4005  if (ExtIdx >= E || CurrentOrder[ExtIdx] != E + 1)
4006  break;
4007  ShouldKeepOrder = false;
4008  CurrentOrder[ExtIdx] = I;
4009  } else {
4010  if (CurrentOrder[I] != E + 1)
4011  break;
4012  CurrentOrder[I] = I;
4013  }
4014  }
4015  if (I < E) {
4016  CurrentOrder.clear();
4017  return false;
4018  }
4019 
4020  return ShouldKeepOrder;
4021 }
4022 
4023 bool BoUpSLP::areAllUsersVectorized(Instruction *I,
4024  ArrayRef<Value *> VectorizedVals) const {
4025  return (I->hasOneUse() && is_contained(VectorizedVals, I)) ||
4026  llvm::all_of(I->users(), [this](User *U) {
4027  return ScalarToTreeEntry.count(U) > 0;
4028  });
4029 }
4030 
4031 static std::pair<InstructionCost, InstructionCost>
4035 
4036  // Calculate the cost of the scalar and vector calls.
4037  SmallVector<Type *, 4> VecTys;
4038  for (Use &Arg : CI->args())
4039  VecTys.push_back(
4040  FixedVectorType::get(Arg->getType(), VecTy->getNumElements()));
4041  FastMathFlags FMF;
4042  if (auto *FPCI = dyn_cast<FPMathOperator>(CI))
4043  FMF = FPCI->getFastMathFlags();
4045  IntrinsicCostAttributes CostAttrs(ID, VecTy, Arguments, VecTys, FMF,
4046  dyn_cast<IntrinsicInst>(CI));
4047  auto IntrinsicCost =
4048  TTI->getIntrinsicInstrCost(CostAttrs, TTI::TCK_RecipThroughput);
4049 
4050  auto Shape = VFShape::get(*CI, ElementCount::getFixed(static_cast<unsigned>(
4051  VecTy->getNumElements())),
4052  false /*HasGlobalPred*/);
4053  Function *VecFunc = VFDatabase(*CI).getVectorizedFunction(Shape);
4054  auto LibCost = IntrinsicCost;
4055  if (!CI->isNoBuiltin() && VecFunc) {
4056  // Calculate the cost of the vector library call.
4057  // If the corresponding vector call is cheaper, return its cost.
4058  LibCost = TTI->getCallInstrCost(nullptr, VecTy, VecTys,
4059  TTI::TCK_RecipThroughput);
4060  }
4061  return {IntrinsicCost, LibCost};
4062 }
4063 
4064 /// Compute the cost of creating a vector of type \p VecTy containing the
4065 /// extracted values from \p VL.
4066 static InstructionCost
4070  unsigned NumOfParts = TTI.getNumberOfParts(VecTy);
4071 
4072  if (ShuffleKind != TargetTransformInfo::SK_PermuteSingleSrc || !NumOfParts ||
4073  VecTy->getNumElements() < NumOfParts)
4074  return TTI.getShuffleCost(ShuffleKind, VecTy, Mask);
4075 
4076  bool AllConsecutive = true;
4077  unsigned EltsPerVector = VecTy->getNumElements() / NumOfParts;
4078  unsigned Idx = -1;
4079  InstructionCost Cost = 0;
4080 
4081  // Process extracts in blocks of EltsPerVector to check if the source vector
4082  // operand can be re-used directly. If not, add the cost of creating a shuffle
4083  // to extract the values into a vector register.
4084  for (auto *V : VL) {
4085  ++Idx;
4086 
4087  // Reached the start of a new vector registers.
4088  if (Idx % EltsPerVector == 0) {
4089  AllConsecutive = true;
4090  continue;
4091  }
4092 
4093  // Check all extracts for a vector register on the target directly
4094  // extract values in order.
4095  unsigned CurrentIdx = *getExtractIndex(cast<Instruction>(V));
4096  unsigned PrevIdx = *getExtractIndex(cast<Instruction>(VL[Idx - 1]));
4097  AllConsecutive &= PrevIdx + 1 == CurrentIdx &&
4098  CurrentIdx % EltsPerVector == Idx % EltsPerVector;
4099 
4100  if (AllConsecutive)
4101  continue;
4102 
4103  // Skip all indices, except for the last index per vector block.
4104  if ((Idx + 1) % EltsPerVector != 0 && Idx + 1 != VL.size())
4105  continue;
4106 
4107  // If we have a series of extracts which are not consecutive and hence
4108  // cannot re-use the source vector register directly, compute the shuffle
4109  // cost to extract the a vector with EltsPerVector elements.
4110  Cost += TTI.getShuffleCost(
4111  TargetTransformInfo::SK_PermuteSingleSrc,
4112  FixedVectorType::get(VecTy->getElementType(), EltsPerVector));
4113  }
4114  return Cost;
4115 }
4116 
4117 /// Build shuffle mask for shuffle graph entries and lists of main and alternate
4118 /// operations operands.
4119 static void
4121  ArrayRef<int> ReusesIndices,
4122  const function_ref<bool(Instruction *)> IsAltOp,
4124  SmallVectorImpl<Value *> *OpScalars = nullptr,
4125  SmallVectorImpl<Value *> *AltScalars = nullptr) {
4126  unsigned Sz = VL.size();
4127  Mask.assign(Sz, UndefMaskElem);
4128  SmallVector<int> OrderMask;
4129  if (!ReorderIndices.empty())
4130  inversePermutation(ReorderIndices, OrderMask);
4131  for (unsigned I = 0; I < Sz; ++I) {
4132  unsigned Idx = I;
4133  if (!ReorderIndices.empty())
4134  Idx = OrderMask[I];
4135  auto *OpInst = cast<Instruction>(VL[Idx]);
4136  if (IsAltOp(OpInst)) {
4137  Mask[I] = Sz + Idx;
4138  if (AltScalars)
4139  AltScalars->push_back(OpInst);
4140  } else {
4141  Mask[I] = Idx;
4142  if (OpScalars)
4143  OpScalars->push_back(OpInst);
4144  }
4145  }
4146  if (!ReusesIndices.empty()) {
4147  SmallVector<int> NewMask(ReusesIndices.size(), UndefMaskElem);
4148  transform(ReusesIndices, NewMask.begin(), [&Mask](int Idx) {
4149  return Idx != UndefMaskElem ? Mask[Idx] : UndefMaskElem;
4150  });
4151  Mask.swap(NewMask);
4152  }
4153 }
4154 
4155 InstructionCost BoUpSLP::getEntryCost(const TreeEntry *E,
4156  ArrayRef<Value *> VectorizedVals) {
4157  ArrayRef<Value*> VL = E->Scalars;
4158 
4159  Type *ScalarTy = VL[0]->getType();
4160  if (StoreInst *SI = dyn_cast<StoreInst>(VL[0]))
4161  ScalarTy = SI->getValueOperand()->getType();
4162  else if (CmpInst *CI = dyn_cast<CmpInst>(VL[0]))
4163  ScalarTy = CI->getOperand(0)->getType();
4164  else if (auto *IE = dyn_cast<InsertElementInst>(VL[0]))
4165  ScalarTy = IE->getOperand(1)->getType();
4166  auto *VecTy = FixedVectorType::get(ScalarTy, VL.size());
4167  TTI::TargetCostKind CostKind = TTI::TCK_RecipThroughput;
4168 
4169  // If we have computed a smaller type for the expression, update VecTy so
4170  // that the costs will be accurate.
4171  if (MinBWs.count(VL[0]))
4172  VecTy = FixedVectorType::get(
4173  IntegerType::get(F->getContext(), MinBWs[VL[0]].first), VL.size());
4174  auto *FinalVecTy = VecTy;
4175 
4176  unsigned ReuseShuffleNumbers = E->ReuseShuffleIndices.size();
4177  bool NeedToShuffleReuses = !E->ReuseShuffleIndices.empty();
4178  if (NeedToShuffleReuses)
4179  FinalVecTy =
4180  FixedVectorType::get(VecTy->getElementType(), ReuseShuffleNumbers);
4181  // FIXME: it tries to fix a problem with MSVC buildbots.
4182  TargetTransformInfo &TTIRef = *TTI;
4183  auto &&AdjustExtractsCost = [this, &TTIRef, CostKind, VL, VecTy,
4184  VectorizedVals](InstructionCost &Cost,
4185  bool IsGather) {
4186  DenseMap<Value *, int> ExtractVectorsTys;
4187  for (auto *V : VL) {
4188  // If all users of instruction are going to be vectorized and this
4189  // instruction itself is not going to be vectorized, consider this
4190  // instruction as dead and remove its cost from the final cost of the
4191  // vectorized tree.
4192  if (!areAllUsersVectorized(cast<Instruction>(V), VectorizedVals) ||
4193  (IsGather && ScalarToTreeEntry.count(V)))
4194  continue;
4195  auto *EE = cast<ExtractElementInst>(V);
4196  unsigned Idx = *getExtractIndex(EE);
4197  if (TTIRef.getNumberOfParts(VecTy) !=
4198  TTIRef.getNumberOfParts(EE->getVectorOperandType())) {
4199  auto It =
4200  ExtractVectorsTys.try_emplace(EE->getVectorOperand(), Idx).first;
4201  It->getSecond() = std::min<int>(It->second, Idx);
4202  }
4203  // Take credit for instruction that will become dead.
4204  if (EE->hasOneUse()) {
4205  Instruction *Ext = EE->user_back();
4206  if ((isa<SExtInst>(Ext) || isa<ZExtInst>(Ext)) &&
4207  all_of(Ext->users(),
4208  [](User *U) { return isa<GetElementPtrInst>(U); })) {
4209  // Use getExtractWithExtendCost() to calculate the cost of
4210  // extractelement/ext pair.
4211  Cost -=
4212  TTIRef.getExtractWithExtendCost(Ext->getOpcode(), Ext->getType(),
4213  EE->getVectorOperandType(), Idx);
4214  // Add back the cost of s|zext which is subtracted separately.
4215  Cost += TTIRef.getCastInstrCost(
4216  Ext->getOpcode(), Ext->getType(), EE->getType(),
4217  TTI::getCastContextHint(Ext), CostKind, Ext);
4218  continue;
4219  }
4220  }
4221  Cost -= TTIRef.getVectorInstrCost(Instruction::ExtractElement,
4222  EE->getVectorOperandType(), Idx);
4223  }
4224  // Add a cost for subvector extracts/inserts if required.
4225  for (const auto &Data : ExtractVectorsTys) {
4226  auto *EEVTy = cast<FixedVectorType>(Data.first->getType());
4227  unsigned NumElts = VecTy->getNumElements();
4228  if (TTIRef.getNumberOfParts(EEVTy) > TTIRef.getNumberOfParts(VecTy)) {
4229  unsigned Idx = (Data.second / NumElts) * NumElts;
4230  unsigned EENumElts = EEVTy->getNumElements();
4231  if (Idx + NumElts <= EENumElts) {
4232  Cost +=
4233  TTIRef.getShuffleCost(TargetTransformInfo::SK_ExtractSubvector,
4234  EEVTy, None, Idx, VecTy);
4235  } else {
4236  // Need to round up the subvector type vectorization factor to avoid a
4237  // crash in cost model functions. Make SubVT so that Idx + VF of SubVT
4238  // <= EENumElts.
4239  auto *SubVT =
4240  FixedVectorType::get(VecTy->getElementType(), EENumElts - Idx);
4241  Cost +=
4242  TTIRef.getShuffleCost(TargetTransformInfo::SK_ExtractSubvector,
4243  EEVTy, None, Idx, SubVT);
4244  }
4245  } else {
4246  Cost += TTIRef.getShuffleCost(TargetTransformInfo::SK_InsertSubvector,
4247  VecTy, None, 0, EEVTy);
4248  }
4249  }
4250  };
4251  if (E->State == TreeEntry::NeedToGather) {
4252  if (allConstant(VL))
4253  return 0;
4254  if (isa<InsertElementInst>(VL[0]))
4255  return InstructionCost::getInvalid();
4259  isGatherShuffledEntry(E, Mask, Entries);
4260  if (Shuffle.hasValue()) {
4261  InstructionCost GatherCost = 0;
4262  if (ShuffleVectorInst::isIdentityMask(Mask)) {
4263  // Perfect match in the graph, will reuse the previously vectorized
4264  // node. Cost is 0.
4265  LLVM_DEBUG(
4266  dbgs()
4267  << "SLP: perfect diamond match for gather bundle that starts with "
4268  << *VL.front() << ".\n");
4269  if (NeedToShuffleReuses)
4270  GatherCost =
4271  TTI->getShuffleCost(TargetTransformInfo::SK_PermuteSingleSrc,
4272  FinalVecTy, E->ReuseShuffleIndices);
4273  } else {
4274  LLVM_DEBUG(dbgs() << "SLP: shuffled " << Entries.size()
4275  << " entries for bundle that starts with "
4276  << *VL.front() << ".\n");
4277  // Detected that instead of gather we can emit a shuffle of single/two
4278  // previously vectorized nodes. Add the cost of the permutation rather
4279  // than gather.
4280  ::addMask(Mask, E->ReuseShuffleIndices);
4281  GatherCost = TTI->getShuffleCost(*Shuffle, FinalVecTy, Mask);
4282  }
4283  return GatherCost;
4284  }
4285  if (isSplat(VL)) {
4286  // Found the broadcasting of the single scalar, calculate the cost as the
4287  // broadcast.
4288  return TTI->getShuffleCost(TargetTransformInfo::SK_Broadcast, VecTy);
4289  }
4290  if (E->getOpcode() == Instruction::ExtractElement && allSameType(VL) &&
4291  allSameBlock(VL) &&
4292  !isa<ScalableVectorType>(
4293  cast<ExtractElementInst>(E->getMainOp())->getVectorOperandType())) {
4294  // Check that gather of extractelements can be represented as just a
4295  // shuffle of a single/two vectors the scalars are extracted from.
4298  isShuffle(VL, Mask);
4299  if (ShuffleKind.hasValue()) {
4300  // Found the bunch of extractelement instructions that must be gathered
4301  // into a vector and can be represented as a permutation elements in a
4302  // single input vector or of 2 input vectors.
4303  InstructionCost Cost =
4304  computeExtractCost(VL, VecTy, *ShuffleKind, Mask, *TTI);
4305  AdjustExtractsCost(Cost, /*IsGather=*/true);
4306  if (NeedToShuffleReuses)
4307  Cost += TTI->getShuffleCost(TargetTransformInfo::SK_PermuteSingleSrc,
4308  FinalVecTy, E->ReuseShuffleIndices);
4309  return Cost;
4310  }
4311  }
4312  InstructionCost ReuseShuffleCost = 0;
4313  if (NeedToShuffleReuses)
4314  ReuseShuffleCost = TTI->getShuffleCost(
4315  TTI::SK_PermuteSingleSrc, FinalVecTy, E->ReuseShuffleIndices);
4316  // Improve gather cost for gather of loads, if we can group some of the
4317  // loads into vector loads.
4318  if (VL.size() > 2 && E->getOpcode() == Instruction::Load &&
4319  !E->isAltShuffle()) {
4320  BoUpSLP::ValueSet VectorizedLoads;
4321  unsigned StartIdx = 0;
4322  unsigned VF = VL.size() / 2;
4323  unsigned VectorizedCnt = 0;
4324  unsigned ScatterVectorizeCnt = 0;
4325  const unsigned Sz = DL->getTypeSizeInBits(E->getMainOp()->getType());
4326  for (unsigned MinVF = getMinVF(2 * Sz); VF >= MinVF; VF /= 2) {
4327  for (unsigned Cnt = StartIdx, End = VL.size(); Cnt + VF <= End;
4328  Cnt += VF) {
4329  ArrayRef<Value *> Slice = VL.slice(Cnt, VF);
4330  if (!VectorizedLoads.count(Slice.front()) &&
4331  !VectorizedLoads.count(Slice.back()) && allSameBlock(Slice)) {
4332  SmallVector<Value *> PointerOps;
4333  OrdersType CurrentOrder;
4334  LoadsState LS = canVectorizeLoads(Slice, Slice.front(), *TTI, *DL,
4335  *SE, CurrentOrder, PointerOps);
4336  switch (LS) {
4337  case LoadsState::Vectorize:
4338  case LoadsState::ScatterVectorize:
4339  // Mark the vectorized loads so that we don't vectorize them
4340  // again.
4341  if (LS == LoadsState::Vectorize)
4342  ++VectorizedCnt;
4343  else
4344  ++ScatterVectorizeCnt;
4345  VectorizedLoads.insert(Slice.begin(), Slice.end());
4346  // If we vectorized initial block, no need to try to vectorize it
4347  // again.
4348  if (Cnt == StartIdx)
4349  StartIdx += VF;
4350  break;
4351  case LoadsState::Gather:
4352  break;
4353  }
4354  }
4355  }
4356  // Check if the whole array was vectorized already - exit.
4357  if (StartIdx >= VL.size())
4358  break;
4359  // Found vectorizable parts - exit.
4360  if (!VectorizedLoads.empty())
4361  break;
4362  }
4363  if (!VectorizedLoads.empty()) {
4364  InstructionCost GatherCost = 0;
4365  // Get the cost for gathered loads.
4366  for (unsigned I = 0, End = VL.size(); I < End; I += VF) {
4367  if (VectorizedLoads.contains(VL[I]))
4368  continue;
4369  GatherCost += getGatherCost(VL.slice(I, VF));
4370  }
4371  // The cost for vectorized loads.
4372  InstructionCost ScalarsCost = 0;
4373  for (Value *V : VectorizedLoads) {
4374  auto *LI = cast<LoadInst>(V);
4375  ScalarsCost += TTI->getMemoryOpCost(
4376  Instruction::Load, LI->getType(), LI->getAlign(),
4377  LI->getPointerAddressSpace(), CostKind, LI);
4378  }
4379  auto *LI = cast<LoadInst>(E->getMainOp());
4380  auto *LoadTy = FixedVectorType::get(LI->getType(), VF);
4381  Align Alignment = LI->getAlign();
4382  GatherCost +=
4383  VectorizedCnt *
4384  TTI->getMemoryOpCost(Instruction::Load, LoadTy, Alignment,
4385  LI->getPointerAddressSpace(), CostKind, LI);
4386  GatherCost += ScatterVectorizeCnt *
4388  Instruction::Load, LoadTy, LI->getPointerOperand(),
4389  /*VariableMask=*/false, Alignment, CostKind, LI);
4390  // Add the cost for the subvectors shuffling.
4391  GatherCost += ((VL.size() - VF) / VF) *
4392  TTI->getShuffleCost(TTI::SK_Select, VecTy);
4393  return ReuseShuffleCost + GatherCost - ScalarsCost;
4394  }
4395  }
4396  return ReuseShuffleCost + getGatherCost(VL);
4397  }
4398  InstructionCost CommonCost = 0;
4400  if (!E->ReorderIndices.empty()) {
4401  SmallVector<int> NewMask;
4402  if (E->getOpcode() == Instruction::Store) {
4403  // For stores the order is actually a mask.
4404  NewMask.resize(E->ReorderIndices.size());
4405  copy(E->ReorderIndices, NewMask.begin());
4406  } else {
4407  inversePermutation(E->ReorderIndices, NewMask);
4408  }
4409  ::addMask(Mask, NewMask);
4410  }
4411  if (NeedToShuffleReuses)
4412  ::addMask(Mask, E->ReuseShuffleIndices);
4413  if (!Mask.empty() && !ShuffleVectorInst::isIdentityMask(Mask))
4414  CommonCost =
4415  TTI->getShuffleCost(TTI::SK_PermuteSingleSrc, FinalVecTy, Mask);
4416  assert((E->State == TreeEntry::Vectorize ||
4417  E->State == TreeEntry::ScatterVectorize) &&
4418  "Unhandled state");
4419  assert(E->getOpcode() && allSameType(VL) && allSameBlock(VL) && "Invalid VL");
4420  Instruction *VL0 = E->getMainOp();
4421  unsigned ShuffleOrOp =
4422  E->isAltShuffle() ? (unsigned)Instruction::ShuffleVector : E->getOpcode();
4423  switch (ShuffleOrOp) {
4424  case Instruction::PHI:
4425  return 0;
4426 
4427  case Instruction::ExtractValue:
4428  case Instruction::ExtractElement: {
4429  // The common cost of removal ExtractElement/ExtractValue instructions +
4430  // the cost of shuffles, if required to resuffle the original vector.
4431  if (NeedToShuffleReuses) {
4432  unsigned Idx = 0;
4433  for (unsigned I : E->ReuseShuffleIndices) {
4434  if (ShuffleOrOp == Instruction::ExtractElement) {
4435  auto *EE = cast<ExtractElementInst>(VL[I]);
4436  CommonCost -= TTI->getVectorInstrCost(Instruction::ExtractElement,
4437  EE->getVectorOperandType(),
4438  *getExtractIndex(EE));
4439  } else {
4440  CommonCost -= TTI->getVectorInstrCost(Instruction::ExtractElement,
4441  VecTy, Idx);
4442  ++Idx;
4443  }
4444  }
4445  Idx = ReuseShuffleNumbers;
4446  for (Value *V : VL) {
4447  if (ShuffleOrOp == Instruction::ExtractElement) {
4448  auto *EE = cast<ExtractElementInst>(V);
4449  CommonCost += TTI->getVectorInstrCost(Instruction::ExtractElement,
4450  EE->getVectorOperandType(),
4451  *getExtractIndex(EE));
4452  } else {
4453  --Idx;
4454  CommonCost += TTI->getVectorInstrCost(Instruction::ExtractElement,
4455  VecTy, Idx);
4456  }
4457  }
4458  }
4459  if (ShuffleOrOp == Instruction::ExtractValue) {
4460  for (unsigned I = 0, E = VL.size(); I < E; ++I) {
4461  auto *EI = cast<Instruction>(VL[I]);
4462  // Take credit for instruction that will become dead.
4463  if (EI->hasOneUse()) {
4464  Instruction *Ext = EI->user_back();
4465  if ((isa<SExtInst>(Ext) || isa<ZExtInst>(Ext)) &&
4466  all_of(Ext->users(),
4467  [](User *U) { return isa<GetElementPtrInst>(U); })) {
4468  // Use getExtractWithExtendCost() to calculate the cost of
4469  // extractelement/ext pair.
4470  CommonCost -= TTI->getExtractWithExtendCost(
4471  Ext->getOpcode(), Ext->getType(), VecTy, I);
4472  // Add back the cost of s|zext which is subtracted separately.
4473  CommonCost += TTI->getCastInstrCost(
4474  Ext->getOpcode(), Ext->getType(), EI->getType(),
4475  TTI::getCastContextHint(Ext), CostKind, Ext);
4476  continue;
4477  }
4478  }
4479  CommonCost -=
4480  TTI->getVectorInstrCost(Instruction::ExtractElement, VecTy, I);
4481  }
4482  } else {
4483  AdjustExtractsCost(CommonCost, /*IsGather=*/false);
4484  }
4485  return CommonCost;
4486  }
4487  case Instruction::InsertElement: {
4488  assert(E->ReuseShuffleIndices.empty() &&
4489  "Unique insertelements only are expected.");
4490  auto *SrcVecTy = cast<FixedVectorType>(VL0->getType());
4491 
4492  unsigned const NumElts = SrcVecTy->getNumElements();
4493  unsigned const NumScalars = VL.size();
4494  APInt DemandedElts = APInt::getZero(NumElts);
4495  // TODO: Add support for Instruction::InsertValue.
4497  if (!E->ReorderIndices.empty()) {
4498  inversePermutation(E->ReorderIndices, Mask);
4499  Mask.append(NumElts - NumScalars, UndefMaskElem);
4500  } else {
4501  Mask.assign(NumElts, UndefMaskElem);
4502  std::iota(Mask.begin(), std::next(Mask.begin(), NumScalars), 0);
4503  }
4504  unsigned Offset = *getInsertIndex(VL0, 0);
4505  bool IsIdentity = true;
4506  SmallVector<int> PrevMask(NumElts, UndefMaskElem);
4507  Mask.swap(PrevMask);
4508  for (unsigned I = 0; I < NumScalars; ++I) {
4509  Optional<int> InsertIdx = getInsertIndex(VL[PrevMask[I]], 0);
4510  if (!InsertIdx || *InsertIdx == UndefMaskElem)
4511  continue;
4512  DemandedElts.setBit(*InsertIdx);
4513  IsIdentity &= *InsertIdx - Offset == I;
4514  Mask[*InsertIdx - Offset] = I;
4515  }
4516  assert(Offset < NumElts && "Failed to find vector index offset");
4517 
4518  InstructionCost Cost = 0;
4519  Cost -= TTI->getScalarizationOverhead(SrcVecTy, DemandedElts,
4520  /*Insert*/ true, /*Extract*/ false);
4521 
4522  if (IsIdentity && NumElts != NumScalars && Offset % NumScalars != 0) {
4523  // FIXME: Replace with SK_InsertSubvector once it is properly supported.
4524  unsigned Sz = PowerOf2Ceil(Offset + NumScalars);
4525  Cost += TTI->getShuffleCost(
4526  TargetTransformInfo::SK_PermuteSingleSrc,
4527  FixedVectorType::get(SrcVecTy->getElementType(), Sz));
4528  } else if (!IsIdentity) {
4529  auto *FirstInsert =
4530  cast<Instruction>(*find_if(E->Scalars, [E](Value *V) {
4531  return !is_contained(E->Scalars,
4532  cast<Instruction>(V)->getOperand(0));
4533  }));
4534  if (isa<UndefValue>(FirstInsert->getOperand(0))) {
4535  Cost += TTI->getShuffleCost(TTI::SK_PermuteSingleSrc, SrcVecTy, Mask);
4536  } else {
4537  SmallVector<int> InsertMask(NumElts);
4538  std::iota(InsertMask.begin(), InsertMask.end(), 0);
4539  for (unsigned I = 0; I < NumElts; I++) {
4540  if (Mask[I] != UndefMaskElem)
4541  InsertMask[Offset + I] = NumElts + I;
4542  }
4543  Cost +=
4544  TTI->getShuffleCost(TTI::SK_PermuteTwoSrc, SrcVecTy, InsertMask);
4545  }
4546  }
4547 
4548  return Cost;
4549  }
4550  case Instruction::ZExt:
4551  case Instruction::SExt:
4552  case Instruction::FPToUI:
4553  case Instruction::FPToSI:
4554  case Instruction::FPExt:
4555  case Instruction::PtrToInt:
4556  case Instruction::IntToPtr:
4557  case Instruction::SIToFP:
4558  case Instruction::UIToFP:
4559  case Instruction::Trunc:
4560  case Instruction::FPTrunc:
4561  case Instruction::BitCast: {
4562  Type *SrcTy = VL0->getOperand(0)->getType();
4563  InstructionCost ScalarEltCost =
4564  TTI->getCastInstrCost(E->getOpcode(), ScalarTy, SrcTy,
4565  TTI::getCastContextHint(VL0), CostKind, VL0);
4566  if (NeedToShuffleReuses) {
4567  CommonCost -= (ReuseShuffleNumbers - VL.size()) * ScalarEltCost;
4568  }
4569 
4570  // Calculate the cost of this instruction.
4571  InstructionCost ScalarCost = VL.size() * ScalarEltCost;
4572 
4573  auto *SrcVecTy = FixedVectorType::get(SrcTy, VL.size());
4574  InstructionCost VecCost = 0;
4575  // Check if the values are candidates to demote.
4576  if (!MinBWs.count(VL0) || VecTy != SrcVecTy) {
4577  VecCost = CommonCost + TTI->getCastInstrCost(
4578  E->getOpcode(), VecTy, SrcVecTy,
4579  TTI::getCastContextHint(VL0), CostKind, VL0);
4580  }
4581  LLVM_DEBUG(dumpTreeCosts(E, CommonCost, VecCost, ScalarCost));
4582  return VecCost - ScalarCost;
4583  }
4584  case Instruction::FCmp:
4585  case Instruction::ICmp:
4586  case Instruction::Select: {
4587  // Calculate the cost of this instruction.
4588  InstructionCost ScalarEltCost =
4589  TTI->getCmpSelInstrCost(E->getOpcode(), ScalarTy, Builder.getInt1Ty(),
4590  CmpInst::BAD_ICMP_PREDICATE, CostKind, VL0);
4591  if (NeedToShuffleReuses) {
4592  CommonCost -= (ReuseShuffleNumbers - VL.size()) * ScalarEltCost;
4593  }
4594  auto *MaskTy = FixedVectorType::get(Builder.getInt1Ty(), VL.size());
4595  InstructionCost ScalarCost = VecTy->getNumElements() * ScalarEltCost;
4596 
4597  // Check if all entries in VL are either compares or selects with compares
4598  // as condition that have the same predicates.
4599  CmpInst::Predicate VecPred = CmpInst::BAD_ICMP_PREDICATE;
4600  bool First = true;
4601  for (auto *V : VL) {
4602  CmpInst::Predicate CurrentPred;
4603  auto MatchCmp = m_Cmp(CurrentPred, m_Value(), m_Value());
4604  if ((!match(V, m_Select(MatchCmp, m_Value(), m_Value())) &&
4605  !match(V, MatchCmp)) ||
4606  (!First && VecPred != CurrentPred)) {
4607  VecPred = CmpInst::BAD_ICMP_PREDICATE;
4608  break;
4609  }
4610  First = false;
4611  VecPred = CurrentPred;
4612  }
4613 
4615  E->getOpcode(), VecTy, MaskTy, VecPred, CostKind, VL0);
4616  // Check if it is possible and profitable to use min/max for selects in
4617  // VL.
4618  //
4619  auto IntrinsicAndUse = canConvertToMinOrMaxIntrinsic(VL);
4620  if (IntrinsicAndUse.first != Intrinsic::not_intrinsic) {
4621  IntrinsicCostAttributes CostAttrs(IntrinsicAndUse.first, VecTy,
4622  {VecTy, VecTy});
4623  InstructionCost IntrinsicCost =
4624  TTI->getIntrinsicInstrCost(CostAttrs, CostKind);
4625  // If the selects are the only uses of the compares, they will be dead
4626  // and we can adjust the cost by removing their cost.
4627  if (IntrinsicAndUse.second)
4628  IntrinsicCost -=
4629  TTI->getCmpSelInstrCost(Instruction::ICmp, VecTy, MaskTy,
4630  CmpInst::BAD_ICMP_PREDICATE, CostKind);
4631  VecCost = std::min(VecCost, IntrinsicCost);
4632  }
4633  LLVM_DEBUG(dumpTreeCosts(E, CommonCost, VecCost, ScalarCost));
4634  return CommonCost + VecCost - ScalarCost;
4635  }
4636  case Instruction::FNeg:
4637  case Instruction::Add:
4638  case Instruction::FAdd:
4639  case Instruction::Sub:
4640  case Instruction::FSub:
4641  case Instruction::Mul:
4642  case Instruction::FMul:
4643  case Instruction::UDiv:
4644  case Instruction::SDiv:
4645  case Instruction::FDiv:
4646  case Instruction::URem:
4647  case Instruction::SRem:
4648  case Instruction::FRem:
4649  case Instruction::Shl:
4650  case Instruction::LShr:
4651  case Instruction::AShr:
4652  case Instruction::And:
4653  case Instruction::Or:
4654  case Instruction::Xor: {
4655  // Certain instructions can be cheaper to vectorize if they have a
4656  // constant second vector operand.
4658  TargetTransformInfo::OK_AnyValue;
4660  TargetTransformInfo::OK_UniformConstantValue;
4662  TargetTransformInfo::OP_None;
4664  TargetTransformInfo::OP_PowerOf2;
4665 
4666  // If all operands are exactly the same ConstantInt then set the
4667  // operand kind to OK_UniformConstantValue.
4668  // If instead not all operands are constants, then set the operand kind
4669  // to OK_AnyValue. If all operands are constants but not the same,
4670  // then set the operand kind to OK_NonUniformConstantValue.
4671  ConstantInt *CInt0 = nullptr;
4672  for (unsigned i = 0, e = VL.size(); i < e; ++i) {
4673  const Instruction *I = cast<Instruction>(VL[i]);
4674  unsigned OpIdx = isa<BinaryOperator>(I) ? 1 : 0;
4675  ConstantInt *CInt = dyn_cast<ConstantInt>(I->getOperand(OpIdx));
4676  if (!CInt) {
4677  Op2VK = TargetTransformInfo::OK_AnyValue;
4678  Op2VP = TargetTransformInfo::OP_None;
4679  break;
4680  }
4681  if (Op2VP == TargetTransformInfo::OP_PowerOf2 &&
4682  !CInt->getValue().isPowerOf2())
4683  Op2VP = TargetTransformInfo::OP_None;
4684  if (i == 0) {
4685  CInt0 = CInt;
4686  continue;
4687  }
4688  if (CInt0 != CInt)
4689  Op2VK = TargetTransformInfo::OK_NonUniformConstantValue;
4690  }
4691 
4693  InstructionCost ScalarEltCost =
4694  TTI->getArithmeticInstrCost(E->getOpcode(), ScalarTy, CostKind, Op1VK,
4695  Op2VK, Op1VP, Op2VP, Operands, VL0);
4696  if (NeedToShuffleReuses) {
4697  CommonCost -= (ReuseShuffleNumbers - VL.size()) * ScalarEltCost;
4698  }
4699  InstructionCost ScalarCost = VecTy->getNumElements() * ScalarEltCost;
4700  InstructionCost VecCost =
4701  TTI->getArithmeticInstrCost(E->getOpcode(), VecTy, CostKind, Op1VK,
4702  Op2VK, Op1VP, Op2VP, Operands, VL0);
4703  LLVM_DEBUG(dumpTreeCosts(E, CommonCost, VecCost, ScalarCost));
4704  return CommonCost + VecCost - ScalarCost;
4705  }
4706  case Instruction::GetElementPtr: {
4708  TargetTransformInfo::OK_AnyValue;
4710  TargetTransformInfo::OK_UniformConstantValue;
4711 
4712  InstructionCost ScalarEltCost = TTI->getArithmeticInstrCost(
4713  Instruction::Add, ScalarTy, CostKind, Op1VK, Op2VK);
4714  if (NeedToShuffleReuses) {
4715  CommonCost -= (ReuseShuffleNumbers - VL.size()) * ScalarEltCost;
4716  }
4717  InstructionCost ScalarCost = VecTy->getNumElements() * ScalarEltCost;
4719  Instruction::Add, VecTy, CostKind, Op1VK, Op2VK);
4720  LLVM_DEBUG(dumpTreeCosts(E, CommonCost, VecCost, ScalarCost));
4721  return CommonCost + VecCost - ScalarCost;
4722  }
4723  case Instruction::Load: {
4724  // Cost of wide load - cost of scalar loads.
4725  Align Alignment = cast<LoadInst>(VL0)->getAlign();
4726  InstructionCost ScalarEltCost = TTI->getMemoryOpCost(
4727  Instruction::Load, ScalarTy, Alignment, 0, CostKind, VL0);
4728  if (NeedToShuffleReuses) {
4729  CommonCost -= (ReuseShuffleNumbers - VL.size()) * ScalarEltCost;
4730  }
4731  InstructionCost ScalarLdCost = VecTy->getNumElements() * ScalarEltCost;
4732  InstructionCost VecLdCost;
4733  if (E->State == TreeEntry::Vectorize) {
4734  VecLdCost = TTI->getMemoryOpCost(Instruction::Load, VecTy, Alignment, 0,
4735  CostKind, VL0);
4736  } else {
4737  assert(E->State == TreeEntry::ScatterVectorize && "Unknown EntryState");
4738  Align CommonAlignment = Alignment;
4739  for (Value *V : VL)
4740  CommonAlignment =
4741  commonAlignment(CommonAlignment, cast<LoadInst>(V)->getAlign());
4742  VecLdCost = TTI->getGatherScatterOpCost(
4743  Instruction::Load, VecTy, cast<LoadInst>(VL0)->getPointerOperand(),
4744  /*VariableMask=*/false, CommonAlignment, CostKind, VL0);
4745  }
4746  LLVM_DEBUG(dumpTreeCosts(E, CommonCost, VecLdCost, ScalarLdCost));
4747  return CommonCost + VecLdCost - ScalarLdCost;
4748  }
4749  case Instruction::Store: {
4750  // We know that we can merge the stores. Calculate the cost.
4751  bool IsReorder = !E->ReorderIndices.empty();
4752  auto *SI =
4753  cast<StoreInst>(IsReorder ? VL[E->ReorderIndices.front()] : VL0);
4754  Align Alignment = SI->getAlign();
4755  InstructionCost ScalarEltCost = TTI->getMemoryOpCost(
4756  Instruction::Store, ScalarTy, Alignment, 0, CostKind, VL0);
4757  InstructionCost ScalarStCost = VecTy->getNumElements() * ScalarEltCost;
4758  InstructionCost VecStCost = TTI->getMemoryOpCost(
4759  Instruction::Store, VecTy, Alignment, 0, CostKind, VL0);
4760  LLVM_DEBUG(dumpTreeCosts(E, CommonCost, VecStCost, ScalarStCost));
4761  return CommonCost + VecStCost - ScalarStCost;
4762  }
4763  case Instruction::Call: {
4764  CallInst *CI = cast<CallInst>(VL0);
4766 
4767  // Calculate the cost of the scalar and vector calls.
4768  IntrinsicCostAttributes CostAttrs(ID, *CI, 1);
4769  InstructionCost ScalarEltCost =
4770  TTI->getIntrinsicInstrCost(CostAttrs, CostKind);
4771  if (NeedToShuffleReuses) {
4772  CommonCost -= (ReuseShuffleNumbers - VL.size()) * ScalarEltCost;
4773  }
4774  InstructionCost ScalarCallCost = VecTy->getNumElements() * ScalarEltCost;
4775 
4776  auto VecCallCosts = getVectorCallCosts(CI, VecTy, TTI, TLI);
4777  InstructionCost VecCallCost =
4778  std::min(VecCallCosts.first, VecCallCosts.second);
4779 
4780  LLVM_DEBUG(dbgs() << "SLP: Call cost " << VecCallCost - ScalarCallCost
4781  << " (" << VecCallCost << "-" << ScalarCallCost << ")"
4782  << " for " << *CI << "\n");
4783 
4784  return CommonCost + VecCallCost - ScalarCallCost;
4785  }
4786  case Instruction::ShuffleVector: {
4787  assert(E->isAltShuffle() &&
4788  ((Instruction::isBinaryOp(E->getOpcode()) &&
4789  Instruction::isBinaryOp(E->getAltOpcode())) ||
4790  (Instruction::isCast(E->getOpcode()) &&
4791  Instruction::isCast(E->getAltOpcode()))) &&
4792  "Invalid Shuffle Vector Operand");
4793  InstructionCost ScalarCost = 0;
4794  if (NeedToShuffleReuses) {
4795  for (unsigned Idx : E->ReuseShuffleIndices) {
4796  Instruction *I = cast<Instruction>(VL[Idx]);
4797  CommonCost -= TTI->getInstructionCost(I, CostKind);
4798  }
4799  for (Value *V : VL) {
4800  Instruction *I = cast<Instruction>(V);
4801  CommonCost += TTI->getInstructionCost(I, CostKind);
4802  }
4803  }
4804  for (Value *V : VL) {
4805  Instruction *I = cast<Instruction>(V);
4806  assert(E->isOpcodeOrAlt(I) && "Unexpected main/alternate opcode");
4807  ScalarCost += TTI->getInstructionCost(I, CostKind);
4808  }
4809  // VecCost is equal to sum of the cost of creating 2 vectors
4810  // and the cost of creating shuffle.
4811  InstructionCost VecCost = 0;
4812  if (Instruction::isBinaryOp(E->getOpcode())) {
4813  VecCost = TTI->getArithmeticInstrCost(E->getOpcode(), VecTy, CostKind);
4814  VecCost += TTI->getArithmeticInstrCost(E->getAltOpcode(), VecTy,
4815  CostKind);
4816  } else {
4817  Type *Src0SclTy = E->getMainOp()->getOperand(0)->getType();
4818  Type *Src1SclTy = E->getAltOp()->getOperand(0)->getType();
4819  auto *Src0Ty = FixedVectorType::get(Src0SclTy, VL.size());
4820  auto *Src1Ty = FixedVectorType::get(Src1SclTy, VL.size());
4821  VecCost = TTI->getCastInstrCost(E->getOpcode(), VecTy, Src0Ty,
4823  VecCost += TTI->getCastInstrCost(E->getAltOpcode(), VecTy, Src1Ty,
4825  }
4826 
4829  E->Scalars, E->ReorderIndices, E->ReuseShuffleIndices,
4830  [E](Instruction *I) {
4831  assert(E->isOpcodeOrAlt(I) && "Unexpected main/alternate opcode");
4832  return I->getOpcode() == E->getAltOpcode();
4833  },
4834  Mask);
4835  CommonCost =
4836  TTI->getShuffleCost(TargetTransformInfo::SK_Select, FinalVecTy, Mask);
4837  LLVM_DEBUG(dumpTreeCosts(E, CommonCost, VecCost, ScalarCost));
4838  return CommonCost + VecCost - ScalarCost;
4839  }
4840  default:
4841  llvm_unreachable("Unknown instruction");
4842  }
4843 }
4844 
4845 bool BoUpSLP::isFullyVectorizableTinyTree() const {
4846  LLVM_DEBUG(dbgs() << "SLP: Check whether the tree with height "
4847  << VectorizableTree.size() << " is fully vectorizable .\n");
4848 
4849  // We only handle trees of heights 1 and 2.
4850  if (VectorizableTree.size() == 1 &&
4851  VectorizableTree[0]->State == TreeEntry::Vectorize)
4852  return true;
4853 
4854  if (VectorizableTree.size() != 2)
4855  return false;
4856 
4857  // Handle splat and all-constants stores. Also try to vectorize tiny trees
4858  // with the second gather nodes if they have less scalar operands rather than
4859  // the initial tree element (may be profitable to shuffle the second gather)
4860  // or they are extractelements, which form shuffle.
4862  if (VectorizableTree[0]->State == TreeEntry::Vectorize &&
4863  (allConstant(VectorizableTree[1]->Scalars) ||
4864  isSplat(VectorizableTree[1]->Scalars) ||
4865  (VectorizableTree[1]->State == TreeEntry::NeedToGather &&
4866  VectorizableTree[1]->Scalars.size() <
4867  VectorizableTree[0]->Scalars.size()) ||
4868  (VectorizableTree[1]->State == TreeEntry::NeedToGather &&
4869  VectorizableTree[1]->getOpcode() == Instruction::ExtractElement &&
4870  isShuffle(VectorizableTree[1]->Scalars, Mask))))
4871  return true;
4872 
4873  // Gathering cost would be too much for tiny trees.
4874  if (VectorizableTree[0]->State == TreeEntry::NeedToGather ||
4875  VectorizableTree[1]->State == TreeEntry::NeedToGather)
4876  return false;
4877 
4878  return true;
4879 }
4880 
4881 static bool isLoadCombineCandidateImpl(Value *Root, unsigned NumElts,
4883  bool MustMatchOrInst) {
4884  // Look past the root to find a source value. Arbitrarily follow the
4885  // path through operand 0 of any 'or'. Also, peek through optional
4886  // shift-left-by-multiple-of-8-bits.
4887  Value *ZextLoad = Root;
4888  const APInt *ShAmtC;
4889  bool FoundOr = false;
4890  while (!isa<ConstantExpr>(ZextLoad) &&
4891  (match(ZextLoad, m_Or(m_Value(), m_Value())) ||
4892  (match(ZextLoad, m_Shl(m_Value(), m_APInt(ShAmtC))) &&
4893  ShAmtC->urem(8) == 0))) {
4894  auto *BinOp = cast<BinaryOperator>(ZextLoad);
4895  ZextLoad = BinOp->getOperand(0);
4896  if (BinOp->getOpcode() == Instruction::Or)
4897  FoundOr = true;
4898  }
4899  // Check if the input is an extended load of the required or/shift expression.
4900  Value *LoadPtr;
4901  if ((MustMatchOrInst && !FoundOr) || ZextLoad == Root ||
4902  !match(ZextLoad, m_ZExt(m_Load(m_Value(LoadPtr)))))
4903  return false;
4904 
4905  // Require that the total load bit width is a legal integer type.
4906  // For example, <8 x i8> --> i64 is a legal integer on a 64-bit target.
4907  // But <16 x i8> --> i128 is not, so the backend probably can't reduce it.
4908  Type *SrcTy = LoadPtr->getType()->getPointerElementType();
4909  unsigned LoadBitWidth = SrcTy->getIntegerBitWidth() * NumElts;
4910  if (!TTI->isTypeLegal(IntegerType::get(Root->getContext(), LoadBitWidth)))
4911  return false;
4912 
4913  // Everything matched - assume that we can fold the whole sequence using
4914  // load combining.
4915  LLVM_DEBUG(dbgs() << "SLP: Assume load combining for tree starting at "
4916  << *(cast<Instruction>(Root)) << "\n");
4917 
4918  return true;
4919 }
4920 
4921 bool BoUpSLP::isLoadCombineReductionCandidate(RecurKind RdxKind) const {
4922  if (RdxKind != RecurKind::Or)
4923  return false;
4924 
4925  unsigned NumElts = VectorizableTree[0]->Scalars.size();
4926  Value *FirstReduced = VectorizableTree[0]->Scalars[0];
4927  return isLoadCombineCandidateImpl(FirstReduced, NumElts, TTI,
4928  /* MatchOr */ false);
4929 }
4930 
4932  // Peek through a final sequence of stores and check if all operations are
4933  // likely to be load-combined.
4934  unsigned NumElts = VectorizableTree[0]->Scalars.size();
4935  for (Value *Scalar : VectorizableTree[0]->Scalars) {
4936  Value *X;
4937  if (!match(Scalar, m_Store(m_Value(X), m_Value())) ||
4938  !isLoadCombineCandidateImpl(X, NumElts, TTI, /* MatchOr */ true))
4939  return false;
4940  }
4941  return true;
4942 }
4943 
4944 bool BoUpSLP::isTreeTinyAndNotFullyVectorizable() const {
4945  // No need to vectorize inserts of gathered values.
4946  if (VectorizableTree.size() == 2 &&
4947  isa<InsertElementInst>(VectorizableTree[0]->Scalars[0]) &&
4948  VectorizableTree[1]->State == TreeEntry::NeedToGather)
4949  return true;
4950 
4951  // We can vectorize the tree if its size is greater than or equal to the
4952  // minimum size specified by the MinTreeSize command line option.
4953  if (VectorizableTree.size() >= MinTreeSize)
4954  return false;
4955 
4956  // If we have a tiny tree (a tree whose size is less than MinTreeSize), we
4957  // can vectorize it if we can prove it fully vectorizable.
4958  if (isFullyVectorizableTinyTree())
4959  return false;
4960 
4961  assert(VectorizableTree.empty()
4962  ? ExternalUses.empty()
4963  : true && "We shouldn't have any external users");
4964 
4965  // Otherwise, we can't vectorize the tree. It is both tiny and not fully
4966  // vectorizable.
4967  return true;
4968 }
4969 
4970 InstructionCost BoUpSLP::getSpillCost() const {
4971  // Walk from the bottom of the tree to the top, tracking which values are
4972  // live. When we see a call instruction that is not part of our tree,
4973  // query TTI to see if there is a cost to keeping values live over it
4974  // (for example, if spills and fills are required).
4975  unsigned BundleWidth = VectorizableTree.front()->Scalars.size();
4976  InstructionCost Cost = 0;
4977 
4978  SmallPtrSet<Instruction*, 4> LiveValues;
4979  Instruction *PrevInst = nullptr;
4980 
4981  // The entries in VectorizableTree are not necessarily ordered by their
4982  // position in basic blocks. Collect them and order them by dominance so later
4983  // instructions are guaranteed to be visited first. For instructions in
4984  // different basic blocks, we only scan to the beginning of the block, so
4985  // their order does not matter, as long as all instructions in a basic block
4986  // are grouped together. Using dominance ensures a deterministic order.
4987  SmallVector<Instruction *, 16> OrderedScalars;
4988  for (const auto &TEPtr : VectorizableTree) {
4989  Instruction *Inst = dyn_cast<Instruction>(TEPtr->Scalars[0]);
4990  if (!Inst)
4991  continue;
4992  OrderedScalars.push_back(Inst);
4993  }
4994  llvm::sort(OrderedScalars, [&](Instruction *A, Instruction *B) {
4995  auto *NodeA = DT->getNode(A->getParent());
4996  auto *NodeB = DT->getNode(B->getParent());
4997  assert(NodeA && "Should only process reachable instructions");
4998  assert(NodeB && "Should only process reachable instructions");
4999  assert((NodeA == NodeB) == (NodeA->getDFSNumIn() == NodeB->getDFSNumIn()) &&
5000  "Different nodes should have different DFS numbers");
5001  if (NodeA != NodeB)
5002  return NodeA->getDFSNumIn() < NodeB->getDFSNumIn();
5003  return B->comesBefore(A);
5004  });
5005 
5006  for (Instruction *Inst : OrderedScalars) {
5007  if (!PrevInst) {
5008  PrevInst = Inst;
5009  continue;
5010  }
5011 
5012  // Update LiveValues.
5013  LiveValues.erase(PrevInst);
5014  for (auto &J : PrevInst->operands()) {
5015  if (isa<Instruction>(&*J) && getTreeEntry(&*J))
5016  LiveValues.insert(cast<Instruction>(&*J));
5017  }
5018 
5019  LLVM_DEBUG({
5020  dbgs() << "SLP: #LV: " << LiveValues.size();
5021  for (auto *X : LiveValues)
5022  dbgs() << " " << X->getName();
5023  dbgs() << ", Looking at ";
5024  Inst->dump();
5025  });
5026 
5027  // Now find the sequence of instructions between PrevInst and Inst.
5028  unsigned NumCalls = 0;
5029  BasicBlock::reverse_iterator InstIt = ++Inst->getIterator().getReverse(),
5030  PrevInstIt =
5031  PrevInst->getIterator().getReverse();
5032  while (InstIt != PrevInstIt) {
5033  if (PrevInstIt == PrevInst->getParent()->rend()) {
5034  PrevInstIt = Inst->getParent()->rbegin();
5035  continue;
5036  }
5037 
5038  // Debug information does not impact spill cost.
5039  if ((isa<CallInst>(&*PrevInstIt) &&
5040  !isa<DbgInfoIntrinsic>(&*PrevInstIt)) &&
5041  &*PrevInstIt != PrevInst)
5042  NumCalls++;
5043 
5044  ++PrevInstIt;
5045  }
5046 
5047  if (NumCalls) {
5049  for (auto *II : LiveValues) {
5050  auto *ScalarTy = II->getType();
5051  if (auto *VectorTy = dyn_cast<FixedVectorType>(ScalarTy))
5052  ScalarTy = VectorTy->getElementType();
5053  V.push_back(FixedVectorType::get(ScalarTy, BundleWidth));
5054  }
5055  Cost += NumCalls * TTI->getCostOfKeepingLiveOverCall(V);
5056  }
5057 
5058  PrevInst = Inst;
5059  }
5060 
5061  return Cost;
5062 }
5063 
5064 InstructionCost BoUpSLP::getTreeCost(ArrayRef<Value *> VectorizedVals) {
5065  InstructionCost Cost = 0;
5066  LLVM_DEBUG(dbgs() << "SLP: Calculating cost for tree of size "
5067  << VectorizableTree.size() << ".\n");
5068 
5069  unsigned BundleWidth = VectorizableTree[0]->Scalars.size();
5070 
5071  for (unsigned I = 0, E = VectorizableTree.size(); I < E; ++I) {
5072  TreeEntry &TE = *VectorizableTree[I].get();
5073 
5074  InstructionCost C = getEntryCost(&TE, VectorizedVals);
5075  Cost += C;
5076  LLVM_DEBUG(dbgs() << "SLP: Adding cost " << C
5077  << " for bundle that starts with " << *TE.Scalars[0]
5078  << ".\n"
5079  << "SLP: Current total cost = " << Cost << "\n");
5080  }
5081 
5082  SmallPtrSet<Value *, 16> ExtractCostCalculated;
5083  InstructionCost ExtractCost = 0;
5085  SmallVector<SmallVector<int>> ShuffleMask;
5086  SmallVector<Value *> FirstUsers;
5087  SmallVector<APInt> DemandedElts;
5088  for (ExternalUser &EU : ExternalUses) {
5089  // We only add extract cost once for the same scalar.
5090  if (!ExtractCostCalculated.insert(EU.Scalar).second)
5091  continue;
5092 
5093  // Uses by ephemeral values are free (because the ephemeral value will be
5094  // removed prior to code generation, and so the extraction will be
5095  // removed as well).
5096  if (EphValues.count(EU.User))
5097  continue;
5098 
5099  // No extract cost for vector "scalar"
5100  if (isa<FixedVectorType>(EU.Scalar->getType()))
5101  continue;
5102 
5103  // Already counted the cost for external uses when tried to adjust the cost
5104  // for extractelements, no need to add it again.
5105  if (isa<ExtractElementInst>(EU.Scalar))
5106  continue;
5107 
5108  // If found user is an insertelement, do not calculate extract cost but try
5109  // to detect it as a final shuffled/identity match.
5110  if (EU.User && isa<InsertElementInst>(EU.User)) {
5111  if (auto *FTy = dyn_cast<FixedVectorType>(EU.User->getType())) {
5112  Optional<int> InsertIdx = getInsertIndex(EU.User, 0);
5113  if (!InsertIdx || *InsertIdx == UndefMaskElem)
5114  continue;
5115  Value *VU = EU.User;
5116  auto *It = find_if(FirstUsers, [VU](Value *V) {
5117  // Checks if 2 insertelements are from the same buildvector.
5118  if (VU->getType() != V->getType())
5119  return false;
5120  auto *IE1 = cast<InsertElementInst>(VU);
5121  auto *IE2 = cast<InsertElementInst>(V);
5122  // Go though of insertelement instructions trying to find either VU as
5123  // the original vector for IE2 or V as the original vector for IE1.
5124  do {
5125  if (IE1 == VU || IE2 == V)
5126  return true;
5127  if (IE1)
5128  IE1 = dyn_cast<InsertElementInst>(IE1->getOperand(0));
5129  if (IE2)
5130  IE2 = dyn_cast<InsertElementInst>(IE2->getOperand(0));
5131  } while (IE1 || IE2);
5132  return false;
5133  });
5134  int VecId = -1;
5135  if (It == FirstUsers.end()) {
5136  VF.push_back(FTy->getNumElements());
5137  ShuffleMask.emplace_back(VF.back(), UndefMaskElem);
5138  FirstUsers.push_back(EU.User);
5139  DemandedElts.push_back(APInt::getZero(VF.back()));
5140  VecId = FirstUsers.size() - 1;
5141  } else {
5142  VecId = std::distance(FirstUsers.begin(), It);
5143  }
5144  int Idx = *InsertIdx;
5145  ShuffleMask[VecId][Idx] = EU.Lane;
5146  DemandedElts[VecId].setBit(Idx);
5147  }
5148  }
5149 
5150  // If we plan to rewrite the tree in a smaller type, we will need to sign
5151  // extend the extracted value back to the original type. Here, we account
5152  // for the extract and the added cost of the sign extend if needed.
5153  auto *VecTy = FixedVectorType::get(EU.Scalar->getType(), BundleWidth);
5154  auto *ScalarRoot = VectorizableTree[0]->Scalars[0];
5155  if (MinBWs.count(ScalarRoot)) {
5156  auto *MinTy = IntegerType::get(F->getContext(), MinBWs[ScalarRoot].first);
5157  auto Extend =
5158  MinBWs[ScalarRoot].second ? Instruction::SExt : Instruction::ZExt;
5159  VecTy = FixedVectorType::get(MinTy, BundleWidth);
5160  ExtractCost += TTI->getExtractWithExtendCost(Extend, EU.Scalar->getType(),
5161  VecTy, EU.Lane);
5162  } else {
5163  ExtractCost +=
5164  TTI->getVectorInstrCost(Instruction::ExtractElement, VecTy, EU.Lane);
5165  }
5166  }
5167 
5168  InstructionCost SpillCost = getSpillCost();
5169  Cost += SpillCost + ExtractCost;
5170  for (int I = 0, E = FirstUsers.size(); I < E; ++I) {
5171  // For the very first element - simple shuffle of the source vector.
5172  int Limit = ShuffleMask[I].size() * 2;
5173  if (I == 0 &&
5174  all_of(ShuffleMask[I], [Limit](int Idx) { return Idx < Limit; }) &&
5175  !ShuffleVectorInst::isIdentityMask(ShuffleMask[I])) {
5177  TTI::SK_PermuteSingleSrc,
5178  cast<FixedVectorType>(FirstUsers[I]->getType()), ShuffleMask[I]);
5179  LLVM_DEBUG(dbgs() << "SLP: Adding cost " << C
5180  << " for final shuffle of insertelement external users "
5181  << *VectorizableTree.front()->Scalars.front() << ".\n"
5182  << "SLP: Current total cost = " << Cost << "\n");
5183  Cost += C;
5184  continue;
5185  }
5186  // Other elements - permutation of 2 vectors (the initial one and the next
5187  // Ith incoming vector).
5188  unsigned VF = ShuffleMask[I].size();
5189  for (unsigned Idx = 0; Idx < VF; ++Idx) {
5190  int &Mask = ShuffleMask[I][Idx];
5191  Mask = Mask == UndefMaskElem ? Idx : VF + Mask;
5192  }
5194  TTI::SK_PermuteTwoSrc, cast<FixedVectorType>(FirstUsers[I]->getType()),
5195  ShuffleMask[I]);
5196  LLVM_DEBUG(
5197  dbgs()
5198  << "SLP: Adding cost " << C
5199  << " for final shuffle of vector node and external insertelement users "
5200  << *VectorizableTree.front()->Scalars.front() << ".\n"
5201  << "SLP: Current total cost = " << Cost << "\n");
5202  Cost += C;
5204  cast<FixedVectorType>(FirstUsers[I]->getType()), DemandedElts[I],
5205  /*Insert*/ true,
5206  /*Extract*/ false);
5207  Cost -= InsertCost;
5208  LLVM_DEBUG(dbgs() << "SLP: subtracting the cost " << InsertCost
5209  << " for insertelements gather.\n"
5210  << "SLP: Current total cost = " << Cost << "\n");
5211  }
5212 
5213 #ifndef NDEBUG
5214  SmallString<256> Str;
5215  {
5216  raw_svector_ostream OS(Str);
5217  OS << "SLP: Spill Cost = " << SpillCost << ".\n"
5218  << "SLP: Extract Cost = " << ExtractCost << ".\n"
5219  << "SLP: Total Cost = " << Cost << ".\n";
5220  }
5221  LLVM_DEBUG(dbgs() << Str);
5222  if (ViewSLPTree)
5223  ViewGraph(this, "SLP" + F->getName(), false, Str);
5224 #endif
5225 
5226  return Cost;
5227 }
5228 
5230 BoUpSLP::isGatherShuffledEntry(const TreeEntry *TE, SmallVectorImpl<int> &Mask,
5232  // TODO: currently checking only for Scalars in the tree entry, need to count
5233  // reused elements too for better cost estimation.
5234  Mask.assign(TE->Scalars.size(), UndefMaskElem);
5235  Entries.clear();
5236  // Build a lists of values to tree entries.
5238  for (const std::unique_ptr<TreeEntry> &EntryPtr : VectorizableTree) {
5239  if (EntryPtr.get() == TE)
5240  break;
5241  if (EntryPtr->State != TreeEntry::NeedToGather)
5242  continue;
5243  for (Value *V : EntryPtr->Scalars)
5244  ValueToTEs.try_emplace(V).first->getSecond().insert(EntryPtr.get());
5245  }
5246  // Find all tree entries used by the gathered values. If no common entries
5247  // found - not a shuffle.
5248  // Here we build a set of tree nodes for each gathered value and trying to
5249  // find the intersection between these sets. If we have at least one common
5250  // tree node for each gathered value - we have just a permutation of the
5251  // single vector. If we have 2 different sets, we're in situation where we