LLVM  16.0.0git
VPlanRecipes.cpp
Go to the documentation of this file.
1 //===- VPlanRecipes.cpp - Implementations for VPlan recipes ---------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 ///
9 /// \file
10 /// This file contains implementations for different VPlan recipes.
11 ///
12 //===----------------------------------------------------------------------===//
13 
14 #include "VPlan.h"
15 #include "llvm/ADT/STLExtras.h"
16 #include "llvm/ADT/SmallVector.h"
17 #include "llvm/ADT/Twine.h"
19 #include "llvm/IR/BasicBlock.h"
20 #include "llvm/IR/IRBuilder.h"
21 #include "llvm/IR/Instruction.h"
22 #include "llvm/IR/Instructions.h"
23 #include "llvm/IR/Type.h"
24 #include "llvm/IR/Value.h"
25 #include "llvm/Support/Casting.h"
27 #include "llvm/Support/Debug.h"
31 #include <cassert>
32 
33 using namespace llvm;
34 
36 
38 
39 #define LV_NAME "loop-vectorize"
40 #define DEBUG_TYPE LV_NAME
41 
43  switch (getVPDefID()) {
44  case VPWidenMemoryInstructionSC: {
45  return cast<VPWidenMemoryInstructionRecipe>(this)->isStore();
46  }
47  case VPReplicateSC:
48  case VPWidenCallSC:
49  return cast<Instruction>(getVPSingleValue()->getUnderlyingValue())
50  ->mayWriteToMemory();
51  case VPBranchOnMaskSC:
52  return false;
53  case VPWidenIntOrFpInductionSC:
54  case VPWidenCanonicalIVSC:
55  case VPWidenPHISC:
56  case VPBlendSC:
57  case VPWidenSC:
58  case VPWidenGEPSC:
59  case VPReductionSC:
60  case VPWidenSelectSC: {
61  const Instruction *I =
62  dyn_cast_or_null<Instruction>(getVPSingleValue()->getUnderlyingValue());
63  (void)I;
64  assert((!I || !I->mayWriteToMemory()) &&
65  "underlying instruction may write to memory");
66  return false;
67  }
68  default:
69  return true;
70  }
71 }
72 
74  switch (getVPDefID()) {
75  case VPWidenMemoryInstructionSC: {
76  return !cast<VPWidenMemoryInstructionRecipe>(this)->isStore();
77  }
78  case VPReplicateSC:
79  case VPWidenCallSC:
80  return cast<Instruction>(getVPSingleValue()->getUnderlyingValue())
81  ->mayReadFromMemory();
82  case VPBranchOnMaskSC:
83  return false;
84  case VPWidenIntOrFpInductionSC:
85  case VPWidenCanonicalIVSC:
86  case VPWidenPHISC:
87  case VPBlendSC:
88  case VPWidenSC:
89  case VPWidenGEPSC:
90  case VPReductionSC:
91  case VPWidenSelectSC: {
92  const Instruction *I =
93  dyn_cast_or_null<Instruction>(getVPSingleValue()->getUnderlyingValue());
94  (void)I;
95  assert((!I || !I->mayReadFromMemory()) &&
96  "underlying instruction may read from memory");
97  return false;
98  }
99  default:
100  return true;
101  }
102 }
103 
105  switch (getVPDefID()) {
106  case VPPredInstPHISC:
107  return false;
108  case VPWidenIntOrFpInductionSC:
109  case VPWidenPointerInductionSC:
110  case VPWidenCanonicalIVSC:
111  case VPWidenPHISC:
112  case VPBlendSC:
113  case VPWidenSC:
114  case VPWidenGEPSC:
115  case VPReductionSC:
116  case VPWidenSelectSC:
117  case VPScalarIVStepsSC: {
118  const Instruction *I =
119  dyn_cast_or_null<Instruction>(getVPSingleValue()->getUnderlyingValue());
120  (void)I;
121  assert((!I || !I->mayHaveSideEffects()) &&
122  "underlying instruction has side-effects");
123  return false;
124  }
125  case VPReplicateSC: {
126  auto *R = cast<VPReplicateRecipe>(this);
127  return R->getUnderlyingInstr()->mayHaveSideEffects();
128  }
129  default:
130  return true;
131  }
132 }
133 
135  auto Lane = VPLane::getLastLaneForVF(State.VF);
136  VPValue *ExitValue = getOperand(0);
137  if (Plan.isUniformAfterVectorization(ExitValue))
138  Lane = VPLane::getFirstLane();
139  Phi->addIncoming(State.get(ExitValue, VPIteration(State.UF - 1, Lane)),
140  State.Builder.GetInsertBlock());
141 }
142 
144  assert(!Parent && "Recipe already in some VPBasicBlock");
145  assert(InsertPos->getParent() &&
146  "Insertion position not in any VPBasicBlock");
147  Parent = InsertPos->getParent();
148  Parent->getRecipeList().insert(InsertPos->getIterator(), this);
149 }
150 
153  assert(!Parent && "Recipe already in some VPBasicBlock");
154  assert(I == BB.end() || I->getParent() == &BB);
155  Parent = &BB;
156  BB.getRecipeList().insert(I, this);
157 }
158 
160  assert(!Parent && "Recipe already in some VPBasicBlock");
161  assert(InsertPos->getParent() &&
162  "Insertion position not in any VPBasicBlock");
163  Parent = InsertPos->getParent();
164  Parent->getRecipeList().insertAfter(InsertPos->getIterator(), this);
165 }
166 
168  assert(getParent() && "Recipe not in any VPBasicBlock");
170  Parent = nullptr;
171 }
172 
174  assert(getParent() && "Recipe not in any VPBasicBlock");
175  return getParent()->getRecipeList().erase(getIterator());
176 }
177 
180  insertAfter(InsertPos);
181 }
182 
186  insertBefore(BB, I);
187 }
188 
189 void VPInstruction::generateInstruction(VPTransformState &State,
190  unsigned Part) {
191  IRBuilderBase &Builder = State.Builder;
192  Builder.SetCurrentDebugLocation(DL);
193 
195  Value *A = State.get(getOperand(0), Part);
196  Value *B = State.get(getOperand(1), Part);
197  Value *V =
198  Builder.CreateBinOp((Instruction::BinaryOps)getOpcode(), A, B, Name);
199  State.set(this, V, Part);
200  return;
201  }
202 
203  switch (getOpcode()) {
204  case VPInstruction::Not: {
205  Value *A = State.get(getOperand(0), Part);
206  Value *V = Builder.CreateNot(A, Name);
207  State.set(this, V, Part);
208  break;
209  }
210  case VPInstruction::ICmpULE: {
211  Value *IV = State.get(getOperand(0), Part);
212  Value *TC = State.get(getOperand(1), Part);
213  Value *V = Builder.CreateICmpULE(IV, TC, Name);
214  State.set(this, V, Part);
215  break;
216  }
217  case Instruction::Select: {
218  Value *Cond = State.get(getOperand(0), Part);
219  Value *Op1 = State.get(getOperand(1), Part);
220  Value *Op2 = State.get(getOperand(2), Part);
221  Value *V = Builder.CreateSelect(Cond, Op1, Op2, Name);
222  State.set(this, V, Part);
223  break;
224  }
226  // Get first lane of vector induction variable.
227  Value *VIVElem0 = State.get(getOperand(0), VPIteration(Part, 0));
228  // Get the original loop tripcount.
229  Value *ScalarTC = State.get(getOperand(1), Part);
230 
231  auto *Int1Ty = Type::getInt1Ty(Builder.getContext());
232  auto *PredTy = VectorType::get(Int1Ty, State.VF);
233  Instruction *Call = Builder.CreateIntrinsic(
234  Intrinsic::get_active_lane_mask, {PredTy, ScalarTC->getType()},
235  {VIVElem0, ScalarTC}, nullptr, Name);
236  State.set(this, Call, Part);
237  break;
238  }
240  // Generate code to combine the previous and current values in vector v3.
241  //
242  // vector.ph:
243  // v_init = vector(..., ..., ..., a[-1])
244  // br vector.body
245  //
246  // vector.body
247  // i = phi [0, vector.ph], [i+4, vector.body]
248  // v1 = phi [v_init, vector.ph], [v2, vector.body]
249  // v2 = a[i, i+1, i+2, i+3];
250  // v3 = vector(v1(3), v2(0, 1, 2))
251 
252  // For the first part, use the recurrence phi (v1), otherwise v2.
253  auto *V1 = State.get(getOperand(0), 0);
254  Value *PartMinus1 = Part == 0 ? V1 : State.get(getOperand(1), Part - 1);
255  if (!PartMinus1->getType()->isVectorTy()) {
256  State.set(this, PartMinus1, Part);
257  } else {
258  Value *V2 = State.get(getOperand(1), Part);
259  State.set(this, Builder.CreateVectorSplice(PartMinus1, V2, -1, Name),
260  Part);
261  }
262  break;
263  }
266  Value *Next = nullptr;
267  if (Part == 0) {
269  auto *Phi = State.get(getOperand(0), 0);
270  // The loop step is equal to the vectorization factor (num of SIMD
271  // elements) times the unroll factor (num of SIMD instructions).
272  Value *Step =
273  createStepForVF(Builder, Phi->getType(), State.VF, State.UF);
274  Next = Builder.CreateAdd(Phi, Step, Name, IsNUW, false);
275  } else {
276  Next = State.get(this, 0);
277  }
278 
279  State.set(this, Next, Part);
280  break;
281  }
282 
286  auto *IV = State.get(getOperand(0), VPIteration(0, 0));
287  if (Part == 0) {
288  State.set(this, IV, Part);
289  break;
290  }
291 
292  // The canonical IV is incremented by the vectorization factor (num of SIMD
293  // elements) times the unroll part.
294  Value *Step = createStepForVF(Builder, IV->getType(), State.VF, Part);
295  Value *Next = Builder.CreateAdd(IV, Step, Name, IsNUW, false);
296  State.set(this, Next, Part);
297  break;
298  }
300  if (Part != 0)
301  break;
302 
303  Value *Cond = State.get(getOperand(0), VPIteration(Part, 0));
304  VPRegionBlock *ParentRegion = getParent()->getParent();
305  VPBasicBlock *Header = ParentRegion->getEntryBasicBlock();
306 
307  // Replace the temporary unreachable terminator with a new conditional
308  // branch, hooking it up to backward destination for exiting blocks now and
309  // to forward destination(s) later when they are created.
310  BranchInst *CondBr =
311  Builder.CreateCondBr(Cond, Builder.GetInsertBlock(), nullptr);
312 
313  if (getParent()->isExiting())
314  CondBr->setSuccessor(1, State.CFG.VPBB2IRBB[Header]);
315 
316  CondBr->setSuccessor(0, nullptr);
317  Builder.GetInsertBlock()->getTerminator()->eraseFromParent();
318  break;
319  }
321  if (Part != 0)
322  break;
323  // First create the compare.
324  Value *IV = State.get(getOperand(0), Part);
325  Value *TC = State.get(getOperand(1), Part);
326  Value *Cond = Builder.CreateICmpEQ(IV, TC);
327 
328  // Now create the branch.
329  auto *Plan = getParent()->getPlan();
330  VPRegionBlock *TopRegion = Plan->getVectorLoopRegion();
331  VPBasicBlock *Header = TopRegion->getEntry()->getEntryBasicBlock();
332 
333  // Replace the temporary unreachable terminator with a new conditional
334  // branch, hooking it up to backward destination (the header) now and to the
335  // forward destination (the exit/middle block) later when it is created.
336  // Note that CreateCondBr expects a valid BB as first argument, so we need
337  // to set it to nullptr later.
338  BranchInst *CondBr = Builder.CreateCondBr(Cond, Builder.GetInsertBlock(),
339  State.CFG.VPBB2IRBB[Header]);
340  CondBr->setSuccessor(0, nullptr);
341  Builder.GetInsertBlock()->getTerminator()->eraseFromParent();
342  break;
343  }
344  default:
345  llvm_unreachable("Unsupported opcode for instruction");
346  }
347 }
348 
350  assert(!State.Instance && "VPInstruction executing an Instance");
352  State.Builder.setFastMathFlags(FMF);
353  for (unsigned Part = 0; Part < State.UF; ++Part)
354  generateInstruction(State, Part);
355 }
356 
357 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
358 void VPInstruction::dump() const {
359  VPSlotTracker SlotTracker(getParent()->getPlan());
360  print(dbgs(), "", SlotTracker);
361 }
362 
364  VPSlotTracker &SlotTracker) const {
365  O << Indent << "EMIT ";
366 
367  if (hasResult()) {
369  O << " = ";
370  }
371 
372  switch (getOpcode()) {
373  case VPInstruction::Not:
374  O << "not";
375  break;
377  O << "icmp ule";
378  break;
380  O << "combined load";
381  break;
383  O << "combined store";
384  break;
386  O << "active lane mask";
387  break;
389  O << "first-order splice";
390  break;
392  O << "VF * UF + ";
393  break;
395  O << "VF * UF +(nuw) ";
396  break;
398  O << "branch-on-cond";
399  break;
401  O << "VF * Part + ";
402  break;
404  O << "VF * Part +(nuw) ";
405  break;
407  O << "branch-on-count ";
408  break;
409  default:
411  }
412 
413  O << FMF;
414 
415  for (const VPValue *Operand : operands()) {
416  O << " ";
417  Operand->printAsOperand(O, SlotTracker);
418  }
419 
420  if (DL) {
421  O << ", !dbg ";
422  DL.print(O);
423  }
424 }
425 #endif
426 
428  // Make sure the VPInstruction is a floating-point operation.
429  assert((Opcode == Instruction::FAdd || Opcode == Instruction::FMul ||
430  Opcode == Instruction::FNeg || Opcode == Instruction::FSub ||
431  Opcode == Instruction::FDiv || Opcode == Instruction::FRem ||
432  Opcode == Instruction::FCmp) &&
433  "this op can't take fast-math flags");
434  FMF = FMFNew;
435 }
436 
437 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
439  VPSlotTracker &SlotTracker) const {
440  O << Indent << "WIDEN-CALL ";
441 
442  auto *CI = cast<CallInst>(getUnderlyingInstr());
443  if (CI->getType()->isVoidTy())
444  O << "void ";
445  else {
447  O << " = ";
448  }
449 
450  O << "call @" << CI->getCalledFunction()->getName() << "(";
452  O << ")";
453 }
454 
456  VPSlotTracker &SlotTracker) const {
457  O << Indent << "WIDEN-SELECT ";
459  O << " = select ";
461  O << ", ";
463  O << ", ";
465  O << (InvariantCond ? " (condition is loop invariant)" : "");
466 }
467 #endif
468 
470  auto &I = *cast<SelectInst>(getUnderlyingInstr());
471  State.setDebugLocFromInst(&I);
472 
473  // The condition can be loop invariant but still defined inside the
474  // loop. This means that we can't just use the original 'cond' value.
475  // We have to take the 'vectorized' value and pick the first lane.
476  // Instcombine will make this a no-op.
477  auto *InvarCond =
478  InvariantCond ? State.get(getOperand(0), VPIteration(0, 0)) : nullptr;
479 
480  for (unsigned Part = 0; Part < State.UF; ++Part) {
481  Value *Cond = InvarCond ? InvarCond : State.get(getOperand(0), Part);
482  Value *Op0 = State.get(getOperand(1), Part);
483  Value *Op1 = State.get(getOperand(2), Part);
484  Value *Sel = State.Builder.CreateSelect(Cond, Op0, Op1);
485  State.set(this, Sel, Part);
486  State.addMetadata(Sel, &I);
487  }
488 }
489 
491  auto &I = *cast<Instruction>(getUnderlyingValue());
492  auto &Builder = State.Builder;
493  switch (I.getOpcode()) {
494  case Instruction::Call:
495  case Instruction::Br:
496  case Instruction::PHI:
497  case Instruction::GetElementPtr:
498  case Instruction::Select:
499  llvm_unreachable("This instruction is handled by a different recipe.");
500  case Instruction::UDiv:
501  case Instruction::SDiv:
502  case Instruction::SRem:
503  case Instruction::URem:
504  case Instruction::Add:
505  case Instruction::FAdd:
506  case Instruction::Sub:
507  case Instruction::FSub:
508  case Instruction::FNeg:
509  case Instruction::Mul:
510  case Instruction::FMul:
511  case Instruction::FDiv:
512  case Instruction::FRem:
513  case Instruction::Shl:
514  case Instruction::LShr:
515  case Instruction::AShr:
516  case Instruction::And:
517  case Instruction::Or:
518  case Instruction::Xor: {
519  // Just widen unops and binops.
520  State.setDebugLocFromInst(&I);
521 
522  for (unsigned Part = 0; Part < State.UF; ++Part) {
524  for (VPValue *VPOp : operands())
525  Ops.push_back(State.get(VPOp, Part));
526 
527  Value *V = Builder.CreateNAryOp(I.getOpcode(), Ops);
528 
529  if (auto *VecOp = dyn_cast<Instruction>(V)) {
530  VecOp->copyIRFlags(&I);
531 
532  // If the instruction is vectorized and was in a basic block that needed
533  // predication, we can't propagate poison-generating flags (nuw/nsw,
534  // exact, etc.). The control flow has been linearized and the
535  // instruction is no longer guarded by the predicate, which could make
536  // the flag properties to no longer hold.
537  if (State.MayGeneratePoisonRecipes.contains(this))
538  VecOp->dropPoisonGeneratingFlags();
539  }
540 
541  // Use this vector value for all users of the original instruction.
542  State.set(this, V, Part);
543  State.addMetadata(V, &I);
544  }
545 
546  break;
547  }
548  case Instruction::Freeze: {
549  State.setDebugLocFromInst(&I);
550 
551  for (unsigned Part = 0; Part < State.UF; ++Part) {
552  Value *Op = State.get(getOperand(0), Part);
553 
554  Value *Freeze = Builder.CreateFreeze(Op);
555  State.set(this, Freeze, Part);
556  }
557  break;
558  }
559  case Instruction::ICmp:
560  case Instruction::FCmp: {
561  // Widen compares. Generate vector compares.
562  bool FCmp = (I.getOpcode() == Instruction::FCmp);
563  auto *Cmp = cast<CmpInst>(&I);
564  State.setDebugLocFromInst(Cmp);
565  for (unsigned Part = 0; Part < State.UF; ++Part) {
566  Value *A = State.get(getOperand(0), Part);
567  Value *B = State.get(getOperand(1), Part);
568  Value *C = nullptr;
569  if (FCmp) {
570  // Propagate fast math flags.
572  Builder.setFastMathFlags(Cmp->getFastMathFlags());
573  C = Builder.CreateFCmp(Cmp->getPredicate(), A, B);
574  } else {
575  C = Builder.CreateICmp(Cmp->getPredicate(), A, B);
576  }
577  State.set(this, C, Part);
578  State.addMetadata(C, &I);
579  }
580 
581  break;
582  }
583 
584  case Instruction::ZExt:
585  case Instruction::SExt:
586  case Instruction::FPToUI:
587  case Instruction::FPToSI:
588  case Instruction::FPExt:
589  case Instruction::PtrToInt:
590  case Instruction::IntToPtr:
591  case Instruction::SIToFP:
592  case Instruction::UIToFP:
593  case Instruction::Trunc:
594  case Instruction::FPTrunc:
595  case Instruction::BitCast: {
596  auto *CI = cast<CastInst>(&I);
597  State.setDebugLocFromInst(CI);
598 
599  /// Vectorize casts.
600  Type *DestTy = (State.VF.isScalar())
601  ? CI->getType()
602  : VectorType::get(CI->getType(), State.VF);
603 
604  for (unsigned Part = 0; Part < State.UF; ++Part) {
605  Value *A = State.get(getOperand(0), Part);
606  Value *Cast = Builder.CreateCast(CI->getOpcode(), A, DestTy);
607  State.set(this, Cast, Part);
608  State.addMetadata(Cast, &I);
609  }
610  break;
611  }
612  default:
613  // This instruction is not vectorized by simple widening.
614  LLVM_DEBUG(dbgs() << "LV: Found an unhandled instruction: " << I);
615  llvm_unreachable("Unhandled instruction!");
616  } // end of switch.
617 }
618 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
620  VPSlotTracker &SlotTracker) const {
621  O << Indent << "WIDEN ";
623  O << " = " << getUnderlyingInstr()->getOpcodeName() << " ";
625 }
626 
628  VPSlotTracker &SlotTracker) const {
629  O << Indent << "WIDEN-INDUCTION";
630  if (getTruncInst()) {
631  O << "\\l\"";
632  O << " +\n" << Indent << "\" " << VPlanIngredient(IV) << "\\l\"";
633  O << " +\n" << Indent << "\" ";
635  } else
636  O << " " << VPlanIngredient(IV);
637 
638  O << ", ";
640 }
641 #endif
642 
644  auto *StartC = dyn_cast<ConstantInt>(getStartValue()->getLiveInIRValue());
645  auto *StepC = dyn_cast<SCEVConstant>(getInductionDescriptor().getStep());
646  return StartC && StartC->isZero() && StepC && StepC->isOne();
647 }
648 
650  return cast<VPCanonicalIVPHIRecipe>(getOperand(0));
651 }
652 
654  auto *CanIV = getCanonicalIV();
655  // The start value of the steps-recipe must match the start value of the
656  // canonical induction and it must step by 1.
657  if (CanIV->getStartValue() != getStartValue())
658  return false;
659  auto *StepVPV = getStepValue();
660  if (StepVPV->getDef())
661  return false;
662  auto *StepC = dyn_cast_or_null<ConstantInt>(StepVPV->getLiveInIRValue());
663  return StepC && StepC->isOne();
664 }
665 
666 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
668  VPSlotTracker &SlotTracker) const {
669  O << Indent;
671  O << Indent << "= SCALAR-STEPS ";
673 }
674 #endif
675 
677  auto *GEP = cast<GetElementPtrInst>(getUnderlyingInstr());
678  // Construct a vector GEP by widening the operands of the scalar GEP as
679  // necessary. We mark the vector GEP 'inbounds' if appropriate. A GEP
680  // results in a vector of pointers when at least one operand of the GEP
681  // is vector-typed. Thus, to keep the representation compact, we only use
682  // vector-typed operands for loop-varying values.
683 
684  if (State.VF.isVector() && IsPtrLoopInvariant && IsIndexLoopInvariant.all()) {
685  // If we are vectorizing, but the GEP has only loop-invariant operands,
686  // the GEP we build (by only using vector-typed operands for
687  // loop-varying values) would be a scalar pointer. Thus, to ensure we
688  // produce a vector of pointers, we need to either arbitrarily pick an
689  // operand to broadcast, or broadcast a clone of the original GEP.
690  // Here, we broadcast a clone of the original.
691  //
692  // TODO: If at some point we decide to scalarize instructions having
693  // loop-invariant operands, this special case will no longer be
694  // required. We would add the scalarization decision to
695  // collectLoopScalars() and teach getVectorValue() to broadcast
696  // the lane-zero scalar value.
697  auto *Clone = State.Builder.Insert(GEP->clone());
698  for (unsigned Part = 0; Part < State.UF; ++Part) {
699  Value *EntryPart = State.Builder.CreateVectorSplat(State.VF, Clone);
700  State.set(this, EntryPart, Part);
701  State.addMetadata(EntryPart, GEP);
702  }
703  } else {
704  // If the GEP has at least one loop-varying operand, we are sure to
705  // produce a vector of pointers. But if we are only unrolling, we want
706  // to produce a scalar GEP for each unroll part. Thus, the GEP we
707  // produce with the code below will be scalar (if VF == 1) or vector
708  // (otherwise). Note that for the unroll-only case, we still maintain
709  // values in the vector mapping with initVector, as we do for other
710  // instructions.
711  for (unsigned Part = 0; Part < State.UF; ++Part) {
712  // The pointer operand of the new GEP. If it's loop-invariant, we
713  // won't broadcast it.
714  auto *Ptr = IsPtrLoopInvariant
715  ? State.get(getOperand(0), VPIteration(0, 0))
716  : State.get(getOperand(0), Part);
717 
718  // Collect all the indices for the new GEP. If any index is
719  // loop-invariant, we won't broadcast it.
720  SmallVector<Value *, 4> Indices;
721  for (unsigned I = 1, E = getNumOperands(); I < E; I++) {
722  VPValue *Operand = getOperand(I);
723  if (IsIndexLoopInvariant[I - 1])
724  Indices.push_back(State.get(Operand, VPIteration(0, 0)));
725  else
726  Indices.push_back(State.get(Operand, Part));
727  }
728 
729  // If the GEP instruction is vectorized and was in a basic block that
730  // needed predication, we can't propagate the poison-generating 'inbounds'
731  // flag. The control flow has been linearized and the GEP is no longer
732  // guarded by the predicate, which could make the 'inbounds' properties to
733  // no longer hold.
734  bool IsInBounds =
735  GEP->isInBounds() && State.MayGeneratePoisonRecipes.count(this) == 0;
736 
737  // Create the new GEP. Note that this GEP may be a scalar if VF == 1,
738  // but it should be a vector, otherwise.
739  auto *NewGEP = State.Builder.CreateGEP(GEP->getSourceElementType(), Ptr,
740  Indices, "", IsInBounds);
741  assert((State.VF.isScalar() || NewGEP->getType()->isVectorTy()) &&
742  "NewGEP is not a pointer vector");
743  State.set(this, NewGEP, Part);
744  State.addMetadata(NewGEP, GEP);
745  }
746  }
747 }
748 
749 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
751  VPSlotTracker &SlotTracker) const {
752  O << Indent << "WIDEN-GEP ";
753  O << (IsPtrLoopInvariant ? "Inv" : "Var");
754  size_t IndicesNumber = IsIndexLoopInvariant.size();
755  for (size_t I = 0; I < IndicesNumber; ++I)
756  O << "[" << (IsIndexLoopInvariant[I] ? "Inv" : "Var") << "]";
757 
758  O << " ";
760  O << " = getelementptr ";
762 }
763 #endif
764 
766  State.setDebugLocFromInst(Phi);
767  // We know that all PHIs in non-header blocks are converted into
768  // selects, so we don't have to worry about the insertion order and we
769  // can just use the builder.
770  // At this point we generate the predication tree. There may be
771  // duplications since this is a simple recursive scan, but future
772  // optimizations will clean it up.
773 
774  unsigned NumIncoming = getNumIncomingValues();
775 
776  // Generate a sequence of selects of the form:
777  // SELECT(Mask3, In3,
778  // SELECT(Mask2, In2,
779  // SELECT(Mask1, In1,
780  // In0)))
781  // Note that Mask0 is never used: lanes for which no path reaches this phi and
782  // are essentially undef are taken from In0.
783  VectorParts Entry(State.UF);
784  for (unsigned In = 0; In < NumIncoming; ++In) {
785  for (unsigned Part = 0; Part < State.UF; ++Part) {
786  // We might have single edge PHIs (blocks) - use an identity
787  // 'select' for the first PHI operand.
788  Value *In0 = State.get(getIncomingValue(In), Part);
789  if (In == 0)
790  Entry[Part] = In0; // Initialize with the first incoming value.
791  else {
792  // Select between the current value and the previous incoming edge
793  // based on the incoming mask.
794  Value *Cond = State.get(getMask(In), Part);
795  Entry[Part] =
796  State.Builder.CreateSelect(Cond, In0, Entry[Part], "predphi");
797  }
798  }
799  }
800  for (unsigned Part = 0; Part < State.UF; ++Part)
801  State.set(this, Entry[Part], Part);
802 }
803 
804 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
806  VPSlotTracker &SlotTracker) const {
807  O << Indent << "BLEND ";
808  Phi->printAsOperand(O, false);
809  O << " =";
810  if (getNumIncomingValues() == 1) {
811  // Not a User of any mask: not really blending, this is a
812  // single-predecessor phi.
813  O << " ";
815  } else {
816  for (unsigned I = 0, E = getNumIncomingValues(); I < E; ++I) {
817  O << " ";
819  O << "/";
821  }
822  }
823 }
824 
826  VPSlotTracker &SlotTracker) const {
827  O << Indent << "REDUCE ";
829  O << " = ";
831  O << " +";
832  if (isa<FPMathOperator>(getUnderlyingInstr()))
834  O << " reduce." << Instruction::getOpcodeName(RdxDesc->getOpcode()) << " (";
836  if (getCondOp()) {
837  O << ", ";
839  }
840  O << ")";
841  if (RdxDesc->IntermediateStore)
842  O << " (with final reduction value stored in invariant address sank "
843  "outside of loop)";
844 }
845 
847  VPSlotTracker &SlotTracker) const {
848  O << Indent << (IsUniform ? "CLONE " : "REPLICATE ");
849 
850  if (!getUnderlyingInstr()->getType()->isVoidTy()) {
852  O << " = ";
853  }
854  if (auto *CB = dyn_cast<CallBase>(getUnderlyingInstr())) {
855  O << "call @" << CB->getCalledFunction()->getName() << "(";
857  O, [&O, &SlotTracker](VPValue *Op) {
858  Op->printAsOperand(O, SlotTracker);
859  });
860  O << ")";
861  } else {
864  }
865 
866  if (AlsoPack)
867  O << " (S->V)";
868 }
869 #endif
870 
872  assert(State.Instance && "Branch on Mask works only on single instance.");
873 
874  unsigned Part = State.Instance->Part;
875  unsigned Lane = State.Instance->Lane.getKnownLane();
876 
877  Value *ConditionBit = nullptr;
878  VPValue *BlockInMask = getMask();
879  if (BlockInMask) {
880  ConditionBit = State.get(BlockInMask, Part);
881  if (ConditionBit->getType()->isVectorTy())
882  ConditionBit = State.Builder.CreateExtractElement(
883  ConditionBit, State.Builder.getInt32(Lane));
884  } else // Block in mask is all-one.
885  ConditionBit = State.Builder.getTrue();
886 
887  // Replace the temporary unreachable terminator with a new conditional branch,
888  // whose two destinations will be set later when they are created.
889  auto *CurrentTerminator = State.CFG.PrevBB->getTerminator();
890  assert(isa<UnreachableInst>(CurrentTerminator) &&
891  "Expected to replace unreachable terminator with conditional branch.");
892  auto *CondBr = BranchInst::Create(State.CFG.PrevBB, nullptr, ConditionBit);
893  CondBr->setSuccessor(0, nullptr);
894  ReplaceInstWithInst(CurrentTerminator, CondBr);
895 }
896 
898  assert(State.Instance && "Predicated instruction PHI works per instance.");
899  Instruction *ScalarPredInst =
900  cast<Instruction>(State.get(getOperand(0), *State.Instance));
901  BasicBlock *PredicatedBB = ScalarPredInst->getParent();
902  BasicBlock *PredicatingBB = PredicatedBB->getSinglePredecessor();
903  assert(PredicatingBB && "Predicated block has no single predecessor.");
904  assert(isa<VPReplicateRecipe>(getOperand(0)) &&
905  "operand must be VPReplicateRecipe");
906 
907  // By current pack/unpack logic we need to generate only a single phi node: if
908  // a vector value for the predicated instruction exists at this point it means
909  // the instruction has vector users only, and a phi for the vector value is
910  // needed. In this case the recipe of the predicated instruction is marked to
911  // also do that packing, thereby "hoisting" the insert-element sequence.
912  // Otherwise, a phi node for the scalar value is needed.
913  unsigned Part = State.Instance->Part;
914  if (State.hasVectorValue(getOperand(0), Part)) {
915  Value *VectorValue = State.get(getOperand(0), Part);
916  InsertElementInst *IEI = cast<InsertElementInst>(VectorValue);
917  PHINode *VPhi = State.Builder.CreatePHI(IEI->getType(), 2);
918  VPhi->addIncoming(IEI->getOperand(0), PredicatingBB); // Unmodified vector.
919  VPhi->addIncoming(IEI, PredicatedBB); // New vector with inserted element.
920  if (State.hasVectorValue(this, Part))
921  State.reset(this, VPhi, Part);
922  else
923  State.set(this, VPhi, Part);
924  // NOTE: Currently we need to update the value of the operand, so the next
925  // predicated iteration inserts its generated value in the correct vector.
926  State.reset(getOperand(0), VPhi, Part);
927  } else {
928  Type *PredInstType = getOperand(0)->getUnderlyingValue()->getType();
929  PHINode *Phi = State.Builder.CreatePHI(PredInstType, 2);
930  Phi->addIncoming(PoisonValue::get(ScalarPredInst->getType()),
931  PredicatingBB);
932  Phi->addIncoming(ScalarPredInst, PredicatedBB);
933  if (State.hasScalarValue(this, *State.Instance))
934  State.reset(this, Phi, *State.Instance);
935  else
936  State.set(this, Phi, *State.Instance);
937  // NOTE: Currently we need to update the value of the operand, so the next
938  // predicated iteration inserts its generated value in the correct vector.
939  State.reset(getOperand(0), Phi, *State.Instance);
940  }
941 }
942 
943 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
945  VPSlotTracker &SlotTracker) const {
946  O << Indent << "PHI-PREDICATED-INSTRUCTION ";
948  O << " = ";
950 }
951 
953  VPSlotTracker &SlotTracker) const {
954  O << Indent << "WIDEN ";
955 
956  if (!isStore()) {
958  O << " = ";
959  }
960  O << Instruction::getOpcodeName(Ingredient.getOpcode()) << " ";
961 
963 }
964 #endif
965 
967  Value *Start = getStartValue()->getLiveInIRValue();
968  PHINode *EntryPart = PHINode::Create(
969  Start->getType(), 2, "index", &*State.CFG.PrevBB->getFirstInsertionPt());
970 
971  BasicBlock *VectorPH = State.CFG.getPreheaderBBFor(this);
972  EntryPart->addIncoming(Start, VectorPH);
973  EntryPart->setDebugLoc(DL);
974  for (unsigned Part = 0, UF = State.UF; Part < UF; ++Part)
975  State.set(this, EntryPart, Part);
976 }
977 
978 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
980  VPSlotTracker &SlotTracker) const {
981  O << Indent << "EMIT ";
983  O << " = CANONICAL-INDUCTION";
984 }
985 #endif
986 
988  bool IsUniform = vputils::onlyFirstLaneUsed(this);
989  return all_of(users(),
990  [&](const VPUser *U) { return U->usesScalars(this); }) &&
991  (IsUniform || !VF.isScalable());
992 }
993 
994 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
996  VPSlotTracker &SlotTracker) const {
997  O << Indent << "EMIT ";
999  O << " = WIDEN-POINTER-INDUCTION ";
1001  O << ", " << *IndDesc.getStep();
1002 }
1003 #endif
1004 
1006  assert(!State.Instance && "cannot be used in per-lane");
1007  const DataLayout &DL = State.CFG.PrevBB->getModule()->getDataLayout();
1008  SCEVExpander Exp(SE, DL, "induction");
1009 
1010  Value *Res = Exp.expandCodeFor(Expr, Expr->getType(),
1011  &*State.Builder.GetInsertPoint());
1012 
1013  for (unsigned Part = 0, UF = State.UF; Part < UF; ++Part)
1014  State.set(this, Res, Part);
1015 }
1016 
1017 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
1019  VPSlotTracker &SlotTracker) const {
1020  O << Indent << "EMIT ";
1022  O << " = EXPAND SCEV " << *Expr;
1023 }
1024 #endif
1025 
1027  Value *CanonicalIV = State.get(getOperand(0), 0);
1028  Type *STy = CanonicalIV->getType();
1030  ElementCount VF = State.VF;
1031  Value *VStart = VF.isScalar()
1032  ? CanonicalIV
1033  : Builder.CreateVectorSplat(VF, CanonicalIV, "broadcast");
1034  for (unsigned Part = 0, UF = State.UF; Part < UF; ++Part) {
1035  Value *VStep = createStepForVF(Builder, STy, VF, Part);
1036  if (VF.isVector()) {
1037  VStep = Builder.CreateVectorSplat(VF, VStep);
1038  VStep =
1039  Builder.CreateAdd(VStep, Builder.CreateStepVector(VStep->getType()));
1040  }
1041  Value *CanonicalVectorIV = Builder.CreateAdd(VStart, VStep, "vec.iv");
1042  State.set(this, CanonicalVectorIV, Part);
1043  }
1044 }
1045 
1046 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
1048  VPSlotTracker &SlotTracker) const {
1049  O << Indent << "EMIT ";
1051  O << " = WIDEN-CANONICAL-INDUCTION ";
1053 }
1054 #endif
1055 
1057  auto &Builder = State.Builder;
1058  // Create a vector from the initial value.
1059  auto *VectorInit = getStartValue()->getLiveInIRValue();
1060 
1061  Type *VecTy = State.VF.isScalar()
1062  ? VectorInit->getType()
1063  : VectorType::get(VectorInit->getType(), State.VF);
1064 
1065  BasicBlock *VectorPH = State.CFG.getPreheaderBBFor(this);
1066  if (State.VF.isVector()) {
1067  auto *IdxTy = Builder.getInt32Ty();
1068  auto *One = ConstantInt::get(IdxTy, 1);
1070  Builder.SetInsertPoint(VectorPH->getTerminator());
1071  auto *RuntimeVF = getRuntimeVF(Builder, IdxTy, State.VF);
1072  auto *LastIdx = Builder.CreateSub(RuntimeVF, One);
1073  VectorInit = Builder.CreateInsertElement(
1074  PoisonValue::get(VecTy), VectorInit, LastIdx, "vector.recur.init");
1075  }
1076 
1077  // Create a phi node for the new recurrence.
1078  PHINode *EntryPart = PHINode::Create(
1079  VecTy, 2, "vector.recur", &*State.CFG.PrevBB->getFirstInsertionPt());
1080  EntryPart->addIncoming(VectorInit, VectorPH);
1081  State.set(this, EntryPart, 0);
1082 }
1083 
1084 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
1086  VPSlotTracker &SlotTracker) const {
1087  O << Indent << "FIRST-ORDER-RECURRENCE-PHI ";
1089  O << " = phi ";
1091 }
1092 #endif
1093 
1095  PHINode *PN = cast<PHINode>(getUnderlyingValue());
1096  auto &Builder = State.Builder;
1097 
1098  // In order to support recurrences we need to be able to vectorize Phi nodes.
1099  // Phi nodes have cycles, so we need to vectorize them in two stages. This is
1100  // stage #1: We create a new vector PHI node with no incoming edges. We'll use
1101  // this value when we vectorize all of the instructions that use the PHI.
1102  bool ScalarPHI = State.VF.isScalar() || IsInLoop;
1103  Type *VecTy =
1104  ScalarPHI ? PN->getType() : VectorType::get(PN->getType(), State.VF);
1105 
1106  BasicBlock *HeaderBB = State.CFG.PrevBB;
1107  assert(State.CurrentVectorLoop->getHeader() == HeaderBB &&
1108  "recipe must be in the vector loop header");
1109  unsigned LastPartForNewPhi = isOrdered() ? 1 : State.UF;
1110  for (unsigned Part = 0; Part < LastPartForNewPhi; ++Part) {
1111  Value *EntryPart =
1112  PHINode::Create(VecTy, 2, "vec.phi", &*HeaderBB->getFirstInsertionPt());
1113  State.set(this, EntryPart, Part);
1114  }
1115 
1116  BasicBlock *VectorPH = State.CFG.getPreheaderBBFor(this);
1117 
1118  // Reductions do not have to start at zero. They can start with
1119  // any loop invariant values.
1120  VPValue *StartVPV = getStartValue();
1121  Value *StartV = StartVPV->getLiveInIRValue();
1122 
1123  Value *Iden = nullptr;
1124  RecurKind RK = RdxDesc.getRecurrenceKind();
1127  // MinMax reduction have the start value as their identify.
1128  if (ScalarPHI) {
1129  Iden = StartV;
1130  } else {
1132  Builder.SetInsertPoint(VectorPH->getTerminator());
1133  StartV = Iden =
1134  Builder.CreateVectorSplat(State.VF, StartV, "minmax.ident");
1135  }
1136  } else {
1137  Iden = RdxDesc.getRecurrenceIdentity(RK, VecTy->getScalarType(),
1138  RdxDesc.getFastMathFlags());
1139 
1140  if (!ScalarPHI) {
1141  Iden = Builder.CreateVectorSplat(State.VF, Iden);
1143  Builder.SetInsertPoint(VectorPH->getTerminator());
1144  Constant *Zero = Builder.getInt32(0);
1145  StartV = Builder.CreateInsertElement(Iden, StartV, Zero);
1146  }
1147  }
1148 
1149  for (unsigned Part = 0; Part < LastPartForNewPhi; ++Part) {
1150  Value *EntryPart = State.get(this, Part);
1151  // Make sure to add the reduction start value only to the
1152  // first unroll part.
1153  Value *StartVal = (Part == 0) ? StartV : Iden;
1154  cast<PHINode>(EntryPart)->addIncoming(StartVal, VectorPH);
1155  }
1156 }
1157 
1158 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
1160  VPSlotTracker &SlotTracker) const {
1161  O << Indent << "WIDEN-REDUCTION-PHI ";
1162 
1164  O << " = phi ";
1166 }
1167 #endif
1168 
1171  "Non-native vplans are not expected to have VPWidenPHIRecipes.");
1172 
1173  // Currently we enter here in the VPlan-native path for non-induction
1174  // PHIs where all control flow is uniform. We simply widen these PHIs.
1175  // Create a vector phi with no operands - the vector phi operands will be
1176  // set at the end of vector code generation.
1177  VPBasicBlock *Parent = getParent();
1178  VPRegionBlock *LoopRegion = Parent->getEnclosingLoopRegion();
1179  unsigned StartIdx = 0;
1180  // For phis in header blocks of loop regions, use the index of the value
1181  // coming from the preheader.
1182  if (LoopRegion->getEntryBasicBlock() == Parent) {
1183  for (unsigned I = 0; I < getNumOperands(); ++I) {
1184  if (getIncomingBlock(I) ==
1185  LoopRegion->getSinglePredecessor()->getExitingBasicBlock())
1186  StartIdx = I;
1187  }
1188  }
1189  Value *Op0 = State.get(getOperand(StartIdx), 0);
1190  Type *VecTy = Op0->getType();
1191  Value *VecPhi = State.Builder.CreatePHI(VecTy, 2, "vec.phi");
1192  State.set(this, VecPhi, 0);
1193 }
1194 
1195 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
1197  VPSlotTracker &SlotTracker) const {
1198  O << Indent << "WIDEN-PHI ";
1199 
1200  auto *OriginalPhi = cast<PHINode>(getUnderlyingValue());
1201  // Unless all incoming values are modeled in VPlan print the original PHI
1202  // directly.
1203  // TODO: Remove once all VPWidenPHIRecipe instances keep all relevant incoming
1204  // values as VPValues.
1205  if (getNumOperands() != OriginalPhi->getNumOperands()) {
1206  O << VPlanIngredient(OriginalPhi);
1207  return;
1208  }
1209 
1211  O << " = phi ";
1213 }
1214 #endif
1215 
1216 // TODO: It would be good to use the existing VPWidenPHIRecipe instead and
1217 // remove VPActiveLaneMaskPHIRecipe.
1219  BasicBlock *VectorPH = State.CFG.getPreheaderBBFor(this);
1220  for (unsigned Part = 0, UF = State.UF; Part < UF; ++Part) {
1221  Value *StartMask = State.get(getOperand(0), Part);
1222  PHINode *EntryPart =
1223  State.Builder.CreatePHI(StartMask->getType(), 2, "active.lane.mask");
1224  EntryPart->addIncoming(StartMask, VectorPH);
1225  EntryPart->setDebugLoc(DL);
1226  State.set(this, EntryPart, Part);
1227  }
1228 }
1229 
1230 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
1232  VPSlotTracker &SlotTracker) const {
1233  O << Indent << "ACTIVE-LANE-MASK-PHI ";
1234 
1236  O << " = phi ";
1238 }
1239 #endif
llvm::VPBasicBlock::getRecipeList
RecipeListTy & getRecipeList()
Returns a reference to the list of recipes.
Definition: VPlan.h:2062
llvm::VPInstruction::getOpcode
unsigned getOpcode() const
Definition: VPlan.h:847
llvm::VPRecipeBase::moveAfter
void moveAfter(VPRecipeBase *MovePos)
Unlink this recipe from its current VPBasicBlock and insert it into the VPBasicBlock that MovePos liv...
Definition: VPlanRecipes.cpp:178
llvm::VPInstruction::SLPStore
@ SLPStore
Definition: VPlan.h:783
llvm::VPBlockBase::getSinglePredecessor
VPBlockBase * getSinglePredecessor() const
Definition: VPlan.h:498
llvm::VPReductionRecipe::print
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
Definition: VPlanRecipes.cpp:825
llvm::VPBlockBase::getPlan
VPlan * getPlan()
Definition: VPlan.cpp:134
llvm::VPWidenIntOrFpInductionRecipe::getStepValue
VPValue * getStepValue()
Returns the step value of the induction.
Definition: VPlan.h:1088
llvm::VPBlendRecipe::getNumIncomingValues
unsigned getNumIncomingValues() const
Return the number of incoming values, taking into account that a single incoming value has no mask.
Definition: VPlan.h:1388
llvm::VPBlendRecipe::getIncomingValue
VPValue * getIncomingValue(unsigned Idx) const
Return incoming value number Idx.
Definition: VPlan.h:1391
llvm
This is an optimization pass for GlobalISel generic memory operations.
Definition: AddressRanges.h:18
llvm::VPRecipeBase::getUnderlyingInstr
Instruction * getUnderlyingInstr()
Returns the underlying instruction, if the recipe is a VPValue or nullptr otherwise.
Definition: VPlan.h:716
llvm::Type::getInt1Ty
static IntegerType * getInt1Ty(LLVMContext &C)
Definition: Type.cpp:236
llvm::VPCanonicalIVPHIRecipe
Canonical scalar induction phi of the vector loop.
Definition: VPlan.h:1832
llvm::make_range
iterator_range< T > make_range(T x, T y)
Convenience function for iterating over sub-ranges.
Definition: iterator_range.h:53
llvm::VPTransformState::CFG
struct llvm::VPTransformState::CFGState CFG
llvm::VPRecipeBase::eraseFromParent
iplist< VPRecipeBase >::iterator eraseFromParent()
This method unlinks 'this' from the containing basic block and deletes it.
Definition: VPlanRecipes.cpp:173
llvm::VPLane::getFirstLane
static VPLane getFirstLane()
Definition: VPlan.h:132
llvm::VPWidenCanonicalIVRecipe::execute
void execute(VPTransformState &State) override
Generate a canonical vector induction variable of the vector loop, with start = {<Part*VF,...
Definition: VPlanRecipes.cpp:1026
llvm::VPPredInstPHIRecipe::execute
void execute(VPTransformState &State) override
Generates phi nodes for live-outs as needed to retain SSA form.
Definition: VPlanRecipes.cpp:897
llvm::VPBlendRecipe::print
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
Definition: VPlanRecipes.cpp:805
llvm::DataLayout
A parsed version of the target data layout string in and methods for querying it.
Definition: DataLayout.h:113
llvm::VPTransformState::Builder
IRBuilderBase & Builder
Hold a reference to the IRBuilder used to generate output IR code.
Definition: VPlan.h:355
ScalarEvolutionExpander.h
llvm::ElementCount
Definition: TypeSize.h:404
llvm::VPBasicBlock
VPBasicBlock serves as the leaf of the Hierarchical Control-Flow Graph.
Definition: VPlan.h:2015
llvm::SCEVExpander
This class uses information about analyze scalars to rewrite expressions in canonical form.
Definition: ScalarEvolutionExpander.h:50
llvm::VPWidenIntOrFpInductionRecipe::print
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
Definition: VPlanRecipes.cpp:627
llvm::VPValue::getLiveInIRValue
Value * getLiveInIRValue()
Returns the underlying IR value, if this VPValue is defined outside the scope of VPlan.
Definition: VPlanValue.h:188
llvm::Type::getScalarType
Type * getScalarType() const
If this is a vector type, return the element type, otherwise return 'this'.
Definition: Type.h:314
llvm::SmallVector
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
Definition: SmallVector.h:1183
llvm::VPCanonicalIVPHIRecipe::print
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
Definition: VPlanRecipes.cpp:979
llvm::VPDef::getVPSingleValue
VPValue * getVPSingleValue()
Returns the only VPValue defined by the VPDef.
Definition: VPlanValue.h:388
llvm::VPTransformState::get
Value * get(VPValue *Def, unsigned Part)
Get the generated Value for a given VPValue and a given Part.
Definition: LoopVectorize.cpp:9778
llvm::VPBranchOnMaskRecipe::execute
void execute(VPTransformState &State) override
Generate the extraction of the appropriate bit from the block mask and the conditional branch.
Definition: VPlanRecipes.cpp:871
llvm::IRBuilderBase::setFastMathFlags
void setFastMathFlags(FastMathFlags NewFMF)
Set the fast-math flags to be used with generated fp-math operators.
Definition: IRBuilder.h:288
llvm::IRBuilder
This provides a uniform API for creating instructions and inserting them into a basic block: either a...
Definition: IRBuilder.h:2507
llvm::VPWidenPointerInductionRecipe::onlyScalarsGenerated
bool onlyScalarsGenerated(ElementCount VF)
Returns true if only scalar values will be generated.
Definition: VPlanRecipes.cpp:987
llvm::VPInstruction::setFastMathFlags
void setFastMathFlags(FastMathFlags FMFNew)
Set the fast-math flags.
Definition: VPlanRecipes.cpp:427
llvm::VPTransformState::set
void set(VPValue *Def, Value *V, unsigned Part)
Set the generated Value for a given VPValue and a given Part.
Definition: VPlan.h:260
llvm::RecurrenceDescriptor::getOpcode
static unsigned getOpcode(RecurKind Kind)
Returns the opcode corresponding to the RecurrenceKind.
Definition: IVDescriptors.cpp:1102
llvm::VPTransformState::CFGState::getPreheaderBBFor
BasicBlock * getPreheaderBBFor(VPRecipeBase *R)
Returns the BasicBlock* mapped to the pre-header of the loop region containing R.
Definition: VPlan.cpp:225
llvm::VPBlendRecipe::execute
void execute(VPTransformState &State) override
Generate the phi/select nodes.
Definition: VPlanRecipes.cpp:765
llvm::interleaveComma
void interleaveComma(const Container &c, StreamT &os, UnaryFunctor each_fn)
Definition: STLExtras.h:1913
llvm::Type
The instances of the Type class are immutable: once they are created, they are never changed.
Definition: Type.h:45
llvm::VPWidenIntOrFpInductionRecipe::getTruncInst
TruncInst * getTruncInst()
Returns the first defined value as TruncInst, if it is one or nullptr otherwise.
Definition: VPlan.h:1093
llvm::VPInstruction::SLPLoad
@ SLPLoad
Definition: VPlan.h:782
llvm::VPInstruction::FirstOrderRecurrenceSplice
@ FirstOrderRecurrenceSplice
Definition: VPlan.h:777
llvm::VPUser::op_begin
operand_iterator op_begin()
Definition: VPlanValue.h:278
llvm::VPWidenMemoryInstructionRecipe::print
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
Definition: VPlanRecipes.cpp:952
llvm::VPInstruction::Not
@ Not
Definition: VPlan.h:780
llvm::iplist_impl::remove
pointer remove(iterator &IT)
Definition: ilist.h:252
llvm::VPInstruction::ICmpULE
@ ICmpULE
Definition: VPlan.h:781
llvm::BasicBlock::getSinglePredecessor
const BasicBlock * getSinglePredecessor() const
Return the predecessor of this block if it has a single predecessor block.
Definition: BasicBlock.cpp:261
STLExtras.h
llvm::VPUser::operands
operand_range operands()
Definition: VPlanValue.h:282
llvm::VPRegionBlock::getEntry
const VPBlockBase * getEntry() const
Definition: VPlan.h:2178
llvm::FastMathFlags
Convenience struct for specifying and reasoning about fast-math flags.
Definition: FMF.h:21
llvm::VPTransformState::hasScalarValue
bool hasScalarValue(VPValue *Def, VPIteration Instance)
Definition: VPlan.h:249
llvm::LinearPolySize::isScalable
bool isScalable() const
Returns whether the size is scaled by a runtime quantity (vscale).
Definition: TypeSize.h:298
llvm::VPBasicBlock::getEnclosingLoopRegion
VPRegionBlock * getEnclosingLoopRegion()
Definition: VPlan.cpp:417
llvm::iplist
An intrusive list with ownership and callbacks specified/controlled by ilist_traits,...
Definition: ilist.h:391
llvm::VPScalarIVStepsRecipe::isCanonical
bool isCanonical() const
Returns true if the induction is canonical, i.e.
Definition: VPlanRecipes.cpp:653
llvm::VPValue::users
user_range users()
Definition: VPlanValue.h:163
LLVM_DEBUG
#define LLVM_DEBUG(X)
Definition: Debug.h:101
llvm::VPWidenPHIRecipe::execute
void execute(VPTransformState &State) override
Generate the phi/select nodes.
Definition: VPlanRecipes.cpp:1169
llvm::VPScalarIVStepsRecipe::getCanonicalIV
VPCanonicalIVPHIRecipe * getCanonicalIV() const
Definition: VPlanRecipes.cpp:649
llvm::BranchInst::setSuccessor
void setSuccessor(unsigned idx, BasicBlock *NewSucc)
Definition: Instructions.h:3197
llvm::iplist_impl::erase
iterator erase(iterator where)
Definition: ilist.h:268
llvm::BasicBlock
LLVM Basic Block Representation.
Definition: BasicBlock.h:55
llvm::VPLiveOut::fixPhi
void fixPhi(VPlan &Plan, VPTransformState &State)
Fixup the wrapped LCSSA phi node in the unique exit block.
Definition: VPlanRecipes.cpp:134
llvm::VPWidenIntOrFpInductionRecipe::getInductionDescriptor
const InductionDescriptor & getInductionDescriptor() const
Returns the induction descriptor for the recipe.
Definition: VPlan.h:1103
llvm::RecurKind
RecurKind
These are the kinds of recurrences that we support.
Definition: IVDescriptors.h:35
llvm::VPRecipeBase::mayWriteToMemory
bool mayWriteToMemory() const
Returns true if the recipe may write to memory.
Definition: VPlanRecipes.cpp:42
llvm::dbgs
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
Definition: Debug.cpp:163
llvm::ElementCount::isScalar
bool isScalar() const
Counting predicates.
Definition: TypeSize.h:414
llvm::VPPredInstPHIRecipe::print
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
Definition: VPlanRecipes.cpp:944
Instruction.h
llvm::VPTransformState
VPTransformState holds information passed down when "executing" a VPlan, needed for generating the ou...
Definition: VPlan.h:201
CommandLine.h
llvm::VPReplicateRecipe::print
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
Definition: VPlanRecipes.cpp:846
llvm::Instruction::getOpcode
unsigned getOpcode() const
Returns a member of one of the enums like Instruction::Add.
Definition: Instruction.h:157
llvm::all_of
bool all_of(R &&range, UnaryPredicate P)
Provide wrappers to std::all_of which take ranges instead of having to pass begin/end explicitly.
Definition: STLExtras.h:1618
llvm::VPWidenPHIRecipe::getIncomingBlock
VPBasicBlock * getIncomingBlock(unsigned I)
Returns the I th incoming VPBasicBlock.
Definition: VPlan.h:1272
llvm::createStepForVF
Value * createStepForVF(IRBuilderBase &B, Type *Ty, ElementCount VF, int64_t Step)
Return a value for Step multiplied by VF.
Definition: LoopVectorize.cpp:941
E
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
llvm::IRBuilderBase::CreateGEP
Value * CreateGEP(Type *Ty, Value *Ptr, ArrayRef< Value * > IdxList, const Twine &Name="", bool IsInBounds=false)
Definition: IRBuilder.h:1740
llvm::VPInstruction::ActiveLaneMask
@ ActiveLaneMask
Definition: VPlan.h:784
llvm::RecurrenceDescriptor::isSelectCmpRecurrenceKind
static bool isSelectCmpRecurrenceKind(RecurKind Kind)
Returns true if the recurrence kind is of the form select(cmp(),x,y) where one of (x,...
Definition: IVDescriptors.h:236
C
(vector float) vec_cmpeq(*A, *B) C
Definition: README_ALTIVEC.txt:86
llvm::VPInstruction::CanonicalIVIncrementForPart
@ CanonicalIVIncrementForPart
Definition: VPlan.h:789
Twine.h
getOpcode
static Optional< unsigned > getOpcode(ArrayRef< VPValue * > Values)
Returns the opcode of Values or ~0 if they do not all agree.
Definition: VPlanSLP.cpp:190
llvm::VPRecipeBase::moveBefore
void moveBefore(VPBasicBlock &BB, iplist< VPRecipeBase >::iterator I)
Unlink this recipe and insert into BB before I.
Definition: VPlanRecipes.cpp:183
llvm::ReplaceInstWithInst
void ReplaceInstWithInst(BasicBlock::InstListType &BIL, BasicBlock::iterator &BI, Instruction *I)
Replace the instruction specified by BI with the instruction specified by I.
Definition: BasicBlockUtils.cpp:478
llvm::Type::isVectorTy
bool isVectorTy() const
True if this is an instance of VectorType.
Definition: Type.h:232
llvm::InsertElementInst
This instruction inserts a single (scalar) element into a VectorType value.
Definition: Instructions.h:1936
llvm::BasicBlock::getFirstInsertionPt
const_iterator getFirstInsertionPt() const
Returns an iterator to the first instruction in this block that is suitable for inserting a non-PHI i...
Definition: BasicBlock.cpp:246
B
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
llvm::SmallBitVector::all
bool all() const
Returns true if all bits are set.
Definition: SmallBitVector.h:216
llvm::VPWidenCanonicalIVRecipe::print
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
Definition: VPlanRecipes.cpp:1047
llvm::Instruction::getOpcodeName
const char * getOpcodeName() const
Definition: Instruction.h:159
llvm::RecurrenceDescriptor::getRecurrenceIdentity
Value * getRecurrenceIdentity(RecurKind K, Type *Tp, FastMathFlags FMF) const
Returns identity corresponding to the RecurrenceKind.
Definition: IVDescriptors.cpp:1051
llvm::Instruction
Definition: Instruction.h:42
EnableVPlanNativePath
cl::opt< bool > EnableVPlanNativePath
llvm::RecurrenceDescriptor::getRecurrenceKind
RecurKind getRecurrenceKind() const
Definition: IVDescriptors.h:195
llvm::VPInstruction::BranchOnCond
@ BranchOnCond
Definition: VPlan.h:792
llvm::raw_ostream
This class implements an extremely fast bulk output stream that can only output to a stream.
Definition: raw_ostream.h:52
llvm::VPScalarIVStepsRecipe::getStartValue
VPValue * getStartValue() const
Definition: VPlan.h:2001
llvm::VPValue
Definition: VPlanValue.h:44
llvm::ConstantInt::get
static Constant * get(Type *Ty, uint64_t V, bool IsSigned=false)
If Ty is a vector type, return a Constant with a splat of the given value.
Definition: Constants.cpp:879
llvm::BasicBlock::getModule
const Module * getModule() const
Return the module owning the function this basic block belongs to, or nullptr if the function does no...
Definition: BasicBlock.cpp:147
VPlan.h
llvm::MCID::Call
@ Call
Definition: MCInstrDesc.h:155
IVDescriptors.h
llvm::VPUser::getNumOperands
unsigned getNumOperands() const
Definition: VPlanValue.h:256
Type.h
llvm::VPRegionBlock
VPRegionBlock represents a collection of VPBasicBlocks and VPRegionBlocks which form a Single-Entry-S...
Definition: VPlan.h:2139
llvm::VPWidenMemoryInstructionRecipe::isStore
bool isStore() const
Returns true if this recipe is a store.
Definition: VPlan.h:1760
llvm::VPTransformState::setDebugLocFromInst
void setDebugLocFromInst(const Value *V)
Set the debug location in the builder using the debug location in V.
Definition: VPlan.cpp:250
llvm::IRBuilderBase::FastMathFlagGuard
Definition: IRBuilder.h:372
llvm::tgtok::In
@ In
Definition: TGLexer.h:51
llvm::VPActiveLaneMaskPHIRecipe::execute
void execute(VPTransformState &State) override
Generate the active lane mask phi of the vector loop.
Definition: VPlanRecipes.cpp:1218
BasicBlock.h
llvm::cl::opt< bool >
llvm::VPRecipeBase::removeFromParent
void removeFromParent()
This method unlinks 'this' from the containing basic block, but does not delete it.
Definition: VPlanRecipes.cpp:167
llvm::RISCVFenceField::O
@ O
Definition: RISCVBaseInfo.h:240
llvm::VPRecipeBase::mayHaveSideEffects
bool mayHaveSideEffects() const
Returns true if the recipe may have side-effects.
Definition: VPlanRecipes.cpp:104
llvm::Constant
This is an important base class in LLVM.
Definition: Constant.h:41
llvm::IRBuilderBase::GetInsertPoint
BasicBlock::iterator GetInsertPoint() const
Definition: IRBuilder.h:174
llvm::VPExpandSCEVRecipe::execute
void execute(VPTransformState &State) override
Generate a canonical vector induction variable of the vector loop, with.
Definition: VPlanRecipes.cpp:1005
llvm::VPReductionRecipe::getCondOp
VPValue * getCondOp() const
The VPValue of the condition for the block.
Definition: VPlan.h:1538
llvm::VPlan::isUniformAfterVectorization
bool isUniformAfterVectorization(VPValue *VPV) const
Returns true if VPV is uniform after vectorization.
Definition: VPlan.h:2684
llvm::VPWidenGEPRecipe::execute
void execute(VPTransformState &State) override
Generate the gep nodes.
Definition: VPlanRecipes.cpp:676
llvm::VPInstruction::execute
void execute(VPTransformState &State) override
Generate the instruction.
Definition: VPlanRecipes.cpp:349
llvm::VPReductionPHIRecipe::print
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
Definition: VPlanRecipes.cpp:1159
llvm::IRBuilderBase::getInt32
ConstantInt * getInt32(uint32_t C)
Get a constant 32-bit value.
Definition: IRBuilder.h:461
llvm::PHINode::addIncoming
void addIncoming(Value *V, BasicBlock *BB)
Add an incoming value to the end of the PHI list.
Definition: Instructions.h:2814
llvm::VPLane::getLastLaneForVF
static VPLane getLastLaneForVF(const ElementCount &VF)
Definition: VPlan.h:134
llvm::BranchInst::Create
static BranchInst * Create(BasicBlock *IfTrue, Instruction *InsertBefore=nullptr)
Definition: Instructions.h:3155
llvm::VPTransformState::UF
unsigned UF
Definition: VPlan.h:210
llvm::VPInstruction::print
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the VPInstruction to O.
Definition: VPlanRecipes.cpp:363
I
#define I(x, y, z)
Definition: MD5.cpp:58
llvm::VPIteration
VPIteration represents a single point in the iteration space of the output (vectorized and/or unrolle...
Definition: VPlan.h:184
llvm::VPHeaderPHIRecipe::getStartValue
VPValue * getStartValue()
Returns the start value of the phi, if one is set.
Definition: VPlan.h:1163
llvm::IRBuilderBase::CreateSelect
Value * CreateSelect(Value *C, Value *True, Value *False, const Twine &Name="", Instruction *MDFrom=nullptr)
Definition: IRBuilder.cpp:1063
llvm::VPBlendRecipe::getMask
VPValue * getMask(unsigned Idx) const
Return mask number Idx.
Definition: VPlan.h:1394
llvm::VPWidenGEPRecipe::print
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
Definition: VPlanRecipes.cpp:750
llvm::Instruction::setDebugLoc
void setDebugLoc(DebugLoc Loc)
Set the debug location information for this instruction.
Definition: Instruction.h:344
IRBuilder.h
assert
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
llvm::IRBuilderBase::CreateExtractElement
Value * CreateExtractElement(Value *Vec, Value *Idx, const Twine &Name="")
Definition: IRBuilder.h:2298
llvm::VPInstruction::hasResult
bool hasResult() const
Definition: VPlan.h:871
Ptr
@ Ptr
Definition: TargetLibraryInfo.cpp:60
llvm::Instruction::isBinaryOp
bool isBinaryOp() const
Definition: Instruction.h:162
llvm::Type::isVoidTy
bool isVoidTy() const
Return true if this is 'void'.
Definition: Type.h:139
llvm::IRBuilderBase::getTrue
ConstantInt * getTrue()
Get the constant value for i1 true.
Definition: IRBuilder.h:441
llvm::VPUser::printOperands
void printOperands(raw_ostream &O, VPSlotTracker &SlotTracker) const
Print the operands to O.
Definition: VPlan.cpp:1010
llvm::VPWidenRecipe::print
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
Definition: VPlanRecipes.cpp:619
llvm::ElementCount::isVector
bool isVector() const
One or more elements.
Definition: TypeSize.h:416
Builder
assume Assume Builder
Definition: AssumeBundleBuilder.cpp:651
llvm::Value::printAsOperand
void printAsOperand(raw_ostream &O, bool PrintType=true, const Module *M=nullptr) const
Print the name of this Value out to the specified raw_ostream.
Definition: AsmWriter.cpp:4671
llvm::VPWidenIntOrFpInductionRecipe::isCanonical
bool isCanonical() const
Returns true if the induction is canonical, i.e.
Definition: VPlanRecipes.cpp:643
llvm::VPWidenCallRecipe::print
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
Definition: VPlanRecipes.cpp:438
llvm::IRBuilderBase::CreatePHI
PHINode * CreatePHI(Type *Ty, unsigned NumReservedValues, const Twine &Name="")
Definition: IRBuilder.h:2236
llvm::VPTransformState::reset
void reset(VPValue *Def, Value *V, unsigned Part)
Reset an existing vector value for Def and a given Part.
Definition: VPlan.h:268
llvm::VPScalarIVStepsRecipe::print
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
Definition: VPlanRecipes.cpp:667
llvm::VPTransformState::addMetadata
void addMetadata(Instruction *To, Instruction *From)
Add metadata from one instruction to another.
Definition: VPlan.cpp:238
llvm::getRuntimeVF
Value * getRuntimeVF(IRBuilderBase &B, Type *Ty, ElementCount VF)
Return the runtime value for VF.
Definition: LoopVectorize.cpp:949
llvm::VPReductionRecipe::getVecOp
VPValue * getVecOp() const
The VPValue of the vector value to be reduced.
Definition: VPlan.h:1536
llvm::Instruction::getFastMathFlags
FastMathFlags getFastMathFlags() const
Convenience function for getting all the fast-math flags, which must be an operator which supports th...
Definition: Instruction.cpp:289
Mul
BinaryOperator * Mul
Definition: X86PartialReduction.cpp:70
llvm::RecurrenceDescriptor::IntermediateStore
StoreInst * IntermediateStore
Reductions may store temporary or final result to an invariant address.
Definition: IVDescriptors.h:273
Cond
SmallVector< MachineOperand, 4 > Cond
Definition: BasicBlockSections.cpp:137
llvm::RecurrenceDescriptor::getFastMathFlags
FastMathFlags getFastMathFlags() const
Definition: IVDescriptors.h:199
llvm_unreachable
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
Definition: ErrorHandling.h:143
llvm::Value::getType
Type * getType() const
All values are typed, get the type of this value.
Definition: Value.h:255
llvm::InductionDescriptor::getStep
const SCEV * getStep() const
Definition: IVDescriptors.h:319
llvm::VPRecipeBase::getParent
VPBasicBlock * getParent()
Definition: VPlan.h:678
llvm::IRBuilderBase::InsertPointGuard
Definition: IRBuilder.h:350
llvm::IRBuilderBase
Common base class shared among various IRBuilders.
Definition: IRBuilder.h:93
llvm::ilist_node_impl::getIterator
self_iterator getIterator()
Definition: ilist_node.h:82
DL
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
Definition: AArch64SLSHardening.cpp:76
llvm::VPCanonicalIVPHIRecipe::execute
void execute(VPTransformState &State) override
Generate the canonical scalar induction phi of the vector loop.
Definition: VPlanRecipes.cpp:966
llvm::VPScalarIVStepsRecipe::getStepValue
VPValue * getStepValue() const
Definition: VPlan.h:2002
llvm::VPWidenSelectRecipe::execute
void execute(VPTransformState &State) override
Produce a widened version of the select instruction.
Definition: VPlanRecipes.cpp:469
llvm::VPInstruction::CanonicalIVIncrement
@ CanonicalIVIncrement
Definition: VPlan.h:785
llvm::VPTransformState::VF
ElementCount VF
The chosen Vectorization and Unroll Factors of the loop being vectorized.
Definition: VPlan.h:209
llvm::VPBlockBase::getEntryBasicBlock
const VPBasicBlock * getEntryBasicBlock() const
Definition: VPlan.cpp:139
llvm::vputils::onlyFirstLaneUsed
bool onlyFirstLaneUsed(VPValue *Def)
Returns true if only the first lane of Def is used.
Definition: VPlan.cpp:1093
llvm::VPDef::getVPValue
VPValue * getVPValue(unsigned I)
Returns the VPValue with index I defined by the VPDef.
Definition: VPlanValue.h:400
llvm::VPInstruction::CanonicalIVIncrementForPartNUW
@ CanonicalIVIncrementForPartNUW
Definition: VPlan.h:790
llvm::VPFirstOrderRecurrencePHIRecipe::print
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
Definition: VPlanRecipes.cpp:1085
llvm::MCID::Select
@ Select
Definition: MCInstrDesc.h:164
llvm::VPReductionPHIRecipe::execute
void execute(VPTransformState &State) override
Generate the phi/select nodes.
Definition: VPlanRecipes.cpp:1094
llvm::NVPTX::PTXLdStInstCode::V2
@ V2
Definition: NVPTX.h:123
llvm::Twine
Twine - A lightweight data structure for efficiently representing the concatenation of temporary valu...
Definition: Twine.h:81
llvm::VPSlotTracker
This class can be used to assign consecutive numbers to all VPValues in a VPlan and allows querying t...
Definition: VPlanValue.h:438
llvm::VPBlockBase::getParent
VPRegionBlock * getParent()
Definition: VPlan.h:457
llvm::VPTransformState::CurrentVectorLoop
Loop * CurrentVectorLoop
The loop object for the current parent region, or nullptr.
Definition: VPlan.h:373
llvm::AMDGPU::SendMsg::Op
Op
Definition: SIDefines.h:348
llvm::VPTransformState::MayGeneratePoisonRecipes
SmallPtrSet< VPRecipeBase *, 16 > MayGeneratePoisonRecipes
Holds recipes that may generate a poison value that is used after vectorization, even when their oper...
Definition: VPlan.h:370
llvm::PHINode::Create
static PHINode * Create(Type *Ty, unsigned NumReservedValues, const Twine &NameStr="", Instruction *InsertBefore=nullptr)
Constructors - NumReservedValues is a hint for the number of incoming edges that this phi node will h...
Definition: Instructions.h:2706
llvm::IRBuilderBase::Insert
InstTy * Insert(InstTy *I, const Twine &Name="") const
Insert and return the specified instruction.
Definition: IRBuilder.h:144
llvm::VPRecipeBase
VPRecipeBase is a base class modeling a sequence of one or more output IR instructions.
Definition: VPlan.h:659
llvm::IRBuilderBase::GetInsertBlock
BasicBlock * GetInsertBlock() const
Definition: IRBuilder.h:173
Casting.h
llvm::LoopBase::getHeader
BlockT * getHeader() const
Definition: LoopInfo.h:104
llvm::DebugLoc::print
void print(raw_ostream &OS) const
prints source location /path/to/file.exe:line:col @[inlined at]
Definition: DebugLoc.cpp:103
llvm::SlotTracker
This class provides computation of slot numbers for LLVM Assembly writing.
Definition: AsmWriter.cpp:658
llvm::VPInstruction::BranchOnCount
@ BranchOnCount
Definition: VPlan.h:791
llvm::VPInstruction::CanonicalIVIncrementNUW
@ CanonicalIVIncrementNUW
Definition: VPlan.h:786
llvm::MCID::Add
@ Add
Definition: MCInstrDesc.h:185
llvm::VPWidenPointerInductionRecipe::print
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
Definition: VPlanRecipes.cpp:995
llvm::iplist_impl::insertAfter
iterator insertAfter(iterator where, pointer New)
Definition: ilist.h:238
llvm::iplist_impl::insert
iterator insert(iterator where, pointer New)
Definition: ilist.h:229
llvm::VPUser
This class augments VPValue with operands which provide the inverse def-use edges from VPValue's user...
Definition: VPlanValue.h:207
llvm::SmallBitVector::size
size_type size() const
Returns the number of bits in this bitvector.
Definition: SmallBitVector.h:195
llvm::Instruction::BinaryOps
BinaryOps
Definition: Instruction.h:766
llvm::InsertElementInst::getType
VectorType * getType() const
Overload to return most specific vector type.
Definition: Instructions.h:1969
Instructions.h
SmallVector.h
llvm::VPUser::getOperand
VPValue * getOperand(unsigned N) const
Definition: VPlanValue.h:257
llvm::VPTransformState::Instance
Optional< VPIteration > Instance
Hold the indices to generate specific scalar instructions.
Definition: VPlan.h:215
llvm::VPRecipeBase::mayReadFromMemory
bool mayReadFromMemory() const
Returns true if the recipe may read from memory.
Definition: VPlanRecipes.cpp:73
llvm::Instruction::getParent
const BasicBlock * getParent() const
Definition: Instruction.h:91
llvm::VPActiveLaneMaskPHIRecipe::print
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
Definition: VPlanRecipes.cpp:1231
llvm::PHINode
Definition: Instructions.h:2664
llvm::BasicBlock::getTerminator
const Instruction * getTerminator() const LLVM_READONLY
Returns the terminator instruction if the block is well formed or null if the block is not well forme...
Definition: BasicBlock.h:119
llvm::VPBlockBase::getExitingBasicBlock
const VPBasicBlock * getExitingBasicBlock() const
Definition: VPlan.cpp:160
llvm::VPFirstOrderRecurrencePHIRecipe::execute
void execute(VPTransformState &State) override
Generate the phi nodes.
Definition: VPlanRecipes.cpp:1056
IV
static const uint32_t IV[8]
Definition: blake3_impl.h:85
llvm::VPTransformState::CFGState::VPBB2IRBB
SmallDenseMap< VPBasicBlock *, BasicBlock * > VPBB2IRBB
A mapping of each VPBasicBlock to the corresponding BasicBlock.
Definition: VPlan.h:339
llvm::IRBuilderBase::CreateVectorSplat
Value * CreateVectorSplat(unsigned NumElts, Value *V, const Twine &Name="")
Return a vector value that contains.
Definition: IRBuilder.cpp:1186
llvm::Module::getDataLayout
const DataLayout & getDataLayout() const
Get the data layout for the module's target platform.
Definition: Module.cpp:398
llvm::SCEV::getType
Type * getType() const
Return the LLVM type of this SCEV expression.
Definition: ScalarEvolution.cpp:392
llvm::VPUser::usesScalars
virtual bool usesScalars(const VPValue *Op) const
Returns true if the VPUser uses scalars of operand Op.
Definition: VPlanValue.h:292
llvm::VPBranchOnMaskRecipe::getMask
VPValue * getMask() const
Return the mask used by this recipe.
Definition: VPlan.h:1648
BB
Common register allocation spilling lr str ldr sxth r3 ldr mla r4 can lr mov lr str ldr sxth r3 mla r4 and then merge mul and lr str ldr sxth r3 mla r4 It also increase the likelihood the store may become dead bb27 Successors according to LLVM BB
Definition: README.txt:39
GEP
Hexagon Common GEP
Definition: HexagonCommonGEP.cpp:171
llvm::VPWidenPHIRecipe::print
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
Definition: VPlanRecipes.cpp:1196
llvm::User::getOperand
Value * getOperand(unsigned i) const
Definition: User.h:169
llvm::RecurrenceDescriptor::isMinMaxRecurrenceKind
static bool isMinMaxRecurrenceKind(RecurKind Kind)
Returns true if the recurrence kind is any min/max kind.
Definition: IVDescriptors.h:230
llvm::BranchInst
Conditional or Unconditional Branch instruction.
Definition: Instructions.h:3099
raw_ostream.h
llvm::VPReductionRecipe::getChainOp
VPValue * getChainOp() const
The VPValue of the scalar Chain being accumulated.
Definition: VPlan.h:1534
llvm::VPExpandSCEVRecipe::print
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
Definition: VPlanRecipes.cpp:1018
BasicBlockUtils.h
llvm::VPWidenRecipe::execute
void execute(VPTransformState &State) override
Produce widened copies of all Ingredients.
Definition: VPlanRecipes.cpp:490
llvm::VPDef::getVPDefID
unsigned getVPDefID() const
Definition: VPlanValue.h:420
llvm::VPInstruction::dump
LLVM_DUMP_METHOD void dump() const
Print the VPInstruction to dbgs() (for debugging).
Definition: VPlanRecipes.cpp:358
Value.h
llvm::VPValue::getUnderlyingValue
Value * getUnderlyingValue()
Return the underlying Value attached to this VPValue.
Definition: VPlanValue.h:84
llvm::VPlan
VPlan models a candidate for vectorization, encoding various decisions take to produce efficient outp...
Definition: VPlan.h:2494
llvm::Value
LLVM Value Representation.
Definition: Value.h:74
llvm::VPRecipeBase::insertBefore
void insertBefore(VPRecipeBase *InsertPos)
Insert an unlinked recipe into a basic block immediately before the specified recipe.
Definition: VPlanRecipes.cpp:143
Debug.h
llvm::VectorType::get
static VectorType * get(Type *ElementType, ElementCount EC)
This static method is the primary way to construct an VectorType.
Definition: Type.cpp:668
llvm::VPReductionPHIRecipe::isOrdered
bool isOrdered() const
Returns true, if the phi is part of an ordered reduction.
Definition: VPlan.h:1357
llvm::VPlanIngredient
Definition: VPlan.h:2786
llvm::VPWidenSelectRecipe::print
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
Definition: VPlanRecipes.cpp:455
llvm::VPValue::printAsOperand
void printAsOperand(raw_ostream &OS, VPSlotTracker &Tracker) const
Definition: VPlan.cpp:995
llvm::VPWidenIntOrFpInductionRecipe::getStartValue
VPValue * getStartValue()
Returns the start value of the induction.
Definition: VPlan.h:1084
llvm::VPTransformState::CFGState::PrevBB
BasicBlock * PrevBB
The previous IR BasicBlock created or used.
Definition: VPlan.h:331
llvm::VPRecipeBase::insertAfter
void insertAfter(VPRecipeBase *InsertPos)
Insert an unlinked Recipe into a basic block immediately after the specified Recipe.
Definition: VPlanRecipes.cpp:159
llvm::VPTransformState::hasVectorValue
bool hasVectorValue(VPValue *Def, unsigned Part)
Definition: VPlan.h:239
llvm::PoisonValue::get
static PoisonValue * get(Type *T)
Static factory methods - Return an 'poison' object of the specified type.
Definition: Constants.cpp:1729