LLVM  16.0.0git
VPlanRecipes.cpp
Go to the documentation of this file.
1 //===- VPlanRecipes.cpp - Implementations for VPlan recipes ---------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 ///
9 /// \file
10 /// This file contains implementations for different VPlan recipes.
11 ///
12 //===----------------------------------------------------------------------===//
13 
14 #include "VPlan.h"
15 #include "llvm/ADT/STLExtras.h"
16 #include "llvm/ADT/SmallVector.h"
17 #include "llvm/ADT/Twine.h"
19 #include "llvm/IR/BasicBlock.h"
20 #include "llvm/IR/IRBuilder.h"
21 #include "llvm/IR/Instruction.h"
22 #include "llvm/IR/Instructions.h"
23 #include "llvm/IR/Type.h"
24 #include "llvm/IR/Value.h"
25 #include "llvm/Support/Casting.h"
27 #include "llvm/Support/Debug.h"
31 #include <cassert>
32 
33 using namespace llvm;
34 
36 
38 
39 #define LV_NAME "loop-vectorize"
40 #define DEBUG_TYPE LV_NAME
41 
43  switch (getVPDefID()) {
44  case VPWidenMemoryInstructionSC: {
45  return cast<VPWidenMemoryInstructionRecipe>(this)->isStore();
46  }
47  case VPReplicateSC:
48  case VPWidenCallSC:
49  return cast<Instruction>(getVPSingleValue()->getUnderlyingValue())
50  ->mayWriteToMemory();
51  case VPBranchOnMaskSC:
52  case VPScalarIVStepsSC:
53  return false;
54  case VPWidenIntOrFpInductionSC:
55  case VPWidenCanonicalIVSC:
56  case VPWidenPHISC:
57  case VPBlendSC:
58  case VPWidenSC:
59  case VPWidenGEPSC:
60  case VPReductionSC:
61  case VPWidenSelectSC: {
62  const Instruction *I =
63  dyn_cast_or_null<Instruction>(getVPSingleValue()->getUnderlyingValue());
64  (void)I;
65  assert((!I || !I->mayWriteToMemory()) &&
66  "underlying instruction may write to memory");
67  return false;
68  }
69  default:
70  return true;
71  }
72 }
73 
75  switch (getVPDefID()) {
76  case VPWidenMemoryInstructionSC: {
77  return !cast<VPWidenMemoryInstructionRecipe>(this)->isStore();
78  }
79  case VPReplicateSC:
80  case VPWidenCallSC:
81  return cast<Instruction>(getVPSingleValue()->getUnderlyingValue())
82  ->mayReadFromMemory();
83  case VPBranchOnMaskSC:
84  case VPScalarIVStepsSC:
85  return false;
86  case VPWidenIntOrFpInductionSC:
87  case VPWidenCanonicalIVSC:
88  case VPWidenPHISC:
89  case VPBlendSC:
90  case VPWidenSC:
91  case VPWidenGEPSC:
92  case VPReductionSC:
93  case VPWidenSelectSC: {
94  const Instruction *I =
95  dyn_cast_or_null<Instruction>(getVPSingleValue()->getUnderlyingValue());
96  (void)I;
97  assert((!I || !I->mayReadFromMemory()) &&
98  "underlying instruction may read from memory");
99  return false;
100  }
101  default:
102  return true;
103  }
104 }
105 
107  switch (getVPDefID()) {
108  case VPDerivedIVSC:
109  case VPPredInstPHISC:
110  return false;
111  case VPWidenIntOrFpInductionSC:
112  case VPWidenPointerInductionSC:
113  case VPWidenCanonicalIVSC:
114  case VPWidenPHISC:
115  case VPBlendSC:
116  case VPWidenSC:
117  case VPWidenGEPSC:
118  case VPReductionSC:
119  case VPWidenSelectSC:
120  case VPScalarIVStepsSC: {
121  const Instruction *I =
122  dyn_cast_or_null<Instruction>(getVPSingleValue()->getUnderlyingValue());
123  (void)I;
124  assert((!I || !I->mayHaveSideEffects()) &&
125  "underlying instruction has side-effects");
126  return false;
127  }
128  case VPReplicateSC: {
129  auto *R = cast<VPReplicateRecipe>(this);
130  return R->getUnderlyingInstr()->mayHaveSideEffects();
131  }
132  default:
133  return true;
134  }
135 }
136 
138  auto Lane = VPLane::getLastLaneForVF(State.VF);
139  VPValue *ExitValue = getOperand(0);
141  Lane = VPLane::getFirstLane();
142  Phi->addIncoming(State.get(ExitValue, VPIteration(State.UF - 1, Lane)),
143  State.Builder.GetInsertBlock());
144 }
145 
147  assert(!Parent && "Recipe already in some VPBasicBlock");
148  assert(InsertPos->getParent() &&
149  "Insertion position not in any VPBasicBlock");
150  Parent = InsertPos->getParent();
151  Parent->getRecipeList().insert(InsertPos->getIterator(), this);
152 }
153 
156  assert(!Parent && "Recipe already in some VPBasicBlock");
157  assert(I == BB.end() || I->getParent() == &BB);
158  Parent = &BB;
159  BB.getRecipeList().insert(I, this);
160 }
161 
163  assert(!Parent && "Recipe already in some VPBasicBlock");
164  assert(InsertPos->getParent() &&
165  "Insertion position not in any VPBasicBlock");
166  Parent = InsertPos->getParent();
167  Parent->getRecipeList().insertAfter(InsertPos->getIterator(), this);
168 }
169 
171  assert(getParent() && "Recipe not in any VPBasicBlock");
173  Parent = nullptr;
174 }
175 
177  assert(getParent() && "Recipe not in any VPBasicBlock");
178  return getParent()->getRecipeList().erase(getIterator());
179 }
180 
183  insertAfter(InsertPos);
184 }
185 
189  insertBefore(BB, I);
190 }
191 
192 void VPInstruction::generateInstruction(VPTransformState &State,
193  unsigned Part) {
194  IRBuilderBase &Builder = State.Builder;
195  Builder.SetCurrentDebugLocation(DL);
196 
198  Value *A = State.get(getOperand(0), Part);
199  Value *B = State.get(getOperand(1), Part);
200  Value *V =
201  Builder.CreateBinOp((Instruction::BinaryOps)getOpcode(), A, B, Name);
202  State.set(this, V, Part);
203  return;
204  }
205 
206  switch (getOpcode()) {
207  case VPInstruction::Not: {
208  Value *A = State.get(getOperand(0), Part);
209  Value *V = Builder.CreateNot(A, Name);
210  State.set(this, V, Part);
211  break;
212  }
213  case VPInstruction::ICmpULE: {
214  Value *IV = State.get(getOperand(0), Part);
215  Value *TC = State.get(getOperand(1), Part);
216  Value *V = Builder.CreateICmpULE(IV, TC, Name);
217  State.set(this, V, Part);
218  break;
219  }
220  case Instruction::Select: {
221  Value *Cond = State.get(getOperand(0), Part);
222  Value *Op1 = State.get(getOperand(1), Part);
223  Value *Op2 = State.get(getOperand(2), Part);
224  Value *V = Builder.CreateSelect(Cond, Op1, Op2, Name);
225  State.set(this, V, Part);
226  break;
227  }
229  // Get first lane of vector induction variable.
230  Value *VIVElem0 = State.get(getOperand(0), VPIteration(Part, 0));
231  // Get the original loop tripcount.
232  Value *ScalarTC = State.get(getOperand(1), Part);
233 
234  auto *Int1Ty = Type::getInt1Ty(Builder.getContext());
235  auto *PredTy = VectorType::get(Int1Ty, State.VF);
236  Instruction *Call = Builder.CreateIntrinsic(
237  Intrinsic::get_active_lane_mask, {PredTy, ScalarTC->getType()},
238  {VIVElem0, ScalarTC}, nullptr, Name);
239  State.set(this, Call, Part);
240  break;
241  }
243  // Generate code to combine the previous and current values in vector v3.
244  //
245  // vector.ph:
246  // v_init = vector(..., ..., ..., a[-1])
247  // br vector.body
248  //
249  // vector.body
250  // i = phi [0, vector.ph], [i+4, vector.body]
251  // v1 = phi [v_init, vector.ph], [v2, vector.body]
252  // v2 = a[i, i+1, i+2, i+3];
253  // v3 = vector(v1(3), v2(0, 1, 2))
254 
255  // For the first part, use the recurrence phi (v1), otherwise v2.
256  auto *V1 = State.get(getOperand(0), 0);
257  Value *PartMinus1 = Part == 0 ? V1 : State.get(getOperand(1), Part - 1);
258  if (!PartMinus1->getType()->isVectorTy()) {
259  State.set(this, PartMinus1, Part);
260  } else {
261  Value *V2 = State.get(getOperand(1), Part);
262  State.set(this, Builder.CreateVectorSplice(PartMinus1, V2, -1, Name),
263  Part);
264  }
265  break;
266  }
269  Value *Next = nullptr;
270  if (Part == 0) {
272  auto *Phi = State.get(getOperand(0), 0);
273  // The loop step is equal to the vectorization factor (num of SIMD
274  // elements) times the unroll factor (num of SIMD instructions).
275  Value *Step =
276  createStepForVF(Builder, Phi->getType(), State.VF, State.UF);
277  Next = Builder.CreateAdd(Phi, Step, Name, IsNUW, false);
278  } else {
279  Next = State.get(this, 0);
280  }
281 
282  State.set(this, Next, Part);
283  break;
284  }
285 
289  auto *IV = State.get(getOperand(0), VPIteration(0, 0));
290  if (Part == 0) {
291  State.set(this, IV, Part);
292  break;
293  }
294 
295  // The canonical IV is incremented by the vectorization factor (num of SIMD
296  // elements) times the unroll part.
297  Value *Step = createStepForVF(Builder, IV->getType(), State.VF, Part);
298  Value *Next = Builder.CreateAdd(IV, Step, Name, IsNUW, false);
299  State.set(this, Next, Part);
300  break;
301  }
303  if (Part != 0)
304  break;
305 
306  Value *Cond = State.get(getOperand(0), VPIteration(Part, 0));
307  VPRegionBlock *ParentRegion = getParent()->getParent();
308  VPBasicBlock *Header = ParentRegion->getEntryBasicBlock();
309 
310  // Replace the temporary unreachable terminator with a new conditional
311  // branch, hooking it up to backward destination for exiting blocks now and
312  // to forward destination(s) later when they are created.
313  BranchInst *CondBr =
314  Builder.CreateCondBr(Cond, Builder.GetInsertBlock(), nullptr);
315 
316  if (getParent()->isExiting())
317  CondBr->setSuccessor(1, State.CFG.VPBB2IRBB[Header]);
318 
319  CondBr->setSuccessor(0, nullptr);
320  Builder.GetInsertBlock()->getTerminator()->eraseFromParent();
321  break;
322  }
324  if (Part != 0)
325  break;
326  // First create the compare.
327  Value *IV = State.get(getOperand(0), Part);
328  Value *TC = State.get(getOperand(1), Part);
329  Value *Cond = Builder.CreateICmpEQ(IV, TC);
330 
331  // Now create the branch.
332  auto *Plan = getParent()->getPlan();
333  VPRegionBlock *TopRegion = Plan->getVectorLoopRegion();
334  VPBasicBlock *Header = TopRegion->getEntry()->getEntryBasicBlock();
335 
336  // Replace the temporary unreachable terminator with a new conditional
337  // branch, hooking it up to backward destination (the header) now and to the
338  // forward destination (the exit/middle block) later when it is created.
339  // Note that CreateCondBr expects a valid BB as first argument, so we need
340  // to set it to nullptr later.
341  BranchInst *CondBr = Builder.CreateCondBr(Cond, Builder.GetInsertBlock(),
342  State.CFG.VPBB2IRBB[Header]);
343  CondBr->setSuccessor(0, nullptr);
344  Builder.GetInsertBlock()->getTerminator()->eraseFromParent();
345  break;
346  }
347  default:
348  llvm_unreachable("Unsupported opcode for instruction");
349  }
350 }
351 
353  assert(!State.Instance && "VPInstruction executing an Instance");
355  State.Builder.setFastMathFlags(FMF);
356  for (unsigned Part = 0; Part < State.UF; ++Part)
357  generateInstruction(State, Part);
358 }
359 
360 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
361 void VPInstruction::dump() const {
362  VPSlotTracker SlotTracker(getParent()->getPlan());
363  print(dbgs(), "", SlotTracker);
364 }
365 
367  VPSlotTracker &SlotTracker) const {
368  O << Indent << "EMIT ";
369 
370  if (hasResult()) {
372  O << " = ";
373  }
374 
375  switch (getOpcode()) {
376  case VPInstruction::Not:
377  O << "not";
378  break;
380  O << "icmp ule";
381  break;
383  O << "combined load";
384  break;
386  O << "combined store";
387  break;
389  O << "active lane mask";
390  break;
392  O << "first-order splice";
393  break;
395  O << "VF * UF + ";
396  break;
398  O << "VF * UF +(nuw) ";
399  break;
401  O << "branch-on-cond";
402  break;
404  O << "VF * Part + ";
405  break;
407  O << "VF * Part +(nuw) ";
408  break;
410  O << "branch-on-count ";
411  break;
412  default:
414  }
415 
416  O << FMF;
417 
418  for (const VPValue *Operand : operands()) {
419  O << " ";
420  Operand->printAsOperand(O, SlotTracker);
421  }
422 
423  if (DL) {
424  O << ", !dbg ";
425  DL.print(O);
426  }
427 }
428 #endif
429 
431  // Make sure the VPInstruction is a floating-point operation.
432  assert((Opcode == Instruction::FAdd || Opcode == Instruction::FMul ||
433  Opcode == Instruction::FNeg || Opcode == Instruction::FSub ||
434  Opcode == Instruction::FDiv || Opcode == Instruction::FRem ||
435  Opcode == Instruction::FCmp) &&
436  "this op can't take fast-math flags");
437  FMF = FMFNew;
438 }
439 
441  auto &CI = *cast<CallInst>(getUnderlyingInstr());
442  assert(!isa<DbgInfoIntrinsic>(CI) &&
443  "DbgInfoIntrinsic should have been dropped during VPlan construction");
444  State.setDebugLocFromInst(&CI);
445 
447  for (Value *ArgOperand : CI.args())
448  Tys.push_back(
449  ToVectorTy(ArgOperand->getType(), State.VF.getKnownMinValue()));
450 
451  for (unsigned Part = 0; Part < State.UF; ++Part) {
452  SmallVector<Type *, 2> TysForDecl = {CI.getType()};
454  for (const auto &I : enumerate(operands())) {
455  // Some intrinsics have a scalar argument - don't replace it with a
456  // vector.
457  Value *Arg;
458  if (VectorIntrinsicID == Intrinsic::not_intrinsic ||
459  !isVectorIntrinsicWithScalarOpAtArg(VectorIntrinsicID, I.index()))
460  Arg = State.get(I.value(), Part);
461  else
462  Arg = State.get(I.value(), VPIteration(0, 0));
463  if (isVectorIntrinsicWithOverloadTypeAtArg(VectorIntrinsicID, I.index()))
464  TysForDecl.push_back(Arg->getType());
465  Args.push_back(Arg);
466  }
467 
468  Function *VectorF;
469  if (VectorIntrinsicID != Intrinsic::not_intrinsic) {
470  // Use vector version of the intrinsic.
471  if (State.VF.isVector())
472  TysForDecl[0] =
473  VectorType::get(CI.getType()->getScalarType(), State.VF);
474  Module *M = State.Builder.GetInsertBlock()->getModule();
475  VectorF = Intrinsic::getDeclaration(M, VectorIntrinsicID, TysForDecl);
476  assert(VectorF && "Can't retrieve vector intrinsic.");
477  } else {
478  // Use vector version of the function call.
479  const VFShape Shape = VFShape::get(CI, State.VF, false /*HasGlobalPred*/);
480 #ifndef NDEBUG
481  assert(VFDatabase(CI).getVectorizedFunction(Shape) != nullptr &&
482  "Can't create vector function.");
483 #endif
484  VectorF = VFDatabase(CI).getVectorizedFunction(Shape);
485  }
487  CI.getOperandBundlesAsDefs(OpBundles);
488  CallInst *V = State.Builder.CreateCall(VectorF, Args, OpBundles);
489 
490  if (isa<FPMathOperator>(V))
491  V->copyFastMathFlags(&CI);
492 
493  State.set(this, V, Part);
494  State.addMetadata(V, &CI);
495  }
496 }
497 
498 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
500  VPSlotTracker &SlotTracker) const {
501  O << Indent << "WIDEN-CALL ";
502 
503  auto *CI = cast<CallInst>(getUnderlyingInstr());
504  if (CI->getType()->isVoidTy())
505  O << "void ";
506  else {
508  O << " = ";
509  }
510 
511  O << "call @" << CI->getCalledFunction()->getName() << "(";
513  O << ")";
514 
515  if (VectorIntrinsicID)
516  O << " (using vector intrinsic)";
517  else
518  O << " (using library function)";
519 }
520 
522  VPSlotTracker &SlotTracker) const {
523  O << Indent << "WIDEN-SELECT ";
525  O << " = select ";
527  O << ", ";
529  O << ", ";
531  O << (InvariantCond ? " (condition is loop invariant)" : "");
532 }
533 #endif
534 
536  auto &I = *cast<SelectInst>(getUnderlyingInstr());
537  State.setDebugLocFromInst(&I);
538 
539  // The condition can be loop invariant but still defined inside the
540  // loop. This means that we can't just use the original 'cond' value.
541  // We have to take the 'vectorized' value and pick the first lane.
542  // Instcombine will make this a no-op.
543  auto *InvarCond =
544  InvariantCond ? State.get(getOperand(0), VPIteration(0, 0)) : nullptr;
545 
546  for (unsigned Part = 0; Part < State.UF; ++Part) {
547  Value *Cond = InvarCond ? InvarCond : State.get(getOperand(0), Part);
548  Value *Op0 = State.get(getOperand(1), Part);
549  Value *Op1 = State.get(getOperand(2), Part);
550  Value *Sel = State.Builder.CreateSelect(Cond, Op0, Op1);
551  State.set(this, Sel, Part);
552  State.addMetadata(Sel, &I);
553  }
554 }
555 
557  auto &I = *cast<Instruction>(getUnderlyingValue());
558  auto &Builder = State.Builder;
559  switch (I.getOpcode()) {
560  case Instruction::Call:
561  case Instruction::Br:
562  case Instruction::PHI:
563  case Instruction::GetElementPtr:
564  case Instruction::Select:
565  llvm_unreachable("This instruction is handled by a different recipe.");
566  case Instruction::UDiv:
567  case Instruction::SDiv:
568  case Instruction::SRem:
569  case Instruction::URem:
570  case Instruction::Add:
571  case Instruction::FAdd:
572  case Instruction::Sub:
573  case Instruction::FSub:
574  case Instruction::FNeg:
575  case Instruction::Mul:
576  case Instruction::FMul:
577  case Instruction::FDiv:
578  case Instruction::FRem:
579  case Instruction::Shl:
580  case Instruction::LShr:
581  case Instruction::AShr:
582  case Instruction::And:
583  case Instruction::Or:
584  case Instruction::Xor: {
585  // Just widen unops and binops.
586  State.setDebugLocFromInst(&I);
587 
588  for (unsigned Part = 0; Part < State.UF; ++Part) {
590  for (VPValue *VPOp : operands())
591  Ops.push_back(State.get(VPOp, Part));
592 
593  Value *V = Builder.CreateNAryOp(I.getOpcode(), Ops);
594 
595  if (auto *VecOp = dyn_cast<Instruction>(V)) {
596  VecOp->copyIRFlags(&I);
597 
598  // If the instruction is vectorized and was in a basic block that needed
599  // predication, we can't propagate poison-generating flags (nuw/nsw,
600  // exact, etc.). The control flow has been linearized and the
601  // instruction is no longer guarded by the predicate, which could make
602  // the flag properties to no longer hold.
603  if (State.MayGeneratePoisonRecipes.contains(this))
604  VecOp->dropPoisonGeneratingFlags();
605  }
606 
607  // Use this vector value for all users of the original instruction.
608  State.set(this, V, Part);
609  State.addMetadata(V, &I);
610  }
611 
612  break;
613  }
614  case Instruction::Freeze: {
615  State.setDebugLocFromInst(&I);
616 
617  for (unsigned Part = 0; Part < State.UF; ++Part) {
618  Value *Op = State.get(getOperand(0), Part);
619 
620  Value *Freeze = Builder.CreateFreeze(Op);
621  State.set(this, Freeze, Part);
622  }
623  break;
624  }
625  case Instruction::ICmp:
626  case Instruction::FCmp: {
627  // Widen compares. Generate vector compares.
628  bool FCmp = (I.getOpcode() == Instruction::FCmp);
629  auto *Cmp = cast<CmpInst>(&I);
630  State.setDebugLocFromInst(Cmp);
631  for (unsigned Part = 0; Part < State.UF; ++Part) {
632  Value *A = State.get(getOperand(0), Part);
633  Value *B = State.get(getOperand(1), Part);
634  Value *C = nullptr;
635  if (FCmp) {
636  // Propagate fast math flags.
638  Builder.setFastMathFlags(Cmp->getFastMathFlags());
639  C = Builder.CreateFCmp(Cmp->getPredicate(), A, B);
640  } else {
641  C = Builder.CreateICmp(Cmp->getPredicate(), A, B);
642  }
643  State.set(this, C, Part);
644  State.addMetadata(C, &I);
645  }
646 
647  break;
648  }
649 
650  case Instruction::ZExt:
651  case Instruction::SExt:
652  case Instruction::FPToUI:
653  case Instruction::FPToSI:
654  case Instruction::FPExt:
655  case Instruction::PtrToInt:
656  case Instruction::IntToPtr:
657  case Instruction::SIToFP:
658  case Instruction::UIToFP:
659  case Instruction::Trunc:
660  case Instruction::FPTrunc:
661  case Instruction::BitCast: {
662  auto *CI = cast<CastInst>(&I);
663  State.setDebugLocFromInst(CI);
664 
665  /// Vectorize casts.
666  Type *DestTy = (State.VF.isScalar())
667  ? CI->getType()
668  : VectorType::get(CI->getType(), State.VF);
669 
670  for (unsigned Part = 0; Part < State.UF; ++Part) {
671  Value *A = State.get(getOperand(0), Part);
672  Value *Cast = Builder.CreateCast(CI->getOpcode(), A, DestTy);
673  State.set(this, Cast, Part);
674  State.addMetadata(Cast, &I);
675  }
676  break;
677  }
678  default:
679  // This instruction is not vectorized by simple widening.
680  LLVM_DEBUG(dbgs() << "LV: Found an unhandled instruction: " << I);
681  llvm_unreachable("Unhandled instruction!");
682  } // end of switch.
683 }
684 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
686  VPSlotTracker &SlotTracker) const {
687  O << Indent << "WIDEN ";
689  const Instruction *UI = getUnderlyingInstr();
690  O << " = " << UI->getOpcodeName() << " ";
691  if (auto *Cmp = dyn_cast<CmpInst>(UI))
692  O << CmpInst::getPredicateName(Cmp->getPredicate()) << " ";
694 }
695 
697  VPSlotTracker &SlotTracker) const {
698  O << Indent << "WIDEN-INDUCTION";
699  if (getTruncInst()) {
700  O << "\\l\"";
701  O << " +\n" << Indent << "\" " << VPlanIngredient(IV) << "\\l\"";
702  O << " +\n" << Indent << "\" ";
704  } else
705  O << " " << VPlanIngredient(IV);
706 
707  O << ", ";
709 }
710 #endif
711 
713  auto *StartC = dyn_cast<ConstantInt>(getStartValue()->getLiveInIRValue());
714  auto *StepC = dyn_cast<SCEVConstant>(getInductionDescriptor().getStep());
715  return StartC && StartC->isZero() && StepC && StepC->isOne();
716 }
717 
718 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
720  VPSlotTracker &SlotTracker) const {
721  O << Indent;
723  O << Indent << "= DERIVED-IV ";
725  O << " + ";
727  O << " * ";
729 
730  if (IndDesc.getStep()->getType() != ResultTy)
731  O << " (truncated to " << *ResultTy << ")";
732 }
733 #endif
734 
735 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
737  VPSlotTracker &SlotTracker) const {
738  O << Indent;
740  O << Indent << "= SCALAR-STEPS ";
742 }
743 #endif
744 
746  auto *GEP = cast<GetElementPtrInst>(getUnderlyingInstr());
747  // Construct a vector GEP by widening the operands of the scalar GEP as
748  // necessary. We mark the vector GEP 'inbounds' if appropriate. A GEP
749  // results in a vector of pointers when at least one operand of the GEP
750  // is vector-typed. Thus, to keep the representation compact, we only use
751  // vector-typed operands for loop-varying values.
752 
753  if (State.VF.isVector() && IsPtrLoopInvariant && IsIndexLoopInvariant.all()) {
754  // If we are vectorizing, but the GEP has only loop-invariant operands,
755  // the GEP we build (by only using vector-typed operands for
756  // loop-varying values) would be a scalar pointer. Thus, to ensure we
757  // produce a vector of pointers, we need to either arbitrarily pick an
758  // operand to broadcast, or broadcast a clone of the original GEP.
759  // Here, we broadcast a clone of the original.
760  //
761  // TODO: If at some point we decide to scalarize instructions having
762  // loop-invariant operands, this special case will no longer be
763  // required. We would add the scalarization decision to
764  // collectLoopScalars() and teach getVectorValue() to broadcast
765  // the lane-zero scalar value.
766  auto *Clone = State.Builder.Insert(GEP->clone());
767  for (unsigned Part = 0; Part < State.UF; ++Part) {
768  Value *EntryPart = State.Builder.CreateVectorSplat(State.VF, Clone);
769  State.set(this, EntryPart, Part);
770  State.addMetadata(EntryPart, GEP);
771  }
772  } else {
773  // If the GEP has at least one loop-varying operand, we are sure to
774  // produce a vector of pointers. But if we are only unrolling, we want
775  // to produce a scalar GEP for each unroll part. Thus, the GEP we
776  // produce with the code below will be scalar (if VF == 1) or vector
777  // (otherwise). Note that for the unroll-only case, we still maintain
778  // values in the vector mapping with initVector, as we do for other
779  // instructions.
780  for (unsigned Part = 0; Part < State.UF; ++Part) {
781  // The pointer operand of the new GEP. If it's loop-invariant, we
782  // won't broadcast it.
783  auto *Ptr = IsPtrLoopInvariant
784  ? State.get(getOperand(0), VPIteration(0, 0))
785  : State.get(getOperand(0), Part);
786 
787  // Collect all the indices for the new GEP. If any index is
788  // loop-invariant, we won't broadcast it.
789  SmallVector<Value *, 4> Indices;
790  for (unsigned I = 1, E = getNumOperands(); I < E; I++) {
791  VPValue *Operand = getOperand(I);
792  if (IsIndexLoopInvariant[I - 1])
793  Indices.push_back(State.get(Operand, VPIteration(0, 0)));
794  else
795  Indices.push_back(State.get(Operand, Part));
796  }
797 
798  // If the GEP instruction is vectorized and was in a basic block that
799  // needed predication, we can't propagate the poison-generating 'inbounds'
800  // flag. The control flow has been linearized and the GEP is no longer
801  // guarded by the predicate, which could make the 'inbounds' properties to
802  // no longer hold.
803  bool IsInBounds =
804  GEP->isInBounds() && State.MayGeneratePoisonRecipes.count(this) == 0;
805 
806  // Create the new GEP. Note that this GEP may be a scalar if VF == 1,
807  // but it should be a vector, otherwise.
808  auto *NewGEP = State.Builder.CreateGEP(GEP->getSourceElementType(), Ptr,
809  Indices, "", IsInBounds);
810  assert((State.VF.isScalar() || NewGEP->getType()->isVectorTy()) &&
811  "NewGEP is not a pointer vector");
812  State.set(this, NewGEP, Part);
813  State.addMetadata(NewGEP, GEP);
814  }
815  }
816 }
817 
818 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
820  VPSlotTracker &SlotTracker) const {
821  O << Indent << "WIDEN-GEP ";
822  O << (IsPtrLoopInvariant ? "Inv" : "Var");
823  size_t IndicesNumber = IsIndexLoopInvariant.size();
824  for (size_t I = 0; I < IndicesNumber; ++I)
825  O << "[" << (IsIndexLoopInvariant[I] ? "Inv" : "Var") << "]";
826 
827  O << " ";
829  O << " = getelementptr ";
831 }
832 #endif
833 
835  State.setDebugLocFromInst(Phi);
836  // We know that all PHIs in non-header blocks are converted into
837  // selects, so we don't have to worry about the insertion order and we
838  // can just use the builder.
839  // At this point we generate the predication tree. There may be
840  // duplications since this is a simple recursive scan, but future
841  // optimizations will clean it up.
842 
843  unsigned NumIncoming = getNumIncomingValues();
844 
845  // Generate a sequence of selects of the form:
846  // SELECT(Mask3, In3,
847  // SELECT(Mask2, In2,
848  // SELECT(Mask1, In1,
849  // In0)))
850  // Note that Mask0 is never used: lanes for which no path reaches this phi and
851  // are essentially undef are taken from In0.
852  VectorParts Entry(State.UF);
853  for (unsigned In = 0; In < NumIncoming; ++In) {
854  for (unsigned Part = 0; Part < State.UF; ++Part) {
855  // We might have single edge PHIs (blocks) - use an identity
856  // 'select' for the first PHI operand.
857  Value *In0 = State.get(getIncomingValue(In), Part);
858  if (In == 0)
859  Entry[Part] = In0; // Initialize with the first incoming value.
860  else {
861  // Select between the current value and the previous incoming edge
862  // based on the incoming mask.
863  Value *Cond = State.get(getMask(In), Part);
864  Entry[Part] =
865  State.Builder.CreateSelect(Cond, In0, Entry[Part], "predphi");
866  }
867  }
868  }
869  for (unsigned Part = 0; Part < State.UF; ++Part)
870  State.set(this, Entry[Part], Part);
871 }
872 
873 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
875  VPSlotTracker &SlotTracker) const {
876  O << Indent << "BLEND ";
877  Phi->printAsOperand(O, false);
878  O << " =";
879  if (getNumIncomingValues() == 1) {
880  // Not a User of any mask: not really blending, this is a
881  // single-predecessor phi.
882  O << " ";
884  } else {
885  for (unsigned I = 0, E = getNumIncomingValues(); I < E; ++I) {
886  O << " ";
888  O << "/";
890  }
891  }
892 }
893 
895  VPSlotTracker &SlotTracker) const {
896  O << Indent << "REDUCE ";
898  O << " = ";
900  O << " +";
901  if (isa<FPMathOperator>(getUnderlyingInstr()))
903  O << " reduce." << Instruction::getOpcodeName(RdxDesc->getOpcode()) << " (";
905  if (getCondOp()) {
906  O << ", ";
908  }
909  O << ")";
910  if (RdxDesc->IntermediateStore)
911  O << " (with final reduction value stored in invariant address sank "
912  "outside of loop)";
913 }
914 
916  VPSlotTracker &SlotTracker) const {
917  O << Indent << (IsUniform ? "CLONE " : "REPLICATE ");
918 
919  if (!getUnderlyingInstr()->getType()->isVoidTy()) {
921  O << " = ";
922  }
923  if (auto *CB = dyn_cast<CallBase>(getUnderlyingInstr())) {
924  O << "call @" << CB->getCalledFunction()->getName() << "(";
926  O, [&O, &SlotTracker](VPValue *Op) {
927  Op->printAsOperand(O, SlotTracker);
928  });
929  O << ")";
930  } else {
933  }
934 
935  if (AlsoPack)
936  O << " (S->V)";
937 }
938 #endif
939 
941  assert(State.Instance && "Branch on Mask works only on single instance.");
942 
943  unsigned Part = State.Instance->Part;
944  unsigned Lane = State.Instance->Lane.getKnownLane();
945 
946  Value *ConditionBit = nullptr;
947  VPValue *BlockInMask = getMask();
948  if (BlockInMask) {
949  ConditionBit = State.get(BlockInMask, Part);
950  if (ConditionBit->getType()->isVectorTy())
951  ConditionBit = State.Builder.CreateExtractElement(
952  ConditionBit, State.Builder.getInt32(Lane));
953  } else // Block in mask is all-one.
954  ConditionBit = State.Builder.getTrue();
955 
956  // Replace the temporary unreachable terminator with a new conditional branch,
957  // whose two destinations will be set later when they are created.
958  auto *CurrentTerminator = State.CFG.PrevBB->getTerminator();
959  assert(isa<UnreachableInst>(CurrentTerminator) &&
960  "Expected to replace unreachable terminator with conditional branch.");
961  auto *CondBr = BranchInst::Create(State.CFG.PrevBB, nullptr, ConditionBit);
962  CondBr->setSuccessor(0, nullptr);
963  ReplaceInstWithInst(CurrentTerminator, CondBr);
964 }
965 
967  assert(State.Instance && "Predicated instruction PHI works per instance.");
968  Instruction *ScalarPredInst =
969  cast<Instruction>(State.get(getOperand(0), *State.Instance));
970  BasicBlock *PredicatedBB = ScalarPredInst->getParent();
971  BasicBlock *PredicatingBB = PredicatedBB->getSinglePredecessor();
972  assert(PredicatingBB && "Predicated block has no single predecessor.");
973  assert(isa<VPReplicateRecipe>(getOperand(0)) &&
974  "operand must be VPReplicateRecipe");
975 
976  // By current pack/unpack logic we need to generate only a single phi node: if
977  // a vector value for the predicated instruction exists at this point it means
978  // the instruction has vector users only, and a phi for the vector value is
979  // needed. In this case the recipe of the predicated instruction is marked to
980  // also do that packing, thereby "hoisting" the insert-element sequence.
981  // Otherwise, a phi node for the scalar value is needed.
982  unsigned Part = State.Instance->Part;
983  if (State.hasVectorValue(getOperand(0), Part)) {
984  Value *VectorValue = State.get(getOperand(0), Part);
985  InsertElementInst *IEI = cast<InsertElementInst>(VectorValue);
986  PHINode *VPhi = State.Builder.CreatePHI(IEI->getType(), 2);
987  VPhi->addIncoming(IEI->getOperand(0), PredicatingBB); // Unmodified vector.
988  VPhi->addIncoming(IEI, PredicatedBB); // New vector with inserted element.
989  if (State.hasVectorValue(this, Part))
990  State.reset(this, VPhi, Part);
991  else
992  State.set(this, VPhi, Part);
993  // NOTE: Currently we need to update the value of the operand, so the next
994  // predicated iteration inserts its generated value in the correct vector.
995  State.reset(getOperand(0), VPhi, Part);
996  } else {
997  Type *PredInstType = getOperand(0)->getUnderlyingValue()->getType();
998  PHINode *Phi = State.Builder.CreatePHI(PredInstType, 2);
999  Phi->addIncoming(PoisonValue::get(ScalarPredInst->getType()),
1000  PredicatingBB);
1001  Phi->addIncoming(ScalarPredInst, PredicatedBB);
1002  if (State.hasScalarValue(this, *State.Instance))
1003  State.reset(this, Phi, *State.Instance);
1004  else
1005  State.set(this, Phi, *State.Instance);
1006  // NOTE: Currently we need to update the value of the operand, so the next
1007  // predicated iteration inserts its generated value in the correct vector.
1008  State.reset(getOperand(0), Phi, *State.Instance);
1009  }
1010 }
1011 
1012 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
1014  VPSlotTracker &SlotTracker) const {
1015  O << Indent << "PHI-PREDICATED-INSTRUCTION ";
1017  O << " = ";
1019 }
1020 
1022  VPSlotTracker &SlotTracker) const {
1023  O << Indent << "WIDEN ";
1024 
1025  if (!isStore()) {
1027  O << " = ";
1028  }
1029  O << Instruction::getOpcodeName(Ingredient.getOpcode()) << " ";
1030 
1032 }
1033 #endif
1034 
1036  Value *Start = getStartValue()->getLiveInIRValue();
1037  PHINode *EntryPart = PHINode::Create(
1038  Start->getType(), 2, "index", &*State.CFG.PrevBB->getFirstInsertionPt());
1039 
1040  BasicBlock *VectorPH = State.CFG.getPreheaderBBFor(this);
1041  EntryPart->addIncoming(Start, VectorPH);
1042  EntryPart->setDebugLoc(DL);
1043  for (unsigned Part = 0, UF = State.UF; Part < UF; ++Part)
1044  State.set(this, EntryPart, Part);
1045 }
1046 
1047 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
1049  VPSlotTracker &SlotTracker) const {
1050  O << Indent << "EMIT ";
1052  O << " = CANONICAL-INDUCTION";
1053 }
1054 #endif
1055 
1057  Type *Ty) const {
1058  if (Ty != getScalarType())
1059  return false;
1060  // The start value of ID must match the start value of this canonical
1061  // induction.
1062  if (getStartValue()->getLiveInIRValue() != ID.getStartValue())
1063  return false;
1064 
1065  ConstantInt *Step = ID.getConstIntStepValue();
1066  // ID must also be incremented by one. IK_IntInduction always increment the
1067  // induction by Step, but the binary op may not be set.
1068  return ID.getKind() == InductionDescriptor::IK_IntInduction && Step &&
1069  Step->isOne();
1070 }
1071 
1073  return IsScalarAfterVectorization &&
1074  (!VF.isScalable() || vputils::onlyFirstLaneUsed(this));
1075 }
1076 
1077 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
1079  VPSlotTracker &SlotTracker) const {
1080  O << Indent << "EMIT ";
1082  O << " = WIDEN-POINTER-INDUCTION ";
1084  O << ", " << *IndDesc.getStep();
1085 }
1086 #endif
1087 
1089  assert(!State.Instance && "cannot be used in per-lane");
1090  const DataLayout &DL = State.CFG.PrevBB->getModule()->getDataLayout();
1091  SCEVExpander Exp(SE, DL, "induction");
1092 
1093  Value *Res = Exp.expandCodeFor(Expr, Expr->getType(),
1094  &*State.Builder.GetInsertPoint());
1095 
1096  for (unsigned Part = 0, UF = State.UF; Part < UF; ++Part)
1097  State.set(this, Res, Part);
1098 }
1099 
1100 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
1102  VPSlotTracker &SlotTracker) const {
1103  O << Indent << "EMIT ";
1105  O << " = EXPAND SCEV " << *Expr;
1106 }
1107 #endif
1108 
1110  Value *CanonicalIV = State.get(getOperand(0), 0);
1111  Type *STy = CanonicalIV->getType();
1113  ElementCount VF = State.VF;
1114  Value *VStart = VF.isScalar()
1115  ? CanonicalIV
1116  : Builder.CreateVectorSplat(VF, CanonicalIV, "broadcast");
1117  for (unsigned Part = 0, UF = State.UF; Part < UF; ++Part) {
1118  Value *VStep = createStepForVF(Builder, STy, VF, Part);
1119  if (VF.isVector()) {
1120  VStep = Builder.CreateVectorSplat(VF, VStep);
1121  VStep =
1122  Builder.CreateAdd(VStep, Builder.CreateStepVector(VStep->getType()));
1123  }
1124  Value *CanonicalVectorIV = Builder.CreateAdd(VStart, VStep, "vec.iv");
1125  State.set(this, CanonicalVectorIV, Part);
1126  }
1127 }
1128 
1129 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
1131  VPSlotTracker &SlotTracker) const {
1132  O << Indent << "EMIT ";
1134  O << " = WIDEN-CANONICAL-INDUCTION ";
1136 }
1137 #endif
1138 
1140  auto &Builder = State.Builder;
1141  // Create a vector from the initial value.
1142  auto *VectorInit = getStartValue()->getLiveInIRValue();
1143 
1144  Type *VecTy = State.VF.isScalar()
1145  ? VectorInit->getType()
1146  : VectorType::get(VectorInit->getType(), State.VF);
1147 
1148  BasicBlock *VectorPH = State.CFG.getPreheaderBBFor(this);
1149  if (State.VF.isVector()) {
1150  auto *IdxTy = Builder.getInt32Ty();
1151  auto *One = ConstantInt::get(IdxTy, 1);
1153  Builder.SetInsertPoint(VectorPH->getTerminator());
1154  auto *RuntimeVF = getRuntimeVF(Builder, IdxTy, State.VF);
1155  auto *LastIdx = Builder.CreateSub(RuntimeVF, One);
1156  VectorInit = Builder.CreateInsertElement(
1157  PoisonValue::get(VecTy), VectorInit, LastIdx, "vector.recur.init");
1158  }
1159 
1160  // Create a phi node for the new recurrence.
1161  PHINode *EntryPart = PHINode::Create(
1162  VecTy, 2, "vector.recur", &*State.CFG.PrevBB->getFirstInsertionPt());
1163  EntryPart->addIncoming(VectorInit, VectorPH);
1164  State.set(this, EntryPart, 0);
1165 }
1166 
1167 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
1169  VPSlotTracker &SlotTracker) const {
1170  O << Indent << "FIRST-ORDER-RECURRENCE-PHI ";
1172  O << " = phi ";
1174 }
1175 #endif
1176 
1178  PHINode *PN = cast<PHINode>(getUnderlyingValue());
1179  auto &Builder = State.Builder;
1180 
1181  // In order to support recurrences we need to be able to vectorize Phi nodes.
1182  // Phi nodes have cycles, so we need to vectorize them in two stages. This is
1183  // stage #1: We create a new vector PHI node with no incoming edges. We'll use
1184  // this value when we vectorize all of the instructions that use the PHI.
1185  bool ScalarPHI = State.VF.isScalar() || IsInLoop;
1186  Type *VecTy =
1187  ScalarPHI ? PN->getType() : VectorType::get(PN->getType(), State.VF);
1188 
1189  BasicBlock *HeaderBB = State.CFG.PrevBB;
1190  assert(State.CurrentVectorLoop->getHeader() == HeaderBB &&
1191  "recipe must be in the vector loop header");
1192  unsigned LastPartForNewPhi = isOrdered() ? 1 : State.UF;
1193  for (unsigned Part = 0; Part < LastPartForNewPhi; ++Part) {
1194  Value *EntryPart =
1195  PHINode::Create(VecTy, 2, "vec.phi", &*HeaderBB->getFirstInsertionPt());
1196  State.set(this, EntryPart, Part);
1197  }
1198 
1199  BasicBlock *VectorPH = State.CFG.getPreheaderBBFor(this);
1200 
1201  // Reductions do not have to start at zero. They can start with
1202  // any loop invariant values.
1203  VPValue *StartVPV = getStartValue();
1204  Value *StartV = StartVPV->getLiveInIRValue();
1205 
1206  Value *Iden = nullptr;
1207  RecurKind RK = RdxDesc.getRecurrenceKind();
1210  // MinMax reduction have the start value as their identify.
1211  if (ScalarPHI) {
1212  Iden = StartV;
1213  } else {
1215  Builder.SetInsertPoint(VectorPH->getTerminator());
1216  StartV = Iden =
1217  Builder.CreateVectorSplat(State.VF, StartV, "minmax.ident");
1218  }
1219  } else {
1220  Iden = RdxDesc.getRecurrenceIdentity(RK, VecTy->getScalarType(),
1221  RdxDesc.getFastMathFlags());
1222 
1223  if (!ScalarPHI) {
1224  Iden = Builder.CreateVectorSplat(State.VF, Iden);
1226  Builder.SetInsertPoint(VectorPH->getTerminator());
1227  Constant *Zero = Builder.getInt32(0);
1228  StartV = Builder.CreateInsertElement(Iden, StartV, Zero);
1229  }
1230  }
1231 
1232  for (unsigned Part = 0; Part < LastPartForNewPhi; ++Part) {
1233  Value *EntryPart = State.get(this, Part);
1234  // Make sure to add the reduction start value only to the
1235  // first unroll part.
1236  Value *StartVal = (Part == 0) ? StartV : Iden;
1237  cast<PHINode>(EntryPart)->addIncoming(StartVal, VectorPH);
1238  }
1239 }
1240 
1241 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
1243  VPSlotTracker &SlotTracker) const {
1244  O << Indent << "WIDEN-REDUCTION-PHI ";
1245 
1247  O << " = phi ";
1249 }
1250 #endif
1251 
1254  "Non-native vplans are not expected to have VPWidenPHIRecipes.");
1255 
1256  // Currently we enter here in the VPlan-native path for non-induction
1257  // PHIs where all control flow is uniform. We simply widen these PHIs.
1258  // Create a vector phi with no operands - the vector phi operands will be
1259  // set at the end of vector code generation.
1260  VPBasicBlock *Parent = getParent();
1261  VPRegionBlock *LoopRegion = Parent->getEnclosingLoopRegion();
1262  unsigned StartIdx = 0;
1263  // For phis in header blocks of loop regions, use the index of the value
1264  // coming from the preheader.
1265  if (LoopRegion->getEntryBasicBlock() == Parent) {
1266  for (unsigned I = 0; I < getNumOperands(); ++I) {
1267  if (getIncomingBlock(I) ==
1268  LoopRegion->getSinglePredecessor()->getExitingBasicBlock())
1269  StartIdx = I;
1270  }
1271  }
1272  Value *Op0 = State.get(getOperand(StartIdx), 0);
1273  Type *VecTy = Op0->getType();
1274  Value *VecPhi = State.Builder.CreatePHI(VecTy, 2, "vec.phi");
1275  State.set(this, VecPhi, 0);
1276 }
1277 
1278 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
1280  VPSlotTracker &SlotTracker) const {
1281  O << Indent << "WIDEN-PHI ";
1282 
1283  auto *OriginalPhi = cast<PHINode>(getUnderlyingValue());
1284  // Unless all incoming values are modeled in VPlan print the original PHI
1285  // directly.
1286  // TODO: Remove once all VPWidenPHIRecipe instances keep all relevant incoming
1287  // values as VPValues.
1288  if (getNumOperands() != OriginalPhi->getNumOperands()) {
1289  O << VPlanIngredient(OriginalPhi);
1290  return;
1291  }
1292 
1294  O << " = phi ";
1296 }
1297 #endif
1298 
1299 // TODO: It would be good to use the existing VPWidenPHIRecipe instead and
1300 // remove VPActiveLaneMaskPHIRecipe.
1302  BasicBlock *VectorPH = State.CFG.getPreheaderBBFor(this);
1303  for (unsigned Part = 0, UF = State.UF; Part < UF; ++Part) {
1304  Value *StartMask = State.get(getOperand(0), Part);
1305  PHINode *EntryPart =
1306  State.Builder.CreatePHI(StartMask->getType(), 2, "active.lane.mask");
1307  EntryPart->addIncoming(StartMask, VectorPH);
1308  EntryPart->setDebugLoc(DL);
1309  State.set(this, EntryPart, Part);
1310  }
1311 }
1312 
1313 #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
1315  VPSlotTracker &SlotTracker) const {
1316  O << Indent << "ACTIVE-LANE-MASK-PHI ";
1317 
1319  O << " = phi ";
1321 }
1322 #endif
llvm::VPBasicBlock::getRecipeList
RecipeListTy & getRecipeList()
Returns a reference to the list of recipes.
Definition: VPlan.h:2112
llvm::VPInstruction::getOpcode
unsigned getOpcode() const
Definition: VPlan.h:851
llvm::VPRecipeBase::moveAfter
void moveAfter(VPRecipeBase *MovePos)
Unlink this recipe from its current VPBasicBlock and insert it into the VPBasicBlock that MovePos liv...
Definition: VPlanRecipes.cpp:181
llvm::VPBlockBase::getSinglePredecessor
VPBlockBase * getSinglePredecessor() const
Definition: VPlan.h:502
llvm::VPReductionRecipe::print
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
Definition: VPlanRecipes.cpp:894
llvm::VPBlockBase::getPlan
VPlan * getPlan()
Definition: VPlan.cpp:142
llvm::VPWidenIntOrFpInductionRecipe::getStepValue
VPValue * getStepValue()
Returns the step value of the induction.
Definition: VPlan.h:1097
llvm::VPBlendRecipe::getNumIncomingValues
unsigned getNumIncomingValues() const
Return the number of incoming values, taking into account that a single incoming value has no mask.
Definition: VPlan.h:1388
llvm::VPBlendRecipe::getIncomingValue
VPValue * getIncomingValue(unsigned Idx) const
Return incoming value number Idx.
Definition: VPlan.h:1391
llvm
This is an optimization pass for GlobalISel generic memory operations.
Definition: AddressRanges.h:18
llvm::VPRecipeBase::getUnderlyingInstr
Instruction * getUnderlyingInstr()
Returns the underlying instruction, if the recipe is a VPValue or nullptr otherwise.
Definition: VPlan.h:720
llvm::Type::getInt1Ty
static IntegerType * getInt1Ty(LLVMContext &C)
Definition: Type.cpp:236
M
We currently emits eax Perhaps this is what we really should generate is Is imull three or four cycles eax eax The current instruction priority is based on pattern complexity The former is more complex because it folds a load so the latter will not be emitted Perhaps we should use AddedComplexity to give LEA32r a higher priority We should always try to match LEA first since the LEA matching code does some estimate to determine whether the match is profitable if we care more about code then imull is better It s two bytes shorter than movl leal On a Pentium M
Definition: README.txt:252
llvm::VPInstruction::Not
@ Not
Definition: VPlan.h:784
llvm::make_range
iterator_range< T > make_range(T x, T y)
Convenience function for iterating over sub-ranges.
Definition: iterator_range.h:53
llvm::VPTransformState::CFG
struct llvm::VPTransformState::CFGState CFG
llvm::VPRecipeBase::eraseFromParent
iplist< VPRecipeBase >::iterator eraseFromParent()
This method unlinks 'this' from the containing basic block and deletes it.
Definition: VPlanRecipes.cpp:176
llvm::VPLane::getFirstLane
static VPLane getFirstLane()
Definition: VPlan.h:136
llvm::VPWidenCanonicalIVRecipe::execute
void execute(VPTransformState &State) override
Generate a canonical vector induction variable of the vector loop, with start = {<Part*VF,...
Definition: VPlanRecipes.cpp:1109
llvm::VPPredInstPHIRecipe::execute
void execute(VPTransformState &State) override
Generates phi nodes for live-outs as needed to retain SSA form.
Definition: VPlanRecipes.cpp:966
llvm::VPBlendRecipe::print
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
Definition: VPlanRecipes.cpp:874
llvm::DataLayout
A parsed version of the target data layout string in and methods for querying it.
Definition: DataLayout.h:113
llvm::VPTransformState::Builder
IRBuilderBase & Builder
Hold a reference to the IRBuilder used to generate output IR code.
Definition: VPlan.h:359
PHI
Rewrite undef for PHI
Definition: AMDGPURewriteUndefForPHI.cpp:101
ScalarEvolutionExpander.h
llvm::VPInstruction::FirstOrderRecurrenceSplice
@ FirstOrderRecurrenceSplice
Definition: VPlan.h:781
llvm::ElementCount
Definition: TypeSize.h:404
llvm::VPBasicBlock
VPBasicBlock serves as the leaf of the Hierarchical Control-Flow Graph.
Definition: VPlan.h:2065
llvm::Function
Definition: Function.h:60
llvm::SCEVExpander
This class uses information about analyze scalars to rewrite expressions in canonical form.
Definition: ScalarEvolutionExpander.h:50
llvm::VPWidenIntOrFpInductionRecipe::print
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
Definition: VPlanRecipes.cpp:696
llvm::VPInstruction::ActiveLaneMask
@ ActiveLaneMask
Definition: VPlan.h:788
llvm::VPValue::getLiveInIRValue
Value * getLiveInIRValue()
Returns the underlying IR value, if this VPValue is defined outside the scope of VPlan.
Definition: VPlanValue.h:197
llvm::Type::getScalarType
Type * getScalarType() const
If this is a vector type, return the element type, otherwise return 'this'.
Definition: Type.h:328
llvm::SmallVector
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
Definition: SmallVector.h:1199
llvm::VPCanonicalIVPHIRecipe::print
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
Definition: VPlanRecipes.cpp:1048
llvm::VPDef::getVPSingleValue
VPValue * getVPSingleValue()
Returns the only VPValue defined by the VPDef.
Definition: VPlanValue.h:404
llvm::VPTransformState::get
Value * get(VPValue *Def, unsigned Part)
Get the generated Value for a given VPValue and a given Part.
Definition: LoopVectorize.cpp:9869
llvm::enumerate
detail::enumerator< R > enumerate(R &&TheRange)
Given an input range, returns a new range whose values are are pair (A,B) such that A is the 0-based ...
Definition: STLExtras.h:2263
llvm::VPBranchOnMaskRecipe::execute
void execute(VPTransformState &State) override
Generate the extraction of the appropriate bit from the block mask and the conditional branch.
Definition: VPlanRecipes.cpp:940
llvm::IRBuilderBase::setFastMathFlags
void setFastMathFlags(FastMathFlags NewFMF)
Set the fast-math flags to be used with generated fp-math operators.
Definition: IRBuilder.h:297
llvm::IRBuilder
This provides a uniform API for creating instructions and inserting them into a basic block: either a...
Definition: IRBuilder.h:2533
llvm::VPWidenPointerInductionRecipe::onlyScalarsGenerated
bool onlyScalarsGenerated(ElementCount VF)
Returns true if only scalar values will be generated.
Definition: VPlanRecipes.cpp:1072
llvm::VPInstruction::setFastMathFlags
void setFastMathFlags(FastMathFlags FMFNew)
Set the fast-math flags.
Definition: VPlanRecipes.cpp:430
llvm::VPTransformState::set
void set(VPValue *Def, Value *V, unsigned Part)
Set the generated Value for a given VPValue and a given Part.
Definition: VPlan.h:264
llvm::isVectorIntrinsicWithScalarOpAtArg
bool isVectorIntrinsicWithScalarOpAtArg(Intrinsic::ID ID, unsigned ScalarOpdIdx)
Identifies if the vector form of the intrinsic has a scalar operand.
Definition: VectorUtils.cpp:101
llvm::RecurrenceDescriptor::getOpcode
static unsigned getOpcode(RecurKind Kind)
Returns the opcode corresponding to the RecurrenceKind.
Definition: IVDescriptors.cpp:1130
llvm::VPTransformState::CFGState::getPreheaderBBFor
BasicBlock * getPreheaderBBFor(VPRecipeBase *R)
Returns the BasicBlock* mapped to the pre-header of the loop region containing R.
Definition: VPlan.cpp:233
llvm::VPBlendRecipe::execute
void execute(VPTransformState &State) override
Generate the phi/select nodes.
Definition: VPlanRecipes.cpp:834
llvm::VPDerivedIVRecipe::getStepValue
VPValue * getStepValue() const
Definition: VPlan.h:2006
llvm::interleaveComma
void interleaveComma(const Container &c, StreamT &os, UnaryFunctor each_fn)
Definition: STLExtras.h:2097
llvm::InductionDescriptor::IK_IntInduction
@ IK_IntInduction
Integer induction variable. Step = C.
Definition: IVDescriptors.h:311
llvm::Type
The instances of the Type class are immutable: once they are created, they are never changed.
Definition: Type.h:45
llvm::VPWidenIntOrFpInductionRecipe::getTruncInst
TruncInst * getTruncInst()
Returns the first defined value as TruncInst, if it is one or nullptr otherwise.
Definition: VPlan.h:1102
llvm::isVectorIntrinsicWithOverloadTypeAtArg
bool isVectorIntrinsicWithOverloadTypeAtArg(Intrinsic::ID ID, unsigned OpdIdx)
Identifies if the vector form of the intrinsic has a operand that has an overloaded type.
Definition: VectorUtils.cpp:119
llvm::VPUser::op_begin
operand_iterator op_begin()
Definition: VPlanValue.h:292
llvm::VPWidenMemoryInstructionRecipe::print
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
Definition: VPlanRecipes.cpp:1021
llvm::iplist_impl::remove
pointer remove(iterator &IT)
Definition: ilist.h:252
llvm::Intrinsic::not_intrinsic
@ not_intrinsic
Definition: Intrinsics.h:45
llvm::BasicBlock::getSinglePredecessor
const BasicBlock * getSinglePredecessor() const
Return the predecessor of this block if it has a single predecessor block.
Definition: BasicBlock.cpp:285
STLExtras.h
llvm::VPDerivedIVRecipe::getCanonicalIV
VPValue * getCanonicalIV() const
Definition: VPlan.h:2005
llvm::VPUser::operands
operand_range operands()
Definition: VPlanValue.h:296
llvm::VPRegionBlock::getEntry
const VPBlockBase * getEntry() const
Definition: VPlan.h:2228
llvm::CmpInst::getPredicateName
static StringRef getPredicateName(Predicate P)
Definition: Instructions.cpp:4096
llvm::FastMathFlags
Convenience struct for specifying and reasoning about fast-math flags.
Definition: FMF.h:21
llvm::VPTransformState::hasScalarValue
bool hasScalarValue(VPValue *Def, VPIteration Instance)
Definition: VPlan.h:253
llvm::LinearPolySize::isScalable
bool isScalable() const
Returns whether the size is scaled by a runtime quantity (vscale).
Definition: TypeSize.h:298
llvm::VPBasicBlock::getEnclosingLoopRegion
VPRegionBlock * getEnclosingLoopRegion()
Definition: VPlan.cpp:425
llvm::iplist
An intrusive list with ownership and callbacks specified/controlled by ilist_traits,...
Definition: ilist.h:391
LLVM_DEBUG
#define LLVM_DEBUG(X)
Definition: Debug.h:101
llvm::VPWidenPHIRecipe::execute
void execute(VPTransformState &State) override
Generate the phi/select nodes.
Definition: VPlanRecipes.cpp:1252
llvm::VPInstruction::SLPLoad
@ SLPLoad
Definition: VPlan.h:786
llvm::BranchInst::setSuccessor
void setSuccessor(unsigned idx, BasicBlock *NewSucc)
Definition: Instructions.h:3230
llvm::iplist_impl::erase
iterator erase(iterator where)
Definition: ilist.h:268
llvm::BasicBlock
LLVM Basic Block Representation.
Definition: BasicBlock.h:55
llvm::VPLiveOut::fixPhi
void fixPhi(VPlan &Plan, VPTransformState &State)
Fixup the wrapped LCSSA phi node in the unique exit block.
Definition: VPlanRecipes.cpp:137
llvm::VPWidenIntOrFpInductionRecipe::getInductionDescriptor
const InductionDescriptor & getInductionDescriptor() const
Returns the induction descriptor for the recipe.
Definition: VPlan.h:1112
llvm::VPCanonicalIVPHIRecipe::getScalarType
const Type * getScalarType() const
Returns the scalar type of the induction.
Definition: VPlan.h:1862
llvm::RecurKind
RecurKind
These are the kinds of recurrences that we support.
Definition: IVDescriptors.h:35
llvm::VPRecipeBase::mayWriteToMemory
bool mayWriteToMemory() const
Returns true if the recipe may write to memory.
Definition: VPlanRecipes.cpp:42
llvm::dbgs
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
Definition: Debug.cpp:163
llvm::ElementCount::isScalar
bool isScalar() const
Counting predicates.
Definition: TypeSize.h:414
llvm::VPPredInstPHIRecipe::print
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
Definition: VPlanRecipes.cpp:1013
Arg
amdgpu Simplify well known AMD library false FunctionCallee Value * Arg
Definition: AMDGPULibCalls.cpp:187
Instruction.h
llvm::VPTransformState
VPTransformState holds information passed down when "executing" a VPlan, needed for generating the ou...
Definition: VPlan.h:205
CommandLine.h
llvm::ConstantInt
This is the shared class of boolean and integer constants.
Definition: Constants.h:79
llvm::VPReplicateRecipe::print
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
Definition: VPlanRecipes.cpp:915
llvm::Instruction::getOpcode
unsigned getOpcode() const
Returns a member of one of the enums like Instruction::Add.
Definition: Instruction.h:169
llvm::InductionDescriptor
A struct for saving information about induction variables.
Definition: IVDescriptors.h:306
llvm::VPWidenPHIRecipe::getIncomingBlock
VPBasicBlock * getIncomingBlock(unsigned I)
Returns the I th incoming VPBasicBlock.
Definition: VPlan.h:1272
llvm::VFDatabase
The Vector Function Database.
Definition: VectorUtils.h:208
llvm::createStepForVF
Value * createStepForVF(IRBuilderBase &B, Type *Ty, ElementCount VF, int64_t Step)
Return a value for Step multiplied by VF.
Definition: LoopVectorize.cpp:955
E
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
llvm::IRBuilderBase::CreateGEP
Value * CreateGEP(Type *Ty, Value *Ptr, ArrayRef< Value * > IdxList, const Twine &Name="", bool IsInBounds=false)
Definition: IRBuilder.h:1764
llvm::VPInstruction::CanonicalIVIncrementForPart
@ CanonicalIVIncrementForPart
Definition: VPlan.h:793
llvm::RecurrenceDescriptor::isSelectCmpRecurrenceKind
static bool isSelectCmpRecurrenceKind(RecurKind Kind)
Returns true if the recurrence kind is of the form select(cmp(),x,y) where one of (x,...
Definition: IVDescriptors.h:239
C
(vector float) vec_cmpeq(*A, *B) C
Definition: README_ALTIVEC.txt:86
Twine.h
llvm::ToVectorTy
Type * ToVectorTy(Type *Scalar, ElementCount EC)
A helper function for converting Scalar types to vector types.
Definition: VectorUtils.h:299
llvm::VPRecipeBase::moveBefore
void moveBefore(VPBasicBlock &BB, iplist< VPRecipeBase >::iterator I)
Unlink this recipe and insert into BB before I.
Definition: VPlanRecipes.cpp:186
llvm::ReplaceInstWithInst
void ReplaceInstWithInst(BasicBlock::InstListType &BIL, BasicBlock::iterator &BI, Instruction *I)
Replace the instruction specified by BI with the instruction specified by I.
Definition: BasicBlockUtils.cpp:558
llvm::Type::isVectorTy
bool isVectorTy() const
True if this is an instance of VectorType.
Definition: Type.h:246
llvm::InsertElementInst
This instruction inserts a single (scalar) element into a VectorType value.
Definition: Instructions.h:1934
llvm::BasicBlock::getFirstInsertionPt
const_iterator getFirstInsertionPt() const
Returns an iterator to the first instruction in this block that is suitable for inserting a non-PHI i...
Definition: BasicBlock.cpp:246
B
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
llvm::SmallBitVector::all
bool all() const
Returns true if all bits are set.
Definition: SmallBitVector.h:216
llvm::VPWidenCanonicalIVRecipe::print
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
Definition: VPlanRecipes.cpp:1130
llvm::Instruction::getOpcodeName
const char * getOpcodeName() const
Definition: Instruction.h:171
llvm::RecurrenceDescriptor::getRecurrenceIdentity
Value * getRecurrenceIdentity(RecurKind K, Type *Tp, FastMathFlags FMF) const
Returns identity corresponding to the RecurrenceKind.
Definition: IVDescriptors.cpp:1075
llvm::Instruction
Definition: Instruction.h:42
EnableVPlanNativePath
cl::opt< bool > EnableVPlanNativePath
llvm::RecurrenceDescriptor::getRecurrenceKind
RecurKind getRecurrenceKind() const
Definition: IVDescriptors.h:198
llvm::raw_ostream
This class implements an extremely fast bulk output stream that can only output to a stream.
Definition: raw_ostream.h:53
llvm::VPValue
Definition: VPlanValue.h:44
llvm::ConstantInt::get
static Constant * get(Type *Ty, uint64_t V, bool IsSigned=false)
If Ty is a vector type, return a Constant with a splat of the given value.
Definition: Constants.cpp:879
llvm::VPWidenCallRecipe::execute
void execute(VPTransformState &State) override
Produce a widened version of the call instruction.
Definition: VPlanRecipes.cpp:440
llvm::BasicBlock::getModule
const Module * getModule() const
Return the module owning the function this basic block belongs to, or nullptr if the function does no...
Definition: BasicBlock.cpp:147
VPlan.h
llvm::VFShape
Contains the information about the kind of vectorization available.
Definition: VectorUtils.h:82
llvm::MCID::Call
@ Call
Definition: MCInstrDesc.h:155
IVDescriptors.h
llvm::VPUser::getNumOperands
unsigned getNumOperands() const
Definition: VPlanValue.h:270
llvm::CallingConv::ID
unsigned ID
LLVM IR allows to use arbitrary numbers as calling convention identifiers.
Definition: CallingConv.h:24
Type.h
llvm::VPRegionBlock
VPRegionBlock represents a collection of VPBasicBlocks and VPRegionBlocks which form a Single-Entry-S...
Definition: VPlan.h:2189
llvm::VPWidenMemoryInstructionRecipe::isStore
bool isStore() const
Returns true if this recipe is a store.
Definition: VPlan.h:1758
llvm::VPTransformState::setDebugLocFromInst
void setDebugLocFromInst(const Value *V)
Set the debug location in the builder using the debug location in V.
Definition: VPlan.cpp:258
llvm::IRBuilderBase::FastMathFlagGuard
Definition: IRBuilder.h:383
llvm::tgtok::In
@ In
Definition: TGLexer.h:51
llvm::VPActiveLaneMaskPHIRecipe::execute
void execute(VPTransformState &State) override
Generate the active lane mask phi of the vector loop.
Definition: VPlanRecipes.cpp:1301
BasicBlock.h
llvm::cl::opt< bool >
llvm::VPRecipeBase::removeFromParent
void removeFromParent()
This method unlinks 'this' from the containing basic block, but does not delete it.
Definition: VPlanRecipes.cpp:170
llvm::RISCVFenceField::O
@ O
Definition: RISCVBaseInfo.h:264
llvm::VPRecipeBase::mayHaveSideEffects
bool mayHaveSideEffects() const
Returns true if the recipe may have side-effects.
Definition: VPlanRecipes.cpp:106
llvm::Constant
This is an important base class in LLVM.
Definition: Constant.h:41
llvm::IRBuilderBase::GetInsertPoint
BasicBlock::iterator GetInsertPoint() const
Definition: IRBuilder.h:175
llvm::VPExpandSCEVRecipe::execute
void execute(VPTransformState &State) override
Generate a canonical vector induction variable of the vector loop, with.
Definition: VPlanRecipes.cpp:1088
llvm::VPInstruction::BranchOnCond
@ BranchOnCond
Definition: VPlan.h:796
llvm::VPReductionRecipe::getCondOp
VPValue * getCondOp() const
The VPValue of the condition for the block.
Definition: VPlan.h:1536
llvm::VPWidenGEPRecipe::execute
void execute(VPTransformState &State) override
Generate the gep nodes.
Definition: VPlanRecipes.cpp:745
llvm::VPInstruction::execute
void execute(VPTransformState &State) override
Generate the instruction.
Definition: VPlanRecipes.cpp:352
llvm::VPReductionPHIRecipe::print
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
Definition: VPlanRecipes.cpp:1242
llvm::IRBuilderBase::getInt32
ConstantInt * getInt32(uint32_t C)
Get a constant 32-bit value.
Definition: IRBuilder.h:472
llvm::PHINode::addIncoming
void addIncoming(Value *V, BasicBlock *BB)
Add an incoming value to the end of the PHI list.
Definition: Instructions.h:2847
llvm::VFDatabase::getVectorizedFunction
Function * getVectorizedFunction(const VFShape &Shape) const
Definition: VectorUtils.h:268
llvm::VPLane::getLastLaneForVF
static VPLane getLastLaneForVF(const ElementCount &VF)
Definition: VPlan.h:138
llvm::BranchInst::Create
static BranchInst * Create(BasicBlock *IfTrue, Instruction *InsertBefore=nullptr)
Definition: Instructions.h:3188
llvm::VPTransformState::UF
unsigned UF
Definition: VPlan.h:214
llvm::VPInstruction::print
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the VPInstruction to O.
Definition: VPlanRecipes.cpp:366
I
#define I(x, y, z)
Definition: MD5.cpp:58
llvm::VPIteration
VPIteration represents a single point in the iteration space of the output (vectorized and/or unrolle...
Definition: VPlan.h:188
llvm::VPHeaderPHIRecipe::getStartValue
VPValue * getStartValue()
Returns the start value of the phi, if one is set.
Definition: VPlan.h:1164
llvm::IRBuilderBase::CreateSelect
Value * CreateSelect(Value *C, Value *True, Value *False, const Twine &Name="", Instruction *MDFrom=nullptr)
Definition: IRBuilder.cpp:1127
llvm::VPBlendRecipe::getMask
VPValue * getMask(unsigned Idx) const
Return mask number Idx.
Definition: VPlan.h:1394
llvm::VPWidenGEPRecipe::print
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
Definition: VPlanRecipes.cpp:819
llvm::Instruction::setDebugLoc
void setDebugLoc(DebugLoc Loc)
Set the debug location information for this instruction.
Definition: Instruction.h:356
IRBuilder.h
assert
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
llvm::VPDerivedIVRecipe::print
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
Definition: VPlanRecipes.cpp:719
llvm::Instruction::getFastMathFlags
FastMathFlags getFastMathFlags() const LLVM_READONLY
Convenience function for getting all the fast-math flags, which must be an operator which supports th...
Definition: Instruction.cpp:326
llvm::IRBuilderBase::CreateExtractElement
Value * CreateExtractElement(Value *Vec, Value *Idx, const Twine &Name="")
Definition: IRBuilder.h:2324
llvm::VPInstruction::hasResult
bool hasResult() const
Definition: VPlan.h:875
Ptr
@ Ptr
Definition: TargetLibraryInfo.cpp:60
llvm::Instruction::isBinaryOp
bool isBinaryOp() const
Definition: Instruction.h:174
llvm::Type::isVoidTy
bool isVoidTy() const
Return true if this is 'void'.
Definition: Type.h:139
llvm::IRBuilderBase::getTrue
ConstantInt * getTrue()
Get the constant value for i1 true.
Definition: IRBuilder.h:452
llvm::VPUser::printOperands
void printOperands(raw_ostream &O, VPSlotTracker &SlotTracker) const
Print the operands to O.
Definition: VPlan.cpp:1020
llvm::Module
A Module instance is used to store all the information related to an LLVM module.
Definition: Module.h:66
llvm::LinearPolySize::getKnownMinValue
ScalarTy getKnownMinValue() const
Returns the minimum value this size can represent.
Definition: TypeSize.h:296
getOpcode
static std::optional< unsigned > getOpcode(ArrayRef< VPValue * > Values)
Returns the opcode of Values or ~0 if they do not all agree.
Definition: VPlanSLP.cpp:191
llvm::VPWidenRecipe::print
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
Definition: VPlanRecipes.cpp:685
llvm::ElementCount::isVector
bool isVector() const
One or more elements.
Definition: TypeSize.h:416
Builder
assume Assume Builder
Definition: AssumeBundleBuilder.cpp:651
llvm::Value::printAsOperand
void printAsOperand(raw_ostream &O, bool PrintType=true, const Module *M=nullptr) const
Print the name of this Value out to the specified raw_ostream.
Definition: AsmWriter.cpp:4760
llvm::VPWidenIntOrFpInductionRecipe::isCanonical
bool isCanonical() const
Returns true if the induction is canonical, i.e.
Definition: VPlanRecipes.cpp:712
llvm::VPWidenCallRecipe::print
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
Definition: VPlanRecipes.cpp:499
llvm::IRBuilderBase::CreatePHI
PHINode * CreatePHI(Type *Ty, unsigned NumReservedValues, const Twine &Name="")
Definition: IRBuilder.h:2261
llvm::VPTransformState::reset
void reset(VPValue *Def, Value *V, unsigned Part)
Reset an existing vector value for Def and a given Part.
Definition: VPlan.h:272
llvm::VPScalarIVStepsRecipe::print
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
Definition: VPlanRecipes.cpp:736
llvm::VPTransformState::addMetadata
void addMetadata(Instruction *To, Instruction *From)
Add metadata from one instruction to another.
Definition: VPlan.cpp:246
llvm::getRuntimeVF
Value * getRuntimeVF(IRBuilderBase &B, Type *Ty, ElementCount VF)
Return the runtime value for VF.
Definition: LoopVectorize.cpp:963
llvm::VPReductionRecipe::getVecOp
VPValue * getVecOp() const
The VPValue of the vector value to be reduced.
Definition: VPlan.h:1534
Mul
BinaryOperator * Mul
Definition: X86PartialReduction.cpp:70
llvm::RecurrenceDescriptor::IntermediateStore
StoreInst * IntermediateStore
Reductions may store temporary or final result to an invariant address.
Definition: IVDescriptors.h:276
Cond
SmallVector< MachineOperand, 4 > Cond
Definition: BasicBlockSections.cpp:138
llvm::RecurrenceDescriptor::getFastMathFlags
FastMathFlags getFastMathFlags() const
Definition: IVDescriptors.h:202
llvm::VPCanonicalIVPHIRecipe::isCanonical
bool isCanonical(const InductionDescriptor &ID, Type *Ty) const
Check if the induction described by ID is canonical, i.e.
Definition: VPlanRecipes.cpp:1056
llvm_unreachable
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
Definition: ErrorHandling.h:143
llvm::Value::getType
Type * getType() const
All values are typed, get the type of this value.
Definition: Value.h:255
llvm::InductionDescriptor::getStep
const SCEV * getStep() const
Definition: IVDescriptors.h:322
llvm::VPInstruction::BranchOnCount
@ BranchOnCount
Definition: VPlan.h:795
llvm::VPRecipeBase::getParent
VPBasicBlock * getParent()
Definition: VPlan.h:682
llvm::IRBuilderBase::InsertPointGuard
Definition: IRBuilder.h:361
llvm::IRBuilderBase
Common base class shared among various IRBuilders.
Definition: IRBuilder.h:94
llvm::ilist_node_impl::getIterator
self_iterator getIterator()
Definition: ilist_node.h:82
DL
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
Definition: AArch64SLSHardening.cpp:76
llvm::VPCanonicalIVPHIRecipe::execute
void execute(VPTransformState &State) override
Generate the canonical scalar induction phi of the vector loop.
Definition: VPlanRecipes.cpp:1035
llvm::Instruction::copyFastMathFlags
void copyFastMathFlags(FastMathFlags FMF)
Convenience function for transferring all fast-math flag values to this instruction,...
Definition: Instruction.cpp:281
llvm::VPWidenSelectRecipe::execute
void execute(VPTransformState &State) override
Produce a widened version of the select instruction.
Definition: VPlanRecipes.cpp:535
llvm::VPTransformState::VF
ElementCount VF
The chosen Vectorization and Unroll Factors of the loop being vectorized.
Definition: VPlan.h:213
llvm::VPBlockBase::getEntryBasicBlock
const VPBasicBlock * getEntryBasicBlock() const
Definition: VPlan.cpp:147
llvm::vputils::onlyFirstLaneUsed
bool onlyFirstLaneUsed(VPValue *Def)
Returns true if only the first lane of Def is used.
Definition: VPlan.cpp:1103
llvm::Intrinsic::getDeclaration
Function * getDeclaration(Module *M, ID id, ArrayRef< Type * > Tys=std::nullopt)
Create or insert an LLVM Function declaration for an intrinsic, and return it.
Definition: Function.cpp:1481
llvm::VPDef::getVPValue
VPValue * getVPValue(unsigned I)
Returns the VPValue with index I defined by the VPDef.
Definition: VPlanValue.h:416
llvm::vputils::isUniformAfterVectorization
bool isUniformAfterVectorization(VPValue *VPV)
Returns true if VPV is uniform after vectorization.
Definition: VPlan.h:3109
llvm::VPFirstOrderRecurrencePHIRecipe::print
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
Definition: VPlanRecipes.cpp:1168
llvm::VPInstruction::CanonicalIVIncrementForPartNUW
@ CanonicalIVIncrementForPartNUW
Definition: VPlan.h:794
llvm::MCID::Select
@ Select
Definition: MCInstrDesc.h:164
llvm::VPReductionPHIRecipe::execute
void execute(VPTransformState &State) override
Generate the phi/select nodes.
Definition: VPlanRecipes.cpp:1177
llvm::NVPTX::PTXLdStInstCode::V2
@ V2
Definition: NVPTX.h:123
llvm::Twine
Twine - A lightweight data structure for efficiently representing the concatenation of temporary valu...
Definition: Twine.h:81
llvm::VPSlotTracker
This class can be used to assign consecutive numbers to all VPValues in a VPlan and allows querying t...
Definition: VPlanValue.h:454
llvm::VPBlockBase::getParent
VPRegionBlock * getParent()
Definition: VPlan.h:461
llvm::VPTransformState::CurrentVectorLoop
Loop * CurrentVectorLoop
The loop object for the current parent region, or nullptr.
Definition: VPlan.h:377
llvm::AMDGPU::SendMsg::Op
Op
Definition: SIDefines.h:348
llvm::VPTransformState::MayGeneratePoisonRecipes
SmallPtrSet< VPRecipeBase *, 16 > MayGeneratePoisonRecipes
Holds recipes that may generate a poison value that is used after vectorization, even when their oper...
Definition: VPlan.h:374
llvm::PHINode::Create
static PHINode * Create(Type *Ty, unsigned NumReservedValues, const Twine &NameStr="", Instruction *InsertBefore=nullptr)
Constructors - NumReservedValues is a hint for the number of incoming edges that this phi node will h...
Definition: Instructions.h:2739
llvm::IRBuilderBase::Insert
InstTy * Insert(InstTy *I, const Twine &Name="") const
Insert and return the specified instruction.
Definition: IRBuilder.h:145
llvm::VPRecipeBase
VPRecipeBase is a base class modeling a sequence of one or more output IR instructions.
Definition: VPlan.h:663
llvm::IRBuilderBase::GetInsertBlock
BasicBlock * GetInsertBlock() const
Definition: IRBuilder.h:174
Casting.h
llvm::LoopBase::getHeader
BlockT * getHeader() const
Definition: LoopInfo.h:105
llvm::VPInstruction::CanonicalIVIncrement
@ CanonicalIVIncrement
Definition: VPlan.h:789
llvm::DebugLoc::print
void print(raw_ostream &OS) const
prints source location /path/to/file.exe:line:col @[inlined at]
Definition: DebugLoc.cpp:103
llvm::SlotTracker
This class provides computation of slot numbers for LLVM Assembly writing.
Definition: AsmWriter.cpp:665
llvm::MCID::Add
@ Add
Definition: MCInstrDesc.h:185
llvm::VPWidenPointerInductionRecipe::print
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
Definition: VPlanRecipes.cpp:1078
llvm::iplist_impl::insertAfter
iterator insertAfter(iterator where, pointer New)
Definition: ilist.h:238
llvm::iplist_impl::insert
iterator insert(iterator where, pointer New)
Definition: ilist.h:229
llvm::SmallBitVector::size
size_type size() const
Returns the number of bits in this bitvector.
Definition: SmallBitVector.h:195
llvm::Instruction::BinaryOps
BinaryOps
Definition: Instruction.h:793
llvm::InsertElementInst::getType
VectorType * getType() const
Overload to return most specific vector type.
Definition: Instructions.h:1967
llvm::VPInstruction::CanonicalIVIncrementNUW
@ CanonicalIVIncrementNUW
Definition: VPlan.h:790
Instructions.h
llvm::VPInstruction::SLPStore
@ SLPStore
Definition: VPlan.h:787
llvm::VPInstruction::ICmpULE
@ ICmpULE
Definition: VPlan.h:785
SmallVector.h
llvm::IRBuilderBase::CreateCall
CallInst * CreateCall(FunctionType *FTy, Value *Callee, ArrayRef< Value * > Args=std::nullopt, const Twine &Name="", MDNode *FPMathTag=nullptr)
Definition: IRBuilder.h:2276
llvm::VPUser::getOperand
VPValue * getOperand(unsigned N) const
Definition: VPlanValue.h:271
llvm::VPTransformState::Instance
Optional< VPIteration > Instance
Hold the indices to generate specific scalar instructions.
Definition: VPlan.h:219
llvm::VPRecipeBase::mayReadFromMemory
bool mayReadFromMemory() const
Returns true if the recipe may read from memory.
Definition: VPlanRecipes.cpp:74
llvm::Instruction::getParent
const BasicBlock * getParent() const
Definition: Instruction.h:91
llvm::VFShape::get
static VFShape get(const CallInst &CI, ElementCount EC, bool HasGlobalPred)
Definition: VectorUtils.h:108
llvm::VPActiveLaneMaskPHIRecipe::print
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
Definition: VPlanRecipes.cpp:1314
llvm::PHINode
Definition: Instructions.h:2697
llvm::BasicBlock::getTerminator
const Instruction * getTerminator() const LLVM_READONLY
Returns the terminator instruction if the block is well formed or null if the block is not well forme...
Definition: BasicBlock.h:119
llvm::VPBlockBase::getExitingBasicBlock
const VPBasicBlock * getExitingBasicBlock() const
Definition: VPlan.cpp:168
llvm::VPFirstOrderRecurrencePHIRecipe::execute
void execute(VPTransformState &State) override
Generate the phi nodes.
Definition: VPlanRecipes.cpp:1139
IV
static const uint32_t IV[8]
Definition: blake3_impl.h:85
llvm::VPTransformState::CFGState::VPBB2IRBB
SmallDenseMap< VPBasicBlock *, BasicBlock * > VPBB2IRBB
A mapping of each VPBasicBlock to the corresponding BasicBlock.
Definition: VPlan.h:343
llvm::IRBuilderBase::CreateVectorSplat
Value * CreateVectorSplat(unsigned NumElts, Value *V, const Twine &Name="")
Return a vector value that contains.
Definition: IRBuilder.cpp:1250
llvm::Module::getDataLayout
const DataLayout & getDataLayout() const
Get the data layout for the module's target platform.
Definition: Module.cpp:399
llvm::SCEV::getType
Type * getType() const
Return the LLVM type of this SCEV expression.
Definition: ScalarEvolution.cpp:403
llvm::CallInst
This class represents a function call, abstracting a target machine's calling convention.
Definition: Instructions.h:1474
llvm::ConstantInt::isOne
bool isOne() const
This is just a convenience method to make client code smaller for a common case.
Definition: Constants.h:200
llvm::VPBranchOnMaskRecipe::getMask
VPValue * getMask() const
Return the mask used by this recipe.
Definition: VPlan.h:1646
BB
Common register allocation spilling lr str ldr sxth r3 ldr mla r4 can lr mov lr str ldr sxth r3 mla r4 and then merge mul and lr str ldr sxth r3 mla r4 It also increase the likelihood the store may become dead bb27 Successors according to LLVM BB
Definition: README.txt:39
GEP
Hexagon Common GEP
Definition: HexagonCommonGEP.cpp:171
llvm::VPDerivedIVRecipe::getStartValue
VPValue * getStartValue() const
Definition: VPlan.h:2004
llvm::VPWidenPHIRecipe::print
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
Definition: VPlanRecipes.cpp:1279
llvm::AMDGPU::HSAMD::Kernel::Key::Args
constexpr char Args[]
Key for Kernel::Metadata::mArgs.
Definition: AMDGPUMetadata.h:394
llvm::User::getOperand
Value * getOperand(unsigned i) const
Definition: User.h:169
llvm::RecurrenceDescriptor::isMinMaxRecurrenceKind
static bool isMinMaxRecurrenceKind(RecurKind Kind)
Returns true if the recurrence kind is any min/max kind.
Definition: IVDescriptors.h:233
llvm::BranchInst
Conditional or Unconditional Branch instruction.
Definition: Instructions.h:3132
raw_ostream.h
llvm::VPReductionRecipe::getChainOp
VPValue * getChainOp() const
The VPValue of the scalar Chain being accumulated.
Definition: VPlan.h:1532
llvm::VPExpandSCEVRecipe::print
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
Definition: VPlanRecipes.cpp:1101
BasicBlockUtils.h
llvm::VPWidenRecipe::execute
void execute(VPTransformState &State) override
Produce widened copies of all Ingredients.
Definition: VPlanRecipes.cpp:556
llvm::VPDef::getVPDefID
unsigned getVPDefID() const
Definition: VPlanValue.h:436
llvm::VPInstruction::dump
LLVM_DUMP_METHOD void dump() const
Print the VPInstruction to dbgs() (for debugging).
Definition: VPlanRecipes.cpp:361
Value.h
llvm::VPValue::getUnderlyingValue
Value * getUnderlyingValue()
Return the underlying Value attached to this VPValue.
Definition: VPlanValue.h:84
llvm::VPlan
VPlan models a candidate for vectorization, encoding various decisions take to produce efficient outp...
Definition: VPlan.h:2544
llvm::Value
LLVM Value Representation.
Definition: Value.h:74
llvm::VPRecipeBase::insertBefore
void insertBefore(VPRecipeBase *InsertPos)
Insert an unlinked recipe into a basic block immediately before the specified recipe.
Definition: VPlanRecipes.cpp:146
Debug.h
llvm::VectorType::get
static VectorType * get(Type *ElementType, ElementCount EC)
This static method is the primary way to construct an VectorType.
Definition: Type.cpp:668
llvm::VPReductionPHIRecipe::isOrdered
bool isOrdered() const
Returns true, if the phi is part of an ordered reduction.
Definition: VPlan.h:1357
llvm::VPlanIngredient
Definition: VPlan.h:2830
llvm::VPWidenSelectRecipe::print
void print(raw_ostream &O, const Twine &Indent, VPSlotTracker &SlotTracker) const override
Print the recipe.
Definition: VPlanRecipes.cpp:521
llvm::VPValue::printAsOperand
void printAsOperand(raw_ostream &OS, VPSlotTracker &Tracker) const
Definition: VPlan.cpp:1005
llvm::VPWidenIntOrFpInductionRecipe::getStartValue
VPValue * getStartValue()
Returns the start value of the induction.
Definition: VPlan.h:1093
llvm::VPTransformState::CFGState::PrevBB
BasicBlock * PrevBB
The previous IR BasicBlock created or used.
Definition: VPlan.h:335
llvm::VPRecipeBase::insertAfter
void insertAfter(VPRecipeBase *InsertPos)
Insert an unlinked Recipe into a basic block immediately after the specified Recipe.
Definition: VPlanRecipes.cpp:162
llvm::VPTransformState::hasVectorValue
bool hasVectorValue(VPValue *Def, unsigned Part)
Definition: VPlan.h:243
llvm::PoisonValue::get
static PoisonValue * get(Type *T)
Static factory methods - Return an 'poison' object of the specified type.
Definition: Constants.cpp:1732