LLVM 17.0.0git
VPlanTransforms.cpp
Go to the documentation of this file.
1//===-- VPlanTransforms.cpp - Utility VPlan to VPlan transforms -----------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8///
9/// \file
10/// This file implements a set of utility VPlan to VPlan transformations.
11///
12//===----------------------------------------------------------------------===//
13
14#include "VPlanTransforms.h"
15#include "VPlanDominatorTree.h"
16#include "VPRecipeBuilder.h"
17#include "VPlanCFG.h"
19#include "llvm/ADT/SetVector.h"
22#include "llvm/IR/Intrinsics.h"
23
24using namespace llvm;
25
27 VPlanPtr &Plan,
29 GetIntOrFpInductionDescriptor,
31 const TargetLibraryInfo &TLI) {
32
34 Plan->getEntry());
35 for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(RPOT)) {
36 VPRecipeBase *Term = VPBB->getTerminator();
37 auto EndIter = Term ? Term->getIterator() : VPBB->end();
38 // Introduce each ingredient into VPlan.
39 for (VPRecipeBase &Ingredient :
40 make_early_inc_range(make_range(VPBB->begin(), EndIter))) {
41
42 VPValue *VPV = Ingredient.getVPSingleValue();
43 Instruction *Inst = cast<Instruction>(VPV->getUnderlyingValue());
44 if (DeadInstructions.count(Inst)) {
45 VPValue DummyValue;
46 VPV->replaceAllUsesWith(&DummyValue);
47 Ingredient.eraseFromParent();
48 continue;
49 }
50
51 VPRecipeBase *NewRecipe = nullptr;
52 if (auto *VPPhi = dyn_cast<VPWidenPHIRecipe>(&Ingredient)) {
53 auto *Phi = cast<PHINode>(VPPhi->getUnderlyingValue());
54 if (const auto *II = GetIntOrFpInductionDescriptor(Phi)) {
55 VPValue *Start = Plan->getOrAddVPValue(II->getStartValue());
56 VPValue *Step =
57 vputils::getOrCreateVPValueForSCEVExpr(*Plan, II->getStep(), SE);
58 NewRecipe =
59 new VPWidenIntOrFpInductionRecipe(Phi, Start, Step, *II, true);
60 } else {
61 Plan->addVPValue(Phi, VPPhi);
62 continue;
63 }
64 } else {
65 assert(isa<VPInstruction>(&Ingredient) &&
66 "only VPInstructions expected here");
67 assert(!isa<PHINode>(Inst) && "phis should be handled above");
68 // Create VPWidenMemoryInstructionRecipe for loads and stores.
69 if (LoadInst *Load = dyn_cast<LoadInst>(Inst)) {
70 NewRecipe = new VPWidenMemoryInstructionRecipe(
71 *Load, Plan->getOrAddVPValue(getLoadStorePointerOperand(Inst)),
72 nullptr /*Mask*/, false /*Consecutive*/, false /*Reverse*/);
73 } else if (StoreInst *Store = dyn_cast<StoreInst>(Inst)) {
74 NewRecipe = new VPWidenMemoryInstructionRecipe(
75 *Store, Plan->getOrAddVPValue(getLoadStorePointerOperand(Inst)),
76 Plan->getOrAddVPValue(Store->getValueOperand()), nullptr /*Mask*/,
77 false /*Consecutive*/, false /*Reverse*/);
78 } else if (GetElementPtrInst *GEP = dyn_cast<GetElementPtrInst>(Inst)) {
79 NewRecipe =
80 new VPWidenGEPRecipe(GEP, Plan->mapToVPValues(GEP->operands()));
81 } else if (CallInst *CI = dyn_cast<CallInst>(Inst)) {
82 NewRecipe =
83 new VPWidenCallRecipe(*CI, Plan->mapToVPValues(CI->args()),
85 } else if (SelectInst *SI = dyn_cast<SelectInst>(Inst)) {
86 NewRecipe =
87 new VPWidenSelectRecipe(*SI, Plan->mapToVPValues(SI->operands()));
88 } else {
89 NewRecipe =
90 new VPWidenRecipe(*Inst, Plan->mapToVPValues(Inst->operands()));
91 }
92 }
93
94 NewRecipe->insertBefore(&Ingredient);
95 if (NewRecipe->getNumDefinedValues() == 1)
96 VPV->replaceAllUsesWith(NewRecipe->getVPSingleValue());
97 else
98 assert(NewRecipe->getNumDefinedValues() == 0 &&
99 "Only recpies with zero or one defined values expected");
100 Ingredient.eraseFromParent();
101 Plan->removeVPValueFor(Inst);
102 for (auto *Def : NewRecipe->definedValues()) {
103 Plan->addVPValue(Inst, Def);
104 }
105 }
106 }
107}
108
109static bool sinkScalarOperands(VPlan &Plan) {
110 auto Iter = vp_depth_first_deep(Plan.getEntry());
111 bool Changed = false;
112 // First, collect the operands of all recipes in replicate blocks as seeds for
113 // sinking.
115 for (VPRegionBlock *VPR : VPBlockUtils::blocksOnly<VPRegionBlock>(Iter)) {
116 VPBasicBlock *EntryVPBB = VPR->getEntryBasicBlock();
117 if (!VPR->isReplicator() || EntryVPBB->getSuccessors().size() != 2)
118 continue;
119 VPBasicBlock *VPBB = dyn_cast<VPBasicBlock>(EntryVPBB->getSuccessors()[0]);
120 if (!VPBB || VPBB->getSingleSuccessor() != VPR->getExitingBasicBlock())
121 continue;
122 for (auto &Recipe : *VPBB) {
123 for (VPValue *Op : Recipe.operands())
124 if (auto *Def = Op->getDefiningRecipe())
125 WorkList.insert(std::make_pair(VPBB, Def));
126 }
127 }
128
129 bool ScalarVFOnly = Plan.hasScalarVFOnly();
130 // Try to sink each replicate or scalar IV steps recipe in the worklist.
131 for (unsigned I = 0; I != WorkList.size(); ++I) {
132 VPBasicBlock *SinkTo;
133 VPRecipeBase *SinkCandidate;
134 std::tie(SinkTo, SinkCandidate) = WorkList[I];
135 if (SinkCandidate->getParent() == SinkTo ||
136 SinkCandidate->mayHaveSideEffects() ||
137 SinkCandidate->mayReadOrWriteMemory())
138 continue;
139 if (auto *RepR = dyn_cast<VPReplicateRecipe>(SinkCandidate)) {
140 if (!ScalarVFOnly && RepR->isUniform())
141 continue;
142 } else if (!isa<VPScalarIVStepsRecipe>(SinkCandidate))
143 continue;
144
145 bool NeedsDuplicating = false;
146 // All recipe users of the sink candidate must be in the same block SinkTo
147 // or all users outside of SinkTo must be uniform-after-vectorization (
148 // i.e., only first lane is used) . In the latter case, we need to duplicate
149 // SinkCandidate.
150 auto CanSinkWithUser = [SinkTo, &NeedsDuplicating,
151 SinkCandidate](VPUser *U) {
152 auto *UI = dyn_cast<VPRecipeBase>(U);
153 if (!UI)
154 return false;
155 if (UI->getParent() == SinkTo)
156 return true;
157 NeedsDuplicating =
158 UI->onlyFirstLaneUsed(SinkCandidate->getVPSingleValue());
159 // We only know how to duplicate VPRecipeRecipes for now.
160 return NeedsDuplicating && isa<VPReplicateRecipe>(SinkCandidate);
161 };
162 if (!all_of(SinkCandidate->getVPSingleValue()->users(), CanSinkWithUser))
163 continue;
164
165 if (NeedsDuplicating) {
166 if (ScalarVFOnly)
167 continue;
168 Instruction *I = cast<Instruction>(
169 cast<VPReplicateRecipe>(SinkCandidate)->getUnderlyingValue());
170 auto *Clone = new VPReplicateRecipe(I, SinkCandidate->operands(), true);
171 // TODO: add ".cloned" suffix to name of Clone's VPValue.
172
173 Clone->insertBefore(SinkCandidate);
174 for (auto *U : to_vector(SinkCandidate->getVPSingleValue()->users())) {
175 auto *UI = cast<VPRecipeBase>(U);
176 if (UI->getParent() == SinkTo)
177 continue;
178
179 for (unsigned Idx = 0; Idx != UI->getNumOperands(); Idx++) {
180 if (UI->getOperand(Idx) != SinkCandidate->getVPSingleValue())
181 continue;
182 UI->setOperand(Idx, Clone);
183 }
184 }
185 }
186 SinkCandidate->moveBefore(*SinkTo, SinkTo->getFirstNonPhi());
187 for (VPValue *Op : SinkCandidate->operands())
188 if (auto *Def = Op->getDefiningRecipe())
189 WorkList.insert(std::make_pair(SinkTo, Def));
190 Changed = true;
191 }
192 return Changed;
193}
194
195/// If \p R is a region with a VPBranchOnMaskRecipe in the entry block, return
196/// the mask.
198 auto *EntryBB = dyn_cast<VPBasicBlock>(R->getEntry());
199 if (!EntryBB || EntryBB->size() != 1 ||
200 !isa<VPBranchOnMaskRecipe>(EntryBB->begin()))
201 return nullptr;
202
203 return cast<VPBranchOnMaskRecipe>(&*EntryBB->begin())->getOperand(0);
204}
205
206/// If \p R is a triangle region, return the 'then' block of the triangle.
208 auto *EntryBB = cast<VPBasicBlock>(R->getEntry());
209 if (EntryBB->getNumSuccessors() != 2)
210 return nullptr;
211
212 auto *Succ0 = dyn_cast<VPBasicBlock>(EntryBB->getSuccessors()[0]);
213 auto *Succ1 = dyn_cast<VPBasicBlock>(EntryBB->getSuccessors()[1]);
214 if (!Succ0 || !Succ1)
215 return nullptr;
216
217 if (Succ0->getNumSuccessors() + Succ1->getNumSuccessors() != 1)
218 return nullptr;
219 if (Succ0->getSingleSuccessor() == Succ1)
220 return Succ0;
221 if (Succ1->getSingleSuccessor() == Succ0)
222 return Succ1;
223 return nullptr;
224}
225
226// Merge replicate regions in their successor region, if a replicate region
227// is connected to a successor replicate region with the same predicate by a
228// single, empty VPBasicBlock.
230 SetVector<VPRegionBlock *> DeletedRegions;
231
232 // Collect replicate regions followed by an empty block, followed by another
233 // replicate region with matching masks to process front. This is to avoid
234 // iterator invalidation issues while merging regions.
236 for (VPRegionBlock *Region1 : VPBlockUtils::blocksOnly<VPRegionBlock>(
237 vp_depth_first_deep(Plan.getEntry()))) {
238 if (!Region1->isReplicator())
239 continue;
240 auto *MiddleBasicBlock =
241 dyn_cast_or_null<VPBasicBlock>(Region1->getSingleSuccessor());
242 if (!MiddleBasicBlock || !MiddleBasicBlock->empty())
243 continue;
244
245 auto *Region2 =
246 dyn_cast_or_null<VPRegionBlock>(MiddleBasicBlock->getSingleSuccessor());
247 if (!Region2 || !Region2->isReplicator())
248 continue;
249
250 VPValue *Mask1 = getPredicatedMask(Region1);
251 VPValue *Mask2 = getPredicatedMask(Region2);
252 if (!Mask1 || Mask1 != Mask2)
253 continue;
254
255 assert(Mask1 && Mask2 && "both region must have conditions");
256 WorkList.push_back(Region1);
257 }
258
259 // Move recipes from Region1 to its successor region, if both are triangles.
260 for (VPRegionBlock *Region1 : WorkList) {
261 if (DeletedRegions.contains(Region1))
262 continue;
263 auto *MiddleBasicBlock = cast<VPBasicBlock>(Region1->getSingleSuccessor());
264 auto *Region2 = cast<VPRegionBlock>(MiddleBasicBlock->getSingleSuccessor());
265
266 VPBasicBlock *Then1 = getPredicatedThenBlock(Region1);
267 VPBasicBlock *Then2 = getPredicatedThenBlock(Region2);
268 if (!Then1 || !Then2)
269 continue;
270
271 // Note: No fusion-preventing memory dependencies are expected in either
272 // region. Such dependencies should be rejected during earlier dependence
273 // checks, which guarantee accesses can be re-ordered for vectorization.
274 //
275 // Move recipes to the successor region.
276 for (VPRecipeBase &ToMove : make_early_inc_range(reverse(*Then1)))
277 ToMove.moveBefore(*Then2, Then2->getFirstNonPhi());
278
279 auto *Merge1 = cast<VPBasicBlock>(Then1->getSingleSuccessor());
280 auto *Merge2 = cast<VPBasicBlock>(Then2->getSingleSuccessor());
281
282 // Move VPPredInstPHIRecipes from the merge block to the successor region's
283 // merge block. Update all users inside the successor region to use the
284 // original values.
285 for (VPRecipeBase &Phi1ToMove : make_early_inc_range(reverse(*Merge1))) {
286 VPValue *PredInst1 =
287 cast<VPPredInstPHIRecipe>(&Phi1ToMove)->getOperand(0);
288 VPValue *Phi1ToMoveV = Phi1ToMove.getVPSingleValue();
289 for (VPUser *U : to_vector(Phi1ToMoveV->users())) {
290 auto *UI = dyn_cast<VPRecipeBase>(U);
291 if (!UI || UI->getParent() != Then2)
292 continue;
293 for (unsigned I = 0, E = U->getNumOperands(); I != E; ++I) {
294 if (Phi1ToMoveV != U->getOperand(I))
295 continue;
296 U->setOperand(I, PredInst1);
297 }
298 }
299
300 Phi1ToMove.moveBefore(*Merge2, Merge2->begin());
301 }
302
303 // Finally, remove the first region.
304 for (VPBlockBase *Pred : make_early_inc_range(Region1->getPredecessors())) {
305 VPBlockUtils::disconnectBlocks(Pred, Region1);
306 VPBlockUtils::connectBlocks(Pred, MiddleBasicBlock);
307 }
308 VPBlockUtils::disconnectBlocks(Region1, MiddleBasicBlock);
309 DeletedRegions.insert(Region1);
310 }
311
312 for (VPRegionBlock *ToDelete : DeletedRegions)
313 delete ToDelete;
314 return !DeletedRegions.empty();
315}
316
318 VPlan &Plan) {
319 Instruction *Instr = PredRecipe->getUnderlyingInstr();
320 // Build the triangular if-then region.
321 std::string RegionName = (Twine("pred.") + Instr->getOpcodeName()).str();
322 assert(Instr->getParent() && "Predicated instruction not in any basic block");
323 auto *BlockInMask = PredRecipe->getMask();
324 auto *BOMRecipe = new VPBranchOnMaskRecipe(BlockInMask);
325 auto *Entry = new VPBasicBlock(Twine(RegionName) + ".entry", BOMRecipe);
326
327 // Replace predicated replicate recipe with a replicate recipe without a
328 // mask but in the replicate region.
329 auto *RecipeWithoutMask = new VPReplicateRecipe(
330 PredRecipe->getUnderlyingInstr(),
331 make_range(PredRecipe->op_begin(), std::prev(PredRecipe->op_end())),
332 PredRecipe->isUniform());
333 auto *Pred = new VPBasicBlock(Twine(RegionName) + ".if", RecipeWithoutMask);
334
335 VPPredInstPHIRecipe *PHIRecipe = nullptr;
336 if (PredRecipe->getNumUsers() != 0) {
337 PHIRecipe = new VPPredInstPHIRecipe(RecipeWithoutMask);
338 PredRecipe->replaceAllUsesWith(PHIRecipe);
339 PHIRecipe->setOperand(0, RecipeWithoutMask);
340 }
341 PredRecipe->eraseFromParent();
342 auto *Exiting = new VPBasicBlock(Twine(RegionName) + ".continue", PHIRecipe);
343 VPRegionBlock *Region = new VPRegionBlock(Entry, Exiting, RegionName, true);
344
345 // Note: first set Entry as region entry and then connect successors starting
346 // from it in order, to propagate the "parent" of each VPBasicBlock.
347 VPBlockUtils::insertTwoBlocksAfter(Pred, Exiting, Entry);
348 VPBlockUtils::connectBlocks(Pred, Exiting);
349
350 return Region;
351}
352
353static void addReplicateRegions(VPlan &Plan) {
355 for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(
356 vp_depth_first_deep(Plan.getEntry()))) {
357 for (VPRecipeBase &R : *VPBB)
358 if (auto *RepR = dyn_cast<VPReplicateRecipe>(&R)) {
359 if (RepR->isPredicated())
360 WorkList.push_back(RepR);
361 }
362 }
363
364 unsigned BBNum = 0;
365 for (VPReplicateRecipe *RepR : WorkList) {
366 VPBasicBlock *CurrentBlock = RepR->getParent();
367 VPBasicBlock *SplitBlock = CurrentBlock->splitAt(RepR->getIterator());
368
369 BasicBlock *OrigBB = RepR->getUnderlyingInstr()->getParent();
371 OrigBB->hasName() ? OrigBB->getName() + "." + Twine(BBNum++) : "");
372 // Record predicated instructions for above packing optimizations.
374 Region->setParent(CurrentBlock->getParent());
376 VPBlockUtils::connectBlocks(CurrentBlock, Region);
378 }
379}
380
382 // Convert masked VPReplicateRecipes to if-then region blocks.
384
385 bool ShouldSimplify = true;
386 while (ShouldSimplify) {
387 ShouldSimplify = sinkScalarOperands(Plan);
388 ShouldSimplify |= mergeReplicateRegionsIntoSuccessors(Plan);
389 ShouldSimplify |= VPlanTransforms::mergeBlocksIntoPredecessors(Plan);
390 }
391}
394 for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(
395 vp_depth_first_deep(Plan.getEntry()))) {
396 auto *PredVPBB =
397 dyn_cast_or_null<VPBasicBlock>(VPBB->getSinglePredecessor());
398 if (PredVPBB && PredVPBB->getNumSuccessors() == 1)
399 WorkList.push_back(VPBB);
400 }
401
402 for (VPBasicBlock *VPBB : WorkList) {
403 VPBasicBlock *PredVPBB = cast<VPBasicBlock>(VPBB->getSinglePredecessor());
404 for (VPRecipeBase &R : make_early_inc_range(*VPBB))
405 R.moveBefore(*PredVPBB, PredVPBB->end());
406 VPBlockUtils::disconnectBlocks(PredVPBB, VPBB);
407 auto *ParentRegion = cast_or_null<VPRegionBlock>(VPBB->getParent());
408 if (ParentRegion && ParentRegion->getExiting() == VPBB)
409 ParentRegion->setExiting(PredVPBB);
410 for (auto *Succ : to_vector(VPBB->successors())) {
412 VPBlockUtils::connectBlocks(PredVPBB, Succ);
413 }
414 delete VPBB;
415 }
416 return !WorkList.empty();
417}
418
420 for (auto &Phi : Plan.getVectorLoopRegion()->getEntryBasicBlock()->phis()) {
421 auto *IV = dyn_cast<VPWidenIntOrFpInductionRecipe>(&Phi);
422 if (!IV || IV->getTruncInst())
423 continue;
424
425 // A sequence of IR Casts has potentially been recorded for IV, which
426 // *must be bypassed* when the IV is vectorized, because the vectorized IV
427 // will produce the desired casted value. This sequence forms a def-use
428 // chain and is provided in reverse order, ending with the cast that uses
429 // the IV phi. Search for the recipe of the last cast in the chain and
430 // replace it with the original IV. Note that only the final cast is
431 // expected to have users outside the cast-chain and the dead casts left
432 // over will be cleaned up later.
433 auto &Casts = IV->getInductionDescriptor().getCastInsts();
434 VPValue *FindMyCast = IV;
435 for (Instruction *IRCast : reverse(Casts)) {
436 VPRecipeBase *FoundUserCast = nullptr;
437 for (auto *U : FindMyCast->users()) {
438 auto *UserCast = cast<VPRecipeBase>(U);
439 if (UserCast->getNumDefinedValues() == 1 &&
440 UserCast->getVPSingleValue()->getUnderlyingValue() == IRCast) {
441 FoundUserCast = UserCast;
442 break;
443 }
444 }
445 FindMyCast = FoundUserCast->getVPSingleValue();
446 }
447 FindMyCast->replaceAllUsesWith(IV);
448 }
449}
450
452 VPCanonicalIVPHIRecipe *CanonicalIV = Plan.getCanonicalIV();
453 VPWidenCanonicalIVRecipe *WidenNewIV = nullptr;
454 for (VPUser *U : CanonicalIV->users()) {
455 WidenNewIV = dyn_cast<VPWidenCanonicalIVRecipe>(U);
456 if (WidenNewIV)
457 break;
458 }
459
460 if (!WidenNewIV)
461 return;
462
464 for (VPRecipeBase &Phi : HeaderVPBB->phis()) {
465 auto *WidenOriginalIV = dyn_cast<VPWidenIntOrFpInductionRecipe>(&Phi);
466
467 if (!WidenOriginalIV || !WidenOriginalIV->isCanonical() ||
468 WidenOriginalIV->getScalarType() != WidenNewIV->getScalarType())
469 continue;
470
471 // Replace WidenNewIV with WidenOriginalIV if WidenOriginalIV provides
472 // everything WidenNewIV's users need. That is, WidenOriginalIV will
473 // generate a vector phi or all users of WidenNewIV demand the first lane
474 // only.
475 if (WidenOriginalIV->needsVectorIV() ||
476 vputils::onlyFirstLaneUsed(WidenNewIV)) {
477 WidenNewIV->replaceAllUsesWith(WidenOriginalIV);
478 WidenNewIV->eraseFromParent();
479 return;
480 }
481 }
482}
483
486 Plan.getEntry());
487
488 for (VPBasicBlock *VPBB : reverse(VPBlockUtils::blocksOnly<VPBasicBlock>(RPOT))) {
489 // The recipes in the block are processed in reverse order, to catch chains
490 // of dead recipes.
491 for (VPRecipeBase &R : make_early_inc_range(reverse(*VPBB))) {
492 if (R.mayHaveSideEffects() || any_of(R.definedValues(), [](VPValue *V) {
493 return V->getNumUsers() > 0;
494 }))
495 continue;
496 R.eraseFromParent();
497 }
498 }
499}
500
504 bool HasOnlyVectorVFs = !Plan.hasVF(ElementCount::getFixed(1));
505 for (VPRecipeBase &Phi : HeaderVPBB->phis()) {
506 auto *WideIV = dyn_cast<VPWidenIntOrFpInductionRecipe>(&Phi);
507 if (!WideIV)
508 continue;
509 if (HasOnlyVectorVFs && none_of(WideIV->users(), [WideIV](VPUser *U) {
510 return U->usesScalars(WideIV);
511 }))
512 continue;
513
514 auto IP = HeaderVPBB->getFirstNonPhi();
515 VPCanonicalIVPHIRecipe *CanonicalIV = Plan.getCanonicalIV();
516 Type *ResultTy = WideIV->getPHINode()->getType();
517 if (Instruction *TruncI = WideIV->getTruncInst())
518 ResultTy = TruncI->getType();
519 const InductionDescriptor &ID = WideIV->getInductionDescriptor();
520 VPValue *Step =
521 vputils::getOrCreateVPValueForSCEVExpr(Plan, ID.getStep(), SE);
522 VPValue *BaseIV = CanonicalIV;
523 if (!CanonicalIV->isCanonical(ID, ResultTy)) {
524 BaseIV = new VPDerivedIVRecipe(ID, WideIV->getStartValue(), CanonicalIV,
525 Step, ResultTy);
526 HeaderVPBB->insert(BaseIV->getDefiningRecipe(), IP);
527 }
528
529 VPScalarIVStepsRecipe *Steps = new VPScalarIVStepsRecipe(ID, BaseIV, Step);
530 HeaderVPBB->insert(Steps, IP);
531
532 // Update scalar users of IV to use Step instead. Use SetVector to ensure
533 // the list of users doesn't contain duplicates.
534 SetVector<VPUser *> Users(WideIV->user_begin(), WideIV->user_end());
535 for (VPUser *U : Users) {
536 if (HasOnlyVectorVFs && !U->usesScalars(WideIV))
537 continue;
538 for (unsigned I = 0, E = U->getNumOperands(); I != E; I++) {
539 if (U->getOperand(I) != WideIV)
540 continue;
541 U->setOperand(I, Steps);
542 }
543 }
544 }
545}
546
549
550 for (VPRecipeBase &R :
552 auto *ExpR = dyn_cast<VPExpandSCEVRecipe>(&R);
553 if (!ExpR)
554 continue;
555
556 auto I = SCEV2VPV.insert({ExpR->getSCEV(), ExpR});
557 if (I.second)
558 continue;
559 ExpR->replaceAllUsesWith(I.first->second);
560 ExpR->eraseFromParent();
561 }
562}
563
565 VPInstruction *Not = dyn_cast<VPInstruction>(Term->getOperand(0));
566 if (!Not || Not->getOpcode() != VPInstruction::Not)
567 return false;
568
569 VPInstruction *ALM = dyn_cast<VPInstruction>(Not->getOperand(0));
570 return ALM && ALM->getOpcode() == VPInstruction::ActiveLaneMask;
571}
572
574 unsigned BestUF,
576 assert(Plan.hasVF(BestVF) && "BestVF is not available in Plan");
577 assert(Plan.hasUF(BestUF) && "BestUF is not available in Plan");
578 VPBasicBlock *ExitingVPBB =
580 auto *Term = dyn_cast<VPInstruction>(&ExitingVPBB->back());
581 // Try to simplify the branch condition if TC <= VF * UF when preparing to
582 // execute the plan for the main vector loop. We only do this if the
583 // terminator is:
584 // 1. BranchOnCount, or
585 // 2. BranchOnCond where the input is Not(ActiveLaneMask).
586 if (!Term || (Term->getOpcode() != VPInstruction::BranchOnCount &&
587 (Term->getOpcode() != VPInstruction::BranchOnCond ||
589 return;
590
591 Type *IdxTy =
593 const SCEV *TripCount = createTripCountSCEV(IdxTy, PSE);
594 ScalarEvolution &SE = *PSE.getSE();
595 const SCEV *C =
596 SE.getConstant(TripCount->getType(), BestVF.getKnownMinValue() * BestUF);
597 if (TripCount->isZero() ||
598 !SE.isKnownPredicate(CmpInst::ICMP_ULE, TripCount, C))
599 return;
600
601 LLVMContext &Ctx = SE.getContext();
602 auto *BOC =
605 Term->eraseFromParent();
606 ExitingVPBB->appendRecipe(BOC);
607 Plan.setVF(BestVF);
608 Plan.setUF(BestUF);
609 // TODO: Further simplifications are possible
610 // 1. Replace inductions with constants.
611 // 2. Replace vector loop region with VPBasicBlock.
612}
613
615 auto *Region = dyn_cast_or_null<VPRegionBlock>(R->getParent()->getParent());
616 if (Region && Region->isReplicator()) {
617 assert(Region->getNumSuccessors() == 1 &&
618 Region->getNumPredecessors() == 1 && "Expected SESE region!");
619 assert(R->getParent()->size() == 1 &&
620 "A recipe in an original replicator region must be the only "
621 "recipe in its block");
622 return Region;
623 }
624 return nullptr;
625}
626
627static bool properlyDominates(const VPRecipeBase *A, const VPRecipeBase *B,
628 VPDominatorTree &VPDT) {
629 if (A == B)
630 return false;
631
632 auto LocalComesBefore = [](const VPRecipeBase *A, const VPRecipeBase *B) {
633 for (auto &R : *A->getParent()) {
634 if (&R == A)
635 return true;
636 if (&R == B)
637 return false;
638 }
639 llvm_unreachable("recipe not found");
640 };
641 const VPBlockBase *ParentA = A->getParent();
642 const VPBlockBase *ParentB = B->getParent();
643 if (ParentA == ParentB)
644 return LocalComesBefore(A, B);
645
646 const VPRegionBlock *RegionA =
647 GetReplicateRegion(const_cast<VPRecipeBase *>(A));
648 const VPRegionBlock *RegionB =
649 GetReplicateRegion(const_cast<VPRecipeBase *>(B));
650 if (RegionA)
651 ParentA = RegionA->getExiting();
652 if (RegionB)
653 ParentB = RegionB->getExiting();
654 return VPDT.properlyDominates(ParentA, ParentB);
655}
656
657// Sink users of \p FOR after the recipe defining the previous value \p Previous
658// of the recurrence.
659static void
661 VPRecipeBase *Previous,
662 VPDominatorTree &VPDT) {
663 // Collect recipes that need sinking.
666 Seen.insert(Previous);
667 auto TryToPushSinkCandidate = [&](VPRecipeBase *SinkCandidate) {
668 assert(
669 SinkCandidate != Previous &&
670 "The previous value cannot depend on the users of the recurrence phi.");
671 if (isa<VPHeaderPHIRecipe>(SinkCandidate) ||
672 !Seen.insert(SinkCandidate).second ||
673 properlyDominates(Previous, SinkCandidate, VPDT))
674 return;
675
676 WorkList.push_back(SinkCandidate);
677 };
678
679 // Recursively sink users of FOR after Previous.
680 WorkList.push_back(FOR);
681 for (unsigned I = 0; I != WorkList.size(); ++I) {
682 VPRecipeBase *Current = WorkList[I];
683 assert(Current->getNumDefinedValues() == 1 &&
684 "only recipes with a single defined value expected");
685 for (VPUser *User : Current->getVPSingleValue()->users()) {
686 if (auto *R = dyn_cast<VPRecipeBase>(User))
687 TryToPushSinkCandidate(R);
688 }
689 }
690
691 // Keep recipes to sink ordered by dominance so earlier instructions are
692 // processed first.
693 sort(WorkList, [&VPDT](const VPRecipeBase *A, const VPRecipeBase *B) {
694 return properlyDominates(A, B, VPDT);
695 });
696
697 for (VPRecipeBase *SinkCandidate : WorkList) {
698 if (SinkCandidate == FOR)
699 continue;
700
701 SinkCandidate->moveAfter(Previous);
702 Previous = SinkCandidate;
703 }
704}
705
707 VPBuilder &Builder) {
708 VPDominatorTree VPDT;
709 VPDT.recalculate(Plan);
710
712 for (VPRecipeBase &R :
714 if (auto *FOR = dyn_cast<VPFirstOrderRecurrencePHIRecipe>(&R))
715 RecurrencePhis.push_back(FOR);
716
717 for (VPFirstOrderRecurrencePHIRecipe *FOR : RecurrencePhis) {
719 VPRecipeBase *Previous = FOR->getBackedgeValue()->getDefiningRecipe();
720 // Fixed-order recurrences do not contain cycles, so this loop is guaranteed
721 // to terminate.
722 while (auto *PrevPhi =
723 dyn_cast_or_null<VPFirstOrderRecurrencePHIRecipe>(Previous)) {
724 assert(PrevPhi->getParent() == FOR->getParent());
725 assert(SeenPhis.insert(PrevPhi).second);
726 Previous = PrevPhi->getBackedgeValue()->getDefiningRecipe();
727 }
728
729 sinkRecurrenceUsersAfterPrevious(FOR, Previous, VPDT);
730
731 // Introduce a recipe to combine the incoming and previous values of a
732 // fixed-order recurrence.
733 VPBasicBlock *InsertBlock = Previous->getParent();
734 if (isa<VPHeaderPHIRecipe>(Previous))
735 Builder.setInsertPoint(InsertBlock, InsertBlock->getFirstNonPhi());
736 else
737 Builder.setInsertPoint(InsertBlock, std::next(Previous->getIterator()));
738
739 auto *RecurSplice = cast<VPInstruction>(
741 {FOR, FOR->getBackedgeValue()}));
742
743 FOR->replaceAllUsesWith(RecurSplice);
744 // Set the first operand of RecurSplice to FOR again, after replacing
745 // all users.
746 RecurSplice->setOperand(0, FOR);
747 }
748}
ReachingDefAnalysis InstSet & ToRemove
assume Assume Builder
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
Returns the sub type a function will return at a given Idx Should correspond to the result type of an ExtractValue instruction executed with just that one unsigned Idx
Hexagon Common GEP
iv Induction Variable Users
Definition: IVUsers.cpp:48
#define I(x, y, z)
Definition: MD5.cpp:58
This file builds on the ADT/GraphTraits.h file to build a generic graph post order iterator.
@ SI
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
This file implements a set that has insertion order iteration characteristics.
This file implements dominator tree analysis for a single level of a VPlan's H-CFG.
static bool canSimplifyBranchOnCond(VPInstruction *Term)
static bool sinkScalarOperands(VPlan &Plan)
static void sinkRecurrenceUsersAfterPrevious(VPFirstOrderRecurrencePHIRecipe *FOR, VPRecipeBase *Previous, VPDominatorTree &VPDT)
static bool mergeReplicateRegionsIntoSuccessors(VPlan &Plan)
static void addReplicateRegions(VPlan &Plan)
static VPRegionBlock * GetReplicateRegion(VPRecipeBase *R)
static bool properlyDominates(const VPRecipeBase *A, const VPRecipeBase *B, VPDominatorTree &VPDT)
static VPRegionBlock * createReplicateRegion(VPReplicateRecipe *PredRecipe, VPlan &Plan)
static VPBasicBlock * getPredicatedThenBlock(VPRegionBlock *R)
If R is a triangle region, return the 'then' block of the triangle.
VPValue * getPredicatedMask(VPRegionBlock *R)
If R is a region with a VPBranchOnMaskRecipe in the entry block, return the mask.
This file provides utility VPlan to VPlan transformations.
static const uint32_t IV[8]
Definition: blake3_impl.h:77
LLVM Basic Block Representation.
Definition: BasicBlock.h:56
const Function * getParent() const
Return the enclosing method, or null if none.
Definition: BasicBlock.h:112
This class represents a function call, abstracting a target machine's calling convention.
@ ICMP_ULE
unsigned less or equal
Definition: InstrTypes.h:744
static ConstantInt * getTrue(LLVMContext &Context)
Definition: Constants.cpp:833
std::pair< iterator, bool > insert(const std::pair< KeyT, ValueT > &KV)
Definition: DenseMap.h:220
Core dominator tree base class.
void recalculate(ParentType &Func)
recalculate - compute a dominator tree for the given function
bool properlyDominates(const DomTreeNodeBase< NodeT > *A, const DomTreeNodeBase< NodeT > *B) const
properlyDominates - Returns true iff A dominates B and A != B.
static constexpr ElementCount getFixed(ScalarTy MinVal)
Definition: TypeSize.h:291
an instruction for type-safe pointer arithmetic to access elements of arrays and structs
Definition: Instructions.h:940
A struct for saving information about induction variables.
const BasicBlock * getParent() const
Definition: Instruction.h:90
const char * getOpcodeName() const
Definition: Instruction.h:170
This is an important class for using LLVM in a threaded context.
Definition: LLVMContext.h:67
An instruction for reading from memory.
Definition: Instructions.h:177
An interface layer with SCEV used to manage how we see SCEV expressions for values in the context of ...
ScalarEvolution * getSE() const
Returns the ScalarEvolution analysis used.
This class represents an analyzed expression in the program.
bool isZero() const
Return true if the expression is a constant zero.
Type * getType() const
Return the LLVM type of this SCEV expression.
The main scalar evolution driver.
const SCEV * getConstant(ConstantInt *V)
bool isKnownPredicate(ICmpInst::Predicate Pred, const SCEV *LHS, const SCEV *RHS)
Test if the given expression is known to satisfy the condition described by Pred, LHS,...
LLVMContext & getContext() const
This class represents the LLVM 'select' instruction.
A vector that has set insertion semantics.
Definition: SetVector.h:51
size_type size() const
Determine the number of elements in the SetVector.
Definition: SetVector.h:88
bool contains(const key_type &key) const
Check if the SetVector contains the given key.
Definition: SetVector.h:213
bool insert(const value_type &X)
Insert a new element into the SetVector.
Definition: SetVector.h:152
bool empty() const
Determine if the SetVector is empty or not.
Definition: SetVector.h:83
A templated base class for SmallPtrSet which provides the typesafe interface that is common across al...
Definition: SmallPtrSet.h:344
size_type count(ConstPtrType Ptr) const
count - Return 1 if the specified pointer is in the set, 0 otherwise.
Definition: SmallPtrSet.h:383
std::pair< iterator, bool > insert(PtrType Ptr)
Inserts Ptr if and only if there is no element in the container equal to Ptr.
Definition: SmallPtrSet.h:365
SmallPtrSet - This class implements a set which is optimized for holding SmallSize or less elements.
Definition: SmallPtrSet.h:450
bool empty() const
Definition: SmallVector.h:94
size_t size() const
Definition: SmallVector.h:91
void push_back(const T &Elt)
Definition: SmallVector.h:416
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
Definition: SmallVector.h:1200
An instruction for storing to memory.
Definition: Instructions.h:301
Provides information about what library functions are available for the current target.
Twine - A lightweight data structure for efficiently representing the concatenation of temporary valu...
Definition: Twine.h:81
The instances of the Type class are immutable: once they are created, they are never changed.
Definition: Type.h:45
op_range operands()
Definition: User.h:242
VPBasicBlock serves as the leaf of the Hierarchical Control-Flow Graph.
Definition: VPlan.h:1956
void appendRecipe(VPRecipeBase *Recipe)
Augment the existing recipes of a VPBasicBlock with an additional Recipe as the last recipe.
Definition: VPlan.h:2024
iterator end()
Definition: VPlan.h:1987
iterator_range< iterator > phis()
Returns an iterator range over the PHI-like recipes in the block.
Definition: VPlan.h:2034
iterator getFirstNonPhi()
Return the position of the first non-phi node recipe in the block.
Definition: VPlan.cpp:207
VPBasicBlock * splitAt(iterator SplitAt)
Split current block at SplitAt by inserting a new block between the current block and its successors ...
Definition: VPlan.cpp:402
const VPRecipeBase & back() const
Definition: VPlan.h:1999
void insert(VPRecipeBase *Recipe, iterator InsertPt)
Definition: VPlan.h:2015
VPBlockBase is the building block of the Hierarchical Control-Flow Graph.
Definition: VPlan.h:390
VPRegionBlock * getParent()
Definition: VPlan.h:462
const VPBasicBlock * getExitingBasicBlock() const
Definition: VPlan.cpp:172
const VPBasicBlock * getEntryBasicBlock() const
Definition: VPlan.cpp:151
VPBlockBase * getSingleSuccessor() const
Definition: VPlan.h:497
const VPBlocksTy & getSuccessors() const
Definition: VPlan.h:487
static void insertTwoBlocksAfter(VPBlockBase *IfTrue, VPBlockBase *IfFalse, VPBlockBase *BlockPtr)
Insert disconnected VPBlockBases IfTrue and IfFalse after BlockPtr.
Definition: VPlan.h:2518
static void disconnectBlocks(VPBlockBase *From, VPBlockBase *To)
Disconnect VPBlockBases From and To bi-directionally.
Definition: VPlan.h:2546
static void connectBlocks(VPBlockBase *From, VPBlockBase *To)
Connect VPBlockBases From and To bi-directionally.
Definition: VPlan.h:2535
A recipe for generating conditional branches on the bits of a mask.
Definition: VPlan.h:1573
VPlan-based builder utility analogous to IRBuilder.
Canonical scalar induction phi of the vector loop.
Definition: VPlan.h:1775
bool isCanonical(const InductionDescriptor &ID, Type *Ty) const
Check if the induction described by ID is canonical, i.e.
unsigned getNumDefinedValues() const
Returns the number of values defined by the VPDef.
Definition: VPlanValue.h:411
ArrayRef< VPValue * > definedValues()
Returns an ArrayRef of the values defined by the VPDef.
Definition: VPlanValue.h:406
VPValue * getVPSingleValue()
Returns the only VPValue defined by the VPDef.
Definition: VPlanValue.h:384
A recipe for converting the canonical IV value to the corresponding value of an IV with different sta...
Definition: VPlan.h:1878
VPValue * getStartValue()
Returns the start value of the phi, if one is set.
Definition: VPlan.h:1083
This is a concrete Recipe that models a single VPlan-level instruction.
Definition: VPlan.h:779
@ FirstOrderRecurrenceSplice
Definition: VPlan.h:785
unsigned getOpcode() const
Definition: VPlan.h:837
VPPredInstPHIRecipe is a recipe for generating the phi nodes needed when control converges back from ...
Definition: VPlan.h:1620
VPRecipeBase is a base class modeling a sequence of one or more output IR instructions.
Definition: VPlan.h:666
bool mayReadOrWriteMemory() const
Returns true if the recipe may read from or write to memory.
Definition: VPlan.h:753
bool mayHaveSideEffects() const
Returns true if the recipe may have side-effects.
Instruction * getUnderlyingInstr()
Returns the underlying instruction, if the recipe is a VPValue or nullptr otherwise.
Definition: VPlan.h:721
VPBasicBlock * getParent()
Definition: VPlan.h:683
void moveBefore(VPBasicBlock &BB, iplist< VPRecipeBase >::iterator I)
Unlink this recipe and insert into BB before I.
void insertBefore(VPRecipeBase *InsertPos)
Insert an unlinked recipe into a basic block immediately before the specified recipe.
iplist< VPRecipeBase >::iterator eraseFromParent()
This method unlinks 'this' from the containing basic block and deletes it.
VPRegionBlock represents a collection of VPBasicBlocks and VPRegionBlocks which form a Single-Entry-S...
Definition: VPlan.h:2080
const VPBlockBase * getEntry() const
Definition: VPlan.h:2119
const VPBlockBase * getExiting() const
Definition: VPlan.h:2131
VPReplicateRecipe replicates a given instruction producing multiple scalar copies of the original sca...
Definition: VPlan.h:1510
bool isUniform() const
Definition: VPlan.h:1542
VPValue * getMask()
Return the mask of a predicated VPReplicateRecipe.
Definition: VPlan.h:1566
A recipe for handling phi nodes of integer and floating-point inductions, producing their scalar valu...
Definition: VPlan.h:1921
This class augments VPValue with operands which provide the inverse def-use edges from VPValue's user...
Definition: VPlanValue.h:201
operand_range operands()
Definition: VPlanValue.h:276
void setOperand(unsigned I, VPValue *New)
Definition: VPlanValue.h:256
operand_iterator op_end()
Definition: VPlanValue.h:274
operand_iterator op_begin()
Definition: VPlanValue.h:272
VPValue * getOperand(unsigned N) const
Definition: VPlanValue.h:251
Value * getUnderlyingValue()
Return the underlying Value attached to this VPValue.
Definition: VPlanValue.h:84
VPRecipeBase * getDefiningRecipe()
Returns the recipe defining this VPValue or nullptr if it is not defined by a recipe,...
Definition: VPlan.cpp:116
void replaceAllUsesWith(VPValue *New)
Definition: VPlan.cpp:1014
unsigned getNumUsers() const
Definition: VPlanValue.h:119
Value * getLiveInIRValue()
Returns the underlying IR value, if this VPValue is defined outside the scope of VPlan.
Definition: VPlanValue.h:177
user_range users()
Definition: VPlanValue.h:147
A recipe for widening Call instructions.
Definition: VPlan.h:932
A Recipe for widening the canonical induction variable of the vector loop.
Definition: VPlan.h:1847
const Type * getScalarType() const
Returns the scalar type of the induction.
Definition: VPlan.h:1869
A recipe for handling GEP instructions.
Definition: VPlan.h:993
A recipe for handling phi nodes of integer and floating-point inductions, producing their vector valu...
Definition: VPlan.h:1107
A Recipe for widening load/store operations.
Definition: VPlan.h:1653
VPWidenRecipe is a recipe for producing a copy of vector type its ingredient.
Definition: VPlan.h:911
VPlan models a candidate for vectorization, encoding various decisions take to produce efficient outp...
Definition: VPlan.h:2177
VPBlockBase * getEntry()
Definition: VPlan.h:2240
VPValue * getOrAddExternalDef(Value *V)
Get the existing or add a new external definition for V.
Definition: VPlan.h:2296
VPRegionBlock * getVectorLoopRegion()
Returns the VPRegionBlock of the vector loop.
Definition: VPlan.h:2370
bool hasVF(ElementCount VF)
Definition: VPlan.h:2278
bool hasUF(unsigned UF) const
Definition: VPlan.h:2282
void setVF(ElementCount VF)
Definition: VPlan.h:2272
bool hasScalarVFOnly() const
Definition: VPlan.h:2280
VPCanonicalIVPHIRecipe * getCanonicalIV()
Returns the canonical induction recipe of the vector loop.
Definition: VPlan.h:2378
void setUF(unsigned UF)
Definition: VPlan.h:2284
Type * getType() const
All values are typed, get the type of this value.
Definition: Value.h:255
void setName(const Twine &Name)
Change the name of the value.
Definition: Value.cpp:377
bool hasName() const
Definition: Value.h:261
StringRef getName() const
Return a constant reference to the value's name.
Definition: Value.cpp:308
constexpr ScalarTy getKnownMinValue() const
Returns the minimum value this quantity can represent.
Definition: TypeSize.h:163
An efficient, type-erasing, non-owning reference to a callable.
self_iterator getIterator()
Definition: ilist_node.h:82
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
@ C
The default llvm calling convention, compatible with C.
Definition: CallingConv.h:34
VPValue * getOrCreateVPValueForSCEVExpr(VPlan &Plan, const SCEV *Expr, ScalarEvolution &SE)
Get or create a VPValue that corresponds to the expansion of Expr.
Definition: VPlan.cpp:1134
bool onlyFirstLaneUsed(VPValue *Def)
Returns true if only the first lane of Def is used.
Definition: VPlan.cpp:1129
This is an optimization pass for GlobalISel generic memory operations.
Definition: AddressRanges.h:18
bool all_of(R &&range, UnaryPredicate P)
Provide wrappers to std::all_of which take ranges instead of having to pass begin/end explicitly.
Definition: STLExtras.h:1819
Intrinsic::ID getVectorIntrinsicIDForCall(const CallInst *CI, const TargetLibraryInfo *TLI)
Returns intrinsic ID for call.
const Value * getLoadStorePointerOperand(const Value *V)
A helper function that returns the pointer operand of a load or store instruction.
iterator_range< T > make_range(T x, T y)
Convenience function for iterating over sub-ranges.
iterator_range< early_inc_iterator_impl< detail::IterOfRange< RangeT > > > make_early_inc_range(RangeT &&Range)
Make a range that does early increment to allow mutation of the underlying range without disrupting i...
Definition: STLExtras.h:748
iterator_range< df_iterator< VPBlockDeepTraversalWrapper< VPBlockBase * > > > vp_depth_first_deep(VPBlockBase *G)
Returns an iterator range to traverse the graph starting at G in depth-first order while traversing t...
Definition: VPlanCFG.h:225
bool any_of(R &&range, UnaryPredicate P)
Provide wrappers to std::any_of which take ranges instead of having to pass begin/end explicitly.
Definition: STLExtras.h:1826
auto reverse(ContainerTy &&C)
Definition: STLExtras.h:511
void sort(IteratorTy Start, IteratorTy End)
Definition: STLExtras.h:1744
std::unique_ptr< VPlan > VPlanPtr
Definition: VPlan.h:104
bool none_of(R &&Range, UnaryPredicate P)
Provide wrappers to std::none_of which take ranges instead of having to pass begin/end explicitly.
Definition: STLExtras.h:1833
SmallVector< ValueTypeFromRangeType< R >, Size > to_vector(R &&Range)
Given a range of type R, iterate the entire range and return a SmallVector with elements of the vecto...
Definition: SmallVector.h:1298
BasicBlock * SplitBlock(BasicBlock *Old, Instruction *SplitPt, DominatorTree *DT, LoopInfo *LI=nullptr, MemorySSAUpdater *MSSAU=nullptr, const Twine &BBName="", bool Before=false)
Split the specified block at the specified instruction.
const SCEV * createTripCountSCEV(Type *IdxTy, PredicatedScalarEvolution &PSE)
A recipe for handling first-order recurrence phis.
Definition: VPlan.h:1267
A recipe for widening select instructions.
Definition: VPlan.h:965
static void VPInstructionsToVPRecipes(VPlanPtr &Plan, function_ref< const InductionDescriptor *(PHINode *)> GetIntOrFpInductionDescriptor, SmallPtrSetImpl< Instruction * > &DeadInstructions, ScalarEvolution &SE, const TargetLibraryInfo &TLI)
Replaces the VPInstructions in Plan with corresponding widen recipes.
static void createAndOptimizeReplicateRegions(VPlan &Plan)
Wrap predicated VPReplicateRecipes with a mask operand in an if-then region block and remove the mask...
static void removeRedundantInductionCasts(VPlan &Plan)
Remove redundant casts of inductions.
static void adjustFixedOrderRecurrences(VPlan &Plan, VPBuilder &Builder)
Sink users of fixed-order recurrences after the recipe defining their previous value.
static void optimizeInductions(VPlan &Plan, ScalarEvolution &SE)
If any user of a VPWidenIntOrFpInductionRecipe needs scalar values, provide them by building scalar s...
static bool mergeBlocksIntoPredecessors(VPlan &Plan)
Remove redundant VPBasicBlocks by merging them into their predecessor if the predecessor has a single...
static void removeRedundantExpandSCEVRecipes(VPlan &Plan)
Remove redundant EpxandSCEVRecipes in Plan's entry block by replacing them with already existing reci...
static void removeRedundantCanonicalIVs(VPlan &Plan)
Try to replace VPWidenCanonicalIVRecipes with a widened canonical IV recipe, if it exists.
static void removeDeadRecipes(VPlan &Plan)
static void optimizeForVFAndUF(VPlan &Plan, ElementCount BestVF, unsigned BestUF, PredicatedScalarEvolution &PSE)
Optimize Plan based on BestVF and BestUF.