LLVM 19.0.0git
BasicBlockUtils.cpp
Go to the documentation of this file.
1//===- BasicBlockUtils.cpp - BasicBlock Utilities --------------------------==//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This family of functions perform manipulations on basic blocks, and
10// instructions contained within basic blocks.
11//
12//===----------------------------------------------------------------------===//
13
15#include "llvm/ADT/ArrayRef.h"
18#include "llvm/ADT/Twine.h"
19#include "llvm/Analysis/CFG.h"
24#include "llvm/IR/BasicBlock.h"
25#include "llvm/IR/CFG.h"
26#include "llvm/IR/Constants.h"
27#include "llvm/IR/DebugInfo.h"
29#include "llvm/IR/Dominators.h"
30#include "llvm/IR/Function.h"
31#include "llvm/IR/InstrTypes.h"
32#include "llvm/IR/Instruction.h"
35#include "llvm/IR/IRBuilder.h"
36#include "llvm/IR/LLVMContext.h"
37#include "llvm/IR/Type.h"
38#include "llvm/IR/User.h"
39#include "llvm/IR/Value.h"
40#include "llvm/IR/ValueHandle.h"
43#include "llvm/Support/Debug.h"
46#include <cassert>
47#include <cstdint>
48#include <string>
49#include <utility>
50#include <vector>
51
52using namespace llvm;
53
54#define DEBUG_TYPE "basicblock-utils"
55
57 "max-deopt-or-unreachable-succ-check-depth", cl::init(8), cl::Hidden,
58 cl::desc("Set the maximum path length when checking whether a basic block "
59 "is followed by a block that either has a terminating "
60 "deoptimizing call or is terminated with an unreachable"));
61
65 bool KeepOneInputPHIs) {
66 for (auto *BB : BBs) {
67 // Loop through all of our successors and make sure they know that one
68 // of their predecessors is going away.
69 SmallPtrSet<BasicBlock *, 4> UniqueSuccessors;
70 for (BasicBlock *Succ : successors(BB)) {
71 Succ->removePredecessor(BB, KeepOneInputPHIs);
72 if (Updates && UniqueSuccessors.insert(Succ).second)
73 Updates->push_back({DominatorTree::Delete, BB, Succ});
74 }
75
76 // Zap all the instructions in the block.
77 while (!BB->empty()) {
78 Instruction &I = BB->back();
79 // If this instruction is used, replace uses with an arbitrary value.
80 // Because control flow can't get here, we don't care what we replace the
81 // value with. Note that since this block is unreachable, and all values
82 // contained within it must dominate their uses, that all uses will
83 // eventually be removed (they are themselves dead).
84 if (!I.use_empty())
85 I.replaceAllUsesWith(PoisonValue::get(I.getType()));
86 BB->back().eraseFromParent();
87 }
88 new UnreachableInst(BB->getContext(), BB);
89 assert(BB->size() == 1 &&
90 isa<UnreachableInst>(BB->getTerminator()) &&
91 "The successor list of BB isn't empty before "
92 "applying corresponding DTU updates.");
93 }
94}
95
97 bool KeepOneInputPHIs) {
98 DeleteDeadBlocks({BB}, DTU, KeepOneInputPHIs);
99}
100
101void llvm::DeleteDeadBlocks(ArrayRef <BasicBlock *> BBs, DomTreeUpdater *DTU,
102 bool KeepOneInputPHIs) {
103#ifndef NDEBUG
104 // Make sure that all predecessors of each dead block is also dead.
105 SmallPtrSet<BasicBlock *, 4> Dead(BBs.begin(), BBs.end());
106 assert(Dead.size() == BBs.size() && "Duplicating blocks?");
107 for (auto *BB : Dead)
108 for (BasicBlock *Pred : predecessors(BB))
109 assert(Dead.count(Pred) && "All predecessors must be dead!");
110#endif
111
113 detachDeadBlocks(BBs, DTU ? &Updates : nullptr, KeepOneInputPHIs);
114
115 if (DTU)
116 DTU->applyUpdates(Updates);
117
118 for (BasicBlock *BB : BBs)
119 if (DTU)
120 DTU->deleteBB(BB);
121 else
122 BB->eraseFromParent();
123}
124
126 bool KeepOneInputPHIs) {
128
129 // Mark all reachable blocks.
130 for (BasicBlock *BB : depth_first_ext(&F, Reachable))
131 (void)BB/* Mark all reachable blocks */;
132
133 // Collect all dead blocks.
134 std::vector<BasicBlock*> DeadBlocks;
135 for (BasicBlock &BB : F)
136 if (!Reachable.count(&BB))
137 DeadBlocks.push_back(&BB);
138
139 // Delete the dead blocks.
140 DeleteDeadBlocks(DeadBlocks, DTU, KeepOneInputPHIs);
141
142 return !DeadBlocks.empty();
143}
144
146 MemoryDependenceResults *MemDep) {
147 if (!isa<PHINode>(BB->begin()))
148 return false;
149
150 while (PHINode *PN = dyn_cast<PHINode>(BB->begin())) {
151 if (PN->getIncomingValue(0) != PN)
152 PN->replaceAllUsesWith(PN->getIncomingValue(0));
153 else
154 PN->replaceAllUsesWith(PoisonValue::get(PN->getType()));
155
156 if (MemDep)
157 MemDep->removeInstruction(PN); // Memdep updates AA itself.
158
159 PN->eraseFromParent();
160 }
161 return true;
162}
163
165 MemorySSAUpdater *MSSAU) {
166 // Recursively deleting a PHI may cause multiple PHIs to be deleted
167 // or RAUW'd undef, so use an array of WeakTrackingVH for the PHIs to delete.
169 for (PHINode &PN : BB->phis())
170 PHIs.push_back(&PN);
171
172 bool Changed = false;
173 for (unsigned i = 0, e = PHIs.size(); i != e; ++i)
174 if (PHINode *PN = dyn_cast_or_null<PHINode>(PHIs[i].operator Value*()))
175 Changed |= RecursivelyDeleteDeadPHINode(PN, TLI, MSSAU);
176
177 return Changed;
178}
179
181 LoopInfo *LI, MemorySSAUpdater *MSSAU,
183 bool PredecessorWithTwoSuccessors,
184 DominatorTree *DT) {
185 if (BB->hasAddressTaken())
186 return false;
187
188 // Can't merge if there are multiple predecessors, or no predecessors.
189 BasicBlock *PredBB = BB->getUniquePredecessor();
190 if (!PredBB) return false;
191
192 // Don't break self-loops.
193 if (PredBB == BB) return false;
194
195 // Don't break unwinding instructions or terminators with other side-effects.
196 Instruction *PTI = PredBB->getTerminator();
197 if (PTI->isSpecialTerminator() || PTI->mayHaveSideEffects())
198 return false;
199
200 // Can't merge if there are multiple distinct successors.
201 if (!PredecessorWithTwoSuccessors && PredBB->getUniqueSuccessor() != BB)
202 return false;
203
204 // Currently only allow PredBB to have two predecessors, one being BB.
205 // Update BI to branch to BB's only successor instead of BB.
206 BranchInst *PredBB_BI;
207 BasicBlock *NewSucc = nullptr;
208 unsigned FallThruPath;
209 if (PredecessorWithTwoSuccessors) {
210 if (!(PredBB_BI = dyn_cast<BranchInst>(PTI)))
211 return false;
212 BranchInst *BB_JmpI = dyn_cast<BranchInst>(BB->getTerminator());
213 if (!BB_JmpI || !BB_JmpI->isUnconditional())
214 return false;
215 NewSucc = BB_JmpI->getSuccessor(0);
216 FallThruPath = PredBB_BI->getSuccessor(0) == BB ? 0 : 1;
217 }
218
219 // Can't merge if there is PHI loop.
220 for (PHINode &PN : BB->phis())
221 if (llvm::is_contained(PN.incoming_values(), &PN))
222 return false;
223
224 LLVM_DEBUG(dbgs() << "Merging: " << BB->getName() << " into "
225 << PredBB->getName() << "\n");
226
227 // Begin by getting rid of unneeded PHIs.
228 SmallVector<AssertingVH<Value>, 4> IncomingValues;
229 if (isa<PHINode>(BB->front())) {
230 for (PHINode &PN : BB->phis())
231 if (!isa<PHINode>(PN.getIncomingValue(0)) ||
232 cast<PHINode>(PN.getIncomingValue(0))->getParent() != BB)
233 IncomingValues.push_back(PN.getIncomingValue(0));
234 FoldSingleEntryPHINodes(BB, MemDep);
235 }
236
237 if (DT) {
238 assert(!DTU && "cannot use both DT and DTU for updates");
239 DomTreeNode *PredNode = DT->getNode(PredBB);
240 DomTreeNode *BBNode = DT->getNode(BB);
241 if (PredNode) {
242 assert(BBNode && "PredNode unreachable but BBNode reachable?");
243 for (DomTreeNode *C : to_vector(BBNode->children()))
244 C->setIDom(PredNode);
245 }
246 }
247 // DTU update: Collect all the edges that exit BB.
248 // These dominator edges will be redirected from Pred.
249 std::vector<DominatorTree::UpdateType> Updates;
250 if (DTU) {
251 assert(!DT && "cannot use both DT and DTU for updates");
252 // To avoid processing the same predecessor more than once.
254 SmallPtrSet<BasicBlock *, 2> SuccsOfPredBB(succ_begin(PredBB),
255 succ_end(PredBB));
256 Updates.reserve(Updates.size() + 2 * succ_size(BB) + 1);
257 // Add insert edges first. Experimentally, for the particular case of two
258 // blocks that can be merged, with a single successor and single predecessor
259 // respectively, it is beneficial to have all insert updates first. Deleting
260 // edges first may lead to unreachable blocks, followed by inserting edges
261 // making the blocks reachable again. Such DT updates lead to high compile
262 // times. We add inserts before deletes here to reduce compile time.
263 for (BasicBlock *SuccOfBB : successors(BB))
264 // This successor of BB may already be a PredBB's successor.
265 if (!SuccsOfPredBB.contains(SuccOfBB))
266 if (SeenSuccs.insert(SuccOfBB).second)
267 Updates.push_back({DominatorTree::Insert, PredBB, SuccOfBB});
268 SeenSuccs.clear();
269 for (BasicBlock *SuccOfBB : successors(BB))
270 if (SeenSuccs.insert(SuccOfBB).second)
271 Updates.push_back({DominatorTree::Delete, BB, SuccOfBB});
272 Updates.push_back({DominatorTree::Delete, PredBB, BB});
273 }
274
275 Instruction *STI = BB->getTerminator();
276 Instruction *Start = &*BB->begin();
277 // If there's nothing to move, mark the starting instruction as the last
278 // instruction in the block. Terminator instruction is handled separately.
279 if (Start == STI)
280 Start = PTI;
281
282 // Move all definitions in the successor to the predecessor...
283 PredBB->splice(PTI->getIterator(), BB, BB->begin(), STI->getIterator());
284
285 if (MSSAU)
286 MSSAU->moveAllAfterMergeBlocks(BB, PredBB, Start);
287
288 // Make all PHI nodes that referred to BB now refer to Pred as their
289 // source...
290 BB->replaceAllUsesWith(PredBB);
291
292 if (PredecessorWithTwoSuccessors) {
293 // Delete the unconditional branch from BB.
294 BB->back().eraseFromParent();
295
296 // Update branch in the predecessor.
297 PredBB_BI->setSuccessor(FallThruPath, NewSucc);
298 } else {
299 // Delete the unconditional branch from the predecessor.
300 PredBB->back().eraseFromParent();
301
302 // Move terminator instruction.
303 BB->back().moveBeforePreserving(*PredBB, PredBB->end());
304
305 // Terminator may be a memory accessing instruction too.
306 if (MSSAU)
307 if (MemoryUseOrDef *MUD = cast_or_null<MemoryUseOrDef>(
308 MSSAU->getMemorySSA()->getMemoryAccess(PredBB->getTerminator())))
309 MSSAU->moveToPlace(MUD, PredBB, MemorySSA::End);
310 }
311 // Add unreachable to now empty BB.
312 new UnreachableInst(BB->getContext(), BB);
313
314 // Inherit predecessors name if it exists.
315 if (!PredBB->hasName())
316 PredBB->takeName(BB);
317
318 if (LI)
319 LI->removeBlock(BB);
320
321 if (MemDep)
323
324 if (DTU)
325 DTU->applyUpdates(Updates);
326
327 if (DT) {
328 assert(succ_empty(BB) &&
329 "successors should have been transferred to PredBB");
330 DT->eraseNode(BB);
331 }
332
333 // Finally, erase the old block and update dominator info.
334 DeleteDeadBlock(BB, DTU);
335
336 return true;
337}
338
341 LoopInfo *LI) {
342 assert(!MergeBlocks.empty() && "MergeBlocks should not be empty");
343
344 bool BlocksHaveBeenMerged = false;
345 while (!MergeBlocks.empty()) {
346 BasicBlock *BB = *MergeBlocks.begin();
347 BasicBlock *Dest = BB->getSingleSuccessor();
348 if (Dest && (!L || L->contains(Dest))) {
349 BasicBlock *Fold = Dest->getUniquePredecessor();
350 (void)Fold;
351 if (MergeBlockIntoPredecessor(Dest, DTU, LI)) {
352 assert(Fold == BB &&
353 "Expecting BB to be unique predecessor of the Dest block");
354 MergeBlocks.erase(Dest);
355 BlocksHaveBeenMerged = true;
356 } else
357 MergeBlocks.erase(BB);
358 } else
359 MergeBlocks.erase(BB);
360 }
361 return BlocksHaveBeenMerged;
362}
363
364/// Remove redundant instructions within sequences of consecutive dbg.value
365/// instructions. This is done using a backward scan to keep the last dbg.value
366/// describing a specific variable/fragment.
367///
368/// BackwardScan strategy:
369/// ----------------------
370/// Given a sequence of consecutive DbgValueInst like this
371///
372/// dbg.value ..., "x", FragmentX1 (*)
373/// dbg.value ..., "y", FragmentY1
374/// dbg.value ..., "x", FragmentX2
375/// dbg.value ..., "x", FragmentX1 (**)
376///
377/// then the instruction marked with (*) can be removed (it is guaranteed to be
378/// obsoleted by the instruction marked with (**) as the latter instruction is
379/// describing the same variable using the same fragment info).
380///
381/// Possible improvements:
382/// - Check fully overlapping fragments and not only identical fragments.
383/// - Support dbg.declare. dbg.label, and possibly other meta instructions being
384/// part of the sequence of consecutive instructions.
386 SmallVector<DPValue *, 8> ToBeRemoved;
388 for (auto &I : reverse(*BB)) {
389 for (DbgRecord &DR : reverse(I.getDbgRecordRange())) {
390 if (isa<DPLabel>(DR)) {
391 // Emulate existing behaviour (see comment below for dbg.declares).
392 // FIXME: Don't do this.
393 VariableSet.clear();
394 continue;
395 }
396
397 DPValue &DPV = cast<DPValue>(DR);
398 // Skip declare-type records, as the debug intrinsic method only works
399 // on dbg.value intrinsics.
400 if (DPV.getType() == DPValue::LocationType::Declare) {
401 // The debug intrinsic method treats dbg.declares are "non-debug"
402 // instructions (i.e., a break in a consecutive range of debug
403 // intrinsics). Emulate that to create identical outputs. See
404 // "Possible improvements" above.
405 // FIXME: Delete the line below.
406 VariableSet.clear();
407 continue;
408 }
409
410 DebugVariable Key(DPV.getVariable(), DPV.getExpression(),
411 DPV.getDebugLoc()->getInlinedAt());
412 auto R = VariableSet.insert(Key);
413 // If the same variable fragment is described more than once it is enough
414 // to keep the last one (i.e. the first found since we for reverse
415 // iteration).
416 if (R.second)
417 continue;
418
419 if (DPV.isDbgAssign()) {
420 // Don't delete dbg.assign intrinsics that are linked to instructions.
421 if (!at::getAssignmentInsts(&DPV).empty())
422 continue;
423 // Unlinked dbg.assign intrinsics can be treated like dbg.values.
424 }
425
426 ToBeRemoved.push_back(&DPV);
427 continue;
428 }
429 // Sequence with consecutive dbg.value instrs ended. Clear the map to
430 // restart identifying redundant instructions if case we find another
431 // dbg.value sequence.
432 VariableSet.clear();
433 }
434
435 for (auto &DPV : ToBeRemoved)
436 DPV->eraseFromParent();
437
438 return !ToBeRemoved.empty();
439}
440
442 if (BB->IsNewDbgInfoFormat)
444
447 for (auto &I : reverse(*BB)) {
448 if (DbgValueInst *DVI = dyn_cast<DbgValueInst>(&I)) {
449 DebugVariable Key(DVI->getVariable(),
450 DVI->getExpression(),
451 DVI->getDebugLoc()->getInlinedAt());
452 auto R = VariableSet.insert(Key);
453 // If the variable fragment hasn't been seen before then we don't want
454 // to remove this dbg intrinsic.
455 if (R.second)
456 continue;
457
458 if (auto *DAI = dyn_cast<DbgAssignIntrinsic>(DVI)) {
459 // Don't delete dbg.assign intrinsics that are linked to instructions.
460 if (!at::getAssignmentInsts(DAI).empty())
461 continue;
462 // Unlinked dbg.assign intrinsics can be treated like dbg.values.
463 }
464
465 // If the same variable fragment is described more than once it is enough
466 // to keep the last one (i.e. the first found since we for reverse
467 // iteration).
468 ToBeRemoved.push_back(DVI);
469 continue;
470 }
471 // Sequence with consecutive dbg.value instrs ended. Clear the map to
472 // restart identifying redundant instructions if case we find another
473 // dbg.value sequence.
474 VariableSet.clear();
475 }
476
477 for (auto &Instr : ToBeRemoved)
478 Instr->eraseFromParent();
479
480 return !ToBeRemoved.empty();
481}
482
483/// Remove redundant dbg.value instructions using a forward scan. This can
484/// remove a dbg.value instruction that is redundant due to indicating that a
485/// variable has the same value as already being indicated by an earlier
486/// dbg.value.
487///
488/// ForwardScan strategy:
489/// ---------------------
490/// Given two identical dbg.value instructions, separated by a block of
491/// instructions that isn't describing the same variable, like this
492///
493/// dbg.value X1, "x", FragmentX1 (**)
494/// <block of instructions, none being "dbg.value ..., "x", ...">
495/// dbg.value X1, "x", FragmentX1 (*)
496///
497/// then the instruction marked with (*) can be removed. Variable "x" is already
498/// described as being mapped to the SSA value X1.
499///
500/// Possible improvements:
501/// - Keep track of non-overlapping fragments.
503 SmallVector<DPValue *, 8> ToBeRemoved;
505 VariableMap;
506 for (auto &I : *BB) {
507 for (DPValue &DPV : filterDbgVars(I.getDbgRecordRange())) {
508 if (DPV.getType() == DPValue::LocationType::Declare)
509 continue;
510 DebugVariable Key(DPV.getVariable(), std::nullopt,
511 DPV.getDebugLoc()->getInlinedAt());
512 auto VMI = VariableMap.find(Key);
513 // A dbg.assign with no linked instructions can be treated like a
514 // dbg.value (i.e. can be deleted).
515 bool IsDbgValueKind =
516 (!DPV.isDbgAssign() || at::getAssignmentInsts(&DPV).empty());
517
518 // Update the map if we found a new value/expression describing the
519 // variable, or if the variable wasn't mapped already.
520 SmallVector<Value *, 4> Values(DPV.location_ops());
521 if (VMI == VariableMap.end() || VMI->second.first != Values ||
522 VMI->second.second != DPV.getExpression()) {
523 if (IsDbgValueKind)
524 VariableMap[Key] = {Values, DPV.getExpression()};
525 else
526 VariableMap[Key] = {Values, nullptr};
527 continue;
528 }
529 // Don't delete dbg.assign intrinsics that are linked to instructions.
530 if (!IsDbgValueKind)
531 continue;
532 // Found an identical mapping. Remember the instruction for later removal.
533 ToBeRemoved.push_back(&DPV);
534 }
535 }
536
537 for (auto *DPV : ToBeRemoved)
538 DPV->eraseFromParent();
539
540 return !ToBeRemoved.empty();
541}
542
544 assert(BB->isEntryBlock() && "expected entry block");
545 SmallVector<DPValue *, 8> ToBeRemoved;
546 DenseSet<DebugVariable> SeenDefForAggregate;
547 // Returns the DebugVariable for DVI with no fragment info.
548 auto GetAggregateVariable = [](const DPValue &DPV) {
549 return DebugVariable(DPV.getVariable(), std::nullopt,
550 DPV.getDebugLoc().getInlinedAt());
551 };
552
553 // Remove undef dbg.assign intrinsics that are encountered before
554 // any non-undef intrinsics from the entry block.
555 for (auto &I : *BB) {
556 for (DPValue &DPV : filterDbgVars(I.getDbgRecordRange())) {
557 if (!DPV.isDbgValue() && !DPV.isDbgAssign())
558 continue;
559 bool IsDbgValueKind =
560 (DPV.isDbgValue() || at::getAssignmentInsts(&DPV).empty());
561 DebugVariable Aggregate = GetAggregateVariable(DPV);
562 if (!SeenDefForAggregate.contains(Aggregate)) {
563 bool IsKill = DPV.isKillLocation() && IsDbgValueKind;
564 if (!IsKill) {
565 SeenDefForAggregate.insert(Aggregate);
566 } else if (DPV.isDbgAssign()) {
567 ToBeRemoved.push_back(&DPV);
568 }
569 }
570 }
571 }
572
573 for (DPValue *DPV : ToBeRemoved)
574 DPV->eraseFromParent();
575
576 return !ToBeRemoved.empty();
577}
578
580 if (BB->IsNewDbgInfoFormat)
582
585 VariableMap;
586 for (auto &I : *BB) {
587 if (DbgValueInst *DVI = dyn_cast<DbgValueInst>(&I)) {
588 DebugVariable Key(DVI->getVariable(), std::nullopt,
589 DVI->getDebugLoc()->getInlinedAt());
590 auto VMI = VariableMap.find(Key);
591 auto *DAI = dyn_cast<DbgAssignIntrinsic>(DVI);
592 // A dbg.assign with no linked instructions can be treated like a
593 // dbg.value (i.e. can be deleted).
594 bool IsDbgValueKind = (!DAI || at::getAssignmentInsts(DAI).empty());
595
596 // Update the map if we found a new value/expression describing the
597 // variable, or if the variable wasn't mapped already.
598 SmallVector<Value *, 4> Values(DVI->getValues());
599 if (VMI == VariableMap.end() || VMI->second.first != Values ||
600 VMI->second.second != DVI->getExpression()) {
601 // Use a sentinel value (nullptr) for the DIExpression when we see a
602 // linked dbg.assign so that the next debug intrinsic will never match
603 // it (i.e. always treat linked dbg.assigns as if they're unique).
604 if (IsDbgValueKind)
605 VariableMap[Key] = {Values, DVI->getExpression()};
606 else
607 VariableMap[Key] = {Values, nullptr};
608 continue;
609 }
610
611 // Don't delete dbg.assign intrinsics that are linked to instructions.
612 if (!IsDbgValueKind)
613 continue;
614 ToBeRemoved.push_back(DVI);
615 }
616 }
617
618 for (auto &Instr : ToBeRemoved)
619 Instr->eraseFromParent();
620
621 return !ToBeRemoved.empty();
622}
623
624/// Remove redundant undef dbg.assign intrinsic from an entry block using a
625/// forward scan.
626/// Strategy:
627/// ---------------------
628/// Scanning forward, delete dbg.assign intrinsics iff they are undef, not
629/// linked to an intrinsic, and don't share an aggregate variable with a debug
630/// intrinsic that didn't meet the criteria. In other words, undef dbg.assigns
631/// that come before non-undef debug intrinsics for the variable are
632/// deleted. Given:
633///
634/// dbg.assign undef, "x", FragmentX1 (*)
635/// <block of instructions, none being "dbg.value ..., "x", ...">
636/// dbg.value %V, "x", FragmentX2
637/// <block of instructions, none being "dbg.value ..., "x", ...">
638/// dbg.assign undef, "x", FragmentX1
639///
640/// then (only) the instruction marked with (*) can be removed.
641/// Possible improvements:
642/// - Keep track of non-overlapping fragments.
644 if (BB->IsNewDbgInfoFormat)
646
647 assert(BB->isEntryBlock() && "expected entry block");
649 DenseSet<DebugVariable> SeenDefForAggregate;
650 // Returns the DebugVariable for DVI with no fragment info.
651 auto GetAggregateVariable = [](DbgValueInst *DVI) {
652 return DebugVariable(DVI->getVariable(), std::nullopt,
653 DVI->getDebugLoc()->getInlinedAt());
654 };
655
656 // Remove undef dbg.assign intrinsics that are encountered before
657 // any non-undef intrinsics from the entry block.
658 for (auto &I : *BB) {
659 DbgValueInst *DVI = dyn_cast<DbgValueInst>(&I);
660 if (!DVI)
661 continue;
662 auto *DAI = dyn_cast<DbgAssignIntrinsic>(DVI);
663 bool IsDbgValueKind = (!DAI || at::getAssignmentInsts(DAI).empty());
664 DebugVariable Aggregate = GetAggregateVariable(DVI);
665 if (!SeenDefForAggregate.contains(Aggregate)) {
666 bool IsKill = DVI->isKillLocation() && IsDbgValueKind;
667 if (!IsKill) {
668 SeenDefForAggregate.insert(Aggregate);
669 } else if (DAI) {
670 ToBeRemoved.push_back(DAI);
671 }
672 }
673 }
674
675 for (DbgAssignIntrinsic *DAI : ToBeRemoved)
676 DAI->eraseFromParent();
677
678 return !ToBeRemoved.empty();
679}
680
682 bool MadeChanges = false;
683 // By using the "backward scan" strategy before the "forward scan" strategy we
684 // can remove both dbg.value (2) and (3) in a situation like this:
685 //
686 // (1) dbg.value V1, "x", DIExpression()
687 // ...
688 // (2) dbg.value V2, "x", DIExpression()
689 // (3) dbg.value V1, "x", DIExpression()
690 //
691 // The backward scan will remove (2), it is made obsolete by (3). After
692 // getting (2) out of the way, the foward scan will remove (3) since "x"
693 // already is described as having the value V1 at (1).
695 if (BB->isEntryBlock() &&
697 MadeChanges |= removeUndefDbgAssignsFromEntryBlock(BB);
699
700 if (MadeChanges)
701 LLVM_DEBUG(dbgs() << "Removed redundant dbg instrs from: "
702 << BB->getName() << "\n");
703 return MadeChanges;
704}
705
707 Instruction &I = *BI;
708 // Replaces all of the uses of the instruction with uses of the value
709 I.replaceAllUsesWith(V);
710
711 // Make sure to propagate a name if there is one already.
712 if (I.hasName() && !V->hasName())
713 V->takeName(&I);
714
715 // Delete the unnecessary instruction now...
716 BI = BI->eraseFromParent();
717}
718
720 Instruction *I) {
721 assert(I->getParent() == nullptr &&
722 "ReplaceInstWithInst: Instruction already inserted into basic block!");
723
724 // Copy debug location to newly added instruction, if it wasn't already set
725 // by the caller.
726 if (!I->getDebugLoc())
727 I->setDebugLoc(BI->getDebugLoc());
728
729 // Insert the new instruction into the basic block...
730 BasicBlock::iterator New = I->insertInto(BB, BI);
731
732 // Replace all uses of the old instruction, and delete it.
734
735 // Move BI back to point to the newly inserted instruction
736 BI = New;
737}
738
740 // Remember visited blocks to avoid infinite loop
742 unsigned Depth = 0;
744 VisitedBlocks.insert(BB).second) {
745 if (isa<UnreachableInst>(BB->getTerminator()) ||
747 return true;
748 BB = BB->getUniqueSuccessor();
749 }
750 return false;
751}
752
755 ReplaceInstWithInst(From->getParent(), BI, To);
756}
757
759 LoopInfo *LI, MemorySSAUpdater *MSSAU,
760 const Twine &BBName) {
761 unsigned SuccNum = GetSuccessorNumber(BB, Succ);
762
763 Instruction *LatchTerm = BB->getTerminator();
764
767
768 if ((isCriticalEdge(LatchTerm, SuccNum, Options.MergeIdenticalEdges))) {
769 // If it is a critical edge, and the succesor is an exception block, handle
770 // the split edge logic in this specific function
771 if (Succ->isEHPad())
772 return ehAwareSplitEdge(BB, Succ, nullptr, nullptr, Options, BBName);
773
774 // If this is a critical edge, let SplitKnownCriticalEdge do it.
775 return SplitKnownCriticalEdge(LatchTerm, SuccNum, Options, BBName);
776 }
777
778 // If the edge isn't critical, then BB has a single successor or Succ has a
779 // single pred. Split the block.
780 if (BasicBlock *SP = Succ->getSinglePredecessor()) {
781 // If the successor only has a single pred, split the top of the successor
782 // block.
783 assert(SP == BB && "CFG broken");
784 SP = nullptr;
785 return SplitBlock(Succ, &Succ->front(), DT, LI, MSSAU, BBName,
786 /*Before=*/true);
787 }
788
789 // Otherwise, if BB has a single successor, split it at the bottom of the
790 // block.
791 assert(BB->getTerminator()->getNumSuccessors() == 1 &&
792 "Should have a single succ!");
793 return SplitBlock(BB, BB->getTerminator(), DT, LI, MSSAU, BBName);
794}
795
797 if (auto *II = dyn_cast<InvokeInst>(TI))
798 II->setUnwindDest(Succ);
799 else if (auto *CS = dyn_cast<CatchSwitchInst>(TI))
800 CS->setUnwindDest(Succ);
801 else if (auto *CR = dyn_cast<CleanupReturnInst>(TI))
802 CR->setUnwindDest(Succ);
803 else
804 llvm_unreachable("unexpected terminator instruction");
805}
806
808 BasicBlock *NewPred, PHINode *Until) {
809 int BBIdx = 0;
810 for (PHINode &PN : DestBB->phis()) {
811 // We manually update the LandingPadReplacement PHINode and it is the last
812 // PHI Node. So, if we find it, we are done.
813 if (Until == &PN)
814 break;
815
816 // Reuse the previous value of BBIdx if it lines up. In cases where we
817 // have multiple phi nodes with *lots* of predecessors, this is a speed
818 // win because we don't have to scan the PHI looking for TIBB. This
819 // happens because the BB list of PHI nodes are usually in the same
820 // order.
821 if (PN.getIncomingBlock(BBIdx) != OldPred)
822 BBIdx = PN.getBasicBlockIndex(OldPred);
823
824 assert(BBIdx != -1 && "Invalid PHI Index!");
825 PN.setIncomingBlock(BBIdx, NewPred);
826 }
827}
828
830 LandingPadInst *OriginalPad,
831 PHINode *LandingPadReplacement,
833 const Twine &BBName) {
834
835 auto *PadInst = Succ->getFirstNonPHI();
836 if (!LandingPadReplacement && !PadInst->isEHPad())
837 return SplitEdge(BB, Succ, Options.DT, Options.LI, Options.MSSAU, BBName);
838
839 auto *LI = Options.LI;
841 // Check if extra modifications will be required to preserve loop-simplify
842 // form after splitting. If it would require splitting blocks with IndirectBr
843 // terminators, bail out if preserving loop-simplify form is requested.
844 if (Options.PreserveLoopSimplify && LI) {
845 if (Loop *BBLoop = LI->getLoopFor(BB)) {
846
847 // The only way that we can break LoopSimplify form by splitting a
848 // critical edge is when there exists some edge from BBLoop to Succ *and*
849 // the only edge into Succ from outside of BBLoop is that of NewBB after
850 // the split. If the first isn't true, then LoopSimplify still holds,
851 // NewBB is the new exit block and it has no non-loop predecessors. If the
852 // second isn't true, then Succ was not in LoopSimplify form prior to
853 // the split as it had a non-loop predecessor. In both of these cases,
854 // the predecessor must be directly in BBLoop, not in a subloop, or again
855 // LoopSimplify doesn't hold.
856 for (BasicBlock *P : predecessors(Succ)) {
857 if (P == BB)
858 continue; // The new block is known.
859 if (LI->getLoopFor(P) != BBLoop) {
860 // Loop is not in LoopSimplify form, no need to re simplify after
861 // splitting edge.
862 LoopPreds.clear();
863 break;
864 }
865 LoopPreds.push_back(P);
866 }
867 // Loop-simplify form can be preserved, if we can split all in-loop
868 // predecessors.
869 if (any_of(LoopPreds, [](BasicBlock *Pred) {
870 return isa<IndirectBrInst>(Pred->getTerminator());
871 })) {
872 return nullptr;
873 }
874 }
875 }
876
877 auto *NewBB =
878 BasicBlock::Create(BB->getContext(), BBName, BB->getParent(), Succ);
879 setUnwindEdgeTo(BB->getTerminator(), NewBB);
880 updatePhiNodes(Succ, BB, NewBB, LandingPadReplacement);
881
882 if (LandingPadReplacement) {
883 auto *NewLP = OriginalPad->clone();
884 auto *Terminator = BranchInst::Create(Succ, NewBB);
885 NewLP->insertBefore(Terminator);
886 LandingPadReplacement->addIncoming(NewLP, NewBB);
887 } else {
888 Value *ParentPad = nullptr;
889 if (auto *FuncletPad = dyn_cast<FuncletPadInst>(PadInst))
890 ParentPad = FuncletPad->getParentPad();
891 else if (auto *CatchSwitch = dyn_cast<CatchSwitchInst>(PadInst))
892 ParentPad = CatchSwitch->getParentPad();
893 else if (auto *CleanupPad = dyn_cast<CleanupPadInst>(PadInst))
894 ParentPad = CleanupPad->getParentPad();
895 else if (auto *LandingPad = dyn_cast<LandingPadInst>(PadInst))
896 ParentPad = LandingPad->getParent();
897 else
898 llvm_unreachable("handling for other EHPads not implemented yet");
899
900 auto *NewCleanupPad = CleanupPadInst::Create(ParentPad, {}, BBName, NewBB);
901 CleanupReturnInst::Create(NewCleanupPad, Succ, NewBB);
902 }
903
904 auto *DT = Options.DT;
905 auto *MSSAU = Options.MSSAU;
906 if (!DT && !LI)
907 return NewBB;
908
909 if (DT) {
910 DomTreeUpdater DTU(DT, DomTreeUpdater::UpdateStrategy::Lazy);
912
913 Updates.push_back({DominatorTree::Insert, BB, NewBB});
914 Updates.push_back({DominatorTree::Insert, NewBB, Succ});
915 Updates.push_back({DominatorTree::Delete, BB, Succ});
916
917 DTU.applyUpdates(Updates);
918 DTU.flush();
919
920 if (MSSAU) {
921 MSSAU->applyUpdates(Updates, *DT);
922 if (VerifyMemorySSA)
923 MSSAU->getMemorySSA()->verifyMemorySSA();
924 }
925 }
926
927 if (LI) {
928 if (Loop *BBLoop = LI->getLoopFor(BB)) {
929 // If one or the other blocks were not in a loop, the new block is not
930 // either, and thus LI doesn't need to be updated.
931 if (Loop *SuccLoop = LI->getLoopFor(Succ)) {
932 if (BBLoop == SuccLoop) {
933 // Both in the same loop, the NewBB joins loop.
934 SuccLoop->addBasicBlockToLoop(NewBB, *LI);
935 } else if (BBLoop->contains(SuccLoop)) {
936 // Edge from an outer loop to an inner loop. Add to the outer loop.
937 BBLoop->addBasicBlockToLoop(NewBB, *LI);
938 } else if (SuccLoop->contains(BBLoop)) {
939 // Edge from an inner loop to an outer loop. Add to the outer loop.
940 SuccLoop->addBasicBlockToLoop(NewBB, *LI);
941 } else {
942 // Edge from two loops with no containment relation. Because these
943 // are natural loops, we know that the destination block must be the
944 // header of its loop (adding a branch into a loop elsewhere would
945 // create an irreducible loop).
946 assert(SuccLoop->getHeader() == Succ &&
947 "Should not create irreducible loops!");
948 if (Loop *P = SuccLoop->getParentLoop())
949 P->addBasicBlockToLoop(NewBB, *LI);
950 }
951 }
952
953 // If BB is in a loop and Succ is outside of that loop, we may need to
954 // update LoopSimplify form and LCSSA form.
955 if (!BBLoop->contains(Succ)) {
956 assert(!BBLoop->contains(NewBB) &&
957 "Split point for loop exit is contained in loop!");
958
959 // Update LCSSA form in the newly created exit block.
960 if (Options.PreserveLCSSA) {
961 createPHIsForSplitLoopExit(BB, NewBB, Succ);
962 }
963
964 if (!LoopPreds.empty()) {
966 Succ, LoopPreds, "split", DT, LI, MSSAU, Options.PreserveLCSSA);
967 if (Options.PreserveLCSSA)
968 createPHIsForSplitLoopExit(LoopPreds, NewExitBB, Succ);
969 }
970 }
971 }
972 }
973
974 return NewBB;
975}
976
978 BasicBlock *SplitBB, BasicBlock *DestBB) {
979 // SplitBB shouldn't have anything non-trivial in it yet.
980 assert((SplitBB->getFirstNonPHI() == SplitBB->getTerminator() ||
981 SplitBB->isLandingPad()) &&
982 "SplitBB has non-PHI nodes!");
983
984 // For each PHI in the destination block.
985 for (PHINode &PN : DestBB->phis()) {
986 int Idx = PN.getBasicBlockIndex(SplitBB);
987 assert(Idx >= 0 && "Invalid Block Index");
988 Value *V = PN.getIncomingValue(Idx);
989
990 // If the input is a PHI which already satisfies LCSSA, don't create
991 // a new one.
992 if (const PHINode *VP = dyn_cast<PHINode>(V))
993 if (VP->getParent() == SplitBB)
994 continue;
995
996 // Otherwise a new PHI is needed. Create one and populate it.
997 PHINode *NewPN = PHINode::Create(PN.getType(), Preds.size(), "split");
998 BasicBlock::iterator InsertPos =
999 SplitBB->isLandingPad() ? SplitBB->begin()
1000 : SplitBB->getTerminator()->getIterator();
1001 NewPN->insertBefore(InsertPos);
1002 for (BasicBlock *BB : Preds)
1003 NewPN->addIncoming(V, BB);
1004
1005 // Update the original PHI.
1006 PN.setIncomingValue(Idx, NewPN);
1007 }
1008}
1009
1010unsigned
1013 unsigned NumBroken = 0;
1014 for (BasicBlock &BB : F) {
1015 Instruction *TI = BB.getTerminator();
1016 if (TI->getNumSuccessors() > 1 && !isa<IndirectBrInst>(TI))
1017 for (unsigned i = 0, e = TI->getNumSuccessors(); i != e; ++i)
1018 if (SplitCriticalEdge(TI, i, Options))
1019 ++NumBroken;
1020 }
1021 return NumBroken;
1022}
1023
1025 DomTreeUpdater *DTU, DominatorTree *DT,
1026 LoopInfo *LI, MemorySSAUpdater *MSSAU,
1027 const Twine &BBName, bool Before) {
1028 if (Before) {
1029 DomTreeUpdater LocalDTU(DT, DomTreeUpdater::UpdateStrategy::Lazy);
1030 return splitBlockBefore(Old, SplitPt,
1031 DTU ? DTU : (DT ? &LocalDTU : nullptr), LI, MSSAU,
1032 BBName);
1033 }
1034 BasicBlock::iterator SplitIt = SplitPt;
1035 while (isa<PHINode>(SplitIt) || SplitIt->isEHPad()) {
1036 ++SplitIt;
1037 assert(SplitIt != SplitPt->getParent()->end());
1038 }
1039 std::string Name = BBName.str();
1040 BasicBlock *New = Old->splitBasicBlock(
1041 SplitIt, Name.empty() ? Old->getName() + ".split" : Name);
1042
1043 // The new block lives in whichever loop the old one did. This preserves
1044 // LCSSA as well, because we force the split point to be after any PHI nodes.
1045 if (LI)
1046 if (Loop *L = LI->getLoopFor(Old))
1047 L->addBasicBlockToLoop(New, *LI);
1048
1049 if (DTU) {
1051 // Old dominates New. New node dominates all other nodes dominated by Old.
1052 SmallPtrSet<BasicBlock *, 8> UniqueSuccessorsOfOld;
1053 Updates.push_back({DominatorTree::Insert, Old, New});
1054 Updates.reserve(Updates.size() + 2 * succ_size(New));
1055 for (BasicBlock *SuccessorOfOld : successors(New))
1056 if (UniqueSuccessorsOfOld.insert(SuccessorOfOld).second) {
1057 Updates.push_back({DominatorTree::Insert, New, SuccessorOfOld});
1058 Updates.push_back({DominatorTree::Delete, Old, SuccessorOfOld});
1059 }
1060
1061 DTU->applyUpdates(Updates);
1062 } else if (DT)
1063 // Old dominates New. New node dominates all other nodes dominated by Old.
1064 if (DomTreeNode *OldNode = DT->getNode(Old)) {
1065 std::vector<DomTreeNode *> Children(OldNode->begin(), OldNode->end());
1066
1067 DomTreeNode *NewNode = DT->addNewBlock(New, Old);
1068 for (DomTreeNode *I : Children)
1069 DT->changeImmediateDominator(I, NewNode);
1070 }
1071
1072 // Move MemoryAccesses still tracked in Old, but part of New now.
1073 // Update accesses in successor blocks accordingly.
1074 if (MSSAU)
1075 MSSAU->moveAllAfterSpliceBlocks(Old, New, &*(New->begin()));
1076
1077 return New;
1078}
1079
1081 DominatorTree *DT, LoopInfo *LI,
1082 MemorySSAUpdater *MSSAU, const Twine &BBName,
1083 bool Before) {
1084 return SplitBlockImpl(Old, SplitPt, /*DTU=*/nullptr, DT, LI, MSSAU, BBName,
1085 Before);
1086}
1088 DomTreeUpdater *DTU, LoopInfo *LI,
1089 MemorySSAUpdater *MSSAU, const Twine &BBName,
1090 bool Before) {
1091 return SplitBlockImpl(Old, SplitPt, DTU, /*DT=*/nullptr, LI, MSSAU, BBName,
1092 Before);
1093}
1094
1096 DomTreeUpdater *DTU, LoopInfo *LI,
1097 MemorySSAUpdater *MSSAU,
1098 const Twine &BBName) {
1099
1100 BasicBlock::iterator SplitIt = SplitPt;
1101 while (isa<PHINode>(SplitIt) || SplitIt->isEHPad())
1102 ++SplitIt;
1103 std::string Name = BBName.str();
1104 BasicBlock *New = Old->splitBasicBlock(
1105 SplitIt, Name.empty() ? Old->getName() + ".split" : Name,
1106 /* Before=*/true);
1107
1108 // The new block lives in whichever loop the old one did. This preserves
1109 // LCSSA as well, because we force the split point to be after any PHI nodes.
1110 if (LI)
1111 if (Loop *L = LI->getLoopFor(Old))
1112 L->addBasicBlockToLoop(New, *LI);
1113
1114 if (DTU) {
1116 // New dominates Old. The predecessor nodes of the Old node dominate
1117 // New node.
1118 SmallPtrSet<BasicBlock *, 8> UniquePredecessorsOfOld;
1119 DTUpdates.push_back({DominatorTree::Insert, New, Old});
1120 DTUpdates.reserve(DTUpdates.size() + 2 * pred_size(New));
1121 for (BasicBlock *PredecessorOfOld : predecessors(New))
1122 if (UniquePredecessorsOfOld.insert(PredecessorOfOld).second) {
1123 DTUpdates.push_back({DominatorTree::Insert, PredecessorOfOld, New});
1124 DTUpdates.push_back({DominatorTree::Delete, PredecessorOfOld, Old});
1125 }
1126
1127 DTU->applyUpdates(DTUpdates);
1128
1129 // Move MemoryAccesses still tracked in Old, but part of New now.
1130 // Update accesses in successor blocks accordingly.
1131 if (MSSAU) {
1132 MSSAU->applyUpdates(DTUpdates, DTU->getDomTree());
1133 if (VerifyMemorySSA)
1134 MSSAU->getMemorySSA()->verifyMemorySSA();
1135 }
1136 }
1137 return New;
1138}
1139
1140/// Update DominatorTree, LoopInfo, and LCCSA analysis information.
1143 DomTreeUpdater *DTU, DominatorTree *DT,
1144 LoopInfo *LI, MemorySSAUpdater *MSSAU,
1145 bool PreserveLCSSA, bool &HasLoopExit) {
1146 // Update dominator tree if available.
1147 if (DTU) {
1148 // Recalculation of DomTree is needed when updating a forward DomTree and
1149 // the Entry BB is replaced.
1150 if (NewBB->isEntryBlock() && DTU->hasDomTree()) {
1151 // The entry block was removed and there is no external interface for
1152 // the dominator tree to be notified of this change. In this corner-case
1153 // we recalculate the entire tree.
1154 DTU->recalculate(*NewBB->getParent());
1155 } else {
1156 // Split block expects NewBB to have a non-empty set of predecessors.
1158 SmallPtrSet<BasicBlock *, 8> UniquePreds;
1159 Updates.push_back({DominatorTree::Insert, NewBB, OldBB});
1160 Updates.reserve(Updates.size() + 2 * Preds.size());
1161 for (auto *Pred : Preds)
1162 if (UniquePreds.insert(Pred).second) {
1163 Updates.push_back({DominatorTree::Insert, Pred, NewBB});
1164 Updates.push_back({DominatorTree::Delete, Pred, OldBB});
1165 }
1166 DTU->applyUpdates(Updates);
1167 }
1168 } else if (DT) {
1169 if (OldBB == DT->getRootNode()->getBlock()) {
1170 assert(NewBB->isEntryBlock());
1171 DT->setNewRoot(NewBB);
1172 } else {
1173 // Split block expects NewBB to have a non-empty set of predecessors.
1174 DT->splitBlock(NewBB);
1175 }
1176 }
1177
1178 // Update MemoryPhis after split if MemorySSA is available
1179 if (MSSAU)
1180 MSSAU->wireOldPredecessorsToNewImmediatePredecessor(OldBB, NewBB, Preds);
1181
1182 // The rest of the logic is only relevant for updating the loop structures.
1183 if (!LI)
1184 return;
1185
1186 if (DTU && DTU->hasDomTree())
1187 DT = &DTU->getDomTree();
1188 assert(DT && "DT should be available to update LoopInfo!");
1189 Loop *L = LI->getLoopFor(OldBB);
1190
1191 // If we need to preserve loop analyses, collect some information about how
1192 // this split will affect loops.
1193 bool IsLoopEntry = !!L;
1194 bool SplitMakesNewLoopHeader = false;
1195 for (BasicBlock *Pred : Preds) {
1196 // Preds that are not reachable from entry should not be used to identify if
1197 // OldBB is a loop entry or if SplitMakesNewLoopHeader. Unreachable blocks
1198 // are not within any loops, so we incorrectly mark SplitMakesNewLoopHeader
1199 // as true and make the NewBB the header of some loop. This breaks LI.
1200 if (!DT->isReachableFromEntry(Pred))
1201 continue;
1202 // If we need to preserve LCSSA, determine if any of the preds is a loop
1203 // exit.
1204 if (PreserveLCSSA)
1205 if (Loop *PL = LI->getLoopFor(Pred))
1206 if (!PL->contains(OldBB))
1207 HasLoopExit = true;
1208
1209 // If we need to preserve LoopInfo, note whether any of the preds crosses
1210 // an interesting loop boundary.
1211 if (!L)
1212 continue;
1213 if (L->contains(Pred))
1214 IsLoopEntry = false;
1215 else
1216 SplitMakesNewLoopHeader = true;
1217 }
1218
1219 // Unless we have a loop for OldBB, nothing else to do here.
1220 if (!L)
1221 return;
1222
1223 if (IsLoopEntry) {
1224 // Add the new block to the nearest enclosing loop (and not an adjacent
1225 // loop). To find this, examine each of the predecessors and determine which
1226 // loops enclose them, and select the most-nested loop which contains the
1227 // loop containing the block being split.
1228 Loop *InnermostPredLoop = nullptr;
1229 for (BasicBlock *Pred : Preds) {
1230 if (Loop *PredLoop = LI->getLoopFor(Pred)) {
1231 // Seek a loop which actually contains the block being split (to avoid
1232 // adjacent loops).
1233 while (PredLoop && !PredLoop->contains(OldBB))
1234 PredLoop = PredLoop->getParentLoop();
1235
1236 // Select the most-nested of these loops which contains the block.
1237 if (PredLoop && PredLoop->contains(OldBB) &&
1238 (!InnermostPredLoop ||
1239 InnermostPredLoop->getLoopDepth() < PredLoop->getLoopDepth()))
1240 InnermostPredLoop = PredLoop;
1241 }
1242 }
1243
1244 if (InnermostPredLoop)
1245 InnermostPredLoop->addBasicBlockToLoop(NewBB, *LI);
1246 } else {
1247 L->addBasicBlockToLoop(NewBB, *LI);
1248 if (SplitMakesNewLoopHeader)
1249 L->moveToHeader(NewBB);
1250 }
1251}
1252
1253/// Update the PHI nodes in OrigBB to include the values coming from NewBB.
1254/// This also updates AliasAnalysis, if available.
1255static void UpdatePHINodes(BasicBlock *OrigBB, BasicBlock *NewBB,
1257 bool HasLoopExit) {
1258 // Otherwise, create a new PHI node in NewBB for each PHI node in OrigBB.
1259 SmallPtrSet<BasicBlock *, 16> PredSet(Preds.begin(), Preds.end());
1260 for (BasicBlock::iterator I = OrigBB->begin(); isa<PHINode>(I); ) {
1261 PHINode *PN = cast<PHINode>(I++);
1262
1263 // Check to see if all of the values coming in are the same. If so, we
1264 // don't need to create a new PHI node, unless it's needed for LCSSA.
1265 Value *InVal = nullptr;
1266 if (!HasLoopExit) {
1267 InVal = PN->getIncomingValueForBlock(Preds[0]);
1268 for (unsigned i = 0, e = PN->getNumIncomingValues(); i != e; ++i) {
1269 if (!PredSet.count(PN->getIncomingBlock(i)))
1270 continue;
1271 if (!InVal)
1272 InVal = PN->getIncomingValue(i);
1273 else if (InVal != PN->getIncomingValue(i)) {
1274 InVal = nullptr;
1275 break;
1276 }
1277 }
1278 }
1279
1280 if (InVal) {
1281 // If all incoming values for the new PHI would be the same, just don't
1282 // make a new PHI. Instead, just remove the incoming values from the old
1283 // PHI.
1285 [&](unsigned Idx) {
1286 return PredSet.contains(PN->getIncomingBlock(Idx));
1287 },
1288 /* DeletePHIIfEmpty */ false);
1289
1290 // Add an incoming value to the PHI node in the loop for the preheader
1291 // edge.
1292 PN->addIncoming(InVal, NewBB);
1293 continue;
1294 }
1295
1296 // If the values coming into the block are not the same, we need a new
1297 // PHI.
1298 // Create the new PHI node, insert it into NewBB at the end of the block
1299 PHINode *NewPHI =
1300 PHINode::Create(PN->getType(), Preds.size(), PN->getName() + ".ph", BI->getIterator());
1301
1302 // NOTE! This loop walks backwards for a reason! First off, this minimizes
1303 // the cost of removal if we end up removing a large number of values, and
1304 // second off, this ensures that the indices for the incoming values aren't
1305 // invalidated when we remove one.
1306 for (int64_t i = PN->getNumIncomingValues() - 1; i >= 0; --i) {
1307 BasicBlock *IncomingBB = PN->getIncomingBlock(i);
1308 if (PredSet.count(IncomingBB)) {
1309 Value *V = PN->removeIncomingValue(i, false);
1310 NewPHI->addIncoming(V, IncomingBB);
1311 }
1312 }
1313
1314 PN->addIncoming(NewPHI, NewBB);
1315 }
1316}
1317
1319 BasicBlock *OrigBB, ArrayRef<BasicBlock *> Preds, const char *Suffix1,
1320 const char *Suffix2, SmallVectorImpl<BasicBlock *> &NewBBs,
1321 DomTreeUpdater *DTU, DominatorTree *DT, LoopInfo *LI,
1322 MemorySSAUpdater *MSSAU, bool PreserveLCSSA);
1323
1324static BasicBlock *
1326 const char *Suffix, DomTreeUpdater *DTU,
1327 DominatorTree *DT, LoopInfo *LI,
1328 MemorySSAUpdater *MSSAU, bool PreserveLCSSA) {
1329 // Do not attempt to split that which cannot be split.
1330 if (!BB->canSplitPredecessors())
1331 return nullptr;
1332
1333 // For the landingpads we need to act a bit differently.
1334 // Delegate this work to the SplitLandingPadPredecessors.
1335 if (BB->isLandingPad()) {
1337 std::string NewName = std::string(Suffix) + ".split-lp";
1338
1339 SplitLandingPadPredecessorsImpl(BB, Preds, Suffix, NewName.c_str(), NewBBs,
1340 DTU, DT, LI, MSSAU, PreserveLCSSA);
1341 return NewBBs[0];
1342 }
1343
1344 // Create new basic block, insert right before the original block.
1346 BB->getContext(), BB->getName() + Suffix, BB->getParent(), BB);
1347
1348 // The new block unconditionally branches to the old block.
1349 BranchInst *BI = BranchInst::Create(BB, NewBB);
1350
1351 Loop *L = nullptr;
1352 BasicBlock *OldLatch = nullptr;
1353 // Splitting the predecessors of a loop header creates a preheader block.
1354 if (LI && LI->isLoopHeader(BB)) {
1355 L = LI->getLoopFor(BB);
1356 // Using the loop start line number prevents debuggers stepping into the
1357 // loop body for this instruction.
1358 BI->setDebugLoc(L->getStartLoc());
1359
1360 // If BB is the header of the Loop, it is possible that the loop is
1361 // modified, such that the current latch does not remain the latch of the
1362 // loop. If that is the case, the loop metadata from the current latch needs
1363 // to be applied to the new latch.
1364 OldLatch = L->getLoopLatch();
1365 } else
1367
1368 // Move the edges from Preds to point to NewBB instead of BB.
1369 for (BasicBlock *Pred : Preds) {
1370 // This is slightly more strict than necessary; the minimum requirement
1371 // is that there be no more than one indirectbr branching to BB. And
1372 // all BlockAddress uses would need to be updated.
1373 assert(!isa<IndirectBrInst>(Pred->getTerminator()) &&
1374 "Cannot split an edge from an IndirectBrInst");
1375 Pred->getTerminator()->replaceSuccessorWith(BB, NewBB);
1376 }
1377
1378 // Insert a new PHI node into NewBB for every PHI node in BB and that new PHI
1379 // node becomes an incoming value for BB's phi node. However, if the Preds
1380 // list is empty, we need to insert dummy entries into the PHI nodes in BB to
1381 // account for the newly created predecessor.
1382 if (Preds.empty()) {
1383 // Insert dummy values as the incoming value.
1384 for (BasicBlock::iterator I = BB->begin(); isa<PHINode>(I); ++I)
1385 cast<PHINode>(I)->addIncoming(PoisonValue::get(I->getType()), NewBB);
1386 }
1387
1388 // Update DominatorTree, LoopInfo, and LCCSA analysis information.
1389 bool HasLoopExit = false;
1390 UpdateAnalysisInformation(BB, NewBB, Preds, DTU, DT, LI, MSSAU, PreserveLCSSA,
1391 HasLoopExit);
1392
1393 if (!Preds.empty()) {
1394 // Update the PHI nodes in BB with the values coming from NewBB.
1395 UpdatePHINodes(BB, NewBB, Preds, BI, HasLoopExit);
1396 }
1397
1398 if (OldLatch) {
1399 BasicBlock *NewLatch = L->getLoopLatch();
1400 if (NewLatch != OldLatch) {
1401 MDNode *MD = OldLatch->getTerminator()->getMetadata("llvm.loop");
1402 NewLatch->getTerminator()->setMetadata("llvm.loop", MD);
1403 // It's still possible that OldLatch is the latch of another inner loop,
1404 // in which case we do not remove the metadata.
1405 Loop *IL = LI->getLoopFor(OldLatch);
1406 if (IL && IL->getLoopLatch() != OldLatch)
1407 OldLatch->getTerminator()->setMetadata("llvm.loop", nullptr);
1408 }
1409 }
1410
1411 return NewBB;
1412}
1413
1416 const char *Suffix, DominatorTree *DT,
1417 LoopInfo *LI, MemorySSAUpdater *MSSAU,
1418 bool PreserveLCSSA) {
1419 return SplitBlockPredecessorsImpl(BB, Preds, Suffix, /*DTU=*/nullptr, DT, LI,
1420 MSSAU, PreserveLCSSA);
1421}
1424 const char *Suffix,
1425 DomTreeUpdater *DTU, LoopInfo *LI,
1426 MemorySSAUpdater *MSSAU,
1427 bool PreserveLCSSA) {
1428 return SplitBlockPredecessorsImpl(BB, Preds, Suffix, DTU,
1429 /*DT=*/nullptr, LI, MSSAU, PreserveLCSSA);
1430}
1431
1433 BasicBlock *OrigBB, ArrayRef<BasicBlock *> Preds, const char *Suffix1,
1434 const char *Suffix2, SmallVectorImpl<BasicBlock *> &NewBBs,
1435 DomTreeUpdater *DTU, DominatorTree *DT, LoopInfo *LI,
1436 MemorySSAUpdater *MSSAU, bool PreserveLCSSA) {
1437 assert(OrigBB->isLandingPad() && "Trying to split a non-landing pad!");
1438
1439 // Create a new basic block for OrigBB's predecessors listed in Preds. Insert
1440 // it right before the original block.
1441 BasicBlock *NewBB1 = BasicBlock::Create(OrigBB->getContext(),
1442 OrigBB->getName() + Suffix1,
1443 OrigBB->getParent(), OrigBB);
1444 NewBBs.push_back(NewBB1);
1445
1446 // The new block unconditionally branches to the old block.
1447 BranchInst *BI1 = BranchInst::Create(OrigBB, NewBB1);
1448 BI1->setDebugLoc(OrigBB->getFirstNonPHI()->getDebugLoc());
1449
1450 // Move the edges from Preds to point to NewBB1 instead of OrigBB.
1451 for (BasicBlock *Pred : Preds) {
1452 // This is slightly more strict than necessary; the minimum requirement
1453 // is that there be no more than one indirectbr branching to BB. And
1454 // all BlockAddress uses would need to be updated.
1455 assert(!isa<IndirectBrInst>(Pred->getTerminator()) &&
1456 "Cannot split an edge from an IndirectBrInst");
1457 Pred->getTerminator()->replaceUsesOfWith(OrigBB, NewBB1);
1458 }
1459
1460 bool HasLoopExit = false;
1461 UpdateAnalysisInformation(OrigBB, NewBB1, Preds, DTU, DT, LI, MSSAU,
1462 PreserveLCSSA, HasLoopExit);
1463
1464 // Update the PHI nodes in OrigBB with the values coming from NewBB1.
1465 UpdatePHINodes(OrigBB, NewBB1, Preds, BI1, HasLoopExit);
1466
1467 // Move the remaining edges from OrigBB to point to NewBB2.
1468 SmallVector<BasicBlock*, 8> NewBB2Preds;
1469 for (pred_iterator i = pred_begin(OrigBB), e = pred_end(OrigBB);
1470 i != e; ) {
1471 BasicBlock *Pred = *i++;
1472 if (Pred == NewBB1) continue;
1473 assert(!isa<IndirectBrInst>(Pred->getTerminator()) &&
1474 "Cannot split an edge from an IndirectBrInst");
1475 NewBB2Preds.push_back(Pred);
1476 e = pred_end(OrigBB);
1477 }
1478
1479 BasicBlock *NewBB2 = nullptr;
1480 if (!NewBB2Preds.empty()) {
1481 // Create another basic block for the rest of OrigBB's predecessors.
1482 NewBB2 = BasicBlock::Create(OrigBB->getContext(),
1483 OrigBB->getName() + Suffix2,
1484 OrigBB->getParent(), OrigBB);
1485 NewBBs.push_back(NewBB2);
1486
1487 // The new block unconditionally branches to the old block.
1488 BranchInst *BI2 = BranchInst::Create(OrigBB, NewBB2);
1489 BI2->setDebugLoc(OrigBB->getFirstNonPHI()->getDebugLoc());
1490
1491 // Move the remaining edges from OrigBB to point to NewBB2.
1492 for (BasicBlock *NewBB2Pred : NewBB2Preds)
1493 NewBB2Pred->getTerminator()->replaceUsesOfWith(OrigBB, NewBB2);
1494
1495 // Update DominatorTree, LoopInfo, and LCCSA analysis information.
1496 HasLoopExit = false;
1497 UpdateAnalysisInformation(OrigBB, NewBB2, NewBB2Preds, DTU, DT, LI, MSSAU,
1498 PreserveLCSSA, HasLoopExit);
1499
1500 // Update the PHI nodes in OrigBB with the values coming from NewBB2.
1501 UpdatePHINodes(OrigBB, NewBB2, NewBB2Preds, BI2, HasLoopExit);
1502 }
1503
1504 LandingPadInst *LPad = OrigBB->getLandingPadInst();
1505 Instruction *Clone1 = LPad->clone();
1506 Clone1->setName(Twine("lpad") + Suffix1);
1507 Clone1->insertInto(NewBB1, NewBB1->getFirstInsertionPt());
1508
1509 if (NewBB2) {
1510 Instruction *Clone2 = LPad->clone();
1511 Clone2->setName(Twine("lpad") + Suffix2);
1512 Clone2->insertInto(NewBB2, NewBB2->getFirstInsertionPt());
1513
1514 // Create a PHI node for the two cloned landingpad instructions only
1515 // if the original landingpad instruction has some uses.
1516 if (!LPad->use_empty()) {
1517 assert(!LPad->getType()->isTokenTy() &&
1518 "Split cannot be applied if LPad is token type. Otherwise an "
1519 "invalid PHINode of token type would be created.");
1520 PHINode *PN = PHINode::Create(LPad->getType(), 2, "lpad.phi", LPad->getIterator());
1521 PN->addIncoming(Clone1, NewBB1);
1522 PN->addIncoming(Clone2, NewBB2);
1523 LPad->replaceAllUsesWith(PN);
1524 }
1525 LPad->eraseFromParent();
1526 } else {
1527 // There is no second clone. Just replace the landing pad with the first
1528 // clone.
1529 LPad->replaceAllUsesWith(Clone1);
1530 LPad->eraseFromParent();
1531 }
1532}
1533
1536 const char *Suffix1, const char *Suffix2,
1538 DomTreeUpdater *DTU, LoopInfo *LI,
1539 MemorySSAUpdater *MSSAU,
1540 bool PreserveLCSSA) {
1541 return SplitLandingPadPredecessorsImpl(OrigBB, Preds, Suffix1, Suffix2,
1542 NewBBs, DTU, /*DT=*/nullptr, LI, MSSAU,
1543 PreserveLCSSA);
1544}
1545
1547 BasicBlock *Pred,
1548 DomTreeUpdater *DTU) {
1549 Instruction *UncondBranch = Pred->getTerminator();
1550 // Clone the return and add it to the end of the predecessor.
1551 Instruction *NewRet = RI->clone();
1552 NewRet->insertInto(Pred, Pred->end());
1553
1554 // If the return instruction returns a value, and if the value was a
1555 // PHI node in "BB", propagate the right value into the return.
1556 for (Use &Op : NewRet->operands()) {
1557 Value *V = Op;
1558 Instruction *NewBC = nullptr;
1559 if (BitCastInst *BCI = dyn_cast<BitCastInst>(V)) {
1560 // Return value might be bitcasted. Clone and insert it before the
1561 // return instruction.
1562 V = BCI->getOperand(0);
1563 NewBC = BCI->clone();
1564 NewBC->insertInto(Pred, NewRet->getIterator());
1565 Op = NewBC;
1566 }
1567
1568 Instruction *NewEV = nullptr;
1569 if (ExtractValueInst *EVI = dyn_cast<ExtractValueInst>(V)) {
1570 V = EVI->getOperand(0);
1571 NewEV = EVI->clone();
1572 if (NewBC) {
1573 NewBC->setOperand(0, NewEV);
1574 NewEV->insertInto(Pred, NewBC->getIterator());
1575 } else {
1576 NewEV->insertInto(Pred, NewRet->getIterator());
1577 Op = NewEV;
1578 }
1579 }
1580
1581 if (PHINode *PN = dyn_cast<PHINode>(V)) {
1582 if (PN->getParent() == BB) {
1583 if (NewEV) {
1584 NewEV->setOperand(0, PN->getIncomingValueForBlock(Pred));
1585 } else if (NewBC)
1586 NewBC->setOperand(0, PN->getIncomingValueForBlock(Pred));
1587 else
1588 Op = PN->getIncomingValueForBlock(Pred);
1589 }
1590 }
1591 }
1592
1593 // Update any PHI nodes in the returning block to realize that we no
1594 // longer branch to them.
1595 BB->removePredecessor(Pred);
1596 UncondBranch->eraseFromParent();
1597
1598 if (DTU)
1599 DTU->applyUpdates({{DominatorTree::Delete, Pred, BB}});
1600
1601 return cast<ReturnInst>(NewRet);
1602}
1603
1605 BasicBlock::iterator SplitBefore,
1606 bool Unreachable,
1607 MDNode *BranchWeights,
1608 DomTreeUpdater *DTU, LoopInfo *LI,
1609 BasicBlock *ThenBlock) {
1611 Cond, SplitBefore, &ThenBlock, /* ElseBlock */ nullptr,
1612 /* UnreachableThen */ Unreachable,
1613 /* UnreachableElse */ false, BranchWeights, DTU, LI);
1614 return ThenBlock->getTerminator();
1615}
1616
1618 BasicBlock::iterator SplitBefore,
1619 bool Unreachable,
1620 MDNode *BranchWeights,
1621 DomTreeUpdater *DTU, LoopInfo *LI,
1622 BasicBlock *ElseBlock) {
1624 Cond, SplitBefore, /* ThenBlock */ nullptr, &ElseBlock,
1625 /* UnreachableThen */ false,
1626 /* UnreachableElse */ Unreachable, BranchWeights, DTU, LI);
1627 return ElseBlock->getTerminator();
1628}
1629
1631 Instruction **ThenTerm,
1632 Instruction **ElseTerm,
1633 MDNode *BranchWeights,
1634 DomTreeUpdater *DTU, LoopInfo *LI) {
1635 BasicBlock *ThenBlock = nullptr;
1636 BasicBlock *ElseBlock = nullptr;
1638 Cond, SplitBefore, &ThenBlock, &ElseBlock, /* UnreachableThen */ false,
1639 /* UnreachableElse */ false, BranchWeights, DTU, LI);
1640
1641 *ThenTerm = ThenBlock->getTerminator();
1642 *ElseTerm = ElseBlock->getTerminator();
1643}
1644
1646 Value *Cond, BasicBlock::iterator SplitBefore, BasicBlock **ThenBlock,
1647 BasicBlock **ElseBlock, bool UnreachableThen, bool UnreachableElse,
1648 MDNode *BranchWeights, DomTreeUpdater *DTU, LoopInfo *LI) {
1649 assert((ThenBlock || ElseBlock) &&
1650 "At least one branch block must be created");
1651 assert((!UnreachableThen || !UnreachableElse) &&
1652 "Split block tail must be reachable");
1653
1655 SmallPtrSet<BasicBlock *, 8> UniqueOrigSuccessors;
1656 BasicBlock *Head = SplitBefore->getParent();
1657 if (DTU) {
1658 UniqueOrigSuccessors.insert(succ_begin(Head), succ_end(Head));
1659 Updates.reserve(4 + 2 * UniqueOrigSuccessors.size());
1660 }
1661
1662 LLVMContext &C = Head->getContext();
1663 BasicBlock *Tail = Head->splitBasicBlock(SplitBefore);
1664 BasicBlock *TrueBlock = Tail;
1665 BasicBlock *FalseBlock = Tail;
1666 bool ThenToTailEdge = false;
1667 bool ElseToTailEdge = false;
1668
1669 // Encapsulate the logic around creation/insertion/etc of a new block.
1670 auto handleBlock = [&](BasicBlock **PBB, bool Unreachable, BasicBlock *&BB,
1671 bool &ToTailEdge) {
1672 if (PBB == nullptr)
1673 return; // Do not create/insert a block.
1674
1675 if (*PBB)
1676 BB = *PBB; // Caller supplied block, use it.
1677 else {
1678 // Create a new block.
1679 BB = BasicBlock::Create(C, "", Head->getParent(), Tail);
1680 if (Unreachable)
1681 (void)new UnreachableInst(C, BB);
1682 else {
1683 (void)BranchInst::Create(Tail, BB);
1684 ToTailEdge = true;
1685 }
1686 BB->getTerminator()->setDebugLoc(SplitBefore->getDebugLoc());
1687 // Pass the new block back to the caller.
1688 *PBB = BB;
1689 }
1690 };
1691
1692 handleBlock(ThenBlock, UnreachableThen, TrueBlock, ThenToTailEdge);
1693 handleBlock(ElseBlock, UnreachableElse, FalseBlock, ElseToTailEdge);
1694
1695 Instruction *HeadOldTerm = Head->getTerminator();
1696 BranchInst *HeadNewTerm =
1697 BranchInst::Create(/*ifTrue*/ TrueBlock, /*ifFalse*/ FalseBlock, Cond);
1698 HeadNewTerm->setMetadata(LLVMContext::MD_prof, BranchWeights);
1699 ReplaceInstWithInst(HeadOldTerm, HeadNewTerm);
1700
1701 if (DTU) {
1702 Updates.emplace_back(DominatorTree::Insert, Head, TrueBlock);
1703 Updates.emplace_back(DominatorTree::Insert, Head, FalseBlock);
1704 if (ThenToTailEdge)
1705 Updates.emplace_back(DominatorTree::Insert, TrueBlock, Tail);
1706 if (ElseToTailEdge)
1707 Updates.emplace_back(DominatorTree::Insert, FalseBlock, Tail);
1708 for (BasicBlock *UniqueOrigSuccessor : UniqueOrigSuccessors)
1709 Updates.emplace_back(DominatorTree::Insert, Tail, UniqueOrigSuccessor);
1710 for (BasicBlock *UniqueOrigSuccessor : UniqueOrigSuccessors)
1711 Updates.emplace_back(DominatorTree::Delete, Head, UniqueOrigSuccessor);
1712 DTU->applyUpdates(Updates);
1713 }
1714
1715 if (LI) {
1716 if (Loop *L = LI->getLoopFor(Head); L) {
1717 if (ThenToTailEdge)
1718 L->addBasicBlockToLoop(TrueBlock, *LI);
1719 if (ElseToTailEdge)
1720 L->addBasicBlockToLoop(FalseBlock, *LI);
1721 L->addBasicBlockToLoop(Tail, *LI);
1722 }
1723 }
1724}
1725
1726std::pair<Instruction*, Value*>
1728 BasicBlock *LoopPred = SplitBefore->getParent();
1729 BasicBlock *LoopBody = SplitBlock(SplitBefore->getParent(), SplitBefore);
1730 BasicBlock *LoopExit = SplitBlock(SplitBefore->getParent(), SplitBefore);
1731
1732 auto *Ty = End->getType();
1733 auto &DL = SplitBefore->getModule()->getDataLayout();
1734 const unsigned Bitwidth = DL.getTypeSizeInBits(Ty);
1735
1736 IRBuilder<> Builder(LoopBody->getTerminator());
1737 auto *IV = Builder.CreatePHI(Ty, 2, "iv");
1738 auto *IVNext =
1739 Builder.CreateAdd(IV, ConstantInt::get(Ty, 1), IV->getName() + ".next",
1740 /*HasNUW=*/true, /*HasNSW=*/Bitwidth != 2);
1741 auto *IVCheck = Builder.CreateICmpEQ(IVNext, End,
1742 IV->getName() + ".check");
1743 Builder.CreateCondBr(IVCheck, LoopExit, LoopBody);
1744 LoopBody->getTerminator()->eraseFromParent();
1745
1746 // Populate the IV PHI.
1747 IV->addIncoming(ConstantInt::get(Ty, 0), LoopPred);
1748 IV->addIncoming(IVNext, LoopBody);
1749
1750 return std::make_pair(LoopBody->getFirstNonPHI(), IV);
1751}
1752
1754 Type *IndexTy, Instruction *InsertBefore,
1755 std::function<void(IRBuilderBase&, Value*)> Func) {
1756
1757 IRBuilder<> IRB(InsertBefore);
1758
1759 if (EC.isScalable()) {
1760 Value *NumElements = IRB.CreateElementCount(IndexTy, EC);
1761
1762 auto [BodyIP, Index] =
1763 SplitBlockAndInsertSimpleForLoop(NumElements, InsertBefore);
1764
1765 IRB.SetInsertPoint(BodyIP);
1766 Func(IRB, Index);
1767 return;
1768 }
1769
1770 unsigned Num = EC.getFixedValue();
1771 for (unsigned Idx = 0; Idx < Num; ++Idx) {
1772 IRB.SetInsertPoint(InsertBefore);
1773 Func(IRB, ConstantInt::get(IndexTy, Idx));
1774 }
1775}
1776
1778 Value *EVL, Instruction *InsertBefore,
1779 std::function<void(IRBuilderBase &, Value *)> Func) {
1780
1781 IRBuilder<> IRB(InsertBefore);
1782 Type *Ty = EVL->getType();
1783
1784 if (!isa<ConstantInt>(EVL)) {
1785 auto [BodyIP, Index] = SplitBlockAndInsertSimpleForLoop(EVL, InsertBefore);
1786 IRB.SetInsertPoint(BodyIP);
1787 Func(IRB, Index);
1788 return;
1789 }
1790
1791 unsigned Num = cast<ConstantInt>(EVL)->getZExtValue();
1792 for (unsigned Idx = 0; Idx < Num; ++Idx) {
1793 IRB.SetInsertPoint(InsertBefore);
1794 Func(IRB, ConstantInt::get(Ty, Idx));
1795 }
1796}
1797
1799 BasicBlock *&IfFalse) {
1800 PHINode *SomePHI = dyn_cast<PHINode>(BB->begin());
1801 BasicBlock *Pred1 = nullptr;
1802 BasicBlock *Pred2 = nullptr;
1803
1804 if (SomePHI) {
1805 if (SomePHI->getNumIncomingValues() != 2)
1806 return nullptr;
1807 Pred1 = SomePHI->getIncomingBlock(0);
1808 Pred2 = SomePHI->getIncomingBlock(1);
1809 } else {
1810 pred_iterator PI = pred_begin(BB), PE = pred_end(BB);
1811 if (PI == PE) // No predecessor
1812 return nullptr;
1813 Pred1 = *PI++;
1814 if (PI == PE) // Only one predecessor
1815 return nullptr;
1816 Pred2 = *PI++;
1817 if (PI != PE) // More than two predecessors
1818 return nullptr;
1819 }
1820
1821 // We can only handle branches. Other control flow will be lowered to
1822 // branches if possible anyway.
1823 BranchInst *Pred1Br = dyn_cast<BranchInst>(Pred1->getTerminator());
1824 BranchInst *Pred2Br = dyn_cast<BranchInst>(Pred2->getTerminator());
1825 if (!Pred1Br || !Pred2Br)
1826 return nullptr;
1827
1828 // Eliminate code duplication by ensuring that Pred1Br is conditional if
1829 // either are.
1830 if (Pred2Br->isConditional()) {
1831 // If both branches are conditional, we don't have an "if statement". In
1832 // reality, we could transform this case, but since the condition will be
1833 // required anyway, we stand no chance of eliminating it, so the xform is
1834 // probably not profitable.
1835 if (Pred1Br->isConditional())
1836 return nullptr;
1837
1838 std::swap(Pred1, Pred2);
1839 std::swap(Pred1Br, Pred2Br);
1840 }
1841
1842 if (Pred1Br->isConditional()) {
1843 // The only thing we have to watch out for here is to make sure that Pred2
1844 // doesn't have incoming edges from other blocks. If it does, the condition
1845 // doesn't dominate BB.
1846 if (!Pred2->getSinglePredecessor())
1847 return nullptr;
1848
1849 // If we found a conditional branch predecessor, make sure that it branches
1850 // to BB and Pred2Br. If it doesn't, this isn't an "if statement".
1851 if (Pred1Br->getSuccessor(0) == BB &&
1852 Pred1Br->getSuccessor(1) == Pred2) {
1853 IfTrue = Pred1;
1854 IfFalse = Pred2;
1855 } else if (Pred1Br->getSuccessor(0) == Pred2 &&
1856 Pred1Br->getSuccessor(1) == BB) {
1857 IfTrue = Pred2;
1858 IfFalse = Pred1;
1859 } else {
1860 // We know that one arm of the conditional goes to BB, so the other must
1861 // go somewhere unrelated, and this must not be an "if statement".
1862 return nullptr;
1863 }
1864
1865 return Pred1Br;
1866 }
1867
1868 // Ok, if we got here, both predecessors end with an unconditional branch to
1869 // BB. Don't panic! If both blocks only have a single (identical)
1870 // predecessor, and THAT is a conditional branch, then we're all ok!
1871 BasicBlock *CommonPred = Pred1->getSinglePredecessor();
1872 if (CommonPred == nullptr || CommonPred != Pred2->getSinglePredecessor())
1873 return nullptr;
1874
1875 // Otherwise, if this is a conditional branch, then we can use it!
1876 BranchInst *BI = dyn_cast<BranchInst>(CommonPred->getTerminator());
1877 if (!BI) return nullptr;
1878
1879 assert(BI->isConditional() && "Two successors but not conditional?");
1880 if (BI->getSuccessor(0) == Pred1) {
1881 IfTrue = Pred1;
1882 IfFalse = Pred2;
1883 } else {
1884 IfTrue = Pred2;
1885 IfFalse = Pred1;
1886 }
1887 return BI;
1888}
1889
1890// After creating a control flow hub, the operands of PHINodes in an outgoing
1891// block Out no longer match the predecessors of that block. Predecessors of Out
1892// that are incoming blocks to the hub are now replaced by just one edge from
1893// the hub. To match this new control flow, the corresponding values from each
1894// PHINode must now be moved a new PHINode in the first guard block of the hub.
1895//
1896// This operation cannot be performed with SSAUpdater, because it involves one
1897// new use: If the block Out is in the list of Incoming blocks, then the newly
1898// created PHI in the Hub will use itself along that edge from Out to Hub.
1899static void reconnectPhis(BasicBlock *Out, BasicBlock *GuardBlock,
1901 BasicBlock *FirstGuardBlock) {
1902 auto I = Out->begin();
1903 while (I != Out->end() && isa<PHINode>(I)) {
1904 auto Phi = cast<PHINode>(I);
1905 auto NewPhi =
1906 PHINode::Create(Phi->getType(), Incoming.size(),
1907 Phi->getName() + ".moved", FirstGuardBlock->begin());
1908 for (auto *In : Incoming) {
1909 Value *V = UndefValue::get(Phi->getType());
1910 if (In == Out) {
1911 V = NewPhi;
1912 } else if (Phi->getBasicBlockIndex(In) != -1) {
1913 V = Phi->removeIncomingValue(In, false);
1914 }
1915 NewPhi->addIncoming(V, In);
1916 }
1917 assert(NewPhi->getNumIncomingValues() == Incoming.size());
1918 if (Phi->getNumOperands() == 0) {
1919 Phi->replaceAllUsesWith(NewPhi);
1920 I = Phi->eraseFromParent();
1921 continue;
1922 }
1923 Phi->addIncoming(NewPhi, GuardBlock);
1924 ++I;
1925 }
1926}
1927
1930
1931// Redirects the terminator of the incoming block to the first guard
1932// block in the hub. The condition of the original terminator (if it
1933// was conditional) and its original successors are returned as a
1934// tuple <condition, succ0, succ1>. The function additionally filters
1935// out successors that are not in the set of outgoing blocks.
1936//
1937// - condition is non-null iff the branch is conditional.
1938// - Succ1 is non-null iff the sole/taken target is an outgoing block.
1939// - Succ2 is non-null iff condition is non-null and the fallthrough
1940// target is an outgoing block.
1941static std::tuple<Value *, BasicBlock *, BasicBlock *>
1943 const BBSetVector &Outgoing) {
1944 assert(isa<BranchInst>(BB->getTerminator()) &&
1945 "Only support branch terminator.");
1946 auto Branch = cast<BranchInst>(BB->getTerminator());
1947 auto Condition = Branch->isConditional() ? Branch->getCondition() : nullptr;
1948
1949 BasicBlock *Succ0 = Branch->getSuccessor(0);
1950 BasicBlock *Succ1 = nullptr;
1951 Succ0 = Outgoing.count(Succ0) ? Succ0 : nullptr;
1952
1953 if (Branch->isUnconditional()) {
1954 Branch->setSuccessor(0, FirstGuardBlock);
1955 assert(Succ0);
1956 } else {
1957 Succ1 = Branch->getSuccessor(1);
1958 Succ1 = Outgoing.count(Succ1) ? Succ1 : nullptr;
1959 assert(Succ0 || Succ1);
1960 if (Succ0 && !Succ1) {
1961 Branch->setSuccessor(0, FirstGuardBlock);
1962 } else if (Succ1 && !Succ0) {
1963 Branch->setSuccessor(1, FirstGuardBlock);
1964 } else {
1965 Branch->eraseFromParent();
1966 BranchInst::Create(FirstGuardBlock, BB);
1967 }
1968 }
1969
1970 assert(Succ0 || Succ1);
1971 return std::make_tuple(Condition, Succ0, Succ1);
1972}
1973// Setup the branch instructions for guard blocks.
1974//
1975// Each guard block terminates in a conditional branch that transfers
1976// control to the corresponding outgoing block or the next guard
1977// block. The last guard block has two outgoing blocks as successors
1978// since the condition for the final outgoing block is trivially
1979// true. So we create one less block (including the first guard block)
1980// than the number of outgoing blocks.
1982 const BBSetVector &Outgoing,
1983 BBPredicates &GuardPredicates) {
1984 // To help keep the loop simple, temporarily append the last
1985 // outgoing block to the list of guard blocks.
1986 GuardBlocks.push_back(Outgoing.back());
1987
1988 for (int i = 0, e = GuardBlocks.size() - 1; i != e; ++i) {
1989 auto Out = Outgoing[i];
1990 assert(GuardPredicates.count(Out));
1991 BranchInst::Create(Out, GuardBlocks[i + 1], GuardPredicates[Out],
1992 GuardBlocks[i]);
1993 }
1994
1995 // Remove the last block from the guard list.
1996 GuardBlocks.pop_back();
1997}
1998
1999/// We are using one integer to represent the block we are branching to. Then at
2000/// each guard block, the predicate was calcuated using a simple `icmp eq`.
2002 const BBSetVector &Incoming, const BBSetVector &Outgoing,
2003 SmallVectorImpl<BasicBlock *> &GuardBlocks, BBPredicates &GuardPredicates) {
2004 auto &Context = Incoming.front()->getContext();
2005 auto FirstGuardBlock = GuardBlocks.front();
2006
2008 "merged.bb.idx", FirstGuardBlock);
2009
2010 for (auto In : Incoming) {
2011 Value *Condition;
2012 BasicBlock *Succ0;
2013 BasicBlock *Succ1;
2014 std::tie(Condition, Succ0, Succ1) =
2015 redirectToHub(In, FirstGuardBlock, Outgoing);
2016 Value *IncomingId = nullptr;
2017 if (Succ0 && Succ1) {
2018 // target_bb_index = Condition ? index_of_succ0 : index_of_succ1.
2019 auto Succ0Iter = find(Outgoing, Succ0);
2020 auto Succ1Iter = find(Outgoing, Succ1);
2021 Value *Id0 = ConstantInt::get(Type::getInt32Ty(Context),
2022 std::distance(Outgoing.begin(), Succ0Iter));
2023 Value *Id1 = ConstantInt::get(Type::getInt32Ty(Context),
2024 std::distance(Outgoing.begin(), Succ1Iter));
2025 IncomingId = SelectInst::Create(Condition, Id0, Id1, "target.bb.idx",
2026 In->getTerminator()->getIterator());
2027 } else {
2028 // Get the index of the non-null successor.
2029 auto SuccIter = Succ0 ? find(Outgoing, Succ0) : find(Outgoing, Succ1);
2030 IncomingId = ConstantInt::get(Type::getInt32Ty(Context),
2031 std::distance(Outgoing.begin(), SuccIter));
2032 }
2033 Phi->addIncoming(IncomingId, In);
2034 }
2035
2036 for (int i = 0, e = Outgoing.size() - 1; i != e; ++i) {
2037 auto Out = Outgoing[i];
2038 auto Cmp = ICmpInst::Create(Instruction::ICmp, ICmpInst::ICMP_EQ, Phi,
2039 ConstantInt::get(Type::getInt32Ty(Context), i),
2040 Out->getName() + ".predicate", GuardBlocks[i]);
2041 GuardPredicates[Out] = Cmp;
2042 }
2043}
2044
2045/// We record the predicate of each outgoing block using a phi of boolean.
2047 const BBSetVector &Incoming, const BBSetVector &Outgoing,
2048 SmallVectorImpl<BasicBlock *> &GuardBlocks, BBPredicates &GuardPredicates,
2049 SmallVectorImpl<WeakVH> &DeletionCandidates) {
2050 auto &Context = Incoming.front()->getContext();
2051 auto BoolTrue = ConstantInt::getTrue(Context);
2052 auto BoolFalse = ConstantInt::getFalse(Context);
2053 auto FirstGuardBlock = GuardBlocks.front();
2054
2055 // The predicate for the last outgoing is trivially true, and so we
2056 // process only the first N-1 successors.
2057 for (int i = 0, e = Outgoing.size() - 1; i != e; ++i) {
2058 auto Out = Outgoing[i];
2059 LLVM_DEBUG(dbgs() << "Creating guard for " << Out->getName() << "\n");
2060
2061 auto Phi =
2063 StringRef("Guard.") + Out->getName(), FirstGuardBlock);
2064 GuardPredicates[Out] = Phi;
2065 }
2066
2067 for (auto *In : Incoming) {
2068 Value *Condition;
2069 BasicBlock *Succ0;
2070 BasicBlock *Succ1;
2071 std::tie(Condition, Succ0, Succ1) =
2072 redirectToHub(In, FirstGuardBlock, Outgoing);
2073
2074 // Optimization: Consider an incoming block A with both successors
2075 // Succ0 and Succ1 in the set of outgoing blocks. The predicates
2076 // for Succ0 and Succ1 complement each other. If Succ0 is visited
2077 // first in the loop below, control will branch to Succ0 using the
2078 // corresponding predicate. But if that branch is not taken, then
2079 // control must reach Succ1, which means that the incoming value of
2080 // the predicate from `In` is true for Succ1.
2081 bool OneSuccessorDone = false;
2082 for (int i = 0, e = Outgoing.size() - 1; i != e; ++i) {
2083 auto Out = Outgoing[i];
2084 PHINode *Phi = cast<PHINode>(GuardPredicates[Out]);
2085 if (Out != Succ0 && Out != Succ1) {
2086 Phi->addIncoming(BoolFalse, In);
2087 } else if (!Succ0 || !Succ1 || OneSuccessorDone) {
2088 // Optimization: When only one successor is an outgoing block,
2089 // the incoming predicate from `In` is always true.
2090 Phi->addIncoming(BoolTrue, In);
2091 } else {
2092 assert(Succ0 && Succ1);
2093 if (Out == Succ0) {
2094 Phi->addIncoming(Condition, In);
2095 } else {
2096 auto Inverted = invertCondition(Condition);
2097 DeletionCandidates.push_back(Condition);
2098 Phi->addIncoming(Inverted, In);
2099 }
2100 OneSuccessorDone = true;
2101 }
2102 }
2103 }
2104}
2105
2106// Capture the existing control flow as guard predicates, and redirect
2107// control flow from \p Incoming block through the \p GuardBlocks to the
2108// \p Outgoing blocks.
2109//
2110// There is one guard predicate for each outgoing block OutBB. The
2111// predicate represents whether the hub should transfer control flow
2112// to OutBB. These predicates are NOT ORTHOGONAL. The Hub evaluates
2113// them in the same order as the Outgoing set-vector, and control
2114// branches to the first outgoing block whose predicate evaluates to true.
2115static void
2117 SmallVectorImpl<WeakVH> &DeletionCandidates,
2118 const BBSetVector &Incoming,
2119 const BBSetVector &Outgoing, const StringRef Prefix,
2120 std::optional<unsigned> MaxControlFlowBooleans) {
2121 BBPredicates GuardPredicates;
2122 auto F = Incoming.front()->getParent();
2123
2124 for (int i = 0, e = Outgoing.size() - 1; i != e; ++i)
2125 GuardBlocks.push_back(
2126 BasicBlock::Create(F->getContext(), Prefix + ".guard", F));
2127
2128 // When we are using an integer to record which target block to jump to, we
2129 // are creating less live values, actually we are using one single integer to
2130 // store the index of the target block. When we are using booleans to store
2131 // the branching information, we need (N-1) boolean values, where N is the
2132 // number of outgoing block.
2133 if (!MaxControlFlowBooleans || Outgoing.size() <= *MaxControlFlowBooleans)
2134 calcPredicateUsingBooleans(Incoming, Outgoing, GuardBlocks, GuardPredicates,
2135 DeletionCandidates);
2136 else
2137 calcPredicateUsingInteger(Incoming, Outgoing, GuardBlocks, GuardPredicates);
2138
2139 setupBranchForGuard(GuardBlocks, Outgoing, GuardPredicates);
2140}
2141
2144 const BBSetVector &Incoming, const BBSetVector &Outgoing,
2145 const StringRef Prefix, std::optional<unsigned> MaxControlFlowBooleans) {
2146 if (Outgoing.size() < 2)
2147 return Outgoing.front();
2148
2150 if (DTU) {
2151 for (auto *In : Incoming) {
2152 for (auto Succ : successors(In))
2153 if (Outgoing.count(Succ))
2154 Updates.push_back({DominatorTree::Delete, In, Succ});
2155 }
2156 }
2157
2158 SmallVector<WeakVH, 8> DeletionCandidates;
2159 convertToGuardPredicates(GuardBlocks, DeletionCandidates, Incoming, Outgoing,
2160 Prefix, MaxControlFlowBooleans);
2161 auto FirstGuardBlock = GuardBlocks.front();
2162
2163 // Update the PHINodes in each outgoing block to match the new control flow.
2164 for (int i = 0, e = GuardBlocks.size(); i != e; ++i)
2165 reconnectPhis(Outgoing[i], GuardBlocks[i], Incoming, FirstGuardBlock);
2166
2167 reconnectPhis(Outgoing.back(), GuardBlocks.back(), Incoming, FirstGuardBlock);
2168
2169 if (DTU) {
2170 int NumGuards = GuardBlocks.size();
2171 assert((int)Outgoing.size() == NumGuards + 1);
2172
2173 for (auto In : Incoming)
2174 Updates.push_back({DominatorTree::Insert, In, FirstGuardBlock});
2175
2176 for (int i = 0; i != NumGuards - 1; ++i) {
2177 Updates.push_back({DominatorTree::Insert, GuardBlocks[i], Outgoing[i]});
2178 Updates.push_back(
2179 {DominatorTree::Insert, GuardBlocks[i], GuardBlocks[i + 1]});
2180 }
2181 Updates.push_back({DominatorTree::Insert, GuardBlocks[NumGuards - 1],
2182 Outgoing[NumGuards - 1]});
2183 Updates.push_back({DominatorTree::Insert, GuardBlocks[NumGuards - 1],
2184 Outgoing[NumGuards]});
2185 DTU->applyUpdates(Updates);
2186 }
2187
2188 for (auto I : DeletionCandidates) {
2189 if (I->use_empty())
2190 if (auto Inst = dyn_cast_or_null<Instruction>(I))
2191 Inst->eraseFromParent();
2192 }
2193
2194 return FirstGuardBlock;
2195}
2196
2198 Value *NewCond = PBI->getCondition();
2199 // If this is a "cmp" instruction, only used for branching (and nowhere
2200 // else), then we can simply invert the predicate.
2201 if (NewCond->hasOneUse() && isa<CmpInst>(NewCond)) {
2202 CmpInst *CI = cast<CmpInst>(NewCond);
2204 } else
2205 NewCond = Builder.CreateNot(NewCond, NewCond->getName() + ".not");
2206
2207 PBI->setCondition(NewCond);
2208 PBI->swapSuccessors();
2209}
2210
2212 for (auto &BB : F) {
2213 auto *Term = BB.getTerminator();
2214 if (!(isa<ReturnInst>(Term) || isa<UnreachableInst>(Term) ||
2215 isa<BranchInst>(Term)))
2216 return false;
2217 }
2218 return true;
2219}
2220
2222 const BasicBlock &Dest) {
2223 assert(Src.getParent() == Dest.getParent());
2224 if (!Src.getParent()->isPresplitCoroutine())
2225 return false;
2226 if (auto *SW = dyn_cast<SwitchInst>(Src.getTerminator()))
2227 if (auto *Intr = dyn_cast<IntrinsicInst>(SW->getCondition()))
2228 return Intr->getIntrinsicID() == Intrinsic::coro_suspend &&
2229 SW->getDefaultDest() == &Dest;
2230 return false;
2231}
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
unsigned Intr
static BasicBlock * SplitBlockPredecessorsImpl(BasicBlock *BB, ArrayRef< BasicBlock * > Preds, const char *Suffix, DomTreeUpdater *DTU, DominatorTree *DT, LoopInfo *LI, MemorySSAUpdater *MSSAU, bool PreserveLCSSA)
static bool DPValuesRemoveRedundantDbgInstrsUsingForwardScan(BasicBlock *BB)
Remove redundant dbg.value instructions using a forward scan.
static void convertToGuardPredicates(SmallVectorImpl< BasicBlock * > &GuardBlocks, SmallVectorImpl< WeakVH > &DeletionCandidates, const BBSetVector &Incoming, const BBSetVector &Outgoing, const StringRef Prefix, std::optional< unsigned > MaxControlFlowBooleans)
static bool removeRedundantDbgInstrsUsingBackwardScan(BasicBlock *BB)
static BasicBlock * SplitBlockImpl(BasicBlock *Old, BasicBlock::iterator SplitPt, DomTreeUpdater *DTU, DominatorTree *DT, LoopInfo *LI, MemorySSAUpdater *MSSAU, const Twine &BBName, bool Before)
static bool DPValuesRemoveUndefDbgAssignsFromEntryBlock(BasicBlock *BB)
static void calcPredicateUsingBooleans(const BBSetVector &Incoming, const BBSetVector &Outgoing, SmallVectorImpl< BasicBlock * > &GuardBlocks, BBPredicates &GuardPredicates, SmallVectorImpl< WeakVH > &DeletionCandidates)
We record the predicate of each outgoing block using a phi of boolean.
static void UpdatePHINodes(BasicBlock *OrigBB, BasicBlock *NewBB, ArrayRef< BasicBlock * > Preds, BranchInst *BI, bool HasLoopExit)
Update the PHI nodes in OrigBB to include the values coming from NewBB.
static bool removeUndefDbgAssignsFromEntryBlock(BasicBlock *BB)
Remove redundant undef dbg.assign intrinsic from an entry block using a forward scan.
static std::tuple< Value *, BasicBlock *, BasicBlock * > redirectToHub(BasicBlock *BB, BasicBlock *FirstGuardBlock, const BBSetVector &Outgoing)
static void setupBranchForGuard(SmallVectorImpl< BasicBlock * > &GuardBlocks, const BBSetVector &Outgoing, BBPredicates &GuardPredicates)
static void UpdateAnalysisInformation(BasicBlock *OldBB, BasicBlock *NewBB, ArrayRef< BasicBlock * > Preds, DomTreeUpdater *DTU, DominatorTree *DT, LoopInfo *LI, MemorySSAUpdater *MSSAU, bool PreserveLCSSA, bool &HasLoopExit)
Update DominatorTree, LoopInfo, and LCCSA analysis information.
static void reconnectPhis(BasicBlock *Out, BasicBlock *GuardBlock, const SetVector< BasicBlock * > &Incoming, BasicBlock *FirstGuardBlock)
static bool DPValuesRemoveRedundantDbgInstrsUsingBackwardScan(BasicBlock *BB)
Remove redundant instructions within sequences of consecutive dbg.value instructions.
static void calcPredicateUsingInteger(const BBSetVector &Incoming, const BBSetVector &Outgoing, SmallVectorImpl< BasicBlock * > &GuardBlocks, BBPredicates &GuardPredicates)
We are using one integer to represent the block we are branching to.
static bool removeRedundantDbgInstrsUsingForwardScan(BasicBlock *BB)
static void SplitLandingPadPredecessorsImpl(BasicBlock *OrigBB, ArrayRef< BasicBlock * > Preds, const char *Suffix1, const char *Suffix2, SmallVectorImpl< BasicBlock * > &NewBBs, DomTreeUpdater *DTU, DominatorTree *DT, LoopInfo *LI, MemorySSAUpdater *MSSAU, bool PreserveLCSSA)
static cl::opt< unsigned > MaxDeoptOrUnreachableSuccessorCheckDepth("max-deopt-or-unreachable-succ-check-depth", cl::init(8), cl::Hidden, cl::desc("Set the maximum path length when checking whether a basic block " "is followed by a block that either has a terminating " "deoptimizing call or is terminated with an unreachable"))
BlockVerifier::State From
This file contains the declarations for the subclasses of Constant, which represent the different fla...
Returns the sub type a function will return at a given Idx Should correspond to the result type of an ExtractValue instruction executed with just that one unsigned Idx
#define LLVM_DEBUG(X)
Definition: Debug.h:101
std::string Name
bool End
Definition: ELF_riscv.cpp:480
This file provides various utilities for inspecting and working with the control flow graph in LLVM I...
static LVOptions Options
Definition: LVOptions.cpp:25
#define F(x, y, z)
Definition: MD5.cpp:55
#define I(x, y, z)
Definition: MD5.cpp:58
LLVMContext & Context
#define P(N)
const SmallVectorImpl< MachineOperand > & Cond
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
This file defines the SmallPtrSet class.
This file defines the SmallVector class.
static const uint32_t IV[8]
Definition: blake3_impl.h:78
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
Definition: ArrayRef.h:41
iterator end() const
Definition: ArrayRef.h:154
size_t size() const
size - Get the array size.
Definition: ArrayRef.h:165
iterator begin() const
Definition: ArrayRef.h:153
bool empty() const
empty - Check if the array is empty.
Definition: ArrayRef.h:160
LLVM Basic Block Representation.
Definition: BasicBlock.h:60
iterator end()
Definition: BasicBlock.h:442
iterator begin()
Instruction iterator methods.
Definition: BasicBlock.h:429
iterator_range< const_phi_iterator > phis() const
Returns a range that iterates over the phis in the basic block.
Definition: BasicBlock.h:498
const LandingPadInst * getLandingPadInst() const
Return the landingpad instruction associated with the landing pad.
Definition: BasicBlock.cpp:663
const_iterator getFirstInsertionPt() const
Returns an iterator to the first instruction in this block that is suitable for inserting a non-PHI i...
Definition: BasicBlock.cpp:396
bool hasAddressTaken() const
Returns true if there are any uses of this basic block other than direct branches,...
Definition: BasicBlock.h:639
const Instruction * getFirstNonPHI() const
Returns a pointer to the first instruction in this block that is not a PHINode instruction.
Definition: BasicBlock.cpp:347
const Instruction & front() const
Definition: BasicBlock.h:452
static BasicBlock * Create(LLVMContext &Context, const Twine &Name="", Function *Parent=nullptr, BasicBlock *InsertBefore=nullptr)
Creates a new BasicBlock.
Definition: BasicBlock.h:198
bool isEntryBlock() const
Return true if this is the entry block of the containing function.
Definition: BasicBlock.cpp:551
BasicBlock * splitBasicBlock(iterator I, const Twine &BBName="", bool Before=false)
Split the basic block into two basic blocks at the specified instruction.
Definition: BasicBlock.cpp:557
const BasicBlock * getUniqueSuccessor() const
Return the successor of this block if it has a unique successor.
Definition: BasicBlock.cpp:477
const BasicBlock * getSinglePredecessor() const
Return the predecessor of this block if it has a single predecessor block.
Definition: BasicBlock.cpp:439
const CallInst * getTerminatingDeoptimizeCall() const
Returns the call instruction calling @llvm.experimental.deoptimize prior to the terminating return in...
Definition: BasicBlock.cpp:311
const BasicBlock * getUniquePredecessor() const
Return the predecessor of this block if it has a unique predecessor block.
Definition: BasicBlock.cpp:447
const BasicBlock * getSingleSuccessor() const
Return the successor of this block if it has a single successor.
Definition: BasicBlock.cpp:469
const Function * getParent() const
Return the enclosing method, or null if none.
Definition: BasicBlock.h:205
const Instruction * getFirstNonPHIOrDbg(bool SkipPseudoOp=true) const
Returns a pointer to the first instruction in this block that is not a PHINode or a debug intrinsic,...
Definition: BasicBlock.cpp:366
InstListType::iterator iterator
Instruction iterators...
Definition: BasicBlock.h:164
LLVMContext & getContext() const
Get the context in which this basic block lives.
Definition: BasicBlock.cpp:155
bool IsNewDbgInfoFormat
Flag recording whether or not this block stores debug-info in the form of intrinsic instructions (fal...
Definition: BasicBlock.h:65
bool isLandingPad() const
Return true if this basic block is a landing pad.
Definition: BasicBlock.cpp:659
bool isEHPad() const
Return true if this basic block is an exception handling block.
Definition: BasicBlock.h:656
const Instruction * getTerminator() const LLVM_READONLY
Returns the terminator instruction if the block is well formed or null if the block is not well forme...
Definition: BasicBlock.h:220
bool canSplitPredecessors() const
Definition: BasicBlock.cpp:525
void splice(BasicBlock::iterator ToIt, BasicBlock *FromBB)
Transfer all instructions from FromBB to this basic block at ToIt.
Definition: BasicBlock.h:612
const Instruction & back() const
Definition: BasicBlock.h:454
void removePredecessor(BasicBlock *Pred, bool KeepOneInputPHIs=false)
Update PHI nodes in this BasicBlock before removal of predecessor Pred.
Definition: BasicBlock.cpp:496
This class represents a no-op cast from one type to another.
Conditional or Unconditional Branch instruction.
void setCondition(Value *V)
void swapSuccessors()
Swap the successors of this branch instruction.
static BranchInst * Create(BasicBlock *IfTrue, BasicBlock::iterator InsertBefore)
bool isConditional() const
BasicBlock * getSuccessor(unsigned i) const
bool isUnconditional() const
void setSuccessor(unsigned idx, BasicBlock *NewSucc)
Value * getCondition() const
static CleanupPadInst * Create(Value *ParentPad, ArrayRef< Value * > Args, const Twine &NameStr, BasicBlock::iterator InsertBefore)
static CleanupReturnInst * Create(Value *CleanupPad, BasicBlock *UnwindBB, BasicBlock::iterator InsertBefore)
This class is the base class for the comparison instructions.
Definition: InstrTypes.h:955
void setPredicate(Predicate P)
Set the predicate for this instruction to the specified value.
Definition: InstrTypes.h:1069
Predicate getInversePredicate() const
For example, EQ -> NE, UGT -> ULE, SLT -> SGE, OEQ -> UNE, UGT -> OLE, OLT -> UGE,...
Definition: InstrTypes.h:1090
static ConstantInt * getTrue(LLVMContext &Context)
Definition: Constants.cpp:849
static ConstantInt * getFalse(LLVMContext &Context)
Definition: Constants.cpp:856
DWARF expression.
Record of a variable value-assignment, aka a non instruction representation of the dbg....
DIExpression * getExpression() const
LocationType getType() const
DILocalVariable * getVariable() const
This class represents an Operation in the Expression.
This represents the llvm.dbg.assign instruction.
Base class for non-instruction debug metadata records that have positions within IR.
DebugLoc getDebugLoc() const
This represents the llvm.dbg.value instruction.
Identifies a unique instance of a variable.
iterator find(const_arg_type_t< KeyT > Val)
Definition: DenseMap.h:155
size_type count(const_arg_type_t< KeyT > Val) const
Return 1 if the specified key is in the map, 0 otherwise.
Definition: DenseMap.h:151
iterator end()
Definition: DenseMap.h:84
Implements a dense probed hash-table based set.
Definition: DenseSet.h:271
iterator_range< iterator > children()
NodeT * getBlock() const
void flush()
Apply all pending updates to available trees and flush all BasicBlocks awaiting deletion.
void recalculate(Function &F)
Notify DTU that the entry block was replaced.
bool hasDomTree() const
Returns true if it holds a DominatorTree.
void applyUpdates(ArrayRef< DominatorTree::UpdateType > Updates)
Submit updates to all available trees.
void deleteBB(BasicBlock *DelBB)
Delete DelBB.
DominatorTree & getDomTree()
Flush DomTree updates and return DomTree.
DomTreeNodeBase< NodeT > * getRootNode()
getRootNode - This returns the entry node for the CFG of the function.
void changeImmediateDominator(DomTreeNodeBase< NodeT > *N, DomTreeNodeBase< NodeT > *NewIDom)
changeImmediateDominator - This method is used to update the dominator tree information when a node's...
DomTreeNodeBase< NodeT > * addNewBlock(NodeT *BB, NodeT *DomBB)
Add a new node to the dominator tree information.
void splitBlock(NodeT *NewBB)
splitBlock - BB is split and now it has one successor.
DomTreeNodeBase< NodeT > * setNewRoot(NodeT *BB)
Add a new node to the forward dominator tree and make it a new root.
void eraseNode(NodeT *BB)
eraseNode - Removes a node from the dominator tree.
DomTreeNodeBase< NodeT > * getNode(const NodeT *BB) const
getNode - return the (Post)DominatorTree node for the specified basic block.
Concrete subclass of DominatorTreeBase that is used to compute a normal dominator tree.
Definition: Dominators.h:162
bool isReachableFromEntry(const Use &U) const
Provide an overload for a Use.
Definition: Dominators.cpp:321
This instruction extracts a struct member or array element value from an aggregate value.
Module * getParent()
Get the module that this global value is contained inside of...
Definition: GlobalValue.h:655
Common base class shared among various IRBuilders.
Definition: IRBuilder.h:94
PHINode * CreatePHI(Type *Ty, unsigned NumReservedValues, const Twine &Name="")
Definition: IRBuilder.h:2380
Value * CreateNot(Value *V, const Twine &Name="")
Definition: IRBuilder.h:1748
Value * CreateICmpEQ(Value *LHS, Value *RHS, const Twine &Name="")
Definition: IRBuilder.h:2224
BranchInst * CreateCondBr(Value *Cond, BasicBlock *True, BasicBlock *False, MDNode *BranchWeights=nullptr, MDNode *Unpredictable=nullptr)
Create a conditional 'br Cond, TrueDest, FalseDest' instruction.
Definition: IRBuilder.h:1114
Value * CreateAdd(Value *LHS, Value *RHS, const Twine &Name="", bool HasNUW=false, bool HasNSW=false)
Definition: IRBuilder.h:1321
Value * CreateElementCount(Type *DstType, ElementCount EC)
Create an expression which evaluates to the number of elements in EC at runtime.
Definition: IRBuilder.cpp:99
void SetInsertPoint(BasicBlock *TheBB)
This specifies that created instructions should be appended to the end of the specified block.
Definition: IRBuilder.h:180
This provides a uniform API for creating instructions and inserting them into a basic block: either a...
Definition: IRBuilder.h:2649
Instruction * clone() const
Create a copy of 'this' instruction that is identical in all ways except the following:
void moveBeforePreserving(Instruction *MovePos)
Perform a moveBefore operation, while signalling that the caller intends to preserve the original ord...
unsigned getNumSuccessors() const LLVM_READONLY
Return the number of successors that this instruction has.
void insertBefore(Instruction *InsertPos)
Insert an unlinked instruction into a basic block immediately before the specified instruction.
const DebugLoc & getDebugLoc() const
Return the debug location for this node as a DebugLoc.
Definition: Instruction.h:453
const Module * getModule() const
Return the module owning the function this instruction belongs to or nullptr it the function does not...
Definition: Instruction.cpp:80
bool isEHPad() const
Return true if the instruction is a variety of EH-block.
Definition: Instruction.h:801
const BasicBlock * getParent() const
Definition: Instruction.h:151
InstListType::iterator eraseFromParent()
This method unlinks 'this' from the containing basic block and deletes it.
MDNode * getMetadata(unsigned KindID) const
Get the metadata of given kind attached to this Instruction.
Definition: Instruction.h:358
bool mayHaveSideEffects() const LLVM_READONLY
Return true if the instruction may have side effects.
void setMetadata(unsigned KindID, MDNode *Node)
Set the metadata of the specified kind to the specified node.
Definition: Metadata.cpp:1633
void setDebugLoc(DebugLoc Loc)
Set the debug location information for this instruction.
Definition: Instruction.h:450
bool isSpecialTerminator() const
Definition: Instruction.h:261
InstListType::iterator insertInto(BasicBlock *ParentBB, InstListType::iterator It)
Inserts an unlinked instruction into ParentBB at position It and returns the iterator of the inserted...
This is an important class for using LLVM in a threaded context.
Definition: LLVMContext.h:67
The landingpad instruction holds all of the information necessary to generate correct exception handl...
BlockT * getLoopLatch() const
If there is a single latch block for this loop, return it.
unsigned getLoopDepth() const
Return the nesting level of this loop.
void addBasicBlockToLoop(BlockT *NewBB, LoopInfoBase< BlockT, LoopT > &LI)
This method is used by other analyses to update loop information.
LoopT * getParentLoop() const
Return the parent loop if it exists or nullptr for top level loops.
void removeBlock(BlockT *BB)
This method completely removes BB from all data structures, including all of the Loop objects it is n...
bool isLoopHeader(const BlockT *BB) const
LoopT * getLoopFor(const BlockT *BB) const
Return the inner most loop that BB lives in.
Represents a single loop in the control flow graph.
Definition: LoopInfo.h:44
Metadata node.
Definition: Metadata.h:1067
Provides a lazy, caching interface for making common memory aliasing information queries,...
void invalidateCachedPredecessors()
Clears the PredIteratorCache info.
void removeInstruction(Instruction *InstToRemove)
Removes an instruction from the dependence analysis, updating the dependence of instructions that pre...
MemorySSA * getMemorySSA() const
Get handle on MemorySSA.
void moveAllAfterSpliceBlocks(BasicBlock *From, BasicBlock *To, Instruction *Start)
From block was spliced into From and To.
void applyUpdates(ArrayRef< CFGUpdate > Updates, DominatorTree &DT, bool UpdateDTFirst=false)
Apply CFG updates, analogous with the DT edge updates.
void moveAllAfterMergeBlocks(BasicBlock *From, BasicBlock *To, Instruction *Start)
From block was merged into To.
void moveToPlace(MemoryUseOrDef *What, BasicBlock *BB, MemorySSA::InsertionPlace Where)
void wireOldPredecessorsToNewImmediatePredecessor(BasicBlock *Old, BasicBlock *New, ArrayRef< BasicBlock * > Preds, bool IdenticalEdgesWereMerged=true)
A new empty BasicBlock (New) now branches directly to Old.
void verifyMemorySSA(VerificationLevel=VerificationLevel::Fast) const
Verify that MemorySSA is self consistent (IE definitions dominate all uses, uses appear in the right ...
Definition: MemorySSA.cpp:1857
MemoryUseOrDef * getMemoryAccess(const Instruction *I) const
Given a memory Mod/Ref'ing instruction, get the MemorySSA access associated with it.
Definition: MemorySSA.h:717
Class that has the common methods + fields of memory uses/defs.
Definition: MemorySSA.h:252
const DataLayout & getDataLayout() const
Get the data layout for the module's target platform.
Definition: Module.h:287
void addIncoming(Value *V, BasicBlock *BB)
Add an incoming value to the end of the PHI list.
void removeIncomingValueIf(function_ref< bool(unsigned)> Predicate, bool DeletePHIIfEmpty=true)
Remove all incoming values for which the predicate returns true.
static PHINode * Create(Type *Ty, unsigned NumReservedValues, const Twine &NameStr, BasicBlock::iterator InsertBefore)
Constructors - NumReservedValues is a hint for the number of incoming edges that this phi node will h...
Value * removeIncomingValue(unsigned Idx, bool DeletePHIIfEmpty=true)
Remove an incoming value.
Value * getIncomingValueForBlock(const BasicBlock *BB) const
BasicBlock * getIncomingBlock(unsigned i) const
Return incoming basic block number i.
Value * getIncomingValue(unsigned i) const
Return incoming value number x.
unsigned getNumIncomingValues() const
Return the number of incoming edges.
static PoisonValue * get(Type *T)
Static factory methods - Return an 'poison' object of the specified type.
Definition: Constants.cpp:1827
Return a value (possibly void), from a function.
static SelectInst * Create(Value *C, Value *S1, Value *S2, const Twine &NameStr, BasicBlock::iterator InsertBefore, Instruction *MDFrom=nullptr)
A vector that has set insertion semantics.
Definition: SetVector.h:57
size_type size() const
Determine the number of elements in the SetVector.
Definition: SetVector.h:98
const value_type & front() const
Return the first element of the SetVector.
Definition: SetVector.h:143
const value_type & back() const
Return the last element of the SetVector.
Definition: SetVector.h:149
size_type count(const key_type &key) const
Count the number of elements of a given key in the SetVector.
Definition: SetVector.h:264
iterator begin()
Get an iterator to the beginning of the SetVector.
Definition: SetVector.h:103
Implements a dense probed hash-table based set with some number of buckets stored inline.
Definition: DenseSet.h:290
size_type size() const
Definition: SmallPtrSet.h:94
A templated base class for SmallPtrSet which provides the typesafe interface that is common across al...
Definition: SmallPtrSet.h:321
bool erase(PtrType Ptr)
erase - If the set contains the specified pointer, remove it and return true, otherwise return false.
Definition: SmallPtrSet.h:356
size_type count(ConstPtrType Ptr) const
count - Return 1 if the specified pointer is in the set, 0 otherwise.
Definition: SmallPtrSet.h:360
std::pair< iterator, bool > insert(PtrType Ptr)
Inserts Ptr if and only if there is no element in the container equal to Ptr.
Definition: SmallPtrSet.h:342
iterator begin() const
Definition: SmallPtrSet.h:380
bool contains(ConstPtrType Ptr) const
Definition: SmallPtrSet.h:366
SmallPtrSet - This class implements a set which is optimized for holding SmallSize or less elements.
Definition: SmallPtrSet.h:427
bool empty() const
Definition: SmallVector.h:94
size_t size() const
Definition: SmallVector.h:91
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
Definition: SmallVector.h:586
reference emplace_back(ArgTypes &&... Args)
Definition: SmallVector.h:950
void reserve(size_type N)
Definition: SmallVector.h:676
void push_back(const T &Elt)
Definition: SmallVector.h:426
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
Definition: SmallVector.h:1209
StringRef - Represent a constant reference to a string, i.e.
Definition: StringRef.h:50
Provides information about what library functions are available for the current target.
Twine - A lightweight data structure for efficiently representing the concatenation of temporary valu...
Definition: Twine.h:81
std::string str() const
Return the twine contents as a std::string.
Definition: Twine.cpp:17
The instances of the Type class are immutable: once they are created, they are never changed.
Definition: Type.h:45
static IntegerType * getInt1Ty(LLVMContext &C)
static IntegerType * getInt32Ty(LLVMContext &C)
bool isTokenTy() const
Return true if this is 'token'.
Definition: Type.h:225
static UndefValue * get(Type *T)
Static factory methods - Return an 'undef' object of the specified type.
Definition: Constants.cpp:1808
This function has undefined behavior.
A Use represents the edge between a Value definition and its users.
Definition: Use.h:43
op_range operands()
Definition: User.h:242
void setOperand(unsigned i, Value *Val)
Definition: User.h:174
LLVM Value Representation.
Definition: Value.h:74
Type * getType() const
All values are typed, get the type of this value.
Definition: Value.h:255
void setName(const Twine &Name)
Change the name of the value.
Definition: Value.cpp:377
bool hasOneUse() const
Return true if there is exactly one use of this value.
Definition: Value.h:434
void replaceAllUsesWith(Value *V)
Change all uses of this to point to a new Value.
Definition: Value.cpp:534
bool use_empty() const
Definition: Value.h:344
bool hasName() const
Definition: Value.h:261
StringRef getName() const
Return a constant reference to the value's name.
Definition: Value.cpp:309
void takeName(Value *V)
Transfer the name from V to this value.
Definition: Value.cpp:383
std::pair< iterator, bool > insert(const ValueT &V)
Definition: DenseSet.h:206
bool contains(const_arg_type_t< ValueT > V) const
Check if the set contains the given element.
Definition: DenseSet.h:185
self_iterator getIterator()
Definition: ilist_node.h:109
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
@ Tail
Attemps to make calls as fast as possible while guaranteeing that tail call optimization can always b...
Definition: CallingConv.h:76
@ C
The default llvm calling convention, compatible with C.
Definition: CallingConv.h:34
AssignmentInstRange getAssignmentInsts(DIAssignID *ID)
Return a range of instructions (typically just one) that have ID as an attachment.
Definition: DebugInfo.cpp:1775
initializer< Ty > init(const Ty &Val)
Definition: CommandLine.h:450
This is an optimization pass for GlobalISel generic memory operations.
Definition: AddressRanges.h:18
void ReplaceInstWithInst(BasicBlock *BB, BasicBlock::iterator &BI, Instruction *I)
Replace the instruction specified by BI with the instruction specified by I.
iterator_range< df_ext_iterator< T, SetTy > > depth_first_ext(const T &G, SetTy &S)
Interval::succ_iterator succ_end(Interval *I)
Definition: Interval.h:102
auto find(R &&Range, const T &Val)
Provide wrappers to std::find which take ranges instead of having to pass begin/end explicitly.
Definition: STLExtras.h:1751
bool RemoveRedundantDbgInstrs(BasicBlock *BB)
Try to remove redundant dbg.value instructions from given basic block.
bool succ_empty(const Instruction *I)
Definition: CFG.h:255
bool IsBlockFollowedByDeoptOrUnreachable(const BasicBlock *BB)
Check if we can prove that all paths starting from this block converge to a block that either has a @...
BranchInst * GetIfCondition(BasicBlock *BB, BasicBlock *&IfTrue, BasicBlock *&IfFalse)
Check whether BB is the merge point of a if-region.
unsigned GetSuccessorNumber(const BasicBlock *BB, const BasicBlock *Succ)
Search for the specified successor of basic block BB and return its position in the terminator instru...
Definition: CFG.cpp:79
void detachDeadBlocks(ArrayRef< BasicBlock * > BBs, SmallVectorImpl< DominatorTree::UpdateType > *Updates, bool KeepOneInputPHIs=false)
Replace contents of every block in BBs with single unreachable instruction.
bool hasOnlySimpleTerminator(const Function &F)
auto successors(const MachineBasicBlock *BB)
ReturnInst * FoldReturnIntoUncondBranch(ReturnInst *RI, BasicBlock *BB, BasicBlock *Pred, DomTreeUpdater *DTU=nullptr)
This method duplicates the specified return instruction into a predecessor which ends in an unconditi...
BasicBlock * splitBlockBefore(BasicBlock *Old, BasicBlock::iterator SplitPt, DomTreeUpdater *DTU, LoopInfo *LI, MemorySSAUpdater *MSSAU, const Twine &BBName="")
Split the specified block at the specified instruction SplitPt.
Instruction * SplitBlockAndInsertIfElse(Value *Cond, BasicBlock::iterator SplitBefore, bool Unreachable, MDNode *BranchWeights=nullptr, DomTreeUpdater *DTU=nullptr, LoopInfo *LI=nullptr, BasicBlock *ElseBlock=nullptr)
Similar to SplitBlockAndInsertIfThen, but the inserted block is on the false path of the branch.
Interval::succ_iterator succ_begin(Interval *I)
succ_begin/succ_end - define methods so that Intervals may be used just like BasicBlocks can with the...
Definition: Interval.h:99
void DeleteDeadBlock(BasicBlock *BB, DomTreeUpdater *DTU=nullptr, bool KeepOneInputPHIs=false)
Delete the specified block, which must have no predecessors.
void ReplaceInstWithValue(BasicBlock::iterator &BI, Value *V)
Replace all uses of an instruction (specified by BI) with a value, then remove and delete the origina...
BasicBlock * SplitKnownCriticalEdge(Instruction *TI, unsigned SuccNum, const CriticalEdgeSplittingOptions &Options=CriticalEdgeSplittingOptions(), const Twine &BBName="")
If it is known that an edge is critical, SplitKnownCriticalEdge can be called directly,...
Interval::pred_iterator pred_end(Interval *I)
Definition: Interval.h:112
bool any_of(R &&range, UnaryPredicate P)
Provide wrappers to std::any_of which take ranges instead of having to pass begin/end explicitly.
Definition: STLExtras.h:1738
bool DeleteDeadPHIs(BasicBlock *BB, const TargetLibraryInfo *TLI=nullptr, MemorySSAUpdater *MSSAU=nullptr)
Examine each PHI in the given block and delete it if it is dead.
auto reverse(ContainerTy &&C)
Definition: STLExtras.h:428
void InvertBranch(BranchInst *PBI, IRBuilderBase &Builder)
bool EliminateUnreachableBlocks(Function &F, DomTreeUpdater *DTU=nullptr, bool KeepOneInputPHIs=false)
Delete all basic blocks from F that are not reachable from its entry node.
bool MergeBlockSuccessorsIntoGivenBlocks(SmallPtrSetImpl< BasicBlock * > &MergeBlocks, Loop *L=nullptr, DomTreeUpdater *DTU=nullptr, LoopInfo *LI=nullptr)
Merge block(s) sucessors, if possible.
void SplitBlockAndInsertIfThenElse(Value *Cond, BasicBlock::iterator SplitBefore, Instruction **ThenTerm, Instruction **ElseTerm, MDNode *BranchWeights=nullptr, DomTreeUpdater *DTU=nullptr, LoopInfo *LI=nullptr)
SplitBlockAndInsertIfThenElse is similar to SplitBlockAndInsertIfThen, but also creates the ElseBlock...
void SplitBlockAndInsertForEachLane(ElementCount EC, Type *IndexTy, Instruction *InsertBefore, std::function< void(IRBuilderBase &, Value *)> Func)
Utility function for performing a given action on each lane of a vector with EC elements.
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
Definition: Debug.cpp:163
Interval::pred_iterator pred_begin(Interval *I)
pred_begin/pred_end - define methods so that Intervals may be used just like BasicBlocks can with the...
Definition: Interval.h:109
BasicBlock * ehAwareSplitEdge(BasicBlock *BB, BasicBlock *Succ, LandingPadInst *OriginalPad=nullptr, PHINode *LandingPadReplacement=nullptr, const CriticalEdgeSplittingOptions &Options=CriticalEdgeSplittingOptions(), const Twine &BBName="")
Split the edge connect the specficed blocks in the case that Succ is an Exception Handling Block.
SmallVector< ValueTypeFromRangeType< R >, Size > to_vector(R &&Range)
Given a range of type R, iterate the entire range and return a SmallVector with elements of the vecto...
Definition: SmallVector.h:1312
void SplitLandingPadPredecessors(BasicBlock *OrigBB, ArrayRef< BasicBlock * > Preds, const char *Suffix, const char *Suffix2, SmallVectorImpl< BasicBlock * > &NewBBs, DomTreeUpdater *DTU=nullptr, LoopInfo *LI=nullptr, MemorySSAUpdater *MSSAU=nullptr, bool PreserveLCSSA=false)
This method transforms the landing pad, OrigBB, by introducing two new basic blocks into the function...
BasicBlock * SplitBlockPredecessors(BasicBlock *BB, ArrayRef< BasicBlock * > Preds, const char *Suffix, DominatorTree *DT, LoopInfo *LI=nullptr, MemorySSAUpdater *MSSAU=nullptr, bool PreserveLCSSA=false)
This method introduces at least one new basic block into the function and moves some of the predecess...
bool VerifyMemorySSA
Enables verification of MemorySSA.
Definition: MemorySSA.cpp:83
void createPHIsForSplitLoopExit(ArrayRef< BasicBlock * > Preds, BasicBlock *SplitBB, BasicBlock *DestBB)
When a loop exit edge is split, LCSSA form may require new PHIs in the new exit block.
bool MergeBlockIntoPredecessor(BasicBlock *BB, DomTreeUpdater *DTU=nullptr, LoopInfo *LI=nullptr, MemorySSAUpdater *MSSAU=nullptr, MemoryDependenceResults *MemDep=nullptr, bool PredecessorWithTwoSuccessors=false, DominatorTree *DT=nullptr)
Attempts to merge a block into its predecessor, if possible.
bool isAssignmentTrackingEnabled(const Module &M)
Return true if assignment tracking is enabled for module M.
Definition: DebugInfo.cpp:2322
BasicBlock * CreateControlFlowHub(DomTreeUpdater *DTU, SmallVectorImpl< BasicBlock * > &GuardBlocks, const SetVector< BasicBlock * > &Predecessors, const SetVector< BasicBlock * > &Successors, const StringRef Prefix, std::optional< unsigned > MaxControlFlowBooleans=std::nullopt)
Given a set of incoming and outgoing blocks, create a "hub" such that every edge from an incoming blo...
std::pair< Instruction *, Value * > SplitBlockAndInsertSimpleForLoop(Value *End, Instruction *SplitBefore)
Insert a for (int i = 0; i < End; i++) loop structure (with the exception that End is assumed > 0,...
DWARFExpression::Operation Op
BasicBlock * SplitCriticalEdge(Instruction *TI, unsigned SuccNum, const CriticalEdgeSplittingOptions &Options=CriticalEdgeSplittingOptions(), const Twine &BBName="")
If this edge is a critical edge, insert a new node to split the critical edge.
bool FoldSingleEntryPHINodes(BasicBlock *BB, MemoryDependenceResults *MemDep=nullptr)
We know that BB has one predecessor.
bool isCriticalEdge(const Instruction *TI, unsigned SuccNum, bool AllowIdenticalEdges=false)
Return true if the specified edge is a critical edge.
Definition: CFG.cpp:95
unsigned SplitAllCriticalEdges(Function &F, const CriticalEdgeSplittingOptions &Options=CriticalEdgeSplittingOptions())
Loop over all of the edges in the CFG, breaking critical edges as they are found.
void updatePhiNodes(BasicBlock *DestBB, BasicBlock *OldPred, BasicBlock *NewPred, PHINode *Until=nullptr)
Replaces all uses of OldPred with the NewPred block in all PHINodes in a block.
bool isPresplitCoroSuspendExitEdge(const BasicBlock &Src, const BasicBlock &Dest)
BasicBlock * SplitBlock(BasicBlock *Old, BasicBlock::iterator SplitPt, DominatorTree *DT, LoopInfo *LI=nullptr, MemorySSAUpdater *MSSAU=nullptr, const Twine &BBName="", bool Before=false)
Split the specified block at the specified instruction.
auto predecessors(const MachineBasicBlock *BB)
bool is_contained(R &&Range, const E &Element)
Returns true if Element is found in Range.
Definition: STLExtras.h:1888
bool RecursivelyDeleteDeadPHINode(PHINode *PN, const TargetLibraryInfo *TLI=nullptr, MemorySSAUpdater *MSSAU=nullptr)
If the specified value is an effectively dead PHI node, due to being a def-use chain of single-use no...
Definition: Local.cpp:644
Instruction * SplitBlockAndInsertIfThen(Value *Cond, BasicBlock::iterator SplitBefore, bool Unreachable, MDNode *BranchWeights=nullptr, DomTreeUpdater *DTU=nullptr, LoopInfo *LI=nullptr, BasicBlock *ThenBlock=nullptr)
Split the containing block at the specified instruction - everything before SplitBefore stays in the ...
unsigned succ_size(const MachineBasicBlock *BB)
Value * invertCondition(Value *Condition)
Invert the given true/false value, possibly reusing an existing copy.
Definition: Local.cpp:4104
void DeleteDeadBlocks(ArrayRef< BasicBlock * > BBs, DomTreeUpdater *DTU=nullptr, bool KeepOneInputPHIs=false)
Delete the specified blocks from BB.
BasicBlock * SplitEdge(BasicBlock *From, BasicBlock *To, DominatorTree *DT=nullptr, LoopInfo *LI=nullptr, MemorySSAUpdater *MSSAU=nullptr, const Twine &BBName="")
Split the edge connecting the specified blocks, and return the newly created basic block between From...
void setUnwindEdgeTo(Instruction *TI, BasicBlock *Succ)
Sets the unwind edge of an instruction to a particular successor.
unsigned pred_size(const MachineBasicBlock *BB)
static auto filterDbgVars(iterator_range< simple_ilist< DbgRecord >::iterator > R)
Filter the DbgRecord range to DPValue types only and downcast.
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
Definition: BitVector.h:860
Option class for critical edge splitting.
CriticalEdgeSplittingOptions & setPreserveLCSSA()
Incoming for lane maks phi as machine instruction, incoming register Reg and incoming block Block are...