LLVM 19.0.0git
SimplifyCFG.cpp
Go to the documentation of this file.
1//===- SimplifyCFG.cpp - Code to perform CFG simplification ---------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// Peephole optimize the CFG.
10//
11//===----------------------------------------------------------------------===//
12
13#include "llvm/ADT/APInt.h"
14#include "llvm/ADT/ArrayRef.h"
15#include "llvm/ADT/DenseMap.h"
16#include "llvm/ADT/MapVector.h"
17#include "llvm/ADT/STLExtras.h"
18#include "llvm/ADT/Sequence.h"
20#include "llvm/ADT/SetVector.h"
23#include "llvm/ADT/Statistic.h"
24#include "llvm/ADT/StringRef.h"
35#include "llvm/IR/Attributes.h"
36#include "llvm/IR/BasicBlock.h"
37#include "llvm/IR/CFG.h"
38#include "llvm/IR/Constant.h"
40#include "llvm/IR/Constants.h"
41#include "llvm/IR/DataLayout.h"
42#include "llvm/IR/DebugInfo.h"
44#include "llvm/IR/Function.h"
45#include "llvm/IR/GlobalValue.h"
47#include "llvm/IR/IRBuilder.h"
48#include "llvm/IR/InstrTypes.h"
49#include "llvm/IR/Instruction.h"
52#include "llvm/IR/LLVMContext.h"
53#include "llvm/IR/MDBuilder.h"
55#include "llvm/IR/Metadata.h"
56#include "llvm/IR/Module.h"
57#include "llvm/IR/NoFolder.h"
58#include "llvm/IR/Operator.h"
61#include "llvm/IR/Type.h"
62#include "llvm/IR/Use.h"
63#include "llvm/IR/User.h"
64#include "llvm/IR/Value.h"
65#include "llvm/IR/ValueHandle.h"
69#include "llvm/Support/Debug.h"
77#include <algorithm>
78#include <cassert>
79#include <climits>
80#include <cstddef>
81#include <cstdint>
82#include <iterator>
83#include <map>
84#include <optional>
85#include <set>
86#include <tuple>
87#include <utility>
88#include <vector>
89
90using namespace llvm;
91using namespace PatternMatch;
92
93#define DEBUG_TYPE "simplifycfg"
94
96 "simplifycfg-require-and-preserve-domtree", cl::Hidden,
97
98 cl::desc("Temorary development switch used to gradually uplift SimplifyCFG "
99 "into preserving DomTree,"));
100
101// Chosen as 2 so as to be cheap, but still to have enough power to fold
102// a select, so the "clamp" idiom (of a min followed by a max) will be caught.
103// To catch this, we need to fold a compare and a select, hence '2' being the
104// minimum reasonable default.
106 "phi-node-folding-threshold", cl::Hidden, cl::init(2),
107 cl::desc(
108 "Control the amount of phi node folding to perform (default = 2)"));
109
111 "two-entry-phi-node-folding-threshold", cl::Hidden, cl::init(4),
112 cl::desc("Control the maximal total instruction cost that we are willing "
113 "to speculatively execute to fold a 2-entry PHI node into a "
114 "select (default = 4)"));
115
116static cl::opt<bool>
117 HoistCommon("simplifycfg-hoist-common", cl::Hidden, cl::init(true),
118 cl::desc("Hoist common instructions up to the parent block"));
119
121 HoistCommonSkipLimit("simplifycfg-hoist-common-skip-limit", cl::Hidden,
122 cl::init(20),
123 cl::desc("Allow reordering across at most this many "
124 "instructions when hoisting"));
125
126static cl::opt<bool>
127 SinkCommon("simplifycfg-sink-common", cl::Hidden, cl::init(true),
128 cl::desc("Sink common instructions down to the end block"));
129
131 "simplifycfg-hoist-cond-stores", cl::Hidden, cl::init(true),
132 cl::desc("Hoist conditional stores if an unconditional store precedes"));
133
135 "simplifycfg-merge-cond-stores", cl::Hidden, cl::init(true),
136 cl::desc("Hoist conditional stores even if an unconditional store does not "
137 "precede - hoist multiple conditional stores into a single "
138 "predicated store"));
139
141 "simplifycfg-merge-cond-stores-aggressively", cl::Hidden, cl::init(false),
142 cl::desc("When merging conditional stores, do so even if the resultant "
143 "basic blocks are unlikely to be if-converted as a result"));
144
146 "speculate-one-expensive-inst", cl::Hidden, cl::init(true),
147 cl::desc("Allow exactly one expensive instruction to be speculatively "
148 "executed"));
149
151 "max-speculation-depth", cl::Hidden, cl::init(10),
152 cl::desc("Limit maximum recursion depth when calculating costs of "
153 "speculatively executed instructions"));
154
155static cl::opt<int>
156 MaxSmallBlockSize("simplifycfg-max-small-block-size", cl::Hidden,
157 cl::init(10),
158 cl::desc("Max size of a block which is still considered "
159 "small enough to thread through"));
160
161// Two is chosen to allow one negation and a logical combine.
163 BranchFoldThreshold("simplifycfg-branch-fold-threshold", cl::Hidden,
164 cl::init(2),
165 cl::desc("Maximum cost of combining conditions when "
166 "folding branches"));
167
169 "simplifycfg-branch-fold-common-dest-vector-multiplier", cl::Hidden,
170 cl::init(2),
171 cl::desc("Multiplier to apply to threshold when determining whether or not "
172 "to fold branch to common destination when vector operations are "
173 "present"));
174
176 "simplifycfg-merge-compatible-invokes", cl::Hidden, cl::init(true),
177 cl::desc("Allow SimplifyCFG to merge invokes together when appropriate"));
178
180 "max-switch-cases-per-result", cl::Hidden, cl::init(16),
181 cl::desc("Limit cases to analyze when converting a switch to select"));
182
183STATISTIC(NumBitMaps, "Number of switch instructions turned into bitmaps");
184STATISTIC(NumLinearMaps,
185 "Number of switch instructions turned into linear mapping");
186STATISTIC(NumLookupTables,
187 "Number of switch instructions turned into lookup tables");
189 NumLookupTablesHoles,
190 "Number of switch instructions turned into lookup tables (holes checked)");
191STATISTIC(NumTableCmpReuses, "Number of reused switch table lookup compares");
192STATISTIC(NumFoldValueComparisonIntoPredecessors,
193 "Number of value comparisons folded into predecessor basic blocks");
194STATISTIC(NumFoldBranchToCommonDest,
195 "Number of branches folded into predecessor basic block");
197 NumHoistCommonCode,
198 "Number of common instruction 'blocks' hoisted up to the begin block");
199STATISTIC(NumHoistCommonInstrs,
200 "Number of common instructions hoisted up to the begin block");
201STATISTIC(NumSinkCommonCode,
202 "Number of common instruction 'blocks' sunk down to the end block");
203STATISTIC(NumSinkCommonInstrs,
204 "Number of common instructions sunk down to the end block");
205STATISTIC(NumSpeculations, "Number of speculative executed instructions");
206STATISTIC(NumInvokes,
207 "Number of invokes with empty resume blocks simplified into calls");
208STATISTIC(NumInvokesMerged, "Number of invokes that were merged together");
209STATISTIC(NumInvokeSetsFormed, "Number of invoke sets that were formed");
210
211namespace {
212
213// The first field contains the value that the switch produces when a certain
214// case group is selected, and the second field is a vector containing the
215// cases composing the case group.
216using SwitchCaseResultVectorTy =
218
219// The first field contains the phi node that generates a result of the switch
220// and the second field contains the value generated for a certain case in the
221// switch for that PHI.
222using SwitchCaseResultsTy = SmallVector<std::pair<PHINode *, Constant *>, 4>;
223
224/// ValueEqualityComparisonCase - Represents a case of a switch.
225struct ValueEqualityComparisonCase {
227 BasicBlock *Dest;
228
229 ValueEqualityComparisonCase(ConstantInt *Value, BasicBlock *Dest)
230 : Value(Value), Dest(Dest) {}
231
232 bool operator<(ValueEqualityComparisonCase RHS) const {
233 // Comparing pointers is ok as we only rely on the order for uniquing.
234 return Value < RHS.Value;
235 }
236
237 bool operator==(BasicBlock *RHSDest) const { return Dest == RHSDest; }
238};
239
240class SimplifyCFGOpt {
242 DomTreeUpdater *DTU;
243 const DataLayout &DL;
244 ArrayRef<WeakVH> LoopHeaders;
246 bool Resimplify;
247
248 Value *isValueEqualityComparison(Instruction *TI);
249 BasicBlock *GetValueEqualityComparisonCases(
250 Instruction *TI, std::vector<ValueEqualityComparisonCase> &Cases);
251 bool SimplifyEqualityComparisonWithOnlyPredecessor(Instruction *TI,
252 BasicBlock *Pred,
253 IRBuilder<> &Builder);
254 bool PerformValueComparisonIntoPredecessorFolding(Instruction *TI, Value *&CV,
255 Instruction *PTI,
256 IRBuilder<> &Builder);
257 bool FoldValueComparisonIntoPredecessors(Instruction *TI,
258 IRBuilder<> &Builder);
259
260 bool simplifyResume(ResumeInst *RI, IRBuilder<> &Builder);
261 bool simplifySingleResume(ResumeInst *RI);
262 bool simplifyCommonResume(ResumeInst *RI);
263 bool simplifyCleanupReturn(CleanupReturnInst *RI);
264 bool simplifyUnreachable(UnreachableInst *UI);
265 bool simplifySwitch(SwitchInst *SI, IRBuilder<> &Builder);
266 bool simplifyIndirectBr(IndirectBrInst *IBI);
267 bool simplifyBranch(BranchInst *Branch, IRBuilder<> &Builder);
268 bool simplifyUncondBranch(BranchInst *BI, IRBuilder<> &Builder);
269 bool simplifyCondBranch(BranchInst *BI, IRBuilder<> &Builder);
270
271 bool tryToSimplifyUncondBranchWithICmpInIt(ICmpInst *ICI,
272 IRBuilder<> &Builder);
273
274 bool hoistCommonCodeFromSuccessors(BasicBlock *BB, bool EqTermsOnly);
275 bool hoistSuccIdenticalTerminatorToSwitchOrIf(
276 Instruction *TI, Instruction *I1,
277 SmallVectorImpl<Instruction *> &OtherSuccTIs);
278 bool SpeculativelyExecuteBB(BranchInst *BI, BasicBlock *ThenBB);
279 bool SimplifyTerminatorOnSelect(Instruction *OldTerm, Value *Cond,
280 BasicBlock *TrueBB, BasicBlock *FalseBB,
281 uint32_t TrueWeight, uint32_t FalseWeight);
282 bool SimplifyBranchOnICmpChain(BranchInst *BI, IRBuilder<> &Builder,
283 const DataLayout &DL);
284 bool SimplifySwitchOnSelect(SwitchInst *SI, SelectInst *Select);
285 bool SimplifyIndirectBrOnSelect(IndirectBrInst *IBI, SelectInst *SI);
286 bool TurnSwitchRangeIntoICmp(SwitchInst *SI, IRBuilder<> &Builder);
287
288public:
289 SimplifyCFGOpt(const TargetTransformInfo &TTI, DomTreeUpdater *DTU,
290 const DataLayout &DL, ArrayRef<WeakVH> LoopHeaders,
291 const SimplifyCFGOptions &Opts)
292 : TTI(TTI), DTU(DTU), DL(DL), LoopHeaders(LoopHeaders), Options(Opts) {
293 assert((!DTU || !DTU->hasPostDomTree()) &&
294 "SimplifyCFG is not yet capable of maintaining validity of a "
295 "PostDomTree, so don't ask for it.");
296 }
297
298 bool simplifyOnce(BasicBlock *BB);
299 bool run(BasicBlock *BB);
300
301 // Helper to set Resimplify and return change indication.
302 bool requestResimplify() {
303 Resimplify = true;
304 return true;
305 }
306};
307
308} // end anonymous namespace
309
310/// Return true if all the PHI nodes in the basic block \p BB
311/// receive compatible (identical) incoming values when coming from
312/// all of the predecessor blocks that are specified in \p IncomingBlocks.
313///
314/// Note that if the values aren't exactly identical, but \p EquivalenceSet
315/// is provided, and *both* of the values are present in the set,
316/// then they are considered equal.
318 BasicBlock *BB, ArrayRef<BasicBlock *> IncomingBlocks,
319 SmallPtrSetImpl<Value *> *EquivalenceSet = nullptr) {
320 assert(IncomingBlocks.size() == 2 &&
321 "Only for a pair of incoming blocks at the time!");
322
323 // FIXME: it is okay if one of the incoming values is an `undef` value,
324 // iff the other incoming value is guaranteed to be a non-poison value.
325 // FIXME: it is okay if one of the incoming values is a `poison` value.
326 return all_of(BB->phis(), [IncomingBlocks, EquivalenceSet](PHINode &PN) {
327 Value *IV0 = PN.getIncomingValueForBlock(IncomingBlocks[0]);
328 Value *IV1 = PN.getIncomingValueForBlock(IncomingBlocks[1]);
329 if (IV0 == IV1)
330 return true;
331 if (EquivalenceSet && EquivalenceSet->contains(IV0) &&
332 EquivalenceSet->contains(IV1))
333 return true;
334 return false;
335 });
336}
337
338/// Return true if it is safe to merge these two
339/// terminator instructions together.
340static bool
342 SmallSetVector<BasicBlock *, 4> *FailBlocks = nullptr) {
343 if (SI1 == SI2)
344 return false; // Can't merge with self!
345
346 // It is not safe to merge these two switch instructions if they have a common
347 // successor, and if that successor has a PHI node, and if *that* PHI node has
348 // conflicting incoming values from the two switch blocks.
349 BasicBlock *SI1BB = SI1->getParent();
350 BasicBlock *SI2BB = SI2->getParent();
351
352 SmallPtrSet<BasicBlock *, 16> SI1Succs(succ_begin(SI1BB), succ_end(SI1BB));
353 bool Fail = false;
354 for (BasicBlock *Succ : successors(SI2BB)) {
355 if (!SI1Succs.count(Succ))
356 continue;
357 if (IncomingValuesAreCompatible(Succ, {SI1BB, SI2BB}))
358 continue;
359 Fail = true;
360 if (FailBlocks)
361 FailBlocks->insert(Succ);
362 else
363 break;
364 }
365
366 return !Fail;
367}
368
369/// Update PHI nodes in Succ to indicate that there will now be entries in it
370/// from the 'NewPred' block. The values that will be flowing into the PHI nodes
371/// will be the same as those coming in from ExistPred, an existing predecessor
372/// of Succ.
373static void AddPredecessorToBlock(BasicBlock *Succ, BasicBlock *NewPred,
374 BasicBlock *ExistPred,
375 MemorySSAUpdater *MSSAU = nullptr) {
376 for (PHINode &PN : Succ->phis())
377 PN.addIncoming(PN.getIncomingValueForBlock(ExistPred), NewPred);
378 if (MSSAU)
379 if (auto *MPhi = MSSAU->getMemorySSA()->getMemoryAccess(Succ))
380 MPhi->addIncoming(MPhi->getIncomingValueForBlock(ExistPred), NewPred);
381}
382
383/// Compute an abstract "cost" of speculating the given instruction,
384/// which is assumed to be safe to speculate. TCC_Free means cheap,
385/// TCC_Basic means less cheap, and TCC_Expensive means prohibitively
386/// expensive.
388 const TargetTransformInfo &TTI) {
389 assert((!isa<Instruction>(I) ||
390 isSafeToSpeculativelyExecute(cast<Instruction>(I))) &&
391 "Instruction is not safe to speculatively execute!");
393}
394
395/// If we have a merge point of an "if condition" as accepted above,
396/// return true if the specified value dominates the block. We
397/// don't handle the true generality of domination here, just a special case
398/// which works well enough for us.
399///
400/// If AggressiveInsts is non-null, and if V does not dominate BB, we check to
401/// see if V (which must be an instruction) and its recursive operands
402/// that do not dominate BB have a combined cost lower than Budget and
403/// are non-trapping. If both are true, the instruction is inserted into the
404/// set and true is returned.
405///
406/// The cost for most non-trapping instructions is defined as 1 except for
407/// Select whose cost is 2.
408///
409/// After this function returns, Cost is increased by the cost of
410/// V plus its non-dominating operands. If that cost is greater than
411/// Budget, false is returned and Cost is undefined.
413 SmallPtrSetImpl<Instruction *> &AggressiveInsts,
415 InstructionCost Budget,
417 unsigned Depth = 0) {
418 // It is possible to hit a zero-cost cycle (phi/gep instructions for example),
419 // so limit the recursion depth.
420 // TODO: While this recursion limit does prevent pathological behavior, it
421 // would be better to track visited instructions to avoid cycles.
423 return false;
424
425 Instruction *I = dyn_cast<Instruction>(V);
426 if (!I) {
427 // Non-instructions dominate all instructions and can be executed
428 // unconditionally.
429 return true;
430 }
431 BasicBlock *PBB = I->getParent();
432
433 // We don't want to allow weird loops that might have the "if condition" in
434 // the bottom of this block.
435 if (PBB == BB)
436 return false;
437
438 // If this instruction is defined in a block that contains an unconditional
439 // branch to BB, then it must be in the 'conditional' part of the "if
440 // statement". If not, it definitely dominates the region.
441 BranchInst *BI = dyn_cast<BranchInst>(PBB->getTerminator());
442 if (!BI || BI->isConditional() || BI->getSuccessor(0) != BB)
443 return true;
444
445 // If we have seen this instruction before, don't count it again.
446 if (AggressiveInsts.count(I))
447 return true;
448
449 // Okay, it looks like the instruction IS in the "condition". Check to
450 // see if it's a cheap instruction to unconditionally compute, and if it
451 // only uses stuff defined outside of the condition. If so, hoist it out.
453 return false;
454
456
457 // Allow exactly one instruction to be speculated regardless of its cost
458 // (as long as it is safe to do so).
459 // This is intended to flatten the CFG even if the instruction is a division
460 // or other expensive operation. The speculation of an expensive instruction
461 // is expected to be undone in CodeGenPrepare if the speculation has not
462 // enabled further IR optimizations.
463 if (Cost > Budget &&
464 (!SpeculateOneExpensiveInst || !AggressiveInsts.empty() || Depth > 0 ||
465 !Cost.isValid()))
466 return false;
467
468 // Okay, we can only really hoist these out if their operands do
469 // not take us over the cost threshold.
470 for (Use &Op : I->operands())
471 if (!dominatesMergePoint(Op, BB, AggressiveInsts, Cost, Budget, TTI,
472 Depth + 1))
473 return false;
474 // Okay, it's safe to do this! Remember this instruction.
475 AggressiveInsts.insert(I);
476 return true;
477}
478
479/// Extract ConstantInt from value, looking through IntToPtr
480/// and PointerNullValue. Return NULL if value is not a constant int.
482 // Normal constant int.
483 ConstantInt *CI = dyn_cast<ConstantInt>(V);
484 if (CI || !isa<Constant>(V) || !V->getType()->isPointerTy() ||
485 DL.isNonIntegralPointerType(V->getType()))
486 return CI;
487
488 // This is some kind of pointer constant. Turn it into a pointer-sized
489 // ConstantInt if possible.
490 IntegerType *PtrTy = cast<IntegerType>(DL.getIntPtrType(V->getType()));
491
492 // Null pointer means 0, see SelectionDAGBuilder::getValue(const Value*).
493 if (isa<ConstantPointerNull>(V))
494 return ConstantInt::get(PtrTy, 0);
495
496 // IntToPtr const int.
497 if (ConstantExpr *CE = dyn_cast<ConstantExpr>(V))
498 if (CE->getOpcode() == Instruction::IntToPtr)
499 if (ConstantInt *CI = dyn_cast<ConstantInt>(CE->getOperand(0))) {
500 // The constant is very likely to have the right type already.
501 if (CI->getType() == PtrTy)
502 return CI;
503 else
504 return cast<ConstantInt>(
505 ConstantFoldIntegerCast(CI, PtrTy, /*isSigned=*/false, DL));
506 }
507 return nullptr;
508}
509
510namespace {
511
512/// Given a chain of or (||) or and (&&) comparison of a value against a
513/// constant, this will try to recover the information required for a switch
514/// structure.
515/// It will depth-first traverse the chain of comparison, seeking for patterns
516/// like %a == 12 or %a < 4 and combine them to produce a set of integer
517/// representing the different cases for the switch.
518/// Note that if the chain is composed of '||' it will build the set of elements
519/// that matches the comparisons (i.e. any of this value validate the chain)
520/// while for a chain of '&&' it will build the set elements that make the test
521/// fail.
522struct ConstantComparesGatherer {
523 const DataLayout &DL;
524
525 /// Value found for the switch comparison
526 Value *CompValue = nullptr;
527
528 /// Extra clause to be checked before the switch
529 Value *Extra = nullptr;
530
531 /// Set of integers to match in switch
533
534 /// Number of comparisons matched in the and/or chain
535 unsigned UsedICmps = 0;
536
537 /// Construct and compute the result for the comparison instruction Cond
538 ConstantComparesGatherer(Instruction *Cond, const DataLayout &DL) : DL(DL) {
539 gather(Cond);
540 }
541
542 ConstantComparesGatherer(const ConstantComparesGatherer &) = delete;
543 ConstantComparesGatherer &
544 operator=(const ConstantComparesGatherer &) = delete;
545
546private:
547 /// Try to set the current value used for the comparison, it succeeds only if
548 /// it wasn't set before or if the new value is the same as the old one
549 bool setValueOnce(Value *NewVal) {
550 if (CompValue && CompValue != NewVal)
551 return false;
552 CompValue = NewVal;
553 return (CompValue != nullptr);
554 }
555
556 /// Try to match Instruction "I" as a comparison against a constant and
557 /// populates the array Vals with the set of values that match (or do not
558 /// match depending on isEQ).
559 /// Return false on failure. On success, the Value the comparison matched
560 /// against is placed in CompValue.
561 /// If CompValue is already set, the function is expected to fail if a match
562 /// is found but the value compared to is different.
563 bool matchInstruction(Instruction *I, bool isEQ) {
564 // If this is an icmp against a constant, handle this as one of the cases.
565 ICmpInst *ICI;
566 ConstantInt *C;
567 if (!((ICI = dyn_cast<ICmpInst>(I)) &&
568 (C = GetConstantInt(I->getOperand(1), DL)))) {
569 return false;
570 }
571
572 Value *RHSVal;
573 const APInt *RHSC;
574
575 // Pattern match a special case
576 // (x & ~2^z) == y --> x == y || x == y|2^z
577 // This undoes a transformation done by instcombine to fuse 2 compares.
578 if (ICI->getPredicate() == (isEQ ? ICmpInst::ICMP_EQ : ICmpInst::ICMP_NE)) {
579 // It's a little bit hard to see why the following transformations are
580 // correct. Here is a CVC3 program to verify them for 64-bit values:
581
582 /*
583 ONE : BITVECTOR(64) = BVZEROEXTEND(0bin1, 63);
584 x : BITVECTOR(64);
585 y : BITVECTOR(64);
586 z : BITVECTOR(64);
587 mask : BITVECTOR(64) = BVSHL(ONE, z);
588 QUERY( (y & ~mask = y) =>
589 ((x & ~mask = y) <=> (x = y OR x = (y | mask)))
590 );
591 QUERY( (y | mask = y) =>
592 ((x | mask = y) <=> (x = y OR x = (y & ~mask)))
593 );
594 */
595
596 // Please note that each pattern must be a dual implication (<--> or
597 // iff). One directional implication can create spurious matches. If the
598 // implication is only one-way, an unsatisfiable condition on the left
599 // side can imply a satisfiable condition on the right side. Dual
600 // implication ensures that satisfiable conditions are transformed to
601 // other satisfiable conditions and unsatisfiable conditions are
602 // transformed to other unsatisfiable conditions.
603
604 // Here is a concrete example of a unsatisfiable condition on the left
605 // implying a satisfiable condition on the right:
606 //
607 // mask = (1 << z)
608 // (x & ~mask) == y --> (x == y || x == (y | mask))
609 //
610 // Substituting y = 3, z = 0 yields:
611 // (x & -2) == 3 --> (x == 3 || x == 2)
612
613 // Pattern match a special case:
614 /*
615 QUERY( (y & ~mask = y) =>
616 ((x & ~mask = y) <=> (x = y OR x = (y | mask)))
617 );
618 */
619 if (match(ICI->getOperand(0),
620 m_And(m_Value(RHSVal), m_APInt(RHSC)))) {
621 APInt Mask = ~*RHSC;
622 if (Mask.isPowerOf2() && (C->getValue() & ~Mask) == C->getValue()) {
623 // If we already have a value for the switch, it has to match!
624 if (!setValueOnce(RHSVal))
625 return false;
626
627 Vals.push_back(C);
628 Vals.push_back(
629 ConstantInt::get(C->getContext(),
630 C->getValue() | Mask));
631 UsedICmps++;
632 return true;
633 }
634 }
635
636 // Pattern match a special case:
637 /*
638 QUERY( (y | mask = y) =>
639 ((x | mask = y) <=> (x = y OR x = (y & ~mask)))
640 );
641 */
642 if (match(ICI->getOperand(0),
643 m_Or(m_Value(RHSVal), m_APInt(RHSC)))) {
644 APInt Mask = *RHSC;
645 if (Mask.isPowerOf2() && (C->getValue() | Mask) == C->getValue()) {
646 // If we already have a value for the switch, it has to match!
647 if (!setValueOnce(RHSVal))
648 return false;
649
650 Vals.push_back(C);
651 Vals.push_back(ConstantInt::get(C->getContext(),
652 C->getValue() & ~Mask));
653 UsedICmps++;
654 return true;
655 }
656 }
657
658 // If we already have a value for the switch, it has to match!
659 if (!setValueOnce(ICI->getOperand(0)))
660 return false;
661
662 UsedICmps++;
663 Vals.push_back(C);
664 return ICI->getOperand(0);
665 }
666
667 // If we have "x ult 3", for example, then we can add 0,1,2 to the set.
668 ConstantRange Span =
670
671 // Shift the range if the compare is fed by an add. This is the range
672 // compare idiom as emitted by instcombine.
673 Value *CandidateVal = I->getOperand(0);
674 if (match(I->getOperand(0), m_Add(m_Value(RHSVal), m_APInt(RHSC)))) {
675 Span = Span.subtract(*RHSC);
676 CandidateVal = RHSVal;
677 }
678
679 // If this is an and/!= check, then we are looking to build the set of
680 // value that *don't* pass the and chain. I.e. to turn "x ugt 2" into
681 // x != 0 && x != 1.
682 if (!isEQ)
683 Span = Span.inverse();
684
685 // If there are a ton of values, we don't want to make a ginormous switch.
686 if (Span.isSizeLargerThan(8) || Span.isEmptySet()) {
687 return false;
688 }
689
690 // If we already have a value for the switch, it has to match!
691 if (!setValueOnce(CandidateVal))
692 return false;
693
694 // Add all values from the range to the set
695 for (APInt Tmp = Span.getLower(); Tmp != Span.getUpper(); ++Tmp)
696 Vals.push_back(ConstantInt::get(I->getContext(), Tmp));
697
698 UsedICmps++;
699 return true;
700 }
701
702 /// Given a potentially 'or'd or 'and'd together collection of icmp
703 /// eq/ne/lt/gt instructions that compare a value against a constant, extract
704 /// the value being compared, and stick the list constants into the Vals
705 /// vector.
706 /// One "Extra" case is allowed to differ from the other.
707 void gather(Value *V) {
708 bool isEQ = match(V, m_LogicalOr(m_Value(), m_Value()));
709
710 // Keep a stack (SmallVector for efficiency) for depth-first traversal
713
714 // Initialize
715 Visited.insert(V);
716 DFT.push_back(V);
717
718 while (!DFT.empty()) {
719 V = DFT.pop_back_val();
720
721 if (Instruction *I = dyn_cast<Instruction>(V)) {
722 // If it is a || (or && depending on isEQ), process the operands.
723 Value *Op0, *Op1;
724 if (isEQ ? match(I, m_LogicalOr(m_Value(Op0), m_Value(Op1)))
725 : match(I, m_LogicalAnd(m_Value(Op0), m_Value(Op1)))) {
726 if (Visited.insert(Op1).second)
727 DFT.push_back(Op1);
728 if (Visited.insert(Op0).second)
729 DFT.push_back(Op0);
730
731 continue;
732 }
733
734 // Try to match the current instruction
735 if (matchInstruction(I, isEQ))
736 // Match succeed, continue the loop
737 continue;
738 }
739
740 // One element of the sequence of || (or &&) could not be match as a
741 // comparison against the same value as the others.
742 // We allow only one "Extra" case to be checked before the switch
743 if (!Extra) {
744 Extra = V;
745 continue;
746 }
747 // Failed to parse a proper sequence, abort now
748 CompValue = nullptr;
749 break;
750 }
751 }
752};
753
754} // end anonymous namespace
755
757 MemorySSAUpdater *MSSAU = nullptr) {
758 Instruction *Cond = nullptr;
759 if (SwitchInst *SI = dyn_cast<SwitchInst>(TI)) {
760 Cond = dyn_cast<Instruction>(SI->getCondition());
761 } else if (BranchInst *BI = dyn_cast<BranchInst>(TI)) {
762 if (BI->isConditional())
763 Cond = dyn_cast<Instruction>(BI->getCondition());
764 } else if (IndirectBrInst *IBI = dyn_cast<IndirectBrInst>(TI)) {
765 Cond = dyn_cast<Instruction>(IBI->getAddress());
766 }
767
768 TI->eraseFromParent();
769 if (Cond)
771}
772
773/// Return true if the specified terminator checks
774/// to see if a value is equal to constant integer value.
775Value *SimplifyCFGOpt::isValueEqualityComparison(Instruction *TI) {
776 Value *CV = nullptr;
777 if (SwitchInst *SI = dyn_cast<SwitchInst>(TI)) {
778 // Do not permit merging of large switch instructions into their
779 // predecessors unless there is only one predecessor.
780 if (!SI->getParent()->hasNPredecessorsOrMore(128 / SI->getNumSuccessors()))
781 CV = SI->getCondition();
782 } else if (BranchInst *BI = dyn_cast<BranchInst>(TI))
783 if (BI->isConditional() && BI->getCondition()->hasOneUse())
784 if (ICmpInst *ICI = dyn_cast<ICmpInst>(BI->getCondition())) {
785 if (ICI->isEquality() && GetConstantInt(ICI->getOperand(1), DL))
786 CV = ICI->getOperand(0);
787 }
788
789 // Unwrap any lossless ptrtoint cast.
790 if (CV) {
791 if (PtrToIntInst *PTII = dyn_cast<PtrToIntInst>(CV)) {
792 Value *Ptr = PTII->getPointerOperand();
793 if (PTII->getType() == DL.getIntPtrType(Ptr->getType()))
794 CV = Ptr;
795 }
796 }
797 return CV;
798}
799
800/// Given a value comparison instruction,
801/// decode all of the 'cases' that it represents and return the 'default' block.
802BasicBlock *SimplifyCFGOpt::GetValueEqualityComparisonCases(
803 Instruction *TI, std::vector<ValueEqualityComparisonCase> &Cases) {
804 if (SwitchInst *SI = dyn_cast<SwitchInst>(TI)) {
805 Cases.reserve(SI->getNumCases());
806 for (auto Case : SI->cases())
807 Cases.push_back(ValueEqualityComparisonCase(Case.getCaseValue(),
808 Case.getCaseSuccessor()));
809 return SI->getDefaultDest();
810 }
811
812 BranchInst *BI = cast<BranchInst>(TI);
813 ICmpInst *ICI = cast<ICmpInst>(BI->getCondition());
814 BasicBlock *Succ = BI->getSuccessor(ICI->getPredicate() == ICmpInst::ICMP_NE);
815 Cases.push_back(ValueEqualityComparisonCase(
816 GetConstantInt(ICI->getOperand(1), DL), Succ));
817 return BI->getSuccessor(ICI->getPredicate() == ICmpInst::ICMP_EQ);
818}
819
820/// Given a vector of bb/value pairs, remove any entries
821/// in the list that match the specified block.
822static void
824 std::vector<ValueEqualityComparisonCase> &Cases) {
825 llvm::erase(Cases, BB);
826}
827
828/// Return true if there are any keys in C1 that exist in C2 as well.
829static bool ValuesOverlap(std::vector<ValueEqualityComparisonCase> &C1,
830 std::vector<ValueEqualityComparisonCase> &C2) {
831 std::vector<ValueEqualityComparisonCase> *V1 = &C1, *V2 = &C2;
832
833 // Make V1 be smaller than V2.
834 if (V1->size() > V2->size())
835 std::swap(V1, V2);
836
837 if (V1->empty())
838 return false;
839 if (V1->size() == 1) {
840 // Just scan V2.
841 ConstantInt *TheVal = (*V1)[0].Value;
842 for (const ValueEqualityComparisonCase &VECC : *V2)
843 if (TheVal == VECC.Value)
844 return true;
845 }
846
847 // Otherwise, just sort both lists and compare element by element.
848 array_pod_sort(V1->begin(), V1->end());
849 array_pod_sort(V2->begin(), V2->end());
850 unsigned i1 = 0, i2 = 0, e1 = V1->size(), e2 = V2->size();
851 while (i1 != e1 && i2 != e2) {
852 if ((*V1)[i1].Value == (*V2)[i2].Value)
853 return true;
854 if ((*V1)[i1].Value < (*V2)[i2].Value)
855 ++i1;
856 else
857 ++i2;
858 }
859 return false;
860}
861
862// Set branch weights on SwitchInst. This sets the metadata if there is at
863// least one non-zero weight.
865 bool IsExpected) {
866 // Check that there is at least one non-zero weight. Otherwise, pass
867 // nullptr to setMetadata which will erase the existing metadata.
868 MDNode *N = nullptr;
869 if (llvm::any_of(Weights, [](uint32_t W) { return W != 0; }))
870 N = MDBuilder(SI->getParent()->getContext())
871 .createBranchWeights(Weights, IsExpected);
872 SI->setMetadata(LLVMContext::MD_prof, N);
873}
874
875// Similar to the above, but for branch and select instructions that take
876// exactly 2 weights.
877static void setBranchWeights(Instruction *I, uint32_t TrueWeight,
878 uint32_t FalseWeight, bool IsExpected) {
879 assert(isa<BranchInst>(I) || isa<SelectInst>(I));
880 // Check that there is at least one non-zero weight. Otherwise, pass
881 // nullptr to setMetadata which will erase the existing metadata.
882 MDNode *N = nullptr;
883 if (TrueWeight || FalseWeight)
884 N = MDBuilder(I->getParent()->getContext())
885 .createBranchWeights(TrueWeight, FalseWeight, IsExpected);
886 I->setMetadata(LLVMContext::MD_prof, N);
887}
888
889/// If TI is known to be a terminator instruction and its block is known to
890/// only have a single predecessor block, check to see if that predecessor is
891/// also a value comparison with the same value, and if that comparison
892/// determines the outcome of this comparison. If so, simplify TI. This does a
893/// very limited form of jump threading.
894bool SimplifyCFGOpt::SimplifyEqualityComparisonWithOnlyPredecessor(
895 Instruction *TI, BasicBlock *Pred, IRBuilder<> &Builder) {
896 Value *PredVal = isValueEqualityComparison(Pred->getTerminator());
897 if (!PredVal)
898 return false; // Not a value comparison in predecessor.
899
900 Value *ThisVal = isValueEqualityComparison(TI);
901 assert(ThisVal && "This isn't a value comparison!!");
902 if (ThisVal != PredVal)
903 return false; // Different predicates.
904
905 // TODO: Preserve branch weight metadata, similarly to how
906 // FoldValueComparisonIntoPredecessors preserves it.
907
908 // Find out information about when control will move from Pred to TI's block.
909 std::vector<ValueEqualityComparisonCase> PredCases;
910 BasicBlock *PredDef =
911 GetValueEqualityComparisonCases(Pred->getTerminator(), PredCases);
912 EliminateBlockCases(PredDef, PredCases); // Remove default from cases.
913
914 // Find information about how control leaves this block.
915 std::vector<ValueEqualityComparisonCase> ThisCases;
916 BasicBlock *ThisDef = GetValueEqualityComparisonCases(TI, ThisCases);
917 EliminateBlockCases(ThisDef, ThisCases); // Remove default from cases.
918
919 // If TI's block is the default block from Pred's comparison, potentially
920 // simplify TI based on this knowledge.
921 if (PredDef == TI->getParent()) {
922 // If we are here, we know that the value is none of those cases listed in
923 // PredCases. If there are any cases in ThisCases that are in PredCases, we
924 // can simplify TI.
925 if (!ValuesOverlap(PredCases, ThisCases))
926 return false;
927
928 if (isa<BranchInst>(TI)) {
929 // Okay, one of the successors of this condbr is dead. Convert it to a
930 // uncond br.
931 assert(ThisCases.size() == 1 && "Branch can only have one case!");
932 // Insert the new branch.
933 Instruction *NI = Builder.CreateBr(ThisDef);
934 (void)NI;
935
936 // Remove PHI node entries for the dead edge.
937 ThisCases[0].Dest->removePredecessor(PredDef);
938
939 LLVM_DEBUG(dbgs() << "Threading pred instr: " << *Pred->getTerminator()
940 << "Through successor TI: " << *TI << "Leaving: " << *NI
941 << "\n");
942
944
945 if (DTU)
946 DTU->applyUpdates(
947 {{DominatorTree::Delete, PredDef, ThisCases[0].Dest}});
948
949 return true;
950 }
951
952 SwitchInstProfUpdateWrapper SI = *cast<SwitchInst>(TI);
953 // Okay, TI has cases that are statically dead, prune them away.
955 for (unsigned i = 0, e = PredCases.size(); i != e; ++i)
956 DeadCases.insert(PredCases[i].Value);
957
958 LLVM_DEBUG(dbgs() << "Threading pred instr: " << *Pred->getTerminator()
959 << "Through successor TI: " << *TI);
960
961 SmallDenseMap<BasicBlock *, int, 8> NumPerSuccessorCases;
962 for (SwitchInst::CaseIt i = SI->case_end(), e = SI->case_begin(); i != e;) {
963 --i;
964 auto *Successor = i->getCaseSuccessor();
965 if (DTU)
966 ++NumPerSuccessorCases[Successor];
967 if (DeadCases.count(i->getCaseValue())) {
968 Successor->removePredecessor(PredDef);
969 SI.removeCase(i);
970 if (DTU)
971 --NumPerSuccessorCases[Successor];
972 }
973 }
974
975 if (DTU) {
976 std::vector<DominatorTree::UpdateType> Updates;
977 for (const std::pair<BasicBlock *, int> &I : NumPerSuccessorCases)
978 if (I.second == 0)
979 Updates.push_back({DominatorTree::Delete, PredDef, I.first});
980 DTU->applyUpdates(Updates);
981 }
982
983 LLVM_DEBUG(dbgs() << "Leaving: " << *TI << "\n");
984 return true;
985 }
986
987 // Otherwise, TI's block must correspond to some matched value. Find out
988 // which value (or set of values) this is.
989 ConstantInt *TIV = nullptr;
990 BasicBlock *TIBB = TI->getParent();
991 for (unsigned i = 0, e = PredCases.size(); i != e; ++i)
992 if (PredCases[i].Dest == TIBB) {
993 if (TIV)
994 return false; // Cannot handle multiple values coming to this block.
995 TIV = PredCases[i].Value;
996 }
997 assert(TIV && "No edge from pred to succ?");
998
999 // Okay, we found the one constant that our value can be if we get into TI's
1000 // BB. Find out which successor will unconditionally be branched to.
1001 BasicBlock *TheRealDest = nullptr;
1002 for (unsigned i = 0, e = ThisCases.size(); i != e; ++i)
1003 if (ThisCases[i].Value == TIV) {
1004 TheRealDest = ThisCases[i].Dest;
1005 break;
1006 }
1007
1008 // If not handled by any explicit cases, it is handled by the default case.
1009 if (!TheRealDest)
1010 TheRealDest = ThisDef;
1011
1012 SmallPtrSet<BasicBlock *, 2> RemovedSuccs;
1013
1014 // Remove PHI node entries for dead edges.
1015 BasicBlock *CheckEdge = TheRealDest;
1016 for (BasicBlock *Succ : successors(TIBB))
1017 if (Succ != CheckEdge) {
1018 if (Succ != TheRealDest)
1019 RemovedSuccs.insert(Succ);
1020 Succ->removePredecessor(TIBB);
1021 } else
1022 CheckEdge = nullptr;
1023
1024 // Insert the new branch.
1025 Instruction *NI = Builder.CreateBr(TheRealDest);
1026 (void)NI;
1027
1028 LLVM_DEBUG(dbgs() << "Threading pred instr: " << *Pred->getTerminator()
1029 << "Through successor TI: " << *TI << "Leaving: " << *NI
1030 << "\n");
1031
1033 if (DTU) {
1035 Updates.reserve(RemovedSuccs.size());
1036 for (auto *RemovedSucc : RemovedSuccs)
1037 Updates.push_back({DominatorTree::Delete, TIBB, RemovedSucc});
1038 DTU->applyUpdates(Updates);
1039 }
1040 return true;
1041}
1042
1043namespace {
1044
1045/// This class implements a stable ordering of constant
1046/// integers that does not depend on their address. This is important for
1047/// applications that sort ConstantInt's to ensure uniqueness.
1048struct ConstantIntOrdering {
1049 bool operator()(const ConstantInt *LHS, const ConstantInt *RHS) const {
1050 return LHS->getValue().ult(RHS->getValue());
1051 }
1052};
1053
1054} // end anonymous namespace
1055
1057 ConstantInt *const *P2) {
1058 const ConstantInt *LHS = *P1;
1059 const ConstantInt *RHS = *P2;
1060 if (LHS == RHS)
1061 return 0;
1062 return LHS->getValue().ult(RHS->getValue()) ? 1 : -1;
1063}
1064
1065/// Get Weights of a given terminator, the default weight is at the front
1066/// of the vector. If TI is a conditional eq, we need to swap the branch-weight
1067/// metadata.
1069 SmallVectorImpl<uint64_t> &Weights) {
1070 MDNode *MD = TI->getMetadata(LLVMContext::MD_prof);
1071 assert(MD && "Invalid branch-weight metadata");
1072 extractFromBranchWeightMD64(MD, Weights);
1073
1074 // If TI is a conditional eq, the default case is the false case,
1075 // and the corresponding branch-weight data is at index 2. We swap the
1076 // default weight to be the first entry.
1077 if (BranchInst *BI = dyn_cast<BranchInst>(TI)) {
1078 assert(Weights.size() == 2);
1079 ICmpInst *ICI = cast<ICmpInst>(BI->getCondition());
1080 if (ICI->getPredicate() == ICmpInst::ICMP_EQ)
1081 std::swap(Weights.front(), Weights.back());
1082 }
1083}
1084
1085/// Keep halving the weights until all can fit in uint32_t.
1087 uint64_t Max = *llvm::max_element(Weights);
1088 if (Max > UINT_MAX) {
1089 unsigned Offset = 32 - llvm::countl_zero(Max);
1090 for (uint64_t &I : Weights)
1091 I >>= Offset;
1092 }
1093}
1094
1096 BasicBlock *BB, BasicBlock *PredBlock, ValueToValueMapTy &VMap) {
1097 Instruction *PTI = PredBlock->getTerminator();
1098
1099 // If we have bonus instructions, clone them into the predecessor block.
1100 // Note that there may be multiple predecessor blocks, so we cannot move
1101 // bonus instructions to a predecessor block.
1102 for (Instruction &BonusInst : *BB) {
1103 if (BonusInst.isTerminator())
1104 continue;
1105
1106 Instruction *NewBonusInst = BonusInst.clone();
1107
1108 if (!isa<DbgInfoIntrinsic>(BonusInst) &&
1109 PTI->getDebugLoc() != NewBonusInst->getDebugLoc()) {
1110 // Unless the instruction has the same !dbg location as the original
1111 // branch, drop it. When we fold the bonus instructions we want to make
1112 // sure we reset their debug locations in order to avoid stepping on
1113 // dead code caused by folding dead branches.
1114 NewBonusInst->setDebugLoc(DebugLoc());
1115 }
1116
1117 RemapInstruction(NewBonusInst, VMap,
1119
1120 // If we speculated an instruction, we need to drop any metadata that may
1121 // result in undefined behavior, as the metadata might have been valid
1122 // only given the branch precondition.
1123 // Similarly strip attributes on call parameters that may cause UB in
1124 // location the call is moved to.
1125 NewBonusInst->dropUBImplyingAttrsAndMetadata();
1126
1127 NewBonusInst->insertInto(PredBlock, PTI->getIterator());
1128 auto Range = NewBonusInst->cloneDebugInfoFrom(&BonusInst);
1129 RemapDbgRecordRange(NewBonusInst->getModule(), Range, VMap,
1131
1132 if (isa<DbgInfoIntrinsic>(BonusInst))
1133 continue;
1134
1135 NewBonusInst->takeName(&BonusInst);
1136 BonusInst.setName(NewBonusInst->getName() + ".old");
1137 VMap[&BonusInst] = NewBonusInst;
1138
1139 // Update (liveout) uses of bonus instructions,
1140 // now that the bonus instruction has been cloned into predecessor.
1141 // Note that we expect to be in a block-closed SSA form for this to work!
1142 for (Use &U : make_early_inc_range(BonusInst.uses())) {
1143 auto *UI = cast<Instruction>(U.getUser());
1144 auto *PN = dyn_cast<PHINode>(UI);
1145 if (!PN) {
1146 assert(UI->getParent() == BB && BonusInst.comesBefore(UI) &&
1147 "If the user is not a PHI node, then it should be in the same "
1148 "block as, and come after, the original bonus instruction.");
1149 continue; // Keep using the original bonus instruction.
1150 }
1151 // Is this the block-closed SSA form PHI node?
1152 if (PN->getIncomingBlock(U) == BB)
1153 continue; // Great, keep using the original bonus instruction.
1154 // The only other alternative is an "use" when coming from
1155 // the predecessor block - here we should refer to the cloned bonus instr.
1156 assert(PN->getIncomingBlock(U) == PredBlock &&
1157 "Not in block-closed SSA form?");
1158 U.set(NewBonusInst);
1159 }
1160 }
1161}
1162
1163bool SimplifyCFGOpt::PerformValueComparisonIntoPredecessorFolding(
1164 Instruction *TI, Value *&CV, Instruction *PTI, IRBuilder<> &Builder) {
1165 BasicBlock *BB = TI->getParent();
1166 BasicBlock *Pred = PTI->getParent();
1167
1169
1170 // Figure out which 'cases' to copy from SI to PSI.
1171 std::vector<ValueEqualityComparisonCase> BBCases;
1172 BasicBlock *BBDefault = GetValueEqualityComparisonCases(TI, BBCases);
1173
1174 std::vector<ValueEqualityComparisonCase> PredCases;
1175 BasicBlock *PredDefault = GetValueEqualityComparisonCases(PTI, PredCases);
1176
1177 // Based on whether the default edge from PTI goes to BB or not, fill in
1178 // PredCases and PredDefault with the new switch cases we would like to
1179 // build.
1181
1182 // Update the branch weight metadata along the way
1184 bool PredHasWeights = hasBranchWeightMD(*PTI);
1185 bool SuccHasWeights = hasBranchWeightMD(*TI);
1186
1187 if (PredHasWeights) {
1188 GetBranchWeights(PTI, Weights);
1189 // branch-weight metadata is inconsistent here.
1190 if (Weights.size() != 1 + PredCases.size())
1191 PredHasWeights = SuccHasWeights = false;
1192 } else if (SuccHasWeights)
1193 // If there are no predecessor weights but there are successor weights,
1194 // populate Weights with 1, which will later be scaled to the sum of
1195 // successor's weights
1196 Weights.assign(1 + PredCases.size(), 1);
1197
1198 SmallVector<uint64_t, 8> SuccWeights;
1199 if (SuccHasWeights) {
1200 GetBranchWeights(TI, SuccWeights);
1201 // branch-weight metadata is inconsistent here.
1202 if (SuccWeights.size() != 1 + BBCases.size())
1203 PredHasWeights = SuccHasWeights = false;
1204 } else if (PredHasWeights)
1205 SuccWeights.assign(1 + BBCases.size(), 1);
1206
1207 if (PredDefault == BB) {
1208 // If this is the default destination from PTI, only the edges in TI
1209 // that don't occur in PTI, or that branch to BB will be activated.
1210 std::set<ConstantInt *, ConstantIntOrdering> PTIHandled;
1211 for (unsigned i = 0, e = PredCases.size(); i != e; ++i)
1212 if (PredCases[i].Dest != BB)
1213 PTIHandled.insert(PredCases[i].Value);
1214 else {
1215 // The default destination is BB, we don't need explicit targets.
1216 std::swap(PredCases[i], PredCases.back());
1217
1218 if (PredHasWeights || SuccHasWeights) {
1219 // Increase weight for the default case.
1220 Weights[0] += Weights[i + 1];
1221 std::swap(Weights[i + 1], Weights.back());
1222 Weights.pop_back();
1223 }
1224
1225 PredCases.pop_back();
1226 --i;
1227 --e;
1228 }
1229
1230 // Reconstruct the new switch statement we will be building.
1231 if (PredDefault != BBDefault) {
1232 PredDefault->removePredecessor(Pred);
1233 if (DTU && PredDefault != BB)
1234 Updates.push_back({DominatorTree::Delete, Pred, PredDefault});
1235 PredDefault = BBDefault;
1236 ++NewSuccessors[BBDefault];
1237 }
1238
1239 unsigned CasesFromPred = Weights.size();
1240 uint64_t ValidTotalSuccWeight = 0;
1241 for (unsigned i = 0, e = BBCases.size(); i != e; ++i)
1242 if (!PTIHandled.count(BBCases[i].Value) && BBCases[i].Dest != BBDefault) {
1243 PredCases.push_back(BBCases[i]);
1244 ++NewSuccessors[BBCases[i].Dest];
1245 if (SuccHasWeights || PredHasWeights) {
1246 // The default weight is at index 0, so weight for the ith case
1247 // should be at index i+1. Scale the cases from successor by
1248 // PredDefaultWeight (Weights[0]).
1249 Weights.push_back(Weights[0] * SuccWeights[i + 1]);
1250 ValidTotalSuccWeight += SuccWeights[i + 1];
1251 }
1252 }
1253
1254 if (SuccHasWeights || PredHasWeights) {
1255 ValidTotalSuccWeight += SuccWeights[0];
1256 // Scale the cases from predecessor by ValidTotalSuccWeight.
1257 for (unsigned i = 1; i < CasesFromPred; ++i)
1258 Weights[i] *= ValidTotalSuccWeight;
1259 // Scale the default weight by SuccDefaultWeight (SuccWeights[0]).
1260 Weights[0] *= SuccWeights[0];
1261 }
1262 } else {
1263 // If this is not the default destination from PSI, only the edges
1264 // in SI that occur in PSI with a destination of BB will be
1265 // activated.
1266 std::set<ConstantInt *, ConstantIntOrdering> PTIHandled;
1267 std::map<ConstantInt *, uint64_t> WeightsForHandled;
1268 for (unsigned i = 0, e = PredCases.size(); i != e; ++i)
1269 if (PredCases[i].Dest == BB) {
1270 PTIHandled.insert(PredCases[i].Value);
1271
1272 if (PredHasWeights || SuccHasWeights) {
1273 WeightsForHandled[PredCases[i].Value] = Weights[i + 1];
1274 std::swap(Weights[i + 1], Weights.back());
1275 Weights.pop_back();
1276 }
1277
1278 std::swap(PredCases[i], PredCases.back());
1279 PredCases.pop_back();
1280 --i;
1281 --e;
1282 }
1283
1284 // Okay, now we know which constants were sent to BB from the
1285 // predecessor. Figure out where they will all go now.
1286 for (unsigned i = 0, e = BBCases.size(); i != e; ++i)
1287 if (PTIHandled.count(BBCases[i].Value)) {
1288 // If this is one we are capable of getting...
1289 if (PredHasWeights || SuccHasWeights)
1290 Weights.push_back(WeightsForHandled[BBCases[i].Value]);
1291 PredCases.push_back(BBCases[i]);
1292 ++NewSuccessors[BBCases[i].Dest];
1293 PTIHandled.erase(BBCases[i].Value); // This constant is taken care of
1294 }
1295
1296 // If there are any constants vectored to BB that TI doesn't handle,
1297 // they must go to the default destination of TI.
1298 for (ConstantInt *I : PTIHandled) {
1299 if (PredHasWeights || SuccHasWeights)
1300 Weights.push_back(WeightsForHandled[I]);
1301 PredCases.push_back(ValueEqualityComparisonCase(I, BBDefault));
1302 ++NewSuccessors[BBDefault];
1303 }
1304 }
1305
1306 // Okay, at this point, we know which new successor Pred will get. Make
1307 // sure we update the number of entries in the PHI nodes for these
1308 // successors.
1309 SmallPtrSet<BasicBlock *, 2> SuccsOfPred;
1310 if (DTU) {
1311 SuccsOfPred = {succ_begin(Pred), succ_end(Pred)};
1312 Updates.reserve(Updates.size() + NewSuccessors.size());
1313 }
1314 for (const std::pair<BasicBlock *, int /*Num*/> &NewSuccessor :
1315 NewSuccessors) {
1316 for (auto I : seq(NewSuccessor.second)) {
1317 (void)I;
1318 AddPredecessorToBlock(NewSuccessor.first, Pred, BB);
1319 }
1320 if (DTU && !SuccsOfPred.contains(NewSuccessor.first))
1321 Updates.push_back({DominatorTree::Insert, Pred, NewSuccessor.first});
1322 }
1323
1324 Builder.SetInsertPoint(PTI);
1325 // Convert pointer to int before we switch.
1326 if (CV->getType()->isPointerTy()) {
1327 CV =
1328 Builder.CreatePtrToInt(CV, DL.getIntPtrType(CV->getType()), "magicptr");
1329 }
1330
1331 // Now that the successors are updated, create the new Switch instruction.
1332 SwitchInst *NewSI = Builder.CreateSwitch(CV, PredDefault, PredCases.size());
1333 NewSI->setDebugLoc(PTI->getDebugLoc());
1334 for (ValueEqualityComparisonCase &V : PredCases)
1335 NewSI->addCase(V.Value, V.Dest);
1336
1337 if (PredHasWeights || SuccHasWeights) {
1338 // Halve the weights if any of them cannot fit in an uint32_t
1339 FitWeights(Weights);
1340
1341 SmallVector<uint32_t, 8> MDWeights(Weights.begin(), Weights.end());
1342
1343 setBranchWeights(NewSI, MDWeights, /*IsExpected=*/false);
1344 }
1345
1347
1348 // Okay, last check. If BB is still a successor of PSI, then we must
1349 // have an infinite loop case. If so, add an infinitely looping block
1350 // to handle the case to preserve the behavior of the code.
1351 BasicBlock *InfLoopBlock = nullptr;
1352 for (unsigned i = 0, e = NewSI->getNumSuccessors(); i != e; ++i)
1353 if (NewSI->getSuccessor(i) == BB) {
1354 if (!InfLoopBlock) {
1355 // Insert it at the end of the function, because it's either code,
1356 // or it won't matter if it's hot. :)
1357 InfLoopBlock =
1358 BasicBlock::Create(BB->getContext(), "infloop", BB->getParent());
1359 BranchInst::Create(InfLoopBlock, InfLoopBlock);
1360 if (DTU)
1361 Updates.push_back(
1362 {DominatorTree::Insert, InfLoopBlock, InfLoopBlock});
1363 }
1364 NewSI->setSuccessor(i, InfLoopBlock);
1365 }
1366
1367 if (DTU) {
1368 if (InfLoopBlock)
1369 Updates.push_back({DominatorTree::Insert, Pred, InfLoopBlock});
1370
1371 Updates.push_back({DominatorTree::Delete, Pred, BB});
1372
1373 DTU->applyUpdates(Updates);
1374 }
1375
1376 ++NumFoldValueComparisonIntoPredecessors;
1377 return true;
1378}
1379
1380/// The specified terminator is a value equality comparison instruction
1381/// (either a switch or a branch on "X == c").
1382/// See if any of the predecessors of the terminator block are value comparisons
1383/// on the same value. If so, and if safe to do so, fold them together.
1384bool SimplifyCFGOpt::FoldValueComparisonIntoPredecessors(Instruction *TI,
1385 IRBuilder<> &Builder) {
1386 BasicBlock *BB = TI->getParent();
1387 Value *CV = isValueEqualityComparison(TI); // CondVal
1388 assert(CV && "Not a comparison?");
1389
1390 bool Changed = false;
1391
1393 while (!Preds.empty()) {
1394 BasicBlock *Pred = Preds.pop_back_val();
1395 Instruction *PTI = Pred->getTerminator();
1396
1397 // Don't try to fold into itself.
1398 if (Pred == BB)
1399 continue;
1400
1401 // See if the predecessor is a comparison with the same value.
1402 Value *PCV = isValueEqualityComparison(PTI); // PredCondVal
1403 if (PCV != CV)
1404 continue;
1405
1407 if (!SafeToMergeTerminators(TI, PTI, &FailBlocks)) {
1408 for (auto *Succ : FailBlocks) {
1409 if (!SplitBlockPredecessors(Succ, TI->getParent(), ".fold.split", DTU))
1410 return false;
1411 }
1412 }
1413
1414 PerformValueComparisonIntoPredecessorFolding(TI, CV, PTI, Builder);
1415 Changed = true;
1416 }
1417 return Changed;
1418}
1419
1420// If we would need to insert a select that uses the value of this invoke
1421// (comments in hoistSuccIdenticalTerminatorToSwitchOrIf explain why we would
1422// need to do this), we can't hoist the invoke, as there is nowhere to put the
1423// select in this case.
1425 Instruction *I1, Instruction *I2) {
1426 for (BasicBlock *Succ : successors(BB1)) {
1427 for (const PHINode &PN : Succ->phis()) {
1428 Value *BB1V = PN.getIncomingValueForBlock(BB1);
1429 Value *BB2V = PN.getIncomingValueForBlock(BB2);
1430 if (BB1V != BB2V && (BB1V == I1 || BB2V == I2)) {
1431 return false;
1432 }
1433 }
1434 }
1435 return true;
1436}
1437
1438// Get interesting characteristics of instructions that
1439// `hoistCommonCodeFromSuccessors` didn't hoist. They restrict what kind of
1440// instructions can be reordered across.
1446
1448 unsigned Flags = 0;
1449 if (I->mayReadFromMemory())
1450 Flags |= SkipReadMem;
1451 // We can't arbitrarily move around allocas, e.g. moving allocas (especially
1452 // inalloca) across stacksave/stackrestore boundaries.
1453 if (I->mayHaveSideEffects() || isa<AllocaInst>(I))
1454 Flags |= SkipSideEffect;
1456 Flags |= SkipImplicitControlFlow;
1457 return Flags;
1458}
1459
1460// Returns true if it is safe to reorder an instruction across preceding
1461// instructions in a basic block.
1462static bool isSafeToHoistInstr(Instruction *I, unsigned Flags) {
1463 // Don't reorder a store over a load.
1464 if ((Flags & SkipReadMem) && I->mayWriteToMemory())
1465 return false;
1466
1467 // If we have seen an instruction with side effects, it's unsafe to reorder an
1468 // instruction which reads memory or itself has side effects.
1469 if ((Flags & SkipSideEffect) &&
1470 (I->mayReadFromMemory() || I->mayHaveSideEffects() || isa<AllocaInst>(I)))
1471 return false;
1472
1473 // Reordering across an instruction which does not necessarily transfer
1474 // control to the next instruction is speculation.
1476 return false;
1477
1478 // Hoisting of llvm.deoptimize is only legal together with the next return
1479 // instruction, which this pass is not always able to do.
1480 if (auto *CB = dyn_cast<CallBase>(I))
1481 if (CB->getIntrinsicID() == Intrinsic::experimental_deoptimize)
1482 return false;
1483
1484 // It's also unsafe/illegal to hoist an instruction above its instruction
1485 // operands
1486 BasicBlock *BB = I->getParent();
1487 for (Value *Op : I->operands()) {
1488 if (auto *J = dyn_cast<Instruction>(Op))
1489 if (J->getParent() == BB)
1490 return false;
1491 }
1492
1493 return true;
1494}
1495
1496static bool passingValueIsAlwaysUndefined(Value *V, Instruction *I, bool PtrValueMayBeModified = false);
1497
1498/// Helper function for hoistCommonCodeFromSuccessors. Return true if identical
1499/// instructions \p I1 and \p I2 can and should be hoisted.
1501 const TargetTransformInfo &TTI) {
1502 // If we're going to hoist a call, make sure that the two instructions
1503 // we're commoning/hoisting are both marked with musttail, or neither of
1504 // them is marked as such. Otherwise, we might end up in a situation where
1505 // we hoist from a block where the terminator is a `ret` to a block where
1506 // the terminator is a `br`, and `musttail` calls expect to be followed by
1507 // a return.
1508 auto *C1 = dyn_cast<CallInst>(I1);
1509 auto *C2 = dyn_cast<CallInst>(I2);
1510 if (C1 && C2)
1511 if (C1->isMustTailCall() != C2->isMustTailCall())
1512 return false;
1513
1515 return false;
1516
1517 // If any of the two call sites has nomerge or convergent attribute, stop
1518 // hoisting.
1519 if (const auto *CB1 = dyn_cast<CallBase>(I1))
1520 if (CB1->cannotMerge() || CB1->isConvergent())
1521 return false;
1522 if (const auto *CB2 = dyn_cast<CallBase>(I2))
1523 if (CB2->cannotMerge() || CB2->isConvergent())
1524 return false;
1525
1526 return true;
1527}
1528
1529/// Hoists DbgVariableRecords from \p I1 and \p OtherInstrs that are identical
1530/// in lock-step to \p TI. This matches how dbg.* intrinsics are hoisting in
1531/// hoistCommonCodeFromSuccessors. e.g. The input:
1532/// I1 DVRs: { x, z },
1533/// OtherInsts: { I2 DVRs: { x, y, z } }
1534/// would result in hoisting only DbgVariableRecord x.
1536 Instruction *TI, Instruction *I1,
1537 SmallVectorImpl<Instruction *> &OtherInsts) {
1538 if (!I1->hasDbgRecords())
1539 return;
1540 using CurrentAndEndIt =
1541 std::pair<DbgRecord::self_iterator, DbgRecord::self_iterator>;
1542 // Vector of {Current, End} iterators.
1544 Itrs.reserve(OtherInsts.size() + 1);
1545 // Helper lambdas for lock-step checks:
1546 // Return true if this Current == End.
1547 auto atEnd = [](const CurrentAndEndIt &Pair) {
1548 return Pair.first == Pair.second;
1549 };
1550 // Return true if all Current are identical.
1551 auto allIdentical = [](const SmallVector<CurrentAndEndIt> &Itrs) {
1552 return all_of(make_first_range(ArrayRef(Itrs).drop_front()),
1554 return Itrs[0].first->isIdenticalToWhenDefined(*I);
1555 });
1556 };
1557
1558 // Collect the iterators.
1559 Itrs.push_back(
1560 {I1->getDbgRecordRange().begin(), I1->getDbgRecordRange().end()});
1561 for (Instruction *Other : OtherInsts) {
1562 if (!Other->hasDbgRecords())
1563 return;
1564 Itrs.push_back(
1565 {Other->getDbgRecordRange().begin(), Other->getDbgRecordRange().end()});
1566 }
1567
1568 // Iterate in lock-step until any of the DbgRecord lists are exausted. If
1569 // the lock-step DbgRecord are identical, hoist all of them to TI.
1570 // This replicates the dbg.* intrinsic behaviour in
1571 // hoistCommonCodeFromSuccessors.
1572 while (none_of(Itrs, atEnd)) {
1573 bool HoistDVRs = allIdentical(Itrs);
1574 for (CurrentAndEndIt &Pair : Itrs) {
1575 // Increment Current iterator now as we may be about to move the
1576 // DbgRecord.
1577 DbgRecord &DR = *Pair.first++;
1578 if (HoistDVRs) {
1579 DR.removeFromParent();
1580 TI->getParent()->insertDbgRecordBefore(&DR, TI->getIterator());
1581 }
1582 }
1583 }
1584}
1585
1586/// Hoist any common code in the successor blocks up into the block. This
1587/// function guarantees that BB dominates all successors. If EqTermsOnly is
1588/// given, only perform hoisting in case both blocks only contain a terminator.
1589/// In that case, only the original BI will be replaced and selects for PHIs are
1590/// added.
1591bool SimplifyCFGOpt::hoistCommonCodeFromSuccessors(BasicBlock *BB,
1592 bool EqTermsOnly) {
1593 // This does very trivial matching, with limited scanning, to find identical
1594 // instructions in the two blocks. In particular, we don't want to get into
1595 // O(N1*N2*...) situations here where Ni are the sizes of these successors. As
1596 // such, we currently just scan for obviously identical instructions in an
1597 // identical order, possibly separated by the same number of non-identical
1598 // instructions.
1599 unsigned int SuccSize = succ_size(BB);
1600 if (SuccSize < 2)
1601 return false;
1602
1603 // If either of the blocks has it's address taken, then we can't do this fold,
1604 // because the code we'd hoist would no longer run when we jump into the block
1605 // by it's address.
1606 for (auto *Succ : successors(BB))
1607 if (Succ->hasAddressTaken() || !Succ->getSinglePredecessor())
1608 return false;
1609
1610 auto *TI = BB->getTerminator();
1611
1612 // The second of pair is a SkipFlags bitmask.
1613 using SuccIterPair = std::pair<BasicBlock::iterator, unsigned>;
1614 SmallVector<SuccIterPair, 8> SuccIterPairs;
1615 for (auto *Succ : successors(BB)) {
1616 BasicBlock::iterator SuccItr = Succ->begin();
1617 if (isa<PHINode>(*SuccItr))
1618 return false;
1619 SuccIterPairs.push_back(SuccIterPair(SuccItr, 0));
1620 }
1621
1622 // Check if only hoisting terminators is allowed. This does not add new
1623 // instructions to the hoist location.
1624 if (EqTermsOnly) {
1625 // Skip any debug intrinsics, as they are free to hoist.
1626 for (auto &SuccIter : make_first_range(SuccIterPairs)) {
1627 auto *INonDbg = &*skipDebugIntrinsics(SuccIter);
1628 if (!INonDbg->isTerminator())
1629 return false;
1630 }
1631 // Now we know that we only need to hoist debug intrinsics and the
1632 // terminator. Let the loop below handle those 2 cases.
1633 }
1634
1635 // Count how many instructions were not hoisted so far. There's a limit on how
1636 // many instructions we skip, serving as a compilation time control as well as
1637 // preventing excessive increase of life ranges.
1638 unsigned NumSkipped = 0;
1639 // If we find an unreachable instruction at the beginning of a basic block, we
1640 // can still hoist instructions from the rest of the basic blocks.
1641 if (SuccIterPairs.size() > 2) {
1642 erase_if(SuccIterPairs,
1643 [](const auto &Pair) { return isa<UnreachableInst>(Pair.first); });
1644 if (SuccIterPairs.size() < 2)
1645 return false;
1646 }
1647
1648 bool Changed = false;
1649
1650 for (;;) {
1651 auto *SuccIterPairBegin = SuccIterPairs.begin();
1652 auto &BB1ItrPair = *SuccIterPairBegin++;
1653 auto OtherSuccIterPairRange =
1654 iterator_range(SuccIterPairBegin, SuccIterPairs.end());
1655 auto OtherSuccIterRange = make_first_range(OtherSuccIterPairRange);
1656
1657 Instruction *I1 = &*BB1ItrPair.first;
1658
1659 // Skip debug info if it is not identical.
1660 bool AllDbgInstsAreIdentical = all_of(OtherSuccIterRange, [I1](auto &Iter) {
1661 Instruction *I2 = &*Iter;
1662 return I1->isIdenticalToWhenDefined(I2);
1663 });
1664 if (!AllDbgInstsAreIdentical) {
1665 while (isa<DbgInfoIntrinsic>(I1))
1666 I1 = &*++BB1ItrPair.first;
1667 for (auto &SuccIter : OtherSuccIterRange) {
1668 Instruction *I2 = &*SuccIter;
1669 while (isa<DbgInfoIntrinsic>(I2))
1670 I2 = &*++SuccIter;
1671 }
1672 }
1673
1674 bool AllInstsAreIdentical = true;
1675 bool HasTerminator = I1->isTerminator();
1676 for (auto &SuccIter : OtherSuccIterRange) {
1677 Instruction *I2 = &*SuccIter;
1678 HasTerminator |= I2->isTerminator();
1679 if (AllInstsAreIdentical && (!I1->isIdenticalToWhenDefined(I2) ||
1680 MMRAMetadata(*I1) != MMRAMetadata(*I2)))
1681 AllInstsAreIdentical = false;
1682 }
1683
1685 for (auto &SuccIter : OtherSuccIterRange)
1686 OtherInsts.push_back(&*SuccIter);
1687
1688 // If we are hoisting the terminator instruction, don't move one (making a
1689 // broken BB), instead clone it, and remove BI.
1690 if (HasTerminator) {
1691 // Even if BB, which contains only one unreachable instruction, is ignored
1692 // at the beginning of the loop, we can hoist the terminator instruction.
1693 // If any instructions remain in the block, we cannot hoist terminators.
1694 if (NumSkipped || !AllInstsAreIdentical) {
1695 hoistLockstepIdenticalDbgVariableRecords(TI, I1, OtherInsts);
1696 return Changed;
1697 }
1698
1699 return hoistSuccIdenticalTerminatorToSwitchOrIf(TI, I1, OtherInsts) ||
1700 Changed;
1701 }
1702
1703 if (AllInstsAreIdentical) {
1704 unsigned SkipFlagsBB1 = BB1ItrPair.second;
1705 AllInstsAreIdentical =
1706 isSafeToHoistInstr(I1, SkipFlagsBB1) &&
1707 all_of(OtherSuccIterPairRange, [=](const auto &Pair) {
1708 Instruction *I2 = &*Pair.first;
1709 unsigned SkipFlagsBB2 = Pair.second;
1710 // Even if the instructions are identical, it may not
1711 // be safe to hoist them if we have skipped over
1712 // instructions with side effects or their operands
1713 // weren't hoisted.
1714 return isSafeToHoistInstr(I2, SkipFlagsBB2) &&
1716 });
1717 }
1718
1719 if (AllInstsAreIdentical) {
1720 BB1ItrPair.first++;
1721 if (isa<DbgInfoIntrinsic>(I1)) {
1722 // The debug location is an integral part of a debug info intrinsic
1723 // and can't be separated from it or replaced. Instead of attempting
1724 // to merge locations, simply hoist both copies of the intrinsic.
1725 hoistLockstepIdenticalDbgVariableRecords(TI, I1, OtherInsts);
1726 // We've just hoisted DbgVariableRecords; move I1 after them (before TI)
1727 // and leave any that were not hoisted behind (by calling moveBefore
1728 // rather than moveBeforePreserving).
1729 I1->moveBefore(TI);
1730 for (auto &SuccIter : OtherSuccIterRange) {
1731 auto *I2 = &*SuccIter++;
1732 assert(isa<DbgInfoIntrinsic>(I2));
1733 I2->moveBefore(TI);
1734 }
1735 } else {
1736 // For a normal instruction, we just move one to right before the
1737 // branch, then replace all uses of the other with the first. Finally,
1738 // we remove the now redundant second instruction.
1739 hoistLockstepIdenticalDbgVariableRecords(TI, I1, OtherInsts);
1740 // We've just hoisted DbgVariableRecords; move I1 after them (before TI)
1741 // and leave any that were not hoisted behind (by calling moveBefore
1742 // rather than moveBeforePreserving).
1743 I1->moveBefore(TI);
1744 for (auto &SuccIter : OtherSuccIterRange) {
1745 Instruction *I2 = &*SuccIter++;
1746 assert(I2 != I1);
1747 if (!I2->use_empty())
1748 I2->replaceAllUsesWith(I1);
1749 I1->andIRFlags(I2);
1750 combineMetadataForCSE(I1, I2, true);
1751 // I1 and I2 are being combined into a single instruction. Its debug
1752 // location is the merged locations of the original instructions.
1753 I1->applyMergedLocation(I1->getDebugLoc(), I2->getDebugLoc());
1754 I2->eraseFromParent();
1755 }
1756 }
1757 if (!Changed)
1758 NumHoistCommonCode += SuccIterPairs.size();
1759 Changed = true;
1760 NumHoistCommonInstrs += SuccIterPairs.size();
1761 } else {
1762 if (NumSkipped >= HoistCommonSkipLimit) {
1763 hoistLockstepIdenticalDbgVariableRecords(TI, I1, OtherInsts);
1764 return Changed;
1765 }
1766 // We are about to skip over a pair of non-identical instructions. Record
1767 // if any have characteristics that would prevent reordering instructions
1768 // across them.
1769 for (auto &SuccIterPair : SuccIterPairs) {
1770 Instruction *I = &*SuccIterPair.first++;
1771 SuccIterPair.second |= skippedInstrFlags(I);
1772 }
1773 ++NumSkipped;
1774 }
1775 }
1776}
1777
1778bool SimplifyCFGOpt::hoistSuccIdenticalTerminatorToSwitchOrIf(
1779 Instruction *TI, Instruction *I1,
1780 SmallVectorImpl<Instruction *> &OtherSuccTIs) {
1781
1782 auto *BI = dyn_cast<BranchInst>(TI);
1783
1784 bool Changed = false;
1785 BasicBlock *TIParent = TI->getParent();
1786 BasicBlock *BB1 = I1->getParent();
1787
1788 // Use only for an if statement.
1789 auto *I2 = *OtherSuccTIs.begin();
1790 auto *BB2 = I2->getParent();
1791 if (BI) {
1792 assert(OtherSuccTIs.size() == 1);
1793 assert(BI->getSuccessor(0) == I1->getParent());
1794 assert(BI->getSuccessor(1) == I2->getParent());
1795 }
1796
1797 // In the case of an if statement, we try to hoist an invoke.
1798 // FIXME: Can we define a safety predicate for CallBr?
1799 // FIXME: Test case llvm/test/Transforms/SimplifyCFG/2009-06-15-InvokeCrash.ll
1800 // removed in 4c923b3b3fd0ac1edebf0603265ca3ba51724937 commit?
1801 if (isa<InvokeInst>(I1) && (!BI || !isSafeToHoistInvoke(BB1, BB2, I1, I2)))
1802 return false;
1803
1804 // TODO: callbr hoisting currently disabled pending further study.
1805 if (isa<CallBrInst>(I1))
1806 return false;
1807
1808 for (BasicBlock *Succ : successors(BB1)) {
1809 for (PHINode &PN : Succ->phis()) {
1810 Value *BB1V = PN.getIncomingValueForBlock(BB1);
1811 for (Instruction *OtherSuccTI : OtherSuccTIs) {
1812 Value *BB2V = PN.getIncomingValueForBlock(OtherSuccTI->getParent());
1813 if (BB1V == BB2V)
1814 continue;
1815
1816 // In the case of an if statement, check for
1817 // passingValueIsAlwaysUndefined here because we would rather eliminate
1818 // undefined control flow then converting it to a select.
1819 if (!BI || passingValueIsAlwaysUndefined(BB1V, &PN) ||
1821 return false;
1822 }
1823 }
1824 }
1825
1826 // Hoist DbgVariableRecords attached to the terminator to match dbg.*
1827 // intrinsic hoisting behaviour in hoistCommonCodeFromSuccessors.
1828 hoistLockstepIdenticalDbgVariableRecords(TI, I1, OtherSuccTIs);
1829 // Clone the terminator and hoist it into the pred, without any debug info.
1830 Instruction *NT = I1->clone();
1831 NT->insertInto(TIParent, TI->getIterator());
1832 if (!NT->getType()->isVoidTy()) {
1833 I1->replaceAllUsesWith(NT);
1834 for (Instruction *OtherSuccTI : OtherSuccTIs)
1835 OtherSuccTI->replaceAllUsesWith(NT);
1836 NT->takeName(I1);
1837 }
1838 Changed = true;
1839 NumHoistCommonInstrs += OtherSuccTIs.size() + 1;
1840
1841 // Ensure terminator gets a debug location, even an unknown one, in case
1842 // it involves inlinable calls.
1844 Locs.push_back(I1->getDebugLoc());
1845 for (auto *OtherSuccTI : OtherSuccTIs)
1846 Locs.push_back(OtherSuccTI->getDebugLoc());
1847 NT->setDebugLoc(DILocation::getMergedLocations(Locs));
1848
1849 // PHIs created below will adopt NT's merged DebugLoc.
1850 IRBuilder<NoFolder> Builder(NT);
1851
1852 // In the case of an if statement, hoisting one of the terminators from our
1853 // successor is a great thing. Unfortunately, the successors of the if/else
1854 // blocks may have PHI nodes in them. If they do, all PHI entries for BB1/BB2
1855 // must agree for all PHI nodes, so we insert select instruction to compute
1856 // the final result.
1857 if (BI) {
1858 std::map<std::pair<Value *, Value *>, SelectInst *> InsertedSelects;
1859 for (BasicBlock *Succ : successors(BB1)) {
1860 for (PHINode &PN : Succ->phis()) {
1861 Value *BB1V = PN.getIncomingValueForBlock(BB1);
1862 Value *BB2V = PN.getIncomingValueForBlock(BB2);
1863 if (BB1V == BB2V)
1864 continue;
1865
1866 // These values do not agree. Insert a select instruction before NT
1867 // that determines the right value.
1868 SelectInst *&SI = InsertedSelects[std::make_pair(BB1V, BB2V)];
1869 if (!SI) {
1870 // Propagate fast-math-flags from phi node to its replacement select.
1871 IRBuilder<>::FastMathFlagGuard FMFGuard(Builder);
1872 if (isa<FPMathOperator>(PN))
1873 Builder.setFastMathFlags(PN.getFastMathFlags());
1874
1875 SI = cast<SelectInst>(Builder.CreateSelect(
1876 BI->getCondition(), BB1V, BB2V,
1877 BB1V->getName() + "." + BB2V->getName(), BI));
1878 }
1879
1880 // Make the PHI node use the select for all incoming values for BB1/BB2
1881 for (unsigned i = 0, e = PN.getNumIncomingValues(); i != e; ++i)
1882 if (PN.getIncomingBlock(i) == BB1 || PN.getIncomingBlock(i) == BB2)
1883 PN.setIncomingValue(i, SI);
1884 }
1885 }
1886 }
1887
1889
1890 // Update any PHI nodes in our new successors.
1891 for (BasicBlock *Succ : successors(BB1)) {
1892 AddPredecessorToBlock(Succ, TIParent, BB1);
1893 if (DTU)
1894 Updates.push_back({DominatorTree::Insert, TIParent, Succ});
1895 }
1896
1897 if (DTU)
1898 for (BasicBlock *Succ : successors(TI))
1899 Updates.push_back({DominatorTree::Delete, TIParent, Succ});
1900
1902 if (DTU)
1903 DTU->applyUpdates(Updates);
1904 return Changed;
1905}
1906
1907// Check lifetime markers.
1908static bool isLifeTimeMarker(const Instruction *I) {
1909 if (auto II = dyn_cast<IntrinsicInst>(I)) {
1910 switch (II->getIntrinsicID()) {
1911 default:
1912 break;
1913 case Intrinsic::lifetime_start:
1914 case Intrinsic::lifetime_end:
1915 return true;
1916 }
1917 }
1918 return false;
1919}
1920
1921// TODO: Refine this. This should avoid cases like turning constant memcpy sizes
1922// into variables.
1924 int OpIdx) {
1925 return !isa<IntrinsicInst>(I);
1926}
1927
1928// All instructions in Insts belong to different blocks that all unconditionally
1929// branch to a common successor. Analyze each instruction and return true if it
1930// would be possible to sink them into their successor, creating one common
1931// instruction instead. For every value that would be required to be provided by
1932// PHI node (because an operand varies in each input block), add to PHIOperands.
1935 DenseMap<Instruction *, SmallVector<Value *, 4>> &PHIOperands) {
1936 // Prune out obviously bad instructions to move. Each instruction must have
1937 // exactly zero or one use, and we check later that use is by a single, common
1938 // PHI instruction in the successor.
1939 bool HasUse = !Insts.front()->user_empty();
1940 for (auto *I : Insts) {
1941 // These instructions may change or break semantics if moved.
1942 if (isa<PHINode>(I) || I->isEHPad() || isa<AllocaInst>(I) ||
1943 I->getType()->isTokenTy())
1944 return false;
1945
1946 // Do not try to sink an instruction in an infinite loop - it can cause
1947 // this algorithm to infinite loop.
1948 if (I->getParent()->getSingleSuccessor() == I->getParent())
1949 return false;
1950
1951 // Conservatively return false if I is an inline-asm instruction. Sinking
1952 // and merging inline-asm instructions can potentially create arguments
1953 // that cannot satisfy the inline-asm constraints.
1954 // If the instruction has nomerge or convergent attribute, return false.
1955 if (const auto *C = dyn_cast<CallBase>(I))
1956 if (C->isInlineAsm() || C->cannotMerge() || C->isConvergent())
1957 return false;
1958
1959 // Each instruction must have zero or one use.
1960 if (HasUse && !I->hasOneUse())
1961 return false;
1962 if (!HasUse && !I->user_empty())
1963 return false;
1964 }
1965
1966 const Instruction *I0 = Insts.front();
1967 const auto I0MMRA = MMRAMetadata(*I0);
1968 for (auto *I : Insts) {
1969 if (!I->isSameOperationAs(I0))
1970 return false;
1971
1972 // swifterror pointers can only be used by a load or store; sinking a load
1973 // or store would require introducing a select for the pointer operand,
1974 // which isn't allowed for swifterror pointers.
1975 if (isa<StoreInst>(I) && I->getOperand(1)->isSwiftError())
1976 return false;
1977 if (isa<LoadInst>(I) && I->getOperand(0)->isSwiftError())
1978 return false;
1979
1980 // Treat MMRAs conservatively. This pass can be quite aggressive and
1981 // could drop a lot of MMRAs otherwise.
1982 if (MMRAMetadata(*I) != I0MMRA)
1983 return false;
1984 }
1985
1986 // All instructions in Insts are known to be the same opcode. If they have a
1987 // use, check that the only user is a PHI or in the same block as the
1988 // instruction, because if a user is in the same block as an instruction we're
1989 // contemplating sinking, it must already be determined to be sinkable.
1990 if (HasUse) {
1991 auto *PNUse = dyn_cast<PHINode>(*I0->user_begin());
1992 auto *Succ = I0->getParent()->getTerminator()->getSuccessor(0);
1993 if (!all_of(Insts, [&PNUse,&Succ](const Instruction *I) -> bool {
1994 auto *U = cast<Instruction>(*I->user_begin());
1995 return (PNUse &&
1996 PNUse->getParent() == Succ &&
1997 PNUse->getIncomingValueForBlock(I->getParent()) == I) ||
1998 U->getParent() == I->getParent();
1999 }))
2000 return false;
2001 }
2002
2003 // Because SROA can't handle speculating stores of selects, try not to sink
2004 // loads, stores or lifetime markers of allocas when we'd have to create a
2005 // PHI for the address operand. Also, because it is likely that loads or
2006 // stores of allocas will disappear when Mem2Reg/SROA is run, don't sink
2007 // them.
2008 // This can cause code churn which can have unintended consequences down
2009 // the line - see https://llvm.org/bugs/show_bug.cgi?id=30244.
2010 // FIXME: This is a workaround for a deficiency in SROA - see
2011 // https://llvm.org/bugs/show_bug.cgi?id=30188
2012 if (isa<StoreInst>(I0) && any_of(Insts, [](const Instruction *I) {
2013 return isa<AllocaInst>(I->getOperand(1)->stripPointerCasts());
2014 }))
2015 return false;
2016 if (isa<LoadInst>(I0) && any_of(Insts, [](const Instruction *I) {
2017 return isa<AllocaInst>(I->getOperand(0)->stripPointerCasts());
2018 }))
2019 return false;
2020 if (isLifeTimeMarker(I0) && any_of(Insts, [](const Instruction *I) {
2021 return isa<AllocaInst>(I->getOperand(1)->stripPointerCasts());
2022 }))
2023 return false;
2024
2025 // For calls to be sinkable, they must all be indirect, or have same callee.
2026 // I.e. if we have two direct calls to different callees, we don't want to
2027 // turn that into an indirect call. Likewise, if we have an indirect call,
2028 // and a direct call, we don't actually want to have a single indirect call.
2029 if (isa<CallBase>(I0)) {
2030 auto IsIndirectCall = [](const Instruction *I) {
2031 return cast<CallBase>(I)->isIndirectCall();
2032 };
2033 bool HaveIndirectCalls = any_of(Insts, IsIndirectCall);
2034 bool AllCallsAreIndirect = all_of(Insts, IsIndirectCall);
2035 if (HaveIndirectCalls) {
2036 if (!AllCallsAreIndirect)
2037 return false;
2038 } else {
2039 // All callees must be identical.
2040 Value *Callee = nullptr;
2041 for (const Instruction *I : Insts) {
2042 Value *CurrCallee = cast<CallBase>(I)->getCalledOperand();
2043 if (!Callee)
2044 Callee = CurrCallee;
2045 else if (Callee != CurrCallee)
2046 return false;
2047 }
2048 }
2049 }
2050
2051 for (unsigned OI = 0, OE = I0->getNumOperands(); OI != OE; ++OI) {
2052 Value *Op = I0->getOperand(OI);
2053 if (Op->getType()->isTokenTy())
2054 // Don't touch any operand of token type.
2055 return false;
2056
2057 auto SameAsI0 = [&I0, OI](const Instruction *I) {
2058 assert(I->getNumOperands() == I0->getNumOperands());
2059 return I->getOperand(OI) == I0->getOperand(OI);
2060 };
2061 if (!all_of(Insts, SameAsI0)) {
2062 if ((isa<Constant>(Op) && !replacingOperandWithVariableIsCheap(I0, OI)) ||
2064 // We can't create a PHI from this GEP.
2065 return false;
2066 for (auto *I : Insts)
2067 PHIOperands[I].push_back(I->getOperand(OI));
2068 }
2069 }
2070 return true;
2071}
2072
2073// Assuming canSinkInstructions(Blocks) has returned true, sink the last
2074// instruction of every block in Blocks to their common successor, commoning
2075// into one instruction.
2077 auto *BBEnd = Blocks[0]->getTerminator()->getSuccessor(0);
2078
2079 // canSinkInstructions returning true guarantees that every block has at
2080 // least one non-terminator instruction.
2082 for (auto *BB : Blocks) {
2083 Instruction *I = BB->getTerminator();
2084 do {
2085 I = I->getPrevNode();
2086 } while (isa<DbgInfoIntrinsic>(I) && I != &BB->front());
2087 if (!isa<DbgInfoIntrinsic>(I))
2088 Insts.push_back(I);
2089 }
2090
2091 // The only checking we need to do now is that all users of all instructions
2092 // are the same PHI node. canSinkInstructions should have checked this but
2093 // it is slightly over-aggressive - it gets confused by commutative
2094 // instructions so double-check it here.
2095 Instruction *I0 = Insts.front();
2096 if (!I0->user_empty()) {
2097 auto *PNUse = dyn_cast<PHINode>(*I0->user_begin());
2098 if (!all_of(Insts, [&PNUse](const Instruction *I) -> bool {
2099 auto *U = cast<Instruction>(*I->user_begin());
2100 return U == PNUse;
2101 }))
2102 return false;
2103 }
2104
2105 // We don't need to do any more checking here; canSinkInstructions should
2106 // have done it all for us.
2107 SmallVector<Value*, 4> NewOperands;
2108 for (unsigned O = 0, E = I0->getNumOperands(); O != E; ++O) {
2109 // This check is different to that in canSinkInstructions. There, we
2110 // cared about the global view once simplifycfg (and instcombine) have
2111 // completed - it takes into account PHIs that become trivially
2112 // simplifiable. However here we need a more local view; if an operand
2113 // differs we create a PHI and rely on instcombine to clean up the very
2114 // small mess we may make.
2115 bool NeedPHI = any_of(Insts, [&I0, O](const Instruction *I) {
2116 return I->getOperand(O) != I0->getOperand(O);
2117 });
2118 if (!NeedPHI) {
2119 NewOperands.push_back(I0->getOperand(O));
2120 continue;
2121 }
2122
2123 // Create a new PHI in the successor block and populate it.
2124 auto *Op = I0->getOperand(O);
2125 assert(!Op->getType()->isTokenTy() && "Can't PHI tokens!");
2126 auto *PN =
2127 PHINode::Create(Op->getType(), Insts.size(), Op->getName() + ".sink");
2128 PN->insertBefore(BBEnd->begin());
2129 for (auto *I : Insts)
2130 PN->addIncoming(I->getOperand(O), I->getParent());
2131 NewOperands.push_back(PN);
2132 }
2133
2134 // Arbitrarily use I0 as the new "common" instruction; remap its operands
2135 // and move it to the start of the successor block.
2136 for (unsigned O = 0, E = I0->getNumOperands(); O != E; ++O)
2137 I0->getOperandUse(O).set(NewOperands[O]);
2138
2139 I0->moveBefore(*BBEnd, BBEnd->getFirstInsertionPt());
2140
2141 // Update metadata and IR flags, and merge debug locations.
2142 for (auto *I : Insts)
2143 if (I != I0) {
2144 // The debug location for the "common" instruction is the merged locations
2145 // of all the commoned instructions. We start with the original location
2146 // of the "common" instruction and iteratively merge each location in the
2147 // loop below.
2148 // This is an N-way merge, which will be inefficient if I0 is a CallInst.
2149 // However, as N-way merge for CallInst is rare, so we use simplified API
2150 // instead of using complex API for N-way merge.
2151 I0->applyMergedLocation(I0->getDebugLoc(), I->getDebugLoc());
2152 combineMetadataForCSE(I0, I, true);
2153 I0->andIRFlags(I);
2154 }
2155
2156 if (!I0->user_empty()) {
2157 // canSinkLastInstruction checked that all instructions were used by
2158 // one and only one PHI node. Find that now, RAUW it to our common
2159 // instruction and nuke it.
2160 auto *PN = cast<PHINode>(*I0->user_begin());
2161 PN->replaceAllUsesWith(I0);
2162 PN->eraseFromParent();
2163 }
2164
2165 // Finally nuke all instructions apart from the common instruction.
2166 for (auto *I : Insts) {
2167 if (I == I0)
2168 continue;
2169 // The remaining uses are debug users, replace those with the common inst.
2170 // In most (all?) cases this just introduces a use-before-def.
2171 assert(I->user_empty() && "Inst unexpectedly still has non-dbg users");
2172 I->replaceAllUsesWith(I0);
2173 I->eraseFromParent();
2174 }
2175
2176 return true;
2177}
2178
2179namespace {
2180
2181 // LockstepReverseIterator - Iterates through instructions
2182 // in a set of blocks in reverse order from the first non-terminator.
2183 // For example (assume all blocks have size n):
2184 // LockstepReverseIterator I([B1, B2, B3]);
2185 // *I-- = [B1[n], B2[n], B3[n]];
2186 // *I-- = [B1[n-1], B2[n-1], B3[n-1]];
2187 // *I-- = [B1[n-2], B2[n-2], B3[n-2]];
2188 // ...
2189 class LockstepReverseIterator {
2192 bool Fail;
2193
2194 public:
2195 LockstepReverseIterator(ArrayRef<BasicBlock*> Blocks) : Blocks(Blocks) {
2196 reset();
2197 }
2198
2199 void reset() {
2200 Fail = false;
2201 Insts.clear();
2202 for (auto *BB : Blocks) {
2203 Instruction *Inst = BB->getTerminator();
2204 for (Inst = Inst->getPrevNode(); Inst && isa<DbgInfoIntrinsic>(Inst);)
2205 Inst = Inst->getPrevNode();
2206 if (!Inst) {
2207 // Block wasn't big enough.
2208 Fail = true;
2209 return;
2210 }
2211 Insts.push_back(Inst);
2212 }
2213 }
2214
2215 bool isValid() const {
2216 return !Fail;
2217 }
2218
2219 void operator--() {
2220 if (Fail)
2221 return;
2222 for (auto *&Inst : Insts) {
2223 for (Inst = Inst->getPrevNode(); Inst && isa<DbgInfoIntrinsic>(Inst);)
2224 Inst = Inst->getPrevNode();
2225 // Already at beginning of block.
2226 if (!Inst) {
2227 Fail = true;
2228 return;
2229 }
2230 }
2231 }
2232
2233 void operator++() {
2234 if (Fail)
2235 return;
2236 for (auto *&Inst : Insts) {
2237 for (Inst = Inst->getNextNode(); Inst && isa<DbgInfoIntrinsic>(Inst);)
2238 Inst = Inst->getNextNode();
2239 // Already at end of block.
2240 if (!Inst) {
2241 Fail = true;
2242 return;
2243 }
2244 }
2245 }
2246
2248 return Insts;
2249 }
2250 };
2251
2252} // end anonymous namespace
2253
2254/// Check whether BB's predecessors end with unconditional branches. If it is
2255/// true, sink any common code from the predecessors to BB.
2257 DomTreeUpdater *DTU) {
2258 // We support two situations:
2259 // (1) all incoming arcs are unconditional
2260 // (2) there are non-unconditional incoming arcs
2261 //
2262 // (2) is very common in switch defaults and
2263 // else-if patterns;
2264 //
2265 // if (a) f(1);
2266 // else if (b) f(2);
2267 //
2268 // produces:
2269 //
2270 // [if]
2271 // / \
2272 // [f(1)] [if]
2273 // | | \
2274 // | | |
2275 // | [f(2)]|
2276 // \ | /
2277 // [ end ]
2278 //
2279 // [end] has two unconditional predecessor arcs and one conditional. The
2280 // conditional refers to the implicit empty 'else' arc. This conditional
2281 // arc can also be caused by an empty default block in a switch.
2282 //
2283 // In this case, we attempt to sink code from all *unconditional* arcs.
2284 // If we can sink instructions from these arcs (determined during the scan
2285 // phase below) we insert a common successor for all unconditional arcs and
2286 // connect that to [end], to enable sinking:
2287 //
2288 // [if]
2289 // / \
2290 // [x(1)] [if]
2291 // | | \
2292 // | | \
2293 // | [x(2)] |
2294 // \ / |
2295 // [sink.split] |
2296 // \ /
2297 // [ end ]
2298 //
2299 SmallVector<BasicBlock*,4> UnconditionalPreds;
2300 bool HaveNonUnconditionalPredecessors = false;
2301 for (auto *PredBB : predecessors(BB)) {
2302 auto *PredBr = dyn_cast<BranchInst>(PredBB->getTerminator());
2303 if (PredBr && PredBr->isUnconditional())
2304 UnconditionalPreds.push_back(PredBB);
2305 else
2306 HaveNonUnconditionalPredecessors = true;
2307 }
2308 if (UnconditionalPreds.size() < 2)
2309 return false;
2310
2311 // We take a two-step approach to tail sinking. First we scan from the end of
2312 // each block upwards in lockstep. If the n'th instruction from the end of each
2313 // block can be sunk, those instructions are added to ValuesToSink and we
2314 // carry on. If we can sink an instruction but need to PHI-merge some operands
2315 // (because they're not identical in each instruction) we add these to
2316 // PHIOperands.
2317 int ScanIdx = 0;
2318 SmallPtrSet<Value*,4> InstructionsToSink;
2320 LockstepReverseIterator LRI(UnconditionalPreds);
2321 while (LRI.isValid() &&
2322 canSinkInstructions(*LRI, PHIOperands)) {
2323 LLVM_DEBUG(dbgs() << "SINK: instruction can be sunk: " << *(*LRI)[0]
2324 << "\n");
2325 InstructionsToSink.insert((*LRI).begin(), (*LRI).end());
2326 ++ScanIdx;
2327 --LRI;
2328 }
2329
2330 // If no instructions can be sunk, early-return.
2331 if (ScanIdx == 0)
2332 return false;
2333
2334 bool followedByDeoptOrUnreachable = IsBlockFollowedByDeoptOrUnreachable(BB);
2335
2336 if (!followedByDeoptOrUnreachable) {
2337 // Okay, we *could* sink last ScanIdx instructions. But how many can we
2338 // actually sink before encountering instruction that is unprofitable to
2339 // sink?
2340 auto ProfitableToSinkInstruction = [&](LockstepReverseIterator &LRI) {
2341 unsigned NumPHIdValues = 0;
2342 for (auto *I : *LRI)
2343 for (auto *V : PHIOperands[I]) {
2344 if (!InstructionsToSink.contains(V))
2345 ++NumPHIdValues;
2346 // FIXME: this check is overly optimistic. We may end up not sinking
2347 // said instruction, due to the very same profitability check.
2348 // See @creating_too_many_phis in sink-common-code.ll.
2349 }
2350 LLVM_DEBUG(dbgs() << "SINK: #phid values: " << NumPHIdValues << "\n");
2351 unsigned NumPHIInsts = NumPHIdValues / UnconditionalPreds.size();
2352 if ((NumPHIdValues % UnconditionalPreds.size()) != 0)
2353 NumPHIInsts++;
2354
2355 return NumPHIInsts <= 1;
2356 };
2357
2358 // We've determined that we are going to sink last ScanIdx instructions,
2359 // and recorded them in InstructionsToSink. Now, some instructions may be
2360 // unprofitable to sink. But that determination depends on the instructions
2361 // that we are going to sink.
2362
2363 // First, forward scan: find the first instruction unprofitable to sink,
2364 // recording all the ones that are profitable to sink.
2365 // FIXME: would it be better, after we detect that not all are profitable.
2366 // to either record the profitable ones, or erase the unprofitable ones?
2367 // Maybe we need to choose (at runtime) the one that will touch least
2368 // instrs?
2369 LRI.reset();
2370 int Idx = 0;
2371 SmallPtrSet<Value *, 4> InstructionsProfitableToSink;
2372 while (Idx < ScanIdx) {
2373 if (!ProfitableToSinkInstruction(LRI)) {
2374 // Too many PHIs would be created.
2375 LLVM_DEBUG(
2376 dbgs() << "SINK: stopping here, too many PHIs would be created!\n");
2377 break;
2378 }
2379 InstructionsProfitableToSink.insert((*LRI).begin(), (*LRI).end());
2380 --LRI;
2381 ++Idx;
2382 }
2383
2384 // If no instructions can be sunk, early-return.
2385 if (Idx == 0)
2386 return false;
2387
2388 // Did we determine that (only) some instructions are unprofitable to sink?
2389 if (Idx < ScanIdx) {
2390 // Okay, some instructions are unprofitable.
2391 ScanIdx = Idx;
2392 InstructionsToSink = InstructionsProfitableToSink;
2393
2394 // But, that may make other instructions unprofitable, too.
2395 // So, do a backward scan, do any earlier instructions become
2396 // unprofitable?
2397 assert(
2398 !ProfitableToSinkInstruction(LRI) &&
2399 "We already know that the last instruction is unprofitable to sink");
2400 ++LRI;
2401 --Idx;
2402 while (Idx >= 0) {
2403 // If we detect that an instruction becomes unprofitable to sink,
2404 // all earlier instructions won't be sunk either,
2405 // so preemptively keep InstructionsProfitableToSink in sync.
2406 // FIXME: is this the most performant approach?
2407 for (auto *I : *LRI)
2408 InstructionsProfitableToSink.erase(I);
2409 if (!ProfitableToSinkInstruction(LRI)) {
2410 // Everything starting with this instruction won't be sunk.
2411 ScanIdx = Idx;
2412 InstructionsToSink = InstructionsProfitableToSink;
2413 }
2414 ++LRI;
2415 --Idx;
2416 }
2417 }
2418
2419 // If no instructions can be sunk, early-return.
2420 if (ScanIdx == 0)
2421 return false;
2422 }
2423
2424 bool Changed = false;
2425
2426 if (HaveNonUnconditionalPredecessors) {
2427 if (!followedByDeoptOrUnreachable) {
2428 // It is always legal to sink common instructions from unconditional
2429 // predecessors. However, if not all predecessors are unconditional,
2430 // this transformation might be pessimizing. So as a rule of thumb,
2431 // don't do it unless we'd sink at least one non-speculatable instruction.
2432 // See https://bugs.llvm.org/show_bug.cgi?id=30244
2433 LRI.reset();
2434 int Idx = 0;
2435 bool Profitable = false;
2436 while (Idx < ScanIdx) {
2437 if (!isSafeToSpeculativelyExecute((*LRI)[0])) {
2438 Profitable = true;
2439 break;
2440 }
2441 --LRI;
2442 ++Idx;
2443 }
2444 if (!Profitable)
2445 return false;
2446 }
2447
2448 LLVM_DEBUG(dbgs() << "SINK: Splitting edge\n");
2449 // We have a conditional edge and we're going to sink some instructions.
2450 // Insert a new block postdominating all blocks we're going to sink from.
2451 if (!SplitBlockPredecessors(BB, UnconditionalPreds, ".sink.split", DTU))
2452 // Edges couldn't be split.
2453 return false;
2454 Changed = true;
2455 }
2456
2457 // Now that we've analyzed all potential sinking candidates, perform the
2458 // actual sink. We iteratively sink the last non-terminator of the source
2459 // blocks into their common successor unless doing so would require too
2460 // many PHI instructions to be generated (currently only one PHI is allowed
2461 // per sunk instruction).
2462 //
2463 // We can use InstructionsToSink to discount values needing PHI-merging that will
2464 // actually be sunk in a later iteration. This allows us to be more
2465 // aggressive in what we sink. This does allow a false positive where we
2466 // sink presuming a later value will also be sunk, but stop half way through
2467 // and never actually sink it which means we produce more PHIs than intended.
2468 // This is unlikely in practice though.
2469 int SinkIdx = 0;
2470 for (; SinkIdx != ScanIdx; ++SinkIdx) {
2471 LLVM_DEBUG(dbgs() << "SINK: Sink: "
2472 << *UnconditionalPreds[0]->getTerminator()->getPrevNode()
2473 << "\n");
2474
2475 // Because we've sunk every instruction in turn, the current instruction to
2476 // sink is always at index 0.
2477 LRI.reset();
2478
2479 if (!sinkLastInstruction(UnconditionalPreds)) {
2480 LLVM_DEBUG(
2481 dbgs()
2482 << "SINK: stopping here, failed to actually sink instruction!\n");
2483 break;
2484 }
2485
2486 NumSinkCommonInstrs++;
2487 Changed = true;
2488 }
2489 if (SinkIdx != 0)
2490 ++NumSinkCommonCode;
2491 return Changed;
2492}
2493
2494namespace {
2495
2496struct CompatibleSets {
2497 using SetTy = SmallVector<InvokeInst *, 2>;
2498
2500
2501 static bool shouldBelongToSameSet(ArrayRef<InvokeInst *> Invokes);
2502
2503 SetTy &getCompatibleSet(InvokeInst *II);
2504
2505 void insert(InvokeInst *II);
2506};
2507
2508CompatibleSets::SetTy &CompatibleSets::getCompatibleSet(InvokeInst *II) {
2509 // Perform a linear scan over all the existing sets, see if the new `invoke`
2510 // is compatible with any particular set. Since we know that all the `invokes`
2511 // within a set are compatible, only check the first `invoke` in each set.
2512 // WARNING: at worst, this has quadratic complexity.
2513 for (CompatibleSets::SetTy &Set : Sets) {
2514 if (CompatibleSets::shouldBelongToSameSet({Set.front(), II}))
2515 return Set;
2516 }
2517
2518 // Otherwise, we either had no sets yet, or this invoke forms a new set.
2519 return Sets.emplace_back();
2520}
2521
2522void CompatibleSets::insert(InvokeInst *II) {
2523 getCompatibleSet(II).emplace_back(II);
2524}
2525
2526bool CompatibleSets::shouldBelongToSameSet(ArrayRef<InvokeInst *> Invokes) {
2527 assert(Invokes.size() == 2 && "Always called with exactly two candidates.");
2528
2529 // Can we theoretically merge these `invoke`s?
2530 auto IsIllegalToMerge = [](InvokeInst *II) {
2531 return II->cannotMerge() || II->isInlineAsm();
2532 };
2533 if (any_of(Invokes, IsIllegalToMerge))
2534 return false;
2535
2536 // Either both `invoke`s must be direct,
2537 // or both `invoke`s must be indirect.
2538 auto IsIndirectCall = [](InvokeInst *II) { return II->isIndirectCall(); };
2539 bool HaveIndirectCalls = any_of(Invokes, IsIndirectCall);
2540 bool AllCallsAreIndirect = all_of(Invokes, IsIndirectCall);
2541 if (HaveIndirectCalls) {
2542 if (!AllCallsAreIndirect)
2543 return false;
2544 } else {
2545 // All callees must be identical.
2546 Value *Callee = nullptr;
2547 for (InvokeInst *II : Invokes) {
2548 Value *CurrCallee = II->getCalledOperand();
2549 assert(CurrCallee && "There is always a called operand.");
2550 if (!Callee)
2551 Callee = CurrCallee;
2552 else if (Callee != CurrCallee)
2553 return false;
2554 }
2555 }
2556
2557 // Either both `invoke`s must not have a normal destination,
2558 // or both `invoke`s must have a normal destination,
2559 auto HasNormalDest = [](InvokeInst *II) {
2560 return !isa<UnreachableInst>(II->getNormalDest()->getFirstNonPHIOrDbg());
2561 };
2562 if (any_of(Invokes, HasNormalDest)) {
2563 // Do not merge `invoke` that does not have a normal destination with one
2564 // that does have a normal destination, even though doing so would be legal.
2565 if (!all_of(Invokes, HasNormalDest))
2566 return false;
2567
2568 // All normal destinations must be identical.
2569 BasicBlock *NormalBB = nullptr;
2570 for (InvokeInst *II : Invokes) {
2571 BasicBlock *CurrNormalBB = II->getNormalDest();
2572 assert(CurrNormalBB && "There is always a 'continue to' basic block.");
2573 if (!NormalBB)
2574 NormalBB = CurrNormalBB;
2575 else if (NormalBB != CurrNormalBB)
2576 return false;
2577 }
2578
2579 // In the normal destination, the incoming values for these two `invoke`s
2580 // must be compatible.
2581 SmallPtrSet<Value *, 16> EquivalenceSet(Invokes.begin(), Invokes.end());
2583 NormalBB, {Invokes[0]->getParent(), Invokes[1]->getParent()},
2584 &EquivalenceSet))
2585 return false;
2586 }
2587
2588#ifndef NDEBUG
2589 // All unwind destinations must be identical.
2590 // We know that because we have started from said unwind destination.
2591 BasicBlock *UnwindBB = nullptr;
2592 for (InvokeInst *II : Invokes) {
2593 BasicBlock *CurrUnwindBB = II->getUnwindDest();
2594 assert(CurrUnwindBB && "There is always an 'unwind to' basic block.");
2595 if (!UnwindBB)
2596 UnwindBB = CurrUnwindBB;
2597 else
2598 assert(UnwindBB == CurrUnwindBB && "Unexpected unwind destination.");
2599 }
2600#endif
2601
2602 // In the unwind destination, the incoming values for these two `invoke`s
2603 // must be compatible.
2605 Invokes.front()->getUnwindDest(),
2606 {Invokes[0]->getParent(), Invokes[1]->getParent()}))
2607 return false;
2608
2609 // Ignoring arguments, these `invoke`s must be identical,
2610 // including operand bundles.
2611 const InvokeInst *II0 = Invokes.front();
2612 for (auto *II : Invokes.drop_front())
2613 if (!II->isSameOperationAs(II0))
2614 return false;
2615
2616 // Can we theoretically form the data operands for the merged `invoke`?
2617 auto IsIllegalToMergeArguments = [](auto Ops) {
2618 Use &U0 = std::get<0>(Ops);
2619 Use &U1 = std::get<1>(Ops);
2620 if (U0 == U1)
2621 return false;
2622 return U0->getType()->isTokenTy() ||
2623 !canReplaceOperandWithVariable(cast<Instruction>(U0.getUser()),
2624 U0.getOperandNo());
2625 };
2626 assert(Invokes.size() == 2 && "Always called with exactly two candidates.");
2627 if (any_of(zip(Invokes[0]->data_ops(), Invokes[1]->data_ops()),
2628 IsIllegalToMergeArguments))
2629 return false;
2630
2631 return true;
2632}
2633
2634} // namespace
2635
2636// Merge all invokes in the provided set, all of which are compatible
2637// as per the `CompatibleSets::shouldBelongToSameSet()`.
2639 DomTreeUpdater *DTU) {
2640 assert(Invokes.size() >= 2 && "Must have at least two invokes to merge.");
2641
2643 if (DTU)
2644 Updates.reserve(2 + 3 * Invokes.size());
2645
2646 bool HasNormalDest =
2647 !isa<UnreachableInst>(Invokes[0]->getNormalDest()->getFirstNonPHIOrDbg());
2648
2649 // Clone one of the invokes into a new basic block.
2650 // Since they are all compatible, it doesn't matter which invoke is cloned.
2651 InvokeInst *MergedInvoke = [&Invokes, HasNormalDest]() {
2652 InvokeInst *II0 = Invokes.front();
2653 BasicBlock *II0BB = II0->getParent();
2654 BasicBlock *InsertBeforeBlock =
2655 II0->getParent()->getIterator()->getNextNode();
2656 Function *Func = II0BB->getParent();
2657 LLVMContext &Ctx = II0->getContext();
2658
2659 BasicBlock *MergedInvokeBB = BasicBlock::Create(
2660 Ctx, II0BB->getName() + ".invoke", Func, InsertBeforeBlock);
2661
2662 auto *MergedInvoke = cast<InvokeInst>(II0->clone());
2663 // NOTE: all invokes have the same attributes, so no handling needed.
2664 MergedInvoke->insertInto(MergedInvokeBB, MergedInvokeBB->end());
2665
2666 if (!HasNormalDest) {
2667 // This set does not have a normal destination,
2668 // so just form a new block with unreachable terminator.
2669 BasicBlock *MergedNormalDest = BasicBlock::Create(
2670 Ctx, II0BB->getName() + ".cont", Func, InsertBeforeBlock);
2671 new UnreachableInst(Ctx, MergedNormalDest);
2672 MergedInvoke->setNormalDest(MergedNormalDest);
2673 }
2674
2675 // The unwind destination, however, remainds identical for all invokes here.
2676
2677 return MergedInvoke;
2678 }();
2679
2680 if (DTU) {
2681 // Predecessor blocks that contained these invokes will now branch to
2682 // the new block that contains the merged invoke, ...
2683 for (InvokeInst *II : Invokes)
2684 Updates.push_back(
2685 {DominatorTree::Insert, II->getParent(), MergedInvoke->getParent()});
2686
2687 // ... which has the new `unreachable` block as normal destination,
2688 // or unwinds to the (same for all `invoke`s in this set) `landingpad`,
2689 for (BasicBlock *SuccBBOfMergedInvoke : successors(MergedInvoke))
2690 Updates.push_back({DominatorTree::Insert, MergedInvoke->getParent(),
2691 SuccBBOfMergedInvoke});
2692
2693 // Since predecessor blocks now unconditionally branch to a new block,
2694 // they no longer branch to their original successors.
2695 for (InvokeInst *II : Invokes)
2696 for (BasicBlock *SuccOfPredBB : successors(II->getParent()))
2697 Updates.push_back(
2698 {DominatorTree::Delete, II->getParent(), SuccOfPredBB});
2699 }
2700
2701 bool IsIndirectCall = Invokes[0]->isIndirectCall();
2702
2703 // Form the merged operands for the merged invoke.
2704 for (Use &U : MergedInvoke->operands()) {
2705 // Only PHI together the indirect callees and data operands.
2706 if (MergedInvoke->isCallee(&U)) {
2707 if (!IsIndirectCall)
2708 continue;
2709 } else if (!MergedInvoke->isDataOperand(&U))
2710 continue;
2711
2712 // Don't create trivial PHI's with all-identical incoming values.
2713 bool NeedPHI = any_of(Invokes, [&U](InvokeInst *II) {
2714 return II->getOperand(U.getOperandNo()) != U.get();
2715 });
2716 if (!NeedPHI)
2717 continue;
2718
2719 // Form a PHI out of all the data ops under this index.
2721 U->getType(), /*NumReservedValues=*/Invokes.size(), "", MergedInvoke->getIterator());
2722 for (InvokeInst *II : Invokes)
2723 PN->addIncoming(II->getOperand(U.getOperandNo()), II->getParent());
2724
2725 U.set(PN);
2726 }
2727
2728 // We've ensured that each PHI node has compatible (identical) incoming values
2729 // when coming from each of the `invoke`s in the current merge set,
2730 // so update the PHI nodes accordingly.
2731 for (BasicBlock *Succ : successors(MergedInvoke))
2732 AddPredecessorToBlock(Succ, /*NewPred=*/MergedInvoke->getParent(),
2733 /*ExistPred=*/Invokes.front()->getParent());
2734
2735 // And finally, replace the original `invoke`s with an unconditional branch
2736 // to the block with the merged `invoke`. Also, give that merged `invoke`
2737 // the merged debugloc of all the original `invoke`s.
2738 DILocation *MergedDebugLoc = nullptr;
2739 for (InvokeInst *II : Invokes) {
2740 // Compute the debug location common to all the original `invoke`s.
2741 if (!MergedDebugLoc)
2742 MergedDebugLoc = II->getDebugLoc();
2743 else
2744 MergedDebugLoc =
2745 DILocation::getMergedLocation(MergedDebugLoc, II->getDebugLoc());
2746
2747 // And replace the old `invoke` with an unconditionally branch
2748 // to the block with the merged `invoke`.
2749 for (BasicBlock *OrigSuccBB : successors(II->getParent()))
2750 OrigSuccBB->removePredecessor(II->getParent());
2751 BranchInst::Create(MergedInvoke->getParent(), II->getParent());
2752 II->replaceAllUsesWith(MergedInvoke);
2753 II->eraseFromParent();
2754 ++NumInvokesMerged;
2755 }
2756 MergedInvoke->setDebugLoc(MergedDebugLoc);
2757 ++NumInvokeSetsFormed;
2758
2759 if (DTU)
2760 DTU->applyUpdates(Updates);
2761}
2762
2763/// If this block is a `landingpad` exception handling block, categorize all
2764/// the predecessor `invoke`s into sets, with all `invoke`s in each set
2765/// being "mergeable" together, and then merge invokes in each set together.
2766///
2767/// This is a weird mix of hoisting and sinking. Visually, it goes from:
2768/// [...] [...]
2769/// | |
2770/// [invoke0] [invoke1]
2771/// / \ / \
2772/// [cont0] [landingpad] [cont1]
2773/// to:
2774/// [...] [...]
2775/// \ /
2776/// [invoke]
2777/// / \
2778/// [cont] [landingpad]
2779///
2780/// But of course we can only do that if the invokes share the `landingpad`,
2781/// edges invoke0->cont0 and invoke1->cont1 are "compatible",
2782/// and the invoked functions are "compatible".
2785 return false;
2786
2787 bool Changed = false;
2788
2789 // FIXME: generalize to all exception handling blocks?
2790 if (!BB->isLandingPad())
2791 return Changed;
2792
2793 CompatibleSets Grouper;
2794
2795 // Record all the predecessors of this `landingpad`. As per verifier,
2796 // the only allowed predecessor is the unwind edge of an `invoke`.
2797 // We want to group "compatible" `invokes` into the same set to be merged.
2798 for (BasicBlock *PredBB : predecessors(BB))
2799 Grouper.insert(cast<InvokeInst>(PredBB->getTerminator()));
2800
2801 // And now, merge `invoke`s that were grouped togeter.
2802 for (ArrayRef<InvokeInst *> Invokes : Grouper.Sets) {
2803 if (Invokes.size() < 2)
2804 continue;
2805 Changed = true;
2806 MergeCompatibleInvokesImpl(Invokes, DTU);
2807 }
2808
2809 return Changed;
2810}
2811
2812namespace {
2813/// Track ephemeral values, which should be ignored for cost-modelling
2814/// purposes. Requires walking instructions in reverse order.
2815class EphemeralValueTracker {
2817
2818 bool isEphemeral(const Instruction *I) {
2819 if (isa<AssumeInst>(I))
2820 return true;
2821 return !I->mayHaveSideEffects() && !I->isTerminator() &&
2822 all_of(I->users(), [&](const User *U) {
2823 return EphValues.count(cast<Instruction>(U));
2824 });
2825 }
2826
2827public:
2828 bool track(const Instruction *I) {
2829 if (isEphemeral(I)) {
2830 EphValues.insert(I);
2831 return true;
2832 }
2833 return false;
2834 }
2835
2836 bool contains(const Instruction *I) const { return EphValues.contains(I); }
2837};
2838} // namespace
2839
2840/// Determine if we can hoist sink a sole store instruction out of a
2841/// conditional block.
2842///
2843/// We are looking for code like the following:
2844/// BrBB:
2845/// store i32 %add, i32* %arrayidx2
2846/// ... // No other stores or function calls (we could be calling a memory
2847/// ... // function).
2848/// %cmp = icmp ult %x, %y
2849/// br i1 %cmp, label %EndBB, label %ThenBB
2850/// ThenBB:
2851/// store i32 %add5, i32* %arrayidx2
2852/// br label EndBB
2853/// EndBB:
2854/// ...
2855/// We are going to transform this into:
2856/// BrBB:
2857/// store i32 %add, i32* %arrayidx2
2858/// ... //
2859/// %cmp = icmp ult %x, %y
2860/// %add.add5 = select i1 %cmp, i32 %add, %add5
2861/// store i32 %add.add5, i32* %arrayidx2
2862/// ...
2863///
2864/// \return The pointer to the value of the previous store if the store can be
2865/// hoisted into the predecessor block. 0 otherwise.
2867 BasicBlock *StoreBB, BasicBlock *EndBB) {
2868 StoreInst *StoreToHoist = dyn_cast<StoreInst>(I);
2869 if (!StoreToHoist)
2870 return nullptr;
2871
2872 // Volatile or atomic.
2873 if (!StoreToHoist->isSimple())
2874 return nullptr;
2875
2876 Value *StorePtr = StoreToHoist->getPointerOperand();
2877 Type *StoreTy = StoreToHoist->getValueOperand()->getType();
2878
2879 // Look for a store to the same pointer in BrBB.
2880 unsigned MaxNumInstToLookAt = 9;
2881 // Skip pseudo probe intrinsic calls which are not really killing any memory
2882 // accesses.
2883 for (Instruction &CurI : reverse(BrBB->instructionsWithoutDebug(true))) {
2884 if (!MaxNumInstToLookAt)
2885 break;
2886 --MaxNumInstToLookAt;
2887
2888 // Could be calling an instruction that affects memory like free().
2889 if (CurI.mayWriteToMemory() && !isa<StoreInst>(CurI))
2890 return nullptr;
2891
2892 if (auto *SI = dyn_cast<StoreInst>(&CurI)) {
2893 // Found the previous store to same location and type. Make sure it is
2894 // simple, to avoid introducing a spurious non-atomic write after an
2895 // atomic write.
2896 if (SI->getPointerOperand() == StorePtr &&
2897 SI->getValueOperand()->getType() == StoreTy && SI->isSimple() &&
2898 SI->getAlign() >= StoreToHoist->getAlign())
2899 // Found the previous store, return its value operand.
2900 return SI->getValueOperand();
2901 return nullptr; // Unknown store.
2902 }
2903
2904 if (auto *LI = dyn_cast<LoadInst>(&CurI)) {
2905 if (LI->getPointerOperand() == StorePtr && LI->getType() == StoreTy &&
2906 LI->isSimple() && LI->getAlign() >= StoreToHoist->getAlign()) {
2907 // Local objects (created by an `alloca` instruction) are always
2908 // writable, so once we are past a read from a location it is valid to
2909 // also write to that same location.
2910 // If the address of the local object never escapes the function, that
2911 // means it's never concurrently read or written, hence moving the store
2912 // from under the condition will not introduce a data race.
2913 auto *AI = dyn_cast<AllocaInst>(getUnderlyingObject(StorePtr));
2914 if (AI && !PointerMayBeCaptured(AI, false, true))
2915 // Found a previous load, return it.
2916 return LI;
2917 }
2918 // The load didn't work out, but we may still find a store.
2919 }
2920 }
2921
2922 return nullptr;
2923}
2924
2925/// Estimate the cost of the insertion(s) and check that the PHI nodes can be
2926/// converted to selects.
2928 BasicBlock *EndBB,
2929 unsigned &SpeculatedInstructions,
2931 const TargetTransformInfo &TTI) {
2933 BB->getParent()->hasMinSize()
2936
2937 bool HaveRewritablePHIs = false;
2938 for (PHINode &PN : EndBB->phis()) {
2939 Value *OrigV = PN.getIncomingValueForBlock(BB);
2940 Value *ThenV = PN.getIncomingValueForBlock(ThenBB);
2941
2942 // FIXME: Try to remove some of the duplication with
2943 // hoistCommonCodeFromSuccessors. Skip PHIs which are trivial.
2944 if (ThenV == OrigV)
2945 continue;
2946
2947 Cost += TTI.getCmpSelInstrCost(Instruction::Select, PN.getType(), nullptr,
2949
2950 // Don't convert to selects if we could remove undefined behavior instead.
2951 if (passingValueIsAlwaysUndefined(OrigV, &PN) ||
2953 return false;
2954
2955 HaveRewritablePHIs = true;
2956 ConstantExpr *OrigCE = dyn_cast<ConstantExpr>(OrigV);
2957 ConstantExpr *ThenCE = dyn_cast<ConstantExpr>(ThenV);
2958 if (!OrigCE && !ThenCE)
2959 continue; // Known cheap (FIXME: Maybe not true for aggregates).
2960
2961 InstructionCost OrigCost = OrigCE ? computeSpeculationCost(OrigCE, TTI) : 0;
2962 InstructionCost ThenCost = ThenCE ? computeSpeculationCost(ThenCE, TTI) : 0;
2963 InstructionCost MaxCost =
2965 if (OrigCost + ThenCost > MaxCost)
2966 return false;
2967
2968 // Account for the cost of an unfolded ConstantExpr which could end up
2969 // getting expanded into Instructions.
2970 // FIXME: This doesn't account for how many operations are combined in the
2971 // constant expression.
2972 ++SpeculatedInstructions;
2973 if (SpeculatedInstructions > 1)
2974 return false;
2975 }
2976
2977 return HaveRewritablePHIs;
2978}
2979
2980/// Speculate a conditional basic block flattening the CFG.
2981///
2982/// Note that this is a very risky transform currently. Speculating
2983/// instructions like this is most often not desirable. Instead, there is an MI
2984/// pass which can do it with full awareness of the resource constraints.
2985/// However, some cases are "obvious" and we should do directly. An example of
2986/// this is speculating a single, reasonably cheap instruction.
2987///
2988/// There is only one distinct advantage to flattening the CFG at the IR level:
2989/// it makes very common but simplistic optimizations such as are common in
2990/// instcombine and the DAG combiner more powerful by removing CFG edges and
2991/// modeling their effects with easier to reason about SSA value graphs.
2992///
2993///
2994/// An illustration of this transform is turning this IR:
2995/// \code
2996/// BB:
2997/// %cmp = icmp ult %x, %y
2998/// br i1 %cmp, label %EndBB, label %ThenBB
2999/// ThenBB:
3000/// %sub = sub %x, %y
3001/// br label BB2
3002/// EndBB:
3003/// %phi = phi [ %sub, %ThenBB ], [ 0, %EndBB ]
3004/// ...
3005/// \endcode
3006///
3007/// Into this IR:
3008/// \code
3009/// BB:
3010/// %cmp = icmp ult %x, %y
3011/// %sub = sub %x, %y
3012/// %cond = select i1 %cmp, 0, %sub
3013/// ...
3014/// \endcode
3015///
3016/// \returns true if the conditional block is removed.
3017bool SimplifyCFGOpt::SpeculativelyExecuteBB(BranchInst *BI,
3018 BasicBlock *ThenBB) {
3019 if (!Options.SpeculateBlocks)
3020 return false;
3021
3022 // Be conservative for now. FP select instruction can often be expensive.
3023 Value *BrCond = BI->getCondition();
3024 if (isa<FCmpInst>(BrCond))
3025 return false;
3026
3027 BasicBlock *BB = BI->getParent();
3028 BasicBlock *EndBB = ThenBB->getTerminator()->getSuccessor(0);
3029 InstructionCost Budget =
3031
3032 // If ThenBB is actually on the false edge of the conditional branch, remember
3033 // to swap the select operands later.
3034 bool Invert = false;
3035 if (ThenBB != BI->getSuccessor(0)) {
3036 assert(ThenBB == BI->getSuccessor(1) && "No edge from 'if' block?");
3037 Invert = true;
3038 }
3039 assert(EndBB == BI->getSuccessor(!Invert) && "No edge from to end block");
3040
3041 // If the branch is non-unpredictable, and is predicted to *not* branch to
3042 // the `then` block, then avoid speculating it.
3043 if (!BI->getMetadata(LLVMContext::MD_unpredictable)) {
3044 uint64_t TWeight, FWeight;
3045 if (extractBranchWeights(*BI, TWeight, FWeight) &&
3046 (TWeight + FWeight) != 0) {
3047 uint64_t EndWeight = Invert ? TWeight : FWeight;
3048 BranchProbability BIEndProb =
3049 BranchProbability::getBranchProbability(EndWeight, TWeight + FWeight);
3051 if (BIEndProb >= Likely)
3052 return false;
3053 }
3054 }
3055
3056 // Keep a count of how many times instructions are used within ThenBB when
3057 // they are candidates for sinking into ThenBB. Specifically:
3058 // - They are defined in BB, and
3059 // - They have no side effects, and
3060 // - All of their uses are in ThenBB.
3061 SmallDenseMap<Instruction *, unsigned, 4> SinkCandidateUseCounts;
3062
3063 SmallVector<Instruction *, 4> SpeculatedDbgIntrinsics;
3064
3065 unsigned SpeculatedInstructions = 0;
3066 Value *SpeculatedStoreValue = nullptr;
3067 StoreInst *SpeculatedStore = nullptr;
3068 EphemeralValueTracker EphTracker;
3069 for (Instruction &I : reverse(drop_end(*ThenBB))) {
3070 // Skip debug info.
3071 if (isa<DbgInfoIntrinsic>(I)) {
3072 SpeculatedDbgIntrinsics.push_back(&I);
3073 continue;
3074 }
3075
3076 // Skip pseudo probes. The consequence is we lose track of the branch
3077 // probability for ThenBB, which is fine since the optimization here takes
3078 // place regardless of the branch probability.
3079 if (isa<PseudoProbeInst>(I)) {
3080 // The probe should be deleted so that it will not be over-counted when
3081 // the samples collected on the non-conditional path are counted towards
3082 // the conditional path. We leave it for the counts inference algorithm to
3083 // figure out a proper count for an unknown probe.
3084 SpeculatedDbgIntrinsics.push_back(&I);
3085 continue;
3086 }
3087
3088 // Ignore ephemeral values, they will be dropped by the transform.
3089 if (EphTracker.track(&I))
3090 continue;
3091
3092 // Only speculatively execute a single instruction (not counting the
3093 // terminator) for now.
3094 ++SpeculatedInstructions;
3095 if (SpeculatedInstructions > 1)
3096 return false;
3097
3098 // Don't hoist the instruction if it's unsafe or expensive.
3100 !(HoistCondStores && (SpeculatedStoreValue = isSafeToSpeculateStore(
3101 &I, BB, ThenBB, EndBB))))
3102 return false;
3103 if (!SpeculatedStoreValue &&
3106 return false;
3107
3108 // Store the store speculation candidate.
3109 if (SpeculatedStoreValue)
3110 SpeculatedStore = cast<StoreInst>(&I);
3111
3112 // Do not hoist the instruction if any of its operands are defined but not
3113 // used in BB. The transformation will prevent the operand from
3114 // being sunk into the use block.
3115 for (Use &Op : I.operands()) {
3116 Instruction *OpI = dyn_cast<Instruction>(Op);
3117 if (!OpI || OpI->getParent() != BB || OpI->mayHaveSideEffects())
3118 continue; // Not a candidate for sinking.
3119
3120 ++SinkCandidateUseCounts[OpI];
3121 }
3122 }
3123
3124 // Consider any sink candidates which are only used in ThenBB as costs for
3125 // speculation. Note, while we iterate over a DenseMap here, we are summing
3126 // and so iteration order isn't significant.
3127 for (const auto &[Inst, Count] : SinkCandidateUseCounts)
3128 if (Inst->hasNUses(Count)) {
3129 ++SpeculatedInstructions;
3130 if (SpeculatedInstructions > 1)
3131 return false;
3132 }
3133
3134 // Check that we can insert the selects and that it's not too expensive to do
3135 // so.
3136 bool Convert = SpeculatedStore != nullptr;
3138 Convert |= validateAndCostRequiredSelects(BB, ThenBB, EndBB,
3139 SpeculatedInstructions,
3140 Cost, TTI);
3141 if (!Convert || Cost > Budget)
3142 return false;
3143
3144 // If we get here, we can hoist the instruction and if-convert.
3145 LLVM_DEBUG(dbgs() << "SPECULATIVELY EXECUTING BB" << *ThenBB << "\n";);
3146
3147 // Insert a select of the value of the speculated store.
3148 if (SpeculatedStoreValue) {
3149 IRBuilder<NoFolder> Builder(BI);
3150 Value *OrigV = SpeculatedStore->getValueOperand();
3151 Value *TrueV = SpeculatedStore->getValueOperand();
3152 Value *FalseV = SpeculatedStoreValue;
3153 if (Invert)
3154 std::swap(TrueV, FalseV);
3155 Value *S = Builder.CreateSelect(
3156 BrCond, TrueV, FalseV, "spec.store.select", BI);
3157 SpeculatedStore->setOperand(0, S);
3158 SpeculatedStore->applyMergedLocation(BI->getDebugLoc(),
3159 SpeculatedStore->getDebugLoc());
3160 // The value stored is still conditional, but the store itself is now
3161 // unconditonally executed, so we must be sure that any linked dbg.assign
3162 // intrinsics are tracking the new stored value (the result of the
3163 // select). If we don't, and the store were to be removed by another pass
3164 // (e.g. DSE), then we'd eventually end up emitting a location describing
3165 // the conditional value, unconditionally.
3166 //
3167 // === Before this transformation ===
3168 // pred:
3169 // store %one, %x.dest, !DIAssignID !1
3170 // dbg.assign %one, "x", ..., !1, ...
3171 // br %cond if.then
3172 //
3173 // if.then:
3174 // store %two, %x.dest, !DIAssignID !2
3175 // dbg.assign %two, "x", ..., !2, ...
3176 //
3177 // === After this transformation ===
3178 // pred:
3179 // store %one, %x.dest, !DIAssignID !1
3180 // dbg.assign %one, "x", ..., !1
3181 /// ...
3182 // %merge = select %cond, %two, %one
3183 // store %merge, %x.dest, !DIAssignID !2
3184 // dbg.assign %merge, "x", ..., !2
3185 auto replaceVariable = [OrigV, S](auto *DbgAssign) {
3186 if (llvm::is_contained(DbgAssign->location_ops(), OrigV))
3187 DbgAssign->replaceVariableLocationOp(OrigV, S);
3188 };
3189 for_each(at::getAssignmentMarkers(SpeculatedStore), replaceVariable);
3190 for_each(at::getDVRAssignmentMarkers(SpeculatedStore), replaceVariable);
3191 }
3192
3193 // Metadata can be dependent on the condition we are hoisting above.
3194 // Strip all UB-implying metadata on the instruction. Drop the debug loc
3195 // to avoid making it appear as if the condition is a constant, which would
3196 // be misleading while debugging.
3197 // Similarly strip attributes that maybe dependent on condition we are
3198 // hoisting above.
3199 for (auto &I : make_early_inc_range(*ThenBB)) {
3200 if (!SpeculatedStoreValue || &I != SpeculatedStore) {
3201 // Don't update the DILocation of dbg.assign intrinsics.
3202 if (!isa<DbgAssignIntrinsic>(&I))
3203 I.setDebugLoc(DebugLoc());
3204 }
3205 I.dropUBImplyingAttrsAndMetadata();
3206
3207 // Drop ephemeral values.
3208 if (EphTracker.contains(&I)) {
3209 I.replaceAllUsesWith(PoisonValue::get(I.getType()));
3210 I.eraseFromParent();
3211 }
3212 }
3213
3214 // Hoist the instructions.
3215 // In "RemoveDIs" non-instr debug-info mode, drop DbgVariableRecords attached
3216 // to these instructions, in the same way that dbg.value intrinsics are
3217 // dropped at the end of this block.
3218 for (auto &It : make_range(ThenBB->begin(), ThenBB->end()))
3219 for (DbgRecord &DR : make_early_inc_range(It.getDbgRecordRange()))
3220 // Drop all records except assign-kind DbgVariableRecords (dbg.assign
3221 // equivalent).
3222 if (DbgVariableRecord *DVR = dyn_cast<DbgVariableRecord>(&DR);
3223 !DVR || !DVR->isDbgAssign())
3224 It.dropOneDbgRecord(&DR);
3225 BB->splice(BI->getIterator(), ThenBB, ThenBB->begin(),
3226 std::prev(ThenBB->end()));
3227
3228 // Insert selects and rewrite the PHI operands.
3229 IRBuilder<NoFolder> Builder(BI);
3230 for (PHINode &PN : EndBB->phis()) {
3231 unsigned OrigI = PN.getBasicBlockIndex(BB);
3232 unsigned ThenI = PN.getBasicBlockIndex(ThenBB);
3233 Value *OrigV = PN.getIncomingValue(OrigI);
3234 Value *ThenV = PN.getIncomingValue(ThenI);
3235
3236 // Skip PHIs which are trivial.
3237 if (OrigV == ThenV)
3238 continue;
3239
3240 // Create a select whose true value is the speculatively executed value and
3241 // false value is the pre-existing value. Swap them if the branch
3242 // destinations were inverted.
3243 Value *TrueV = ThenV, *FalseV = OrigV;
3244 if (Invert)
3245 std::swap(TrueV, FalseV);
3246 Value *V = Builder.CreateSelect(BrCond, TrueV, FalseV, "spec.select", BI);
3247 PN.setIncomingValue(OrigI, V);
3248 PN.setIncomingValue(ThenI, V);
3249 }
3250
3251 // Remove speculated dbg intrinsics.
3252 // FIXME: Is it possible to do this in a more elegant way? Moving/merging the
3253 // dbg value for the different flows and inserting it after the select.
3254 for (Instruction *I : SpeculatedDbgIntrinsics) {
3255 // We still want to know that an assignment took place so don't remove
3256 // dbg.assign intrinsics.
3257 if (!isa<DbgAssignIntrinsic>(I))
3258 I->eraseFromParent();
3259 }
3260
3261 ++NumSpeculations;
3262 return true;
3263}
3264
3265/// Return true if we can thread a branch across this block.
3267 int Size = 0;
3268 EphemeralValueTracker EphTracker;
3269
3270 // Walk the loop in reverse so that we can identify ephemeral values properly
3271 // (values only feeding assumes).
3272 for (Instruction &I : reverse(BB->instructionsWithoutDebug(false))) {
3273 // Can't fold blocks that contain noduplicate or convergent calls.
3274 if (CallInst *CI = dyn_cast<CallInst>(&I))
3275 if (CI->cannotDuplicate() || CI->isConvergent())
3276 return false;
3277
3278 // Ignore ephemeral values which are deleted during codegen.
3279 // We will delete Phis while threading, so Phis should not be accounted in
3280 // block's size.
3281 if (!EphTracker.track(&I) && !isa<PHINode>(I)) {
3282 if (Size++ > MaxSmallBlockSize)
3283 return false; // Don't clone large BB's.
3284 }
3285
3286 // We can only support instructions that do not define values that are
3287 // live outside of the current basic block.
3288 for (User *U : I.users()) {
3289 Instruction *UI = cast<Instruction>(U);
3290 if (UI->getParent() != BB || isa<PHINode>(UI))
3291 return false;
3292 }
3293
3294 // Looks ok, continue checking.
3295 }
3296
3297 return true;
3298}
3299
3301 BasicBlock *To) {
3302 // Don't look past the block defining the value, we might get the value from
3303 // a previous loop iteration.
3304 auto *I = dyn_cast<Instruction>(V);
3305 if (I && I->getParent() == To)
3306 return nullptr;
3307
3308 // We know the value if the From block branches on it.
3309 auto *BI = dyn_cast<BranchInst>(From->getTerminator());
3310 if (BI && BI->isConditional() && BI->getCondition() == V &&
3311 BI->getSuccessor(0) != BI->getSuccessor(1))
3312 return BI->getSuccessor(0) == To ? ConstantInt::getTrue(BI->getContext())
3314
3315 return nullptr;
3316}
3317
3318/// If we have a conditional branch on something for which we know the constant
3319/// value in predecessors (e.g. a phi node in the current block), thread edges
3320/// from the predecessor to their ultimate destination.
3321static std::optional<bool>
3323 const DataLayout &DL,
3324 AssumptionCache *AC) {
3326 BasicBlock *BB = BI->getParent();
3327 Value *Cond = BI->getCondition();
3328 PHINode *PN = dyn_cast<PHINode>(Cond);
3329 if (PN && PN->getParent() == BB) {
3330 // Degenerate case of a single entry PHI.
3331 if (PN->getNumIncomingValues() == 1) {
3333 return true;
3334 }
3335
3336 for (Use &U : PN->incoming_values())
3337 if (auto *CB = dyn_cast<ConstantInt>(U))
3338 KnownValues[CB].insert(PN->getIncomingBlock(U));
3339 } else {
3340 for (BasicBlock *Pred : predecessors(BB)) {
3341 if (ConstantInt *CB = getKnownValueOnEdge(Cond, Pred, BB))
3342 KnownValues[CB].insert(Pred);
3343 }
3344 }
3345
3346 if (KnownValues.empty())
3347 return false;
3348
3349 // Now we know that this block has multiple preds and two succs.
3350 // Check that the block is small enough and values defined in the block are
3351 // not used outside of it.
3353 return false;
3354
3355 for (const auto &Pair : KnownValues) {
3356 // Okay, we now know that all edges from PredBB should be revectored to
3357 // branch to RealDest.
3358 ConstantInt *CB = Pair.first;
3359 ArrayRef<BasicBlock *> PredBBs = Pair.second.getArrayRef();
3360 BasicBlock *RealDest = BI->getSuccessor(!CB->getZExtValue());
3361
3362 if (RealDest == BB)
3363 continue; // Skip self loops.
3364
3365 // Skip if the predecessor's terminator is an indirect branch.
3366 if (any_of(PredBBs, [](BasicBlock *PredBB) {
3367 return isa<IndirectBrInst>(PredBB->getTerminator());
3368 }))
3369 continue;
3370
3371 LLVM_DEBUG({
3372 dbgs() << "Condition " << *Cond << " in " << BB->getName()
3373 << " has value " << *Pair.first << " in predecessors:\n";
3374 for (const BasicBlock *PredBB : Pair.second)
3375 dbgs() << " " << PredBB->getName() << "\n";
3376 dbgs() << "Threading to destination " << RealDest->getName() << ".\n";
3377 });
3378
3379 // Split the predecessors we are threading into a new edge block. We'll
3380 // clone the instructions into this block, and then redirect it to RealDest.
3381 BasicBlock *EdgeBB = SplitBlockPredecessors(BB, PredBBs, ".critedge", DTU);
3382
3383 // TODO: These just exist to reduce test diff, we can drop them if we like.
3384 EdgeBB->setName(RealDest->getName() + ".critedge");
3385 EdgeBB->moveBefore(RealDest);
3386
3387 // Update PHI nodes.
3388 AddPredecessorToBlock(RealDest, EdgeBB, BB);
3389
3390 // BB may have instructions that are being threaded over. Clone these
3391 // instructions into EdgeBB. We know that there will be no uses of the
3392 // cloned instructions outside of EdgeBB.
3393 BasicBlock::iterator InsertPt = EdgeBB->getFirstInsertionPt();
3394 DenseMap<Value *, Value *> TranslateMap; // Track translated values.
3395 TranslateMap[Cond] = CB;
3396
3397 // RemoveDIs: track instructions that we optimise away while folding, so
3398 // that we can copy DbgVariableRecords from them later.
3399 BasicBlock::iterator SrcDbgCursor = BB->begin();
3400 for (BasicBlock::iterator BBI = BB->begin(); &*BBI != BI; ++BBI) {
3401 if (PHINode *PN = dyn_cast<PHINode>(BBI)) {
3402 TranslateMap[PN] = PN->getIncomingValueForBlock(EdgeBB);
3403 continue;
3404 }
3405 // Clone the instruction.
3406 Instruction *N = BBI->clone();
3407 // Insert the new instruction into its new home.
3408 N->insertInto(EdgeBB, InsertPt);
3409
3410 if (BBI->hasName())
3411 N->setName(BBI->getName() + ".c");
3412
3413 // Update operands due to translation.
3414 for (Use &Op : N->operands()) {
3415 DenseMap<Value *, Value *>::iterator PI = TranslateMap.find(Op);
3416 if (PI != TranslateMap.end())
3417 Op = PI->second;
3418 }
3419
3420 // Check for trivial simplification.
3421 if (Value *V = simplifyInstruction(N, {DL, nullptr, nullptr, AC})) {
3422 if (!BBI->use_empty())
3423 TranslateMap[&*BBI] = V;
3424 if (!N->mayHaveSideEffects()) {
3425 N->eraseFromParent(); // Instruction folded away, don't need actual
3426 // inst
3427 N = nullptr;
3428 }
3429 } else {
3430 if (!BBI->use_empty())
3431 TranslateMap[&*BBI] = N;
3432 }
3433 if (N) {
3434 // Copy all debug-info attached to instructions from the last we
3435 // successfully clone, up to this instruction (they might have been
3436 // folded away).
3437 for (; SrcDbgCursor != BBI; ++SrcDbgCursor)
3438 N->cloneDebugInfoFrom(&*SrcDbgCursor);
3439 SrcDbgCursor = std::next(BBI);
3440 // Clone debug-info on this instruction too.
3441 N->cloneDebugInfoFrom(&*BBI);
3442
3443 // Register the new instruction with the assumption cache if necessary.
3444 if (auto *Assume = dyn_cast<AssumeInst>(N))
3445 if (AC)
3446 AC->registerAssumption(Assume);
3447 }
3448 }
3449
3450 for (; &*SrcDbgCursor != BI; ++SrcDbgCursor)
3451 InsertPt->cloneDebugInfoFrom(&*SrcDbgCursor);
3452 InsertPt->cloneDebugInfoFrom(BI);
3453
3454 BB->removePredecessor(EdgeBB);
3455 BranchInst *EdgeBI = cast<BranchInst>(EdgeBB->getTerminator());
3456 EdgeBI->setSuccessor(0, RealDest);
3457 EdgeBI->setDebugLoc(BI->getDebugLoc());
3458
3459 if (DTU) {
3461 Updates.push_back({DominatorTree::Delete, EdgeBB, BB});
3462 Updates.push_back({DominatorTree::Insert, EdgeBB, RealDest});
3463 DTU->applyUpdates(Updates);
3464 }
3465
3466 // For simplicity, we created a separate basic block for the edge. Merge
3467 // it back into the predecessor if possible. This not only avoids
3468 // unnecessary SimplifyCFG iterations, but also makes sure that we don't
3469 // bypass the check for trivial cycles above.
3470 MergeBlockIntoPredecessor(EdgeBB, DTU);
3471
3472 // Signal repeat, simplifying any other constants.
3473 return std::nullopt;
3474 }
3475
3476 return false;
3477}
3478
3480 DomTreeUpdater *DTU,
3481 const DataLayout &DL,
3482 AssumptionCache *AC) {
3483 std::optional<bool> Result;
3484 bool EverChanged = false;
3485 do {
3486 // Note that None means "we changed things, but recurse further."
3487 Result = FoldCondBranchOnValueKnownInPredecessorImpl(BI, DTU, DL, AC);
3488 EverChanged |= Result == std::nullopt || *Result;
3489 } while (Result == std::nullopt);
3490 return EverChanged;
3491}
3492
3493/// Given a BB that starts with the specified two-entry PHI node,
3494/// see if we can eliminate it.
3496 DomTreeUpdater *DTU, const DataLayout &DL) {
3497 // Ok, this is a two entry PHI node. Check to see if this is a simple "if
3498 // statement", which has a very simple dominance structure. Basically, we
3499 // are trying to find the condition that is being branched on, which
3500 // subsequently causes this merge to happen. We really want control
3501 // dependence information for this check, but simplifycfg can't keep it up
3502 // to date, and this catches most of the cases we care about anyway.
3503 BasicBlock *BB = PN->getParent();
3504
3505 BasicBlock *IfTrue, *IfFalse;
3506 BranchInst *DomBI = GetIfCondition(BB, IfTrue, IfFalse);
3507 if (!DomBI)
3508 return false;
3509 Value *IfCond = DomBI->getCondition();
3510 // Don't bother if the branch will be constant folded trivially.
3511 if (isa<ConstantInt>(IfCond))
3512 return false;
3513
3514 BasicBlock *DomBlock = DomBI->getParent();
3517 PN->blocks(), std::back_inserter(IfBlocks), [](BasicBlock *IfBlock) {
3518 return cast<BranchInst>(IfBlock->getTerminator())->isUnconditional();
3519 });
3520 assert((IfBlocks.size() == 1 || IfBlocks.size() == 2) &&
3521 "Will have either one or two blocks to speculate.");
3522
3523 // If the branch is non-unpredictable, see if we either predictably jump to
3524 // the merge bb (if we have only a single 'then' block), or if we predictably
3525 // jump to one specific 'then' block (if we have two of them).
3526 // It isn't beneficial to speculatively execute the code
3527 // from the block that we know is predictably not entered.
3528 if (!DomBI->getMetadata(LLVMContext::MD_unpredictable)) {
3529 uint64_t TWeight, FWeight;
3530 if (extractBranchWeights(*DomBI, TWeight, FWeight) &&
3531 (TWeight + FWeight) != 0) {
3532 BranchProbability BITrueProb =
3533 BranchProbability::getBranchProbability(TWeight, TWeight + FWeight);
3535 BranchProbability BIFalseProb = BITrueProb.getCompl();
3536 if (IfBlocks.size() == 1) {
3537 BranchProbability BIBBProb =
3538 DomBI->getSuccessor(0) == BB ? BITrueProb : BIFalseProb;
3539 if (BIBBProb >= Likely)
3540 return false;
3541 } else {
3542 if (BITrueProb >= Likely || BIFalseProb >= Likely)
3543 return false;
3544 }
3545 }
3546 }
3547
3548 // Don't try to fold an unreachable block. For example, the phi node itself
3549 // can't be the candidate if-condition for a select that we want to form.
3550 if (auto *IfCondPhiInst = dyn_cast<PHINode>(IfCond))
3551 if (IfCondPhiInst->getParent() == BB)
3552 return false;
3553
3554 // Okay, we found that we can merge this two-entry phi node into a select.
3555 // Doing so would require us to fold *all* two entry phi nodes in this block.
3556 // At some point this becomes non-profitable (particularly if the target
3557 // doesn't support cmov's). Only do this transformation if there are two or
3558 // fewer PHI nodes in this block.
3559 unsigned NumPhis = 0;
3560 for (BasicBlock::iterator I = BB->begin(); isa<PHINode>(I); ++NumPhis, ++I)
3561 if (NumPhis > 2)
3562 return false;
3563
3564 // Loop over the PHI's seeing if we can promote them all to select
3565 // instructions. While we are at it, keep track of the instructions
3566 // that need to be moved to the dominating block.
3567 SmallPtrSet<Instruction *, 4> AggressiveInsts;
3569 InstructionCost Budget =
3571
3572 bool Changed = false;
3573 for (BasicBlock::iterator II = BB->begin(); isa<PHINode>(II);) {
3574 PHINode *PN = cast<PHINode>(II++);
3575 if (Value *V = simplifyInstruction(PN, {DL, PN})) {
3576 PN->replaceAllUsesWith(V);
3577 PN->eraseFromParent();
3578 Changed = true;
3579 continue;
3580 }
3581
3582 if (!dominatesMergePoint(PN->getIncomingValue(0), BB, AggressiveInsts,
3583 Cost, Budget, TTI) ||
3584 !dominatesMergePoint(PN->getIncomingValue(1), BB, AggressiveInsts,
3585 Cost, Budget, TTI))
3586 return Changed;
3587 }
3588
3589 // If we folded the first phi, PN dangles at this point. Refresh it. If
3590 // we ran out of PHIs then we simplified them all.
3591 PN = dyn_cast<PHINode>(BB->begin());
3592 if (!PN)
3593 return true;
3594
3595 // Return true if at least one of these is a 'not', and another is either
3596 // a 'not' too, or a constant.
3597 auto CanHoistNotFromBothValues = [](Value *V0, Value *V1) {
3598 if (!match(V0, m_Not(m_Value())))
3599 std::swap(V0, V1);
3600 auto Invertible = m_CombineOr(m_Not(m_Value()), m_AnyIntegralConstant());
3601 return match(V0, m_Not(m_Value())) && match(V1, Invertible);
3602 };
3603
3604 // Don't fold i1 branches on PHIs which contain binary operators or
3605 // (possibly inverted) select form of or/ands, unless one of
3606 // the incoming values is an 'not' and another one is freely invertible.
3607 // These can often be turned into switches and other things.
3608 auto IsBinOpOrAnd = [](Value *V) {
3609 return match(
3610 V, m_CombineOr(
3611 m_BinOp(),
3614 };
3615 if (PN->getType()->isIntegerTy(1) &&
3616 (IsBinOpOrAnd(PN->getIncomingValue(0)) ||
3617 IsBinOpOrAnd(PN->getIncomingValue(1)) || IsBinOpOrAnd(IfCond)) &&
3618 !CanHoistNotFromBothValues(PN->getIncomingValue(0),
3619 PN->getIncomingValue(1)))
3620 return Changed;
3621
3622 // If all PHI nodes are promotable, check to make sure that all instructions
3623 // in the predecessor blocks can be promoted as well. If not, we won't be able
3624 // to get rid of the control flow, so it's not worth promoting to select
3625 // instructions.
3626 for (BasicBlock *IfBlock : IfBlocks)
3627 for (BasicBlock::iterator I = IfBlock->begin(); !I->isTerminator(); ++I)
3628 if (!AggressiveInsts.count(&*I) && !I->isDebugOrPseudoInst()) {
3629 // This is not an aggressive instruction that we can promote.
3630 // Because of this, we won't be able to get rid of the control flow, so
3631 // the xform is not worth it.
3632 return Changed;
3633 }
3634
3635 // If either of the blocks has it's address taken, we can't do this fold.
3636 if (any_of(IfBlocks,
3637 [](BasicBlock *IfBlock) { return IfBlock->hasAddressTaken(); }))
3638 return Changed;
3639
3640 LLVM_DEBUG(dbgs() << "FOUND IF CONDITION! " << *IfCond
3641 << " T: " << IfTrue->getName()
3642 << " F: " << IfFalse->getName() << "\n");
3643
3644 // If we can still promote the PHI nodes after this gauntlet of tests,
3645 // do all of the PHI's now.
3646
3647 // Move all 'aggressive' instructions, which are defined in the
3648 // conditional parts of the if's up to the dominating block.
3649 for (BasicBlock *IfBlock : IfBlocks)
3650 hoistAllInstructionsInto(DomBlock, DomBI, IfBlock);
3651
3652 IRBuilder<NoFolder> Builder(DomBI);
3653 // Propagate fast-math-flags from phi nodes to replacement selects.
3654 IRBuilder<>::FastMathFlagGuard FMFGuard(Builder);
3655 while (PHINode *PN = dyn_cast<PHINode>(BB->begin())) {
3656 if (isa<FPMathOperator>(PN))
3657 Builder.setFastMathFlags(PN->getFastMathFlags());
3658
3659 // Change the PHI node into a select instruction.
3660 Value *TrueVal = PN->getIncomingValueForBlock(IfTrue);
3661 Value *FalseVal = PN->getIncomingValueForBlock(IfFalse);
3662
3663 Value *Sel = Builder.CreateSelect(IfCond, TrueVal, FalseVal, "", DomBI);
3664 PN->replaceAllUsesWith(Sel);
3665 Sel->takeName(PN);
3666 PN->eraseFromParent();
3667 }
3668
3669 // At this point, all IfBlocks are empty, so our if statement
3670 // has been flattened. Change DomBlock to jump directly to our new block to
3671 // avoid other simplifycfg's kicking in on the diamond.
3672 Builder.CreateBr(BB);
3673
3675 if (DTU) {
3676 Updates.push_back({DominatorTree::Insert, DomBlock, BB});
3677 for (auto *Successor : successors(DomBlock))
3678 Updates.push_back({DominatorTree::Delete, DomBlock, Successor});
3679 }
3680
3681 DomBI->eraseFromParent();
3682 if (DTU)
3683 DTU->applyUpdates(Updates);
3684
3685 return true;
3686}
3687
3689 Instruction::BinaryOps Opc, Value *LHS,
3690 Value *RHS, const Twine &Name = "") {
3691 // Try to relax logical op to binary op.
3692 if (impliesPoison(RHS, LHS))
3693 return Builder.CreateBinOp(Opc, LHS, RHS, Name);
3694 if (Opc == Instruction::And)
3695 return Builder.CreateLogicalAnd(LHS, RHS, Name);
3696 if (Opc == Instruction::Or)
3697 return Builder.CreateLogicalOr(LHS, RHS, Name);
3698 llvm_unreachable("Invalid logical opcode");
3699}
3700
3701/// Return true if either PBI or BI has branch weight available, and store
3702/// the weights in {Pred|Succ}{True|False}Weight. If one of PBI and BI does
3703/// not have branch weight, use 1:1 as its weight.
3705 uint64_t &PredTrueWeight,
3706 uint64_t &PredFalseWeight,
3707 uint64_t &SuccTrueWeight,
3708 uint64_t &SuccFalseWeight) {
3709 bool PredHasWeights =
3710 extractBranchWeights(*PBI, PredTrueWeight, PredFalseWeight);
3711 bool SuccHasWeights =
3712 extractBranchWeights(*BI, SuccTrueWeight, SuccFalseWeight);
3713 if (PredHasWeights || SuccHasWeights) {
3714 if (!PredHasWeights)
3715 PredTrueWeight = PredFalseWeight = 1;
3716 if (!SuccHasWeights)
3717 SuccTrueWeight = SuccFalseWeight = 1;
3718 return true;
3719 } else {
3720 return false;
3721 }
3722}
3723
3724/// Determine if the two branches share a common destination and deduce a glue
3725/// that joins the branches' conditions to arrive at the common destination if
3726/// that would be profitable.
3727static std::optional<std::tuple<BasicBlock *, Instruction::BinaryOps, bool>>
3729 const TargetTransformInfo *TTI) {
3730 assert(BI && PBI && BI->isConditional() && PBI->isConditional() &&
3731 "Both blocks must end with a conditional branches.");
3733 "PredBB must be a predecessor of BB.");
3734
3735 // We have the potential to fold the conditions together, but if the
3736 // predecessor branch is predictable, we may not want to merge them.
3737 uint64_t PTWeight, PFWeight;
3738 BranchProbability PBITrueProb, Likely;
3739 if (TTI && !PBI->getMetadata(LLVMContext::MD_unpredictable) &&
3740 extractBranchWeights(*PBI, PTWeight, PFWeight) &&
3741 (PTWeight + PFWeight) != 0) {
3742 PBITrueProb =
3743 BranchProbability::getBranchProbability(PTWeight, PTWeight + PFWeight);
3745 }
3746
3747 if (PBI->getSuccessor(0) == BI->getSuccessor(0)) {
3748 // Speculate the 2nd condition unless the 1st is probably true.
3749 if (PBITrueProb.isUnknown() || PBITrueProb < Likely)
3750 return {{BI->getSuccessor(0), Instruction::Or, false}};
3751 } else if (PBI->getSuccessor(1) == BI->getSuccessor(1)) {
3752 // Speculate the 2nd condition unless the 1st is probably false.
3753 if (PBITrueProb.isUnknown() || PBITrueProb.getCompl() < Likely)
3754 return {{BI->getSuccessor(1), Instruction::And, false}};
3755 } else if (PBI->getSuccessor(0) == BI->getSuccessor(1)) {
3756 // Speculate the 2nd condition unless the 1st is probably true.
3757 if (PBITrueProb.isUnknown() || PBITrueProb < Likely)
3758 return {{BI->getSuccessor(1), Instruction::And, true}};
3759 } else if (PBI->getSuccessor(1) == BI->getSuccessor(0)) {
3760 // Speculate the 2nd condition unless the 1st is probably false.
3761 if (PBITrueProb.isUnknown() || PBITrueProb.getCompl() < Likely)
3762 return {{BI->getSuccessor(0), Instruction::Or, true}};
3763 }
3764 return std::nullopt;
3765}
3766
3768 DomTreeUpdater *DTU,
3769 MemorySSAUpdater *MSSAU,
3770 const TargetTransformInfo *TTI) {
3771 BasicBlock *BB = BI->getParent();
3772 BasicBlock *PredBlock = PBI->getParent();
3773
3774 // Determine if the two branches share a common destination.
3775 BasicBlock *CommonSucc;
3777 bool InvertPredCond;
3778 std::tie(CommonSucc, Opc, InvertPredCond) =
3780
3781 LLVM_DEBUG(dbgs() << "FOLDING BRANCH TO COMMON DEST:\n" << *PBI << *BB);
3782
3783 IRBuilder<> Builder(PBI);
3784 // The builder is used to create instructions to eliminate the branch in BB.
3785 // If BB's terminator has !annotation metadata, add it to the new
3786 // instructions.
3788 {LLVMContext::MD_annotation});
3789
3790 // If we need to invert the condition in the pred block to match, do so now.
3791 if (InvertPredCond) {
3792 InvertBranch(PBI, Builder);
3793 }
3794
3795 BasicBlock *UniqueSucc =
3796 PBI->getSuccessor(0) == BB ? BI->getSuccessor(0) : BI->getSuccessor(1);
3797
3798 // Before cloning instructions, notify the successor basic block that it
3799 // is about to have a new predecessor. This will update PHI nodes,
3800 // which will allow us to update live-out uses of bonus instructions.
3801 AddPredecessorToBlock(UniqueSucc, PredBlock, BB, MSSAU);
3802
3803 // Try to update branch weights.
3804 uint64_t PredTrueWeight, PredFalseWeight, SuccTrueWeight, SuccFalseWeight;
3805 if (extractPredSuccWeights(PBI, BI, PredTrueWeight, PredFalseWeight,
3806 SuccTrueWeight, SuccFalseWeight)) {
3807 SmallVector<uint64_t, 8> NewWeights;
3808
3809 if (PBI->getSuccessor(0) == BB) {
3810 // PBI: br i1 %x, BB, FalseDest
3811 // BI: br i1 %y, UniqueSucc, FalseDest
3812 // TrueWeight is TrueWeight for PBI * TrueWeight for BI.
3813 NewWeights.push_back(PredTrueWeight * SuccTrueWeight);
3814 // FalseWeight is FalseWeight for PBI * TotalWeight for BI +
3815 // TrueWeight for PBI * FalseWeight for BI.
3816 // We assume that total weights of a BranchInst can fit into 32 bits.
3817 // Therefore, we will not have overflow using 64-bit arithmetic.
3818 NewWeights.push_back(PredFalseWeight *
3819 (SuccFalseWeight + SuccTrueWeight) +
3820 PredTrueWeight * SuccFalseWeight);
3821 } else {
3822 // PBI: br i1 %x, TrueDest, BB
3823 // BI: br i1 %y, TrueDest, UniqueSucc
3824 // TrueWeight is TrueWeight for PBI * TotalWeight for BI +
3825 // FalseWeight for PBI * TrueWeight for BI.
3826 NewWeights.push_back(PredTrueWeight * (SuccFalseWeight + SuccTrueWeight) +
3827 PredFalseWeight * SuccTrueWeight);
3828 // FalseWeight is FalseWeight for PBI * FalseWeight for BI.
3829 NewWeights.push_back(PredFalseWeight * SuccFalseWeight);
3830 }
3831
3832 // Halve the weights if any of them cannot fit in an uint32_t
3833 FitWeights(NewWeights);
3834
3835 SmallVector<uint32_t, 8> MDWeights(NewWeights.begin(), NewWeights.end());
3836 setBranchWeights(PBI, MDWeights[0], MDWeights[1], /*IsExpected=*/false);
3837
3838 // TODO: If BB is reachable from all paths through PredBlock, then we
3839 // could replace PBI's branch probabilities with BI's.
3840 } else
3841 PBI->setMetadata(LLVMContext::MD_prof, nullptr);
3842
3843 // Now, update the CFG.
3844 PBI->setSuccessor(PBI->getSuccessor(0) != BB, UniqueSucc);
3845
3846 if (DTU)
3847 DTU->applyUpdates({{DominatorTree::Insert, PredBlock, UniqueSucc},
3848 {DominatorTree::Delete, PredBlock, BB}});
3849
3850 // If BI was a loop latch, it may have had associated loop metadata.
3851 // We need to copy it to the new latch, that is, PBI.
3852 if (MDNode *LoopMD = BI->getMetadata(LLVMContext::MD_loop))
3853 PBI->setMetadata(LLVMContext::MD_loop, LoopMD);
3854
3855 ValueToValueMapTy VMap; // maps original values to cloned values
3857
3858 Module *M = BB->getModule();
3859
3860 if (PredBlock->IsNewDbgInfoFormat) {
3861 PredBlock->getTerminator()->cloneDebugInfoFrom(BB->getTerminator());
3862 for (DbgVariableRecord &DVR :
3864 RemapDbgRecord(M, &DVR, VMap,
3866 }
3867 }
3868
3869 // Now that the Cond was cloned into the predecessor basic block,
3870 // or/and the two conditions together.
3871 Value *BICond = VMap[BI->getCondition()];
3872 PBI->setCondition(
3873 createLogicalOp(Builder, Opc, PBI->getCondition(), BICond, "or.cond"));
3874
3875 ++NumFoldBranchToCommonDest;
3876 return true;
3877}
3878
3879/// Return if an instruction's type or any of its operands' types are a vector
3880/// type.
3881static bool isVectorOp(Instruction &I) {
3882 return I.getType()->isVectorTy() || any_of(I.operands(), [](Use &U) {
3883 return U->getType()->isVectorTy();
3884 });
3885}
3886
3887/// If this basic block is simple enough, and if a predecessor branches to us
3888/// and one of our successors, fold the block into the predecessor and use
3889/// logical operations to pick the right destination.
3891 MemorySSAUpdater *MSSAU,
3892 const TargetTransformInfo *TTI,
3893 unsigned BonusInstThreshold) {
3894 // If this block ends with an unconditional branch,
3895 // let SpeculativelyExecuteBB() deal with it.
3896 if (!BI->isConditional())
3897 return false;
3898
3899 BasicBlock *BB = BI->getParent();
3903
3904 Instruction *Cond = dyn_cast<Instruction>(BI->getCondition());
3905
3906 if (!Cond ||
3907 (!isa<CmpInst>(Cond) && !isa<BinaryOperator>(Cond) &&
3908 !isa<SelectInst>(Cond)) ||
3909 Cond->getParent() != BB || !Cond->hasOneUse())
3910 return false;
3911
3912 // Finally, don't infinitely unroll conditional loops.
3913 if (is_contained(successors(BB), BB))
3914 return false;
3915
3916 // With which predecessors will we want to deal with?
3918 for (BasicBlock *PredBlock : predecessors(BB)) {
3919 BranchInst *PBI = dyn_cast<BranchInst>(PredBlock->getTerminator());
3920
3921 // Check that we have two conditional branches. If there is a PHI node in
3922 // the common successor, verify that the same value flows in from both
3923 // blocks.
3924 if (!PBI || PBI->isUnconditional() || !SafeToMergeTerminators(BI, PBI))
3925 continue;
3926
3927 // Determine if the two branches share a common destination.
3928 BasicBlock *CommonSucc;
3930 bool InvertPredCond;
3931 if (auto Recipe = shouldFoldCondBranchesToCommonDestination(BI, PBI, TTI))
3932 std::tie(CommonSucc, Opc, InvertPredCond) = *Recipe;
3933 else
3934 continue;
3935
3936 // Check the cost of inserting the necessary logic before performing the
3937 // transformation.
3938 if (TTI) {
3939 Type *Ty = BI->getCondition()->getType();
3941 if (InvertPredCond && (!PBI->getCondition()->hasOneUse() ||
3942 !isa<CmpInst>(PBI->getCondition())))
3943 Cost += TTI->getArithmeticInstrCost(Instruction::Xor, Ty, CostKind);
3944
3946 continue;
3947 }
3948
3949 // Ok, we do want to deal with this predecessor. Record it.
3950 Preds.emplace_back(PredBlock);
3951 }
3952
3953 // If there aren't any predecessors into which we can fold,
3954 // don't bother checking the cost.
3955 if (Preds.empty())
3956 return false;
3957
3958 // Only allow this transformation if computing the condition doesn't involve
3959 // too many instructions and these involved instructions can be executed
3960 // unconditionally. We denote all involved instructions except the condition
3961 // as "bonus instructions", and only allow this transformation when the
3962 // number of the bonus instructions we'll need to create when cloning into
3963 // each predecessor does not exceed a certain threshold.
3964 unsigned NumBonusInsts = 0;
3965 bool SawVectorOp = false;
3966 const unsigned PredCount = Preds.size();
3967 for (Instruction &I : *BB) {
3968 // Don't check the branch condition comparison itself.
3969 if (&I == Cond)
3970 continue;
3971 // Ignore dbg intrinsics, and the terminator.
3972 if (isa<DbgInfoIntrinsic>(I) || isa<BranchInst>(I))
3973 continue;
3974 // I must be safe to execute unconditionally.
3976 return false;
3977 SawVectorOp |= isVectorOp(I);
3978
3979 // Account for the cost of duplicating this instruction into each
3980 // predecessor. Ignore free instructions.
3981 if (!TTI || TTI->getInstructionCost(&I, CostKind) !=
3983 NumBonusInsts += PredCount;
3984
3985 // Early exits once we reach the limit.
3986 if (NumBonusInsts >
3987 BonusInstThreshold * BranchFoldToCommonDestVectorMultiplier)
3988 return false;
3989 }
3990
3991 auto IsBCSSAUse = [BB, &I](Use &U) {
3992 auto *UI = cast<Instruction>(U.getUser());
3993 if (auto *PN = dyn_cast<PHINode>(UI))
3994 return PN->getIncomingBlock(U) == BB;
3995 return UI->getParent() == BB && I.comesBefore(UI);
3996 };
3997
3998 // Does this instruction require rewriting of uses?
3999 if (!all_of(I.uses(), IsBCSSAUse))
4000 return false;
4001 }
4002 if (NumBonusInsts >
4003 BonusInstThreshold *
4004 (SawVectorOp ? BranchFoldToCommonDestVectorMultiplier : 1))
4005 return false;
4006
4007 // Ok, we have the budget. Perform the transformation.
4008 for (BasicBlock *PredBlock : Preds) {
4009 auto *PBI = cast<BranchInst>(PredBlock->getTerminator());
4010 return performBranchToCommonDestFolding(BI, PBI, DTU, MSSAU, TTI);
4011 }
4012 return false;
4013}
4014
4015// If there is only one store in BB1 and BB2, return it, otherwise return
4016// nullptr.
4018 StoreInst *S = nullptr;
4019 for (auto *BB : {BB1, BB2}) {
4020 if (!BB)
4021 continue;
4022 for (auto &I : *BB)
4023 if (auto *SI = dyn_cast<StoreInst>(&I)) {
4024 if (S)
4025 // Multiple stores seen.
4026 return nullptr;
4027 else
4028 S = SI;
4029 }
4030 }
4031 return S;
4032}
4033
4035 Value *AlternativeV = nullptr) {
4036 // PHI is going to be a PHI node that allows the value V that is defined in
4037 // BB to be referenced in BB's only successor.
4038 //
4039 // If AlternativeV is nullptr, the only value we care about in PHI is V. It
4040 // doesn't matter to us what the other operand is (it'll never get used). We
4041 // could just create a new PHI with an undef incoming value, but that could
4042 // increase register pressure if EarlyCSE/InstCombine can't fold it with some
4043 // other PHI. So here we directly look for some PHI in BB's successor with V
4044 // as an incoming operand. If we find one, we use it, else we create a new
4045 // one.
4046 //
4047 // If AlternativeV is not nullptr, we care about both incoming values in PHI.
4048 // PHI must be exactly: phi <ty> [ %BB, %V ], [ %OtherBB, %AlternativeV]
4049 // where OtherBB is the single other predecessor of BB's only successor.
4050 PHINode *PHI = nullptr;
4051 BasicBlock *Succ = BB->getSingleSuccessor();
4052
4053 for (auto I = Succ->begin(); isa<PHINode>(I); ++I)
4054 if (cast<PHINode>(I)->getIncomingValueForBlock(BB) == V) {
4055 PHI = cast<PHINode>(I);
4056 if (!AlternativeV)
4057 break;
4058
4059 assert(Succ->hasNPredecessors(2));
4060 auto PredI = pred_begin(Succ);
4061 BasicBlock *OtherPredBB = *PredI == BB ? *++PredI : *PredI;
4062 if (PHI->getIncomingValueForBlock(OtherPredBB) == AlternativeV)
4063 break;
4064 PHI = nullptr;
4065 }
4066 if (PHI)
4067 return PHI;
4068
4069 // If V is not an instruction defined in BB, just return it.
4070 if (!AlternativeV &&
4071 (!isa<Instruction>(V) || cast<Instruction>(V)->getParent() != BB))
4072 return V;
4073
4074 PHI = PHINode::Create(V->getType(), 2, "simplifycfg.merge");
4075 PHI->insertBefore(Succ->begin());
4076 PHI->addIncoming(V, BB);
4077 for (BasicBlock *PredBB : predecessors(Succ))
4078 if (PredBB != BB)
4079 PHI->addIncoming(
4080 AlternativeV ? AlternativeV : PoisonValue::get(V->getType()), PredBB);
4081 return PHI;
4082}
4083
4085 BasicBlock *PTB, BasicBlock *PFB, BasicBlock *QTB, BasicBlock *QFB,
4086 BasicBlock *PostBB, Value *Address, bool InvertPCond, bool InvertQCond,
4087 DomTreeUpdater *DTU, const DataLayout &DL, const TargetTransformInfo &TTI) {
4088 // For every pointer, there must be exactly two stores, one coming from
4089 // PTB or PFB, and the other from QTB or QFB. We don't support more than one
4090 // store (to any address) in PTB,PFB or QTB,QFB.
4091 // FIXME: We could relax this restriction with a bit more work and performance
4092 // testing.
4093 StoreInst *PStore = findUniqueStoreInBlocks(PTB, PFB);
4094 StoreInst *QStore = findUniqueStoreInBlocks(QTB, QFB);
4095 if (!PStore || !QStore)
4096 return false;
4097
4098 // Now check the stores are compatible.
4099 if (!QStore->isUnordered() || !PStore->isUnordered() ||
4100 PStore->getValueOperand()->getType() !=
4101 QStore->getValueOperand()->getType())
4102 return false;
4103
4104 // Check that sinking the store won't cause program behavior changes. Sinking
4105 // the store out of the Q blocks won't change any behavior as we're sinking
4106 // from a block to its unconditional successor. But we're moving a store from
4107 // the P blocks down through the middle block (QBI) and past both QFB and QTB.
4108 // So we need to check that there are no aliasing loads or stores in
4109 // QBI, QTB and QFB. We also need to check there are no conflicting memory
4110 // operations between PStore and the end of its parent block.
4111 //
4112 // The ideal way to do this is to query AliasAnalysis, but we don't
4113 // preserve AA currently so that is dangerous. Be super safe and just
4114 // check there are no other memory operations at all.
4115 for (auto &I : *QFB->getSinglePredecessor())
4116 if (I.mayReadOrWriteMemory())
4117 return false;
4118 for (auto &I : *QFB)
4119 if (&I != QStore && I.mayReadOrWriteMemory())
4120 return false;
4121 if (QTB)
4122 for (auto &I : *QTB)
4123 if (&I != QStore && I.mayReadOrWriteMemory())
4124 return false;
4125 for (auto I = BasicBlock::iterator(PStore), E = PStore->getParent()->end();
4126 I != E; ++I)
4127 if (&*I != PStore && I->mayReadOrWriteMemory())
4128 return false;
4129
4130 // If we're not in aggressive mode, we only optimize if we have some
4131 // confidence that by optimizing we'll allow P and/or Q to be if-converted.
4132 auto IsWorthwhile = [&](BasicBlock *BB, ArrayRef<StoreInst *> FreeStores) {
4133 if (!BB)
4134 return true;
4135 // Heuristic: if the block can be if-converted/phi-folded and the
4136 // instructions inside are all cheap (arithmetic/GEPs), it's worthwhile to
4137 // thread this store.
4139 InstructionCost Budget =
4141 for (auto &I : BB->instructionsWithoutDebug(false)) {
4142 // Consider terminator instruction to be free.
4143 if (I.isTerminator())
4144 continue;
4145 // If this is one the stores that we want to speculate out of this BB,
4146 // then don't count it's cost, consider it to be free.
4147 if (auto *S = dyn_cast<StoreInst>(&I))
4148 if (llvm::find(FreeStores, S))
4149 continue;
4150 // Else, we have a white-list of instructions that we are ak speculating.
4151 if (!isa<BinaryOperator>(I) && !isa<GetElementPtrInst>(I))
4152 return false; // Not in white-list - not worthwhile folding.
4153 // And finally, if this is a non-free instruction that we are okay
4154 // speculating, ensure that we consider the speculation budget.
4155 Cost +=
4157 if (Cost > Budget)
4158 return false; // Eagerly refuse to fold as soon as we're out of budget.
4159 }
4160 assert(Cost <= Budget &&
4161 "When we run out of budget we will eagerly return from within the "
4162 "per-instruction loop.");
4163 return true;
4164 };
4165
4166 const std::array<StoreInst *, 2> FreeStores = {PStore, QStore};
4168 (!IsWorthwhile(PTB, FreeStores) || !IsWorthwhile(PFB, FreeStores) ||
4169 !IsWorthwhile(QTB, FreeStores) || !IsWorthwhile(QFB, FreeStores)))
4170 return false;
4171
4172 // If PostBB has more than two predecessors, we need to split it so we can
4173 // sink the store.
4174 if (std::next(pred_begin(PostBB), 2) != pred_end(PostBB)) {
4175 // We know that QFB's only successor is PostBB. And QFB has a single
4176 // predecessor. If QTB exists, then its only successor is also PostBB.
4177 // If QTB does not exist, then QFB's only predecessor has a conditional
4178 // branch to QFB and PostBB.
4179 BasicBlock *TruePred = QTB ? QTB : QFB->getSinglePredecessor();
4180 BasicBlock *NewBB =
4181 SplitBlockPredecessors(PostBB, {QFB, TruePred}, "condstore.split", DTU);
4182 if (!NewBB)
4183 return false;
4184 PostBB = NewBB;
4185 }
4186
4187 // OK, we're going to sink the stores to PostBB. The store has to be
4188 // conditional though, so first create the predicate.
4189 Value *PCond = cast<BranchInst>(PFB->getSinglePredecessor()->getTerminator())
4190 ->getCondition();
4191 Value *QCond = cast<BranchInst>(QFB->getSinglePredecessor()->getTerminator())
4192 ->getCondition();
4193
4195 PStore->getParent());
4197 QStore->getParent(), PPHI);
4198
4199 BasicBlock::iterator PostBBFirst = PostBB->getFirstInsertionPt();
4200 IRBuilder<> QB(PostBB, PostBBFirst);
4201 QB.SetCurrentDebugLocation(PostBBFirst->getStableDebugLoc());
4202
4203 Value *PPred = PStore->getParent() == PTB ? PCond : QB.CreateNot(PCond);
4204 Value *QPred = QStore->getParent() == QTB ? QCond : QB.CreateNot(QCond);
4205
4206 if (InvertPCond)
4207 PPred = QB.CreateNot(PPred);
4208 if (InvertQCond)
4209 QPred = QB.CreateNot(QPred);
4210 Value *CombinedPred = QB.CreateOr(PPred, QPred);
4211
4212 BasicBlock::iterator InsertPt = QB.GetInsertPoint();
4213 auto *T = SplitBlockAndInsertIfThen(CombinedPred, InsertPt,
4214 /*Unreachable=*/false,
4215 /*BranchWeights=*/nullptr, DTU);
4216
4217 QB.SetInsertPoint(T);
4218 StoreInst *SI = cast<StoreInst>(QB.CreateStore(QPHI, Address));
4219 SI->setAAMetadata(PStore->getAAMetadata().merge(QStore->getAAMetadata()));
4220 // Choose the minimum alignment. If we could prove both stores execute, we
4221 // could use biggest one. In this case, though, we only know that one of the
4222 // stores executes. And we don't know it's safe to take the alignment from a
4223 // store that doesn't execute.
4224 SI->setAlignment(std::min(PStore->getAlign(), QStore->getAlign()));
4225
4226 QStore->eraseFromParent();
4227 PStore->eraseFromParent();
4228
4229 return true;
4230}
4231
4233 DomTreeUpdater *DTU, const DataLayout &DL,
4234 const TargetTransformInfo &TTI) {
4235 // The intention here is to find diamonds or triangles (see below) where each
4236 // conditional block contains a store to the same address. Both of these
4237 // stores are conditional, so they can't be unconditionally sunk. But it may
4238 // be profitable to speculatively sink the stores into one merged store at the
4239 // end, and predicate the merged store on the union of the two conditions of
4240 // PBI and QBI.
4241 //
4242 // This can reduce the number of stores executed if both of the conditions are
4243 // true, and can allow the blocks to become small enough to be if-converted.
4244 // This optimization will also chain, so that ladders of test-and-set
4245 // sequences can be if-converted away.
4246 //
4247 // We only deal with simple diamonds or triangles:
4248 //
4249 // PBI or PBI or a combination of the two
4250 // / \ | \
4251 // PTB PFB | PFB
4252 // \ / | /
4253 // QBI QBI
4254 // / \ | \
4255 // QTB QFB | QFB
4256 // \ / | /
4257 // PostBB PostBB
4258 //
4259 // We model triangles as a type of diamond with a nullptr "true" block.
4260 // Triangles are canonicalized so that the fallthrough edge is represented by
4261 // a true condition, as in the diagram above.
4262 BasicBlock *PTB = PBI->getSuccessor(0);
4263 BasicBlock *PFB = PBI->getSuccessor(1);
4264 BasicBlock *QTB = QBI->getSuccessor(0);
4265 BasicBlock *QFB = QBI->getSuccessor(1);
4266 BasicBlock *PostBB = QFB->getSingleSuccessor();
4267
4268 // Make sure we have a good guess for PostBB. If QTB's only successor is
4269 // QFB, then QFB is a better PostBB.
4270 if (QTB->getSingleSuccessor() == QFB)
4271 PostBB = QFB;
4272
4273 // If we couldn't find a good PostBB, stop.
4274 if (!PostBB)
4275 return false;
4276
4277 bool InvertPCond = false, InvertQCond = false;
4278 // Canonicalize fallthroughs to the true branches.
4279 if (PFB == QBI->getParent()) {
4280 std::swap(PFB, PTB);
4281 InvertPCond = true;
4282 }
4283 if (QFB == PostBB) {
4284 std::swap(QFB, QTB);
4285 InvertQCond = true;
4286 }
4287
4288 // From this point on we can assume PTB or QTB may be fallthroughs but PFB
4289 // and QFB may not. Model fallthroughs as a nullptr block.
4290 if (PTB == QBI->getParent())
4291 PTB = nullptr;
4292 if (QTB == PostBB)
4293 QTB = nullptr;
4294
4295 // Legality bailouts. We must have at least the non-fallthrough blocks and
4296 // the post-dominating block, and the non-fallthroughs must only have one
4297 // predecessor.
4298 auto HasOnePredAndOneSucc = [](BasicBlock *BB, BasicBlock *P, BasicBlock *S) {
4299 return BB->getSinglePredecessor() == P && BB->getSingleSuccessor() == S;
4300 };
4301 if (!HasOnePredAndOneSucc(PFB, PBI->getParent(), QBI->getParent()) ||
4302 !HasOnePredAndOneSucc(QFB, QBI->getParent(), PostBB))
4303 return false;
4304 if ((PTB && !HasOnePredAndOneSucc(PTB, PBI->getParent(), QBI->getParent())) ||
4305 (QTB && !HasOnePredAndOneSucc(QTB, QBI->getParent(), PostBB)))
4306 return false;
4307 if (!QBI->getParent()->hasNUses(2))
4308 return false;
4309
4310 // OK, this is a sequence of two diamonds or triangles.
4311 // Check if there are stores in PTB or PFB that are repeated in QTB or QFB.
4312 SmallPtrSet<Value *, 4> PStoreAddresses, QStoreAddresses;
4313 for (auto *BB : {PTB, PFB}) {
4314 if (!BB)
4315 continue;
4316 for (auto &I : *BB)
4317 if (StoreInst *SI = dyn_cast<StoreInst>(&I))
4318 PStoreAddresses.insert(SI->getPointerOperand());
4319 }
4320 for (auto *BB : {QTB, QFB}) {
4321 if (!BB)
4322 continue;
4323 for (auto &I : *BB)
4324 if (StoreInst *SI = dyn_cast<StoreInst>(&I))
4325 QStoreAddresses.insert(SI->getPointerOperand());
4326 }
4327
4328 set_intersect(PStoreAddresses, QStoreAddresses);
4329 // set_intersect mutates PStoreAddresses in place. Rename it here to make it
4330 // clear what it contains.
4331 auto &CommonAddresses = PStoreAddresses;
4332
4333 bool Changed = false;
4334 for (auto *Address : CommonAddresses)
4335 Changed |=
4336 mergeConditionalStoreToAddress(PTB, PFB, QTB, QFB, PostBB, Address,
4337 InvertPCond, InvertQCond, DTU, DL, TTI);
4338 return Changed;
4339}
4340
4341/// If the previous block ended with a widenable branch, determine if reusing
4342/// the target block is profitable and legal. This will have the effect of
4343/// "widening" PBI, but doesn't require us to reason about hosting safety.
4345 DomTreeUpdater *DTU) {
4346 // TODO: This can be generalized in two important ways:
4347 // 1) We can allow phi nodes in IfFalseBB and simply reuse all the input
4348 // values from the PBI edge.
4349 // 2) We can sink side effecting instructions into BI's fallthrough
4350 // successor provided they doesn't contribute to computation of
4351 // BI's condition.
4352 BasicBlock *IfTrueBB = PBI->getSuccessor(0);
4353 BasicBlock *IfFalseBB = PBI->getSuccessor(1);
4354 if (!isWidenableBranch(PBI) || IfTrueBB != BI->getParent() ||
4356 return false;
4357 if (!IfFalseBB->phis().empty())
4358 return false; // TODO
4359 // This helps avoid infinite loop with SimplifyCondBranchToCondBranch which
4360 // may undo the transform done here.
4361 // TODO: There might be a more fine-grained solution to this.
4362 if (!llvm::succ_empty(IfFalseBB))
4363 return false;
4364 // Use lambda to lazily compute expensive condition after cheap ones.
4365 auto NoSideEffects = [](BasicBlock &BB) {
4366 return llvm::none_of(BB, [](const Instruction &I) {
4367 return I.mayWriteToMemory() || I.mayHaveSideEffects();
4368 });
4369 };
4370 if (BI->getSuccessor(1) != IfFalseBB && // no inf looping
4371 BI->getSuccessor(1)->getTerminatingDeoptimizeCall() && // profitability
4372 NoSideEffects(*BI->getParent())) {
4373 auto *OldSuccessor = BI->getSuccessor(1);
4374 OldSuccessor->removePredecessor(BI->getParent());
4375 BI->setSuccessor(1, IfFalseBB);
4376 if (DTU)
4377 DTU->applyUpdates(
4378 {{DominatorTree::Insert, BI->getParent(), IfFalseBB},
4379 {DominatorTree::Delete, BI->getParent(), OldSuccessor}});
4380 return true;
4381 }
4382 if (BI->getSuccessor(0) != IfFalseBB && // no inf looping
4383 BI->getSuccessor(0)->getTerminatingDeoptimizeCall() && // profitability
4384 NoSideEffects(*BI->getParent())) {
4385 auto *OldSuccessor = BI->getSuccessor(0);
4386 OldSuccessor->removePredecessor(BI->getParent());
4387 BI->setSuccessor(0, IfFalseBB);
4388 if (DTU)
4389 DTU->applyUpdates(
4390 {{DominatorTree::Insert, BI->getParent(), IfFalseBB},
4391 {DominatorTree::Delete, BI->getParent(), OldSuccessor}});
4392 return true;
4393 }
4394 return false;
4395}
4396
4397/// If we have a conditional branch as a predecessor of another block,
4398/// this function tries to simplify it. We know
4399/// that PBI and BI are both conditional branches, and BI is in one of the
4400/// successor blocks of PBI - PBI branches to BI.
4402 DomTreeUpdater *DTU,
4403 const DataLayout &DL,
4404 const TargetTransformInfo &TTI) {
4405 assert(PBI->isConditional() && BI->isConditional());
4406 BasicBlock *BB = BI->getParent();
4407
4408 // If this block ends with a branch instruction, and if there is a
4409 // predecessor that ends on a branch of the same condition, make
4410 // this conditional branch redundant.
4411 if (PBI->getCondition() == BI->getCondition() &&
4412 PBI->getSuccessor(0) != PBI->getSuccessor(1)) {
4413 // Okay, the outcome of this conditional branch is statically
4414 // knowable. If this block had a single pred, handle specially, otherwise
4415 // FoldCondBranchOnValueKnownInPredecessor() will handle it.
4416 if (BB->getSinglePredecessor()) {
4417 // Turn this into a branch on constant.
4418 bool CondIsTrue = PBI->getSuccessor(0) == BB;
4419 BI->setCondition(
4420 ConstantInt::get(Type::getInt1Ty(BB->getContext()), CondIsTrue));
4421 return true; // Nuke the branch on constant.
4422 }
4423 }
4424
4425 // If the previous block ended with a widenable branch, determine if reusing
4426 // the target block is profitable and legal. This will have the effect of
4427 // "widening" PBI, but doesn't require us to reason about hosting safety.
4428 if (tryWidenCondBranchToCondBranch(PBI, BI, DTU))
4429 return true;
4430
4431 // If both branches are conditional and both contain stores to the same
4432 // address, remove the stores from the conditionals and create a conditional
4433 // merged store at the end.
4434 if (MergeCondStores && mergeConditionalStores(PBI, BI, DTU, DL, TTI))
4435 return true;
4436
4437 // If this is a conditional branch in an empty block, and if any
4438 // predecessors are a conditional branch to one of our destinations,
4439 // fold the conditions into logical ops and one cond br.
4440
4441 // Ignore dbg intrinsics.
4442 if (&*BB->instructionsWithoutDebug(false).begin() != BI)
4443 return false;
4444
4445 int PBIOp, BIOp;
4446 if (PBI->getSuccessor(0) == BI->getSuccessor(0)) {
4447 PBIOp = 0;
4448 BIOp = 0;
4449 } else if (PBI->getSuccessor(0) == BI->getSuccessor(1)) {
4450 PBIOp = 0;
4451 BIOp = 1;
4452 } else if (PBI->getSuccessor(1) == BI->getSuccessor(0)) {
4453 PBIOp = 1;
4454 BIOp = 0;
4455 } else if (PBI->getSuccessor(1) == BI->getSuccessor(1)) {
4456 PBIOp = 1;
4457 BIOp = 1;
4458 } else {
4459 return false;
4460 }
4461
4462 // Check to make sure that the other destination of this branch
4463 // isn't BB itself. If so, this is an infinite loop that will
4464 // keep getting unwound.
4465 if (PBI->getSuccessor(PBIOp) == BB)
4466 return false;
4467
4468 // If predecessor's branch probability to BB is too low don't merge branches.
4469 SmallVector<uint32_t, 2> PredWeights;
4470 if (!PBI->getMetadata(LLVMContext::MD_unpredictable) &&
4471 extractBranchWeights(*PBI, PredWeights) &&
4472 (static_cast<uint64_t>(PredWeights[0]) + PredWeights[1]) != 0) {
4473
4475 PredWeights[PBIOp],
4476 static_cast<uint64_t>(PredWeights[0]) + PredWeights[1]);
4477
4479 if (CommonDestProb >= Likely)
4480 return false;
4481 }
4482
4483 // Do not perform this transformation if it would require
4484 // insertion of a large number of select instructions. For targets
4485 // without predication/cmovs, this is a big pessimization.
4486
4487 BasicBlock *CommonDest = PBI->getSuccessor(PBIOp);
4488 BasicBlock *RemovedDest = PBI->getSuccessor(PBIOp ^ 1);
4489 unsigned NumPhis = 0;
4490 for (BasicBlock::iterator II = CommonDest->begin(); isa<PHINode>(II);
4491 ++II, ++NumPhis) {
4492 if (NumPhis > 2) // Disable this xform.
4493 return false;
4494 }
4495
4496 // Finally, if everything is ok, fold the branches to logical ops.
4497 BasicBlock *OtherDest = BI->getSuccessor(BIOp ^ 1);
4498
4499 LLVM_DEBUG(dbgs() << "FOLDING BRs:" << *PBI->getParent()
4500 << "AND: " << *BI->getParent());
4501
4503
4504 // If OtherDest *is* BB, then BB is a basic block with a single conditional
4505 // branch in it, where one edge (OtherDest) goes back to itself but the other
4506 // exits. We don't *know* that the program avoids the infinite loop
4507 // (even though that seems likely). If we do this xform naively, we'll end up
4508 // recursively unpeeling the loop. Since we know that (after the xform is
4509 // done) that the block *is* infinite if reached, we just make it an obviously
4510 // infinite loop with no cond branch.
4511 if (OtherDest == BB) {
4512 // Insert it at the end of the function, because it's either code,
4513 // or it won't matter if it's hot. :)
4514 BasicBlock *InfLoopBlock =
4515 BasicBlock::Create(BB->getContext(), "infloop", BB->getParent());
4516 BranchInst::Create(InfLoopBlock, InfLoopBlock);
4517 if (DTU)
4518 Updates.push_back({DominatorTree::Insert, InfLoopBlock, InfLoopBlock});
4519 OtherDest = InfLoopBlock;
4520 }
4521
4522 LLVM_DEBUG(dbgs() << *PBI->getParent()->getParent());
4523
4524 // BI may have other predecessors. Because of this, we leave
4525 // it alone, but modify PBI.
4526
4527 // Make sure we get to CommonDest on True&True directions.
4528 Value *PBICond = PBI->getCondition();
4529 IRBuilder<NoFolder> Builder(PBI);
4530 if (PBIOp)
4531 PBICond = Builder.CreateNot(PBICond, PBICond->getName() + ".not");
4532
4533 Value *BICond = BI->getCondition();
4534 if (BIOp)
4535 BICond = Builder.CreateNot(BICond, BICond->getName() + ".not");
4536
4537 // Merge the conditions.
4538 Value *Cond =
4539 createLogicalOp(Builder, Instruction::Or, PBICond, BICond, "brmerge");
4540
4541 // Modify PBI to branch on the new condition to the new dests.
4542 PBI->setCondition(Cond);
4543 PBI->setSuccessor(0, CommonDest);
4544 PBI->setSuccessor(1, OtherDest);
4545
4546 if (DTU) {
4547 Updates.push_back({DominatorTree::Insert, PBI->getParent(), OtherDest});
4548 Updates.push_back({DominatorTree::Delete, PBI->getParent(), RemovedDest});
4549
4550 DTU->applyUpdates(Updates);
4551 }
4552
4553 // Update branch weight for PBI.
4554 uint64_t PredTrueWeight, PredFalseWeight, SuccTrueWeight, SuccFalseWeight;
4555 uint64_t PredCommon, PredOther, SuccCommon, SuccOther;
4556 bool HasWeights =
4557 extractPredSuccWeights(PBI, BI, PredTrueWeight, PredFalseWeight,
4558 SuccTrueWeight, SuccFalseWeight);
4559 if (HasWeights) {
4560 PredCommon = PBIOp ? PredFalseWeight : PredTrueWeight;
4561 PredOther = PBIOp ? PredTrueWeight : PredFalseWeight;
4562 SuccCommon = BIOp ? SuccFalseWeight : SuccTrueWeight;
4563 SuccOther = BIOp ? SuccTrueWeight : SuccFalseWeight;
4564 // The weight to CommonDest should be PredCommon * SuccTotal +
4565 // PredOther * SuccCommon.
4566 // The weight to OtherDest should be PredOther * SuccOther.
4567 uint64_t NewWeights[2] = {PredCommon * (SuccCommon + SuccOther) +
4568 PredOther * SuccCommon,
4569 PredOther * SuccOther};
4570 // Halve the weights if any of them cannot fit in an uint32_t
4571 FitWeights(NewWeights);
4572
4573 setBranchWeights(PBI, NewWeights[0], NewWeights[1], /*IsExpected=*/false);
4574 }
4575
4576 // OtherDest may have phi nodes. If so, add an entry from PBI's
4577 // block that are identical to the entries for BI's block.
4578 AddPredecessorToBlock(OtherDest, PBI->getParent(), BB);
4579
4580 // We know that the CommonDest already had an edge from PBI to
4581 // it. If it has PHIs though, the PHIs may have different
4582 // entries for BB and PBI's BB. If so, insert a select to make
4583 // them agree.
4584 for (PHINode &PN : CommonDest->phis()) {
4585 Value *BIV = PN.getIncomingValueForBlock(BB);
4586 unsigned PBBIdx = PN.getBasicBlockIndex(PBI->getParent());
4587 Value *PBIV = PN.getIncomingValue(PBBIdx);
4588 if (BIV != PBIV) {
4589 // Insert a select in PBI to pick the right value.
4590 SelectInst *NV = cast<SelectInst>(
4591 Builder.CreateSelect(PBICond, PBIV, BIV, PBIV->getName() + ".mux"));
4592 PN.setIncomingValue(PBBIdx, NV);
4593 // Although the select has the same condition as PBI, the original branch
4594 // weights for PBI do not apply to the new select because the select's
4595 // 'logical' edges are incoming edges of the phi that is eliminated, not
4596 // the outgoing edges of PBI.
4597 if (HasWeights) {
4598 uint64_t PredCommon = PBIOp ? PredFalseWeight : PredTrueWeight;
4599 uint64_t PredOther = PBIOp ? PredTrueWeight : PredFalseWeight;
4600 uint64_t SuccCommon = BIOp ? SuccFalseWeight : SuccTrueWeight;
4601 uint64_t SuccOther = BIOp ? SuccTrueWeight : SuccFalseWeight;
4602 // The weight to PredCommonDest should be PredCommon * SuccTotal.
4603 // The weight to PredOtherDest should be PredOther * SuccCommon.
4604 uint64_t NewWeights[2] = {PredCommon * (SuccCommon + SuccOther),
4605 PredOther * SuccCommon};
4606
4607 FitWeights(NewWeights);
4608
4609 setBranchWeights(NV, NewWeights[0], NewWeights[1],
4610 /*IsExpected=*/false);
4611 }
4612 }
4613 }
4614
4615 LLVM_DEBUG(dbgs() << "INTO: " << *PBI->getParent());
4616 LLVM_DEBUG(dbgs() << *PBI->getParent()->getParent());
4617
4618 // This basic block is probably dead. We know it has at least
4619 // one fewer predecessor.
4620 return true;
4621}
4622
4623// Simplifies a terminator by replacing it with a branch to TrueBB if Cond is
4624// true or to FalseBB if Cond is false.
4625// Takes care of updating the successors and removing the old terminator.
4626// Also makes sure not to introduce new successors by assuming that edges to
4627// non-successor TrueBBs and FalseBBs aren't reachable.
4628bool SimplifyCFGOpt::SimplifyTerminatorOnSelect(Instruction *OldTerm,
4629 Value *Cond, BasicBlock *TrueBB,
4630 BasicBlock *FalseBB,
4631 uint32_t TrueWeight,
4632 uint32_t FalseWeight) {
4633 auto *BB = OldTerm->getParent();
4634 // Remove any superfluous successor edges from the CFG.
4635 // First, figure out which successors to preserve.
4636 // If TrueBB and FalseBB are equal, only try to preserve one copy of that
4637 // successor.
4638 BasicBlock *KeepEdge1 = TrueBB;
4639 BasicBlock *KeepEdge2 = TrueBB != FalseBB ? FalseBB : nullptr;
4640
4641 SmallSetVector<BasicBlock *, 2> RemovedSuccessors;
4642
4643 // Then remove the rest.
4644 for (BasicBlock *Succ : successors(OldTerm)) {
4645 // Make sure only to keep exactly one copy of each edge.
4646 if (Succ == KeepEdge1)
4647 KeepEdge1 = nullptr;
4648 else if (Succ == KeepEdge2)
4649 KeepEdge2 = nullptr;
4650 else {
4651 Succ->removePredecessor(BB,
4652 /*KeepOneInputPHIs=*/true);
4653
4654 if (Succ != TrueBB && Succ != FalseBB)
4655 RemovedSuccessors.insert(Succ);
4656 }
4657 }
4658
4659 IRBuilder<> Builder(OldTerm);
4660 Builder.SetCurrentDebugLocation(OldTerm->getDebugLoc());
4661
4662 // Insert an appropriate new terminator.
4663 if (!KeepEdge1 && !KeepEdge2) {
4664 if (TrueBB == FalseBB) {
4665 // We were only looking for one successor, and it was present.
4666 // Create an unconditional branch to it.
4667 Builder.CreateBr(TrueBB);
4668 } else {
4669 // We found both of the successors we were looking for.
4670 // Create a conditional branch sharing the condition of the select.
4671 BranchInst *NewBI = Builder.CreateCondBr(Cond, TrueBB, FalseBB);
4672 if (TrueWeight != FalseWeight)
4673 setBranchWeights(NewBI, TrueWeight, FalseWeight, /*IsExpected=*/false);
4674 }
4675 } else if (KeepEdge1 && (KeepEdge2 || TrueBB == FalseBB)) {
4676 // Neither of the selected blocks were successors, so this
4677 // terminator must be unreachable.
4678 new UnreachableInst(OldTerm->getContext(), OldTerm->getIterator());
4679 } else {
4680 // One of the selected values was a successor, but the other wasn't.
4681 // Insert an unconditional branch to the one that was found;
4682 // the edge to the one that wasn't must be unreachable.
4683 if (!KeepEdge1) {
4684 // Only TrueBB was found.
4685 Builder.CreateBr(TrueBB);
4686 } else {
4687 // Only FalseBB was found.
4688 Builder.CreateBr(FalseBB);
4689 }
4690 }
4691
4693
4694 if (DTU) {
4696 Updates.reserve(RemovedSuccessors.size());
4697 for (auto *RemovedSuccessor : RemovedSuccessors)
4698 Updates.push_back({DominatorTree::Delete, BB, RemovedSuccessor});
4699 DTU->applyUpdates(Updates);
4700 }
4701
4702 return true;
4703}
4704
4705// Replaces
4706// (switch (select cond, X, Y)) on constant X, Y
4707// with a branch - conditional if X and Y lead to distinct BBs,
4708// unconditional otherwise.
4709bool SimplifyCFGOpt::SimplifySwitchOnSelect(SwitchInst *SI,
4710 SelectInst *Select) {
4711 // Check for constant integer values in the select.
4712 ConstantInt *TrueVal = dyn_cast<ConstantInt>(Select->getTrueValue());
4713 ConstantInt *FalseVal = dyn_cast<ConstantInt>(Select->getFalseValue());
4714 if (!TrueVal || !FalseVal)
4715 return false;
4716
4717 // Find the relevant condition and destinations.
4718 Value *Condition = Select->getCondition();
4719 BasicBlock *TrueBB = SI->findCaseValue(TrueVal)->getCaseSuccessor();
4720 BasicBlock *FalseBB = SI->findCaseValue(FalseVal)->getCaseSuccessor();
4721
4722 // Get weight for TrueBB and FalseBB.
4723 uint32_t TrueWeight = 0, FalseWeight = 0;
4725 bool HasWeights = hasBranchWeightMD(*SI);
4726 if (HasWeights) {
4727 GetBranchWeights(SI, Weights);
4728 if (Weights.size() == 1 + SI->getNumCases()) {
4729 TrueWeight =
4730 (uint32_t)Weights[SI->findCaseValue(TrueVal)->getSuccessorIndex()];
4731 FalseWeight =
4732 (uint32_t)Weights[SI->findCaseValue(FalseVal)->getSuccessorIndex()];
4733 }
4734 }
4735
4736 // Perform the actual simplification.
4737 return SimplifyTerminatorOnSelect(SI, Condition, TrueBB, FalseBB, TrueWeight,
4738 FalseWeight);
4739}
4740
4741// Replaces
4742// (indirectbr (select cond, blockaddress(@fn, BlockA),
4743// blockaddress(@fn, BlockB)))
4744// with
4745// (br cond, BlockA, BlockB).
4746bool SimplifyCFGOpt::SimplifyIndirectBrOnSelect(IndirectBrInst *IBI,
4747 SelectInst *SI) {
4748 // Check that both operands of the select are block addresses.
4749 BlockAddress *TBA = dyn_cast<BlockAddress>(SI->getTrueValue());
4750 BlockAddress *FBA = dyn_cast<BlockAddress>(SI->getFalseValue());
4751 if (!TBA || !FBA)
4752 return false;
4753
4754 // Extract the actual blocks.
4755 BasicBlock *TrueBB = TBA->getBasicBlock();
4756 BasicBlock *FalseBB = FBA->getBasicBlock();
4757
4758 // Perform the actual simplification.
4759 return SimplifyTerminatorOnSelect(IBI, SI->getCondition(), TrueBB, FalseBB, 0,
4760 0);
4761}
4762
4763/// This is called when we find an icmp instruction
4764/// (a seteq/setne with a constant) as the only instruction in a
4765/// block that ends with an uncond branch. We are looking for a very specific
4766/// pattern that occurs when "A == 1 || A == 2 || A == 3" gets simplified. In
4767/// this case, we merge the first two "or's of icmp" into a switch, but then the
4768/// default value goes to an uncond block with a seteq in it, we get something
4769/// like:
4770///
4771/// switch i8 %A, label %DEFAULT [ i8 1, label %end i8 2, label %end ]
4772/// DEFAULT:
4773/// %tmp = icmp eq i8 %A, 92
4774/// br label %end
4775/// end:
4776/// ... = phi i1 [ true, %entry ], [ %tmp, %DEFAULT ], [ true, %entry ]
4777///
4778/// We prefer to split the edge to 'end' so that there is a true/false entry to
4779/// the PHI, merging the third icmp into the switch.
4780bool SimplifyCFGOpt::tryToSimplifyUncondBranchWithICmpInIt(
4781 ICmpInst *ICI, IRBuilder<> &Builder) {
4782 BasicBlock *BB = ICI->getParent();
4783
4784 // If the block has any PHIs in it or the icmp has multiple uses, it is too
4785 // complex.
4786 if (isa<PHINode>(BB->begin()) || !ICI->hasOneUse())
4787 return false;
4788
4789 Value *V = ICI->getOperand(0);
4790 ConstantInt *Cst = cast<ConstantInt>(ICI->getOperand(1));
4791
4792 // The pattern we're looking for is where our only predecessor is a switch on
4793 // 'V' and this block is the default case for the switch. In this case we can
4794 // fold the compared value into the switch to simplify things.
4795 BasicBlock *Pred = BB->getSinglePredecessor();
4796 if (!Pred || !isa<SwitchInst>(Pred->getTerminator()))
4797 return false;
4798
4799 SwitchInst *SI = cast<SwitchInst>(Pred->getTerminator());
4800 if (SI->getCondition() != V)
4801 return false;
4802
4803 // If BB is reachable on a non-default case, then we simply know the value of
4804 // V in this block. Substitute it and constant fold the icmp instruction
4805 // away.
4806 if (SI->getDefaultDest() != BB) {
4807 ConstantInt *VVal = SI->findCaseDest(BB);
4808 assert(VVal && "Should have a unique destination value");
4809 ICI->setOperand(0, VVal);
4810
4811 if (Value *V = simplifyInstruction(ICI, {DL, ICI})) {
4812 ICI->replaceAllUsesWith(V);
4813 ICI->eraseFromParent();
4814 }
4815 // BB is now empty, so it is likely to simplify away.
4816 return requestResimplify();
4817 }
4818
4819 // Ok, the block is reachable from the default dest. If the constant we're
4820 // comparing exists in one of the other edges, then we can constant fold ICI
4821 // and zap it.
4822 if (SI->findCaseValue(Cst) != SI->case_default()) {
4823 Value *V;
4824 if (ICI->getPredicate() == ICmpInst::ICMP_EQ)
4826 else
4828
4829 ICI->replaceAllUsesWith(V);
4830 ICI->eraseFromParent();
4831 // BB is now empty, so it is likely to simplify away.
4832 return requestResimplify();
4833 }
4834
4835 // The use of the icmp has to be in the 'end' block, by the only PHI node in
4836 // the block.
4837 BasicBlock *SuccBlock = BB->getTerminator()->getSuccessor(0);
4838 PHINode *PHIUse = dyn_cast<PHINode>(ICI->user_back());
4839 if (PHIUse == nullptr || PHIUse != &SuccBlock->front() ||
4840 isa<PHINode>(++BasicBlock::iterator(PHIUse)))
4841 return false;
4842
4843 // If the icmp is a SETEQ, then the default dest gets false, the new edge gets
4844 // true in the PHI.
4845 Constant *DefaultCst = ConstantInt::getTrue(BB->getContext());
4846 Constant *NewCst = ConstantInt::getFalse(BB->getContext());
4847
4848 if (ICI->getPredicate() == ICmpInst::ICMP_EQ)
4849 std::swap(DefaultCst, NewCst);
4850
4851 // Replace ICI (which is used by the PHI for the default value) with true or
4852 // false depending on if it is EQ or NE.
4853 ICI->replaceAllUsesWith(DefaultCst);
4854 ICI->eraseFromParent();
4855
4857
4858 // Okay, the switch goes to this block on a default value. Add an edge from
4859 // the switch to the merge point on the compared value.
4860 BasicBlock *NewBB =
4861 BasicBlock::Create(BB->getContext(), "switch.edge", BB->getParent(), BB);
4862 {
4864 auto W0 = SIW.getSuccessorWeight(0);
4866 if (W0) {
4867 NewW = ((uint64_t(*W0) + 1) >> 1);
4868 SIW.setSuccessorWeight(0, *NewW);
4869 }
4870 SIW.addCase(Cst, NewBB, NewW);
4871 if (DTU)
4872 Updates.push_back({DominatorTree::Insert, Pred, NewBB});
4873 }
4874
4875 // NewBB branches to the phi block, add the uncond branch and the phi entry.
4876 Builder.SetInsertPoint(NewBB);
4877 Builder.SetCurrentDebugLocation(SI->getDebugLoc());
4878 Builder.CreateBr(SuccBlock);
4879 PHIUse->addIncoming(NewCst, NewBB);
4880 if (DTU) {
4881 Updates.push_back({DominatorTree::Insert, NewBB, SuccBlock});
4882 DTU->applyUpdates(Updates);
4883 }
4884 return true;
4885}
4886
4887/// The specified branch is a conditional branch.
4888/// Check to see if it is branching on an or/and chain of icmp instructions, and
4889/// fold it into a switch instruction if so.
4890bool SimplifyCFGOpt::SimplifyBranchOnICmpChain(BranchInst *BI,
4891 IRBuilder<> &Builder,
4892 const DataLayout &DL) {
4893 Instruction *Cond = dyn_cast<Instruction>(BI->getCondition());
4894 if (!Cond)
4895 return false;
4896
4897 // Change br (X == 0 | X == 1), T, F into a switch instruction.
4898 // If this is a bunch of seteq's or'd together, or if it's a bunch of
4899 // 'setne's and'ed together, collect them.
4900
4901 // Try to gather values from a chain of and/or to be turned into a switch
4902 ConstantComparesGatherer ConstantCompare(Cond, DL);
4903 // Unpack the result
4904 SmallVectorImpl<ConstantInt *> &Values = ConstantCompare.Vals;
4905 Value *CompVal = ConstantCompare.CompValue;
4906 unsigned UsedICmps = ConstantCompare.UsedICmps;
4907 Value *ExtraCase = ConstantCompare.Extra;
4908
4909 // If we didn't have a multiply compared value, fail.
4910 if (!CompVal)
4911 return false;
4912
4913 // Avoid turning single icmps into a switch.
4914 if (UsedICmps <= 1)
4915 return false;
4916
4917 bool TrueWhenEqual = match(Cond, m_LogicalOr(m_Value(), m_Value()));
4918
4919 // There might be duplicate constants in the list, which the switch
4920 // instruction can't handle, remove them now.
4921 array_pod_sort(Values.begin(), Values.end(), ConstantIntSortPredicate);
4922 Values.erase(std::unique(Values.begin(), Values.end()), Values.end());
4923
4924 // If Extra was used, we require at least two switch values to do the
4925 // transformation. A switch with one value is just a conditional branch.
4926 if (ExtraCase && Values.size() < 2)
4927 return false;
4928
4929 // TODO: Preserve branch weight metadata, similarly to how
4930 // FoldValueComparisonIntoPredecessors preserves it.
4931
4932 // Figure out which block is which destination.
4933 BasicBlock *DefaultBB = BI->getSuccessor(1);
4934 BasicBlock *EdgeBB = BI->getSuccessor(0);
4935 if (!TrueWhenEqual)
4936 std::swap(DefaultBB, EdgeBB);
4937
4938 BasicBlock *BB = BI->getParent();
4939
4940 LLVM_DEBUG(dbgs() << "Converting 'icmp' chain with " << Values.size()
4941 << " cases into SWITCH. BB is:\n"
4942 << *BB);
4943
4945
4946 // If there are any extra values that couldn't be folded into the switch
4947 // then we evaluate them with an explicit branch first. Split the block
4948 // right before the condbr to handle it.
4949 if (ExtraCase) {
4950 BasicBlock *NewBB = SplitBlock(BB, BI, DTU, /*LI=*/nullptr,
4951 /*MSSAU=*/nullptr, "switch.early.test");
4952
4953 // Remove the uncond branch added to the old block.
4954 Instruction *OldTI = BB->getTerminator();
4955 Builder.SetInsertPoint(OldTI);
4956
4957 // There can be an unintended UB if extra values are Poison. Before the
4958 // transformation, extra values may not be evaluated according to the
4959 // condition, and it will not raise UB. But after transformation, we are
4960 // evaluating extra values before checking the condition, and it will raise
4961 // UB. It can be solved by adding freeze instruction to extra values.
4962 AssumptionCache *AC = Options.AC;
4963
4964 if (!isGuaranteedNotToBeUndefOrPoison(ExtraCase, AC, BI, nullptr))
4965 ExtraCase = Builder.CreateFreeze(ExtraCase);
4966
4967 if (TrueWhenEqual)
4968 Builder.CreateCondBr(ExtraCase, EdgeBB, NewBB);
4969 else
4970 Builder.CreateCondBr(ExtraCase, NewBB, EdgeBB);
4971
4972 OldTI->eraseFromParent();
4973
4974 if (DTU)
4975 Updates.push_back({DominatorTree::Insert, BB, EdgeBB});
4976
4977 // If there are PHI nodes in EdgeBB, then we need to add a new entry to them
4978 // for the edge we just added.
4979 AddPredecessorToBlock(EdgeBB, BB, NewBB);
4980
4981 LLVM_DEBUG(dbgs() << " ** 'icmp' chain unhandled condition: " << *ExtraCase
4982 << "\nEXTRABB = " << *BB);
4983 BB = NewBB;
4984 }
4985
4986 Builder.SetInsertPoint(BI);
4987 // Convert pointer to int before we switch.
4988 if (CompVal->getType()->isPointerTy()) {
4989 CompVal = Builder.CreatePtrToInt(
4990 CompVal, DL.getIntPtrType(CompVal->getType()), "magicptr");
4991 }
4992
4993 // Create the new switch instruction now.
4994 SwitchInst *New = Builder.CreateSwitch(CompVal, DefaultBB, Values.size());
4995
4996 // Add all of the 'cases' to the switch instruction.
4997 for (unsigned i = 0, e = Values.size(); i != e; ++i)
4998 New->addCase(Values[i], EdgeBB);
4999
5000 // We added edges from PI to the EdgeBB. As such, if there were any
5001 // PHI nodes in EdgeBB, they need entries to be added corresponding to
5002 // the number of edges added.
5003 for (BasicBlock::iterator BBI = EdgeBB->begin(); isa<PHINode>(BBI); ++BBI) {
5004 PHINode *PN = cast<PHINode>(BBI);
5005 Value *InVal = PN->getIncomingValueForBlock(BB);
5006 for (unsigned i = 0, e = Values.size() - 1; i != e; ++i)
5007 PN->addIncoming(InVal, BB);
5008 }
5009
5010 // Erase the old branch instruction.
5012 if (DTU)
5013 DTU->applyUpdates(Updates);
5014
5015 LLVM_DEBUG(dbgs() << " ** 'icmp' chain result is:\n" << *BB << '\n');
5016 return true;
5017}
5018
5019bool SimplifyCFGOpt::simplifyResume(ResumeInst *RI, IRBuilder<> &Builder) {
5020 if (isa<PHINode>(RI->getValue()))
5021 return simplifyCommonResume(RI);
5022 else if (isa<LandingPadInst>(RI->getParent()->getFirstNonPHI()) &&
5023 RI->getValue() == RI->getParent()->getFirstNonPHI())
5024 // The resume must unwind the exception that caused control to branch here.
5025 return simplifySingleResume(RI);
5026
5027 return false;
5028}
5029
5030// Check if cleanup block is empty
5032 for (Instruction &I : R) {
5033 auto *II = dyn_cast<IntrinsicInst>(&I);
5034 if (!II)
5035 return false;
5036
5037 Intrinsic::ID IntrinsicID = II->getIntrinsicID();
5038 switch (IntrinsicID) {
5039 case Intrinsic::dbg_declare:
5040 case Intrinsic::dbg_value:
5041 case Intrinsic::dbg_label:
5042 case Intrinsic::lifetime_end:
5043 break;
5044 default:
5045 return false;
5046 }
5047 }
5048 return true;
5049}
5050
5051// Simplify resume that is shared by several landing pads (phi of landing pad).
5052bool SimplifyCFGOpt::simplifyCommonResume(ResumeInst *RI) {
5053 BasicBlock *BB = RI->getParent();
5054
5055 // Check that there are no other instructions except for debug and lifetime
5056 // intrinsics between the phi's and resume instruction.
5059 return false;
5060
5061 SmallSetVector<BasicBlock *, 4> TrivialUnwindBlocks;
5062 auto *PhiLPInst = cast<PHINode>(RI->getValue());
5063
5064 // Check incoming blocks to see if any of them are trivial.
5065 for (unsigned Idx = 0, End = PhiLPInst->getNumIncomingValues(); Idx != End;
5066 Idx++) {
5067 auto *IncomingBB = PhiLPInst->getIncomingBlock(Idx);
5068 auto *IncomingValue = PhiLPInst->getIncomingValue(Idx);
5069
5070 // If the block has other successors, we can not delete it because
5071 // it has other dependents.
5072 if (IncomingBB->getUniqueSuccessor() != BB)
5073 continue;
5074
5075 auto *LandingPad = dyn_cast<LandingPadInst>(IncomingBB->getFirstNonPHI());
5076 // Not the landing pad that caused the control to branch here.
5077 if (IncomingValue != LandingPad)
5078 continue;
5079
5081 make_range(LandingPad->getNextNode(), IncomingBB->getTerminator())))
5082 TrivialUnwindBlocks.insert(IncomingBB);
5083 }
5084
5085 // If no trivial unwind blocks, don't do any simplifications.
5086 if (TrivialUnwindBlocks.empty())
5087 return false;
5088
5089 // Turn all invokes that unwind here into calls.
5090 for (auto *TrivialBB : TrivialUnwindBlocks) {
5091 // Blocks that will be simplified should be removed from the phi node.
5092 // Note there could be multiple edges to the resume block, and we need
5093 // to remove them all.
5094 while (PhiLPInst->getBasicBlockIndex(TrivialBB) != -1)
5095 BB->removePredecessor(TrivialBB, true);
5096
5097 for (BasicBlock *Pred :
5099 removeUnwindEdge(Pred, DTU);
5100 ++NumInvokes;
5101 }
5102
5103 // In each SimplifyCFG run, only the current processed block can be erased.
5104 // Otherwise, it will break the iteration of SimplifyCFG pass. So instead
5105 // of erasing TrivialBB, we only remove the branch to the common resume
5106 // block so that we can later erase the resume block since it has no
5107 // predecessors.
5108 TrivialBB->getTerminator()->eraseFromParent();
5109 new UnreachableInst(RI->getContext(), TrivialBB);
5110 if (DTU)
5111 DTU->applyUpdates({{DominatorTree::Delete, TrivialBB, BB}});
5112 }
5113
5114 // Delete the resume block if all its predecessors have been removed.
5115 if (pred_empty(BB))
5116 DeleteDeadBlock(BB, DTU);
5117
5118 return !TrivialUnwindBlocks.empty();
5119}
5120
5121// Simplify resume that is only used by a single (non-phi) landing pad.
5122bool SimplifyCFGOpt::simplifySingleResume(ResumeInst *RI) {
5123 BasicBlock *BB = RI->getParent();
5124 auto *LPInst = cast<LandingPadInst>(BB->getFirstNonPHI());
5125 assert(RI->getValue() == LPInst &&
5126 "Resume must unwind the exception that caused control to here");
5127
5128 // Check that there are no other instructions except for debug intrinsics.
5130 make_range<Instruction *>(LPInst->getNextNode(), RI)))
5131 return false;
5132
5133 // Turn all invokes that unwind here into calls and delete the basic block.
5135 removeUnwindEdge(Pred, DTU);
5136 ++NumInvokes;
5137 }
5138
5139 // The landingpad is now unreachable. Zap it.
5140 DeleteDeadBlock(BB, DTU);
5141 return true;
5142}
5143
5145 // If this is a trivial cleanup pad that executes no instructions, it can be
5146 // eliminated. If the cleanup pad continues to the caller, any predecessor
5147 // that is an EH pad will be updated to continue to the caller and any
5148 // predecessor that terminates with an invoke instruction will have its invoke
5149 // instruction converted to a call instruction. If the cleanup pad being
5150 // simplified does not continue to the caller, each predecessor will be
5151 // updated to continue to the unwind destination of the cleanup pad being
5152 // simplified.
5153 BasicBlock *BB = RI->getParent();
5154 CleanupPadInst *CPInst = RI->getCle