GVN.cpp

Go to the documentation of this file.

//===- GVN.cpp - Eliminate redundant values and loads ---------------------===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//
//
// This pass performs global value numbering to eliminate fully redundant
// instructions.  It also performs simple dead load elimination.
//
// Note that this pass does the value numbering itself; it does not use the
// ValueNumbering analysis passes.
//
//===----------------------------------------------------------------------===//
 
#include "llvm/Transforms/Scalar/GVN.h"
#include "ScalarOptions.h"
#include "llvm/ADT/DenseMap.h"
#include "llvm/ADT/DepthFirstIterator.h"
#include "llvm/ADT/Hashing.h"
#include "llvm/ADT/MapVector.h"
#include "llvm/ADT/PostOrderIterator.h"
#include "llvm/ADT/STLExtras.h"
#include "llvm/ADT/SetVector.h"
#include "llvm/ADT/SmallPtrSet.h"
#include "llvm/ADT/SmallVector.h"
#include "llvm/ADT/Statistic.h"
#include "llvm/Analysis/AliasAnalysis.h"
#include "llvm/Analysis/AssumeBundleQueries.h"
#include "llvm/Analysis/AssumptionCache.h"
#include "llvm/Analysis/CFG.h"
#include "llvm/Analysis/DomTreeUpdater.h"
#include "llvm/Analysis/GlobalsModRef.h"
#include "llvm/Analysis/InstructionPrecedenceTracking.h"
#include "llvm/Analysis/InstructionSimplify.h"
#include "llvm/Analysis/Loads.h"
#include "llvm/Analysis/LoopInfo.h"
#include "llvm/Analysis/MemoryBuiltins.h"
#include "llvm/Analysis/MemoryDependenceAnalysis.h"
#include "llvm/Analysis/MemorySSA.h"
#include "llvm/Analysis/MemorySSAUpdater.h"
#include "llvm/Analysis/OptimizationRemarkEmitter.h"
#include "llvm/Analysis/PHITransAddr.h"
#include "llvm/Analysis/TargetLibraryInfo.h"
#include "llvm/Analysis/ValueTracking.h"
#include "llvm/IR/Attributes.h"
#include "llvm/IR/BasicBlock.h"
#include "llvm/IR/Constant.h"
#include "llvm/IR/Constants.h"
#include "llvm/IR/DebugLoc.h"
#include "llvm/IR/Dominators.h"
#include "llvm/IR/Function.h"
#include "llvm/IR/InstrTypes.h"
#include "llvm/IR/Instruction.h"
#include "llvm/IR/Instructions.h"
#include "llvm/IR/IntrinsicInst.h"
#include "llvm/IR/LLVMContext.h"
#include "llvm/IR/Metadata.h"
#include "llvm/IR/Module.h"
#include "llvm/IR/PassManager.h"
#include "llvm/IR/PatternMatch.h"
#include "llvm/IR/ProfDataUtils.h"
#include "llvm/IR/Type.h"
#include "llvm/IR/Use.h"
#include "llvm/IR/Value.h"
#include "llvm/InitializePasses.h"
#include "llvm/Pass.h"
#include "llvm/Support/Casting.h"
#include "llvm/Support/CommandLine.h"
#include "llvm/Support/Compiler.h"
#include "llvm/Support/Debug.h"
#include "llvm/Support/ErrorHandling.h"
#include "llvm/Support/raw_ostream.h"
#include "llvm/Transforms/Scalar/GVNValueTable.h"
#include "llvm/Transforms/Utils/AssumeBundleBuilder.h"
#include "llvm/Transforms/Utils/BasicBlockUtils.h"
#include "llvm/Transforms/Utils/Local.h"
#include "llvm/Transforms/Utils/SSAUpdater.h"
#include "llvm/Transforms/Utils/VNCoercion.h"
#include <algorithm>
#include <cassert>
#include <cstdint>
#include <optional>
#include <utility>
#include <variant>
 
using namespace llvm;
using namespace llvm::VNCoercion;
using namespace PatternMatch;
 
#define DEBUG_TYPE "gvn"
 
STATISTIC(NumGVNInstr, "Number of instructions deleted");
STATISTIC(NumGVNLoad, "Number of loads deleted");
STATISTIC(NumGVNPRE, "Number of instructions PRE'd");
STATISTIC(NumGVNBlocks, "Number of blocks merged");
STATISTIC(NumGVNSimpl, "Number of instructions simplified");
STATISTIC(NumGVNEqProp, "Number of equalities propagated");
STATISTIC(NumPRELoad, "Number of loads PRE'd");
STATISTIC(NumPRELoopLoad, "Number of loop loads PRE'd");
STATISTIC(NumPRELoadMoved2CEPred,
          "Number of loads moved to predecessor of a critical edge in PRE");
 
STATISTIC(IsValueFullyAvailableInBlockNumSpeculationsMax,
          "Number of blocks speculated as available in "
          "IsValueFullyAvailableInBlock(), max");
STATISTIC(MaxBBSpeculationCutoffReachedTimes,
          "Number of times we we reached gvn-max-block-speculations cut-off "
          "preventing further exploration");
 
struct llvm::GVNValueTable::Expression {
  uint32_t Opcode;
  bool Commutative = false;
  // The type is not necessarily the result type of the expression, it may be
  // any additional type needed to disambiguate the expression.
  Type *Ty = nullptr;
  SmallVector<uint32_t, 4> VarArgs;
 
  AttributeList Attrs;
 
  Expression(uint32_t Op = ~2U) : Opcode(Op) {}
 
  bool operator==(const Expression &Other) const {
    if (Opcode != Other.Opcode)
      return false;
    if (Opcode == ~0U || Opcode == ~1U)
      return true;
    if (Ty != Other.Ty)
      return false;
    if (VarArgs != Other.VarArgs)
      return false;
    if ((!Attrs.isEmpty() || !Other.Attrs.isEmpty()) &&
        !Attrs.intersectWith(Ty->getContext(), Other.Attrs).has_value())
      return false;
    return true;
  }
 
  friend hash_code hash_value(const Expression &Value) {
    return hash_combine(Value.Opcode, Value.Ty,
                        hash_combine_range(Value.VarArgs));
  }
};
 
template <> struct llvm::DenseMapInfo<GVNValueTable::Expression> {
  static unsigned getHashValue(const GVNValueTable::Expression &E) {
    using llvm::hash_value;
 
    return static_cast<unsigned>(hash_value(E));
  }
 
  static bool isEqual(const GVNValueTable::Expression &LHS,
                      const GVNValueTable::Expression &RHS) {
    return LHS == RHS;
  }
};
 
/// A mapping from value numbers to lists of Value*'s that
/// have that value number. Use getLeaders to query it.
class llvm::GVNLeaderMap {
public:
  struct LeaderTableEntry {
    // Use AssertingVH here to catch dangling Value*'s in the leader table.
    // Will crash if the value gets deleted before the AssertingVH is
    // destroyed.
    AssertingVH<Value> Val;
    const BasicBlock *BB;
    LeaderTableEntry(Value *V, const BasicBlock *BB) : Val(V), BB(BB) {}
  };
 
private:
  struct LeaderListNode {
    LeaderTableEntry Entry;
    LeaderListNode *Next;
    LeaderListNode(Value *V, const BasicBlock *BB, LeaderListNode *Next)
        : Entry(V, BB), Next(Next) {}
  };
  DenseMap<uint32_t, LeaderListNode> NumToLeaders;
  BumpPtrAllocator TableAllocator;
 
public:
  class leader_iterator {
    const LeaderListNode *Current;
 
  public:
    using iterator_category = std::forward_iterator_tag;
    using value_type = const LeaderTableEntry;
    using difference_type = std::ptrdiff_t;
    using pointer = value_type *;
    using reference = value_type &;
 
    leader_iterator(const LeaderListNode *C) : Current(C) {}
    leader_iterator &operator++() {
      assert(Current && "Dereferenced end of leader list!");
      Current = Current->Next;
      return *this;
    }
    bool operator==(const leader_iterator &Other) const {
      return Current == Other.Current;
    }
    bool operator!=(const leader_iterator &Other) const {
      return Current != Other.Current;
    }
    reference operator*() const { return Current->Entry; }
  };
 
  iterator_range<leader_iterator> getLeaders(uint32_t N) {
    auto I = NumToLeaders.find(N);
    if (I == NumToLeaders.end()) {
      return iterator_range(leader_iterator(nullptr), leader_iterator(nullptr));
    }
 
    return iterator_range(leader_iterator(&I->second),
                          leader_iterator(nullptr));
  }
 
  LLVM_ABI void insert(uint32_t N, Value *V, const BasicBlock *BB);
  LLVM_ABI void erase(uint32_t N, Instruction *I, const BasicBlock *BB);
  void clear() {
    // Manually destroy non-head nodes (in BumpPtrAllocator) to properly
    // clean up AssertingVH handles before Reset(). Head nodes are destroyed
    // by NumToLeaders.clear() below.
    for (auto &[_, HeadNode] : NumToLeaders) {
      LeaderListNode *N = HeadNode.Next;
      while (N) {
        auto *Next = N->Next;
        N->~LeaderListNode();
        N = Next;
      }
    }
    NumToLeaders.clear();
    TableAllocator.Reset();
  }
};
 
class GVNLegacyPass;
 
/// The core GVN pass object.
///
/// FIXME: We should have a good summary of the GVN algorithm implemented by
/// this particular pass here.
class llvm::GVNPassImpl {
  const ScalarOptions *Opts = nullptr;
  llvm::GVNOptions Options;
 
public:
  struct AvailableValue;
  struct AvailableValueInBlock;
 
  GVNPassImpl(llvm::GVNOptions Options = {}) : Options(Options) {}
 
  /// This removes the specified instruction from
  /// our various maps and marks it for deletion.
  void salvageAndRemoveInstruction(Instruction *I);
 
  DominatorTree &getDominatorTree() const { return *DT; }
  AAResults *getAliasAnalysis() const { return VN.getAliasAnalysis(); }
  MemoryDependenceResults &getMemDep() const { return *MD; }
 
  bool isScalarPREEnabled() const;
  bool isLoadPREEnabled() const;
  bool isLoadInLoopPREEnabled() const;
  bool isLoadPRESplitBackedgeEnabled() const;
  bool isMemDepEnabled() const;
  bool isMemorySSAEnabled() const;
 
private:
  friend class GVNPass;
  friend class ::GVNLegacyPass;
 
  MemoryDependenceResults *MD = nullptr;
  DominatorTree *DT = nullptr;
  const TargetLibraryInfo *TLI = nullptr;
  AssumptionCache *AC = nullptr;
  SetVector<BasicBlock *> DeadBlocks;
  OptimizationRemarkEmitter *ORE = nullptr;
  ImplicitControlFlowTracking *ICF = nullptr;
  LoopInfo *LI = nullptr;
  AAResults *AA = nullptr;
  MemorySSAUpdater *MSSAU = nullptr;
  EarliestEscapeAnalysis *EA = nullptr;
 
  GVNValueTable VN;
 
  GVNLeaderMap LeaderTable;
 
  // Map the block to reversed postorder traversal number. It is used to
  // find back edge easily.
  DenseMap<AssertingVH<BasicBlock>, uint32_t> BlockRPONumber;
 
  // This is set 'true' initially and also when new blocks have been added to
  // the function being analyzed. This boolean is used to control the updating
  // of BlockRPONumber prior to accessing the contents of BlockRPONumber.
  bool InvalidBlockRPONumbers = true;
 
  using LoadDepVect = SmallVector<NonLocalDepResult, 64>;
  using AvailValInBlkVect = SmallVector<AvailableValueInBlock, 64>;
  using UnavailBlkVect = SmallVector<BasicBlock *, 64>;
 
  bool run(Function &F, AssumptionCache &RunAC, DominatorTree &RunDT,
           const TargetLibraryInfo &RunTLI, AAResults &RunAA,
           MemoryDependenceResults *RunMD, LoopInfo &LI,
           OptimizationRemarkEmitter *ORE, MemorySSA *MSSA = nullptr);
 
  // List of critical edges to be split between iterations.
  SmallVector<std::pair<Instruction *, unsigned>, 4> ToSplit;
 
  enum class DepKind {
    Other = 0, // Unknown value.
    Def,       // Exactly overlapping locations.
    Clobber,   // Reaching value superset of needed bits.
    Select,    // Reaching value is a select of two reaching addresses.
  };
 
  // Describe a memory location value, such that there exists a path to a point
  // in the program, along which that memory location is not modified.
  struct ReachingMemVal {
    DepKind Kind;
    BasicBlock *Block;
    const Value *Addr;
    Instruction *Inst;
    int32_t Offset;
    // For DepKind::Select only: the select instruction and the two addresses
    // referenced by the "true" and "false" side of the select-dependent load.
    SelectInst *Sel = nullptr;
    const Value *SelTrueAddr = nullptr;
    const Value *SelFalseAddr = nullptr;
 
    static ReachingMemVal getUnknown(BasicBlock *BB, const Value *Addr,
                                     Instruction *Inst = nullptr) {
      return {DepKind::Other, BB, Addr, Inst, -1};
    }
 
    static ReachingMemVal getDef(const Value *Addr, Instruction *Inst) {
      return {DepKind::Def, Inst->getParent(), Addr, Inst, -1};
    }
 
    static ReachingMemVal getClobber(const Value *Addr, Instruction *Inst,
                                     int32_t Offset = -1) {
      return {DepKind::Clobber, Inst->getParent(), Addr, Inst, Offset};
    }
 
    static ReachingMemVal getSelect(BasicBlock *BB, SelectInst *Sel,
                                    const Value *TrueAddr,
                                    const Value *FalseAddr) {
      return {DepKind::Select, BB,       nullptr, nullptr, -1, Sel,
              TrueAddr,        FalseAddr};
    }
  };
 
  struct DependencyBlockInfo {
    DependencyBlockInfo() = delete;
    DependencyBlockInfo(const PHITransAddr &Addr, MemoryAccess *ClobberMA)
        : Addr(Addr), InitialClobberMA(ClobberMA), ClobberMA(ClobberMA),
          ForceUnknown(false), Visited(false) {}
    PHITransAddr Addr;
    MemoryAccess *InitialClobberMA;
    MemoryAccess *ClobberMA;
    std::optional<ReachingMemVal> MemVal;
    bool ForceUnknown : 1;
    bool Visited : 1;
  };
 
  using DependencyBlockSet = DenseMap<BasicBlock *, DependencyBlockInfo>;
 
  std::optional<GVNPassImpl::ReachingMemVal> scanMemoryAccessesUsers(
      const MemoryLocation &Loc, bool IsInvariantLoad, BasicBlock *BB,
      const SmallVectorImpl<MemoryAccess *> &ClobbersList, MemorySSA &MSSA,
      BatchAAResults &AA, LoadInst *L = nullptr);
 
  std::optional<GVNPassImpl::ReachingMemVal>
  accessMayModifyLocation(MemoryAccess *ClobberMA, const MemoryLocation &Loc,
                          Align LoadAlign, bool IsInvariantLoad, BasicBlock *BB,
                          MemorySSA &MSSA, BatchAAResults &AA);
 
  bool collectPredecessors(BasicBlock *BB, const PHITransAddr &Addr,
                           MemoryAccess *ClobberMA, DependencyBlockSet &Blocks,
                           SmallVectorImpl<BasicBlock *> &Worklist);
 
  void collectClobberList(SmallVectorImpl<MemoryAccess *> &Clobbers,
                          BasicBlock *BB, const DependencyBlockInfo &StartInfo,
                          const DependencyBlockSet &Blocks, MemorySSA &MSSA);
 
  bool findReachingValuesForLoad(LoadInst *Inst,
                                 SmallVectorImpl<ReachingMemVal> &Values,
                                 MemorySSA &MSSA, AAResults &AA);
 
  // Helper functions of redundant load elimination.
  bool processLoad(LoadInst *L);
  bool processMaskedLoad(IntrinsicInst *I);
  bool processNonLocalLoad(LoadInst *L);
  bool processNonLocalLoad(LoadInst *L, SmallVectorImpl<ReachingMemVal> &Deps);
  bool processAssumeIntrinsic(AssumeInst *II);
 
  /// Given a local dependency (Def or Clobber) determine if a value is
  /// available for the load.
  std::optional<AvailableValue>
  analyzeLoadAvailability(LoadInst *Load, const ReachingMemVal &Dep,
                          Value *Address);
 
  /// Given a select-dependency for the load (the load address is a select of
  /// \p TrueAddr and \p FalseAddr guarded by \p Sel), determine whether a
  /// value is available by finding dominating values for both addresses.  If
  /// so, the load can be rematerialized as a select of those two values.
  std::optional<AvailableValue>
  analyzeSelectAvailability(LoadInst *Load, SelectInst *Sel, Value *TrueAddr,
                            Value *FalseAddr, Instruction *From);
 
  /// Given a list of non-local dependencies, determine if a value is
  /// available for the load in each specified block.  If it is, add it to
  /// ValuesPerBlock.  If not, add it to UnavailableBlocks.
  void analyzeLoadAvailability(LoadInst *Load,
                               SmallVectorImpl<ReachingMemVal> &Deps,
                               AvailValInBlkVect &ValuesPerBlock,
                               UnavailBlkVect &UnavailableBlocks);
 
  /// Given a critical edge from Pred to LoadBB, find a load instruction
  /// which is identical to Load from another successor of Pred.
  LoadInst *findLoadToHoistIntoPred(BasicBlock *Pred, BasicBlock *LoadBB,
                                    LoadInst *Load);
 
  bool performLoadPRE(LoadInst *Load, AvailValInBlkVect &ValuesPerBlock,
                      UnavailBlkVect &UnavailableBlocks);
 
  /// Try to replace a load which executes on each loop iteraiton with Phi
  /// translation of load in preheader and load(s) in conditionally executed
  /// paths.
  bool performLoopLoadPRE(LoadInst *Load, AvailValInBlkVect &ValuesPerBlock,
                          UnavailBlkVect &UnavailableBlocks);
 
  /// Eliminates partially redundant \p Load, replacing it with \p
  /// AvailableLoads (connected by Phis if needed).
  void eliminatePartiallyRedundantLoad(
      LoadInst *Load, AvailValInBlkVect &ValuesPerBlock,
      MapVector<BasicBlock *, Value *> &AvailableLoads,
      MapVector<BasicBlock *, LoadInst *> *CriticalEdgePredAndLoad);
 
  // Other helper routines.
  bool processInstruction(Instruction *I);
  bool processBlock(BasicBlock *BB);
  bool replaceWithEquivalentCmp(CmpInst *Cmp);
  bool iterateOnFunction(Function &F);
  bool performPRE(Function &F);
  bool performScalarPRE(Instruction *I);
  bool performScalarPREInsertion(Instruction *Instr, BasicBlock *Pred,
                                 BasicBlock *Curr, unsigned int ValNo);
  Value *findLeader(const BasicBlock *BB, uint32_t Num);
  void cleanupGlobalSets();
  void removeInstruction(Instruction *I);
  void verifyRemoved(const Instruction *I) const;
  bool splitCriticalEdges();
  BasicBlock *splitCriticalEdges(BasicBlock *Pred, BasicBlock *Succ);
  bool
  propagateEquality(Value *LHS, Value *RHS,
                    const std::variant<BasicBlockEdge, Instruction *> &Root);
  bool processFoldableCondBr(CondBrInst *BI);
  void addDeadBlock(BasicBlock *BB);
  void assignValNumForDeadCode();
  void assignBlockRPONumber(Function &F);
};
 
/// Represents a particular available value that we know how to materialize.
/// Materialization of an AvailableValue never fails.  An AvailableValue is
/// implicitly associated with a rematerialization point which is the
/// location of the instruction from which it was formed.
struct GVNPassImpl::AvailableValue {
  enum class ValType {
    SimpleVal, // A simple offsetted value that is accessed.
    LoadVal,   // A value produced by a load.
    MemIntrin, // A memory intrinsic which is loaded from.
    UndefVal,  // A UndefValue representing a value from dead block (which
               // is not yet physically removed from the CFG).
    SelectVal, // A pointer select which is loaded from and for which the load
               // can be replace by a value select.
  };
 
  /// Val - The value that is live out of the block.
  Value *Val;
  /// Kind of the live-out value.
  ValType Kind;
 
  /// Offset - The byte offset in Val that is interesting for the load query.
  unsigned Offset = 0;
  /// V1, V2 - The dominating non-clobbered values of SelectVal.
  Value *V1 = nullptr, *V2 = nullptr;
 
  static AvailableValue get(Value *V, unsigned Offset = 0) {
    AvailableValue Res;
    Res.Val = V;
    Res.Kind = ValType::SimpleVal;
    Res.Offset = Offset;
    return Res;
  }
 
  static AvailableValue getMI(MemIntrinsic *MI, unsigned Offset = 0) {
    AvailableValue Res;
    Res.Val = MI;
    Res.Kind = ValType::MemIntrin;
    Res.Offset = Offset;
    return Res;
  }
 
  static AvailableValue getLoad(LoadInst *Load, unsigned Offset = 0) {
    AvailableValue Res;
    Res.Val = Load;
    Res.Kind = ValType::LoadVal;
    Res.Offset = Offset;
    return Res;
  }
 
  static AvailableValue getUndef() {
    AvailableValue Res;
    Res.Val = nullptr;
    Res.Kind = ValType::UndefVal;
    Res.Offset = 0;
    return Res;
  }
 
  static AvailableValue getSelect(SelectInst *Sel, Value *V1, Value *V2) {
    AvailableValue Res;
    Res.Val = Sel;
    Res.Kind = ValType::SelectVal;
    Res.Offset = 0;
    Res.V1 = V1;
    Res.V2 = V2;
    return Res;
  }
 
  bool isSimpleValue() const { return Kind == ValType::SimpleVal; }
  bool isCoercedLoadValue() const { return Kind == ValType::LoadVal; }
  bool isMemIntrinValue() const { return Kind == ValType::MemIntrin; }
  bool isUndefValue() const { return Kind == ValType::UndefVal; }
  bool isSelectValue() const { return Kind == ValType::SelectVal; }
 
  Value *getSimpleValue() const {
    assert(isSimpleValue() && "Wrong accessor");
    return Val;
  }
 
  LoadInst *getCoercedLoadValue() const {
    assert(isCoercedLoadValue() && "Wrong accessor");
    return cast<LoadInst>(Val);
  }
 
  MemIntrinsic *getMemIntrinValue() const {
    assert(isMemIntrinValue() && "Wrong accessor");
    return cast<MemIntrinsic>(Val);
  }
 
  SelectInst *getSelectInstr() const {
    assert(isSelectValue() && "Wrong accessor");
    return cast<SelectInst>(Val);
  }
 
  /// Emit code at the specified insertion point to adjust the value defined
  /// here to the specified type. This handles various coercion cases.
  Value *MaterializeAdjustedValue(LoadInst *Load, Instruction *InsertPt) const;
};
 
/// Represents an AvailableValue which can be rematerialized at the end of
/// the associated BasicBlock.
struct GVNPassImpl::AvailableValueInBlock {
  /// BB - The basic block in question.
  BasicBlock *BB = nullptr;
 
  /// AV - The actual available value.
  AvailableValue AV;
 
  static AvailableValueInBlock get(BasicBlock *BB, AvailableValue &&AV) {
    AvailableValueInBlock Res;
    Res.BB = BB;
    Res.AV = std::move(AV);
    return Res;
  }
 
  static AvailableValueInBlock get(BasicBlock *BB, Value *V,
                                   unsigned Offset = 0) {
    return get(BB, AvailableValue::get(V, Offset));
  }
 
  static AvailableValueInBlock getUndef(BasicBlock *BB) {
    return get(BB, AvailableValue::getUndef());
  }
 
  /// Emit code at the end of this block to adjust the value defined here to
  /// the specified type. This handles various coercion cases.
  Value *MaterializeAdjustedValue(LoadInst *Load) const {
    return AV.MaterializeAdjustedValue(Load, BB->getTerminator());
  }
};
 
//===----------------------------------------------------------------------===//
//                     ValueTable Internal Functions
//===----------------------------------------------------------------------===//
 
GVNValueTable::Expression GVNValueTable::createExpr(Instruction *I) {
  Expression E;
  E.Ty = I->getType();
  E.Opcode = I->getOpcode();
  if (const GCRelocateInst *GCR = dyn_cast<GCRelocateInst>(I)) {
    // gc.relocate is 'special' call: its second and third operands are
    // not real values, but indices into statepoint's argument list.
    // Use the refered to values for purposes of identity.
    E.VarArgs.push_back(lookupOrAdd(GCR->getOperand(0)));
    E.VarArgs.push_back(lookupOrAdd(GCR->getBasePtr()));
    E.VarArgs.push_back(lookupOrAdd(GCR->getDerivedPtr()));
  } else {
    for (Use &Op : I->operands())
      E.VarArgs.push_back(lookupOrAdd(Op));
  }
  if (I->isCommutative()) {
    // Ensure that commutative instructions that only differ by a permutation
    // of their operands get the same value number by sorting the operand value
    // numbers.  Since commutative operands are the 1st two operands it is more
    // efficient to sort by hand rather than using, say, std::sort.
    assert(I->getNumOperands() >= 2 && "Unsupported commutative instruction!");
    if (E.VarArgs[0] > E.VarArgs[1])
      std::swap(E.VarArgs[0], E.VarArgs[1]);
    E.Commutative = true;
  }
 
  if (auto *IVI = dyn_cast<InsertValueInst>(I)) {
    E.VarArgs.append(IVI->idx_begin(), IVI->idx_end());
  } else if (auto *SVI = dyn_cast<ShuffleVectorInst>(I)) {
    ArrayRef<int> ShuffleMask = SVI->getShuffleMask();
    E.VarArgs.append(ShuffleMask.begin(), ShuffleMask.end());
  } else if (auto *CB = dyn_cast<CallBase>(I)) {
    E.Attrs = CB->getAttributes();
  }
 
  return E;
}
 
GVNValueTable::Expression
GVNValueTable::createCmpExpr(unsigned Opcode, CmpInst::Predicate Predicate,
                             Value *LHS, Value *RHS) {
  assert((Opcode == Instruction::ICmp || Opcode == Instruction::FCmp) &&
         "Not a comparison!");
  Expression E;
  E.Ty = CmpInst::makeCmpResultType(LHS->getType());
  E.VarArgs.push_back(lookupOrAdd(LHS));
  E.VarArgs.push_back(lookupOrAdd(RHS));
 
  // Sort the operand value numbers so x<y and y>x get the same value number.
  if (E.VarArgs[0] > E.VarArgs[1]) {
    std::swap(E.VarArgs[0], E.VarArgs[1]);
    Predicate = CmpInst::getSwappedPredicate(Predicate);
  }
  E.Opcode = (Opcode << 8) | Predicate;
  E.Commutative = true;
  return E;
}
 
GVNValueTable::Expression
GVNValueTable::createExtractValueExpr(ExtractValueInst *EI) {
  assert(EI && "Not an ExtractValueInst?");
  Expression E;
  E.Ty = EI->getType();
  E.Opcode = 0;
 
  WithOverflowInst *WO = dyn_cast<WithOverflowInst>(EI->getAggregateOperand());
  if (WO != nullptr && EI->getNumIndices() == 1 && *EI->idx_begin() == 0) {
    // EI is an extract from one of our with.overflow intrinsics. Synthesize
    // a semantically equivalent expression instead of an extract value
    // expression.
    E.Opcode = WO->getBinaryOp();
    E.VarArgs.push_back(lookupOrAdd(WO->getLHS()));
    E.VarArgs.push_back(lookupOrAdd(WO->getRHS()));
    return E;
  }
 
  // Not a recognised intrinsic. Fall back to producing an extract value
  // expression.
  E.Opcode = EI->getOpcode();
  for (Use &Op : EI->operands())
    E.VarArgs.push_back(lookupOrAdd(Op));
 
  append_range(E.VarArgs, EI->indices());
 
  return E;
}
 
GVNValueTable::Expression GVNValueTable::createGEPExpr(GetElementPtrInst *GEP) {
  Expression E;
  Type *PtrTy = GEP->getType()->getScalarType();
  const DataLayout &DL = GEP->getDataLayout();
  unsigned BitWidth = DL.getIndexTypeSizeInBits(PtrTy);
  SmallMapVector<Value *, APInt, 4> VariableOffsets;
  APInt ConstantOffset(BitWidth, 0);
  if (GEP->collectOffset(DL, BitWidth, VariableOffsets, ConstantOffset)) {
    // Convert into offset representation, to recognize equivalent address
    // calculations that use different type encoding.
    LLVMContext &Context = GEP->getContext();
    E.Opcode = GEP->getOpcode();
    E.Ty = nullptr;
    E.VarArgs.push_back(lookupOrAdd(GEP->getPointerOperand()));
    for (const auto &[V, Scale] : VariableOffsets) {
      E.VarArgs.push_back(lookupOrAdd(V));
      E.VarArgs.push_back(lookupOrAdd(ConstantInt::get(Context, Scale)));
    }
    if (!ConstantOffset.isZero())
      E.VarArgs.push_back(
          lookupOrAdd(ConstantInt::get(Context, ConstantOffset)));
  } else {
    // If converting to offset representation fails (for scalable vectors),
    // fall back to type-based implementation.
    E.Opcode = GEP->getOpcode();
    E.Ty = GEP->getSourceElementType();
    for (Use &Op : GEP->operands())
      E.VarArgs.push_back(lookupOrAdd(Op));
  }
  return E;
}
 
//===----------------------------------------------------------------------===//
//                     ValueTable External Functions
//===----------------------------------------------------------------------===//
 
GVNValueTable::GVNValueTable() = default;
GVNValueTable::GVNValueTable(const GVNValueTable &) = default;
GVNValueTable::GVNValueTable(GVNValueTable &&) = default;
GVNValueTable::~GVNValueTable() = default;
GVNValueTable &GVNValueTable::operator=(const GVNValueTable &Arg) = default;
 
/// add - Insert a value into the table with a specified value number.
void GVNValueTable::add(Value *V, uint32_t Num) {
  ValueNumbering.insert(std::make_pair(V, Num));
  if (PHINode *PN = dyn_cast<PHINode>(V))
    NumberingPhi[Num] = PN;
}
 
/// Include the incoming memory state into the hash of the expression for the
/// given instruction. If the incoming memory state is:
/// * LiveOnEntry, add the value number of the entry block,
/// * a MemoryPhi, add the value number of the basic block corresponding to that
/// MemoryPhi,
/// * a MemoryDef, add the value number of the memory setting instruction.
void GVNValueTable::addMemoryStateToExp(Instruction *I, Expression &Exp) {
  assert(MSSA && "addMemoryStateToExp should not be called without MemorySSA");
  assert(MSSA->getMemoryAccess(I) && "Instruction does not access memory");
  MemoryAccess *MA = MSSA->getSkipSelfWalker()->getClobberingMemoryAccess(I);
  Exp.VarArgs.push_back(lookupOrAdd(MA));
}
 
uint32_t GVNValueTable::lookupOrAddCall(CallInst *C) {
  // FIXME: Currently the calls which may access the thread id may
  // be considered as not accessing the memory. But this is
  // problematic for coroutines, since coroutines may resume in a
  // different thread. So we disable the optimization here for the
  // correctness. However, it may block many other correct
  // optimizations. Revert this one when we detect the memory
  // accessing kind more precisely.
  if (C->getFunction()->isPresplitCoroutine()) {
    ValueNumbering[C] = NextValueNumber;
    return NextValueNumber++;
  }
 
  // Do not combine convergent calls since they implicitly depend on the set of
  // threads that is currently executing, and they might be in different basic
  // blocks.
  if (C->isConvergent()) {
    ValueNumbering[C] = NextValueNumber;
    return NextValueNumber++;
  }
 
  // Conservatively assign unique value numbers to calls with operand bundles.
  // TODO: Bundle names could be included in the value numbering expression to
  // allow combining calls with identical bundles.
  if (C->hasOperandBundles()) {
    ValueNumbering[C] = NextValueNumber;
    return NextValueNumber++;
  }
 
  if (AA->doesNotAccessMemory(C)) {
    Expression Exp = createExpr(C);
    uint32_t E = assignExpNewValueNum(Exp).first;
    ValueNumbering[C] = E;
    return E;
  }
 
  if (MD && AA->onlyReadsMemory(C)) {
    Expression Exp = createExpr(C);
    auto [E, IsValNumNew] = assignExpNewValueNum(Exp);
    if (IsValNumNew) {
      ValueNumbering[C] = E;
      return E;
    }
 
    MemDepResult LocalDep = MD->getDependency(C);
 
    if (!LocalDep.isDef() && !LocalDep.isNonLocal()) {
      ValueNumbering[C] = NextValueNumber;
      return NextValueNumber++;
    }
 
    if (LocalDep.isDef()) {
      // For masked load/store intrinsics, the local_dep may actually be
      // a normal load or store instruction.
      CallInst *LocalDepCall = dyn_cast<CallInst>(LocalDep.getInst());
 
      if (!LocalDepCall || LocalDepCall->arg_size() != C->arg_size()) {
        ValueNumbering[C] = NextValueNumber;
        return NextValueNumber++;
      }
 
      for (unsigned I = 0, E = C->arg_size(); I < E; ++I) {
        uint32_t CVN = lookupOrAdd(C->getArgOperand(I));
        uint32_t LocalDepCallVN = lookupOrAdd(LocalDepCall->getArgOperand(I));
        if (CVN != LocalDepCallVN) {
          ValueNumbering[C] = NextValueNumber;
          return NextValueNumber++;
        }
      }
 
      uint32_t V = lookupOrAdd(LocalDepCall);
      ValueNumbering[C] = V;
      return V;
    }
 
    // Non-local case.
    const MemoryDependenceResults::NonLocalDepInfo &Deps =
        MD->getNonLocalCallDependency(C);
    // FIXME: Move the checking logic to MemDep!
    CallInst *CDep = nullptr;
 
    // Check to see if we have a single dominating call instruction that is
    // identical to C.
    for (const NonLocalDepEntry &I : Deps) {
      if (I.getResult().isNonLocal())
        continue;
 
      // We don't handle non-definitions.  If we already have a call, reject
      // instruction dependencies.
      if (!I.getResult().isDef() || CDep != nullptr) {
        CDep = nullptr;
        break;
      }
 
      CallInst *NonLocalDepCall = dyn_cast<CallInst>(I.getResult().getInst());
      // FIXME: All duplicated with non-local case.
      if (NonLocalDepCall && DT->properlyDominates(I.getBB(), C->getParent())) {
        CDep = NonLocalDepCall;
        continue;
      }
 
      CDep = nullptr;
      break;
    }
 
    if (!CDep) {
      ValueNumbering[C] = NextValueNumber;
      return NextValueNumber++;
    }
 
    if (CDep->arg_size() != C->arg_size()) {
      ValueNumbering[C] = NextValueNumber;
      return NextValueNumber++;
    }
    for (unsigned I = 0, E = C->arg_size(); I < E; ++I) {
      uint32_t CVN = lookupOrAdd(C->getArgOperand(I));
      uint32_t CDepVN = lookupOrAdd(CDep->getArgOperand(I));
      if (CVN != CDepVN) {
        ValueNumbering[C] = NextValueNumber;
        return NextValueNumber++;
      }
    }
 
    uint32_t V = lookupOrAdd(CDep);
    ValueNumbering[C] = V;
    return V;
  }
 
  if (MSSA && IsMSSAEnabled && AA->onlyReadsMemory(C)) {
    Expression Exp = createExpr(C);
    addMemoryStateToExp(C, Exp);
    auto [V, _] = assignExpNewValueNum(Exp);
    ValueNumbering[C] = V;
    return V;
  }
 
  ValueNumbering[C] = NextValueNumber;
  return NextValueNumber++;
}
 
/// Returns the value number for the specified load or store instruction.
uint32_t GVNValueTable::computeLoadStoreVN(Instruction *I) {
  if (!MSSA || !IsMSSAEnabled) {
    ValueNumbering[I] = NextValueNumber;
    return NextValueNumber++;
  }
 
  Expression Exp;
  Exp.Ty = I->getType();
  Exp.Opcode = I->getOpcode();
  for (Use &Op : I->operands())
    Exp.VarArgs.push_back(lookupOrAdd(Op));
  addMemoryStateToExp(I, Exp);
 
  auto [V, _] = assignExpNewValueNum(Exp);
  ValueNumbering[I] = V;
  return V;
}
 
/// Returns true if a value number exists for the specified value.
bool GVNValueTable::exists(Value *V) const {
  return ValueNumbering.contains(V);
}
 
uint32_t GVNValueTable::lookupOrAdd(MemoryAccess *MA) {
  return MSSA->isLiveOnEntryDef(MA) || isa<MemoryPhi>(MA)
             ? lookupOrAdd(MA->getBlock())
             : lookupOrAdd(cast<MemoryUseOrDef>(MA)->getMemoryInst());
}
 
/// lookupOrAdd - Returns the value number for the specified value, assigning
/// it a new number if it did not have one before.
uint32_t GVNValueTable::lookupOrAdd(Value *V) {
  auto VI = ValueNumbering.find(V);
  if (VI != ValueNumbering.end())
    return VI->second;
 
  auto *I = dyn_cast<Instruction>(V);
  if (!I) {
    ValueNumbering[V] = NextValueNumber;
    if (isa<BasicBlock>(V))
      NumberingBB[NextValueNumber] = cast<BasicBlock>(V);
    return NextValueNumber++;
  }
 
  Expression Exp;
  switch (I->getOpcode()) {
    case Instruction::Call:
      return lookupOrAddCall(cast<CallInst>(I));
    case Instruction::FNeg:
    case Instruction::Add:
    case Instruction::FAdd:
    case Instruction::Sub:
    case Instruction::FSub:
    case Instruction::Mul:
    case Instruction::FMul:
    case Instruction::UDiv:
    case Instruction::SDiv:
    case Instruction::FDiv:
    case Instruction::URem:
    case Instruction::SRem:
    case Instruction::FRem:
    case Instruction::Shl:
    case Instruction::LShr:
    case Instruction::AShr:
    case Instruction::And:
    case Instruction::Or:
    case Instruction::Xor:
    case Instruction::Trunc:
    case Instruction::ZExt:
    case Instruction::SExt:
    case Instruction::FPToUI:
    case Instruction::FPToSI:
    case Instruction::UIToFP:
    case Instruction::SIToFP:
    case Instruction::FPTrunc:
    case Instruction::FPExt:
    case Instruction::PtrToInt:
    case Instruction::PtrToAddr:
    case Instruction::IntToPtr:
    case Instruction::AddrSpaceCast:
    case Instruction::BitCast:
    case Instruction::Select:
    case Instruction::Freeze:
    case Instruction::ExtractElement:
    case Instruction::InsertElement:
    case Instruction::ShuffleVector:
    case Instruction::InsertValue:
      Exp = createExpr(I);
      break;
    case Instruction::ICmp:
    case Instruction::FCmp:
      Exp = createCmpExpr(I->getOpcode(), cast<CmpInst>(I)->getPredicate(),
                          I->getOperand(0), I->getOperand(1));
      break;
    case Instruction::GetElementPtr:
      Exp = createGEPExpr(cast<GetElementPtrInst>(I));
      break;
    case Instruction::ExtractValue:
      Exp = createExtractValueExpr(cast<ExtractValueInst>(I));
      break;
    case Instruction::PHI:
      ValueNumbering[V] = NextValueNumber;
      NumberingPhi[NextValueNumber] = cast<PHINode>(V);
      return NextValueNumber++;
    case Instruction::Load:
    case Instruction::Store:
      return computeLoadStoreVN(I);
    default:
      ValueNumbering[V] = NextValueNumber;
      return NextValueNumber++;
  }
 
  uint32_t E = assignExpNewValueNum(Exp).first;
  ValueNumbering[V] = E;
  return E;
}
 
/// Returns the value number of the specified value. Fails if
/// the value has not yet been numbered.
uint32_t GVNValueTable::lookup(Value *V, bool Verify) const {
  auto VI = ValueNumbering.find(V);
  if (Verify) {
    assert(VI != ValueNumbering.end() && "Value not numbered?");
    return VI->second;
  }
  return (VI != ValueNumbering.end()) ? VI->second : 0;
}
 
/// Returns the value number of the given comparison,
/// assigning it a new number if it did not have one before.  Useful when
/// we deduced the result of a comparison, but don't immediately have an
/// instruction realizing that comparison to hand.
uint32_t GVNValueTable::lookupOrAddCmp(unsigned Opcode,
                                       CmpInst::Predicate Predicate, Value *LHS,
                                       Value *RHS) {
  Expression Exp = createCmpExpr(Opcode, Predicate, LHS, RHS);
  return assignExpNewValueNum(Exp).first;
}
 
uint32_t GVNValueTable::lookupCmp(unsigned Opcode, CmpInst::Predicate Predicate,
                                  Value *LHS, Value *RHS) {
  Expression Exp = createCmpExpr(Opcode, Predicate, LHS, RHS);
  return ExpressionNumbering.lookup(Exp);
}
 
/// Returns the value number of ptrtoint \p Ptr to \Ty.
uint32_t GVNValueTable::lookupPtrToInt(Value *Ptr, Type *Ty) {
  Expression Exp(Instruction::PtrToInt);
  Exp.Ty = Ty;
  Exp.VarArgs.push_back(lookupOrAdd(Ptr));
  return ExpressionNumbering.lookup(Exp);
}
 
/// Remove all entries from the ValueTable.
void GVNValueTable::clear() {
  ValueNumbering.clear();
  ExpressionNumbering.clear();
  NumberingPhi.clear();
  NumberingBB.clear();
  PhiTranslateTable.clear();
  NextValueNumber = 1;
  Expressions.clear();
  ExprIdx.clear();
  NextExprNumber = 0;
}
 
/// Remove a value from the value numbering.
void GVNValueTable::erase(Value *V) {
  uint32_t Num = ValueNumbering.lookup(V);
  ValueNumbering.erase(V);
  // If V is PHINode, V <--> value number is an one-to-one mapping.
  if (isa<PHINode>(V))
    NumberingPhi.erase(Num);
  else if (isa<BasicBlock>(V))
    NumberingBB.erase(Num);
}
 
/// verifyRemoved - Verify that the value is removed from all internal data
/// structures.
void GVNValueTable::verifyRemoved(const Value *V) const {
  assert(!ValueNumbering.contains(V) &&
         "Inst still occurs in value numbering map!");
}
 
//===----------------------------------------------------------------------===//
//                     LeaderMap External Functions
//===----------------------------------------------------------------------===//
 
/// Push a new Value to the LeaderTable onto the list for its value number.
void GVNLeaderMap::insert(uint32_t N, Value *V, const BasicBlock *BB) {
  const auto &[It, Inserted] = NumToLeaders.try_emplace(N, V, BB, nullptr);
  if (!Inserted) {
    // Key already exists: insert new node after the head.
    auto *NewSlot = TableAllocator.Allocate<LeaderListNode>();
    new (NewSlot) LeaderListNode(V, BB, It->second.Next);
    It->second.Next = NewSlot;
  }
}
 
/// Scan the list of values corresponding to a given
/// value number, and remove the given instruction if encountered.
void GVNLeaderMap::erase(uint32_t N, Instruction *I, const BasicBlock *BB) {
  auto It = NumToLeaders.find(N);
  if (It == NumToLeaders.end())
    return;
 
  LeaderListNode *Prev = nullptr;
  LeaderListNode *Curr = &It->second;
 
  while (Curr && (Curr->Entry.Val != I || Curr->Entry.BB != BB)) {
    Prev = Curr;
    Curr = Curr->Next;
  }
 
  if (!Curr)
    return;
 
  if (Prev) {
    // Non-head node: unlink and destroy.
    Prev->Next = Curr->Next;
    Curr->~LeaderListNode();
    TableAllocator.Deallocate<LeaderListNode>(Curr);
  } else {
    // Head node (stored by value in DenseMap).
    if (!Curr->Next) {
      // Only node; erase from map (DenseMap calls the destructor).
      NumToLeaders.erase(It);
    } else {
      // Move second node's data into head, then destroy second node.
      LeaderListNode *Next = Curr->Next;
      Curr->Entry.Val = std::move(Next->Entry.Val);
      Curr->Entry.BB = Next->Entry.BB;
      Curr->Next = Next->Next;
      Next->~LeaderListNode();
      TableAllocator.Deallocate<LeaderListNode>(Next);
    }
  }
}
 
//===----------------------------------------------------------------------===//
//                                GVN Pass
//===----------------------------------------------------------------------===//
 
bool GVNPassImpl::isScalarPREEnabled() const {
  return Options.AllowScalarPRE.value_or(Opts->enable_scalar_pre);
}
 
bool GVNPassImpl::isLoadPREEnabled() const {
  return Options.AllowLoadPRE.value_or(Opts->enable_load_pre);
}
 
bool GVNPassImpl::isLoadInLoopPREEnabled() const {
  return Options.AllowLoadInLoopPRE.value_or(Opts->enable_load_in_loop_pre);
}
 
bool GVNPassImpl::isLoadPRESplitBackedgeEnabled() const {
  return Options.AllowLoadPRESplitBackedge.value_or(
      Opts->enable_split_backedge_in_load_pre);
}
 
bool GVNPassImpl::isMemDepEnabled() const {
  // MemDep and MemorySSA are mutually exclusive. parseGVNOptions() enforces
  // this for pass parameters, but the -enable-gvn-{memdep,memoryssa} cl::opt
  // overrides default independently, so honor MemorySSA winning here too.
  if (isMemorySSAEnabled())
    return Options.AllowMemDep.value_or(false);
  return Options.AllowMemDep.value_or(valueOr(Opts->enable_gvn_memdep, true));
}
 
bool GVNPassImpl::isMemorySSAEnabled() const {
  return Options.AllowMemorySSA.value_or(Opts->enable_gvn_memoryssa);
}
 
GVNPass::GVNPass(GVNOptions Options)
    : Impl(std::make_unique<GVNPassImpl>(Options)) {}
 
GVNPass::~GVNPass() = default;
 
GVNPass::GVNPass(GVNPass &&) noexcept = default;
 
GVNPass &GVNPass::operator=(GVNPass &&) noexcept = default;
 
PreservedAnalyses GVNPass::run(Function &F, FunctionAnalysisManager &AM) {
  // FIXME: The order of evaluation of these 'getResult' calls is very
  // significant! Re-ordering these variables will cause GVN when run alone to
  // be less effective! We should fix memdep and basic-aa to not exhibit this
  // behavior, but until then don't change the order here.
  Impl->Opts = &getScalarOptions(F);
  auto &AC = AM.getResult<AssumptionAnalysis>(F);
  auto &DT = AM.getResult<DominatorTreeAnalysis>(F);
  auto &TLI = AM.getResult<TargetLibraryAnalysis>(F);
  auto &AA = AM.getResult<AAManager>(F);
  auto *MemDep = Impl->isMemDepEnabled()
                     ? &AM.getResult<MemoryDependenceAnalysis>(F)
                     : nullptr;
  auto &LI = AM.getResult<LoopAnalysis>(F);
  auto *MSSA = AM.getCachedResult<MemorySSAAnalysis>(F);
  if (Impl->isMemorySSAEnabled() && !MSSA) {
    assert(!MemDep &&
           "On-demand computation of MemSSA implies that MemDep is disabled!");
    MSSA = &AM.getResult<MemorySSAAnalysis>(F);
  }
  auto &ORE = AM.getResult<OptimizationRemarkEmitterAnalysis>(F);
  bool Changed = Impl->run(F, AC, DT, TLI, AA, MemDep, LI, &ORE,
                           MSSA ? &MSSA->getMSSA() : nullptr);
  if (!Changed)
    return PreservedAnalyses::all();
  PreservedAnalyses PA;
  PA.preserve<DominatorTreeAnalysis>();
  PA.preserve<TargetLibraryAnalysis>();
  if (MSSA)
    PA.preserve<MemorySSAAnalysis>();
  PA.preserve<LoopAnalysis>();
  return PA;
}
 
void GVNPassImpl::salvageAndRemoveInstruction(Instruction *I) {
  salvageKnowledge(I, AC);
  salvageDebugInfo(*I);
  removeInstruction(I);
}
 
void GVNPass::printPipeline(
    raw_ostream &OS, function_ref<StringRef(StringRef)> MapClassName2PassName) {
  static_cast<PassInfoMixin<GVNPass> *>(this)->printPipeline(
      OS, MapClassName2PassName);
 
  const GVNOptions &Options = Impl->Options;
  OS << '<';
  if (Options.AllowScalarPRE != std::nullopt)
    OS << (*Options.AllowScalarPRE ? "" : "no-") << "scalar-pre;";
  if (Options.AllowLoadPRE != std::nullopt)
    OS << (*Options.AllowLoadPRE ? "" : "no-") << "load-pre;";
  if (Options.AllowLoadPRESplitBackedge != std::nullopt)
    OS << (*Options.AllowLoadPRESplitBackedge ? "" : "no-")
       << "split-backedge-load-pre;";
  if (Options.AllowMemDep != std::nullopt)
    OS << (*Options.AllowMemDep ? "" : "no-") << "memdep;";
  if (Options.AllowMemorySSA != std::nullopt)
    OS << (*Options.AllowMemorySSA ? "" : "no-") << "memoryssa";
  OS << '>';
}
 
enum class AvailabilityState : char {
  /// We know the block *is not* fully available. This is a fixpoint.
  Unavailable = 0,
  /// We know the block *is* fully available. This is a fixpoint.
  Available = 1,
  /// We do not know whether the block is fully available or not,
  /// but we are currently speculating that it will be.
  /// If it would have turned out that the block was, in fact, not fully
  /// available, this would have been cleaned up into an Unavailable.
  SpeculativelyAvailable = 2,
};
 
/// Return true if we can prove that the value
/// we're analyzing is fully available in the specified block.  As we go, keep
/// track of which blocks we know are fully alive in FullyAvailableBlocks.  This
/// map is actually a tri-state map with the following values:
///   0) we know the block *is not* fully available.
///   1) we know the block *is* fully available.
///   2) we do not know whether the block is fully available or not, but we are
///      currently speculating that it will be.
static bool isValueFullyAvailableInBlock(
    const ScalarOptions &Opts, BasicBlock *BB,
    DenseMap<BasicBlock *, AvailabilityState> &FullyAvailableBlocks) {
  SmallVector<BasicBlock *, 32> Worklist;
  std::optional<BasicBlock *> UnavailableBB;
 
  // The number of times we didn't find an entry for a block in a map and
  // optimistically inserted an entry marking block as speculatively available.
  unsigned NumNewNewSpeculativelyAvailableBBs = 0;
 
#ifndef NDEBUG
  SmallPtrSet<BasicBlock *, 32> NewSpeculativelyAvailableBBs;
  SmallVector<BasicBlock *, 32> AvailableBBs;
#endif
 
  Worklist.emplace_back(BB);
  while (!Worklist.empty()) {
    BasicBlock *CurrBB = Worklist.pop_back_val(); // LoadFO - depth-first!
    // Optimistically assume that the block is Speculatively Available and check
    // to see if we already know about this block in one lookup.
    std::pair<DenseMap<BasicBlock *, AvailabilityState>::iterator, bool> IV =
        FullyAvailableBlocks.try_emplace(
            CurrBB, AvailabilityState::SpeculativelyAvailable);
    AvailabilityState &State = IV.first->second;
 
    // Did the entry already exist for this block?
    if (!IV.second) {
      if (State == AvailabilityState::Unavailable) {
        UnavailableBB = CurrBB;
        break; // Backpropagate unavailability info.
      }
 
#ifndef NDEBUG
      AvailableBBs.emplace_back(CurrBB);
#endif
      continue; // Don't recurse further, but continue processing worklist.
    }
 
    // No entry found for block.
    ++NumNewNewSpeculativelyAvailableBBs;
    bool OutOfBudget =
        NumNewNewSpeculativelyAvailableBBs > Opts.gvn_max_block_speculations;
 
    // If we have exhausted our budget, mark this block as unavailable.
    // Also, if this block has no predecessors, the value isn't live-in here.
    if (OutOfBudget || pred_empty(CurrBB)) {
      MaxBBSpeculationCutoffReachedTimes += (int)OutOfBudget;
      State = AvailabilityState::Unavailable;
      UnavailableBB = CurrBB;
      break; // Backpropagate unavailability info.
    }
 
    // Tentatively consider this block as speculatively available.
#ifndef NDEBUG
    NewSpeculativelyAvailableBBs.insert(CurrBB);
#endif
    // And further recurse into block's predecessors, in depth-first order!
    Worklist.append(pred_begin(CurrBB), pred_end(CurrBB));
  }
 
#if LLVM_ENABLE_STATS
  IsValueFullyAvailableInBlockNumSpeculationsMax.updateMax(
      NumNewNewSpeculativelyAvailableBBs);
#endif
 
  // If the block isn't marked as fixpoint yet
  // (the Unavailable and Available states are fixpoints).
  auto MarkAsFixpointAndEnqueueSuccessors =
      [&](BasicBlock *BB, AvailabilityState FixpointState) {
        auto It = FullyAvailableBlocks.find(BB);
        if (It == FullyAvailableBlocks.end())
          return; // Never queried this block, leave as-is.
        switch (AvailabilityState &State = It->second) {
        case AvailabilityState::Unavailable:
        case AvailabilityState::Available:
          return; // Don't backpropagate further, continue processing worklist.
        case AvailabilityState::SpeculativelyAvailable: // Fix it!
          State = FixpointState;
#ifndef NDEBUG
          assert(NewSpeculativelyAvailableBBs.erase(BB) &&
                 "Found a speculatively available successor leftover?");
#endif
          // Queue successors for further processing.
          Worklist.append(succ_begin(BB), succ_end(BB));
          return;
        }
      };
 
  if (UnavailableBB) {
    // Okay, we have encountered an unavailable block.
    // Mark speculatively available blocks reachable from UnavailableBB as
    // unavailable as well. Paths are terminated when they reach blocks not in
    // FullyAvailableBlocks or they are not marked as speculatively available.
    Worklist.clear();
    Worklist.append(succ_begin(*UnavailableBB), succ_end(*UnavailableBB));
    while (!Worklist.empty())
      MarkAsFixpointAndEnqueueSuccessors(Worklist.pop_back_val(),
                                         AvailabilityState::Unavailable);
  }
 
#ifndef NDEBUG
  Worklist.clear();
  for (BasicBlock *AvailableBB : AvailableBBs)
    Worklist.append(succ_begin(AvailableBB), succ_end(AvailableBB));
  while (!Worklist.empty())
    MarkAsFixpointAndEnqueueSuccessors(Worklist.pop_back_val(),
                                       AvailabilityState::Available);
 
  assert(NewSpeculativelyAvailableBBs.empty() &&
         "Must have fixed all the new speculatively available blocks.");
#endif
 
  return !UnavailableBB;
}
 
using AvailableValue = GVNPassImpl::AvailableValue;
using AvailableValueInBlock = GVNPassImpl::AvailableValueInBlock;
 
/// If the specified OldValue exists in ValuesPerBlock, replace its value with
/// NewValue.
static void replaceValuesPerBlockEntry(
    SmallVectorImpl<AvailableValueInBlock> &ValuesPerBlock, Value *OldValue,
    Value *NewValue) {
  for (AvailableValueInBlock &V : ValuesPerBlock) {
    if (V.AV.Val == OldValue)
      V.AV.Val = NewValue;
    if (V.AV.isSelectValue()) {
      if (V.AV.V1 == OldValue)
        V.AV.V1 = NewValue;
      if (V.AV.V2 == OldValue)
        V.AV.V2 = NewValue;
    }
  }
}
 
/// Given a set of loads specified by ValuesPerBlock,
/// construct SSA form, allowing us to eliminate Load.  This returns the value
/// that should be used at Load's definition site.
static Value *
constructSSAForLoadSet(LoadInst *Load,
                       SmallVectorImpl<AvailableValueInBlock> &ValuesPerBlock,
                       DominatorTree &DT) {
  // Check for the fully redundant, dominating load case.  In this case, we can
  // just use the dominating value directly.
  if (ValuesPerBlock.size() == 1 &&
      DT.properlyDominates(ValuesPerBlock[0].BB, Load->getParent())) {
    assert(!ValuesPerBlock[0].AV.isUndefValue() &&
           "Dead BB dominate this block");
    return ValuesPerBlock[0].MaterializeAdjustedValue(Load);
  }
 
  // Otherwise, we have to construct SSA form.
  SmallVector<PHINode*, 8> NewPHIs;
  SSAUpdater SSAUpdate(&NewPHIs);
  SSAUpdate.Initialize(Load->getType(), Load->getName());
 
  for (const AvailableValueInBlock &AV : ValuesPerBlock) {
    BasicBlock *BB = AV.BB;
 
    if (AV.AV.isUndefValue())
      continue;
 
    if (SSAUpdate.HasValueForBlock(BB))
      continue;
 
    // If the value is the load that we will be eliminating, and the block it's
    // available in is the block that the load is in, then don't add it as
    // SSAUpdater will resolve the value to the relevant phi which may let it
    // avoid phi construction entirely if there's actually only one value.
    if (BB == Load->getParent() &&
        ((AV.AV.isSimpleValue() && AV.AV.getSimpleValue() == Load) ||
         (AV.AV.isCoercedLoadValue() && AV.AV.getCoercedLoadValue() == Load)))
      continue;
 
    SSAUpdate.AddAvailableValue(BB, AV.MaterializeAdjustedValue(Load));
  }
 
  // Perform PHI construction.
  return SSAUpdate.GetValueInMiddleOfBlock(Load->getParent());
}
 
Value *AvailableValue::MaterializeAdjustedValue(LoadInst *Load,
                                                Instruction *InsertPt) const {
  Value *Res;
  Type *LoadTy = Load->getType();
  const DataLayout &DL = Load->getDataLayout();
  if (isSimpleValue()) {
    Res = getSimpleValue();
    if (Res->getType() != LoadTy) {
      Res = getValueForLoad(Res, Offset, LoadTy, InsertPt, Load->getFunction());
 
      LLVM_DEBUG(dbgs() << "GVN COERCED NONLOCAL VAL:\nOffset: " << Offset
                        << "  " << *getSimpleValue() << '\n'
                        << *Res << '\n'
                        << "\n\n\n");
    }
  } else if (isCoercedLoadValue()) {
    LoadInst *CoercedLoad = getCoercedLoadValue();
    if (CoercedLoad->getType() == LoadTy && Offset == 0) {
      Res = CoercedLoad;
      combineMetadataForCSE(CoercedLoad, Load, false);
    } else {
      Res = getValueForLoad(CoercedLoad, Offset, LoadTy, InsertPt,
                            Load->getFunction());
      // We are adding a new user for this load, for which the original
      // metadata may not hold. Additionally, the new load may have a different
      // size and type, so their metadata cannot be combined in any
      // straightforward way.
      // Drop all metadata that is not known to cause immediate UB on violation,
      // unless the load has !noundef, in which case all metadata violations
      // will be promoted to UB.
      // !noalias and !alias.scope are kept: the load is not moved and still
      // accesses the same memory, and these are independent of the load type
      // and offset, so they remain valid for the coerced result.
      if (!CoercedLoad->hasMetadata(LLVMContext::MD_noundef))
        CoercedLoad->dropUnknownNonDebugMetadata(
            {LLVMContext::MD_dereferenceable,
             LLVMContext::MD_dereferenceable_or_null,
             LLVMContext::MD_invariant_load, LLVMContext::MD_invariant_group,
             LLVMContext::MD_alias_scope, LLVMContext::MD_noalias});
      LLVM_DEBUG(dbgs() << "GVN COERCED NONLOCAL LOAD:\nOffset: " << Offset
                        << "  " << *getCoercedLoadValue() << '\n'
                        << *Res << '\n'
                        << "\n\n\n");
    }
  } else if (isMemIntrinValue()) {
    Res = getMemInstValueForLoad(getMemIntrinValue(), Offset, LoadTy,
                                 InsertPt, DL);
    LLVM_DEBUG(dbgs() << "GVN COERCED NONLOCAL MEM INTRIN:\nOffset: " << Offset
                      << "  " << *getMemIntrinValue() << '\n'
                      << *Res << '\n'
                      << "\n\n\n");
  } else if (isSelectValue()) {
    // Introduce a new value select for a load from an eligible pointer select.
    SelectInst *Sel = getSelectInstr();
    assert(V1 && V2 && "both value operands of the select must be present");
    Res = SelectInst::Create(Sel->getCondition(), V1, V2, "",
                             InsertPt->getIterator(),
                             ProfcheckDisableMetadataFixes ? nullptr : Sel);
    // We use the DebugLoc from the original load here, as this instruction
    // materializes the value that would previously have been loaded.
    cast<SelectInst>(Res)->setDebugLoc(Load->getDebugLoc());
  } else {
    llvm_unreachable("Should not materialize value from dead block");
  }
  assert(Res && "failed to materialize?");
  return Res;
}
 
static bool isLifetimeStart(const Instruction *Inst) {
  if (const IntrinsicInst* II = dyn_cast<IntrinsicInst>(Inst))
    return II->getIntrinsicID() == Intrinsic::lifetime_start;
  return false;
}
 
/// Assuming To can be reached from both From and Between, does Between lie on
/// every path from From to To?
static bool liesBetween(const Instruction *From, Instruction *Between,
                        const Instruction *To, const DominatorTree *DT) {
  if (From->getParent() == Between->getParent())
    return DT->dominates(From, Between);
  SmallPtrSet<BasicBlock *, 1> Exclusion;
  Exclusion.insert(Between->getParent());
  return !isPotentiallyReachable(From, To, &Exclusion, DT);
}
 
static const Instruction *findMayClobberedPtrAccess(LoadInst *Load,
                                                    const DominatorTree *DT) {
  Value *PtrOp = Load->getPointerOperand();
  if (!PtrOp->hasUseList())
    return nullptr;
 
  Instruction *OtherAccess = nullptr;
 
  for (auto *U : PtrOp->users()) {
    if (U != Load && (isa<LoadInst>(U) || isa<StoreInst>(U))) {
      auto *I = cast<Instruction>(U);
      if (I->getFunction() == Load->getFunction() && DT->dominates(I, Load)) {
        // Use the most immediately dominating value.
        if (OtherAccess) {
          if (DT->dominates(OtherAccess, I))
            OtherAccess = I;
          else
            assert(U == OtherAccess || DT->dominates(I, OtherAccess));
        } else
          OtherAccess = I;
      }
    }
  }
 
  if (OtherAccess)
    return OtherAccess;
 
  // There is no dominating use, check if we can find a closest non-dominating
  // use that lies between any other potentially available use and Load.
  for (auto *U : PtrOp->users()) {
    if (U != Load && (isa<LoadInst>(U) || isa<StoreInst>(U))) {
      auto *I = cast<Instruction>(U);
      if (I->getFunction() == Load->getFunction() &&
          isPotentiallyReachable(I, Load, nullptr, DT)) {
        if (OtherAccess) {
          if (liesBetween(OtherAccess, I, Load, DT)) {
            OtherAccess = I;
          } else if (!liesBetween(I, OtherAccess, Load, DT)) {
            // These uses are both partially available at Load were it not for
            // the clobber, but neither lies strictly after the other.
            OtherAccess = nullptr;
            break;
          } // else: keep current OtherAccess since it lies between U and
          // Load.
        } else {
          OtherAccess = I;
        }
      }
    }
  }
 
  return OtherAccess;
}
 
/// Try to locate the three instruction involved in a missed
/// load-elimination case that is due to an intervening store.
static void reportMayClobberedLoad(LoadInst *Load, Instruction *DepInst,
                                   const DominatorTree *DT,
                                   OptimizationRemarkEmitter *ORE) {
  using namespace ore;
 
  OptimizationRemarkMissed R(DEBUG_TYPE, "LoadClobbered", Load);
  R << "load of type " << NV("Type", Load->getType()) << " not eliminated"
    << setExtraArgs();
 
  const Instruction *OtherAccess = findMayClobberedPtrAccess(Load, DT);
  if (OtherAccess)
    R << " in favor of " << NV("OtherAccess", OtherAccess);
 
  R << " because it is clobbered by " << NV("ClobberedBy", DepInst);
 
  ORE->emit(R);
}
 
// Find a dominating value for Loc memory location in the extended basic block
// (chain of basic blocks with single predecessors) starting From instruction.
// Returns the value from a matching load or a simple store to the same pointer.
static Value *findDominatingValue(const ScalarOptions &Opts,
                                  const MemoryLocation &Loc, Type *LoadTy,
                                  Instruction *From, AAResults *AA) {
  uint32_t NumVisitedInsts = 0;
  BasicBlock *FromBB = From->getParent();
  BatchAAResults BatchAA(*AA);
  for (BasicBlock *BB = FromBB; BB; BB = BB->getSinglePredecessor())
    for (auto *Inst = BB == FromBB ? From : BB->getTerminator();
         Inst != nullptr; Inst = Inst->getPrevNode()) {
      // Stop the search if limit is reached.
      if (++NumVisitedInsts > Opts.gvn_max_num_visited_insts)
        return nullptr;
      if (isModSet(BatchAA.getModRefInfo(Inst, Loc))) {
        // A simple store to the exact location can forward its value.
        if (auto *SI = dyn_cast<StoreInst>(Inst))
          if (SI->isSimple() && SI->getPointerOperand() == Loc.Ptr &&
              SI->getValueOperand()->getType() == LoadTy)
            return SI->getValueOperand();
        return nullptr;
      }
      if (auto *LI = dyn_cast<LoadInst>(Inst))
        if (LI->getPointerOperand() == Loc.Ptr && LI->getType() == LoadTy)
          return LI;
    }
  return nullptr;
}
 
std::optional<AvailableValue>
GVNPassImpl::analyzeSelectAvailability(LoadInst *Load, SelectInst *Sel,
                                       Value *TrueAddr, Value *FalseAddr,
                                       Instruction *From) {
  assert(TrueAddr->getType() == Load->getPointerOperandType() &&
         "Invalid address type of true side of select dependency");
  assert(FalseAddr->getType() == Load->getPointerOperandType() &&
         "Invalid address type of false side of select dependency");
  // We can convert a load through a select address into a select of the two
  // loaded values only if both sides have a dominating, non-clobbered value of
  // the right type in the extended basic block ending at From.
  auto Loc = MemoryLocation::get(Load);
  Value *V1 = findDominatingValue(*Opts, Loc.getWithNewPtr(TrueAddr),
                                  Load->getType(), From, getAliasAnalysis());
  if (!V1)
    return std::nullopt;
  Value *V2 = findDominatingValue(*Opts, Loc.getWithNewPtr(FalseAddr),
                                  Load->getType(), From, getAliasAnalysis());
  if (!V2)
    return std::nullopt;
  return AvailableValue::getSelect(Sel, V1, V2);
}
 
std::optional<AvailableValue>
GVNPassImpl::analyzeLoadAvailability(LoadInst *Load, const ReachingMemVal &Dep,
                                     Value *Address) {
  assert(Load->isUnordered() && "rules below are incorrect for ordered access");
  assert((Dep.Kind == DepKind::Def || Dep.Kind == DepKind::Clobber) &&
         "expected a local dependence");
 
  Instruction *DepInst = Dep.Inst;
 
  const DataLayout &DL = Load->getDataLayout();
  if (Dep.Kind == DepKind::Clobber) {
    // If the dependence is to a store that writes to a superset of the bits
    // read by the load, we can extract the bits we need for the load from the
    // stored value.
    if (StoreInst *DepSI = dyn_cast<StoreInst>(DepInst)) {
      // Can't forward from non-atomic to atomic without violating memory model.
      if (Address && Load->isAtomic() <= DepSI->isAtomic()) {
        int Offset =
            analyzeLoadFromClobberingStore(Load->getType(), Address, DepSI, DL);
        if (Offset != -1)
          return AvailableValue::get(DepSI->getValueOperand(), Offset);
      }
    }
 
    // Check to see if we have something like this:
    //    load i32* P
    //    load i8* (P+1)
    // if we have this, replace the later with an extraction from the former.
    if (LoadInst *DepLoad = dyn_cast<LoadInst>(DepInst)) {
      // If this is a clobber and L is the first instruction in its block, then
      // we have the first instruction in the entry block.
      // Can't forward from non-atomic to atomic without violating memory model.
      if (DepLoad != Load && Address &&
          Load->isAtomic() <= DepLoad->isAtomic()) {
        Type *LoadType = Load->getType();
        int Offset = Dep.Offset;
 
        if (!isMemorySSAEnabled()) {
          // If MD reported clobber, check it was nested.
          if (canCoerceMustAliasedValueToLoad(DepLoad, LoadType,
                                              DepLoad->getFunction())) {
            const auto ClobberOff = MD->getClobberOffset(DepLoad);
            // GVN has no deal with a negative offset.
            Offset = (ClobberOff == std::nullopt || *ClobberOff < 0)
                         ? -1
                         : *ClobberOff;
          }
        } else {
          if (!canCoerceMustAliasedValueToLoad(DepLoad, LoadType,
                                               DepLoad->getFunction()) ||
              Offset < 0)
            Offset = -1;
        }
        if (Offset == -1)
          Offset =
              analyzeLoadFromClobberingLoad(LoadType, Address, DepLoad, DL);
        if (Offset != -1)
          return AvailableValue::getLoad(DepLoad, Offset);
      }
    }
 
    // If the clobbering value is a memset/memcpy/memmove, see if we can
    // forward a value on from it.
    if (MemIntrinsic *DepMI = dyn_cast<MemIntrinsic>(DepInst)) {
      if (Address && !Load->isAtomic()) {
        int Offset = analyzeLoadFromClobberingMemInst(Load->getType(), Address,
                                                      DepMI, DL);
        if (Offset != -1)
          return AvailableValue::getMI(DepMI, Offset);
      }
    }
 
    // Nothing known about this clobber, have to be conservative.
    LLVM_DEBUG(
        // fast print dep, using operator<< on instruction is too slow.
        dbgs() << "GVN: load "; Load->printAsOperand(dbgs());
        dbgs() << " is clobbered by " << *DepInst << '\n';);
    if (ORE->allowExtraAnalysis(DEBUG_TYPE))
      reportMayClobberedLoad(Load, DepInst, DT, ORE);
 
    return std::nullopt;
  }
  assert(Dep.Kind == DepKind::Def && "follows from above");
 
  // Loading the alloca -> undef.
  // Loading immediately after lifetime begin -> undef.
  if (isa<AllocaInst>(DepInst) || isLifetimeStart(DepInst))
    return AvailableValue::get(UndefValue::get(Load->getType()));
 
  if (Constant *InitVal =
          getInitialValueOfAllocation(DepInst, TLI, Load->getType()))
    return AvailableValue::get(InitVal);
 
  if (StoreInst *S = dyn_cast<StoreInst>(DepInst)) {
    // Reject loads and stores that are to the same address but are of
    // different types if we have to. If the stored value is convertable to
    // the loaded value, we can reuse it.
    if (!canCoerceMustAliasedValueToLoad(S->getValueOperand(), Load->getType(),
                                         S->getFunction()))
      return std::nullopt;
 
    // Can't forward from non-atomic to atomic without violating memory model.
    if (S->isAtomic() < Load->isAtomic())
      return std::nullopt;
 
    return AvailableValue::get(S->getValueOperand());
  }
 
  if (LoadInst *LD = dyn_cast<LoadInst>(DepInst)) {
    // If the types mismatch and we can't handle it, reject reuse of the load.
    // If the stored value is larger or equal to the loaded value, we can reuse
    // it.
    if (!canCoerceMustAliasedValueToLoad(LD, Load->getType(),
                                         LD->getFunction()))
      return std::nullopt;
 
    // Can't forward from non-atomic to atomic without violating memory model.
    if (LD->isAtomic() < Load->isAtomic())
      return std::nullopt;
 
    return AvailableValue::getLoad(LD);
  }
 
  // Check if load with Addr dependent from select can be converted to select
  // between load values. There must be no instructions between the found
  // loads and DepInst that may clobber the loads.
  if (auto *Sel = dyn_cast<SelectInst>(DepInst)) {
    assert(Sel->getType() == Load->getPointerOperandType());
    if (auto AV = analyzeSelectAvailability(Load, Sel, Sel->getTrueValue(),
                                            Sel->getFalseValue(), DepInst))
      return AV;
    return std::nullopt;
  }
 
  // Unknown def - must be conservative.
  LLVM_DEBUG(
      // fast print dep, using operator<< on instruction is too slow.
      dbgs() << "GVN: load "; Load->printAsOperand(dbgs());
      dbgs() << " has unknown def " << *DepInst << '\n';);
  return std::nullopt;
}
 
void GVNPassImpl::analyzeLoadAvailability(LoadInst *Load,
                                          SmallVectorImpl<ReachingMemVal> &Deps,
                                          AvailValInBlkVect &ValuesPerBlock,
                                          UnavailBlkVect &UnavailableBlocks) {
  // Filter out useless results (non-locals, etc).  Keep track of the blocks
  // where we have a value available in repl, also keep track of whether we see
  // dependencies that produce an unknown value for the load (such as a call
  // that could potentially clobber the load).
  for (const auto &Dep : Deps) {
    BasicBlock *DepBB = Dep.Block;
 
    if (DeadBlocks.count(DepBB)) {
      // Dead dependent mem-op disguise as a load evaluating the same value
      // as the load in question.
      ValuesPerBlock.push_back(AvailableValueInBlock::getUndef(DepBB));
      continue;
    }
 
    if (Dep.Kind == DepKind::Other) {
      UnavailableBlocks.push_back(DepBB);
      continue;
    }
 
    // The load address is a select in this block: try to rematerialize the
    // load as a select of the two reaching values (one per side).  The values
    // are searched for at the end of DepBB.
    if (Dep.Kind == DepKind::Select) {
      if (auto AV = analyzeSelectAvailability(
              Load, Dep.Sel, const_cast<Value *>(Dep.SelTrueAddr),
              const_cast<Value *>(Dep.SelFalseAddr), DepBB->getTerminator())) {
        ValuesPerBlock.push_back(
            AvailableValueInBlock::get(DepBB, std::move(*AV)));
      } else {
        UnavailableBlocks.push_back(DepBB);
      }
      continue;
    }
 
    // The address being loaded in this non-local block may not be the same as
    // the pointer operand of the load if PHI translation occurs.  Make sure
    // to consider the right address.
    if (auto AV =
            analyzeLoadAvailability(Load, Dep, const_cast<Value *>(Dep.Addr))) {
      // subtlety: because we know this was a non-local dependency, we know
      // it's safe to materialize anywhere between the instruction within
      // DepInfo and the end of it's block.
      ValuesPerBlock.push_back(
          AvailableValueInBlock::get(DepBB, std::move(*AV)));
    } else {
      UnavailableBlocks.push_back(DepBB);
    }
  }
 
  assert(Deps.size() == ValuesPerBlock.size() + UnavailableBlocks.size() &&
         "post condition violation");
}
 
/// Given the following code, v1 is partially available on some edges, but not
/// available on the edge from PredBB. This function tries to find if there is
/// another identical load in the other successor of PredBB.
///
///      v0 = load %addr
///      br %LoadBB
///
///   LoadBB:
///      v1 = load %addr
///      ...
///
///   PredBB:
///      ...
///      br %cond, label %LoadBB, label %SuccBB
///
///   SuccBB:
///      v2 = load %addr
///      ...
///
LoadInst *GVNPassImpl::findLoadToHoistIntoPred(BasicBlock *Pred,
                                               BasicBlock *LoadBB,
                                               LoadInst *Load) {
  // For simplicity we handle a Pred has 2 successors only.
  auto *Term = Pred->getTerminator();
  if (Term->getNumSuccessors() != 2 || Term->isSpecialTerminator())
    return nullptr;
  auto *SuccBB = Term->getSuccessor(0);
  if (SuccBB == LoadBB)
    SuccBB = Term->getSuccessor(1);
  if (!SuccBB->getSinglePredecessor())
    return nullptr;
 
  unsigned int NumInsts = Opts->gvn_max_num_insns;
  for (Instruction &Inst : *SuccBB) {
    if (isa<PseudoProbeInst>(Inst))
      continue;
    if (--NumInsts == 0)
      return nullptr;
 
    if (!Inst.isIdenticalTo(Load))
      continue;
 
    bool HasLocalDep = true;
    if (!isMemorySSAEnabled()) {
      MemDepResult Dep = MD->getDependency(&Inst);
      HasLocalDep = !Dep.isNonLocal();
    } else {
      auto *MSSA = MSSAU->getMemorySSA();
      // Do not hoist if the identical load has ordering constraint.
      if (auto *MA = MSSA->getMemoryAccess(&Inst); MA && isa<MemoryUse>(MA)) {
        auto *Clobber = MSSA->getWalker()->getClobberingMemoryAccess(MA);
        HasLocalDep = Clobber->getBlock() == SuccBB;
      }
    }
 
    // If an identical load doesn't depends on any local instructions, it can
    // be safely moved to PredBB.
    // Also check for the implicit control flow instructions. See the comments
    // in performLoadPRE for details.
    if (!HasLocalDep && !ICF->isDominatedByICFIFromSameBlock(&Inst))
      return cast<LoadInst>(&Inst);
 
    // Otherwise there is something in the same BB clobbers the memory, we can't
    // move this and later load to PredBB.
    return nullptr;
  }
 
  return nullptr;
}
 
void GVNPassImpl::eliminatePartiallyRedundantLoad(
    LoadInst *Load, AvailValInBlkVect &ValuesPerBlock,
    MapVector<BasicBlock *, Value *> &AvailableLoads,
    MapVector<BasicBlock *, LoadInst *> *CriticalEdgePredAndLoad) {
  for (const auto &AvailableLoad : AvailableLoads) {
    BasicBlock *UnavailableBlock = AvailableLoad.first;
    Value *LoadPtr = AvailableLoad.second;
 
    auto *NewLoad =
        new LoadInst(Load->getType(), LoadPtr, Load->getName() + ".pre",
                     Load->getProperties(),
                     UnavailableBlock->getTerminator()->getIterator());
    NewLoad->setDebugLoc(Load->getDebugLoc());
    if (MSSAU) {
      auto *NewAccess = MSSAU->createMemoryAccessInBB(
          NewLoad, nullptr, NewLoad->getParent(), MemorySSA::BeforeTerminator);
      if (auto *NewDef = dyn_cast<MemoryDef>(NewAccess))
        MSSAU->insertDef(NewDef, /*RenameUses=*/true);
      else
        MSSAU->insertUse(cast<MemoryUse>(NewAccess), /*RenameUses=*/true);
    }
 
    // Transfer the old load's AA tags to the new load.
    AAMDNodes Tags = Load->getAAMetadata();
    if (Tags)
      NewLoad->setAAMetadata(Tags);
 
    if (auto *MD = Load->getMetadata(LLVMContext::MD_invariant_load))
      NewLoad->setMetadata(LLVMContext::MD_invariant_load, MD);
    if (auto *InvGroupMD = Load->getMetadata(LLVMContext::MD_invariant_group))
      NewLoad->setMetadata(LLVMContext::MD_invariant_group, InvGroupMD);
    if (auto *RangeMD = Load->getMetadata(LLVMContext::MD_range))
      NewLoad->setMetadata(LLVMContext::MD_range, RangeMD);
    if (auto *NoFPClassMD = Load->getMetadata(LLVMContext::MD_nofpclass))
      NewLoad->setMetadata(LLVMContext::MD_nofpclass, NoFPClassMD);
 
    if (auto *AccessMD = Load->getMetadata(LLVMContext::MD_access_group))
      if (LI->getLoopFor(Load->getParent()) == LI->getLoopFor(UnavailableBlock))
        NewLoad->setMetadata(LLVMContext::MD_access_group, AccessMD);
 
    // We do not propagate the old load's debug location, because the new
    // load now lives in a different BB, and we want to avoid a jumpy line
    // table.
    // FIXME: How do we retain source locations without causing poor debugging
    // behavior?
 
    // Add the newly created load.
    ValuesPerBlock.push_back(
        AvailableValueInBlock::get(UnavailableBlock, NewLoad));
    if (MD)
      MD->invalidateCachedPointerInfo(LoadPtr);
    LLVM_DEBUG(dbgs() << "GVN INSERTED " << *NewLoad << '\n');
 
    // For PredBB in CriticalEdgePredAndLoad we need to replace the uses of old
    // load instruction with the new created load instruction.
    if (CriticalEdgePredAndLoad) {
      auto It = CriticalEdgePredAndLoad->find(UnavailableBlock);
      if (It != CriticalEdgePredAndLoad->end()) {
        ++NumPRELoadMoved2CEPred;
        ICF->insertInstructionTo(NewLoad, UnavailableBlock);
        LoadInst *OldLoad = It->second;
        combineMetadataForCSE(NewLoad, OldLoad, /*DoesKMove=*/true);
        OldLoad->replaceAllUsesWith(NewLoad);
        replaceValuesPerBlockEntry(ValuesPerBlock, OldLoad, NewLoad);
        if (uint32_t ValNo = VN.lookup(OldLoad, false))
          LeaderTable.erase(ValNo, OldLoad, OldLoad->getParent());
        removeInstruction(OldLoad);
      }
    }
  }
 
  // Perform PHI construction.
  Value *V = constructSSAForLoadSet(Load, ValuesPerBlock, getDominatorTree());
  // constructSSAForLoadSet is responsible for combining metadata.
  ICF->removeUsersOf(Load);
  Load->replaceAllUsesWith(V);
  if (isa<PHINode>(V))
    V->takeName(Load);
  if (Instruction *I = dyn_cast<Instruction>(V))
    I->setDebugLoc(Load->getDebugLoc());
  if (MD && V->getType()->isPtrOrPtrVectorTy())
    MD->invalidateCachedPointerInfo(V);
  ORE->emit([&]() {
    return OptimizationRemark(DEBUG_TYPE, "LoadPRE", Load)
           << "load eliminated by PRE";
  });
  salvageAndRemoveInstruction(Load);
}
 
bool GVNPassImpl::performLoadPRE(LoadInst *Load,
                                 AvailValInBlkVect &ValuesPerBlock,
                                 UnavailBlkVect &UnavailableBlocks) {
  // Okay, we have *some* definitions of the value.  This means that the value
  // is available in some of our (transitive) predecessors.  Lets think about
  // doing PRE of this load.  This will involve inserting a new load into the
  // predecessor when it's not available.  We could do this in general, but
  // prefer to not increase code size.  As such, we only do this when we know
  // that we only have to insert *one* load (which means we're basically moving
  // the load, not inserting a new one).
 
  SmallPtrSet<BasicBlock *, 4> Blockers(llvm::from_range, UnavailableBlocks);
 
  // Let's find the first basic block with more than one predecessor.  Walk
  // backwards through predecessors if needed.
  BasicBlock *LoadBB = Load->getParent();
  BasicBlock *TmpBB = LoadBB;
 
  // Check that there is no implicit control flow instructions above our load in
  // its block. If there is an instruction that doesn't always pass the
  // execution to the following instruction, then moving through it may become
  // invalid. For example:
  //
  // int arr[LEN];
  // int index = ???;
  // ...
  // guard(0 <= index && index < LEN);
  // use(arr[index]);
  //
  // It is illegal to move the array access to any point above the guard,
  // because if the index is out of bounds we should deoptimize rather than
  // access the array.
  // Check that there is no guard in this block above our instruction.
  bool MustEnsureSafetyOfSpeculativeExecution =
      ICF->isDominatedByICFIFromSameBlock(Load);
 
  while (TmpBB->getSinglePredecessor()) {
    TmpBB = TmpBB->getSinglePredecessor();
    if (TmpBB == LoadBB) // Infinite (unreachable) loop.
      return false;
    if (Blockers.count(TmpBB))
      return false;
 
    // If any of these blocks has more than one successor (i.e. if the edge we
    // just traversed was critical), then there are other paths through this
    // block along which the load may not be anticipated.  Hoisting the load
    // above this block would be adding the load to execution paths along
    // which it was not previously executed.
    if (TmpBB->getTerminator()->getNumSuccessors() != 1)
      return false;
 
    // Check that there is no implicit control flow in a block above.
    MustEnsureSafetyOfSpeculativeExecution =
        MustEnsureSafetyOfSpeculativeExecution || ICF->hasICF(TmpBB);
  }
 
  assert(TmpBB);
  LoadBB = TmpBB;
 
  // Check to see how many predecessors have the loaded value fully
  // available.
  MapVector<BasicBlock *, Value *> PredLoads;
  DenseMap<BasicBlock *, AvailabilityState> FullyAvailableBlocks;
  for (const AvailableValueInBlock &AV : ValuesPerBlock)
    FullyAvailableBlocks[AV.BB] = AvailabilityState::Available;
  for (BasicBlock *UnavailableBB : UnavailableBlocks)
    FullyAvailableBlocks[UnavailableBB] = AvailabilityState::Unavailable;
 
  // The edge from Pred to LoadBB is a critical edge will be splitted.
  SmallVector<BasicBlock *, 4> CriticalEdgePredSplit;
  // The edge from Pred to LoadBB is a critical edge, another successor of Pred
  // contains a load can be moved to Pred. This data structure maps the Pred to
  // the movable load.
  MapVector<BasicBlock *, LoadInst *> CriticalEdgePredAndLoad;
  for (BasicBlock *Pred : predecessors(LoadBB)) {
    // If any predecessor block is an EH pad that does not allow non-PHI
    // instructions before the terminator, we can't PRE the load.
    if (Pred->getTerminator()->isEHPad()) {
      LLVM_DEBUG(
          dbgs() << "COULD NOT PRE LOAD BECAUSE OF AN EH PAD PREDECESSOR '"
                 << Pred->getName() << "': " << *Load << '\n');
      return false;
    }
 
    if (isValueFullyAvailableInBlock(*Opts, Pred, FullyAvailableBlocks)) {
      continue;
    }
 
    if (Pred->getTerminator()->getNumSuccessors() != 1) {
      if (isa<IndirectBrInst>(Pred->getTerminator())) {
        LLVM_DEBUG(
            dbgs() << "COULD NOT PRE LOAD BECAUSE OF INDBR CRITICAL EDGE '"
                   << Pred->getName() << "': " << *Load << '\n');
        return false;
      }
 
      if (LoadBB->isEHPad()) {
        LLVM_DEBUG(
            dbgs() << "COULD NOT PRE LOAD BECAUSE OF AN EH PAD CRITICAL EDGE '"
                   << Pred->getName() << "': " << *Load << '\n');
        return false;
      }
 
      // Do not split backedge as it will break the canonical loop form.
      if (!isLoadPRESplitBackedgeEnabled())
        if (DT->dominates(LoadBB, Pred)) {
          LLVM_DEBUG(
              dbgs()
              << "COULD NOT PRE LOAD BECAUSE OF A BACKEDGE CRITICAL EDGE '"
              << Pred->getName() << "': " << *Load << '\n');
          return false;
        }
 
      if (LoadInst *LI = findLoadToHoistIntoPred(Pred, LoadBB, Load))
        CriticalEdgePredAndLoad[Pred] = LI;
      else
        CriticalEdgePredSplit.push_back(Pred);
    } else {
      // Only add the predecessors that will not be split for now.
      PredLoads[Pred] = nullptr;
    }
  }
 
  // Decide whether PRE is profitable for this load.
  unsigned NumInsertPreds = PredLoads.size() + CriticalEdgePredSplit.size();
  unsigned NumUnavailablePreds = NumInsertPreds +
      CriticalEdgePredAndLoad.size();
  assert(NumUnavailablePreds != 0 &&
         "Fully available value should already be eliminated!");
  (void)NumUnavailablePreds;
 
  // If we need to insert new load in multiple predecessors, reject it.
  // FIXME: If we could restructure the CFG, we could make a common pred with
  // all the preds that don't have an available Load and insert a new load into
  // that one block.
  if (NumInsertPreds > 1)
      return false;
 
  // Now we know where we will insert load. We must ensure that it is safe
  // to speculatively execute the load at that points.
  if (MustEnsureSafetyOfSpeculativeExecution) {
    if (CriticalEdgePredSplit.size())
      if (!isSafeToSpeculativelyExecute(Load, &*LoadBB->getFirstNonPHIIt(), AC,
                                        DT))
        return false;
    for (auto &PL : PredLoads)
      if (!isSafeToSpeculativelyExecute(Load, PL.first->getTerminator(), AC,
                                        DT))
        return false;
    for (auto &CEP : CriticalEdgePredAndLoad)
      if (!isSafeToSpeculativelyExecute(Load, CEP.first->getTerminator(), AC,
                                        DT))
        return false;
  }
 
  // Split critical edges, and update the unavailable predecessors accordingly.
  for (BasicBlock *OrigPred : CriticalEdgePredSplit) {
    BasicBlock *NewPred = splitCriticalEdges(OrigPred, LoadBB);
    assert(!PredLoads.count(OrigPred) && "Split edges shouldn't be in map!");
    PredLoads[NewPred] = nullptr;
    LLVM_DEBUG(dbgs() << "Split critical edge " << OrigPred->getName() << "->"
                      << LoadBB->getName() << '\n');
  }
 
  for (auto &CEP : CriticalEdgePredAndLoad)
    PredLoads[CEP.first] = nullptr;
 
  // Check if the load can safely be moved to all the unavailable predecessors.
  bool CanDoPRE = true;
  const DataLayout &DL = Load->getDataLayout();
  SmallVector<Instruction*, 8> NewInsts;
  for (auto &PredLoad : PredLoads) {
    BasicBlock *UnavailablePred = PredLoad.first;
 
    // Do PHI translation to get its value in the predecessor if necessary.  The
    // returned pointer (if non-null) is guaranteed to dominate UnavailablePred.
    // We do the translation for each edge we skipped by going from Load's block
    // to LoadBB, otherwise we might miss pieces needing translation.
 
    // If all preds have a single successor, then we know it is safe to insert
    // the load on the pred (?!?), so we can insert code to materialize the
    // pointer if it is not available.
    Value *LoadPtr = Load->getPointerOperand();
    BasicBlock *Cur = Load->getParent();
    while (Cur != LoadBB) {
      PHITransAddr Address(LoadPtr, DL, AC);
      LoadPtr = Address.translateWithInsertion(Cur, Cur->getSinglePredecessor(),
                                               *DT, NewInsts);
      if (!LoadPtr) {
        CanDoPRE = false;
        break;
      }
      Cur = Cur->getSinglePredecessor();
    }
 
    if (LoadPtr) {
      PHITransAddr Address(LoadPtr, DL, AC);
      LoadPtr = Address.translateWithInsertion(LoadBB, UnavailablePred, *DT,
                                               NewInsts);
    }
    // If we couldn't find or insert a computation of this phi translated value,
    // we fail PRE.
    if (!LoadPtr) {
      LLVM_DEBUG(dbgs() << "COULDN'T INSERT PHI TRANSLATED VALUE OF: "
                        << *Load->getPointerOperand() << "\n");
      CanDoPRE = false;
      break;
    }
 
    PredLoad.second = LoadPtr;
  }
 
  if (!CanDoPRE) {
    while (!NewInsts.empty()) {
      // Erase instructions generated by the failed PHI translation before
      // trying to number them. PHI translation might insert instructions
      // in basic blocks other than the current one, and we delete them
      // directly, as salvageAndRemoveInstruction only allows removing from the
      // current basic block.
      NewInsts.pop_back_val()->eraseFromParent();
    }
    // HINT: Don't revert the edge-splitting as following transformation may
    // also need to split these critical edges.
    return !CriticalEdgePredSplit.empty();
  }
 
  // Okay, we can eliminate this load by inserting a reload in the predecessor
  // and using PHI construction to get the value in the other predecessors, do
  // it.
  LLVM_DEBUG(dbgs() << "GVN REMOVING PRE LOAD: " << *Load << '\n');
  LLVM_DEBUG(if (!NewInsts.empty()) dbgs() << "INSERTED " << NewInsts.size()
                                           << " INSTS: " << *NewInsts.back()
                                           << '\n');
 
  // Assign value numbers to the new instructions.
  for (Instruction *I : NewInsts) {
    // Instructions that have been inserted in predecessor(s) to materialize
    // the load address do not retain their original debug locations. Doing
    // so could lead to confusing (but correct) source attributions.
    I->updateLocationAfterHoist();
 
    // FIXME: We really _ought_ to insert these value numbers into their
    // parent's availability map.  However, in doing so, we risk getting into
    // ordering issues.  If a block hasn't been processed yet, we would be
    // marking a value as AVAIL-IN, which isn't what we intend.
    VN.lookupOrAdd(I);
  }
 
  eliminatePartiallyRedundantLoad(Load, ValuesPerBlock, PredLoads,
                                  &CriticalEdgePredAndLoad);
  ++NumPRELoad;
  return true;
}
 
bool GVNPassImpl::performLoopLoadPRE(LoadInst *Load,
                                     AvailValInBlkVect &ValuesPerBlock,
                                     UnavailBlkVect &UnavailableBlocks) {
  const Loop *L = LI->getLoopFor(Load->getParent());
  // TODO: Generalize to other loop blocks that dominate the latch.
  if (!L || L->getHeader() != Load->getParent())
    return false;
 
  BasicBlock *Preheader = L->getLoopPreheader();
  BasicBlock *Latch = L->getLoopLatch();
  if (!Preheader || !Latch)
    return false;
 
  Value *LoadPtr = Load->getPointerOperand();
  // Must be available in preheader.
  if (!L->isLoopInvariant(LoadPtr))
    return false;
 
  // We plan to hoist the load to preheader without introducing a new fault.
  // In order to do it, we need to prove that we cannot side-exit the loop
  // once loop header is first entered before execution of the load.
  if (ICF->isDominatedByICFIFromSameBlock(Load))
    return false;
 
  BasicBlock *LoopBlock = nullptr;
  for (auto *Blocker : UnavailableBlocks) {
    // Blockers from outside the loop are handled in preheader.
    if (!L->contains(Blocker))
      continue;
 
    // Only allow one loop block. Loop header is not less frequently executed
    // than each loop block, and likely it is much more frequently executed. But
    // in case of multiple loop blocks, we need extra information (such as block
    // frequency info) to understand whether it is profitable to PRE into
    // multiple loop blocks.
    if (LoopBlock)
      return false;
 
    // A reload in a blocker inside an inner loop can run once per inner
    // iteration, which is more often than L's header. Reload in the exit block
    // of the inner loop that is directly inside L instead. That exit block is
    // on every path from the blocker back to L's header and it runs at most
    // once per iteration of L.
    BasicBlock *ReloadBB = Blocker;
    if (const Loop *Inner = LI->getLoopFor(Blocker); Inner != L) {
      // The blocker is inside Inner, which may itself be nested further down.
      // Walk parents until Inner is the loop directly inside L that contains
      // it.
      while (Inner->getParentLoop() != L)
        Inner = Inner->getParentLoop();
 
      // PRE is not performed if the inner loop's exit edges do not all go to
      // one block, or if that block is not directly inside L, which means it
      // is the header of a sibling loop, where the reload can run more often
      // than L's header.
      ReloadBB = Inner->getUniqueExitBlock();
      if (!ReloadBB || LI->getLoopFor(ReloadBB) != L)
        return false;
    }
 
    // Blocks that dominate the latch execute on every single iteration, maybe
    // except the last one. So PREing into these blocks doesn't make much sense
    // in most cases. But the blocks that do not necessarily execute on each
    // iteration are sometimes much colder than the header, and this is when
    // PRE is potentially profitable.
    if (DT->dominates(ReloadBB, Latch))
      return false;
 
    // Make sure that the terminator itself doesn't clobber.
    if (ReloadBB->getTerminator()->mayWriteToMemory())
      return false;
 
    LoopBlock = ReloadBB;
  }
 
  if (!LoopBlock)
    return false;
 
  // Make sure the memory at this pointer cannot be freed, therefore we can
  // safely reload from it after clobber.
  //
  // The header load has already dereferenced LoadPtr on this iteration, so
  // only a deallocation between that load and the reload in LoopBlock can make
  // the same address unsafe to read again. Check every path between these two
  // points for an instruction that may deallocate the memory.
  if (LoadPtr->canBeFreed() &&
      !willNotFreeBetween(Load, LoopBlock->getTerminator(), DT))
    return false;
 
  // TODO: Support critical edge splitting if blocker has more than 1 successor.
  MapVector<BasicBlock *, Value *> AvailableLoads;
  AvailableLoads[LoopBlock] = LoadPtr;
  AvailableLoads[Preheader] = LoadPtr;
 
  LLVM_DEBUG(dbgs() << "GVN REMOVING PRE LOOP LOAD: " << *Load << '\n');
  eliminatePartiallyRedundantLoad(Load, ValuesPerBlock, AvailableLoads,
                                  /*CriticalEdgePredAndLoad*/ nullptr);
  ++NumPRELoopLoad;
  return true;
}
 
static void reportLoadElim(LoadInst *Load, Value *AvailableValue,
                           OptimizationRemarkEmitter *ORE) {
  using namespace ore;
 
  ORE->emit([&]() {
    return OptimizationRemark(DEBUG_TYPE, "LoadElim", Load)
           << "load of type " << NV("Type", Load->getType()) << " eliminated"
           << setExtraArgs() << " in favor of "
           << NV("InfavorOfValue", AvailableValue);
  });
}
 
/// Attempt to eliminate a load whose dependencies are
/// non-local by performing PHI construction.
bool GVNPassImpl::processNonLocalLoad(LoadInst *Load) {
  // Non-local speculations are not allowed under asan.
  if (Load->getFunction()->hasFnAttribute(Attribute::SanitizeAddress) ||
      Load->getFunction()->hasFnAttribute(Attribute::SanitizeHWAddress))
    return false;
 
  // Find the non-local dependencies of the load.
  LoadDepVect Deps;
  MD->getNonLocalPointerDependency(Load, Deps);
 
  // If we had to process more than one hundred blocks to find the
  // dependencies, this load isn't worth worrying about.  Optimizing
  // it will be too expensive.
  unsigned NumDeps = Deps.size();
  if (NumDeps > Opts->gvn_max_num_deps)
    return false;
 
  SmallVector<ReachingMemVal, 64> MemVals;
  MemVals.reserve(Deps.size());
 
  for (const NonLocalDepResult &Dep : Deps) {
    const auto &R = Dep.getResult();
    SelectAddr SelAddr = Dep.getAddress();
    BasicBlock *BB = Dep.getBB();
    Instruction *Inst = R.getInst();
    if (R.isSelect()) {
      auto [Sel, Addrs] = SelAddr.getSelectAndAddrs();
      MemVals.emplace_back(
          ReachingMemVal::getSelect(BB, Sel, Addrs.first, Addrs.second));
      continue;
    }
    Value *Address = SelAddr.getAddr();
    if (R.isClobber())
      MemVals.emplace_back(ReachingMemVal::getClobber(Address, Inst));
    else if (R.isDef())
      MemVals.emplace_back(ReachingMemVal::getDef(Address, Inst));
    else
      MemVals.emplace_back(ReachingMemVal::getUnknown(BB, Address, Inst));
  }
 
  return processNonLocalLoad(Load, MemVals);
}
 
bool GVNPassImpl::processNonLocalLoad(LoadInst *Load,
                                      SmallVectorImpl<ReachingMemVal> &Deps) {
  // If we had a phi translation failure, we'll have a single entry which is a
  // clobber in the current block.  Reject this early.
  if (Deps.size() == 1 && Deps[0].Kind == DepKind::Other) {
    LLVM_DEBUG(dbgs() << "GVN: non-local load "; Load->printAsOperand(dbgs());
               dbgs() << " has unknown dependencies\n";);
    return false;
  }
 
  bool Changed = false;
  // This is a limited form of scalar PRE for load indices. If this load follows
  // a GEP, see if we can PRE the indices before analyzing.
  if (isScalarPREEnabled()) {
    if (GetElementPtrInst *GEP =
            dyn_cast<GetElementPtrInst>(Load->getOperand(0))) {
      for (Use &U : GEP->indices())
        // Instructions inserted by GVN during this iteration (e.g. coercion
        // casts from MaterializeAdjustedValue) may not have value numbers yet,
        // so they are skipped.
        if (Instruction *I = dyn_cast<Instruction>(U.get()); I && VN.exists(I))
          Changed |= performScalarPRE(I);
    }
  }
 
  // Step 1: Analyze the availability of the load.
  AvailValInBlkVect ValuesPerBlock;
  UnavailBlkVect UnavailableBlocks;
  analyzeLoadAvailability(Load, Deps, ValuesPerBlock, UnavailableBlocks);
 
  // If we have no predecessors that produce a known value for this load, exit
  // early.
  if (ValuesPerBlock.empty())
    return Changed;
 
  // Step 2: Eliminate fully redundancy.
  //
  // If all of the instructions we depend on produce a known value for this
  // load, then it is fully redundant and we can use PHI insertion to compute
  // its value.  Insert PHIs and remove the fully redundant value now.
  if (UnavailableBlocks.empty()) {
    LLVM_DEBUG(dbgs() << "GVN REMOVING NONLOCAL LOAD: " << *Load << '\n');
 
    // Perform PHI construction.
    Value *V = constructSSAForLoadSet(Load, ValuesPerBlock, getDominatorTree());
    // constructSSAForLoadSet is responsible for combining metadata.
    ICF->removeUsersOf(Load);
    Load->replaceAllUsesWith(V);
 
    if (isa<PHINode>(V))
      V->takeName(Load);
    if (Instruction *I = dyn_cast<Instruction>(V))
      // If instruction I has debug info, then we should not update it.
      // Also, if I has a null DebugLoc, then it is still potentially incorrect
      // to propagate Load's DebugLoc because Load may not post-dominate I.
      if (Load->getDebugLoc() && Load->getParent() == I->getParent())
        I->setDebugLoc(Load->getDebugLoc());
    if (MD && V->getType()->isPtrOrPtrVectorTy())
      MD->invalidateCachedPointerInfo(V);
    ++NumGVNLoad;
    reportLoadElim(Load, V, ORE);
    salvageAndRemoveInstruction(Load);
    return true;
  }
 
  // Step 3: Eliminate partial redundancy.
  if (!isLoadPREEnabled())
    return Changed;
  if (!isLoadInLoopPREEnabled() && LI->getLoopFor(Load->getParent()))
    return Changed;
 
  if (performLoopLoadPRE(Load, ValuesPerBlock, UnavailableBlocks) ||
      performLoadPRE(Load, ValuesPerBlock, UnavailableBlocks))
    return true;
 
  return Changed;
}
 
bool GVNPassImpl::processAssumeIntrinsic(AssumeInst *IntrinsicI) {
  Value *V = IntrinsicI->getArgOperand(0);
 
  if (ConstantInt *Cond = dyn_cast<ConstantInt>(V)) {
    if (Cond->isZero()) {
      Type *Int8Ty = Type::getInt8Ty(V->getContext());
      Type *PtrTy = PointerType::get(V->getContext(), 0);
      // Insert a new store to null instruction before the load to indicate that
      // this code is not reachable.  FIXME: We could insert unreachable
      // instruction directly because we can modify the CFG.
      auto *NewS =
          new StoreInst(PoisonValue::get(Int8Ty), Constant::getNullValue(PtrTy),
                        IntrinsicI->getIterator());
      if (MSSAU) {
        const MemoryUseOrDef *FirstNonDom = nullptr;
        const auto *AL =
            MSSAU->getMemorySSA()->getBlockAccesses(IntrinsicI->getParent());
 
        // If there are accesses in the current basic block, find the first one
        // that does not come before NewS. The new memory access is inserted
        // after the found access or before the terminator if no such access is
        // found.
        if (AL) {
          for (const auto &Acc : *AL) {
            if (auto *Current = dyn_cast<MemoryUseOrDef>(&Acc))
              if (!Current->getMemoryInst()->comesBefore(NewS)) {
                FirstNonDom = Current;
                break;
              }
          }
        }
 
        auto *NewDef =
            FirstNonDom ? MSSAU->createMemoryAccessBefore(
                              NewS, nullptr,
                              const_cast<MemoryUseOrDef *>(FirstNonDom))
                        : MSSAU->createMemoryAccessInBB(
                              NewS, nullptr,
                              NewS->getParent(), MemorySSA::BeforeTerminator);
 
        MSSAU->insertDef(cast<MemoryDef>(NewDef), /*RenameUses=*/false);
      }
    }
    if (isAssumeWithEmptyBundle(*IntrinsicI)) {
      salvageAndRemoveInstruction(IntrinsicI);
      return true;
    }
    return false;
  }
 
  if (isa<Constant>(V)) {
    // If it's not false, and constant, it must evaluate to true. This means our
    // assume is assume(true), and thus, pointless, and we don't want to do
    // anything more here.
    return false;
  }
 
  Constant *True = ConstantInt::getTrue(V->getContext());
  return propagateEquality(V, True, IntrinsicI);
}
 
static void patchAndReplaceAllUsesWith(Instruction *I, Value *Repl) {
  patchReplacementInstruction(I, Repl);
  I->replaceAllUsesWith(Repl);
}
 
/// If a load has !invariant.group, try to find the most-dominating instruction
/// with the same metadata and equivalent pointer (modulo bitcasts and zero
/// GEPs). If one is found that dominates the load, its value can be reused.
static Instruction *findInvariantGroupValue(LoadInst *L, DominatorTree &DT) {
  Value *PointerOperand = L->getPointerOperand()->stripPointerCasts();
 
  // It's not safe to walk the use list of a global value because function
  // passes aren't allowed to look outside their functions.
  // FIXME: this could be fixed by filtering instructions from outside of
  // current function.
  if (isa<Constant>(PointerOperand))
    return nullptr;
 
  // Queue to process all pointers that are equivalent to load operand.
  SmallVector<Value *, 8> PointerUsesQueue;
  PointerUsesQueue.push_back(PointerOperand);
 
  Instruction *MostDominatingInstruction = L;
 
  // FIXME: This loop is potentially O(n^2) due to repeated dominates checks.
  while (!PointerUsesQueue.empty()) {
    Value *Ptr = PointerUsesQueue.pop_back_val();
    assert(Ptr && !isa<GlobalValue>(Ptr) &&
           "Null or GlobalValue should not be inserted");
 
    for (User *U : Ptr->users()) {
      auto *I = dyn_cast<Instruction>(U);
      if (!I || I == L || !DT.dominates(I, MostDominatingInstruction))
        continue;
 
      // Add bitcasts and zero GEPs to queue.
      // TODO: Should drop bitcast?
      if (isa<BitCastInst>(I) ||
          (isa<GetElementPtrInst>(I) &&
           cast<GetElementPtrInst>(I)->hasAllZeroIndices())) {
        PointerUsesQueue.push_back(I);
        continue;
      }
 
      // If we hit a load/store with an invariant.group metadata and the same
      // pointer operand, we can assume that value pointed to by the pointer
      // operand didn't change.
      if (I->hasMetadata(LLVMContext::MD_invariant_group) &&
          Ptr == getLoadStorePointerOperand(I) && !I->isVolatile())
        MostDominatingInstruction = I;
    }
  }
 
  return MostDominatingInstruction != L ? MostDominatingInstruction : nullptr;
}
 
/// Return the memory location accessed by the (masked) load/store instruction
/// `I`, if the instruction could potentially provide a useful value for
/// eliminating the load.
static std::optional<MemoryLocation>
maybeLoadStoreLocation(Instruction *I, bool AllowStores,
                       const TargetLibraryInfo *TLI) {
  if (auto *LI = dyn_cast<LoadInst>(I))
    return MemoryLocation::get(LI);
 
  if (auto *II = dyn_cast<IntrinsicInst>(I)) {
    switch (II->getIntrinsicID()) {
    case Intrinsic::masked_load:
      return MemoryLocation::getForArgument(II, 0, TLI);
    case Intrinsic::masked_store:
      if (AllowStores)
        return MemoryLocation::getForArgument(II, 1, TLI);
      return std::nullopt;
    default:
      break;
    }
  }
 
  if (!AllowStores)
    return std::nullopt;
 
  if (auto *SI = dyn_cast<StoreInst>(I))
    return MemoryLocation::get(SI);
  return std::nullopt;
}
 
/// Scan the users of each MemoryAccess in `ClobbersList` that belong to `BB`,
/// looking for memory reads whose location aliases `Loc` and dominates our
/// load.
std::optional<GVNPassImpl::ReachingMemVal> GVNPassImpl::scanMemoryAccessesUsers(
    const MemoryLocation &Loc, bool IsInvariantLoad, BasicBlock *BB,
    const SmallVectorImpl<MemoryAccess *> &ClobbersList, MemorySSA &MSSA,
    BatchAAResults &AA, LoadInst *L) {
 
  // Prefer a candidate that is closer to the load within the same block.
  auto UpdateChoice = [&](std::optional<ReachingMemVal> &Choice,
                          AliasResult &AR, Instruction *Candidate) {
    if (!Choice) {
      if (AR == AliasResult::PartialAlias)
        Choice = ReachingMemVal::getClobber(Loc.Ptr, Candidate, AR.getOffset());
      else
        Choice = ReachingMemVal::getDef(Loc.Ptr, Candidate);
      return;
    }
    if (!MSSA.locallyDominates(MSSA.getMemoryAccess(Choice->Inst),
                               MSSA.getMemoryAccess(Candidate)))
      return;
 
    if (AR == AliasResult::PartialAlias) {
      Choice->Kind = DepKind::Clobber;
      Choice->Offset = AR.getOffset();
    } else {
      Choice->Kind = DepKind::Def;
      Choice->Offset = -1;
    }
 
    Choice->Inst = Candidate;
    Choice->Block = Candidate->getParent();
  };
 
  std::optional<ReachingMemVal> ReachingVal;
  for (MemoryAccess *MA : ClobbersList) {
    unsigned Scanned = 0;
    for (User *U : MA->users()) {
      if (++Scanned >= Opts->gvn_scan_users_limit)
        return ReachingMemVal::getUnknown(BB, Loc.Ptr);
 
      auto *UseOrDef = dyn_cast<MemoryUseOrDef>(U);
      if (!UseOrDef || UseOrDef->getBlock() != BB)
        continue;
 
      Instruction *MemI = UseOrDef->getMemoryInst();
      if (MemI == L ||
          (L && !MSSA.locallyDominates(UseOrDef, MSSA.getMemoryAccess(L))))
        continue;
 
      if (auto MaybeLoc = maybeLoadStoreLocation(MemI, IsInvariantLoad, TLI)) {
        AliasResult AR = AA.alias(*MaybeLoc, Loc);
        // If the locations do not certainly alias, we cannot possibly infer the
        // following load loads the same value.
        if (AR == AliasResult::NoAlias || AR == AliasResult::MayAlias)
          continue;
 
        // Locations partially overlap, but neither is a subset of the other, or
        // the second location is before the first.
        if (AR == AliasResult::PartialAlias &&
            (!AR.hasOffset() || AR.getOffset() < 0))
          continue;
 
        // Found candidate, the new load memory location and the given location
        // must alias: precise overlap, or subset with non-negative offset.
        UpdateChoice(ReachingVal, AR, MemI);
      }
    }
    if (ReachingVal)
      break;
  }
 
  return ReachingVal;
}
 
/// Check if a given MemoryAccess (usually a MemoryDef) actually modifies a
/// given location. Returns a ReachingMemVal describing the dependency.
std::optional<GVNPassImpl::ReachingMemVal> GVNPassImpl::accessMayModifyLocation(
    MemoryAccess *ClobberMA, const MemoryLocation &Loc, Align LoadAlign,
    bool IsInvariantLoad, BasicBlock *BB, MemorySSA &MSSA, BatchAAResults &AA) {
  assert(ClobberMA->getBlock() == BB);
 
  // If the clobbering access is the entry memory state, we cannot say anything
  // about the content of the memory, except when we are accessing a local
  // object, which can be turned later into producing `undef`.
  if (MSSA.isLiveOnEntryDef(ClobberMA)) {
    if (auto *Alloc = dyn_cast<AllocaInst>(getUnderlyingObject(Loc.Ptr)))
      if (Alloc->getParent() == BB)
        return ReachingMemVal::getDef(Loc.Ptr, const_cast<AllocaInst *>(Alloc));
    return ReachingMemVal::getUnknown(BB, Loc.Ptr);
  }
 
  // Loads from "constant" memory can't be clobbered.
  if (IsInvariantLoad || AA.pointsToConstantMemory(Loc))
    return std::nullopt;
 
  auto GetOrdering = [](const Instruction *I) {
    if (auto *L = dyn_cast<LoadInst>(I))
      return L->getOrdering();
    return cast<StoreInst>(I)->getOrdering();
  };
  Instruction *ClobberI = cast<MemoryDef>(ClobberMA)->getMemoryInst();
 
  // Check if the clobbering access is a load or a store that we can reuse.
  if (auto MaybeLoc = maybeLoadStoreLocation(ClobberI, true, TLI)) {
    AliasResult AR = AA.alias(*MaybeLoc, Loc);
    if (AR == AliasResult::MustAlias)
      return ReachingMemVal::getDef(Loc.Ptr, ClobberI);
 
    if (AR == AliasResult::NoAlias) {
      // If the locations do not alias we may still be able to skip over the
      // clobbering instruction, even if it is atomic.
      // The original load is either non-atomic or unordered. We can reorder
      // these across non-atomic, unordered or monotonic loads or across any
      // store.
      if (!ClobberI->isAtomic() ||
          !isStrongerThan(GetOrdering(ClobberI), AtomicOrdering::Monotonic) ||
          isa<StoreInst>(ClobberI))
        return std::nullopt;
      return ReachingMemVal::getClobber(Loc.Ptr, ClobberI);
    }
 
    // Skip over volatile loads (the original load is non-volatile, non-atomic).
    if (!ClobberI->isAtomic() && isa<LoadInst>(ClobberI))
      return std::nullopt;
 
    // A store that writes back a value already at the memory location leaves
    // the latter unchanged.
    if (auto *SI = dyn_cast<StoreInst>(ClobberI))
      if (isStorePreservingMemoryLocation(SI, Loc, LoadAlign, AA,
                                          Opts->gvn_max_num_insns))
        return std::nullopt;
 
    if (AR == AliasResult::MayAlias ||
        (AR == AliasResult::PartialAlias &&
         (!AR.hasOffset() || AR.getOffset() < 0)))
      return ReachingMemVal::getClobber(Loc.Ptr, ClobberI);
 
    // The only option left is a store of the superset of the required bits.
    assert(AR == AliasResult::PartialAlias && AR.hasOffset() &&
           AR.getOffset() > 0 &&
           "Must be the superset/partial overlap case with positive offset");
    return ReachingMemVal::getClobber(Loc.Ptr, ClobberI, AR.getOffset());
  }
 
  if (auto *II = dyn_cast<IntrinsicInst>(ClobberI)) {
    if (II->getIntrinsicID() == Intrinsic::lifetime_start) {
      MemoryLocation IIObjLoc = MemoryLocation::getForArgument(II, 0, TLI);
      if (AA.isMustAlias(IIObjLoc, Loc))
        return ReachingMemVal::getDef(Loc.Ptr, ClobberI);
      return std::nullopt;
    }
  }
 
  // If we are at a malloc-like function call, we can turn the load into `undef`
  // or zero.
  if (isNoAliasCall(ClobberI)) {
    const Value *Obj = getUnderlyingObject(Loc.Ptr);
    if (Obj == ClobberI || AA.isMustAlias(ClobberI, Loc.Ptr))
      return ReachingMemVal::getDef(Loc.Ptr, ClobberI);
  }
 
  // Can reorder loads across a release fence.
  if (auto *FI = dyn_cast<FenceInst>(ClobberI))
    if (FI->getOrdering() == AtomicOrdering::Release)
      return std::nullopt;
 
  // See if the clobber instruction (e.g., a generic call) may modify the
  // location.
  ModRefInfo MR = AA.getModRefInfo(ClobberI, Loc);
  // If may modify the location, analyze deeper, to exclude accesses to
  // non-escaping local allocations.
  if (MR == ModRefInfo::NoModRef || MR == ModRefInfo::Ref)
    return std::nullopt;
 
  // Conservatively assume the clobbering memory access may overwrite the
  // location.
  return ReachingMemVal::getClobber(Loc.Ptr, ClobberI);
}
 
/// Collect the predecessors of block, while doing phi-translation of the memory
/// address and the memory clobber. Return false if the block should be marked
/// as clobbering the memory location in an unknown way.
bool GVNPassImpl::collectPredecessors(BasicBlock *BB, const PHITransAddr &Addr,
                                      MemoryAccess *ClobberMA,
                                      DependencyBlockSet &Blocks,
                                      SmallVectorImpl<BasicBlock *> &Worklist) {
  if (Addr.needsPHITranslationFromBlock(BB) &&
      !Addr.isPotentiallyPHITranslatable())
    return false;
 
  auto *MPhi =
      ClobberMA->getBlock() == BB ? dyn_cast<MemoryPhi>(ClobberMA) : nullptr;
  SmallVector<std::pair<BasicBlock *, DependencyBlockInfo>, 8> Preds;
  for (BasicBlock *Pred : predecessors(BB)) {
    // Skip unreachable predecessors.
    if (!DT->isReachableFromEntry(Pred))
      continue;
 
    // Skip already visited predecessors.
    if (llvm::any_of(Preds, [Pred](const auto &P) { return P.first == Pred; }))
      continue;
 
    PHITransAddr TransAddr = Addr;
    if (TransAddr.needsPHITranslationFromBlock(BB))
      TransAddr.translateValue(BB, Pred, DT, false);
 
    auto It = Blocks.find(Pred);
    if (It != Blocks.end()) {
      // If we reach a visited block with a different address, set the
      // current block as clobbering the memory location in an unknown way
      // (by returning false).
      if (It->second.Addr.getAddr() != TransAddr.getAddr())
        return false;
      // Otherwise, just stop the traversal.
      continue;
    }
 
    Preds.emplace_back(
        Pred, DependencyBlockInfo(TransAddr,
                                  MPhi ? MPhi->getIncomingValueForBlock(Pred)
                                       : ClobberMA));
  }
 
  // We collected the predecessors and stored them in Preds. Now, populate the
  // worklist with the predecessors found, and cache the eventual translated
  // address for each block.
  for (auto &P : Preds) {
    [[maybe_unused]] auto It =
        Blocks.try_emplace(P.first, std::move(P.second)).first;
    Worklist.push_back(P.first);
  }
 
  return true;
}
 
/// Build a list of MemoryAccesses whose users could potentially alias the
/// memory location being queried. Starts from StartInfo's initial clobber,
/// walk the use-def chain to the final clobber. If the chain extends beyond
/// `BB`, continue into that block but only if it is in the previously collected
/// set.
void GVNPassImpl::collectClobberList(SmallVectorImpl<MemoryAccess *> &Clobbers,
                                     BasicBlock *BB,
                                     const DependencyBlockInfo &StartInfo,
                                     const DependencyBlockSet &Blocks,
                                     MemorySSA &MSSA) {
  MemoryAccess *MA = StartInfo.InitialClobberMA;
  MemoryAccess *LastMA = StartInfo.ClobberMA;
 
  for (;;) {
    while (MA != LastMA) {
      Clobbers.push_back(MA);
      MA = cast<MemoryUseOrDef>(MA)->getDefiningAccess();
    }
    Clobbers.push_back(MA);
 
    if (MSSA.isLiveOnEntryDef(MA) ||
        (MA->getBlock() == BB && !isa<MemoryPhi>(MA)))
      break;
 
    // If the final clobber in the current block is a MemoryPhi, go to the
    // immediate dominator; otherwise, just get to the block containing the
    // final clobber.
    if (MA->getBlock() == BB)
      BB = DT->getNode(BB)->getIDom()->getBlock();
    else
      BB = MA->getBlock();
 
    auto It = Blocks.find(BB);
    if (It == Blocks.end())
      break;
 
    MA = It->second.InitialClobberMA;
    LastMA = It->second.ClobberMA;
    if (MA == Clobbers.back())
      Clobbers.pop_back();
  }
}
 
/// Entrypoint for the MemorySSA-based redundant load elimination algorithm.
/// Given as input a load instruction, the function computes the set of reaching
/// memory values, one per predecessor path, that analyzeLoadAvailability can
/// later use to establish whether the load may be eliminated. A reaching value
/// may be of the following descriptor kind:
/// * Def: a precise instruction that produces the exact bits the load would
/// read (e.g., an equivalent load or a MustAlias store);
/// * Clobber: a write that clobbers a superset of the bits the load would read
/// (e.g., a memset over a larger region);
/// * Other: we know which block defines the memory location in some way, but
/// could not identify a precise instruction (e.g., memory already live at
/// function entry).
bool GVNPassImpl::findReachingValuesForLoad(
    LoadInst *L, SmallVectorImpl<ReachingMemVal> &Values, MemorySSA &MSSA,
    AAResults &AAR) {
  BatchAAResults AA(AAR, EA);
  BasicBlock *StartBlock = L->getParent();
  bool IsInvariantLoad = L->hasMetadata(LLVMContext::MD_invariant_load);
  // TODO: Simplify later work by just getClobberingMemoryAccess().
  MemoryAccess *ClobberMA = MSSA.getMemoryAccess(L)->getDefiningAccess();
  const MemoryLocation Loc = MemoryLocation::get(L);
 
  // Fast path for load tagged with !invariant.group.
  if (L->hasMetadata(LLVMContext::MD_invariant_group)) {
    if (Instruction *G = findInvariantGroupValue(L, *DT)) {
      Values.emplace_back(
          ReachingMemVal::getDef(getLoadStorePointerOperand(G), G));
      return true;
    }
  }
 
  // Phase 1. First off, look for a local dependency to avoid having to
  // disambiguate between before the load and after the load of the starting
  // block (as the load may be visited from a backedge).
  for (;;) {
    // Scan users of the clobbering memory access.
    if (auto RMV = scanMemoryAccessesUsers(
            Loc, IsInvariantLoad, StartBlock,
            SmallVector<MemoryAccess *, 1>{ClobberMA}, MSSA, AA, L)) {
      Values.emplace_back(*RMV);
      return true;
    }
 
    // Exit from here, and proceed visiting predecessors if the clobbering
    // access is non-local or is a MemoryPhi.
    if (ClobberMA->getBlock() != StartBlock || isa<MemoryPhi>(ClobberMA))
      break;
 
    // Check if the clobber actually aliases the load location.
    if (auto RMV =
            accessMayModifyLocation(ClobberMA, Loc, L->getAlign(),
                                    IsInvariantLoad, StartBlock, MSSA, AA)) {
      Values.emplace_back(*RMV);
      return true;
    }
 
    // It may happen that the clobbering memory access does not actually
    // clobber our load location, transition to its defining memory access.
    ClobberMA = cast<MemoryUseOrDef>(ClobberMA)->getDefiningAccess();
  }
 
  // Non-local speculations are not allowed under ASan.
  if (L->getFunction()->hasFnAttribute(Attribute::SanitizeAddress) ||
      L->getFunction()->hasFnAttribute(Attribute::SanitizeHWAddress))
    return false;
 
  // Phase 2. Walk backwards through the CFG, collecting all the blocks that
  // contain an instruction that modifies the load memory location, or that lie
  // on a path between a clobbering block and our load. Start off by collecting
  // the predecessors of `StartBlock`. All the visited blocks are stored in a
  // the set `Blocks`. If possible, the memory address maintained for the block
  // visited does get phi-translated.
  DependencyBlockSet Blocks;
  SmallVector<BasicBlock *, 16> InitialWorklist;
  const DataLayout &DL = L->getDataLayout();
  if (!collectPredecessors(StartBlock,
                           PHITransAddr(L->getPointerOperand(), DL, AC),
                           ClobberMA, Blocks, InitialWorklist))
    return false;
 
  // Do a bottom-up DFS.
  auto Worklist = InitialWorklist;
  while (!Worklist.empty()) {
    // Match MemDep's cutoff for expensive non-local queries.
    if (Blocks.size() > Opts->gvn_max_num_reaching_blocks)
      return false;
    auto *BB = Worklist.pop_back_val();
    DependencyBlockInfo &Info = Blocks.find(BB)->second;
 
    // Phi-translation may have failed.
    if (!Info.Addr.getAddr())
      continue;
 
    // If the clobbering memory access is in the current block and it indeed
    // clobbers our load location, record the dependency and do not visit the
    // predecessors of this block further, continue with the blocks in the
    // worklist.
    if (Info.ClobberMA->getBlock() == BB && !isa<MemoryPhi>(Info.ClobberMA)) {
      const MemoryLocation BBLoc = Loc.getWithNewPtr(Info.Addr.getAddr());
      if (auto RMV =
              accessMayModifyLocation(Info.ClobberMA, BBLoc, L->getAlign(),
                                      IsInvariantLoad, BB, MSSA, AA)) {
        Info.MemVal = RMV;
        continue;
      }
      assert(!MSSA.isLiveOnEntryDef(Info.ClobberMA) &&
             "LiveOnEntry aliases everything");
 
      // If, however, the clobbering memory access does not actually clobber
      // our load location, transition to its defining memory access, but
      // keep examining the same basic block.
      Info.ClobberMA =
          cast<MemoryUseOrDef>(Info.ClobberMA)->getDefiningAccess();
      Worklist.emplace_back(BB);
      continue;
    }
 
    // At this point we know the current block is "transparent", i.e. the memory
    // location is not modified when execution goes through this block.
    // Continue to its predecessors, unless a predecessor has already been
    // visited with a different address. We currently cannot represent such a
    // dependency.
    if (BB == StartBlock && Info.Addr.getAddr() != L->getPointerOperand()) {
      Info.ForceUnknown = true;
      continue;
    }
    if (BB != StartBlock &&
        !collectPredecessors(BB, Info.Addr, Info.ClobberMA, Blocks, Worklist))
      Info.ForceUnknown = true;
  }
 
  // Phase 3. We have collected all the blocks that either write a value to the
  // memory location of the load, or there exists a path to the load, along
  // which the memory location is not modified. Perform a second DFS to find
  // load-to-load dependencies; namely, look at the dominating memory reads,
  // that alias our load. These are the MemoryUses that are users of the
  // MemoryDefs we previously identified. If no memory read is encountered,
  // either confirm the clobbering write found before or set to unknown.
  Worklist = InitialWorklist;
  for (BasicBlock *BB : Worklist) {
    DependencyBlockInfo &Info = Blocks.find(BB)->second;
    Info.Visited = true;
  }
 
  SmallVector<MemoryAccess *> Clobbers;
  while (!Worklist.empty()) {
    auto *BB = Worklist.pop_back_val();
    DependencyBlockInfo &Info = Blocks.find(BB)->second;
 
    // If phi-translation failed, assume the memory location is modified in
    // unknown way.
    if (!Info.Addr.getAddr()) {
      Values.push_back(ReachingMemVal::getUnknown(BB, nullptr));
      continue;
    }
 
    Clobbers.clear();
    collectClobberList(Clobbers, BB, Info, Blocks, MSSA);
    if (auto RMV =
            scanMemoryAccessesUsers(Loc.getWithNewPtr(Info.Addr.getAddr()),
                                    IsInvariantLoad, BB, Clobbers, MSSA, AA)) {
      Values.push_back(*RMV);
      continue;
    }
 
    // If no reusable memory use was found, and the current block is not
    // transparent, use the already established memory def.
    if (Info.MemVal) {
      Values.push_back(*Info.MemVal);
      continue;
    }
 
    if (Info.ForceUnknown) {
      Values.push_back(ReachingMemVal::getUnknown(BB, Info.Addr.getAddr()));
      continue;
    }
 
    // If the current block is transparent, continue to its predecessors.
    for (BasicBlock *Pred : predecessors(BB)) {
      auto It = Blocks.find(Pred);
      if (It == Blocks.end())
        continue;
      DependencyBlockInfo &PredInfo = It->second;
      if (PredInfo.Visited)
        continue;
      PredInfo.Visited = true;
      Worklist.push_back(Pred);
    }
  }
 
  return true;
}
 
/// Attempt to eliminate a load, first by eliminating it
/// locally, and then attempting non-local elimination if that fails.
bool GVNPassImpl::processLoad(LoadInst *L) {
  if (!MD && !isMemorySSAEnabled())
    return false;
 
  // This code hasn't been audited for ordered or volatile memory access.
  if (!L->isUnordered())
    return false;
 
  if (L->getType()->isTokenLikeTy())
    return false;
 
  if (L->use_empty()) {
    salvageAndRemoveInstruction(L);
    return true;
  }
 
  ReachingMemVal MemVal = ReachingMemVal::getUnknown(nullptr, nullptr);
  if (!isMemorySSAEnabled()) {
    // ... to a pointer that has been loaded from before...
    MemDepResult Dep = MD->getDependency(L);
 
    // If it is defined in another block, try harder.
    if (Dep.isNonLocal())
      return processNonLocalLoad(L);
 
    // Only handle the local case below.
    if (Dep.isDef())
      MemVal = ReachingMemVal::getDef(L->getPointerOperand(), Dep.getInst());
    else if (Dep.isClobber())
      MemVal =
          ReachingMemVal::getClobber(L->getPointerOperand(), Dep.getInst());
  } else {
    SmallVector<ReachingMemVal, 8> MemVals;
    if (!findReachingValuesForLoad(L, MemVals, *MSSAU->getMemorySSA(), *AA))
      return false; // Too many dependencies.
    assert(MemVals.size() && "Expected at least an unknown value");
    if (MemVals.size() > 1 || MemVals[0].Block != L->getParent())
      return processNonLocalLoad(L, MemVals);
 
    MemVal = MemVals[0];
  }
 
  if (MemVal.Kind == DepKind::Other) {
    // This might be a NonFuncLocal or an Unknown.
    LLVM_DEBUG(
        // fast print dep, using operator<< on instruction is too slow.
        dbgs() << "GVN: load "; L->printAsOperand(dbgs());
        dbgs() << " has unknown dependence\n";);
    return false;
  }
 
  auto AV = analyzeLoadAvailability(L, MemVal, L->getPointerOperand());
  if (!AV)
    return false;
 
  Value *AvailableValue = AV->MaterializeAdjustedValue(L, L);
 
  // MaterializeAdjustedValue is responsible for combining metadata.
  ICF->removeUsersOf(L);
  L->replaceAllUsesWith(AvailableValue);
  if (MSSAU)
    MSSAU->removeMemoryAccess(L);
  ++NumGVNLoad;
  reportLoadElim(L, AvailableValue, ORE);
  salvageAndRemoveInstruction(L);
  // Tell MDA to reexamine the reused pointer since we might have more
  // information after forwarding it.
  if (MD && AvailableValue->getType()->isPtrOrPtrVectorTy())
    MD->invalidateCachedPointerInfo(AvailableValue);
  return true;
}
 
// Attempt to process masked loads which have loaded from
// masked stores with the same mask
bool GVNPassImpl::processMaskedLoad(IntrinsicInst *I) {
  BatchAAResults BatchAA(*AA);
  Instruction *DepInst = nullptr;
  MemoryLocation LoadLoc = MemoryLocation::getForArgument(I, 0, TLI);
 
  if (!isMemorySSAEnabled()) {
    if (!MD)
      return false;
    MemDepResult Dep = MD->getDependency(I);
    DepInst = Dep.getInst();
    if (!DepInst || !Dep.isLocal() || !Dep.isDef())
      return false;
  } else {
    auto *MSSA = MSSAU->getMemorySSA();
    auto *Def = dyn_cast<MemoryDef>(
        MSSA->getWalker()->getClobberingMemoryAccess(I, BatchAA));
    // We may want to extend this to non-local forwarding as well. For now, keep
    // parity with MemoryDependenceAnalysis.
    if (!Def || MSSA->isLiveOnEntryDef(Def) ||
        Def->getBlock() != I->getParent())
      return false;
    DepInst = Def->getMemoryInst();
  }
 
  Value *Mask = I->getOperand(1);
  Value *Passthrough = I->getOperand(2);
  Value *StoreVal;
  if (!match(DepInst,
             m_MaskedStore(m_Value(StoreVal), m_Value(), m_Specific(Mask))) ||
      StoreVal->getType() != I->getType())
    return false;
 
  if (isMemorySSAEnabled()) {
    // Ensure the masked store and load access the same exact memory location.
    MemoryLocation StoreLoc =
        MemoryLocation::getForArgument(cast<IntrinsicInst>(DepInst), 1, TLI);
    if (!BatchAA.isMustAlias(StoreLoc, LoadLoc))
      return false;
  }
 
  // Remove the load but generate a select for the passthrough
  Value *OpToForward = llvm::SelectInst::Create(Mask, StoreVal, Passthrough, "",
                                                I->getIterator());
 
  ICF->removeUsersOf(I);
  I->replaceAllUsesWith(OpToForward);
  salvageAndRemoveInstruction(I);
  ++NumGVNLoad;
  return true;
}
 
/// Return a pair the first field showing the value number of \p Exp and the
/// second field showing whether it is a value number newly created.
std::pair<uint32_t, bool> GVNValueTable::assignExpNewValueNum(Expression &Exp) {
  uint32_t &E = ExpressionNumbering[Exp];
  bool CreateNewValNum = !E;
  if (CreateNewValNum) {
    Expressions.push_back(Exp);
    if (ExprIdx.size() < NextValueNumber + 1)
      ExprIdx.resize(NextValueNumber * 2);
    E = NextValueNumber;
    ExprIdx[NextValueNumber++] = NextExprNumber++;
  }
  return {E, CreateNewValNum};
}
 
/// Return whether all the values related with the same \p num are
/// defined in \p BB.
bool GVNValueTable::areAllValsInBB(uint32_t Num, const BasicBlock *BB,
                                   GVNLeaderMap &LeaderTable) {
  return all_of(
      LeaderTable.getLeaders(Num),
      [=](const GVNLeaderMap::LeaderTableEntry &L) { return L.BB == BB; });
}
 
/// Wrap phiTranslateImpl to provide caching functionality.
uint32_t GVNValueTable::phiTranslate(const BasicBlock *Pred,
                                     const BasicBlock *PhiBlock, uint32_t Num,
                                     GVNLeaderMap &LeaderTable) {
  auto FindRes = PhiTranslateTable.find({Num, Pred});
  if (FindRes != PhiTranslateTable.end())
    return FindRes->second;
  uint32_t NewNum = phiTranslateImpl(Pred, PhiBlock, Num, LeaderTable);
  PhiTranslateTable.insert({{Num, Pred}, NewNum});
  return NewNum;
}
 
// Return true if the value number \p Num and NewNum have equal value.
// Return false if the result is unknown.
bool GVNValueTable::areCallValsEqual(uint32_t Num, uint32_t NewNum,
                                     const BasicBlock *Pred,
                                     const BasicBlock *PhiBlock,
                                     GVNLeaderMap &LeaderTable) {
  CallInst *Call = nullptr;
  auto Leaders = LeaderTable.getLeaders(Num);
  for (const auto &Entry : Leaders) {
    Call = dyn_cast<CallInst>(&*Entry.Val);
    if (Call && Call->getParent() == PhiBlock)
      break;
  }
 
  if (AA->doesNotAccessMemory(Call))
    return true;
 
  if (!MD || !AA->onlyReadsMemory(Call))
    return false;
 
  MemDepResult LocalDep = MD->getDependency(Call);
  if (!LocalDep.isNonLocal())
    return false;
 
  const MemoryDependenceResults::NonLocalDepInfo &Deps =
      MD->getNonLocalCallDependency(Call);
 
  // Check to see if the Call has no function local clobber.
  for (const NonLocalDepEntry &D : Deps) {
    if (D.getResult().isNonFuncLocal())
      return true;
  }
  return false;
}
 
/// Translate value number \p Num using phis, so that it has the values of
/// the phis in BB.
uint32_t GVNValueTable::phiTranslateImpl(const BasicBlock *Pred,
                                         const BasicBlock *PhiBlock,
                                         uint32_t Num,
                                         GVNLeaderMap &LeaderTable) {
  // See if we can refine the value number by looking at the PN incoming value
  // for the given predecessor.
  if (PHINode *PN = NumberingPhi[Num]) {
    if (PN->getParent() != PhiBlock)
      return Num;
    for (unsigned I = 0; I != PN->getNumIncomingValues(); ++I) {
      if (PN->getIncomingBlock(I) != Pred)
        continue;
      if (uint32_t TransVal = lookup(PN->getIncomingValue(I), false))
        return TransVal;
    }
    return Num;
  }
 
  if (BasicBlock *BB = NumberingBB[Num]) {
    assert(MSSA && "NumberingBB is non-empty only when using MemorySSA");
    // Value numbers of basic blocks are used to represent memory state in
    // load/store instructions and read-only function calls when said state is
    // set by a MemoryPhi.
    if (BB != PhiBlock)
      return Num;
    MemoryPhi *MPhi = MSSA->getMemoryAccess(BB);
    for (unsigned i = 0, N = MPhi->getNumIncomingValues(); i != N; ++i) {
      if (MPhi->getIncomingBlock(i) != Pred)
        continue;
      MemoryAccess *MA = MPhi->getIncomingValue(i);
      if (auto *PredPhi = dyn_cast<MemoryPhi>(MA))
        return lookupOrAdd(PredPhi->getBlock());
      if (MSSA->isLiveOnEntryDef(MA))
        return lookupOrAdd(&BB->getParent()->getEntryBlock());
      return lookupOrAdd(cast<MemoryUseOrDef>(MA)->getMemoryInst());
    }
    llvm_unreachable(
        "CFG/MemorySSA mismatch: predecessor not found among incoming blocks");
  }
 
  // If there is any value related with Num is defined in a BB other than
  // PhiBlock, it cannot depend on a phi in PhiBlock without going through
  // a backedge. We can do an early exit in that case to save compile time.
  if (!areAllValsInBB(Num, PhiBlock, LeaderTable))
    return Num;
 
  if (Num >= ExprIdx.size() || ExprIdx[Num] == 0)
    return Num;
  Expression Exp = Expressions[ExprIdx[Num]];
 
  for (unsigned I = 0; I < Exp.VarArgs.size(); I++) {
    // For InsertValue and ExtractValue, some varargs are index numbers
    // instead of value numbers. Those index numbers should not be
    // translated.
    if ((I > 1 && Exp.Opcode == Instruction::InsertValue) ||
        (I > 0 && Exp.Opcode == Instruction::ExtractValue) ||
        (I > 1 && Exp.Opcode == Instruction::ShuffleVector))
      continue;
    Exp.VarArgs[I] = phiTranslate(Pred, PhiBlock, Exp.VarArgs[I], LeaderTable);
  }
 
  if (Exp.Commutative) {
    assert(Exp.VarArgs.size() >= 2 && "Unsupported commutative instruction!");
    if (Exp.VarArgs[0] > Exp.VarArgs[1]) {
      std::swap(Exp.VarArgs[0], Exp.VarArgs[1]);
      uint32_t Opcode = Exp.Opcode >> 8;
      if (Opcode == Instruction::ICmp || Opcode == Instruction::FCmp)
        Exp.Opcode = (Opcode << 8) |
                     CmpInst::getSwappedPredicate(
                         static_cast<CmpInst::Predicate>(Exp.Opcode & 255));
    }
  }
 
  if (uint32_t NewNum = ExpressionNumbering[Exp]) {
    if (Exp.Opcode == Instruction::Call && NewNum != Num)
      return areCallValsEqual(Num, NewNum, Pred, PhiBlock, LeaderTable) ? NewNum
                                                                        : Num;
    return NewNum;
  }
  return Num;
}
 
/// Erase stale entry from phiTranslate cache so phiTranslate can be computed
/// again.
void GVNValueTable::eraseTranslateCacheEntry(uint32_t Num,
                                             const BasicBlock &CurrBlock) {
  for (const BasicBlock *Pred : predecessors(&CurrBlock))
    PhiTranslateTable.erase({Num, Pred});
}
 
// In order to find a leader for a given value number at a
// specific basic block, we first obtain the list of all Values for that number,
// and then scan the list to find one whose block dominates the block in
// question.  This is fast because dominator tree queries consist of only
// a few comparisons of DFS numbers.
Value *GVNPassImpl::findLeader(const BasicBlock *BB, uint32_t Num) {
  auto Leaders = LeaderTable.getLeaders(Num);
  if (Leaders.empty())
    return nullptr;
 
  Value *Val = nullptr;
  for (const auto &Entry : Leaders) {
    if (DT->dominates(Entry.BB, BB)) {
      Val = Entry.Val;
      if (isa<Constant>(Val))
        return Val;
    }
  }
 
  return Val;
}
 
/// There is an edge from 'Src' to 'Dst'.  Return
/// true if every path from the entry block to 'Dst' passes via this edge.  In
/// particular 'Dst' must not be reachable via another edge from 'Src'.
static bool isOnlyReachableViaThisEdge(const BasicBlockEdge &E,
                                       DominatorTree *DT) {
  // While in theory it is interesting to consider the case in which Dst has
  // more than one predecessor, because Dst might be part of a loop which is
  // only reachable from Src, in practice it is pointless since at the time
  // GVN runs all such loops have preheaders, which means that Dst will have
  // been changed to have only one predecessor, namely Src.
  const BasicBlock *Pred = E.getEnd()->getSinglePredecessor();
  assert((!Pred || Pred == E.getStart()) &&
         "No edge between these basic blocks!");
  return Pred != nullptr;
}
 
void GVNPassImpl::assignBlockRPONumber(Function &F) {
  BlockRPONumber.clear();
  uint32_t NextBlockNumber = 1;
  ReversePostOrderTraversal<Function *> RPOT(&F);
  for (BasicBlock *BB : RPOT)
    BlockRPONumber[BB] = NextBlockNumber++;
  InvalidBlockRPONumbers = false;
}
 
/// The given values are known to be equal in every use
/// dominated by 'Root'.  Exploit this, for example by replacing 'LHS' with
/// 'RHS' everywhere in the scope.  Returns whether a change was made.
/// The Root may either be a basic block edge (for conditions) or an
/// instruction (for assumes).
bool GVNPassImpl::propagateEquality(
    Value *LHS, Value *RHS,
    const std::variant<BasicBlockEdge, Instruction *> &Root) {
  SmallVector<std::pair<Value*, Value*>, 4> Worklist;
  SmallDenseSet<std::pair<Value *, Value *>, 4> Visited;
  Worklist.push_back(std::make_pair(LHS, RHS));
  bool Changed = false;
  SmallVector<const BasicBlock *> DominatedBlocks;
  if (const BasicBlockEdge *Edge = std::get_if<BasicBlockEdge>(&Root)) {
    // For speed, compute a conservative fast approximation to
    // DT->dominates(Root, Root.getEnd());
    if (isOnlyReachableViaThisEdge(*Edge, DT))
      DominatedBlocks.push_back(Edge->getEnd());
  } else {
    Instruction *I = std::get<Instruction *>(Root);
    for (const auto *Node : DT->getNode(I->getParent())->children())
      DominatedBlocks.push_back(Node->getBlock());
  }
 
  while (!Worklist.empty()) {
    std::pair<Value*, Value*> Item = Worklist.pop_back_val();
    LHS = Item.first; RHS = Item.second;
 
    if (LHS == RHS)
      continue;
    assert(LHS->getType() == RHS->getType() && "Equality but unequal types!");
 
    // Don't try to propagate equalities between constants.
    if (isa<Constant>(LHS) && isa<Constant>(RHS))
      continue;
 
    // Prefer a constant on the right-hand side, or an Argument if no constants.
    if (isa<Constant>(LHS) || (isa<Argument>(LHS) && !isa<Constant>(RHS)))
      std::swap(LHS, RHS);
    assert((isa<Argument>(LHS) || isa<Instruction>(LHS)) && "Unexpected value!");
    const DataLayout &DL =
        isa<Argument>(LHS)
            ? cast<Argument>(LHS)->getParent()->getDataLayout()
            : cast<Instruction>(LHS)->getDataLayout();
 
    // If there is no obvious reason to prefer the left-hand side over the
    // right-hand side, ensure the longest lived term is on the right-hand side,
    // so the shortest lived term will be replaced by the longest lived.
    // This tends to expose more simplifications.
    uint32_t LVN = VN.lookupOrAdd(LHS);
    if ((isa<Argument>(LHS) && isa<Argument>(RHS)) ||
        (isa<Instruction>(LHS) && isa<Instruction>(RHS))) {
      // Move the 'oldest' value to the right-hand side, using the value number
      // as a proxy for age.
      uint32_t RVN = VN.lookupOrAdd(RHS);
      if (LVN < RVN) {
        std::swap(LHS, RHS);
        LVN = RVN;
      }
    }
 
    if (!Visited.insert({LHS, RHS}).second)
      continue;
 
    // If value numbering later sees that an instruction in the scope is equal
    // to 'LHS' then ensure it will be turned into 'RHS'.  In order to preserve
    // the invariant that instructions only occur in the leader table for their
    // own value number (this is used by removeFromLeaderTable), do not do this
    // if RHS is an instruction (if an instruction in the scope is morphed into
    // LHS then it will be turned into RHS by the next GVN iteration anyway, so
    // using the leader table is about compiling faster, not optimizing better).
    // The leader table only tracks basic blocks, not edges. Only add to if we
    // have the simple case where the edge dominates the end.
    if (!isa<Instruction>(RHS) && canReplacePointersIfEqual(LHS, RHS, DL))
      for (const BasicBlock *BB : DominatedBlocks)
        LeaderTable.insert(LVN, RHS, BB);
 
    // Replace all occurrences of 'LHS' with 'RHS' everywhere in the scope.  As
    // LHS always has at least one use that is not dominated by Root, this will
    // never do anything if LHS has only one use.
    if (!LHS->hasOneUse()) {
      // Create a callback that captures the DL.
      auto CanReplacePointersCallBack = [&DL](const Use &U, const Value *To) {
        return canReplacePointersInUseIfEqual(U, To, DL);
      };
      unsigned NumReplacements;
      if (const BasicBlockEdge *Edge = std::get_if<BasicBlockEdge>(&Root))
        NumReplacements = replaceDominatedUsesWithIf(
            LHS, RHS, *DT, *Edge, CanReplacePointersCallBack);
      else
        NumReplacements = replaceDominatedUsesWithIf(
            LHS, RHS, *DT, std::get<Instruction *>(Root),
            CanReplacePointersCallBack);
 
      if (NumReplacements > 0) {
        Changed = true;
        NumGVNEqProp += NumReplacements;
        // Cached information for anything that uses LHS will be invalid.
        if (MD)
          MD->invalidateCachedPointerInfo(LHS);
      }
    }
 
    // Now try to deduce additional equalities from this one. For example, if
    // the known equality was "(A != B)" == "false" then it follows that A and B
    // are equal in the scope. Only boolean equalities with an explicit true or
    // false RHS are currently supported.
    if (!RHS->getType()->isIntegerTy(1))
      // Not a boolean equality - bail out.
      continue;
    ConstantInt *CI = dyn_cast<ConstantInt>(RHS);
    if (!CI)
      // RHS neither 'true' nor 'false' - bail out.
      continue;
    // Whether RHS equals 'true'.  Otherwise it equals 'false'.
    bool IsKnownTrue = CI->isMinusOne();
    bool IsKnownFalse = !IsKnownTrue;
 
    // If "A && B" is known true then both A and B are known true.  If "A || B"
    // is known false then both A and B are known false.
    Value *A, *B;
    if ((IsKnownTrue && match(LHS, m_LogicalAnd(m_Value(A), m_Value(B)))) ||
        (IsKnownFalse && match(LHS, m_LogicalOr(m_Value(A), m_Value(B))))) {
      Worklist.push_back(std::make_pair(A, RHS));
      Worklist.push_back(std::make_pair(B, RHS));
      continue;
    }
 
    // If we are propagating an equality like "(A == B)" == "true" then also
    // propagate the equality A == B.  When propagating a comparison such as
    // "(A >= B)" == "true", replace all instances of "A < B" with "false".
    if (CmpInst *Cmp = dyn_cast<CmpInst>(LHS)) {
      Value *Op0 = Cmp->getOperand(0), *Op1 = Cmp->getOperand(1);
 
      // If "A == B" is known true, or "A != B" is known false, then replace
      // A with B everywhere in the scope.  For floating point operations, we
      // have to be careful since equality does not always imply equivalance.
      if (Cmp->isEquivalence(IsKnownFalse))
        Worklist.push_back(std::make_pair(Op0, Op1));
 
      // If "A >= B" is known true, replace "A < B" with false everywhere.
      CmpInst::Predicate NotPred = Cmp->getInversePredicate();
      Constant *NotVal = ConstantInt::get(Cmp->getType(), IsKnownFalse);
      // Since we don't have the instruction "A < B" immediately to hand, work
      // out the value number that it would have and use that to find an
      // appropriate instruction (if any).
      uint32_t NextNum = VN.getNextUnusedValueNumber();
      uint32_t Num = VN.lookupOrAddCmp(Cmp->getOpcode(), NotPred, Op0, Op1);
      // If the number we were assigned was brand new then there is no point in
      // looking for an instruction realizing it: there cannot be one!
      if (Num < NextNum) {
        for (const auto &Entry : LeaderTable.getLeaders(Num)) {
          // Only look at leaders that either dominate the start of the edge,
          // or are dominated by the end. This check is not necessary for
          // correctness, it only discards cases for which the following
          // use replacement will not work anyway.
          if (const BasicBlockEdge *Edge = std::get_if<BasicBlockEdge>(&Root)) {
            if (!DT->dominates(Entry.BB, Edge->getStart()) &&
                !DT->dominates(Edge->getEnd(), Entry.BB))
              continue;
          } else {
            auto *InstBB = std::get<Instruction *>(Root)->getParent();
            if (!DT->dominates(Entry.BB, InstBB) &&
                !DT->dominates(InstBB, Entry.BB))
              continue;
          }
 
          Value *NotCmp = Entry.Val;
          if (NotCmp && isa<Instruction>(NotCmp)) {
            unsigned NumReplacements;
            if (const BasicBlockEdge *Edge = std::get_if<BasicBlockEdge>(&Root))
              NumReplacements =
                  replaceDominatedUsesWith(NotCmp, NotVal, *DT, *Edge);
            else
              NumReplacements = replaceDominatedUsesWith(
                  NotCmp, NotVal, *DT, std::get<Instruction *>(Root));
            Changed |= NumReplacements > 0;
            NumGVNEqProp += NumReplacements;
            // Cached information for anything that uses NotCmp will be invalid.
            if (MD)
              MD->invalidateCachedPointerInfo(NotCmp);
          }
        }
      }
      // Ensure that any instruction in scope that gets the "A < B" value number
      // is replaced with false.
      // The leader table only tracks basic blocks, not edges. Only add to if we
      // have the simple case where the edge dominates the end.
      for (const BasicBlock *BB : DominatedBlocks)
        LeaderTable.insert(Num, NotVal, BB);
 
      continue;
    }
 
    // Propagate equalities that results from truncation with no unsigned wrap
    // like (trunc nuw i64 %v to i1) == "true" or (trunc nuw i64 %v to i1) ==
    // "false"
    if (match(LHS, m_NUWTrunc(m_Value(A)))) {
      Worklist.emplace_back(A, ConstantInt::get(A->getType(), IsKnownTrue));
      continue;
    }
 
    if (match(LHS, m_Not(m_Value(A)))) {
      Worklist.emplace_back(A, ConstantInt::get(A->getType(), !IsKnownTrue));
      continue;
    }
  }
 
  return Changed;
}
 
bool GVNPassImpl::replaceWithEquivalentCmp(CmpInst *Cmp) {
  auto FindCmpLeader = [&](CmpInst::Predicate Pred) -> Value * {
    uint32_t Num = VN.lookupCmp(Cmp->getOpcode(), Pred, Cmp->getOperand(0),
                                Cmp->getOperand(1));
    if (Num != 0)
      return findLeader(Cmp->getParent(), Num);
    return nullptr;
  };
 
  // Substitute cmp instruction with not if possible.
  if (Value *Repl = FindCmpLeader(Cmp->getInversePredicate())) {
    patchReplacementInstruction(Cmp, Repl);
    BinaryOperator *Not = BinaryOperator::CreateNot(
        Repl, Repl->getName() + ".not", Cmp->getIterator());
    Not->setDebugLoc(Cmp->getDebugLoc());
    Cmp->replaceAllUsesWith(Not);
    salvageAndRemoveInstruction(Cmp);
    return true;
  }
 
  // Substitute icmp samesign upred with icmp spred
  auto *ICmp = dyn_cast<ICmpInst>(Cmp);
  if (ICmp && ICmp->hasSameSign() && !ICmp->isEquality()) {
    if (Value *Repl = FindCmpLeader(
            ICmpInst::getFlippedSignednessPredicate(ICmp->getPredicate()))) {
      patchAndReplaceAllUsesWith(Cmp, Repl);
      salvageAndRemoveInstruction(Cmp);
      return true;
    }
  }
  return false;
}
 
/// When calculating availability, handle an instruction
/// by inserting it into the appropriate sets.
bool GVNPassImpl::processInstruction(Instruction *I) {
  // If the instruction can be easily simplified then do so now in preference
  // to value numbering it.  Value numbering often exposes redundancies, for
  // example if it determines that %y is equal to %x then the instruction
  // "%z = and i32 %x, %y" becomes "%z = and i32 %x, %x" which we now simplify.
  const DataLayout &DL = I->getDataLayout();
  if (Value *V = simplifyInstruction(I, {DL, TLI, DT, AC})) {
    bool Changed = false;
    if (!I->use_empty()) {
      // Simplification can cause a special instruction to become not special.
      // For example, devirtualization to a willreturn function.
      ICF->removeUsersOf(I);
      I->replaceAllUsesWith(V);
      Changed = true;
    }
    if (isInstructionTriviallyDead(I, TLI)) {
      salvageAndRemoveInstruction(I);
      Changed = true;
    }
    if (Changed) {
      if (MD && V->getType()->isPtrOrPtrVectorTy())
        MD->invalidateCachedPointerInfo(V);
      ++NumGVNSimpl;
      return true;
    }
  }
 
  if (auto *Assume = dyn_cast<AssumeInst>(I))
    return processAssumeIntrinsic(Assume);
 
  if (LoadInst *Load = dyn_cast<LoadInst>(I)) {
    if (processLoad(Load))
      return true;
 
    unsigned Num = VN.lookupOrAdd(Load);
    LeaderTable.insert(Num, Load, Load->getParent());
    return false;
  }
 
  if (match(I, m_Intrinsic<Intrinsic::masked_load>()) &&
      processMaskedLoad(cast<IntrinsicInst>(I)))
    return true;
 
  // For conditional branches, we can perform simple conditional propagation on
  // the condition value itself.
  if (CondBrInst *BI = dyn_cast<CondBrInst>(I)) {
    if (isa<Constant>(BI->getCondition()))
      return processFoldableCondBr(BI);
 
    Value *BranchCond = BI->getCondition();
    BasicBlock *TrueSucc = BI->getSuccessor(0);
    BasicBlock *FalseSucc = BI->getSuccessor(1);
    // Avoid multiple edges early.
    if (TrueSucc == FalseSucc)
      return false;
 
    BasicBlock *Parent = BI->getParent();
    bool Changed = false;
 
    Value *TrueVal = ConstantInt::getTrue(TrueSucc->getContext());
    BasicBlockEdge TrueE(Parent, TrueSucc);
    Changed |= propagateEquality(BranchCond, TrueVal, TrueE);
 
    Value *FalseVal = ConstantInt::getFalse(FalseSucc->getContext());
    BasicBlockEdge FalseE(Parent, FalseSucc);
    Changed |= propagateEquality(BranchCond, FalseVal, FalseE);
 
    return Changed;
  }
 
  // For switches, propagate the case values into the case destinations.
  if (SwitchInst *SI = dyn_cast<SwitchInst>(I)) {
    Value *SwitchCond = SI->getCondition();
    BasicBlock *Parent = SI->getParent();
    bool Changed = false;
 
    // Remember how many outgoing edges there are to every successor.
    SmallDenseMap<BasicBlock *, unsigned, 16> SwitchEdges;
    for (BasicBlock *Succ : successors(Parent))
      ++SwitchEdges[Succ];
 
    for (const auto &Case : SI->cases()) {
      BasicBlock *Dst = Case.getCaseSuccessor();
      // If there is only a single edge, propagate the case value into it.
      if (SwitchEdges.lookup(Dst) == 1) {
        BasicBlockEdge E(Parent, Dst);
        Changed |= propagateEquality(SwitchCond, Case.getCaseValue(), E);
      }
    }
    return Changed;
  }
 
  // Instructions with void type don't return a value, so there's
  // no point in trying to find redundancies in them.
  if (I->getType()->isVoidTy())
    return false;
 
  uint32_t NextNum = VN.getNextUnusedValueNumber();
  unsigned Num = VN.lookupOrAdd(I);
 
  // Allocations are always uniquely numbered, so we can save time and memory
  // by fast failing them.
  if (isa<AllocaInst>(I) || I->isTerminator() || isa<PHINode>(I)) {
    LeaderTable.insert(Num, I, I->getParent());
    return false;
  }
 
  // A ptrtoaddr and a ptrtoint of the same pointer compute the same value when
  // the address width equals the pointer representation width.
  if (auto *PTA = dyn_cast<PtrToAddrInst>(I)) {
    const DataLayout &DL = I->getDataLayout();
    unsigned AS = PTA->getPointerAddressSpace();
    if (DL.getAddressSizeInBits(AS) == DL.getPointerSizeInBits(AS) &&
        !DL.hasUnstableRepresentation(AS)) {
      uint32_t PTINum =
          VN.lookupPtrToInt(PTA->getPointerOperand(), PTA->getType());
      if (Value *PTI = findLeader(I->getParent(), PTINum)) {
        patchAndReplaceAllUsesWith(I, PTI);
        salvageAndRemoveInstruction(I);
        return true;
      }
    }
  }
 
  // Perform fast-path value-number based elimination of values inherited from
  // dominators, unless the number we were assigned was a brand new VN, then
  // we don't need to do a lookup to see if the number already exists somewhere
  // in the domtree: it can't!
  Value *Repl = Num < NextNum ? findLeader(I->getParent(), Num) : nullptr;
  if (!Repl) {
    if (auto *Cmp = dyn_cast<CmpInst>(I); Cmp && replaceWithEquivalentCmp(Cmp))
      return true;
 
    // Failure, just remember this instance for future use.
    LeaderTable.insert(Num, I, I->getParent());
    return false;
  }
 
  if (Repl == I) {
    // If I was the result of a shortcut PRE, it might already be in the table
    // and the best replacement for itself. Nothing to do.
    return false;
  }
 
  // Remove it!
  patchAndReplaceAllUsesWith(I, Repl);
  if (MD && Repl->getType()->isPtrOrPtrVectorTy())
    MD->invalidateCachedPointerInfo(Repl);
  salvageAndRemoveInstruction(I);
  return true;
}
 
/// runOnFunction - This is the main transformation entry point for a function.
bool GVNPassImpl::run(Function &F, AssumptionCache &RunAC, DominatorTree &RunDT,
                      const TargetLibraryInfo &RunTLI, AAResults &RunAA,
                      MemoryDependenceResults *RunMD, LoopInfo &LI,
                      OptimizationRemarkEmitter *RunORE, MemorySSA *MSSA) {
  // MemDep and MemorySSA are mutually exclusive. isMemDepEnabled() silently
  // lets MemorySSA win for the common single-flag case, but an explicit
  // request for both via -enable-gvn-{memdep,memoryssa} is a contradiction we
  // reject rather than resolve arbitrarily.
  if (Opts->enable_gvn_memdep == BoolOrDefault::True &&
      Opts->enable_gvn_memoryssa)
    report_fatal_error("GVN: -enable-gvn-memdep and -enable-gvn-memoryssa are "
                       "mutually exclusive",
                       /*gen_crash_diag=*/false);
  AC = &RunAC;
  DT = &RunDT;
  VN.setDomTree(DT);
  TLI = &RunTLI;
  AA = &RunAA;
  VN.setAliasAnalysis(&RunAA);
  MD = RunMD;
  ImplicitControlFlowTracking ImplicitCFT;
  ICF = &ImplicitCFT;
  this->LI = &LI;
  VN.setMemDep(MD);
  // Propagate the MSSA-enabled flag so the value-numbering paths in
  // lookupOrAddCall() and computeLoadStoreVN(), which depends on whether
  // IsMSSAEnabled is turned on.
  VN.setMemorySSA(MSSA, isMemorySSAEnabled());
  ORE = RunORE;
  InvalidBlockRPONumbers = true;
  MemorySSAUpdater Updater(MSSA);
  MSSAU = MSSA ? &Updater : nullptr;
 
  bool Changed = false;
  bool ShouldContinue = true;
 
  DomTreeUpdater DTU(DT, DomTreeUpdater::UpdateStrategy::Lazy);
  // Merge unconditional branches, allowing PRE to catch more
  // optimization opportunities.
  for (BasicBlock &BB : make_early_inc_range(F)) {
    bool RemovedBlock = MergeBlockIntoPredecessor(&BB, &DTU, &LI, MSSAU, MD);
    if (RemovedBlock)
      ++NumGVNBlocks;
 
    Changed |= RemovedBlock;
  }
  DTU.flush();
 
  EarliestEscapeAnalysis RunEA(*DT, &LI);
  EA = MSSA ? &RunEA : nullptr;
 
  unsigned Iteration = 0;
  while (ShouldContinue) {
    LLVM_DEBUG(dbgs() << "GVN iteration: " << Iteration << "\n");
    (void) Iteration;
    ShouldContinue = iterateOnFunction(F);
    Changed |= ShouldContinue;
    ++Iteration;
  }
 
  if (isScalarPREEnabled()) {
    // Fabricate val-num for dead-code in order to suppress assertion in
    // performPRE().
    assignValNumForDeadCode();
    bool PREChanged = true;
    while (PREChanged) {
      PREChanged = performPRE(F);
      Changed |= PREChanged;
    }
  }
 
  // FIXME: Should perform GVN again after PRE does something.  PRE can move
  // computations into blocks where they become fully redundant.  Note that
  // we can't do this until PRE's critical edge splitting updates memdep.
  // Actually, when this happens, we should just fully integrate PRE into GVN.
 
  cleanupGlobalSets();
  // Do not cleanup DeadBlocks in cleanupGlobalSets() as it's called for each
  // iteration.
  DeadBlocks.clear();
 
  if (MSSA && VerifyMemorySSA)
    MSSA->verifyMemorySSA();
 
  return Changed;
}
 
bool GVNPassImpl::processBlock(BasicBlock *BB) {
  if (DeadBlocks.count(BB))
    return false;
 
  bool ChangedFunction = false;
 
  // Since we may not have visited the input blocks of the phis, we can't
  // use our normal hash approach for phis.  Instead, simply look for
  // obvious duplicates.  The first pass of GVN will tend to create
  // identical phis, and the second or later passes can eliminate them.
  SmallPtrSet<PHINode *, 8> PHINodesToRemove;
  ChangedFunction |= EliminateDuplicatePHINodes(BB, PHINodesToRemove);
  for (PHINode *PN : PHINodesToRemove) {
    removeInstruction(PN);
  }
  for (Instruction &Inst : make_early_inc_range(*BB))
    ChangedFunction |= processInstruction(&Inst);
  return ChangedFunction;
}
 
// Instantiate an expression in a predecessor that lacked it.
bool GVNPassImpl::performScalarPREInsertion(Instruction *Instr,
                                            BasicBlock *Pred, BasicBlock *Curr,
                                            unsigned int ValNo) {
  // Because we are going top-down through the block, all value numbers
  // will be available in the predecessor by the time we need them.  Any
  // that weren't originally present will have been instantiated earlier
  // in this loop.
  bool Success = true;
  for (unsigned I = 0, E = Instr->getNumOperands(); I != E; ++I) {
    Value *Op = Instr->getOperand(I);
    if (isa<Argument>(Op) || isa<Constant>(Op) || isa<GlobalValue>(Op))
      continue;
    // This could be a newly inserted instruction, in which case, we won't
    // find a value number, and should give up before we hurt ourselves.
    // FIXME: Rewrite the infrastructure to let it easier to value number
    // and process newly inserted instructions.
    if (!VN.exists(Op)) {
      Success = false;
      break;
    }
    uint32_t TValNo = VN.phiTranslate(Pred, Curr, VN.lookup(Op), LeaderTable);
    if (Value *V = findLeader(Pred, TValNo)) {
      Instr->setOperand(I, V);
    } else {
      Success = false;
      break;
    }
  }
 
  // Fail out if we encounter an operand that is not available in
  // the PRE predecessor.  This is typically because of loads which
  // are not value numbered precisely.
  if (!Success)
    return false;
 
  Instr->insertBefore(Pred->getTerminator()->getIterator());
  Instr->setName(Instr->getName() + ".pre");
  Instr->setDebugLoc(Instr->getDebugLoc());
 
  ICF->insertInstructionTo(Instr, Pred);
 
  unsigned Num = VN.lookupOrAdd(Instr);
  VN.add(Instr, Num);
 
  // Update the availability map to include the new instruction.
  LeaderTable.insert(Num, Instr, Pred);
  return true;
}
 
bool GVNPassImpl::performScalarPRE(Instruction *CurInst) {
  if (isa<AllocaInst>(CurInst) || CurInst->isTerminator() ||
      isa<PHINode>(CurInst) || CurInst->getType()->isVoidTy() ||
      CurInst->mayReadFromMemory() || CurInst->mayHaveSideEffects() ||
      CurInst->getType()->isTokenLikeTy())
    return false;
 
  // Don't do PRE on compares. The PHI would prevent CodeGenPrepare from
  // sinking the compare again, and it would force the code generator to
  // move the i1 from processor flags or predicate registers into a general
  // purpose register.
  if (isa<CmpInst>(CurInst))
    return false;
 
  // Don't do PRE on GEPs. The inserted PHI would prevent CodeGenPrepare from
  // sinking the addressing mode computation back to its uses. Extending the
  // GEP's live range increases the register pressure, and therefore it can
  // introduce unnecessary spills.
  //
  // This doesn't prevent Load PRE. PHI translation will make the GEP available
  // to the load by moving it to the predecessor block if necessary.
  if (isa<GetElementPtrInst>(CurInst))
    return false;
 
  if (auto *CallB = dyn_cast<CallBase>(CurInst)) {
    // We don't currently value number ANY inline asm calls.
    if (CallB->isInlineAsm())
      return false;
  }
 
  uint32_t ValNo = VN.lookup(CurInst);
 
  // Look for the predecessors for PRE opportunities.  We're
  // only trying to solve the basic diamond case, where
  // a value is computed in the successor and one predecessor,
  // but not the other.  We also explicitly disallow cases
  // where the successor is its own predecessor, because they're
  // more complicated to get right.
  unsigned NumWith = 0;
  unsigned NumWithout = 0;
  BasicBlock *PREPred = nullptr;
  BasicBlock *CurrentBlock = CurInst->getParent();
 
  // Update the RPO numbers for this function.
  if (InvalidBlockRPONumbers)
    assignBlockRPONumber(*CurrentBlock->getParent());
 
  SmallVector<std::pair<Value *, BasicBlock *>, 8> PredMap;
  for (BasicBlock *P : predecessors(CurrentBlock)) {
    // We're not interested in PRE where blocks with predecessors that are
    // not reachable.
    if (!DT->isReachableFromEntry(P)) {
      NumWithout = 2;
      break;
    }
    // It is not safe to do PRE when P->CurrentBlock is a loop backedge.
    assert(BlockRPONumber.count(P) && BlockRPONumber.count(CurrentBlock) &&
           "Invalid BlockRPONumber map.");
    if (BlockRPONumber[P] >= BlockRPONumber[CurrentBlock]) {
      NumWithout = 2;
      break;
    }
 
    uint32_t TValNo = VN.phiTranslate(P, CurrentBlock, ValNo, LeaderTable);
    Value *PredV = findLeader(P, TValNo);
    if (!PredV) {
      PredMap.push_back(std::make_pair(static_cast<Value *>(nullptr), P));
      PREPred = P;
      ++NumWithout;
    } else if (PredV == CurInst) {
      // CurInst dominates this predecessor.
      NumWithout = 2;
      break;
    } else {
      PredMap.push_back(std::make_pair(PredV, P));
      ++NumWith;
    }
  }
 
  // Don't do PRE when it might increase code size, i.e. when
  // we would need to insert instructions in more than one pred.
  if (NumWithout > 1 || NumWith == 0)
    return false;
 
  // We may have a case where all predecessors have the instruction,
  // and we just need to insert a phi node. Otherwise, perform
  // insertion.
  Instruction *PREInstr = nullptr;
 
  if (NumWithout != 0) {
    if (!isSafeToSpeculativelyExecute(CurInst)) {
      // It is only valid to insert a new instruction if the current instruction
      // is always executed. An instruction with implicit control flow could
      // prevent us from doing it. If we cannot speculate the execution, then
      // PRE should be prohibited.
      if (ICF->isDominatedByICFIFromSameBlock(CurInst))
        return false;
    }
 
    // Don't do PRE across indirect branch.
    if (isa<IndirectBrInst>(PREPred->getTerminator()))
      return false;
 
    // We can't do PRE safely on a critical edge, so instead we schedule
    // the edge to be split and perform the PRE the next time we iterate
    // on the function.
    unsigned SuccNum = GetSuccessorNumber(PREPred, CurrentBlock);
    if (isCriticalEdge(PREPred->getTerminator(), SuccNum)) {
      ToSplit.push_back(std::make_pair(PREPred->getTerminator(), SuccNum));
      return false;
    }
    // We need to insert somewhere, so let's give it a shot.
    PREInstr = CurInst->clone();
    if (!performScalarPREInsertion(PREInstr, PREPred, CurrentBlock, ValNo)) {
      // If we failed insertion, make sure we remove the instruction.
#ifndef NDEBUG
      verifyRemoved(PREInstr);
#endif
      PREInstr->deleteValue();
      return false;
    }
  }
 
  // Either we should have filled in the PRE instruction, or we should
  // not have needed insertions.
  assert(PREInstr != nullptr || NumWithout == 0);
 
  ++NumGVNPRE;
 
  // Create a PHI to make the value available in this block.
  PHINode *Phi = PHINode::Create(CurInst->getType(), PredMap.size(),
                                 CurInst->getName() + ".pre-phi");
  Phi->insertBefore(CurrentBlock->begin());
  for (auto &[V, BB] : PredMap) {
    if (V) {
      // If we use an existing value in this phi, we have to patch the original
      // value because the phi will be used to replace a later value.
      patchReplacementInstruction(CurInst, V);
      Phi->addIncoming(V, BB);
    } else
      Phi->addIncoming(PREInstr, PREPred);
  }
 
  VN.add(Phi, ValNo);
  // After creating a new PHI for ValNo, the phi translate result for ValNo will
  // be changed, so erase the related stale entries in phi translate cache.
  VN.eraseTranslateCacheEntry(ValNo, *CurrentBlock);
  LeaderTable.insert(ValNo, Phi, CurrentBlock);
  Phi->setDebugLoc(CurInst->getDebugLoc());
  CurInst->replaceAllUsesWith(Phi);
  if (MD && Phi->getType()->isPtrOrPtrVectorTy())
    MD->invalidateCachedPointerInfo(Phi);
  LeaderTable.erase(ValNo, CurInst, CurrentBlock);
 
  LLVM_DEBUG(dbgs() << "GVN PRE removed: " << *CurInst << '\n');
  removeInstruction(CurInst);
 
  return true;
}
 
/// Perform a purely local form of PRE that looks for diamond
/// control flow patterns and attempts to perform simple PRE at the join point.
bool GVNPassImpl::performPRE(Function &F) {
  bool Changed = false;
  for (BasicBlock *CurrentBlock : depth_first(&F.getEntryBlock())) {
    // Nothing to PRE in the entry block.
    if (CurrentBlock == &F.getEntryBlock())
      continue;
 
    // Don't perform PRE on an EH pad.
    if (CurrentBlock->isEHPad())
      continue;
 
    for (BasicBlock::iterator BI = CurrentBlock->begin(),
                              BE = CurrentBlock->end();
         BI != BE;) {
      Instruction *CurInst = &*BI++;
      Changed |= performScalarPRE(CurInst);
    }
  }
 
  if (splitCriticalEdges())
    Changed = true;
 
  return Changed;
}
 
/// Split the critical edge connecting the given two blocks, and return
/// the block inserted to the critical edge.
BasicBlock *GVNPassImpl::splitCriticalEdges(BasicBlock *Pred,
                                            BasicBlock *Succ) {
  // GVN does not require loop-simplify, do not try to preserve it if it is not
  // possible.
  BasicBlock *BB = SplitCriticalEdge(
      Pred, Succ,
      CriticalEdgeSplittingOptions(DT, LI, MSSAU).unsetPreserveLoopSimplify());
  if (BB) {
    if (MD)
      MD->invalidateCachedPredecessors();
    InvalidBlockRPONumbers = true;
  }
  return BB;
}
 
/// Split critical edges found during the previous
/// iteration that may enable further optimization.
bool GVNPassImpl::splitCriticalEdges() {
  if (ToSplit.empty())
    return false;
 
  bool Changed = false;
  do {
    std::pair<Instruction *, unsigned> Edge = ToSplit.pop_back_val();
    Changed |= SplitCriticalEdge(Edge.first, Edge.second,
                                 CriticalEdgeSplittingOptions(DT, LI, MSSAU)) !=
               nullptr;
  } while (!ToSplit.empty());
  if (Changed) {
    if (MD)
      MD->invalidateCachedPredecessors();
    InvalidBlockRPONumbers = true;
  }
  return Changed;
}
 
/// Executes one iteration of GVN.
bool GVNPassImpl::iterateOnFunction(Function &F) {
  cleanupGlobalSets();
 
  // Top-down walk of the dominator tree.
  bool Changed = false;
  // Needed for value numbering with phi construction to work.
  // RPOT walks the graph in its constructor and will not be invalidated during
  // processBlock.
  ReversePostOrderTraversal<Function *> RPOT(&F);
 
  for (BasicBlock *BB : RPOT)
    Changed |= processBlock(BB);
 
  return Changed;
}
 
void GVNPassImpl::cleanupGlobalSets() {
  VN.clear();
  LeaderTable.clear();
  BlockRPONumber.clear();
  ICF->clear();
  InvalidBlockRPONumbers = true;
}
 
void GVNPassImpl::removeInstruction(Instruction *I) {
  VN.erase(I);
  if (MD) MD->removeInstruction(I);
  if (EA)
    EA->removeInstruction(I);
  if (MSSAU)
    MSSAU->removeMemoryAccess(I);
#ifndef NDEBUG
  verifyRemoved(I);
#endif
  ICF->removeInstruction(I);
  I->eraseFromParent();
  ++NumGVNInstr;
}
 
/// Verify that the specified instruction does not occur in our
/// internal data structures.
void GVNPassImpl::verifyRemoved(const Instruction *Inst) const {
  VN.verifyRemoved(Inst);
}
 
/// BB is declared dead, which implied other blocks become dead as well. This
/// function is to add all these blocks to "DeadBlocks". For the dead blocks'
/// live successors, update their phi nodes by replacing the operands
/// corresponding to dead blocks with UndefVal.
void GVNPassImpl::addDeadBlock(BasicBlock *BB) {
  SmallVector<BasicBlock *, 4> NewDead;
  SmallSetVector<BasicBlock *, 4> DF;
 
  NewDead.push_back(BB);
  while (!NewDead.empty()) {
    BasicBlock *D = NewDead.pop_back_val();
    if (DeadBlocks.count(D))
      continue;
 
    // All blocks dominated by D are dead.
    SmallVector<BasicBlock *, 8> Dom;
    DT->getDescendants(D, Dom);
    DeadBlocks.insert_range(Dom);
 
    // Figure out the dominance-frontier(D).
    for (BasicBlock *B : Dom) {
      for (BasicBlock *S : successors(B)) {
        if (DeadBlocks.count(S))
          continue;
 
        bool AllPredDead = true;
        for (BasicBlock *P : predecessors(S))
          if (!DeadBlocks.count(P)) {
            AllPredDead = false;
            break;
          }
 
        if (!AllPredDead) {
          // S could be proved dead later on. That is why we don't update phi
          // operands at this moment.
          DF.insert(S);
        } else {
          // While S is not dominated by D, it is dead by now. This could take
          // place if S already have a dead predecessor before D is declared
          // dead.
          NewDead.push_back(S);
        }
      }
    }
  }
 
  // For the dead blocks' live successors, update their phi nodes by replacing
  // the operands corresponding to dead blocks with UndefVal.
  for (BasicBlock *B : DF) {
    if (DeadBlocks.count(B))
      continue;
 
    // First, split the critical edges. This might also create additional blocks
    // to preserve LoopSimplify form and adjust edges accordingly.
    SmallVector<BasicBlock *, 4> Preds(predecessors(B));
    for (BasicBlock *P : Preds) {
      if (!DeadBlocks.count(P))
        continue;
 
      if (is_contained(successors(P), B) &&
          isCriticalEdge(P->getTerminator(), B)) {
        if (BasicBlock *S = splitCriticalEdges(P, B))
          DeadBlocks.insert(P = S);
      }
    }
 
    // Now poison the incoming values from the dead predecessors.
    for (BasicBlock *P : predecessors(B)) {
      if (!DeadBlocks.count(P))
        continue;
      for (PHINode &Phi : B->phis()) {
        Phi.setIncomingValueForBlock(P, PoisonValue::get(Phi.getType()));
        if (MD)
          MD->invalidateCachedPointerInfo(&Phi);
      }
    }
  }
}
 
// If the given branch is recognized as a foldable branch (i.e. conditional
// branch with constant condition), it will perform following analyses and
// transformation.
//  1) If the dead out-coming edge is a critical-edge, split it. Let
//     R be the target of the dead out-coming edge.
//  1) Identify the set of dead blocks implied by the branch's dead outcoming
//     edge. The result of this step will be {X| X is dominated by R}
//  2) Identify those blocks which haves at least one dead predecessor. The
//     result of this step will be dominance-frontier(R).
//  3) Update the PHIs in DF(R) by replacing the operands corresponding to
//     dead blocks with "UndefVal" in an hope these PHIs will optimized away.
//
// Return true iff *NEW* dead code are found.
bool GVNPassImpl::processFoldableCondBr(CondBrInst *BI) {
  // If a branch has two identical successors, we cannot declare either dead.
  if (BI->getSuccessor(0) == BI->getSuccessor(1))
    return false;
 
  ConstantInt *Cond = dyn_cast<ConstantInt>(BI->getCondition());
  if (!Cond)
    return false;
 
  BasicBlock *DeadRoot =
      Cond->getZExtValue() ? BI->getSuccessor(1) : BI->getSuccessor(0);
  if (DeadBlocks.count(DeadRoot))
    return false;
 
  if (!DeadRoot->getSinglePredecessor())
    DeadRoot = splitCriticalEdges(BI->getParent(), DeadRoot);
 
  addDeadBlock(DeadRoot);
  return true;
}
 
// performPRE() will trigger assert if it comes across an instruction without
// associated val-num. As it normally has far more live instructions than dead
// instructions, it makes more sense just to "fabricate" a val-number for the
// dead code than checking if instruction involved is dead or not.
void GVNPassImpl::assignValNumForDeadCode() {
  for (BasicBlock *BB : DeadBlocks) {
    for (Instruction &Inst : *BB) {
      unsigned ValNum = VN.lookupOrAdd(&Inst);
      LeaderTable.insert(ValNum, &Inst, BB);
    }
  }
}
 
class GVNLegacyPass : public FunctionPass {
public:
  static char ID; // Pass identification, replacement for typeid.
 
  // Always uses MemDep, ignoring -enable-gvn-memdep and -enable-gvn-memoryssa.
  explicit GVNLegacyPass(bool ScalarPRE = true)
      : FunctionPass(ID),
        Impl(GVNOptions().setMemDep(true).setMemorySSA(false).setScalarPRE(
            ScalarPRE)) {
    initializeGVNLegacyPassPass(*PassRegistry::getPassRegistry());
  }
 
  bool runOnFunction(Function &F) override {
    if (skipFunction(F))
      return false;
 
    Impl.Opts = &getScalarOptions(F);
    auto *MSSAWP = getAnalysisIfAvailable<MemorySSAWrapperPass>();
    return Impl.run(
        F, getAnalysis<AssumptionCacheTracker>().getAssumptionCache(F),
        getAnalysis<DominatorTreeWrapperPass>().getDomTree(),
        getAnalysis<TargetLibraryInfoWrapperPass>().getTLI(F),
        getAnalysis<AAResultsWrapperPass>().getAAResults(),
        &getAnalysis<MemoryDependenceWrapperPass>().getMemDep(),
        getAnalysis<LoopInfoWrapperPass>().getLoopInfo(),
        &getAnalysis<OptimizationRemarkEmitterWrapperPass>().getORE(),
        MSSAWP ? &MSSAWP->getMSSA() : nullptr);
  }
 
  void getAnalysisUsage(AnalysisUsage &AU) const override {
    AU.addRequired<AssumptionCacheTracker>();
    AU.addRequired<DominatorTreeWrapperPass>();
    AU.addRequired<TargetLibraryInfoWrapperPass>();
    AU.addRequired<LoopInfoWrapperPass>();
    AU.addRequired<MemoryDependenceWrapperPass>();
    AU.addRequired<AAResultsWrapperPass>();
    AU.addPreserved<DominatorTreeWrapperPass>();
    AU.addPreserved<GlobalsAAWrapperPass>();
    AU.addPreserved<TargetLibraryInfoWrapperPass>();
    AU.addPreserved<LoopInfoWrapperPass>();
    AU.addRequired<OptimizationRemarkEmitterWrapperPass>();
    AU.addPreserved<MemorySSAWrapperPass>();
  }
 
private:
  GVNPassImpl Impl;
};
 
char GVNLegacyPass::ID = 0;
 
INITIALIZE_PASS_BEGIN(GVNLegacyPass, "gvn", "Global Value Numbering", false, false)
INITIALIZE_PASS_DEPENDENCY(AssumptionCacheTracker)
INITIALIZE_PASS_DEPENDENCY(MemoryDependenceWrapperPass)
INITIALIZE_PASS_DEPENDENCY(DominatorTreeWrapperPass)
INITIALIZE_PASS_DEPENDENCY(TargetLibraryInfoWrapperPass)
INITIALIZE_PASS_DEPENDENCY(AAResultsWrapperPass)
INITIALIZE_PASS_DEPENDENCY(GlobalsAAWrapperPass)
INITIALIZE_PASS_DEPENDENCY(OptimizationRemarkEmitterWrapperPass)
INITIALIZE_PASS_END(GVNLegacyPass, "gvn", "Global Value Numbering", false, false)
 
// The public interface to this file...
FunctionPass *llvm::createGVNPass() { return new GVNLegacyPass(); }
FunctionPass *llvm::createGVNPass(bool ScalarPRE) {
  return new GVNLegacyPass(ScalarPRE);
}