LLVM 20.0.0git
Macros | Functions | Variables
RewriteStatepointsForGC.cpp File Reference
#include "llvm/Transforms/Scalar/RewriteStatepointsForGC.h"
#include "llvm/ADT/ArrayRef.h"
#include "llvm/ADT/DenseMap.h"
#include "llvm/ADT/DenseSet.h"
#include "llvm/ADT/MapVector.h"
#include "llvm/ADT/STLExtras.h"
#include "llvm/ADT/Sequence.h"
#include "llvm/ADT/SetVector.h"
#include "llvm/ADT/SmallSet.h"
#include "llvm/ADT/SmallVector.h"
#include "llvm/ADT/StringRef.h"
#include "llvm/ADT/iterator_range.h"
#include "llvm/Analysis/DomTreeUpdater.h"
#include "llvm/Analysis/TargetLibraryInfo.h"
#include "llvm/Analysis/TargetTransformInfo.h"
#include "llvm/IR/Argument.h"
#include "llvm/IR/AttributeMask.h"
#include "llvm/IR/Attributes.h"
#include "llvm/IR/BasicBlock.h"
#include "llvm/IR/CallingConv.h"
#include "llvm/IR/Constant.h"
#include "llvm/IR/Constants.h"
#include "llvm/IR/DataLayout.h"
#include "llvm/IR/DerivedTypes.h"
#include "llvm/IR/Dominators.h"
#include "llvm/IR/Function.h"
#include "llvm/IR/GCStrategy.h"
#include "llvm/IR/IRBuilder.h"
#include "llvm/IR/InstIterator.h"
#include "llvm/IR/InstrTypes.h"
#include "llvm/IR/Instruction.h"
#include "llvm/IR/Instructions.h"
#include "llvm/IR/IntrinsicInst.h"
#include "llvm/IR/Intrinsics.h"
#include "llvm/IR/LLVMContext.h"
#include "llvm/IR/MDBuilder.h"
#include "llvm/IR/Metadata.h"
#include "llvm/IR/Module.h"
#include "llvm/IR/Statepoint.h"
#include "llvm/IR/Type.h"
#include "llvm/IR/User.h"
#include "llvm/IR/Value.h"
#include "llvm/IR/ValueHandle.h"
#include "llvm/Support/Casting.h"
#include "llvm/Support/CommandLine.h"
#include "llvm/Support/Compiler.h"
#include "llvm/Support/Debug.h"
#include "llvm/Support/ErrorHandling.h"
#include "llvm/Support/raw_ostream.h"
#include "llvm/Transforms/Utils/BasicBlockUtils.h"
#include "llvm/Transforms/Utils/Local.h"
#include "llvm/Transforms/Utils/PromoteMemToReg.h"
#include <algorithm>
#include <cassert>
#include <cstddef>
#include <cstdint>
#include <iterator>
#include <optional>
#include <set>
#include <string>
#include <utility>
#include <vector>

Go to the source code of this file.

Macros

#define DEBUG_TYPE   "rewrite-statepoints-for-gc"
 

Functions

static void stripNonValidData (Module &M)
 The IR fed into RewriteStatepointsForGC may have had attributes and metadata implying dereferenceability that are no longer valid/correct after RewriteStatepointsForGC has run.
 
static std::unique_ptr< GCStrategyfindGCStrategy (Function &F)
 Looks up the GC strategy for a given function, returning null if the function doesn't have a GC tag.
 
static bool shouldRewriteStatepointsIn (Function &F)
 Returns true if this function should be rewritten by this pass.
 
static ArrayRef< UseGetDeoptBundleOperands (const CallBase *Call)
 
static void computeLiveInValues (DominatorTree &DT, Function &F, GCPtrLivenessData &Data, GCStrategy *GC)
 Compute the live-in set for every basic block in the function.
 
static void findLiveSetAtInst (Instruction *inst, GCPtrLivenessData &Data, StatepointLiveSetTy &out, GCStrategy *GC)
 Given results from the dataflow liveness computation, find the set of live Values at a particular instruction.
 
static bool isGCPointerType (Type *T, GCStrategy *GC)
 
static bool isHandledGCPointerType (Type *T, GCStrategy *GC)
 
static bool containsGCPtrType (Type *Ty, GCStrategy *GC)
 Returns true if this type contains a gc pointer whether we know how to handle that type or not.
 
static bool isUnhandledGCPointerType (Type *Ty, GCStrategy *GC)
 
static std::string suffixed_name_or (Value *V, StringRef Suffix, StringRef DefaultName)
 
static void analyzeParsePointLiveness (DominatorTree &DT, GCPtrLivenessData &OriginalLivenessData, CallBase *Call, PartiallyConstructedSafepointRecord &Result, GCStrategy *GC)
 
static bool isKnownBase (Value *V, const IsKnownBaseMapTy &KnownBases)
 Returns true if V is a known base.
 
static void setKnownBase (Value *V, bool IsKnownBase, IsKnownBaseMapTy &KnownBases)
 Caches the IsKnownBase flag for a value and asserts that it wasn't present in the cache before.
 
static ValuefindBaseDefiningValue (Value *I, DefiningValueMapTy &Cache, IsKnownBaseMapTy &KnownBases)
 Helper function for findBasePointer - Will return a value which either a) defines the base pointer for the input, b) blocks the simple search (i.e.
 
static ValuefindBaseDefiningValueOfVector (Value *I, DefiningValueMapTy &Cache, IsKnownBaseMapTy &KnownBases)
 Return a base defining value for the 'Index' element of the given vector instruction 'I'.
 
static ValuefindBaseDefiningValueCached (Value *I, DefiningValueMapTy &Cache, IsKnownBaseMapTy &KnownBases)
 Returns the base defining value for this value.
 
static ValuefindBaseOrBDV (Value *I, DefiningValueMapTy &Cache, IsKnownBaseMapTy &KnownBases)
 Return a base pointer for this value if known.
 
static bool isOriginalBaseResult (Value *V)
 This value is a base pointer that is not generated by RS4GC, i.e.
 
static bool areBothVectorOrScalar (Value *First, Value *Second)
 
static raw_ostreamoperator<< (raw_ostream &OS, const BDVState &State)
 
static ValuefindBasePointer (Value *I, DefiningValueMapTy &Cache, IsKnownBaseMapTy &KnownBases)
 For a given value or instruction, figure out what base ptr its derived from.
 
static void findBasePointers (const StatepointLiveSetTy &live, PointerToBaseTy &PointerToBase, DominatorTree *DT, DefiningValueMapTy &DVCache, IsKnownBaseMapTy &KnownBases)
 
static void findBasePointers (DominatorTree &DT, DefiningValueMapTy &DVCache, CallBase *Call, PartiallyConstructedSafepointRecord &result, PointerToBaseTy &PointerToBase, IsKnownBaseMapTy &KnownBases)
 Find the required based pointers (and adjust the live set) for the given parse point.
 
static void recomputeLiveInValues (GCPtrLivenessData &RevisedLivenessData, CallBase *Call, PartiallyConstructedSafepointRecord &result, PointerToBaseTy &PointerToBase, GCStrategy *GC)
 Given an updated version of the dataflow liveness results, update the liveset and base pointer maps for the call site CS.
 
static void recomputeLiveInValues (Function &F, DominatorTree &DT, ArrayRef< CallBase * > toUpdate, MutableArrayRef< struct PartiallyConstructedSafepointRecord > records, PointerToBaseTy &PointerToBase, GCStrategy *GC)
 
static InstructionrematerializeChain (ArrayRef< Instruction * > ChainToBase, Instruction *InsertBefore, Value *RootOfChain, Value *AlternateLiveBase)
 
static BasicBlocknormalizeForInvokeSafepoint (BasicBlock *BB, BasicBlock *InvokeParent, DominatorTree &DT)
 
static AttributeList legalizeCallAttributes (CallBase *Call, bool IsMemIntrinsic, AttributeList StatepointAL)
 
static void CreateGCRelocates (ArrayRef< Value * > LiveVariables, ArrayRef< Value * > BasePtrs, Instruction *StatepointToken, IRBuilder<> &Builder, GCStrategy *GC)
 Helper function to place all gc relocates necessary for the given statepoint.
 
static StringRef getDeoptLowering (CallBase *Call)
 
static void makeStatepointExplicitImpl (CallBase *Call, const SmallVectorImpl< Value * > &BasePtrs, const SmallVectorImpl< Value * > &LiveVariables, PartiallyConstructedSafepointRecord &Result, std::vector< DeferredReplacement > &Replacements, const PointerToBaseTy &PointerToBase, GCStrategy *GC)
 
static void makeStatepointExplicit (DominatorTree &DT, CallBase *Call, PartiallyConstructedSafepointRecord &Result, std::vector< DeferredReplacement > &Replacements, const PointerToBaseTy &PointerToBase, GCStrategy *GC)
 
static void insertRelocationStores (iterator_range< Value::user_iterator > GCRelocs, DenseMap< Value *, AllocaInst * > &AllocaMap, DenseSet< Value * > &VisitedLiveValues)
 
static void insertRematerializationStores (const RematerializedValueMapTy &RematerializedValues, DenseMap< Value *, AllocaInst * > &AllocaMap, DenseSet< Value * > &VisitedLiveValues)
 
static void relocationViaAlloca (Function &F, DominatorTree &DT, ArrayRef< Value * > Live, ArrayRef< PartiallyConstructedSafepointRecord > Records)
 Do all the relocation update via allocas and mem2reg.
 
template<typename T >
static void unique_unsorted (SmallVectorImpl< T > &Vec)
 Implement a unique function which doesn't require we sort the input vector.
 
static void insertUseHolderAfter (CallBase *Call, const ArrayRef< Value * > Values, SmallVectorImpl< CallInst * > &Holders)
 Insert holders so that each Value is obviously live through the entire lifetime of the call.
 
static void findLiveReferences (Function &F, DominatorTree &DT, ArrayRef< CallBase * > toUpdate, MutableArrayRef< struct PartiallyConstructedSafepointRecord > records, GCStrategy *GC)
 
static ValuefindRematerializableChainToBasePointer (SmallVectorImpl< Instruction * > &ChainToBase, Value *CurrentValue)
 
static InstructionCost chainToBasePointerCost (SmallVectorImpl< Instruction * > &Chain, TargetTransformInfo &TTI)
 
static bool AreEquivalentPhiNodes (PHINode &OrigRootPhi, PHINode &AlternateRootPhi)
 
static void findRematerializationCandidates (PointerToBaseTy PointerToBase, RematCandTy &RematerizationCandidates, TargetTransformInfo &TTI)
 
static void rematerializeLiveValuesAtUses (RematCandTy &RematerizationCandidates, MutableArrayRef< PartiallyConstructedSafepointRecord > Records, PointerToBaseTy &PointerToBase)
 
static void rematerializeLiveValues (CallBase *Call, PartiallyConstructedSafepointRecord &Info, PointerToBaseTy &PointerToBase, RematCandTy &RematerizationCandidates, TargetTransformInfo &TTI)
 
static bool inlineGetBaseAndOffset (Function &F, SmallVectorImpl< CallInst * > &Intrinsics, DefiningValueMapTy &DVCache, IsKnownBaseMapTy &KnownBases)
 
static bool insertParsePoints (Function &F, DominatorTree &DT, TargetTransformInfo &TTI, SmallVectorImpl< CallBase * > &ToUpdate, DefiningValueMapTy &DVCache, IsKnownBaseMapTy &KnownBases)
 
static AttributeMask getParamAndReturnAttributesToRemove ()
 
static void stripNonValidAttributesFromPrototype (Function &F)
 
static void stripInvalidMetadataFromInstruction (Instruction &I)
 Certain metadata on instructions are invalid after running RS4GC.
 
static void stripNonValidDataFromBody (Function &F)
 
static void computeLiveInValues (BasicBlock::reverse_iterator Begin, BasicBlock::reverse_iterator End, SetVector< Value * > &LiveTmp, GCStrategy *GC)
 Compute the live-in set for the location rbegin starting from the live-out set of the basic block.
 
static void computeLiveOutSeed (BasicBlock *BB, SetVector< Value * > &LiveTmp, GCStrategy *GC)
 
static SetVector< Value * > computeKillSet (BasicBlock *BB, GCStrategy *GC)
 
static void checkBasicSSA (DominatorTree &DT, SetVector< Value * > &Live, Instruction *TI, bool TermOkay=false)
 Check that the items in 'Live' dominate 'TI'.
 
static void checkBasicSSA (DominatorTree &DT, GCPtrLivenessData &Data, BasicBlock &BB)
 Check that all the liveness sets used during the computation of liveness obey basic SSA properties.
 

Variables

static cl::opt< boolPrintLiveSet ("spp-print-liveset", cl::Hidden, cl::init(false))
 
static cl::opt< boolPrintLiveSetSize ("spp-print-liveset-size", cl::Hidden, cl::init(false))
 
static cl::opt< boolPrintBasePointers ("spp-print-base-pointers", cl::Hidden, cl::init(false))
 
static cl::opt< unsignedRematerializationThreshold ("spp-rematerialization-threshold", cl::Hidden, cl::init(6))
 
static bool ClobberNonLive = false
 
static cl::opt< bool, trueClobberNonLiveOverride ("rs4gc-clobber-non-live", cl::location(ClobberNonLive), cl::Hidden)
 
static cl::opt< boolAllowStatepointWithNoDeoptInfo ("rs4gc-allow-statepoint-with-no-deopt-info", cl::Hidden, cl::init(true))
 
static cl::opt< boolRematDerivedAtUses ("rs4gc-remat-derived-at-uses", cl::Hidden, cl::init(true))
 
static constexpr Attribute::AttrKind FnAttrsToStrip []
 

Macro Definition Documentation

◆ DEBUG_TYPE

#define DEBUG_TYPE   "rewrite-statepoints-for-gc"

Definition at line 78 of file RewriteStatepointsForGC.cpp.

Function Documentation

◆ analyzeParsePointLiveness()

static void analyzeParsePointLiveness ( DominatorTree DT,
GCPtrLivenessData &  OriginalLivenessData,
CallBase Call,
PartiallyConstructedSafepointRecord &  Result,
GCStrategy GC 
)
static

◆ areBothVectorOrScalar()

static bool areBothVectorOrScalar ( Value First,
Value Second 
)
static

Definition at line 682 of file RewriteStatepointsForGC.cpp.

References llvm::First, and llvm::Value::getType().

Referenced by findBasePointer().

◆ AreEquivalentPhiNodes()

static bool AreEquivalentPhiNodes ( PHINode OrigRootPhi,
PHINode AlternateRootPhi 
)
static

◆ chainToBasePointerCost()

static InstructionCost chainToBasePointerCost ( SmallVectorImpl< Instruction * > &  Chain,
TargetTransformInfo TTI 
)
static

◆ checkBasicSSA() [1/2]

static void checkBasicSSA ( DominatorTree DT,
GCPtrLivenessData &  Data,
BasicBlock BB 
)
static

Check that all the liveness sets used during the computation of liveness obey basic SSA properties.

This is useful for finding cases where we miss a def.

Definition at line 3268 of file RewriteStatepointsForGC.cpp.

References checkBasicSSA(), llvm::Data, and llvm::BasicBlock::getTerminator().

◆ checkBasicSSA() [2/2]

static void checkBasicSSA ( DominatorTree DT,
SetVector< Value * > &  Live,
Instruction TI,
bool  TermOkay = false 
)
static

Check that the items in 'Live' dominate 'TI'.

This is used as a basic validation check for the liveness computation.

Definition at line 3250 of file RewriteStatepointsForGC.cpp.

References assert(), llvm::DominatorTree::dominates(), I, and Live.

Referenced by checkBasicSSA(), and computeLiveInValues().

◆ computeKillSet()

static SetVector< Value * > computeKillSet ( BasicBlock BB,
GCStrategy GC 
)
static

◆ computeLiveInValues() [1/2]

static void computeLiveInValues ( BasicBlock::reverse_iterator  Begin,
BasicBlock::reverse_iterator  End,
SetVector< Value * > &  LiveTmp,
GCStrategy GC 
)
static

Compute the live-in set for the location rbegin starting from the live-out set of the basic block.

Definition at line 3189 of file RewriteStatepointsForGC.cpp.

References assert(), End, I, llvm::SetVector< T, Vector, Set, N >::insert(), isHandledGCPointerType(), isUnhandledGCPointerType(), llvm::make_range(), and llvm::SetVector< T, Vector, Set, N >::remove().

◆ computeLiveInValues() [2/2]

static void computeLiveInValues ( DominatorTree DT,
Function F,
GCPtrLivenessData &  Data,
GCStrategy GC 
)
static

◆ computeLiveOutSeed()

static void computeLiveOutSeed ( BasicBlock BB,
SetVector< Value * > &  LiveTmp,
GCStrategy GC 
)
static

◆ containsGCPtrType()

static bool containsGCPtrType ( Type Ty,
GCStrategy GC 
)
static

Returns true if this type contains a gc pointer whether we know how to handle that type or not.

Definition at line 283 of file RewriteStatepointsForGC.cpp.

References llvm::any_of(), containsGCPtrType(), and isGCPointerType().

◆ CreateGCRelocates()

static void CreateGCRelocates ( ArrayRef< Value * >  LiveVariables,
ArrayRef< Value * >  BasePtrs,
Instruction StatepointToken,
IRBuilder<> &  Builder,
GCStrategy GC 
)
static

Helper function to place all gc relocates necessary for the given statepoint.

Inputs: liveVariables - list of variables to be relocated. basePtrs - base pointers. statepointToken - statepoint instruction to which relocates should be bound. Builder - Llvm IR builder to be used to construct new calls.

Definition at line 1498 of file RewriteStatepointsForGC.cpp.

References assert(), llvm::ArrayRef< T >::begin(), llvm::CallingConv::Cold, llvm::DenseMapBase< DerivedT, KeyT, ValueT, KeyInfoT, BucketT >::count(), llvm::IRBuilderBase::CreateCall(), llvm::ArrayRef< T >::end(), llvm::find(), llvm::PointerType::get(), llvm::FixedVectorType::get(), llvm::Intrinsic::getDeclaration(), llvm::IRBuilderBase::getInt32(), llvm::Instruction::getModule(), getNumElements(), isHandledGCPointerType(), llvm::CallBase::setCallingConv(), llvm::ArrayRef< T >::size(), and suffixed_name_or().

Referenced by makeStatepointExplicitImpl().

◆ findBaseDefiningValue()

static Value * findBaseDefiningValue ( Value I,
DefiningValueMapTy &  Cache,
IsKnownBaseMapTy &  KnownBases 
)
static

Helper function for findBasePointer - Will return a value which either a) defines the base pointer for the input, b) blocks the simple search (i.e.

a PHI or Select of two derived pointers), or c) involves a change from pointer to vector type or back.

Definition at line 451 of file RewriteStatepointsForGC.cpp.

References assert(), findBaseDefiningValue(), findBaseDefiningValueOfVector(), GEP, llvm::ConstantPointerNull::get(), I, II, llvm_unreachable, and setKnownBase().

Referenced by findBaseDefiningValue(), findBaseDefiningValueCached(), and findBaseDefiningValueOfVector().

◆ findBaseDefiningValueCached()

static Value * findBaseDefiningValueCached ( Value I,
DefiningValueMapTy &  Cache,
IsKnownBaseMapTy &  KnownBases 
)
static

Returns the base defining value for this value.

Definition at line 625 of file RewriteStatepointsForGC.cpp.

References assert(), llvm::dbgs(), findBaseDefiningValue(), I, and LLVM_DEBUG.

Referenced by findBaseOrBDV().

◆ findBaseDefiningValueOfVector()

static Value * findBaseDefiningValueOfVector ( Value I,
DefiningValueMapTy &  Cache,
IsKnownBaseMapTy &  KnownBases 
)
static

Return a base defining value for the 'Index' element of the given vector instruction 'I'.

If Index is null, returns a BDV for the entire vector 'I'. As an optimization, this method will try to determine when the element is known to already be a base pointer. If this can be established, the second value in the returned pair will be true. Note that either a vector or a pointer typed value can be returned. For the former, the vector returned is a BDV (and possibly a base) of the entire vector 'I'. If the later, the return pointer is a BDV (or possibly a base) for the particular element in 'I'.

Definition at line 353 of file RewriteStatepointsForGC.cpp.

References assert(), findBaseDefiningValue(), GEP, llvm::ConstantAggregateZero::get(), I, and setKnownBase().

Referenced by findBaseDefiningValue().

◆ findBaseOrBDV()

static Value * findBaseOrBDV ( Value I,
DefiningValueMapTy &  Cache,
IsKnownBaseMapTy &  KnownBases 
)
static

Return a base pointer for this value if known.

Otherwise, return it's base defining value.

Definition at line 642 of file RewriteStatepointsForGC.cpp.

References findBaseDefiningValueCached(), and I.

Referenced by findBasePointer().

◆ findBasePointer()

static Value * findBasePointer ( Value I,
DefiningValueMapTy &  Cache,
IsKnownBaseMapTy &  KnownBases 
)
static

◆ findBasePointers() [1/2]

static void findBasePointers ( const StatepointLiveSetTy &  live,
PointerToBaseTy &  PointerToBase,
DominatorTree DT,
DefiningValueMapTy &  DVCache,
IsKnownBaseMapTy &  KnownBases 
)
static

◆ findBasePointers() [2/2]

static void findBasePointers ( DominatorTree DT,
DefiningValueMapTy &  DVCache,
CallBase Call,
PartiallyConstructedSafepointRecord &  result,
PointerToBaseTy &  PointerToBase,
IsKnownBaseMapTy &  KnownBases 
)
static

Find the required based pointers (and adjust the live set) for the given parse point.

Definition at line 1319 of file RewriteStatepointsForGC.cpp.

References findBasePointers(), and llvm::LLVMContext::OB_deopt.

◆ findGCStrategy()

static std::unique_ptr< GCStrategy > findGCStrategy ( Function F)
static

Looks up the GC strategy for a given function, returning null if the function doesn't have a GC tag.

The strategy is stored in the cache.

Definition at line 2996 of file RewriteStatepointsForGC.cpp.

References F, and llvm::getGCStrategy().

Referenced by insertParsePoints(), and shouldRewriteStatepointsIn().

◆ findLiveReferences()

static void findLiveReferences ( Function F,
DominatorTree DT,
ArrayRef< CallBase * >  toUpdate,
MutableArrayRef< struct PartiallyConstructedSafepointRecord >  records,
GCStrategy GC 
)
static

◆ findLiveSetAtInst()

static void findLiveSetAtInst ( Instruction inst,
GCPtrLivenessData &  Data,
StatepointLiveSetTy &  out,
GCStrategy GC 
)
static

◆ findRematerializableChainToBasePointer()

static Value * findRematerializableChainToBasePointer ( SmallVectorImpl< Instruction * > &  ChainToBase,
Value CurrentValue 
)
static

◆ findRematerializationCandidates()

static void findRematerializationCandidates ( PointerToBaseTy  PointerToBase,
RematCandTy &  RematerizationCandidates,
TargetTransformInfo TTI 
)
static

◆ GetDeoptBundleOperands()

static ArrayRef< Use > GetDeoptBundleOperands ( const CallBase Call)
static

◆ getDeoptLowering()

static StringRef getDeoptLowering ( CallBase Call)
static

◆ getParamAndReturnAttributesToRemove()

static AttributeMask getParamAndReturnAttributesToRemove ( )
static

◆ inlineGetBaseAndOffset()

static bool inlineGetBaseAndOffset ( Function F,
SmallVectorImpl< CallInst * > &  Intrinsics,
DefiningValueMapTy &  DVCache,
IsKnownBaseMapTy &  KnownBases 
)
static

◆ insertParsePoints()

static bool insertParsePoints ( Function F,
DominatorTree DT,
TargetTransformInfo TTI,
SmallVectorImpl< CallBase * > &  ToUpdate,
DefiningValueMapTy &  DVCache,
IsKnownBaseMapTy &  KnownBases 
)
static

◆ insertRelocationStores()

static void insertRelocationStores ( iterator_range< Value::user_iterator GCRelocs,
DenseMap< Value *, AllocaInst * > &  AllocaMap,
DenseSet< Value * > &  VisitedLiveValues 
)
static

◆ insertRematerializationStores()

static void insertRematerializationStores ( const RematerializedValueMapTy &  RematerializedValues,
DenseMap< Value *, AllocaInst * > &  AllocaMap,
DenseSet< Value * > &  VisitedLiveValues 
)
static

◆ insertUseHolderAfter()

static void insertUseHolderAfter ( CallBase Call,
const ArrayRef< Value * >  Values,
SmallVectorImpl< CallInst * > &  Holders 
)
static

Insert holders so that each Value is obviously live through the entire lifetime of the call.

Definition at line 2221 of file RewriteStatepointsForGC.cpp.

References llvm::CallInst::Create(), llvm::ArrayRef< T >::empty(), llvm::FunctionType::get(), llvm::Type::getVoidTy(), II, and llvm::SmallVectorTemplateBase< T, bool >::push_back().

Referenced by insertParsePoints().

◆ isGCPointerType()

static bool isGCPointerType ( Type T,
GCStrategy GC 
)
static

Definition at line 254 of file RewriteStatepointsForGC.cpp.

References assert().

◆ isHandledGCPointerType()

static bool isHandledGCPointerType ( Type T,
GCStrategy GC 
)
static

◆ isKnownBase()

static bool isKnownBase ( Value V,
const IsKnownBaseMapTy &  KnownBases 
)
static

Returns true if V is a known base.

Definition at line 665 of file RewriteStatepointsForGC.cpp.

References assert().

Referenced by findBasePointer().

◆ isOriginalBaseResult()

static bool isOriginalBaseResult ( Value V)
static

This value is a base pointer that is not generated by RS4GC, i.e.

it already exists in the code.

Definition at line 657 of file RewriteStatepointsForGC.cpp.

Referenced by findBasePointer().

◆ isUnhandledGCPointerType()

static bool isUnhandledGCPointerType ( Type Ty,
GCStrategy GC 
)
static

◆ legalizeCallAttributes()

static AttributeList legalizeCallAttributes ( CallBase Call,
bool  IsMemIntrinsic,
AttributeList  StatepointAL 
)
static

◆ makeStatepointExplicit()

static void makeStatepointExplicit ( DominatorTree DT,
CallBase Call,
PartiallyConstructedSafepointRecord &  Result,
std::vector< DeferredReplacement > &  Replacements,
const PointerToBaseTy &  PointerToBase,
GCStrategy GC 
)
static

◆ makeStatepointExplicitImpl()

static void makeStatepointExplicitImpl ( CallBase Call,
const SmallVectorImpl< Value * > &  BasePtrs,
const SmallVectorImpl< Value * > &  LiveVariables,
PartiallyConstructedSafepointRecord &  Result,
std::vector< DeferredReplacement > &  Replacements,
const PointerToBaseTy &  PointerToBase,
GCStrategy GC 
)
static

Definition at line 1642 of file RewriteStatepointsForGC.cpp.

References assert(), llvm::sampleprof::Base, llvm::BasicBlock::begin(), llvm::SmallVectorImpl< T >::clear(), CreateGCRelocates(), llvm::IRBuilderBase::CreateGCResult(), llvm::IRBuilderBase::CreateGCStatepointCall(), llvm::IRBuilderBase::CreateGCStatepointInvoke(), llvm::IRBuilderBase::CreatePtrToInt(), llvm::IRBuilderBase::CreateSub(), llvm::StatepointDirectives::DefaultStatepointID, llvm::DeoptLiveIn, DL, F, llvm::GCTransition, llvm::AttributeList::get(), llvm::ConstantPointerNull::get(), llvm::FunctionType::get(), llvm::CallBase::getAttributes(), llvm::FunctionCallee::getCallee(), llvm::Value::getContext(), getDeoptLowering(), llvm::BasicBlock::getFirstInsertionPt(), llvm::Type::getIntNTy(), llvm::BasicBlock::getLandingPadInst(), llvm::BasicBlock::getUniquePredecessor(), llvm::Type::getVoidTy(), II, legalizeCallAttributes(), llvm_unreachable, Name, llvm::None, llvm::StatepointDirectives::NumPatchBytes, llvm::LLVMContext::OB_deopt, llvm::LLVMContext::OB_gc_transition, llvm::parseStatepointDirectivesFromAttrs(), llvm::SmallVectorTemplateBase< T, bool >::push_back(), llvm::AttributeList::ReturnIndex, llvm::CallBase::setAttributes(), llvm::CallBase::setCallingConv(), llvm::IRBuilderBase::SetCurrentDebugLocation(), llvm::IRBuilderBase::SetInsertPoint(), llvm::Value::setName(), llvm::CallInst::setTailCallKind(), llvm::SmallVectorBase< Size_T >::size(), and llvm::StatepointDirectives::StatepointID.

Referenced by makeStatepointExplicit().

◆ normalizeForInvokeSafepoint()

static BasicBlock * normalizeForInvokeSafepoint ( BasicBlock BB,
BasicBlock InvokeParent,
DominatorTree DT 
)
static

◆ operator<<()

static raw_ostream & operator<< ( raw_ostream OS,
const BDVState &  State 
)
static

Definition at line 796 of file RewriteStatepointsForGC.cpp.

References OS.

◆ recomputeLiveInValues() [1/2]

static void recomputeLiveInValues ( Function F,
DominatorTree DT,
ArrayRef< CallBase * >  toUpdate,
MutableArrayRef< struct PartiallyConstructedSafepointRecord >  records,
PointerToBaseTy &  PointerToBase,
GCStrategy GC 
)
static

◆ recomputeLiveInValues() [2/2]

static void recomputeLiveInValues ( GCPtrLivenessData &  RevisedLivenessData,
CallBase Call,
PartiallyConstructedSafepointRecord &  result,
PointerToBaseTy &  PointerToBase,
GCStrategy GC 
)
static

Given an updated version of the dataflow liveness results, update the liveset and base pointer maps for the call site CS.

Definition at line 3361 of file RewriteStatepointsForGC.cpp.

References findLiveSetAtInst(), and Info.

Referenced by insertParsePoints(), and recomputeLiveInValues().

◆ relocationViaAlloca()

static void relocationViaAlloca ( Function F,
DominatorTree DT,
ArrayRef< Value * >  Live,
ArrayRef< PartiallyConstructedSafepointRecord >  Records 
)
static

◆ rematerializeChain()

static Instruction * rematerializeChain ( ArrayRef< Instruction * >  ChainToBase,
Instruction InsertBefore,
Value RootOfChain,
Value AlternateLiveBase 
)
static

◆ rematerializeLiveValues()

static void rematerializeLiveValues ( CallBase Call,
PartiallyConstructedSafepointRecord &  Info,
PointerToBaseTy &  PointerToBase,
RematCandTy &  RematerizationCandidates,
TargetTransformInfo TTI 
)
static

◆ rematerializeLiveValuesAtUses()

static void rematerializeLiveValuesAtUses ( RematCandTy &  RematerizationCandidates,
MutableArrayRef< PartiallyConstructedSafepointRecord >  Records,
PointerToBaseTy &  PointerToBase 
)
static

◆ setKnownBase()

static void setKnownBase ( Value V,
bool  IsKnownBase,
IsKnownBaseMapTy &  KnownBases 
)
static

Caches the IsKnownBase flag for a value and asserts that it wasn't present in the cache before.

Definition at line 671 of file RewriteStatepointsForGC.cpp.

References assert().

Referenced by findBaseDefiningValue(), findBaseDefiningValueOfVector(), and findBasePointer().

◆ shouldRewriteStatepointsIn()

static bool shouldRewriteStatepointsIn ( Function F)
static

Returns true if this function should be rewritten by this pass.

The main point of this function is as an extension point for custom logic.

Definition at line 3005 of file RewriteStatepointsForGC.cpp.

References assert(), F, and findGCStrategy().

Referenced by llvm::RewriteStatepointsForGC::run(), llvm::RewriteStatepointsForGC::runOnFunction(), and stripNonValidData().

◆ stripInvalidMetadataFromInstruction()

static void stripInvalidMetadataFromInstruction ( Instruction I)
static

Certain metadata on instructions are invalid after running RS4GC.

Optimizations that run after RS4GC can incorrectly use this metadata to optimize functions. We drop such metadata on the instruction.

Definition at line 2918 of file RewriteStatepointsForGC.cpp.

References I.

Referenced by stripNonValidDataFromBody().

◆ stripNonValidAttributesFromPrototype()

static void stripNonValidAttributesFromPrototype ( Function F)
static

◆ stripNonValidData()

static void stripNonValidData ( Module M)
static

The IR fed into RewriteStatepointsForGC may have had attributes and metadata implying dereferenceability that are no longer valid/correct after RewriteStatepointsForGC has run.

This is because semantically, after RewriteStatepointsForGC runs, all calls to gc.statepoint "free" the entire heap. stripNonValidData (conservatively) restores correctness by erasing all attributes in the module that externally imply dereferenceability. Similar reasoning also applies to the noalias attributes and metadata. gc.statepoint can touch the entire heap including noalias objects. Apart from attributes and metadata, we also remove instructions that imply constant physical memory: llvm.invariant.start.

Definition at line 3016 of file RewriteStatepointsForGC.cpp.

References llvm::any_of(), assert(), F, shouldRewriteStatepointsIn(), stripNonValidAttributesFromPrototype(), and stripNonValidDataFromBody().

Referenced by llvm::RewriteStatepointsForGC::run().

◆ stripNonValidDataFromBody()

static void stripNonValidDataFromBody ( Function F)
static

◆ suffixed_name_or()

static std::string suffixed_name_or ( Value V,
StringRef  Suffix,
StringRef  DefaultName 
)
static

◆ unique_unsorted()

template<typename T >
static void unique_unsorted ( SmallVectorImpl< T > &  Vec)
static

Implement a unique function which doesn't require we sort the input vector.

Doing so has the effect of changing the output of a couple of tests in ways which make them less useful in testing fused safepoints.

Definition at line 2214 of file RewriteStatepointsForGC.cpp.

References llvm::erase_if(), and llvm::SmallSet< T, N, C >::insert().

Referenced by insertParsePoints().

Variable Documentation

◆ AllowStatepointWithNoDeoptInfo

cl::opt< bool > AllowStatepointWithNoDeoptInfo("rs4gc-allow-statepoint-with-no-deopt-info", cl::Hidden, cl::init(true)) ( "rs4gc-allow-statepoint-with-no-deopt-info"  ,
cl::Hidden  ,
cl::init(true  
)
static

◆ ClobberNonLive

bool ClobberNonLive = false
static

Definition at line 101 of file RewriteStatepointsForGC.cpp.

Referenced by relocationViaAlloca().

◆ ClobberNonLiveOverride

cl::opt< bool, true > ClobberNonLiveOverride("rs4gc-clobber-non-live", cl::location(ClobberNonLive), cl::Hidden) ( "rs4gc-clobber-non-live"  ,
cl::location(ClobberNonLive ,
cl::Hidden   
)
static

◆ FnAttrsToStrip

constexpr Attribute::AttrKind FnAttrsToStrip[]
staticconstexpr
Initial value:
=
{Attribute::Memory, Attribute::NoSync, Attribute::NoFree}

Definition at line 1448 of file RewriteStatepointsForGC.cpp.

Referenced by legalizeCallAttributes(), and stripNonValidAttributesFromPrototype().

◆ PrintBasePointers

cl::opt< bool > PrintBasePointers("spp-print-base-pointers", cl::Hidden, cl::init(false)) ( "spp-print-base-pointers"  ,
cl::Hidden  ,
cl::init(false)   
)
static

Referenced by insertParsePoints().

◆ PrintLiveSet

cl::opt< bool > PrintLiveSet("spp-print-liveset", cl::Hidden, cl::init(false)) ( "spp-print-liveset"  ,
cl::Hidden  ,
cl::init(false)   
)
static

◆ PrintLiveSetSize

cl::opt< bool > PrintLiveSetSize("spp-print-liveset-size", cl::Hidden, cl::init(false)) ( "spp-print-liveset-size"  ,
cl::Hidden  ,
cl::init(false)   
)
static

◆ RematDerivedAtUses

cl::opt< bool > RematDerivedAtUses("rs4gc-remat-derived-at-uses", cl::Hidden, cl::init(true)) ( "rs4gc-remat-derived-at-uses"  ,
cl::Hidden  ,
cl::init(true  
)
static

◆ RematerializationThreshold

cl::opt< unsigned > RematerializationThreshold("spp-rematerialization-threshold", cl::Hidden, cl::init(6)) ( "spp-rematerialization-threshold"  ,
cl::Hidden  ,
cl::init(6)   
)
static