56 #define DEBUG_TYPE "statepoint-lowering" 58 STATISTIC(NumSlotsAllocatedForStatepoints,
59 "Number of stack slots allocated for statepoints");
60 STATISTIC(NumOfStatepoints,
"Number of statepoint nodes encountered");
62 "Maximum number of stack slots required for a singe statepoint");
74 assert(PendingGCRelocateCalls.empty() &&
75 "Trying to visit statepoint before finished processing previous one");
77 NextSlotToAllocate = 0;
81 AllocatedStackSlots.
clear();
87 AllocatedStackSlots.
clear();
88 assert(PendingGCRelocateCalls.empty() &&
89 "cleared before statepoint sequence completed");
95 NumSlotsAllocatedForStatepoints++;
105 const size_t NumSlots = AllocatedStackSlots.
size();
106 assert(NextSlotToAllocate <= NumSlots &&
"Broken invariant");
112 for (; NextSlotToAllocate < NumSlots; NextSlotToAllocate++) {
113 if (!AllocatedStackSlots.
test(NextSlotToAllocate)) {
116 AllocatedStackSlots.
set(NextSlotToAllocate);
126 const unsigned FI = cast<FrameIndexSDNode>(SpillSlot)->getIndex();
130 AllocatedStackSlots.
resize(AllocatedStackSlots.
size()+1,
true);
135 StatepointMaxSlotsRequired.updateMax(
148 if (LookUpDepth <= 0)
152 if (
const auto *Relocate = dyn_cast<GCRelocateInst>(Val)) {
153 const auto &SpillMap =
156 auto It = SpillMap.find(Relocate->getDerivedPtr());
157 if (It == SpillMap.end())
164 if (
const BitCastInst *Cast = dyn_cast<BitCastInst>(Val))
170 if (
const PHINode *Phi = dyn_cast<PHINode>(Val)) {
173 for (
auto &IncomingValue : Phi->incoming_values()) {
179 if (MergedResult.
hasValue() && *MergedResult != *SpillSlot)
182 MergedResult = SpillSlot;
227 if (isa<ConstantSDNode>(Incoming) || isa<FrameIndexSDNode>(Incoming)) {
238 const int LookUpDepth = 6;
246 auto SlotIt =
find(StatepointSlots, *Index);
247 assert(SlotIt != StatepointSlots.end() &&
248 "Value spilled to the unknown stack slot");
251 const int Offset = std::distance(StatepointSlots.begin(), SlotIt);
285 for (
size_t i = 0,
e = Ptrs.
size(); i <
e; i++) {
287 auto SeenIt = Seen.
find(SD);
289 if (SeenIt == Seen.
end()) {
316 SDValue ReturnValue, CallEndVal;
317 std::tie(ReturnValue, CallEndVal) =
358 MFI.getObjectAlignment(FI.
getIndex()));
367 static std::tuple<SDValue, SDValue, MachineMemOperand*>
377 int Index = cast<FrameIndexSDNode>(Loc)->getIndex();
388 "Bad spill: stack slot does not match!");
408 return std::make_tuple(Loc, Chain, MMO);
436 "Incoming value is a frame index!");
444 }
else if (LiveInOnly) {
461 if (
auto *MMO = std::get<2>(Res))
463 Chain = std::get<1>(Res);;
483 if (
auto *GFI = Builder.
GFI) {
492 if (Opt.hasValue()) {
494 "non gc managed base pointer found in statepoint");
499 if (Opt.hasValue()) {
501 "non gc managed derived pointer found in statepoint");
506 assert(SI.
Bases.empty() &&
"No gc specified, so cannot relocate pointers!");
507 assert(SI.
Ptrs.empty() &&
"No gc specified, so cannot relocate pointers!");
521 const bool LiveInDeopt =
524 auto isGCValue =[&](
const Value *V) {
534 if (!LiveInDeopt || isGCValue(V))
537 for (
unsigned i = 0; i < SI.
Bases.size(); ++i) {
561 const bool LiveInValue = LiveInDeopt && !isGCValue(V);
570 for (
unsigned i = 0; i < SI.
Bases.size(); ++i) {
573 Ops, MemRefs, Builder);
577 Ops, MemRefs, Builder);
590 "Incoming value is a frame index!");
612 SpillMap.SlotMap[V] = cast<FrameIndexSDNode>(Loc)->getIndex();
620 SpillMap.SlotMap[V] =
None;
642 StatepointLowering.startNewStatepoint(*
this);
649 StatepointLowering.scheduleRelocCall(*Reloc);
673 std::tie(ReturnVal, CallNode) =
684 if (CallHasIncomingGlue) {
696 const bool IsGCTransition =
699 if (IsGCTransition) {
713 if (CallHasIncomingGlue)
721 Chain = GCTransitionStart.
getValue(0);
722 Glue = GCTransitionStart.
getValue(1);
737 unsigned NumCallRegArgs =
748 if (CallHasIncomingGlue)
749 RegMaskIt = CallNode->
op_end() - 2;
751 RegMaskIt = CallNode->
op_end() - 1;
760 "Unknown flag used");
764 Ops.
insert(Ops.
end(), LoweredMetaArgs.begin(), LoweredMetaArgs.end());
781 DAG.getMachineNode(TargetOpcode::STATEPOINT, getCurSDLoc(), NodeTys, Ops);
782 DAG.setNodeMemRefs(StatepointMCNode, MemRefs);
784 SDNode *SinkNode = StatepointMCNode;
790 if (IsGCTransition) {
811 SinkNode = GCTransitionStart.
getNode();
815 DAG.ReplaceAllUsesWith(CallNode, SinkNode);
817 DAG.DeleteNode(CallNode);
835 "anyregcc is not supported on statepoints!");
844 assert(GFI->getStrategy().useStatepoints() &&
845 "GCStrategy does not expect to encounter statepoints");
857 const auto &TLI = DAG.getTargetLoweringInfo();
858 const auto &DL = DAG.getDataLayout();
861 ActualCallee = DAG.getConstant(0, getCurSDLoc(), TLI.getPointerTy(DL, AS));
867 populateCallLoweringInfo(SI.CLI, ISP.
getCall(),
873 SI.GCRelocates.push_back(Relocate);
874 SI.Bases.push_back(Relocate->getBasePtr());
875 SI.Ptrs.push_back(Relocate->getDerivedPtr());
880 SI.GCTransitionArgs =
884 SI.StatepointFlags = ISP.
getFlags();
886 SI.EHPadBB = EHPadBB;
888 SDValue ReturnValue = LowerAsSTATEPOINT(SI);
893 if (!RetTy->
isVoidTy() && GCResult) {
903 unsigned Reg = FuncInfo.CreateRegs(RetTy);
904 RegsForValue RFV(*DAG.getContext(), DAG.getTargetLoweringInfo(),
905 DAG.getDataLayout(),
Reg, RetTy,
907 SDValue Chain = DAG.getEntryNode();
909 RFV.getCopyToRegs(ReturnValue, DAG, getCurSDLoc(), Chain,
nullptr);
910 PendingExports.push_back(Chain);
921 setValue(ISP.
getInstruction(), DAG.getIntPtrConstant(-1, getCurSDLoc()));
927 bool VarArgDisallowed,
bool ForceVoidReturnTy) {
930 populateCallLoweringInfo(
934 if (!VarArgDisallowed)
942 SI.
ID = SD.StatepointID.getValueOr(DefaultID);
952 if (
SDValue ReturnVal = LowerAsSTATEPOINT(SI)) {
953 ReturnVal = lowerRangeToAssertZExt(DAG, *Call, ReturnVal);
954 setValue(Call, ReturnVal);
960 LowerCallSiteWithDeoptBundleImpl(Call, Callee, EHPadBB,
965 void SelectionDAGBuilder::visitGCResult(
const GCResultInst &CI) {
980 cast<FunctionType>(CalleeType->
getElementType())->getReturnType();
984 setValue(&CI, CopyFromReg);
986 setValue(&CI, getValue(I));
990 void SelectionDAGBuilder::visitGCRelocate(
const GCRelocateInst &Relocate) {
997 StatepointLowering.relocCallVisited(Relocate);
1000 if (
auto IsManaged = GFI->getStrategy().isGCManagedPointer(Ty))
1001 assert(*IsManaged &&
"Non gc managed pointer relocated!");
1005 SDValue SD = getValue(DerivedPtr);
1007 auto &SpillMap = FuncInfo.StatepointSpillMaps[Relocate.
getStatepoint()];
1008 auto SlotIt = SpillMap.find(DerivedPtr);
1009 assert(SlotIt != SpillMap.end() &&
"Relocating not lowered gc value");
1014 if (!DerivedPtrLocation) {
1015 setValue(&Relocate, SD);
1019 unsigned Index = *DerivedPtrLocation;
1020 SDValue SpillSlot = DAG.getTargetFrameIndex(Index, getFrameIndexTy());
1027 auto &MF = DAG.getMachineFunction();
1028 auto &MFI = MF.getFrameInfo();
1032 MFI.getObjectSize(Index),
1033 MFI.getObjectAlignment(Index));
1035 auto LoadVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),
1038 SDValue SpillLoad = DAG.getLoad(LoadVT, getCurSDLoc(), Chain,
1039 SpillSlot, LoadMMO);
1041 DAG.setRoot(SpillLoad.
getValue(1));
1044 setValue(&Relocate, SpillLoad);
1048 const auto &TLI = DAG.getTargetLoweringInfo();
1049 SDValue Callee = DAG.getExternalSymbol(TLI.getLibcallName(RTLIB::DEOPTIMIZE),
1050 TLI.getPointerTy(DAG.getDataLayout()));
1055 LowerCallSiteWithDeoptBundleImpl(CI, Callee,
nullptr,
1064 if (DAG.getTarget().Options.TrapUnreachable)
SDValue getStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr, MachinePointerInfo PtrInfo, unsigned Alignment=0, MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes())
Helper function to build ISD::STORE nodes.
Represents calls to the gc.result intrinsic.
EVT getValueType() const
Return the ValueType of the referenced return value.
unsigned getOpcode() const
Return the SelectionDAG opcode value for this node.
This class represents an incoming formal argument to a Function.
This class represents lattice values for constants.
void ExportFromCurrentBlock(const Value *V)
ExportFromCurrentBlock - If this condition isn't known to be exported from the current basic block...
void LowerDeoptimizeCall(const CallInst *CI)
TargetLowering::CallLoweringInfo CLI
Information regarding the underlying call instruction.
StatepointDirectives parseStatepointDirectivesFromAttrs(AttributeList AS)
Parse out statepoint directives from the function attributes present in AS.
SmallVector< const GCRelocateInst *, 16 > GCRelocates
The set of gc.relocate calls associated with this gc.statepoint.
SelectionDAGBuilder - This is the common target-independent lowering implementation that is parameter...
A specialization of it's base class for read only access to a gc.statepoint.
void push_back(const T &Elt)
void LowerCallSiteWithDeoptBundle(const CallBase *Call, SDValue Callee, const BasicBlock *EHPadBB)
Type * getActualReturnType() const
Return the type of the value returned by the call underlying the statepoint.
This class represents a function call, abstracting a target machine's calling convention.
void LowerDeoptimizingReturn()
static std::pair< SDValue, SDNode * > lowerCallFromStatepointLoweringInfo(SelectionDAGBuilder::StatepointLoweringInfo &SI, SelectionDAGBuilder &Builder, SmallVectorImpl< SDValue > &PendingExports)
Extract call from statepoint, lower it and return pointer to the call node.
void resize(unsigned N, bool t=false)
Grow or shrink the bitvector.
STATISTIC(NumFunctions, "Total number of functions")
Base class for all callable instructions (InvokeInst and CallInst) Holds everything related to callin...
unsigned getPointerAddressSpace() const
Get the address space of this pointer or pointer vector type.
SDNode * getNode() const
get the SDNode which holds the desired result
const SDValue & setRoot(SDValue N)
Set the current root tag of the SelectionDAG.
CallBaseTy * getCall() const
Return the underlying call instruction.
unsigned getValueSizeInBits() const
Returns the size of the value in bits.
static void reservePreviousStackSlotForValue(const Value *IncomingValue, SelectionDAGBuilder &Builder)
Try to find existing copies of the incoming values in stack slots used for statepoint spilling...
Keep track of frame indices allocated for statepoints as they could be used across basic block bounda...
const CallBase * getStatepoint() const
The statepoint with which this gc.relocate is associated.
SDValue getValue(const Value *V)
getValue - Return an SDValue for the given Value.
bool test(unsigned Idx) const
void verify()
Asserts if this statepoint is malformed.
unsigned ID
The ID that the resulting STATEPOINT instruction has to report.
SDValue getRoot()
Return the current virtual root of the Selection DAG, flushing any PendingLoad items.
InstructionTy * getInstruction() const
A description of a memory reference used in the backend.
uint64_t StatepointFlags
Flags associated with the meta arguments being lowered.
Value * getDerivedPtr() const
ArrayRef< const Use > GCTransitionArgs
The list of gc transition arguments present in the gc.statepoint being lowered.
CallLoweringInfo & setChain(SDValue InChain)
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
op_iterator op_end() const
unsigned getStoreSize() const
Return the number of bytes overwritten by a store of the specified value type.
The MachineFrameInfo class represents an abstract stack frame until prolog/epilog code is inserted...
static void removeDuplicateGCPtrs(SmallVectorImpl< const Value *> &Bases, SmallVectorImpl< const Value *> &Ptrs, SmallVectorImpl< const GCRelocateInst *> &Relocs, SelectionDAGBuilder &Builder, FunctionLoweringInfo::StatepointSpillMap &SSM)
Remove any duplicate (as SDValues) from the derived pointer pairs.
const GCResultInst * getGCResult() const
Get the experimental_gc_result call tied to this statepoint.
arg_iterator gc_args_begin() const
MachineMemOperand * getMachineMemOperand(MachinePointerInfo PtrInfo, MachineMemOperand::Flags f, uint64_t s, unsigned base_alignment, const AAMDNodes &AAInfo=AAMDNodes(), const MDNode *Ranges=nullptr, SyncScope::ID SSID=SyncScope::System, AtomicOrdering Ordering=AtomicOrdering::NotAtomic, AtomicOrdering FailureOrdering=AtomicOrdering::NotAtomic)
getMachineMemOperand - Allocate a new MachineMemOperand.
SDValue LowerAsSTATEPOINT(StatepointLoweringInfo &SI)
Lower SLI into a STATEPOINT instruction.
This represents a list of ValueType's that has been intern'd by a SelectionDAG.
int getArgumentFrameIndex(const Argument *A)
getArgumentFrameIndex - Get frame index for the byval argument.
unsigned getSizeInBits() const
Return the size of the specified value type in bits.
Type * getType() const
All values are typed, get the type of this value.
MachineFunction & getMachineFunction() const
SDValue getTargetFrameIndex(int FI, EVT VT)
virtual Optional< bool > isGCManagedPointer(const Type *Ty) const
If the type specified can be reliably distinguished, returns true for pointers to GC managed location...
This class represents a no-op cast from one type to another.
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory)...
op_iterator op_begin() const
SDValue getTargetConstant(uint64_t Val, const SDLoc &DL, EVT VT, bool isOpaque=false)
int64_t getObjectSize(int ObjectIdx) const
Return the size of the specified object.
Class to represent pointers.
Optional< OperandBundleUse > getOperandBundle(StringRef Name) const
Return an operand bundle by name, if present.
static const uint64_t DeoptBundleStatepointID
Type * getScalarType() const
If this is a vector type, return the element type, otherwise return 'this'.
iterator find(const_arg_type_t< KeyT > Val)
bool isVoidTy() const
Return true if this is 'void'.
The memory access is volatile.
uint64_t getID() const
Return the ID associated with this statepoint.
unsigned getObjectAlignment(int ObjectIdx) const
Return the alignment of the specified stack object.
arg_iterator deopt_end() const
void reserveStackSlot(int Offset)
std::pair< SDValue, SDValue > lowerInvokable(TargetLowering::CallLoweringInfo &CLI, const BasicBlock *EHPadBB=nullptr)
const Instruction * StatepointInstr
The gc.statepoint instruction.
GC_TRANSITION_START/GC_TRANSITION_END - These operators mark the beginning and end of GC transition s...
ArrayRef< const Use > DeoptState
The deoptimization state associated with this gc.statepoint call, if any.
static std::tuple< SDValue, SDValue, MachineMemOperand * > spillIncomingStatepointValue(SDValue Incoming, SDValue Chain, SelectionDAGBuilder &Builder)
Spill a value incoming to the statepoint.
LLVM Basic Block Representation.
The instances of the Type class are immutable: once they are created, they are never changed...
MachineFrameInfo & getFrameInfo()
getFrameInfo - Return the frame info object for the current function.
static void pushStackMapConstant(SmallVectorImpl< SDValue > &Ops, SelectionDAGBuilder &Builder, uint64_t Value)
const SDValue & getOperand(unsigned Num) const
bool isStackSlotAllocated(int Offset)
bool isPointerTy() const
True if this is an instance of PointerType.
FunctionType * getFunctionType() const
void LowerCallSiteWithDeoptBundleImpl(const CallBase *Call, SDValue Callee, const BasicBlock *EHPadBB, bool VarArgDisallowed, bool ForceVoidReturnTy)
SDValue getLocation(SDValue Val)
Returns the spill location of a value incoming to the current statepoint.
SDValue allocateStackSlot(EVT ValueType, SelectionDAGBuilder &Builder)
Get a stack slot we can use to store an value of type ValueType.
static Type * getVoidTy(LLVMContext &C)
void clear()
Clear the memory usage of this object.
TRAP - Trapping instruction.
amdgpu Simplify well known AMD library false FunctionCallee Value * Arg
SDNode * getGluedNode() const
If this node has a glue operand, return the node to which the glue operand points.
int getNumCallArgs() const
Number of arguments to be passed to the actual callee.
static Optional< int > findPreviousSpillSlot(const Value *Val, SelectionDAGBuilder &Builder, int LookUpDepth)
Utility function for reservePreviousStackSlotForValue.
DenseMap< const Instruction *, StatepointSpillMap > StatepointSpillMaps
Maps gc.statepoint instructions to their corresponding StatepointSpillMap instances.
auto find(R &&Range, const T &Val) -> decltype(adl_begin(Range))
Provide wrappers to std::find which take ranges instead of having to pass begin/end explicitly...
unsigned getNumOperands() const
Return the number of values used by this operation.
SDLoc getCurSDLoc() const
void setLocation(SDValue Val, SDValue Location)
SmallVector< unsigned, 50 > StatepointStackSlots
StatepointStackSlots - A list of temporary stack slots (frame indices) used to spill values at a stat...
SDValue CreateStackTemporary(EVT VT, unsigned minAlign=1)
Create a stack temporary, suitable for holding the specified value type.
void markAsStatepointSpillSlotObjectIndex(int ObjectIdx)
The memory access writes data.
void clear()
Clear all bits.
arg_iterator gc_args_end() const
This struct represents the registers (physical or virtual) that a particular set of values is assigne...
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small...
DenseMap< const Value *, const Value * > DuplicateMap
Maps llvm IR values to the values they were de-duplicated to.
An SDNode that represents everything that will be needed to construct a MachineInstr.
Indicates that this statepoint is a transition from GC-aware code to code that is not GC-aware...
Wrapper class for IR location info (IR ordering and DebugLoc) to be passed into SDNode creation funct...
size_t size() const
Returns the number of bits in this bitvector.
Represents one node in the SelectionDAG.
AttributeList getAttributes() const
Return the parameter attributes for this call.
GCFunctionInfo * GFI
Garbage collection metadata for the function.
amdgpu Simplify well known AMD library false FunctionCallee Callee
static void lowerIncomingStatepointValue(SDValue Incoming, bool LiveInOnly, SmallVectorImpl< SDValue > &Ops, SmallVectorImpl< MachineMemOperand *> &MemRefs, SelectionDAGBuilder &Builder)
Lower a single value incoming to a statepoint node.
iterator insert(iterator I, T &&Elt)
The memory access reads data.
static MachinePointerInfo getFixedStack(MachineFunction &MF, int FI, int64_t Offset=0)
Return a MachinePointerInfo record that refers to the specified FrameIndex.
User::op_iterator arg_begin()
Return the iterator pointing to the beginning of the argument list.
Represents a use of a SDNode.
void LowerStatepoint(ImmutableStatepoint ISP, const BasicBlock *EHPadBB=nullptr)
GCStrategy describes a garbage collector algorithm's code generation requirements, and provides overridable hooks for those needs which cannot be abstractly described.
unsigned getNumArgOperands() const
uint32_t getNumPatchBytes() const
Return the number of patchable bytes associated with this statepoint.
CallingConv::ID getCallingConv() const
LOAD and STORE have token chains as their first operand, then the same operands as an LLVM load/store...
void startNewStatepoint(SelectionDAGBuilder &Builder)
Reset all state tracking for a newly encountered safepoint.
SmallVector< const Value *, 16 > Bases
Bases[i] is the base pointer for Ptrs[i].
SDValue getValue(unsigned R) const
StatepointLoweringState StatepointLowering
State used while lowering a statepoint sequence (gc_statepoint, gc_relocate, and gc_result).
CopyFromReg - This node indicates that the input value is a virtual or physical register that is defi...
std::vector< const GCRelocateInst * > getRelocates() const
Get list of all gc reloactes linked to this statepoint May contain several relocations for the same b...
unsigned NumPatchBytes
The number of patchable bytes the call needs to get lowered into.
uint64_t getFlags() const
ValueTy * getCalledValue() const
Return the value actually being called or invoked.
FunctionLoweringInfo & FuncInfo
Information about the function as a whole.
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
const BasicBlock * EHPadBB
The exception handling unwind destination, in case this represents an invoke of gc.statepoint.
SDValue getFrameIndex(int FI, EVT VT, bool isTarget=false)
Represents calls to the gc.relocate intrinsic.
Mark the deopt arguments associated with the statepoint as only being "live-in".
LLVM Value Representation.
static void lowerStatepointMetaArgs(SmallVectorImpl< SDValue > &Ops, SmallVectorImpl< MachineMemOperand *> &MemRefs, SelectionDAGBuilder::StatepointLoweringInfo &SI, SelectionDAGBuilder &Builder)
Lower deopt state and gc pointer arguments of the statepoint.
static MachineMemOperand * getMachineMemOperand(MachineFunction &MF, FrameIndexSDNode &FI)
ArrayRef< const Use > GCArgs
The full list of gc arguments to the gc.statepoint being lowered.
MVT getFrameIndexTy()
Returns the type of FrameIndex and TargetFrameIndex nodes.
Unlike LLVM values, Selection DAG nodes may return multiple values as the result of a computation...
A bitmask that includes all valid flags.
SmallVector< const Value *, 16 > Ptrs
Type * getElementType() const
arg_iterator deopt_begin() const
Describes a gc.statepoint or a gc.statepoint like thing for the purposes of lowering into a STATEPOIN...
This file describes how to lower LLVM code to machine code.
const BasicBlock * getParent() const
A discriminated union of two or more pointer types, with the discriminator in the low bit of the poin...
bool is_contained(R &&Range, const E &Element)
Wrapper function around std::find to detect if an element exists in a container.