#include "StatepointLowering.h"
#include "SelectionDAGBuilder.h"
#include "SelectionDAGOptions.h"
#include "llvm/ADT/ArrayRef.h"
#include "llvm/ADT/STLExtras.h"
#include "llvm/ADT/SetVector.h"
#include "llvm/ADT/SmallBitVector.h"
#include "llvm/ADT/SmallSet.h"
#include "llvm/ADT/SmallVector.h"
#include "llvm/ADT/Statistic.h"
#include "llvm/CodeGen/FunctionLoweringInfo.h"
#include "llvm/CodeGen/GCMetadata.h"
#include "llvm/CodeGen/ISDOpcodes.h"
#include "llvm/CodeGen/MachineFrameInfo.h"
#include "llvm/CodeGen/MachineFunction.h"
#include "llvm/CodeGen/MachineMemOperand.h"
#include "llvm/CodeGen/SelectionDAG.h"
#include "llvm/CodeGen/SelectionDAGNodes.h"
#include "llvm/CodeGen/StackMaps.h"
#include "llvm/CodeGen/TargetLowering.h"
#include "llvm/CodeGen/TargetOpcodes.h"
#include "llvm/CodeGenTypes/MachineValueType.h"
#include "llvm/IR/CallingConv.h"
#include "llvm/IR/DerivedTypes.h"
#include "llvm/IR/GCStrategy.h"
#include "llvm/IR/Instruction.h"
#include "llvm/IR/Instructions.h"
#include "llvm/IR/LLVMContext.h"
#include "llvm/IR/Statepoint.h"
#include "llvm/IR/Type.h"
#include "llvm/Support/Casting.h"
#include "llvm/Support/CommandLine.h"
#include "llvm/Target/TargetMachine.h"
#include "llvm/Target/TargetOptions.h"
#include <cassert>
#include <cstddef>
#include <cstdint>
#include <iterator>
#include <tuple>
#include <utility>

Macros
#define	DEBUG_TYPE "statepoint-lowering"

Typedefs
typedef FunctionLoweringInfo::StatepointRelocationRecord	RecordType

Functions
	STATISTIC (NumSlotsAllocatedForStatepoints, "Number of stack slots allocated for statepoints")
	STATISTIC (NumOfStatepoints, "Number of statepoint nodes encountered")
	STATISTIC (StatepointMaxSlotsRequired, "Maximum number of stack slots required for a singe statepoint")
static void	pushStackMapConstant (SmallVectorImpl< SDValue > &Ops, SelectionDAGBuilder &Builder, uint64_t Value)
static std::optional< int >	findPreviousSpillSlot (const Value *Val, SelectionDAGBuilder &Builder, int LookUpDepth)
	Utility function for reservePreviousStackSlotForValue.
static bool	willLowerDirectly (SDValue Incoming)
	Return true if-and-only-if the given SDValue can be lowered as either a constant argument or a stack reference.
static void	reservePreviousStackSlotForValue (const Value *IncomingValue, SelectionDAGBuilder &Builder)
	Try to find existing copies of the incoming values in stack slots used for statepoint spilling.
static std::pair< SDValue, SDNode * >	lowerCallFromStatepointLoweringInfo (SelectionDAGBuilder::StatepointLoweringInfo &SI, SelectionDAGBuilder &Builder)
	Extract call from statepoint, lower it and return pointer to the call node.
static MachineMemOperand *	getMachineMemOperand (MachineFunction &MF, FrameIndexSDNode &FI)
static std::tuple< SDValue, SDValue, MachineMemOperand * >	spillIncomingStatepointValue (SDValue Incoming, SDValue Chain, SelectionDAGBuilder &Builder)
	Spill a value incoming to the statepoint.
static void	lowerIncomingStatepointValue (SDValue Incoming, bool RequireSpillSlot, SmallVectorImpl< SDValue > &Ops, SmallVectorImpl< MachineMemOperand * > &MemRefs, SelectionDAGBuilder &Builder)
	Lower a single value incoming to a statepoint node.
static bool	isGCValue (const Value *V, SelectionDAGBuilder &Builder)
	Return true if value V represents the GC value.
static void	lowerStatepointMetaArgs (SmallVectorImpl< SDValue > &Ops, SmallVectorImpl< MachineMemOperand * > &MemRefs, SmallVectorImpl< SDValue > &GCPtrs, DenseMap< SDValue, int > &LowerAsVReg, SelectionDAGBuilder::StatepointLoweringInfo &SI, SelectionDAGBuilder &Builder)
	Lower deopt state and gc pointer arguments of the statepoint.
static std::pair< const GCResultInst , const GCResultInst >	getGCResultLocality (const GCStatepointInst &S)
	Return two gc.results if present.

Variables
static constexpr uint32_t	UndefStackMapValue = 0xFEFEFEFE

Macro Definition Documentation

◆ DEBUG_TYPE

#define DEBUG_TYPE "statepoint-lowering"

Definition at line 57 of file StatepointLowering.cpp.

Typedef Documentation

◆ RecordType

typedef FunctionLoweringInfo::StatepointRelocationRecord RecordType

Definition at line 69 of file StatepointLowering.cpp.

Function Documentation

◆ findPreviousSpillSlot()

std::optional< int > findPreviousSpillSlot	(	const Value *	Val,
		SelectionDAGBuilder &	Builder,
		int	LookUpDepth )

static

Utility function for reservePreviousStackSlotForValue.

Tries to find stack slot index to which we have spilled value for previous statepoints. LookUpDepth specifies maximum DFS depth this function is allowed to look.

Definition at line 153 of file StatepointLowering.cpp.

References assert(), llvm::cast(), llvm::dyn_cast(), findPreviousSpillSlot(), llvm::isa(), and llvm::FunctionLoweringInfo::StatepointRelocationRecord::Spill.

Referenced by findPreviousSpillSlot(), and reservePreviousStackSlotForValue().

◆ getGCResultLocality()

std::pair< const GCResultInst *, const GCResultInst * > getGCResultLocality ( const GCStatepointInst & S )

static

Return two gc.results if present.

First result is a block local gc.result, second result is a non-block local gc.result. Corresponding entry will be nullptr if not present.

Definition at line 1027 of file StatepointLowering.cpp.

References llvm::dyn_cast(), llvm::ilist_detail::node_parent_access< NodeTy, ParentTy >::getParent(), and llvm::Instruction::users().

Referenced by llvm::SelectionDAGBuilder::LowerStatepoint().

◆ getMachineMemOperand()

MachineMemOperand * getMachineMemOperand	(	MachineFunction &	MF,
		FrameIndexSDNode &	FI )

static

Definition at line 377 of file StatepointLowering.cpp.

References llvm::MachinePointerInfo::getFixedStack(), llvm::MachineFunction::getFrameInfo(), llvm::FrameIndexSDNode::getIndex(), llvm::MachineFunction::getMachineMemOperand(), llvm::FunctionLoweringInfo::MF, llvm::MachineMemOperand::MOLoad, llvm::MachineMemOperand::MOStore, and llvm::MachineMemOperand::MOVolatile.

Referenced by llvm::MachineFunction::getMachineMemOperand(), llvm::MachineFunction::getMachineMemOperand(), llvm::MachineFunction::getMachineMemOperand(), llvm::MachineFunction::getMachineMemOperand(), llvm::MachineFunction::getMachineMemOperand(), llvm::MachineFunction::getMachineMemOperand(), llvm::MachineFunction::getMachineMemOperand(), lowerIncomingStatepointValue(), lowerStatepointMetaArgs(), and spillIncomingStatepointValue().

◆ isGCValue()

bool isGCValue	(	const Value *	V,
		SelectionDAGBuilder &	Builder )

static

Return true if value V represents the GC value.

The behavior is conservative in case it is not sure that value is not GC the function returns true.

Definition at line 521 of file StatepointLowering.cpp.

References V.

Referenced by llvm::SelectionDAGBuilder::LowerStatepoint(), and lowerStatepointMetaArgs().

◆ lowerCallFromStatepointLoweringInfo()

std::pair< SDValue, SDNode * > lowerCallFromStatepointLoweringInfo	(	SelectionDAGBuilder::StatepointLoweringInfo &	SI,
		SelectionDAGBuilder &	Builder )

static

Extract call from statepoint, lower it and return pointer to the call node.

Also update NodeMap so that getValue(statepoint) will reference lowered call result

Definition at line 337 of file StatepointLowering.cpp.

References assert(), llvm::ISD::CALLSEQ_END, llvm::ISD::CopyFromReg, llvm::ISD::EH_LABEL, llvm::SDValue::getNode(), llvm::SDNode::getOpcode(), llvm::SDNode::getOperand(), and llvm::ISD::LOAD.

Referenced by llvm::SelectionDAGBuilder::LowerAsSTATEPOINT().

◆ lowerIncomingStatepointValue()

void lowerIncomingStatepointValue	(	SDValue	Incoming,
		bool	RequireSpillSlot,
		SmallVectorImpl< SDValue > &	Ops,
		SmallVectorImpl< MachineMemOperand * > &	MemRefs,
		SelectionDAGBuilder &	Builder )

static

Lower a single value incoming to a statepoint node.

This value can be either a deopt value or a gc value, the handling is the same. We special case constants and allocas, then fall back to spilling if required.

Definition at line 443 of file StatepointLowering.cpp.

References AbstractManglingParser< Derived, Alloc >::Ops, assert(), C(), llvm::dyn_cast(), getMachineMemOperand(), llvm::EVT::getSizeInBits(), llvm::SDValue::getValueType(), llvm::SDValue::isUndef(), llvm_unreachable, llvm::FunctionLoweringInfo::MF, llvm::SmallVectorTemplateBase< T, bool >::push_back(), pushStackMapConstant(), spillIncomingStatepointValue(), and willLowerDirectly().

Referenced by lowerStatepointMetaArgs().

◆ lowerStatepointMetaArgs()

void lowerStatepointMetaArgs	(	SmallVectorImpl< SDValue > &	Ops,
		SmallVectorImpl< MachineMemOperand * > &	MemRefs,
		SmallVectorImpl< SDValue > &	GCPtrs,
		DenseMap< SDValue, int > &	LowerAsVReg,
		SelectionDAGBuilder::StatepointLoweringInfo &	SI,
		SelectionDAGBuilder &	Builder )

static

Lower deopt state and gc pointer arguments of the statepoint.

The actual lowering is described in lowerIncomingStatepointValue. This function is responsible for lowering everything in the right position and playing some tricks to avoid redundant stack manipulation where possible. On completion, 'Ops' will contain ready to use operands for machine code statepoint. The chain nodes will have already been created and the DAG root will be set to the last value spilled (if any were).

Definition at line 539 of file StatepointLowering.cpp.

References AbstractManglingParser< Derived, Alloc >::Ops, assert(), llvm::sampleprof::Base, llvm::SmallVectorTemplateCommon< T, typename >::begin(), llvm::DenseMapBase< StorageT, KeyT, ValueT, KeyInfoT, BucketT >::count(), llvm::SmallSet< T, N, C >::count(), llvm::dbgs(), llvm::DeoptLiveIn, llvm::SDValue::dump(), llvm::dyn_cast(), llvm::dyn_cast_or_null(), llvm::SmallVectorTemplateCommon< T, typename >::end(), getMachineMemOperand(), llvm::SDValue::getNode(), llvm::SDValue::getValue(), llvm::SDValue::getValueType(), llvm::SetVector< T, Vector, Set, N >::insert(), llvm::SmallSet< T, N, C >::insert(), isGCValue(), llvm::EVT::isVector(), LLVM_DEBUG, lowerIncomingStatepointValue(), llvm::FunctionLoweringInfo::MF, llvm::SmallVectorTemplateBase< T, bool >::push_back(), pushStackMapConstant(), reservePreviousStackSlotForValue(), llvm::DenseMapBase< StorageT, KeyT, ValueT, KeyInfoT, BucketT >::size(), llvm::SetVector< T, Vector, Set, N >::size(), llvm::SmallVectorTemplateCommon< T, typename >::size(), llvm::SetVector< T, Vector, Set, N >::takeVector(), uint64_t, V, and willLowerDirectly().

Referenced by llvm::SelectionDAGBuilder::LowerAsSTATEPOINT().

◆ pushStackMapConstant()

void pushStackMapConstant	(	SmallVectorImpl< SDValue > &	Ops,
		SelectionDAGBuilder &	Builder,
		uint64_t	Value )

static

Definition at line 71 of file StatepointLowering.cpp.

References AbstractManglingParser< Derived, Alloc >::Ops, and uint64_t.

Referenced by llvm::SelectionDAGBuilder::LowerAsSTATEPOINT(), lowerIncomingStatepointValue(), and lowerStatepointMetaArgs().

◆ reservePreviousStackSlotForValue()

void reservePreviousStackSlotForValue	(	const Value *	IncomingValue,
		SelectionDAGBuilder &	Builder )

static

Try to find existing copies of the incoming values in stack slots used for statepoint spilling.

If we can find a spill slot for the incoming value, mark that slot as allocated, and reuse the same slot for this safepoint. This helps to avoid series of loads and stores that only serve to reshuffle values on the stack between calls.

Definition at line 287 of file StatepointLowering.cpp.

References assert(), llvm::find(), findPreviousSpillSlot(), llvm::SDValue::getNode(), llvm::SDValue::getValue(), llvm::Offset, and willLowerDirectly().

Referenced by lowerStatepointMetaArgs().

◆ spillIncomingStatepointValue()

std::tuple< SDValue, SDValue, MachineMemOperand * > spillIncomingStatepointValue	(	SDValue	Incoming,
		SDValue	Chain,
		SelectionDAGBuilder &	Builder )

static

Spill a value incoming to the statepoint.

It might be either part of vmstate or gcstate. In both cases unconditionally spill it on the stack unless it is a null constant. Return pair with first element being frame index containing saved value and second element with outgoing chain from the emitted store

Definition at line 395 of file StatepointLowering.cpp.

References assert(), llvm::cast(), llvm::MachinePointerInfo::getFixedStack(), getMachineMemOperand(), llvm::MachineFunction::getMachineMemOperand(), llvm::MachineFrameInfo::getObjectAlign(), llvm::MachineFrameInfo::getObjectSize(), llvm::SDValue::getValueSizeInBits(), llvm::SDValue::getValueType(), llvm::FunctionLoweringInfo::MF, and llvm::MachineMemOperand::MOStore.

Referenced by lowerIncomingStatepointValue().

◆ STATISTIC() [1/3]

STATISTIC	(	NumOfStatepoints	,
		"Number of statepoint nodes encountered"	)

◆ STATISTIC() [2/3]

STATISTIC	(	NumSlotsAllocatedForStatepoints	,
		"Number of stack slots allocated for statepoints"	)

◆ STATISTIC() [3/3]

STATISTIC	(	StatepointMaxSlotsRequired	,
		"Maximum number of stack slots required for a singe statepoint"	)

◆ willLowerDirectly()

bool willLowerDirectly ( SDValue Incoming )

static

Return true if-and-only-if the given SDValue can be lowered as either a constant argument or a stack reference.

The key point is that the value doesn't need to be spilled or tracked as a vreg use.

Definition at line 240 of file StatepointLowering.cpp.

References llvm::EVT::getSizeInBits(), llvm::SDValue::getValueType(), llvm::isa(), llvm::isIntOrFPConstant(), and llvm::SDValue::isUndef().

Referenced by llvm::SelectionDAGBuilder::LowerAsSTATEPOINT(), lowerIncomingStatepointValue(), lowerStatepointMetaArgs(), reservePreviousStackSlotForValue(), and llvm::FunctionLoweringInfo::StatepointDirectLeaf::StatepointDirectLeaf().

Variable Documentation

◆ UndefStackMapValue

uint32_t UndefStackMapValue = 0xFEFEFEFE

staticconstexpr

Definition at line 67 of file StatepointLowering.cpp.

Referenced by llvm::FunctionLoweringInfo::StatepointDirectLeaf::rematerialize().

Macros

Typedefs

Functions

Variables

Macro Definition Documentation

◆ DEBUG_TYPE

Typedef Documentation

◆ RecordType

Function Documentation

◆ findPreviousSpillSlot()

◆ getGCResultLocality()

◆ getMachineMemOperand()

◆ isGCValue()

◆ lowerCallFromStatepointLoweringInfo()

◆ lowerIncomingStatepointValue()

◆ lowerStatepointMetaArgs()

◆ pushStackMapConstant()

◆ reservePreviousStackSlotForValue()

◆ spillIncomingStatepointValue()

◆ STATISTIC() [1/3]

◆ STATISTIC() [2/3]

◆ STATISTIC() [3/3]

◆ willLowerDirectly()

Variable Documentation

◆ UndefStackMapValue