#include "X86.h"
#include "X86MachineFunctionInfo.h"
#include "X86RegisterInfo.h"
#include "X86Subtarget.h"
#include "X86TargetMachine.h"
#include "llvm/ADT/Statistic.h"
#include "llvm/CodeGen/MachineModuleInfo.h"
#include "llvm/CodeGen/SelectionDAGISel.h"
#include "llvm/Config/llvm-config.h"
#include "llvm/IR/ConstantRange.h"
#include "llvm/IR/Function.h"
#include "llvm/IR/Instructions.h"
#include "llvm/IR/Intrinsics.h"
#include "llvm/IR/IntrinsicsX86.h"
#include "llvm/IR/Type.h"
#include "llvm/Support/Debug.h"
#include "llvm/Support/ErrorHandling.h"
#include "llvm/Support/KnownBits.h"
#include "llvm/Support/MathExtras.h"
#include <cstdint>
#include "X86GenDAGISel.inc"

Macros
#define	DEBUG_TYPE "x86-isel"

#define	PASS_NAME "X86 DAG->DAG Instruction Selection"

#define	GET_ND_IF_ENABLED(OPC) (Subtarget->hasNDD() ? OPC##_ND : OPC)

#define	GET_EGPR_IF_ENABLED(OPC) (Subtarget->hasEGPR() ? OPC##_EVEX : OPC)

#define	VPTESTM_CASE(VT, SUFFIX)

#define	VPTESTM_BROADCAST_CASES(SUFFIX)

#define	VPTESTM_FULL_CASES(SUFFIX)

Functions
	STATISTIC (NumLoadMoved, "Number of loads moved below TokenFactor")

static bool	isLegalMaskCompare (SDNode N, const X86Subtarget Subtarget)

static void	moveBelowOrigChain (SelectionDAG *CurDAG, SDValue Load, SDValue Call, SDValue OrigChain)
	Replace the original chain operand of the call with load's chain operand and move load below the call's chain operand.

static bool	isCalleeLoad (SDValue Callee, SDValue &Chain, bool HasCallSeq)
	Return true if call address is a load and it can be moved below CALLSEQ_START and the chains leading up to the call.

static bool	isEndbrImm64 (uint64_t Imm)

static bool	needBWI (MVT VT)

static bool	isDispSafeForFrameIndex (int64_t Val)

static void	insertDAGNode (SelectionDAG &DAG, SDValue Pos, SDValue N)

static bool	foldMaskAndShiftToExtract (SelectionDAG &DAG, SDValue N, uint64_t Mask, SDValue Shift, SDValue X, X86ISelAddressMode &AM)

static bool	foldMaskedShiftToScaledMask (SelectionDAG &DAG, SDValue N, X86ISelAddressMode &AM)

static bool	foldMaskAndShiftToScale (SelectionDAG &DAG, SDValue N, uint64_t Mask, SDValue Shift, SDValue X, X86ISelAddressMode &AM)

static bool	foldMaskedShiftToBEXTR (SelectionDAG &DAG, SDValue N, uint64_t Mask, SDValue Shift, SDValue X, X86ISelAddressMode &AM, const X86Subtarget &Subtarget)

static bool	mayUseCarryFlag (X86::CondCode CC)

static bool	isFusableLoadOpStorePattern (StoreSDNode StoreNode, SDValue StoredVal, SelectionDAG CurDAG, unsigned LoadOpNo, LoadSDNode *&LoadNode, SDValue &InputChain)
	Check whether or not the chain ending in StoreNode is suitable for doing the {load; op; store} to modify transformation.

static unsigned	getVPTESTMOpc (MVT TestVT, bool IsTestN, bool FoldedLoad, bool FoldedBCast, bool Masked)

Variables
static cl::opt< bool >	AndImmShrink ("x86-and-imm-shrink", cl::init(true), cl::desc("Enable setting constant bits to reduce size of mask immediates"), cl::Hidden)

static cl::opt< bool >	EnablePromoteAnyextLoad ("x86-promote-anyext-load", cl::init(true), cl::desc("Enable promoting aligned anyext load to wider load"), cl::Hidden)

cl::opt< bool >	IndirectBranchTracking

Macro Definition Documentation

◆ DEBUG_TYPE

#define DEBUG_TYPE "x86-isel"

Definition at line 37 of file X86ISelDAGToDAG.cpp.

◆ GET_EGPR_IF_ENABLED

#define GET_EGPR_IF_ENABLED ( OPC ) (Subtarget->hasEGPR() ? OPC##_EVEX : OPC)

◆ GET_ND_IF_ENABLED

#define GET_ND_IF_ENABLED ( OPC ) (Subtarget->hasNDD() ? OPC##_ND : OPC)

◆ PASS_NAME

#define PASS_NAME "X86 DAG->DAG Instruction Selection"

Definition at line 38 of file X86ISelDAGToDAG.cpp.

◆ VPTESTM_BROADCAST_CASES

#define VPTESTM_BROADCAST_CASES ( SUFFIX )

Value:

default: llvm_unreachable("Unexpected VT!"); \
VPTESTM_CASE(v4i32, DZ128##SUFFIX) \
VPTESTM_CASE(v2i64, QZ128##SUFFIX) \
VPTESTM_CASE(v8i32, DZ256##SUFFIX) \
VPTESTM_CASE(v4i64, QZ256##SUFFIX) \
VPTESTM_CASE(v16i32, DZ##SUFFIX) \
VPTESTM_CASE(v8i64, QZ##SUFFIX)

◆ VPTESTM_CASE

#define VPTESTM_CASE	(	VT,
		SUFFIX
	)

Value:

case MVT::VT: \
  if (Masked) \
    return IsTestN ? X86::VPTESTNM##SUFFIX##k: X86::VPTESTM##SUFFIX##k; \
  return IsTestN ? X86::VPTESTNM##SUFFIX : X86::VPTESTM##SUFFIX;

◆ VPTESTM_FULL_CASES

#define VPTESTM_FULL_CASES ( SUFFIX )

Value:

VPTESTM_BROADCAST_CASES(SUFFIX) \
VPTESTM_CASE(v16i8, BZ128##SUFFIX) \
VPTESTM_CASE(v8i16, WZ128##SUFFIX) \
VPTESTM_CASE(v32i8, BZ256##SUFFIX) \
VPTESTM_CASE(v16i16, WZ256##SUFFIX) \
VPTESTM_CASE(v64i8, BZ##SUFFIX) \
VPTESTM_CASE(v32i16, WZ##SUFFIX)

Function Documentation

◆ foldMaskAndShiftToExtract()

static bool foldMaskAndShiftToExtract	(	SelectionDAG &	DAG,
		SDValue	N,
		uint64_t	Mask,
		SDValue	Shift,
		SDValue	X,
		X86ISelAddressMode &	AM
	)

static

Definition at line 1989 of file X86ISelDAGToDAG.cpp.

References llvm::And, llvm::ISD::AND, DL, llvm::SelectionDAG::getConstant(), llvm::SDValue::getConstantOperandVal(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SelectionDAG::getZExtOrTrunc(), llvm::SDValue::hasOneUse(), insertDAGNode(), N, llvm::SelectionDAG::RemoveDeadNode(), llvm::SelectionDAG::ReplaceAllUsesWith(), llvm::ISD::SHL, llvm::ISD::SRL, and X.

◆ foldMaskAndShiftToScale()

static bool foldMaskAndShiftToScale	(	SelectionDAG &	DAG,
		SDValue	N,
		uint64_t	Mask,
		SDValue	Shift,
		SDValue	X,
		X86ISelAddressMode &	AM
	)

static

◆ foldMaskedShiftToBEXTR()

static bool foldMaskedShiftToBEXTR	(	SelectionDAG &	DAG,
		SDValue	N,
		uint64_t	Mask,
		SDValue	Shift,
		SDValue	X,
		X86ISelAddressMode &	AM,
		const X86Subtarget &	Subtarget
	)

static

Definition at line 2220 of file X86ISelDAGToDAG.cpp.

References llvm::ISD::AND, DL, llvm::SelectionDAG::getConstant(), llvm::SDValue::getConstantOperandVal(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SelectionDAG::getZExtOrTrunc(), llvm::SDValue::hasOneUse(), insertDAGNode(), llvm::isShiftedMask_64(), N, llvm::SelectionDAG::RemoveDeadNode(), llvm::SelectionDAG::ReplaceAllUsesWith(), llvm::ISD::SHL, llvm::ISD::SRL, and X.

◆ foldMaskedShiftToScaledMask()

static bool foldMaskedShiftToScaledMask	(	SelectionDAG &	DAG,
		SDValue	N,
		X86ISelAddressMode &	AM
	)

static

Definition at line 2036 of file X86ISelDAGToDAG.cpp.

References llvm::ISD::AND, llvm::ISD::ANY_EXTEND, DL, llvm::SelectionDAG::getConstant(), llvm::SDValue::getConstantOperandVal(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDValue::getSimpleValueType(), llvm::SDValue::hasOneUse(), insertDAGNode(), N, llvm::SelectionDAG::RemoveDeadNode(), llvm::SelectionDAG::ReplaceAllUsesWith(), llvm::ISD::SHL, and X.

◆ getVPTESTMOpc()

static unsigned getVPTESTMOpc	(	MVT	TestVT,
		bool	IsTestN,
		bool	FoldedLoad,
		bool	FoldedBCast,
		bool	Masked
	)

static

Definition at line 4758 of file X86ISelDAGToDAG.cpp.

References llvm::MVT::SimpleTy, VPTESTM_BROADCAST_CASES, and VPTESTM_FULL_CASES.

◆ insertDAGNode()

static void insertDAGNode	(	SelectionDAG &	DAG,
		SDValue	Pos,
		SDValue	N
	)

static

Definition at line 1971 of file X86ISelDAGToDAG.cpp.

References llvm::ilist_node_impl< OptionsT >::getIterator(), llvm::SDValue::getNode(), llvm::SDNode::getNodeId(), llvm::SelectionDAGISel::getUninvalidatedNodeId(), llvm::SelectionDAGISel::InvalidateNodeId(), N, and llvm::SelectionDAG::RepositionNode().

Referenced by foldMaskAndShiftToExtract(), foldMaskAndShiftToScale(), foldMaskedShiftToBEXTR(), and foldMaskedShiftToScaledMask().

◆ isCalleeLoad()

static bool isCalleeLoad	(	SDValue	Callee,
		SDValue &	Chain,
		bool	HasCallSeq
	)

static

Return true if call address is a load and it can be moved below CALLSEQ_START and the chains leading up to the call.

Return the CALLSEQ_START by reference as a second output. In the case of a tail call, there isn't a callseq node between the call chain and the load.

Definition at line 828 of file X86ISelDAGToDAG.cpp.

References llvm::ISD::CALLSEQ_START, llvm::SDValue::getNode(), llvm::SDValue::getNumOperands(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDValue::hasOneUse(), llvm::ISD::NON_EXTLOAD, llvm::ISD::TokenFactor, and llvm::ISD::UNINDEXED.

◆ isDispSafeForFrameIndex()

static bool isDispSafeForFrameIndex ( int64_t Val )

static

Definition at line 1719 of file X86ISelDAGToDAG.cpp.

◆ isEndbrImm64()

static bool isEndbrImm64 ( uint64_t Imm )

static

Definition at line 865 of file X86ISelDAGToDAG.cpp.

References llvm::is_contained().

◆ isFusableLoadOpStorePattern()

static bool isFusableLoadOpStorePattern	(	StoreSDNode *	StoreNode,
		SDValue	StoredVal,
		SelectionDAG *	CurDAG,
		unsigned	LoadOpNo,
		LoadSDNode *&	LoadNode,
		SDValue &	InputChain
	)

static

Check whether or not the chain ending in StoreNode is suitable for doing the {load; op; store} to modify transformation.

Definition at line 3374 of file X86ISelDAGToDAG.cpp.

References llvm::LoadSDNode::getBasePtr(), llvm::StoreSDNode::getBasePtr(), llvm::MemSDNode::getChain(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDValue::getNumOperands(), llvm::LoadSDNode::getOffset(), llvm::StoreSDNode::getOffset(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDNode::getOperand(), llvm::SDValue::getResNo(), llvm::SDNode::hasNUsesOfValue(), llvm::SDNode::hasPredecessorHelper(), llvm::MemSDNode::isNonTemporal(), llvm::ISD::isNormalLoad(), llvm::ISD::isNormalStore(), llvm::SDNode::ops(), llvm::SmallVectorTemplateBase< T, bool >::push_back(), and llvm::ISD::TokenFactor.

◆ isLegalMaskCompare()

static bool isLegalMaskCompare	(	SDNode *	N,
		const X86Subtarget *	Subtarget
	)

static

Definition at line 588 of file X86ISelDAGToDAG.cpp.

References llvm::X86ISD::CMPM, llvm::X86ISD::CMPMM, llvm::X86ISD::CMPMM_SAE, llvm::X86ISD::FSETCCM, llvm::X86ISD::FSETCCM_SAE, llvm::EVT::is128BitVector(), llvm::EVT::is256BitVector(), N, llvm::ISD::SETCC, llvm::X86ISD::STRICT_CMPM, llvm::X86ISD::VFPCLASS, and llvm::X86ISD::VFPCLASSS.

◆ mayUseCarryFlag()

static bool mayUseCarryFlag ( X86::CondCode CC )

static

Definition at line 3300 of file X86ISelDAGToDAG.cpp.

References CC, llvm::X86::COND_E, llvm::X86::COND_G, llvm::X86::COND_GE, llvm::X86::COND_L, llvm::X86::COND_LE, llvm::X86::COND_NE, llvm::X86::COND_NO, llvm::X86::COND_NP, llvm::X86::COND_NS, llvm::X86::COND_O, llvm::X86::COND_P, and llvm::X86::COND_S.

◆ moveBelowOrigChain()

static void moveBelowOrigChain	(	SelectionDAG *	CurDAG,
		SDValue	Load,
		SDValue	Call,
		SDValue	OrigChain
	)

static

Replace the original chain operand of the call with load's chain operand and move load below the call's chain operand.

Definition at line 793 of file X86ISelDAGToDAG.cpp.

References llvm::SmallVectorImpl< T >::append(), assert(), llvm::SmallVectorImpl< T >::clear(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDValue::getNumOperands(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDNode::op_begin(), llvm::SDNode::op_end(), llvm::SmallVectorTemplateBase< T, bool >::push_back(), llvm::ISD::TokenFactor, and llvm::SelectionDAG::UpdateNodeOperands().

◆ needBWI()

static bool needBWI ( MVT VT )

static

Definition at line 886 of file X86ISelDAGToDAG.cpp.

◆ STATISTIC()

STATISTIC	(	NumLoadMoved	,
		"Number of loads moved below TokenFactor"
	)

Variable Documentation

◆ AndImmShrink

cl::opt< bool > AndImmShrink("x86-and-imm-shrink", cl::init(true), cl::desc("Enable setting constant bits to reduce size of mask immediates"), cl::Hidden)	(	"x86-and-imm-shrink"	,
		cl::init(true)	,
		cl::desc("Enable setting constant bits to reduce size of mask immediates")	,
		cl::Hidden
	)

static

◆ EnablePromoteAnyextLoad

cl::opt< bool > EnablePromoteAnyextLoad("x86-promote-anyext-load", cl::init(true), cl::desc("Enable promoting aligned anyext load to wider load"), cl::Hidden)	(	"x86-promote-anyext-load"	,
		cl::init(true)	,
		cl::desc("Enable promoting aligned anyext load to wider load")	,
		cl::Hidden
	)

static

◆ IndirectBranchTracking

cl::opt<bool> IndirectBranchTracking

extern

Macros

Functions

Variables

Macro Definition Documentation

◆ DEBUG_TYPE

◆ GET_EGPR_IF_ENABLED

◆ GET_ND_IF_ENABLED

◆ PASS_NAME

◆ VPTESTM_BROADCAST_CASES

◆ VPTESTM_CASE

◆ VPTESTM_FULL_CASES

Function Documentation

◆ foldMaskAndShiftToExtract()

◆ foldMaskAndShiftToScale()

◆ foldMaskedShiftToBEXTR()

◆ foldMaskedShiftToScaledMask()

◆ getVPTESTMOpc()

◆ insertDAGNode()

◆ isCalleeLoad()

◆ isDispSafeForFrameIndex()

◆ isEndbrImm64()

◆ isFusableLoadOpStorePattern()

◆ isLegalMaskCompare()

◆ mayUseCarryFlag()

◆ moveBelowOrigChain()

◆ needBWI()

◆ STATISTIC()

Variable Documentation

◆ AndImmShrink

◆ EnablePromoteAnyextLoad

◆ IndirectBranchTracking