NVPTXISelDAGToDAG.cpp File Reference

#include "NVPTXISelDAGToDAG.h"
#include "NVPTX.h"
#include "NVPTXUtilities.h"
#include "llvm/ADT/APInt.h"
#include "llvm/Analysis/ValueTracking.h"
#include "llvm/CodeGen/ISDOpcodes.h"
#include "llvm/CodeGen/SelectionDAG.h"
#include "llvm/CodeGen/SelectionDAGNodes.h"
#include "llvm/IR/GlobalValue.h"
#include "llvm/IR/Instructions.h"
#include "llvm/IR/IntrinsicsNVPTX.h"
#include "llvm/IR/NVVMIntrinsicUtils.h"
#include "llvm/Support/AtomicOrdering.h"
#include "llvm/Support/CommandLine.h"
#include "llvm/Support/ErrorHandling.h"
#include "llvm/Support/FormatVariadic.h"
#include "llvm/Support/MathExtras.h"
#include <optional>

Go to the source code of this file.

Macros
#define	DEBUG_TYPE   "nvptx-isel"
#define	PASS_NAME   "NVPTX DAG->DAG Pattern Instruction Selection"
#define	TCGEN05_LD_OPCODE(SHAPE, NUM)
#define	CP_ASYNC_BULK_TENSOR_OPCODE(dir, dim, mode, is_s32, suffix)
#define	GET_CP_ASYNC_BULK_TENSOR_OPCODE_S2G_RED(dim, mode, is_ch, is_s32)
#define	GET_CP_ASYNC_BULK_TENSOR_OPCODE_G2S(dim, mode, is_mc, is_ch, is_s32)
#define	TCGEN05_ST_OPCODE(SHAPE, NUM)

Functions
static unsigned	getTcgen05LdOpcode (unsigned IID, bool enablePack)
static std::optional< NVPTX::AddressSpace >	convertAS (unsigned AS)
static bool	canLowerToLDG (const MemSDNode &N, const NVPTXSubtarget &Subtarget, NVPTX::AddressSpace CodeAddrSpace)
static unsigned int	getFenceOp (NVPTX::Ordering O, NVPTX::Scope S, NVPTXSubtarget const *T)
static std::optional< unsigned >	pickOpcodeForVT (MVT::SimpleValueType VT, std::optional< unsigned > Opcode_i16, std::optional< unsigned > Opcode_i32, std::optional< unsigned > Opcode_i64)
static bool	isAddLike (const SDValue V)
static SDValue	stripAssertAlign (SDValue N)
static SDValue	selectBaseADDR (SDValue N, SelectionDAG *DAG)
static SDValue	accumulateOffset (SDValue &Addr, SDLoc DL, SelectionDAG *DAG)
static std::pair< SDValue, SDValue >	selectADDR (SDValue Addr, SelectionDAG *DAG)
static unsigned	getStoreVectorNumElts (SDNode *N)
static unsigned	GetCpAsyncBulkTensorS2GReductionOpcode (size_t Dim, bool IsShared32, bool IsCacheHint, bool IsIm2Col)
static unsigned	GetCpAsyncBulkTensorG2SOpcode (size_t Dim, bool IsShared32, bool IsMultiCast, bool IsCacheHint, bool IsIm2Col)
static size_t	GetDimsFromIntrinsic (unsigned IID)
static unsigned	getTcgen05StOpcode (unsigned IID, bool enableUnpack)

Variables
static cl::opt< bool >	EnableRsqrtOpt ("nvptx-rsqrt-approx-opt", cl::init(true), cl::Hidden, cl::desc("Enable reciprocal sqrt optimization"))

Macro Definition Documentation

CP_ASYNC_BULK_TENSOR_OPCODE

#define CP_ASYNC_BULK_TENSOR_OPCODE	(		dir,
			dim,
			mode,
			is_s32,
			suffix )

Value:

  (is_s32                                                                      \
       ? NVPTX::CP_ASYNC_BULK_TENSOR_##dir##_##dim##_SHARED32_##mode##suffix   \
       : NVPTX::CP_ASYNC_BULK_TENSOR_##dir##_##dim##_##mode##suffix)

Definition at line 1851 of file NVPTXISelDAGToDAG.cpp.

DEBUG_TYPE

#define DEBUG_TYPE   "nvptx-isel"

Definition at line 34 of file NVPTXISelDAGToDAG.cpp.

GET_CP_ASYNC_BULK_TENSOR_OPCODE_G2S

#define GET_CP_ASYNC_BULK_TENSOR_OPCODE_G2S	(		dim,
			mode,
			is_mc,
			is_ch,
			is_s32 )

Value:

  [&]() -> auto {                                                              \
    if (is_mc && is_ch)                                                        \
      return CP_ASYNC_BULK_TENSOR_OPCODE(G2S, dim, mode, is_s32, _MC_CH);      \
    if (is_ch)                                                                 \
      return CP_ASYNC_BULK_TENSOR_OPCODE(G2S, dim, mode, is_s32, _CH);         \
    if (is_mc)                                                                 \
      return CP_ASYNC_BULK_TENSOR_OPCODE(G2S, dim, mode, is_s32, _MC);         \
    return CP_ASYNC_BULK_TENSOR_OPCODE(G2S, dim, mode, is_s32, );              \
  }()

Definition at line 1860 of file NVPTXISelDAGToDAG.cpp.

Referenced by GetCpAsyncBulkTensorG2SOpcode().

GET_CP_ASYNC_BULK_TENSOR_OPCODE_S2G_RED

#define GET_CP_ASYNC_BULK_TENSOR_OPCODE_S2G_RED	(		dim,
			mode,
			is_ch,
			is_s32 )

Value:

  (is_ch ? (CP_ASYNC_BULK_TENSOR_OPCODE(RED, dim, mode, is_s32, _CH))          \
         : (CP_ASYNC_BULK_TENSOR_OPCODE(RED, dim, mode, is_s32, )))

Definition at line 1856 of file NVPTXISelDAGToDAG.cpp.

Referenced by GetCpAsyncBulkTensorS2GReductionOpcode().

PASS_NAME

#define PASS_NAME   "NVPTX DAG->DAG Pattern Instruction Selection"

Definition at line 35 of file NVPTXISelDAGToDAG.cpp.

TCGEN05_LD_OPCODE

#define TCGEN05_LD_OPCODE	(		SHAPE,
			NUM )

Value:

  (enablePack ? NVPTX::TCGEN05_LD_##SHAPE##_##NUM##_PACK                       \
              : NVPTX::TCGEN05_LD_##SHAPE##_##NUM)

Definition at line 189 of file NVPTXISelDAGToDAG.cpp.

Referenced by getTcgen05LdOpcode().

TCGEN05_ST_OPCODE

#define TCGEN05_ST_OPCODE	(		SHAPE,
			NUM )

Value:

  (enableUnpack ? NVPTX::TCGEN05_ST_##SHAPE##_##NUM##_UNPACK                   \
                : NVPTX::TCGEN05_ST_##SHAPE##_##NUM)

Definition at line 2044 of file NVPTXISelDAGToDAG.cpp.

Referenced by getTcgen05StOpcode().

Function Documentation

accumulateOffset()

SDValue accumulateOffset ( SDValue & Addr, SDLoc DL, SelectionDAG * DAG )

static

Definition at line 1057 of file NVPTXISelDAGToDAG.cpp.

References DL, llvm::dyn_cast(), llvm::SDNode::getOperand(), llvm::SDValue::getOperand(), llvm::APInt::getSExtValue(), llvm::SelectionDAG::getSignedTargetConstant(), isAddLike(), llvm::APInt::sext(), and stripAssertAlign().

Referenced by selectADDR().

canLowerToLDG()

bool canLowerToLDG ( const MemSDNode & N, const NVPTXSubtarget & Subtarget, NVPTX::AddressSpace CodeAddrSpace )

static

Definition at line 774 of file NVPTXISelDAGToDAG.cpp.

References llvm::NVPTX::Global, llvm::NVPTXSubtarget::hasLDG(), and N.

convertAS()

std::optional< NVPTX::AddressSpace > convertAS ( unsigned AS )

static

Definition at line 482 of file NVPTXISelDAGToDAG.cpp.

References llvm::NVPTXAS::ADDRESS_SPACE_CONST, llvm::NVPTXAS::ADDRESS_SPACE_GENERIC, llvm::NVPTXAS::ADDRESS_SPACE_GLOBAL, llvm::NVPTXAS::ADDRESS_SPACE_LOCAL, llvm::NVPTXAS::ADDRESS_SPACE_PARAM, llvm::NVPTXAS::ADDRESS_SPACE_SHARED, llvm::NVPTXAS::ADDRESS_SPACE_SHARED_CLUSTER, llvm::NVPTX::Const, llvm::NVPTX::Generic, llvm::NVPTX::Global, llvm::NVPTX::Local, llvm::NVPTX::Param, llvm::NVPTX::Shared, and llvm::NVPTX::SharedCluster.

Referenced by llvm::NVPTXDAGToDAGISel::getAddrSpace().

GetCpAsyncBulkTensorG2SOpcode()

unsigned GetCpAsyncBulkTensorG2SOpcode ( size_t Dim, bool IsShared32, bool IsMultiCast, bool IsCacheHint, bool IsIm2Col )

static

Definition at line 1914 of file NVPTXISelDAGToDAG.cpp.

References D(), GET_CP_ASYNC_BULK_TENSOR_OPCODE_G2S, and llvm_unreachable.

GetCpAsyncBulkTensorS2GReductionOpcode()

unsigned GetCpAsyncBulkTensorS2GReductionOpcode ( size_t Dim, bool IsShared32, bool IsCacheHint, bool IsIm2Col )

static

Definition at line 1871 of file NVPTXISelDAGToDAG.cpp.

References D(), GET_CP_ASYNC_BULK_TENSOR_OPCODE_S2G_RED, and llvm_unreachable.

GetDimsFromIntrinsic()

size_t GetDimsFromIntrinsic ( unsigned IID )

static

Definition at line 1956 of file NVPTXISelDAGToDAG.cpp.

References llvm_unreachable.

getFenceOp()

unsigned int getFenceOp ( NVPTX::Ordering O, NVPTX::Scope S, NVPTXSubtarget const * T )

static

Definition at line 782 of file NVPTXISelDAGToDAG.cpp.

References llvm::NVPTX::Acquire, llvm::NVPTX::AcquireRelease, llvm::NVPTX::Block, llvm::NVPTX::Cluster, llvm::NVPTX::DefaultDevice, llvm::NVPTX::Device, llvm::formatv(), llvm_unreachable, llvm::NVPTX::NotAtomic, llvm::NVPTX::Relaxed, llvm::NVPTX::RelaxedMMIO, llvm::NVPTX::Release, llvm::report_fatal_error(), llvm::NVPTX::SequentiallyConsistent, llvm::NVPTX::System, T, llvm::NVPTX::Thread, and llvm::NVPTX::Volatile.

getStoreVectorNumElts()

unsigned getStoreVectorNumElts ( SDNode * N )

static

Definition at line 1152 of file NVPTXISelDAGToDAG.cpp.

References llvm_unreachable, N, llvm::NVPTXISD::StoreV2, llvm::NVPTXISD::StoreV4, and llvm::NVPTXISD::StoreV8.

getTcgen05LdOpcode()

unsigned getTcgen05LdOpcode ( unsigned IID, bool enablePack )

static

Definition at line 193 of file NVPTXISelDAGToDAG.cpp.

References llvm_unreachable, and TCGEN05_LD_OPCODE.

getTcgen05StOpcode()

unsigned getTcgen05StOpcode ( unsigned IID, bool enableUnpack )

static

Definition at line 2048 of file NVPTXISelDAGToDAG.cpp.

References llvm_unreachable, and TCGEN05_ST_OPCODE.

isAddLike()

bool isAddLike ( const SDValue V )

inlinestatic

Definition at line 1029 of file NVPTXISelDAGToDAG.cpp.

References llvm::ISD::ADD, and llvm::ISD::OR.

Referenced by accumulateOffset().

pickOpcodeForVT()

std::optional< unsigned > pickOpcodeForVT ( MVT::SimpleValueType VT, std::optional< unsigned > Opcode_i16, std::optional< unsigned > Opcode_i32, std::optional< unsigned > Opcode_i64 )

static

Definition at line 1005 of file NVPTXISelDAGToDAG.cpp.

selectADDR()

std::pair< SDValue, SDValue > selectADDR ( SDValue Addr, SelectionDAG * DAG )

static

Definition at line 1076 of file NVPTXISelDAGToDAG.cpp.

References accumulateOffset(), llvm::sampleprof::Base, llvm::Offset, and selectBaseADDR().

Referenced by llvm::NVPTXDAGToDAGISel::SelectInlineAsmMemoryOperand().

selectBaseADDR()

SDValue selectBaseADDR ( SDValue N, SelectionDAG * DAG )

static

Definition at line 1042 of file NVPTXISelDAGToDAG.cpp.

References llvm::dyn_cast(), llvm::SelectionDAG::getTargetExternalSymbol(), llvm::SelectionDAG::getTargetFrameIndex(), llvm::SelectionDAG::getTargetGlobalAddress(), N, and stripAssertAlign().

Referenced by selectADDR().

stripAssertAlign()

SDValue stripAssertAlign ( SDValue N )

static

Definition at line 1034 of file NVPTXISelDAGToDAG.cpp.

References llvm::ISD::AssertAlign, and N.

Referenced by accumulateOffset(), and selectBaseADDR().

Variable Documentation

EnableRsqrtOpt

cl::opt< bool > EnableRsqrtOpt("nvptx-rsqrt-approx-opt", cl::init(true), cl::Hidden, cl::desc("Enable reciprocal sqrt optimization")) ( "nvptx-rsqrt-approx-opt" , cl::init(true) , cl::Hidden , cl::desc("Enable reciprocal sqrt optimization")  )

static