RISCVISelLowering.cpp File Reference

#include "RISCVISelLowering.h"
#include "MCTargetDesc/RISCVMatInt.h"
#include "RISCV.h"
#include "RISCVMachineFunctionInfo.h"
#include "RISCVRegisterInfo.h"
#include "RISCVSubtarget.h"
#include "RISCVTargetMachine.h"
#include "llvm/ADT/SmallSet.h"
#include "llvm/ADT/Statistic.h"
#include "llvm/Analysis/MemoryLocation.h"
#include "llvm/Analysis/VectorUtils.h"
#include "llvm/CodeGen/MachineFrameInfo.h"
#include "llvm/CodeGen/MachineFunction.h"
#include "llvm/CodeGen/MachineInstrBuilder.h"
#include "llvm/CodeGen/MachineJumpTableInfo.h"
#include "llvm/CodeGen/MachineRegisterInfo.h"
#include "llvm/CodeGen/SelectionDAGAddressAnalysis.h"
#include "llvm/CodeGen/TargetLoweringObjectFileImpl.h"
#include "llvm/CodeGen/ValueTypes.h"
#include "llvm/IR/DiagnosticInfo.h"
#include "llvm/IR/DiagnosticPrinter.h"
#include "llvm/IR/IRBuilder.h"
#include "llvm/IR/Instructions.h"
#include "llvm/IR/IntrinsicsRISCV.h"
#include "llvm/IR/PatternMatch.h"
#include "llvm/Support/CommandLine.h"
#include "llvm/Support/Debug.h"
#include "llvm/Support/ErrorHandling.h"
#include "llvm/Support/InstructionCost.h"
#include "llvm/Support/KnownBits.h"
#include "llvm/Support/MathExtras.h"
#include "llvm/Support/raw_ostream.h"
#include <optional>
#include "RISCVGenAsmMatcher.inc"
#include "RISCVGenSearchableTables.inc"

Go to the source code of this file.

Classes
struct	VIDSequence

Namespaces
namespace	llvm
	This is an optimization pass for GlobalISel generic memory operations.
namespace	llvm::RISCVVIntrinsicsTable

Macros
#define	DEBUG_TYPE   "riscv-lower"
#define	OP_CASE(NODE)
#define	VP_CASE(NODE)
#define	NODE_NAME_CASE(NODE)
#define	GET_REGISTER_MATCHER
#define	GET_RISCVVIntrinsicsTable_IMPL

Functions
	STATISTIC (NumTailCalls, "Number of tail calls")
static void	translateSetCCForBranch (const SDLoc &DL, SDValue &LHS, SDValue &RHS, ISD::CondCode &CC, SelectionDAG &DAG)
static SDValue	getVLOperand (SDValue Op)
static bool	useRVVForFixedLengthVectorVT (MVT VT, const RISCVSubtarget &Subtarget)
static MVT	getContainerForFixedLengthVector (const TargetLowering &TLI, MVT VT, const RISCVSubtarget &Subtarget)
static MVT	getContainerForFixedLengthVector (SelectionDAG &DAG, MVT VT, const RISCVSubtarget &Subtarget)
static SDValue	convertToScalableVector (EVT VT, SDValue V, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static SDValue	convertFromScalableVector (EVT VT, SDValue V, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static MVT	getMaskTypeFor (MVT VecVT)
	Return the type of the mask type suitable for masking the provided vector type.
static SDValue	getAllOnesMask (MVT VecVT, SDValue VL, const SDLoc &DL, SelectionDAG &DAG)
	Creates an all ones mask suitable for masking a vector of type VecTy with vector length VL.
static SDValue	getVLOp (uint64_t NumElts, const SDLoc &DL, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static std::pair< SDValue, SDValue >	getDefaultScalableVLOps (MVT VecVT, const SDLoc &DL, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static std::pair< SDValue, SDValue >	getDefaultVLOps (uint64_t NumElts, MVT ContainerVT, const SDLoc &DL, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static std::pair< SDValue, SDValue >	getDefaultVLOps (MVT VecVT, MVT ContainerVT, const SDLoc &DL, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static SDValue	lowerFP_TO_INT_SAT (SDValue Op, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static RISCVFPRndMode::RoundingMode	matchRoundingOp (unsigned Opc)
static SDValue	lowerVectorFTRUNC_FCEIL_FFLOOR_FROUND (SDValue Op, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static SDValue	lowerVectorStrictFTRUNC_FCEIL_FFLOOR_FROUND (SDValue Op, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static SDValue	lowerFTRUNC_FCEIL_FFLOOR_FROUND (SDValue Op, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static SDValue	lowerVectorXRINT (SDValue Op, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static SDValue	getVSlidedown (SelectionDAG &DAG, const RISCVSubtarget &Subtarget, const SDLoc &DL, EVT VT, SDValue Merge, SDValue Op, SDValue Offset, SDValue Mask, SDValue VL, unsigned Policy=RISCVII::TAIL_UNDISTURBED_MASK_UNDISTURBED)
static SDValue	getVSlideup (SelectionDAG &DAG, const RISCVSubtarget &Subtarget, const SDLoc &DL, EVT VT, SDValue Merge, SDValue Op, SDValue Offset, SDValue Mask, SDValue VL, unsigned Policy=RISCVII::TAIL_UNDISTURBED_MASK_UNDISTURBED)
static MVT	getLMUL1VT (MVT VT)
static std::optional< uint64_t >	getExactInteger (const APFloat &APF, uint32_t BitWidth)
static std::optional< VIDSequence >	isSimpleVIDSequence (SDValue Op)
static SDValue	matchSplatAsGather (SDValue SplatVal, MVT VT, const SDLoc &DL, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static SDValue	lowerBuildVectorViaDominantValues (SDValue Op, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
	Try and optimize BUILD_VECTORs with "dominant values" - these are values which constitute a large proportion of the elements.
static SDValue	lowerBuildVectorOfConstants (SDValue Op, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static SDValue	lowerBUILD_VECTOR (SDValue Op, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static SDValue	splatPartsI64WithVL (const SDLoc &DL, MVT VT, SDValue Passthru, SDValue Lo, SDValue Hi, SDValue VL, SelectionDAG &DAG)
static SDValue	splatSplitI64WithVL (const SDLoc &DL, MVT VT, SDValue Passthru, SDValue Scalar, SDValue VL, SelectionDAG &DAG)
static SDValue	lowerScalarSplat (SDValue Passthru, SDValue Scalar, SDValue VL, MVT VT, const SDLoc &DL, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static SDValue	lowerScalarInsert (SDValue Scalar, SDValue VL, MVT VT, const SDLoc &DL, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static bool	isDeinterleaveShuffle (MVT VT, MVT ContainerVT, SDValue V1, SDValue V2, ArrayRef< int > Mask, const RISCVSubtarget &Subtarget)
static bool	isInterleaveShuffle (ArrayRef< int > Mask, MVT VT, int &EvenSrc, int &OddSrc, const RISCVSubtarget &Subtarget)
	Is this shuffle interleaving contiguous elements from one vector into the even elements and contiguous elements from another vector into the odd elements.
static int	isElementRotate (int &LoSrc, int &HiSrc, ArrayRef< int > Mask)
	Match shuffles that concatenate two vectors, rotate the concatenation, and then extract the original number of elements from the rotated result.
static SDValue	getDeinterleaveViaVNSRL (const SDLoc &DL, MVT VT, SDValue Src, bool EvenElts, const RISCVSubtarget &Subtarget, SelectionDAG &DAG)
static SDValue	lowerVECTOR_SHUFFLEAsVSlidedown (const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Mask, const RISCVSubtarget &Subtarget, SelectionDAG &DAG)
static SDValue	lowerVECTOR_SHUFFLEAsVSlideup (const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Mask, const RISCVSubtarget &Subtarget, SelectionDAG &DAG)
static SDValue	lowerVECTOR_SHUFFLEAsVSlide1 (const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Mask, const RISCVSubtarget &Subtarget, SelectionDAG &DAG)
	Match v(f)slide1up/down idioms.
static SDValue	getWideningInterleave (SDValue EvenV, SDValue OddV, const SDLoc &DL, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static SDValue	lowerBitreverseShuffle (ShuffleVectorSDNode *SVN, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static SDValue	lowerVECTOR_SHUFFLEAsRotate (ShuffleVectorSDNode *SVN, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static SDValue	lowerVECTOR_SHUFFLE (SDValue Op, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static SDValue	lowerConstant (SDValue Op, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static SDValue	LowerATOMIC_FENCE (SDValue Op, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static SDValue	lowerFMAXIMUM_FMINIMUM (SDValue Op, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static unsigned	getRISCVVLOp (SDValue Op)
	Get a RISC-V target specified VL op for a given SDNode.
static bool	hasMergeOp (unsigned Opcode)
	Return true if a RISC-V target specified op has a merge operand.
static bool	hasMaskOp (unsigned Opcode)
	Return true if a RISC-V target specified op has a mask operand.
static SDValue	SplitVectorOp (SDValue Op, SelectionDAG &DAG)
static SDValue	SplitVPOp (SDValue Op, SelectionDAG &DAG)
static SDValue	SplitVectorReductionOp (SDValue Op, SelectionDAG &DAG)
static SDValue	SplitStrictFPVectorOp (SDValue Op, SelectionDAG &DAG)
static SDValue	getTargetNode (GlobalAddressSDNode *N, const SDLoc &DL, EVT Ty, SelectionDAG &DAG, unsigned Flags)
static SDValue	getTargetNode (BlockAddressSDNode *N, const SDLoc &DL, EVT Ty, SelectionDAG &DAG, unsigned Flags)
static SDValue	getTargetNode (ConstantPoolSDNode *N, const SDLoc &DL, EVT Ty, SelectionDAG &DAG, unsigned Flags)
static SDValue	getTargetNode (JumpTableSDNode *N, const SDLoc &DL, EVT Ty, SelectionDAG &DAG, unsigned Flags)
static std::optional< bool >	matchSetCC (SDValue LHS, SDValue RHS, ISD::CondCode CC, SDValue Val)
static SDValue	combineSelectToBinOp (SDNode *N, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static SDValue	foldBinOpIntoSelectIfProfitable (SDNode *BO, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static std::optional< MVT >	getSmallestVTForIndex (MVT VecVT, unsigned MaxIdx, SDLoc DL, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static SDValue	lowerVectorIntrinsicScalars (SDValue Op, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static SDValue	lowerGetVectorLength (SDNode *N, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static void	getVCIXOperands (SDValue &Op, SelectionDAG &DAG, SmallVector< SDValue > &Ops)
static bool	isValidEGW (int EGS, EVT VT, const RISCVSubtarget &Subtarget)
static unsigned	getRVVReductionOp (unsigned ISDOpcode)
static bool	isNonZeroAVL (SDValue AVL)
static SDValue	lowerReductionSeq (unsigned RVVOpcode, MVT ResVT, SDValue StartValue, SDValue Vec, SDValue Mask, SDValue VL, const SDLoc &DL, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
	Helper to lower a reduction sequence of the form: scalar = reduce_op vec, scalar_start.
static std::tuple< unsigned, SDValue, SDValue >	getRVVFPReductionOpAndOperands (SDValue Op, SelectionDAG &DAG, EVT EltVT, const RISCVSubtarget &Subtarget)
static SDValue	widenVectorOpsToi8 (SDValue N, const SDLoc &DL, SelectionDAG &DAG)
static RISCVISD::NodeType	getRISCVWOpcode (unsigned Opcode)
static SDValue	customLegalizeToWOp (SDNode *N, SelectionDAG &DAG, unsigned ExtOpc=ISD::ANY_EXTEND)
static SDValue	customLegalizeToWOpWithSExt (SDNode *N, SelectionDAG &DAG)
static unsigned	getVecReduceOpcode (unsigned Opc)
	Given a binary operator, return the associative generic ISD::VECREDUCE_OP which corresponds to it.
static SDValue	combineBinOpOfExtractToReduceTree (SDNode *N, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
	Perform two related transforms whose purpose is to incrementally recognize an explode_vector followed by scalar reduction as a vector reduction node.
static SDValue	combineBinOpToReduce (SDNode *N, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static SDValue	transformAddShlImm (SDNode *N, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static SDValue	combineSelectAndUse (SDNode *N, SDValue Slct, SDValue OtherOp, SelectionDAG &DAG, bool AllOnes, const RISCVSubtarget &Subtarget)
static SDValue	combineSelectAndUseCommutative (SDNode *N, SelectionDAG &DAG, bool AllOnes, const RISCVSubtarget &Subtarget)
static SDValue	transformAddImmMulImm (SDNode *N, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static SDValue	combineAddOfBooleanXor (SDNode *N, SelectionDAG &DAG)
static SDValue	performADDCombine (SDNode *N, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static SDValue	combineSubOfBoolean (SDNode *N, SelectionDAG &DAG)
static SDValue	performSUBCombine (SDNode *N, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static SDValue	combineDeMorganOfBoolean (SDNode *N, SelectionDAG &DAG)
static SDValue	performTRUNCATECombine (SDNode *N, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static SDValue	performANDCombine (SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const RISCVSubtarget &Subtarget)
static SDValue	combineOrOfCZERO (SDNode *N, SDValue N0, SDValue N1, SelectionDAG &DAG)
static SDValue	performORCombine (SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const RISCVSubtarget &Subtarget)
static SDValue	performXORCombine (SDNode *N, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static SDValue	performMULCombine (SDNode *N, SelectionDAG &DAG)
static bool	narrowIndex (SDValue &N, ISD::MemIndexType IndexType, SelectionDAG &DAG)
	According to the property that indexed load/store instructions zero-extend their indices, try to narrow the type of index operand.
static SDValue	performSETCCCombine (SDNode *N, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static SDValue	performSIGN_EXTEND_INREGCombine (SDNode *N, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static SDValue	combineBinOp_VLToVWBinOp_VL (SDNode *N, TargetLowering::DAGCombinerInfo &DCI)
	Combine a binary operation to its equivalent VW or VW_W form.
static SDValue	tryMemPairCombine (SelectionDAG &DAG, LSBaseSDNode *LSNode1, LSBaseSDNode *LSNode2, SDValue BasePtr, uint64_t Imm)
static SDValue	performMemPairCombine (SDNode *N, TargetLowering::DAGCombinerInfo &DCI)
static SDValue	performFP_TO_INTCombine (SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const RISCVSubtarget &Subtarget)
static SDValue	performFP_TO_INT_SATCombine (SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const RISCVSubtarget &Subtarget)
static SDValue	performBITREVERSECombine (SDNode *N, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static unsigned	negateFMAOpcode (unsigned Opcode, bool NegMul, bool NegAcc)
static SDValue	combineVFMADD_VLWithVFNEG_VL (SDNode *N, SelectionDAG &DAG)
static SDValue	performVFMADD_VLCombine (SDNode *N, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static SDValue	performVFMUL_VLCombine (SDNode *N, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static SDValue	performFADDSUB_VLCombine (SDNode *N, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static SDValue	performSRACombine (SDNode *N, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static SDValue	tryDemorganOfBooleanCondition (SDValue Cond, SelectionDAG &DAG)
static bool	combine_CC (SDValue &LHS, SDValue &RHS, SDValue &CC, const SDLoc &DL, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static SDValue	tryFoldSelectIntoOp (SDNode *N, SelectionDAG &DAG, SDValue TrueVal, SDValue FalseVal, bool Swapped)
static SDValue	foldSelectOfCTTZOrCTLZ (SDNode *N, SelectionDAG &DAG)
static SDValue	performSELECTCombine (SDNode *N, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static SDValue	performBUILD_VECTORCombine (SDNode *N, SelectionDAG &DAG, const RISCVSubtarget &Subtarget, const RISCVTargetLowering &TLI)
	If we have a build_vector where each lane is binop X, C, where C is a constant (but not necessarily the same constant on all lanes), form binop (build_vector x1, x2, ...), (build_vector c1, c2, c3, ..).
static SDValue	performCONCAT_VECTORSCombine (SDNode *N, SelectionDAG &DAG, const RISCVSubtarget &Subtarget, const RISCVTargetLowering &TLI)
static SDValue	combineToVWMACC (SDNode *N, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
static bool	legalizeScatterGatherIndexType (SDLoc DL, SDValue &Index, ISD::MemIndexType &IndexType, RISCVTargetLowering::DAGCombinerInfo &DCI)
static bool	matchIndexAsShuffle (EVT VT, SDValue Index, SDValue Mask, SmallVector< int > &ShuffleMask)
	Match the index vector of a scatter or gather node as the shuffle mask which performs the rearrangement if possible.
static bool	matchIndexAsWiderOp (EVT VT, SDValue Index, SDValue Mask, Align BaseAlign, const RISCVSubtarget &ST)
	Match the index of a gather or scatter operation as an operation with twice the element width and half the number of elements.
static uint64_t	computeGREVOrGORC (uint64_t x, unsigned ShAmt, bool IsGORC)
static MachineBasicBlock *	emitReadCycleWidePseudo (MachineInstr &MI, MachineBasicBlock *BB)
static MachineBasicBlock *	emitSplitF64Pseudo (MachineInstr &MI, MachineBasicBlock *BB, const RISCVSubtarget &Subtarget)
static MachineBasicBlock *	emitBuildPairF64Pseudo (MachineInstr &MI, MachineBasicBlock *BB, const RISCVSubtarget &Subtarget)
static bool	isSelectPseudo (MachineInstr &MI)
static MachineBasicBlock *	emitQuietFCMP (MachineInstr &MI, MachineBasicBlock *BB, unsigned RelOpcode, unsigned EqOpcode, const RISCVSubtarget &Subtarget)
static MachineBasicBlock *	EmitLoweredCascadedSelect (MachineInstr &First, MachineInstr &Second, MachineBasicBlock *ThisMBB, const RISCVSubtarget &Subtarget)
static MachineBasicBlock *	emitSelectPseudo (MachineInstr &MI, MachineBasicBlock *BB, const RISCVSubtarget &Subtarget)
static MachineBasicBlock *	emitVFROUND_NOEXCEPT_MASK (MachineInstr &MI, MachineBasicBlock *BB, unsigned CVTXOpc, unsigned CVTFOpc)
static MachineBasicBlock *	emitFROUND (MachineInstr &MI, MachineBasicBlock *MBB, const RISCVSubtarget &Subtarget)
static bool	CC_RISCVAssign2XLen (unsigned XLen, CCState &State, CCValAssign VA1, ISD::ArgFlagsTy ArgFlags1, unsigned ValNo2, MVT ValVT2, MVT LocVT2, ISD::ArgFlagsTy ArgFlags2)
static unsigned	allocateRVVReg (MVT ValVT, unsigned ValNo, std::optional< unsigned > FirstMaskArgument, CCState &State, const RISCVTargetLowering &TLI)
template<typename ArgTy >
static std::optional< unsigned >	preAssignMask (const ArgTy &Args)
static SDValue	convertLocVTToValVT (SelectionDAG &DAG, SDValue Val, const CCValAssign &VA, const SDLoc &DL, const RISCVSubtarget &Subtarget)
static SDValue	unpackFromRegLoc (SelectionDAG &DAG, SDValue Chain, const CCValAssign &VA, const SDLoc &DL, const ISD::InputArg &In, const RISCVTargetLowering &TLI)
static SDValue	convertValVTToLocVT (SelectionDAG &DAG, SDValue Val, const CCValAssign &VA, const SDLoc &DL, const RISCVSubtarget &Subtarget)
static SDValue	unpackFromMemLoc (SelectionDAG &DAG, SDValue Chain, const CCValAssign &VA, const SDLoc &DL)
static SDValue	unpackF64OnRV32DSoftABI (SelectionDAG &DAG, SDValue Chain, const CCValAssign &VA, const CCValAssign &HiVA, const SDLoc &DL)
static Align	getPrefTypeAlign (EVT VT, SelectionDAG &DAG)
static Intrinsic::ID	getIntrinsicForMaskedAtomicRMWBinOp (unsigned XLen, AtomicRMWInst::BinOp BinOp)
static Value *	useTpOffset (IRBuilderBase &IRB, unsigned Offset)

Variables
static cl::opt< unsigned >	ExtensionMaxWebSize (DEBUG_TYPE "-ext-max-web-size", cl::Hidden, cl::desc("Give the maximum size (in number of nodes) of the web of " "instructions that we will consider for VW expansion"), cl::init(18))
static cl::opt< bool >	AllowSplatInVW_W (DEBUG_TYPE "-form-vw-w-with-splat", cl::Hidden, cl::desc("Allow the formation of VW_W operations (e.g., " "VWADD_W) with splat constants"), cl::init(false))
static cl::opt< unsigned >	NumRepeatedDivisors (DEBUG_TYPE "-fp-repeated-divisors", cl::Hidden, cl::desc("Set the minimum number of repetitions of a divisor to allow " "transformation to multiplications by the reciprocal"), cl::init(2))
static cl::opt< int >	FPImmCost (DEBUG_TYPE "-fpimm-cost", cl::Hidden, cl::desc("Give the maximum number of instructions that we will " "use for creating a floating-point immediate value"), cl::init(2))
static cl::opt< bool >	RV64LegalI32 ("riscv-experimental-rv64-legal-i32", cl::ReallyHidden, cl::desc("Make i32 a legal type for SelectionDAG on RV64."))
static const MCPhysReg	ArgGPRs []
static const MCPhysReg	ArgFPR16s []
static const MCPhysReg	ArgFPR32s []
static const MCPhysReg	ArgFPR64s []
static const MCPhysReg	ArgVRs []
static const MCPhysReg	ArgVRM2s []
static const MCPhysReg	ArgVRM4s []
static const MCPhysReg	ArgVRM8s [] = {RISCV::V8M8, RISCV::V16M8}
static const Intrinsic::ID	FixedVlsegIntrIds []
static const Intrinsic::ID	FixedVssegIntrIds []

Macro Definition Documentation

DEBUG_TYPE

#define DEBUG_TYPE   "riscv-lower"

Definition at line 50 of file RISCVISelLowering.cpp.

GET_REGISTER_MATCHER

#define GET_REGISTER_MATCHER

Definition at line 19625 of file RISCVISelLowering.cpp.

GET_RISCVVIntrinsicsTable_IMPL

#define GET_RISCVVIntrinsicsTable_IMPL

Definition at line 19763 of file RISCVISelLowering.cpp.

NODE_NAME_CASE

#define NODE_NAME_CASE

(

NODE

)

Value:

  case RISCVISD::NODE:                                                         \
    return "RISCVISD::" #NODE;

OP_CASE

#define OP_CASE

(

NODE

)

Value:

  case ISD::NODE:                                                              \
    return RISCVISD::NODE##_VL;

VP_CASE

#define VP_CASE

(

NODE

)

Value:

  case ISD::VP_##NODE:                                                         \
    return RISCVISD::NODE##_VL;

Function Documentation

allocateRVVReg()

static unsigned allocateRVVReg ( MVT  ValVT, unsigned  ValNo, std::optional< unsigned >  FirstMaskArgument, CCState &  State, const RISCVTargetLowering &  TLI  )

static

Definition at line 16773 of file RISCVISelLowering.cpp.

References llvm::CCState::AllocateReg(), ArgVRM2s, ArgVRM4s, ArgVRM8s, ArgVRs, llvm::TargetLoweringBase::getRegClassFor(), and llvm_unreachable.

Referenced by llvm::RISCV::CC_RISCV(), and llvm::RISCV::CC_RISCV_FastCC().

CC_RISCVAssign2XLen()

static bool CC_RISCVAssign2XLen ( unsigned  XLen, CCState &  State, CCValAssign  VA1, ISD::ArgFlagsTy  ArgFlags1, unsigned  ValNo2, MVT  ValVT2, MVT  LocVT2, ISD::ArgFlagsTy  ArgFlags2  )

static

Definition at line 16736 of file RISCVISelLowering.cpp.

References llvm::CCState::addLoc(), llvm::CCState::AllocateReg(), llvm::CCState::AllocateStack(), ArgGPRs, llvm::CCValAssign::Full, llvm::CCValAssign::getLocVT(), llvm::CCValAssign::getMem(), llvm::ISD::ArgFlagsTy::getNonZeroOrigAlign(), llvm::CCValAssign::getReg(), llvm::CCValAssign::getValNo(), and llvm::CCValAssign::getValVT().

Referenced by llvm::RISCV::CC_RISCV().

combine_CC()

static bool combine_CC ( SDValue &  LHS, SDValue &  RHS, SDValue &  CC, const SDLoc &  DL, SelectionDAG &  DAG, const RISCVSubtarget &  Subtarget  )

static

Definition at line 14013 of file RISCVISelLowering.cpp.

References llvm::ISD::AND, CC, llvm::ISD::Constant, DL, llvm::APInt::getBitsSetFrom(), llvm::SelectionDAG::getCondCode(), llvm::SelectionDAG::getConstant(), llvm::SDValue::getConstantOperandVal(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::ISD::getSetCCInverse(), llvm::RISCVSubtarget::getXLenVT(), llvm::Value::hasOneUse(), llvm::ISD::isIntEqualitySetCC(), llvm::isNullConstant(), llvm::isOneConstant(), llvm::isPowerOf2_64(), LHS, llvm::Log2_64(), llvm::SelectionDAG::MaskedValueIsZero(), RHS, llvm::ISD::SETCC, llvm::ISD::SETEQ, llvm::ISD::SETGE, llvm::ISD::SETLT, llvm::ISD::SHL, llvm::ISD::SRA, llvm::ISD::SRL, translateSetCCForBranch(), tryDemorganOfBooleanCondition(), and llvm::ISD::XOR.

Referenced by llvm::RISCVTargetLowering::PerformDAGCombine().

combineAddOfBooleanXor()

static SDValue combineAddOfBooleanXor ( SDNode *  N, SelectionDAG &  DAG  )

static

Definition at line 12162 of file RISCVISelLowering.cpp.

References DL, llvm::APInt::getBitsSetFrom(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::EVT::getSizeInBits(), llvm::isAllOnesConstant(), llvm::isOneConstant(), llvm::SelectionDAG::MaskedValueIsZero(), N, llvm::ISD::SUB, and llvm::ISD::XOR.

Referenced by performADDCombine().

combineBinOp_VLToVWBinOp_VL()

static SDValue combineBinOp_VLToVWBinOp_VL ( SDNode *  N, TargetLowering::DAGCombinerInfo &  DCI  )

static

Combine a binary operation to its equivalent VW or VW_W form.

The supported combines are: add_vl -> vwadd(u) | vwadd(u)_w sub_vl -> vwsub(u) | vwsub(u)_w mul_vl -> vwmul(u) | vwmul_su vwadd_w(u) -> vwadd(u) vwub_w(u) -> vwadd(u)

Definition at line 13184 of file RISCVISelLowering.cpp.

References llvm::TargetLowering::DAGCombinerInfo::AddToWorklist(), assert(), llvm::TargetLowering::DAGCombinerInfo::DAG, llvm::SmallVectorImpl< T >::emplace_back(), llvm::SmallVectorBase< Size_T >::empty(), ExtensionMaxWebSize, LHS, N, llvm::SmallVectorImpl< T >::pop_back_val(), llvm::SmallVectorTemplateBase< T, bool >::push_back(), llvm::SelectionDAG::ReplaceAllUsesOfValueWith(), llvm::SmallVectorImpl< T >::reserve(), RHS, llvm::SmallVectorBase< Size_T >::size(), and std::swap().

Referenced by llvm::RISCVTargetLowering::PerformDAGCombine().

combineBinOpOfExtractToReduceTree()

static SDValue combineBinOpOfExtractToReduceTree ( SDNode *  N, SelectionDAG &  DAG, const RISCVSubtarget &  Subtarget  )

static

Perform two related transforms whose purpose is to incrementally recognize an explode_vector followed by scalar reduction as a vector reduction node.

This exists to recover from a deficiency in SLP which can't handle forests with multiple roots sharing common nodes. In some cases, one of the trees will be vectorized, and the other will remain (unprofitably) scalarized.

Definition at line 11759 of file RISCVISelLowering.cpp.

References assert(), DL, llvm::ISD::EXTRACT_SUBVECTOR, llvm::ISD::EXTRACT_VECTOR_ELT, llvm::ISD::FADD, llvm::SelectionDAG::getContext(), llvm::RISCVSubtarget::getELen(), llvm::SDNode::getFlags(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::EVT::getScalarSizeInBits(), llvm::SDValue::getValueType(), llvm::ISD::getVecReduceBaseOpcode(), getVecReduceOpcode(), llvm::EVT::getVectorElementType(), llvm::SelectionDAG::getVectorIdxConstant(), llvm::EVT::getVectorNumElements(), llvm::EVT::getVectorVT(), llvm::SDValue::hasOneUse(), llvm::Value::hasOneUse(), llvm::RISCVSubtarget::hasVInstructions(), llvm::EVT::isInteger(), llvm::isNullConstant(), llvm::EVT::isScalableVector(), LHS, N, llvm::SelectionDAG::NewNodesMustHaveLegalTypes, RHS, and std::swap().

Referenced by performADDCombine(), performANDCombine(), llvm::RISCVTargetLowering::PerformDAGCombine(), performORCombine(), and performXORCombine().

combineBinOpToReduce()

static SDValue combineBinOpToReduce ( SDNode *  N, SelectionDAG &  DAG, const RISCVSubtarget &  Subtarget  )

static

Definition at line 11855 of file RISCVISelLowering.cpp.

References llvm::ISD::ADD, llvm::ISD::AND, DL, llvm::ISD::EXTRACT_VECTOR_ELT, llvm::ISD::FADD, llvm::ISD::FMAXNUM, llvm::ISD::FMINNUM, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDValue::getSimpleValueType(), llvm::SelectionDAG::getUNDEF(), llvm::SDValue::getValueType(), llvm::RISCVSubtarget::getXLenVT(), llvm::SDValue::hasOneUse(), llvm::ISD::INSERT_SUBVECTOR, llvm::isNeutralConstant(), isNonZeroAVL(), llvm::isNullConstant(), llvm::SDNode::isUndef(), llvm_unreachable, lowerScalarInsert(), N, llvm::ISD::OR, llvm::ISD::SMAX, llvm::ISD::SMIN, llvm::ISD::UMAX, llvm::ISD::UMIN, llvm::RISCVISD::VECREDUCE_ADD_VL, llvm::RISCVISD::VECREDUCE_AND_VL, llvm::RISCVISD::VECREDUCE_FADD_VL, llvm::RISCVISD::VECREDUCE_FMAX_VL, llvm::RISCVISD::VECREDUCE_FMIN_VL, llvm::RISCVISD::VECREDUCE_OR_VL, llvm::RISCVISD::VECREDUCE_SMAX_VL, llvm::RISCVISD::VECREDUCE_SMIN_VL, llvm::RISCVISD::VECREDUCE_UMAX_VL, llvm::RISCVISD::VECREDUCE_UMIN_VL, llvm::RISCVISD::VECREDUCE_XOR_VL, llvm::RISCVISD::VFMV_S_F_VL, llvm::RISCVISD::VMV_S_X_VL, llvm::RISCVISD::VMV_V_X_VL, and llvm::ISD::XOR.

Referenced by performADDCombine(), performANDCombine(), llvm::RISCVTargetLowering::PerformDAGCombine(), performORCombine(), and performXORCombine().

combineDeMorganOfBoolean()

static SDValue combineDeMorganOfBoolean ( SDNode *  N, SelectionDAG &  DAG  )

static

Definition at line 12273 of file RISCVISelLowering.cpp.

References llvm::ISD::AND, DL, llvm::APInt::getBitsSetFrom(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::EVT::getSizeInBits(), llvm::SDValue::hasOneUse(), llvm::isAllOnesConstant(), llvm::isOneConstant(), llvm::SelectionDAG::MaskedValueIsZero(), N, llvm::ISD::OR, and llvm::ISD::XOR.

Referenced by performANDCombine(), and performORCombine().

combineOrOfCZERO()

static SDValue combineOrOfCZERO ( SDNode *  N, SDValue  N0, SDValue  N1, SelectionDAG &  DAG  )

static

Definition at line 12383 of file RISCVISelLowering.cpp.

References assert(), Cond, llvm::RISCVISD::CZERO_EQZ, llvm::RISCVISD::CZERO_NEZ, DL, llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDValue::hasOneUse(), llvm::isOneConstant(), N, llvm::ISD::OR, and llvm::ISD::XOR.

Referenced by performORCombine().

combineSelectAndUse()

static SDValue combineSelectAndUse ( SDNode *  N, SDValue  Slct, SDValue  OtherOp, SelectionDAG &  DAG, bool  AllOnes, const RISCVSubtarget &  Subtarget  )

static

Definition at line 12017 of file RISCVISelLowering.cpp.

References llvm::AllOnes, llvm::ISD::AND, llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::EVT::getSizeInBits(), llvm::RISCVSubtarget::getXLen(), llvm::SDValue::hasOneUse(), llvm::isAllOnesConstant(), llvm::isNullConstant(), llvm::EVT::isVector(), isZeroOrAllOnes(), N, llvm::ISD::SELECT, llvm::RISCVISD::SELECT_CC, and std::swap().

combineSelectAndUseCommutative()

static SDValue combineSelectAndUseCommutative ( SDNode *  N, SelectionDAG &  DAG, bool  AllOnes, const RISCVSubtarget &  Subtarget  )

static

Definition at line 12082 of file RISCVISelLowering.cpp.

References llvm::AllOnes, combineSelectAndUse(), and N.

combineSelectToBinOp()

static SDValue combineSelectToBinOp ( SDNode *  N, SelectionDAG &  DAG, const RISCVSubtarget &  Subtarget  )

static

Definition at line 6870 of file RISCVISelLowering.cpp.

References llvm::ISD::ADD, llvm::ISD::AND, CC, DL, llvm::SelectionDAG::getAllOnesConstant(), llvm::SelectionDAG::getNegative(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::isAllOnesConstant(), llvm::isNullConstant(), LHS, matchSetCC(), N, llvm::ISD::OR, RHS, and llvm::ISD::SETCC.

combineSubOfBoolean()

static SDValue combineSubOfBoolean ( SDNode *  N, SelectionDAG &  DAG  )

static

Definition at line 12205 of file RISCVISelLowering.cpp.

References llvm::ISD::ADD, DL, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SelectionDAG::getSetCC(), llvm::ISD::getSetCCInverse(), llvm::SDValue::getValueType(), llvm::SDValue::hasOneUse(), llvm::EVT::isInteger(), llvm::isOneConstant(), llvm::APInt::isSignedIntN(), N, llvm::ISD::SETCC, and llvm::ISD::XOR.

Referenced by performSUBCombine().

combineToVWMACC()

static SDValue combineToVWMACC ( SDNode *  N, SelectionDAG &  DAG, const RISCVSubtarget &  Subtarget  )

static

Definition at line 14421 of file RISCVISelLowering.cpp.

References llvm::RISCVISD::ADD_VL, assert(), DL, llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDValue::isUndef(), N, std::swap(), llvm::RISCVISD::VWMACC_VL, llvm::RISCVISD::VWMACCSU_VL, llvm::RISCVISD::VWMACCU_VL, llvm::RISCVISD::VWMUL_VL, llvm::RISCVISD::VWMULSU_VL, and llvm::RISCVISD::VWMULU_VL.

Referenced by llvm::RISCVTargetLowering::PerformDAGCombine().

combineVFMADD_VLWithVFNEG_VL()

static SDValue combineVFMADD_VLWithVFNEG_VL ( SDNode *  N, SelectionDAG &  DAG  )

static

Definition at line 13642 of file RISCVISelLowering.cpp.

References A, B, llvm::CallingConv::C, llvm::RISCVISD::FNEG_VL, llvm::SelectionDAG::getNode(), N, negateFMAOpcode(), and llvm::Offset.

Referenced by performVFMADD_VLCombine().

computeGREVOrGORC()

static uint64_t computeGREVOrGORC ( uint64_t  x, unsigned  ShAmt, bool  IsGORC  )

static

Definition at line 15683 of file RISCVISelLowering.cpp.

Referenced by llvm::RISCVTargetLowering::computeKnownBitsForTargetNode().

convertFromScalableVector()

static SDValue convertFromScalableVector ( EVT  VT, SDValue  V, SelectionDAG &  DAG, const RISCVSubtarget &  Subtarget  )

static

Definition at line 2557 of file RISCVISelLowering.cpp.

References assert(), DL, llvm::ISD::EXTRACT_SUBVECTOR, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::RISCVSubtarget::getXLenVT(), and llvm::EVT::isFixedLengthVector().

convertLocVTToValVT()

static SDValue convertLocVTToValVT ( SelectionDAG &  DAG, SDValue  Val, const CCValAssign &  VA, const SDLoc &  DL, const RISCVSubtarget &  Subtarget  )

static

Definition at line 17098 of file RISCVISelLowering.cpp.

References llvm::CCValAssign::BCvt, llvm::ISD::BITCAST, convertFromScalableVector(), DL, llvm::RISCVISD::FMV_H_X, llvm::RISCVISD::FMV_W_X_RV64, llvm::CCValAssign::Full, llvm::CCValAssign::getLocInfo(), llvm::CCValAssign::getLocVT(), llvm::SelectionDAG::getNode(), llvm::CCValAssign::getValVT(), llvm::MVT::isFixedLengthVector(), llvm::MVT::isInteger(), llvm::MVT::isScalableVector(), llvm_unreachable, RV64LegalI32, and llvm::ISD::TRUNCATE.

convertToScalableVector()

static SDValue convertToScalableVector ( EVT  VT, SDValue  V, SelectionDAG &  DAG, const RISCVSubtarget &  Subtarget  )

static

Definition at line 2545 of file RISCVISelLowering.cpp.

References assert(), DL, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getUNDEF(), llvm::RISCVSubtarget::getXLenVT(), llvm::ISD::INSERT_SUBVECTOR, and llvm::EVT::isScalableVector().

convertValVTToLocVT()

static SDValue convertValVTToLocVT ( SelectionDAG &  DAG, SDValue  Val, const CCValAssign &  VA, const SDLoc &  DL, const RISCVSubtarget &  Subtarget  )

static

Definition at line 17162 of file RISCVISelLowering.cpp.

References llvm::ISD::ANY_EXTEND, llvm::CCValAssign::BCvt, llvm::ISD::BITCAST, convertToScalableVector(), DL, llvm::RISCVISD::FMV_X_ANYEXTH, llvm::RISCVISD::FMV_X_ANYEXTW_RV64, llvm::CCValAssign::Full, llvm::CCValAssign::getLocInfo(), llvm::CCValAssign::getLocVT(), llvm::SelectionDAG::getNode(), llvm::CCValAssign::getValVT(), llvm::MVT::isFixedLengthVector(), llvm::EVT::isInteger(), llvm::EVT::isScalableVector(), llvm_unreachable, and RV64LegalI32.

customLegalizeToWOp()

static SDValue customLegalizeToWOp ( SDNode *  N, SelectionDAG &  DAG, unsigned  ExtOpc = ISD::ANY_EXTEND  )

static

Definition at line 11030 of file RISCVISelLowering.cpp.

References DL, llvm::SelectionDAG::getNode(), getRISCVWOpcode(), N, and llvm::ISD::TRUNCATE.

customLegalizeToWOpWithSExt()

static SDValue customLegalizeToWOpWithSExt ( SDNode *  N, SelectionDAG &  DAG  )

static

Definition at line 11043 of file RISCVISelLowering.cpp.

References llvm::ISD::ANY_EXTEND, DL, llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getValueType(), N, llvm::ISD::SIGN_EXTEND_INREG, and llvm::ISD::TRUNCATE.

Referenced by llvm::RISCVTargetLowering::ReplaceNodeResults().

emitBuildPairF64Pseudo()

static MachineBasicBlock * emitBuildPairF64Pseudo ( MachineInstr &  MI, MachineBasicBlock *  BB, const RISCVSubtarget &  Subtarget  )

static

Definition at line 16052 of file RISCVISelLowering.cpp.

References llvm::MachineInstrBuilder::addFrameIndex(), llvm::MachineInstrBuilder::addImm(), llvm::MachineInstrBuilder::addMemOperand(), llvm::MachineInstrBuilder::addReg(), assert(), llvm::BuildMI(), DL, llvm::MachinePointerInfo::getFixedStack(), llvm::MachineFunction::getInfo(), llvm::TargetSubtargetInfo::getInstrInfo(), llvm::getKillRegState(), llvm::MachineFunction::getMachineMemOperand(), llvm::MachineBasicBlock::getParent(), llvm::TargetSubtargetInfo::getRegisterInfo(), llvm::MachineFunction::getSubtarget(), llvm::MachinePointerInfo::getWithOffset(), llvm::HexagonInstrInfo::loadRegFromStackSlot(), MI, llvm::MachineMemOperand::MOStore, and TII.

Referenced by llvm::RISCVTargetLowering::EmitInstrWithCustomInserter().

emitFROUND()

static MachineBasicBlock * emitFROUND ( MachineInstr &  MI, MachineBasicBlock *  MBB, const RISCVSubtarget &  Subtarget  )

static

Definition at line 16438 of file RISCVISelLowering.cpp.

References llvm::MachineInstrBuilder::addImm(), llvm::MachineInstrBuilder::addMBB(), llvm::MachineInstrBuilder::addReg(), llvm::MachineBasicBlock::addSuccessor(), assert(), llvm::MachineBasicBlock::begin(), llvm::BuildMI(), DL, llvm::MachineBasicBlock::end(), F, llvm::MachineBasicBlock::getBasicBlock(), llvm::RISCVSubtarget::getInstrInfo(), llvm::ilist_node_impl< OptionsT >::getIterator(), llvm::MachineBasicBlock::getParent(), llvm::MachineFunction::getRegInfo(), I, llvm::RISCVSubtarget::is64Bit(), llvm_unreachable, MBB, MI, MRI, llvm::MachineInstr::NoFPExcept, llvm::MachineInstr::setFlag(), llvm::MachineBasicBlock::splice(), TII, and llvm::MachineBasicBlock::transferSuccessorsAndUpdatePHIs().

Referenced by llvm::RISCVTargetLowering::EmitInstrWithCustomInserter().

EmitLoweredCascadedSelect()

static MachineBasicBlock * EmitLoweredCascadedSelect ( MachineInstr &  First, MachineInstr &  Second, MachineBasicBlock *  ThisMBB, const RISCVSubtarget &  Subtarget  )

static

Definition at line 16145 of file RISCVISelLowering.cpp.

References llvm::MachineInstrBuilder::addMBB(), llvm::MachineInstrBuilder::addReg(), llvm::MachineBasicBlock::addSuccessor(), llvm::MachineBasicBlock::begin(), llvm::BuildMI(), DL, llvm::MachineBasicBlock::end(), llvm::MachineInstr::eraseFromParent(), F, llvm::First, llvm::MachineBasicBlock::getBasicBlock(), llvm::MachineOperand::getImm(), llvm::RISCVSubtarget::getInstrInfo(), llvm::ilist_node_impl< OptionsT >::getIterator(), llvm::MachineInstr::getOperand(), llvm::MachineBasicBlock::getParent(), llvm::MachineOperand::getReg(), llvm::MachineBasicBlock::splice(), TII, and llvm::MachineBasicBlock::transferSuccessorsAndUpdatePHIs().

Referenced by emitSelectPseudo().

emitQuietFCMP()

static MachineBasicBlock * emitQuietFCMP ( MachineInstr &  MI, MachineBasicBlock *  BB, unsigned  RelOpcode, unsigned  EqOpcode, const RISCVSubtarget &  Subtarget  )

static

Definition at line 16108 of file RISCVISelLowering.cpp.

References llvm::MachineInstrBuilder::addReg(), llvm::BuildMI(), DL, llvm::TargetSubtargetInfo::getInstrInfo(), llvm::getKillRegState(), llvm::MachineBasicBlock::getParent(), llvm::MachineFunction::getRegInfo(), llvm::MachineFunction::getSubtarget(), llvm::RegState::Kill, MI, MRI, llvm::MachineInstr::NoFPExcept, llvm::MachineInstr::setFlag(), and TII.

Referenced by llvm::RISCVTargetLowering::EmitInstrWithCustomInserter().

emitReadCycleWidePseudo()

static MachineBasicBlock * emitReadCycleWidePseudo ( MachineInstr &  MI, MachineBasicBlock *  BB  )

static

Definition at line 15951 of file RISCVISelLowering.cpp.

References llvm::MachineInstrBuilder::addImm(), llvm::MachineInstrBuilder::addMBB(), llvm::MachineInstrBuilder::addReg(), llvm::MachineBasicBlock::addSuccessor(), assert(), llvm::MachineBasicBlock::begin(), llvm::BuildMI(), llvm::MachineFunction::CreateMachineBasicBlock(), DL, llvm::MachineBasicBlock::end(), llvm::MachineBasicBlock::getBasicBlock(), llvm::TargetSubtargetInfo::getInstrInfo(), llvm::ilist_node_impl< OptionsT >::getIterator(), llvm::MachineBasicBlock::getParent(), llvm::MachineFunction::getRegInfo(), llvm::MachineFunction::getSubtarget(), llvm::MachineFunction::insert(), MI, llvm::MachineBasicBlock::splice(), TII, and llvm::MachineBasicBlock::transferSuccessorsAndUpdatePHIs().

Referenced by llvm::RISCVTargetLowering::EmitInstrWithCustomInserter().

emitSelectPseudo()

static MachineBasicBlock * emitSelectPseudo ( MachineInstr &  MI, MachineBasicBlock *  BB, const RISCVSubtarget &  Subtarget  )

static

Definition at line 16247 of file RISCVISelLowering.cpp.

References llvm::MachineInstrBuilder::addMBB(), llvm::MachineInstrBuilder::addReg(), llvm::MachineBasicBlock::addSuccessor(), llvm::any_of(), llvm::MachineBasicBlock::begin(), llvm::BuildMI(), CC, llvm::MachineInstr::collectDebugValues(), llvm::SmallSet< T, N, C >::count(), DL, E, EmitLoweredCascadedSelect(), llvm::MachineBasicBlock::end(), llvm::MachineInstr::eraseFromParent(), F, llvm::MachineBasicBlock::getBasicBlock(), llvm::RISCVSubtarget::getInstrInfo(), llvm::ilist_node_impl< OptionsT >::getIterator(), llvm::MachineBasicBlock::getParent(), I, llvm::SmallSet< T, N, C >::insert(), llvm::MachineBasicBlock::instr_end(), isSelectPseudo(), LHS, MI, llvm::next_nodbg(), llvm::MachineFunctionProperties::NoPHIs, llvm::MachineBasicBlock::push_back(), RHS, llvm::MachineBasicBlock::splice(), TII, and llvm::MachineBasicBlock::transferSuccessorsAndUpdatePHIs().

emitSplitF64Pseudo()

static MachineBasicBlock * emitSplitF64Pseudo ( MachineInstr &  MI, MachineBasicBlock *  BB, const RISCVSubtarget &  Subtarget  )

static

Definition at line 16013 of file RISCVISelLowering.cpp.

References llvm::MachineInstrBuilder::addFrameIndex(), llvm::MachineInstrBuilder::addImm(), llvm::MachineInstrBuilder::addMemOperand(), assert(), llvm::BuildMI(), DL, llvm::MachinePointerInfo::getFixedStack(), llvm::MachineFunction::getInfo(), llvm::TargetSubtargetInfo::getInstrInfo(), llvm::MachineFunction::getMachineMemOperand(), llvm::MachineBasicBlock::getParent(), llvm::TargetSubtargetInfo::getRegisterInfo(), llvm::MachineFunction::getSubtarget(), llvm::MachinePointerInfo::getWithOffset(), MI, llvm::MachineMemOperand::MOLoad, llvm::HexagonInstrInfo::storeRegToStackSlot(), and TII.

Referenced by llvm::RISCVTargetLowering::EmitInstrWithCustomInserter().

emitVFROUND_NOEXCEPT_MASK()

static MachineBasicBlock * emitVFROUND_NOEXCEPT_MASK ( MachineInstr &  MI, MachineBasicBlock *  BB, unsigned  CVTXOpc, unsigned  CVTFOpc  )

static

Definition at line 16382 of file RISCVISelLowering.cpp.

References llvm::MachineInstrBuilder::add(), llvm::MachineInstrBuilder::addReg(), assert(), llvm::BuildMI(), llvm::MachineOperand::CreateImm(), llvm::MachineOperand::CreateReg(), DL, llvm::TargetSubtargetInfo::getInstrInfo(), llvm::MachineBasicBlock::getParent(), llvm::MachineFunction::getRegInfo(), llvm::TargetSubtargetInfo::getRegisterInfo(), llvm::MachineFunction::getSubtarget(), llvm::RegState::Kill, MI, MRI, TII, and TRI.

Referenced by llvm::RISCVTargetLowering::EmitInstrWithCustomInserter().

foldBinOpIntoSelectIfProfitable()

static SDValue foldBinOpIntoSelectIfProfitable ( SDNode *  BO, SelectionDAG &  DAG, const RISCVSubtarget &  Subtarget  )

static

Definition at line 6936 of file RISCVISelLowering.cpp.

References DL, llvm::SelectionDAG::FoldConstantArithmetic(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDNode::getOpcode(), llvm::SDValue::getOperand(), llvm::SDNode::getOperand(), llvm::SelectionDAG::getSelect(), llvm::SDNode::getValueType(), llvm::SDValue::hasOneUse(), llvm::APInt::isAllOnes(), llvm::ConstantSDNode::isOpaque(), llvm::APInt::isZero(), llvm::ISD::SELECT, and std::swap().

foldSelectOfCTTZOrCTLZ()

static SDValue foldSelectOfCTTZOrCTLZ ( SDNode *  N, SelectionDAG &  DAG  )

static

Definition at line 14156 of file RISCVISelLowering.cpp.

References llvm::ISD::AND, llvm::BitWidth, Cond, llvm::ISD::CTLZ, llvm::ISD::CTLZ_ZERO_UNDEF, llvm::ISD::CTTZ, llvm::ISD::CTTZ_ZERO_UNDEF, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDNode::getOperand(), llvm::SDValue::getValueSizeInBits(), llvm::SDValue::getValueType(), llvm::SelectionDAG::getZExtOrTrunc(), llvm::isNullConstant(), N, llvm::ISD::SETCC, llvm::ISD::SETEQ, llvm::ISD::SETNE, llvm::ISD::TRUNCATE, and llvm::ISD::ZERO_EXTEND.

Referenced by performSELECTCombine().

getAllOnesMask()

static SDValue getAllOnesMask ( MVT  VecVT, SDValue  VL, const SDLoc &  DL, SelectionDAG &  DAG  )

static

Creates an all ones mask suitable for masking a vector of type VecTy with vector length VL.

Definition at line 2579 of file RISCVISelLowering.cpp.

References DL, getMaskTypeFor(), llvm::SelectionDAG::getNode(), and llvm::RISCVISD::VMSET_VL.

Referenced by getDefaultScalableVLOps(), getDefaultVLOps(), and lowerVectorIntrinsicScalars().

getContainerForFixedLengthVector() [1/2]

static MVT getContainerForFixedLengthVector ( const TargetLowering &  TLI, MVT  VT, const RISCVSubtarget &  Subtarget  )

static

Definition at line 2500 of file RISCVISelLowering.cpp.

References assert(), llvm::RISCVSubtarget::getELen(), llvm::RISCVSubtarget::getRealMinVLen(), llvm::MVT::getScalableVectorVT(), llvm::MVT::getVectorElementType(), llvm::MVT::getVectorNumElements(), llvm::MVT::isFixedLengthVector(), llvm::isPowerOf2_32(), llvm::TargetLoweringBase::isTypeLegal(), llvm_unreachable, llvm::RISCV::RVVBitsPerBlock, llvm::MVT::SimpleTy, and useRVVForFixedLengthVectorVT().

getContainerForFixedLengthVector() [2/2]

static MVT getContainerForFixedLengthVector ( SelectionDAG &  DAG, MVT  VT, const RISCVSubtarget &  Subtarget  )

static

Definition at line 2534 of file RISCVISelLowering.cpp.

References getContainerForFixedLengthVector(), and llvm::SelectionDAG::getTargetLoweringInfo().

getDefaultScalableVLOps()

static std::pair< SDValue, SDValue > getDefaultScalableVLOps ( MVT  VecVT, const SDLoc &  DL, SelectionDAG &  DAG, const RISCVSubtarget &  Subtarget  )

static

Definition at line 2591 of file RISCVISelLowering.cpp.

References assert(), DL, getAllOnesMask(), llvm::SelectionDAG::getRegister(), llvm::RISCVSubtarget::getXLenVT(), and llvm::MVT::isScalableVector().

Referenced by getDefaultVLOps().

getDefaultVLOps() [1/2]

static std::pair< SDValue, SDValue > getDefaultVLOps ( MVT  VecVT, MVT  ContainerVT, const SDLoc &  DL, SelectionDAG &  DAG, const RISCVSubtarget &  Subtarget  )

static

Definition at line 2613 of file RISCVISelLowering.cpp.

References assert(), DL, getDefaultScalableVLOps(), getDefaultVLOps(), llvm::MVT::getVectorNumElements(), llvm::MVT::isFixedLengthVector(), and llvm::MVT::isScalableVector().

getDefaultVLOps() [2/2]

static std::pair< SDValue, SDValue > getDefaultVLOps ( uint64_t  NumElts, MVT  ContainerVT, const SDLoc &  DL, SelectionDAG &  DAG, const RISCVSubtarget &  Subtarget  )

static

Definition at line 2600 of file RISCVISelLowering.cpp.

References assert(), DL, getAllOnesMask(), getVLOp(), and llvm::MVT::isScalableVector().

Referenced by getDefaultVLOps(), getDeinterleaveViaVNSRL(), getWideningInterleave(), lowerBUILD_VECTOR(), lowerBuildVectorOfConstants(), lowerBuildVectorViaDominantValues(), lowerFMAXIMUM_FMINIMUM(), lowerFP_TO_INT_SAT(), llvm::RISCVTargetLowering::LowerOperation(), lowerVECTOR_SHUFFLE(), lowerVECTOR_SHUFFLEAsVSlide1(), lowerVECTOR_SHUFFLEAsVSlidedown(), lowerVECTOR_SHUFFLEAsVSlideup(), lowerVectorFTRUNC_FCEIL_FFLOOR_FROUND(), lowerVectorStrictFTRUNC_FCEIL_FFLOOR_FROUND(), lowerVectorXRINT(), matchSplatAsGather(), performFP_TO_INTCombine(), and llvm::RISCVTargetLowering::ReplaceNodeResults().

getDeinterleaveViaVNSRL()

static SDValue getDeinterleaveViaVNSRL ( const SDLoc &  DL, MVT  VT, SDValue  Src, bool  EvenElts, const RISCVSubtarget &  Subtarget, SelectionDAG &  DAG  )

static

Definition at line 4201 of file RISCVISelLowering.cpp.

References assert(), llvm::MVT::changeVectorElementTypeToInteger(), convertFromScalableVector(), convertToScalableVector(), DL, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getConstant(), getContainerForFixedLengthVector(), getDefaultVLOps(), llvm::MVT::getIntegerVT(), llvm::SelectionDAG::getNode(), llvm::MVT::getScalarSizeInBits(), llvm::SelectionDAG::getUNDEF(), llvm::MVT::getVectorElementCount(), llvm::MVT::getVectorElementType(), llvm::MVT::getVectorVT(), llvm::RISCVSubtarget::getXLenVT(), llvm::MVT::isFixedLengthVector(), llvm::RISCVISD::VMV_V_X_VL, and llvm::RISCVISD::VNSRL_VL.

Referenced by lowerVECTOR_SHUFFLE().

getExactInteger()

static std::optional< uint64_t > getExactInteger ( const APFloat &  APF, uint32_t  BitWidth  )

static

Definition at line 3122 of file RISCVISelLowering.cpp.

References llvm::BitWidth, llvm::APFloat::convertToInteger(), llvm::APInt::extractBitsAsZExtValue(), llvm::APFloat::isNegative(), llvm::APFloatBase::opInvalidOp, and llvm::TowardZero.

Referenced by isSimpleVIDSequence().

getIntrinsicForMaskedAtomicRMWBinOp()

static Intrinsic::ID getIntrinsicForMaskedAtomicRMWBinOp ( unsigned  XLen, AtomicRMWInst::BinOp  BinOp  )

static

Definition at line 18744 of file RISCVISelLowering.cpp.

References llvm::AtomicRMWInst::Add, llvm_unreachable, llvm::AtomicRMWInst::Max, llvm::AtomicRMWInst::Min, llvm::AtomicRMWInst::Nand, llvm::AtomicRMWInst::Sub, llvm::AtomicRMWInst::UMax, llvm::AtomicRMWInst::UMin, and llvm::AtomicRMWInst::Xchg.

getLMUL1VT()

static MVT getLMUL1VT ( MVT  VT )

static

Definition at line 3108 of file RISCVISelLowering.cpp.

References assert(), llvm::MVT::getScalableVectorVT(), llvm::MVT::getSizeInBits(), llvm::MVT::getVectorElementType(), and llvm::RISCV::RVVBitsPerBlock.

Referenced by getSmallestVTForIndex(), lowerBUILD_VECTOR(), lowerReductionSeq(), and llvm::RISCVTargetLowering::PerformDAGCombine().

getMaskTypeFor()

static MVT getMaskTypeFor ( MVT  VecVT )

static

Return the type of the mask type suitable for masking the provided vector type.

This is simply an i1 element type vector of the same (possibly scalable) length.

Definition at line 2571 of file RISCVISelLowering.cpp.

References assert(), llvm::MVT::getVectorElementCount(), llvm::MVT::getVectorVT(), and llvm::MVT::isVector().

Referenced by getAllOnesMask(), lowerVectorFTRUNC_FCEIL_FFLOOR_FROUND(), and llvm::RISCVTargetLowering::PerformDAGCombine().

getPrefTypeAlign()

static Align getPrefTypeAlign ( EVT  VT, SelectionDAG &  DAG  )

static

Definition at line 17672 of file RISCVISelLowering.cpp.

References llvm::SelectionDAG::getContext(), llvm::SelectionDAG::getDataLayout(), llvm::DataLayout::getPrefTypeAlign(), and llvm::EVT::getTypeForEVT().

getRISCVVLOp()

static unsigned getRISCVVLOp ( SDValue  Op )

static

Get a RISC-V target specified VL op for a given SDNode.

Definition at line 5392 of file RISCVISelLowering.cpp.

References llvm::ISD::AND, llvm::RISCVISD::AND_VL, llvm::RISCVISD::CTLZ_VL, llvm::ISD::CTLZ_ZERO_UNDEF, llvm::RISCVISD::CTTZ_VL, llvm::ISD::CTTZ_ZERO_UNDEF, llvm::ISD::FMA, llvm::ISD::FMAXNUM, llvm::ISD::FMINNUM, llvm::RISCVISD::FSQRT_VL, llvm_unreachable, OP_CASE, llvm::ISD::OR, llvm::RISCVISD::OR_VL, ROTR, llvm::RISCVISD::SRA_VL, llvm::RISCVISD::SRL_VL, llvm::ISD::STRICT_FMA, llvm::RISCVISD::STRICT_VFMADD_VL, llvm::RISCVISD::VFCVT_RTZ_X_F_VL, llvm::RISCVISD::VFCVT_RTZ_XU_F_VL, llvm::RISCVISD::VFMADD_VL, llvm::RISCVISD::VFMAX_VL, llvm::RISCVISD::VFMIN_VL, llvm::RISCVISD::VMAND_VL, llvm::RISCVISD::VMOR_VL, llvm::RISCVISD::VMXOR_VL, VP_CASE, llvm::RISCVISD::VP_MERGE_VL, llvm::RISCVISD::VSELECT_VL, llvm::RISCVISD::VSEXT_VL, llvm::RISCVISD::VZEXT_VL, llvm::ISD::XOR, and llvm::RISCVISD::XOR_VL.

getRISCVWOpcode()

static RISCVISD::NodeType getRISCVWOpcode ( unsigned  Opcode )

static

Definition at line 11002 of file RISCVISelLowering.cpp.

References llvm::RISCVISD::DIVUW, llvm::RISCVISD::DIVW, llvm_unreachable, Opcode, llvm::RISCVISD::REMUW, llvm::RISCVISD::ROLW, llvm::RISCVISD::RORW, llvm::ISD::ROTL, llvm::ISD::ROTR, llvm::ISD::SDIV, llvm::ISD::SHL, llvm::RISCVISD::SLLW, llvm::ISD::SRA, llvm::RISCVISD::SRAW, llvm::ISD::SRL, llvm::RISCVISD::SRLW, llvm::ISD::UDIV, and llvm::ISD::UREM.

Referenced by customLegalizeToWOp().

getRVVFPReductionOpAndOperands()

static std::tuple< unsigned, SDValue, SDValue > getRVVFPReductionOpAndOperands ( SDValue  Op, SelectionDAG &  DAG, EVT  EltVT, const RISCVSubtarget &  Subtarget  )

static

Definition at line 9057 of file RISCVISelLowering.cpp.

References DL, llvm::ISD::EXTRACT_VECTOR_ELT, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getConstantFP(), llvm::SelectionDAG::getNode(), llvm::RISCVSubtarget::getXLenVT(), llvm_unreachable, Opcode, llvm::ISD::VECREDUCE_FADD, llvm::RISCVISD::VECREDUCE_FADD_VL, llvm::ISD::VECREDUCE_FMAX, llvm::RISCVISD::VECREDUCE_FMAX_VL, llvm::ISD::VECREDUCE_FMIN, llvm::RISCVISD::VECREDUCE_FMIN_VL, llvm::ISD::VECREDUCE_SEQ_FADD, and llvm::RISCVISD::VECREDUCE_SEQ_FADD_VL.

getRVVReductionOp()

static unsigned getRVVReductionOp ( unsigned  ISDOpcode )

static

Definition at line 8842 of file RISCVISelLowering.cpp.

References llvm_unreachable, llvm::ISD::VECREDUCE_ADD, llvm::RISCVISD::VECREDUCE_ADD_VL, llvm::ISD::VECREDUCE_AND, llvm::RISCVISD::VECREDUCE_AND_VL, llvm::RISCVISD::VECREDUCE_FADD_VL, llvm::RISCVISD::VECREDUCE_FMAX_VL, llvm::RISCVISD::VECREDUCE_FMIN_VL, llvm::ISD::VECREDUCE_OR, llvm::RISCVISD::VECREDUCE_OR_VL, llvm::RISCVISD::VECREDUCE_SEQ_FADD_VL, llvm::ISD::VECREDUCE_SMAX, llvm::RISCVISD::VECREDUCE_SMAX_VL, llvm::ISD::VECREDUCE_SMIN, llvm::RISCVISD::VECREDUCE_SMIN_VL, llvm::ISD::VECREDUCE_UMAX, llvm::RISCVISD::VECREDUCE_UMAX_VL, llvm::ISD::VECREDUCE_UMIN, llvm::RISCVISD::VECREDUCE_UMIN_VL, llvm::ISD::VECREDUCE_XOR, and llvm::RISCVISD::VECREDUCE_XOR_VL.

getSmallestVTForIndex()

static std::optional< MVT > getSmallestVTForIndex ( MVT  VecVT, unsigned  MaxIdx, SDLoc  DL, SelectionDAG &  DAG, const RISCVSubtarget &  Subtarget  )

static

Definition at line 7692 of file RISCVISelLowering.cpp.

References assert(), llvm::MVT::bitsGT(), llvm::MVT::getDoubleNumVectorElementsVT(), getLMUL1VT(), llvm::RISCVSubtarget::getRealMinVLen(), llvm::MVT::getScalarSizeInBits(), llvm::MVT::isScalableVector(), and llvm::MVT::isValid().

getTargetNode() [1/4]

static SDValue getTargetNode ( BlockAddressSDNode *  N, const SDLoc &  DL, EVT  Ty, SelectionDAG &  DAG, unsigned  Flags  )

static

Definition at line 6617 of file RISCVISelLowering.cpp.

References llvm::SelectionDAG::getTargetBlockAddress(), and N.

getTargetNode() [2/4]

static SDValue getTargetNode ( ConstantPoolSDNode *  N, const SDLoc &  DL, EVT  Ty, SelectionDAG &  DAG, unsigned  Flags  )

static

Definition at line 6623 of file RISCVISelLowering.cpp.

References llvm::SelectionDAG::getTargetConstantPool(), and N.

getTargetNode() [3/4]

static SDValue getTargetNode ( GlobalAddressSDNode *  N, const SDLoc &  DL, EVT  Ty, SelectionDAG &  DAG, unsigned  Flags  )

static

Definition at line 6612 of file RISCVISelLowering.cpp.

References DL, llvm::SelectionDAG::getTargetGlobalAddress(), and N.

getTargetNode() [4/4]

static SDValue getTargetNode ( JumpTableSDNode *  N, const SDLoc &  DL, EVT  Ty, SelectionDAG &  DAG, unsigned  Flags  )

static

Definition at line 6629 of file RISCVISelLowering.cpp.

References llvm::SelectionDAG::getTargetJumpTable(), and N.

getVCIXOperands()

static void getVCIXOperands ( SDValue &  Op, SelectionDAG &  DAG, SmallVector< SDValue > &  Ops  )

static

Definition at line 8249 of file RISCVISelLowering.cpp.

References convertToScalableVector(), DL, llvm::SelectionDAG::getBitcast(), getContainerForFixedLengthVector(), llvm::MVT::getIntegerVT(), llvm::SelectionDAG::getMachineFunction(), llvm::MachineFunction::getSubtarget(), llvm::MVT::getVectorVT(), and llvm::SmallVectorTemplateBase< T, bool >::push_back().

getVecReduceOpcode()

static unsigned getVecReduceOpcode ( unsigned  Opc )

static

Given a binary operator, return the associative generic ISD::VECREDUCE_OP which corresponds to it.

Definition at line 11726 of file RISCVISelLowering.cpp.

References llvm::ISD::ADD, llvm::ISD::AND, llvm::ISD::FADD, llvm_unreachable, llvm::ISD::OR, llvm::ISD::SMAX, llvm::ISD::SMIN, llvm::ISD::UMAX, llvm::ISD::UMIN, llvm::ISD::VECREDUCE_ADD, llvm::ISD::VECREDUCE_AND, llvm::ISD::VECREDUCE_FADD, llvm::ISD::VECREDUCE_OR, llvm::ISD::VECREDUCE_SMAX, llvm::ISD::VECREDUCE_SMIN, llvm::ISD::VECREDUCE_UMAX, llvm::ISD::VECREDUCE_UMIN, llvm::ISD::VECREDUCE_XOR, and llvm::ISD::XOR.

Referenced by combineBinOpOfExtractToReduceTree().

getVLOp()

static SDValue getVLOp ( uint64_t  NumElts, const SDLoc &  DL, SelectionDAG &  DAG, const RISCVSubtarget &  Subtarget  )

static

Definition at line 2585 of file RISCVISelLowering.cpp.

References DL, llvm::SelectionDAG::getConstant(), and llvm::RISCVSubtarget::getXLenVT().

Referenced by getDefaultVLOps().

getVLOperand()

static SDValue getVLOperand ( SDValue  Op )

static

Definition at line 2415 of file RISCVISelLowering.cpp.

References assert(), llvm::ISD::INTRINSIC_W_CHAIN, llvm::ISD::INTRINSIC_WO_CHAIN, and llvm::RISCVVIntrinsicsTable::RISCVVIntrinsicInfo::VLOperand.

Referenced by lowerVectorIntrinsicScalars().

getVSlidedown()

static SDValue getVSlidedown ( SelectionDAG &  DAG, const RISCVSubtarget &  Subtarget, const SDLoc &  DL, EVT  VT, SDValue  Merge, SDValue  Op, SDValue  Offset, SDValue  Mask, SDValue  VL, unsigned  Policy = RISCVII::TAIL_UNDISTURBED_MASK_UNDISTURBED  )

static

Definition at line 3085 of file RISCVISelLowering.cpp.

References DL, llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getTargetConstant(), llvm::RISCVSubtarget::getXLenVT(), llvm::RISCVII::MASK_AGNOSTIC, Merge, llvm::Offset, llvm::RISCVII::TAIL_AGNOSTIC, and llvm::RISCVISD::VSLIDEDOWN_VL.

Referenced by lowerBUILD_VECTOR(), lowerVECTOR_SHUFFLE(), lowerVECTOR_SHUFFLEAsVSlidedown(), and llvm::RISCVTargetLowering::ReplaceNodeResults().

getVSlideup()

static SDValue getVSlideup ( SelectionDAG &  DAG, const RISCVSubtarget &  Subtarget, const SDLoc &  DL, EVT  VT, SDValue  Merge, SDValue  Op, SDValue  Offset, SDValue  Mask, SDValue  VL, unsigned  Policy = RISCVII::TAIL_UNDISTURBED_MASK_UNDISTURBED  )

static

Definition at line 3097 of file RISCVISelLowering.cpp.

References DL, llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getTargetConstant(), llvm::RISCVSubtarget::getXLenVT(), llvm::RISCVII::MASK_AGNOSTIC, Merge, llvm::Offset, llvm::RISCVII::TAIL_AGNOSTIC, and llvm::RISCVISD::VSLIDEUP_VL.

Referenced by lowerVECTOR_SHUFFLE(), and lowerVECTOR_SHUFFLEAsVSlideup().

getWideningInterleave()

static SDValue getWideningInterleave ( SDValue  EvenV, SDValue  OddV, const SDLoc &  DL, SelectionDAG &  DAG, const RISCVSubtarget &  Subtarget  )

static

Definition at line 4435 of file RISCVISelLowering.cpp.

References llvm::RISCVISD::ADD_VL, assert(), llvm::MVT::changeTypeToInteger(), convertFromScalableVector(), convertToScalableVector(), DL, llvm::SelectionDAG::getAllOnesConstant(), llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getConstant(), getContainerForFixedLengthVector(), getDefaultVLOps(), llvm::RISCVSubtarget::getELen(), llvm::MVT::getIntegerVT(), llvm::SelectionDAG::getNode(), llvm::MVT::getScalarSizeInBits(), llvm::SDValue::getSimpleValueType(), llvm::SelectionDAG::getSplatVector(), llvm::SelectionDAG::getUNDEF(), llvm::MVT::getVectorElementCount(), llvm::MVT::getVectorElementType(), llvm::MVT::getVectorVT(), llvm::RISCVSubtarget::getXLenVT(), llvm::MVT::isFixedLengthVector(), llvm::details::FixedOrScalableQuantity< LeafTy, ValueTy >::multiplyCoefficientBy(), llvm::RISCVISD::VWADDU_VL, llvm::RISCVISD::VWADDU_W_VL, llvm::RISCVISD::VWMULU_VL, and llvm::RISCVISD::VWSLL_VL.

Referenced by lowerVECTOR_SHUFFLE().

hasMaskOp()

static bool hasMaskOp ( unsigned  Opcode )

static

Return true if a RISC-V target specified op has a mask operand.

Definition at line 5550 of file RISCVISelLowering.cpp.

References assert(), llvm::RISCVISD::FIRST_NUMBER, llvm::ISD::FIRST_TARGET_STRICTFP_OPCODE, llvm::RISCVISD::FIRST_VL_VECTOR_OP, llvm::RISCVISD::LAST_RISCV_STRICTFP_OPCODE, llvm::RISCVISD::LAST_VL_VECTOR_OP, Opcode, llvm::RISCVISD::SETCC_VL, llvm::RISCVISD::STRICT_FADD_VL, llvm::RISCVISD::STRICT_VFROUND_NOEXCEPT_VL, llvm::RISCVISD::TRUNCATE_VECTOR_VL, llvm::RISCVISD::VFIRST_VL, and llvm::RISCVISD::VRGATHER_VX_VL.

hasMergeOp()

static bool hasMergeOp ( unsigned  Opcode )

static

Return true if a RISC-V target specified op has a merge operand.

Definition at line 5526 of file RISCVISelLowering.cpp.

References llvm::RISCVISD::ADD_VL, assert(), llvm::RISCVISD::FCOPYSIGN_VL, llvm::RISCVISD::FIRST_NUMBER, llvm::ISD::FIRST_TARGET_STRICTFP_OPCODE, llvm::RISCVISD::FIRST_VL_VECTOR_OP, llvm::RISCVISD::LAST_RISCV_STRICTFP_OPCODE, llvm::RISCVISD::LAST_VL_VECTOR_OP, Opcode, llvm::RISCVISD::SETCC_VL, llvm::RISCVISD::STRICT_FADD_VL, llvm::RISCVISD::STRICT_FDIV_VL, llvm::RISCVISD::VFMAX_VL, llvm::RISCVISD::VFWSUB_W_VL, and llvm::RISCVISD::VWMUL_VL.

isDeinterleaveShuffle()

static bool isDeinterleaveShuffle ( MVT  VT, MVT  ContainerVT, SDValue  V1, SDValue  V2, ArrayRef< int >  Mask, const RISCVSubtarget &  Subtarget  )

static

Definition at line 4046 of file RISCVISelLowering.cpp.

References llvm::ISD::EXTRACT_SUBVECTOR, llvm::SDValue::getConstantOperandVal(), llvm::RISCVSubtarget::getELen(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), and llvm::MVT::getScalarSizeInBits().

Referenced by lowerVECTOR_SHUFFLE().

isElementRotate()

static int isElementRotate ( int &  LoSrc, int &  HiSrc, ArrayRef< int >  Mask  )

static

Match shuffles that concatenate two vectors, rotate the concatenation, and then extract the original number of elements from the rotated result.

This is equivalent to vector.splice or X86's PALIGNR instruction. The returned rotation amount is for a rotate right, where elements move from higher elements to lower elements. LoSrc indicates the first source vector of the rotate or -1 for undef. HiSrc indicates the second vector of the rotate or -1 for undef. At least one of LoSrc and HiSrc will be 0 or 1 if a rotation is found.

NOTE: We talk about rotate to the right which matches how bit shift and rotate instructions are described where LSBs are on the right, but LLVM IR and the table below write vectors with the lowest elements on the left.

Definition at line 4135 of file RISCVISelLowering.cpp.

References assert(), and Size.

Referenced by llvm::RISCVTargetLowering::isShuffleMaskLegal(), and lowerVECTOR_SHUFFLE().

isInterleaveShuffle()

static bool isInterleaveShuffle ( ArrayRef< int >  Mask, MVT  VT, int &  EvenSrc, int &  OddSrc, const RISCVSubtarget &  Subtarget  )

static

Is this shuffle interleaving contiguous elements from one vector into the even elements and contiguous elements from another vector into the odd elements.

EvenSrc will contain the element that should be in the first even element. OddSrc will contain the element that should be in the first odd element. These can be the first element in a source or the element half way through the source.

Definition at line 4091 of file RISCVISelLowering.cpp.

References assert(), llvm::RISCVSubtarget::getELen(), llvm::MVT::getScalarSizeInBits(), llvm::MVT::getVectorNumElements(), llvm::ShuffleVectorInst::isInterleaveMask(), and Size.

Referenced by llvm::RISCVTargetLowering::isShuffleMaskLegal(), and lowerVECTOR_SHUFFLE().

isNonZeroAVL()

static bool isNonZeroAVL ( SDValue  AVL )

static

Definition at line 8964 of file RISCVISelLowering.cpp.

isSelectPseudo()

static bool isSelectPseudo ( MachineInstr &  MI )

static

Definition at line 16092 of file RISCVISelLowering.cpp.

References MI.

Referenced by emitSelectPseudo().

isSimpleVIDSequence()

static std::optional< VIDSequence > isSimpleVIDSequence ( SDValue  Op )

static

Definition at line 3150 of file RISCVISelLowering.cpp.

References assert(), llvm::ISD::BUILD_VECTOR, getExactInteger(), llvm::DWARFExpression::Operation::getNumOperands(), Idx, and llvm::SignExtend64().

Referenced by lowerBuildVectorOfConstants(), and llvm::RISCVTargetLowering::PerformDAGCombine().

isValidEGW()

static bool isValidEGW ( int  EGS, EVT  VT, const RISCVSubtarget &  Subtarget  )

inlinestatic

Definition at line 8272 of file RISCVISelLowering.cpp.

References llvm::details::FixedOrScalableQuantity< LeafTy, ValueTy >::getKnownMinValue(), llvm::RISCVSubtarget::getRealMinVLen(), llvm::EVT::getScalarSizeInBits(), llvm::EVT::getSizeInBits(), and llvm::RISCV::RVVBitsPerBlock.

legalizeScatterGatherIndexType()

static bool legalizeScatterGatherIndexType ( SDLoc  DL, SDValue &  Index, ISD::MemIndexType &  IndexType, RISCVTargetLowering::DAGCombinerInfo &  DCI  )

static

Definition at line 14478 of file RISCVISelLowering.cpp.

References llvm::EVT::bitsLT(), llvm::EVT::changeVectorElementType(), llvm::TargetLowering::DAGCombinerInfo::DAG, DL, llvm::SelectionDAG::getMachineFunction(), llvm::SelectionDAG::getNode(), llvm::MachineFunction::getSubtarget(), llvm::EVT::getVectorElementType(), llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalize(), llvm::ISD::SIGN_EXTEND, and llvm::ISD::UNSIGNED_SCALED.

Referenced by llvm::RISCVTargetLowering::PerformDAGCombine().

LowerATOMIC_FENCE()

static SDValue LowerATOMIC_FENCE ( SDValue  Op, SelectionDAG &  DAG, const RISCVSubtarget &  Subtarget  )

static

Definition at line 5187 of file RISCVISelLowering.cpp.

References llvm::SelectionDAG::getNode(), llvm::ISD::MEMBARRIER, llvm::SequentiallyConsistent, llvm::SyncScope::SingleThread, and llvm::SyncScope::System.

lowerBitreverseShuffle()

static SDValue lowerBitreverseShuffle ( ShuffleVectorSDNode *  SVN, SelectionDAG &  DAG, const RISCVSubtarget &  Subtarget  )

static

Definition at line 4519 of file RISCVISelLowering.cpp.

References assert(), llvm::ISD::BITREVERSE, DL, llvm::ISD::EXTRACT_SUBVECTOR, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getContext(), llvm::EVT::getIntegerVT(), llvm::ShuffleVectorSDNode::getMask(), llvm::SelectionDAG::getNode(), llvm::SDNode::getOperand(), llvm::EVT::getScalarSizeInBits(), llvm::SDNode::getSimpleValueType(), llvm::RISCVSubtarget::getTargetLowering(), llvm::SelectionDAG::getUNDEF(), llvm::MVT::getVectorElementType(), llvm::SelectionDAG::getVectorIdxConstant(), llvm::MVT::getVectorNumElements(), llvm::EVT::getVectorVT(), llvm::ISD::INSERT_SUBVECTOR, llvm::TargetLoweringBase::isOperationLegalOrCustom(), llvm::ShuffleVectorInst::isReverseMask(), llvm::TargetLoweringBase::isTypeLegal(), llvm::SDValue::isUndef(), llvm::PowerOf2Ceil(), llvm::ArrayRef< T >::size(), and llvm::ISD::SRL.

Referenced by lowerVECTOR_SHUFFLE().

lowerBUILD_VECTOR()

static SDValue lowerBUILD_VECTOR ( SDValue  Op, SelectionDAG &  DAG, const RISCVSubtarget &  Subtarget  )

static

Definition at line 3706 of file RISCVISelLowering.cpp.

References llvm::ISD::AND, llvm::ISD::ANY_EXTEND, assert(), llvm::ISD::BUILD_VECTOR, llvm::MVT::changeVectorElementType(), convertFromScalableVector(), convertToScalableVector(), DL, llvm::SelectionDAG::getBuildVector(), llvm::SelectionDAG::getConstant(), getContainerForFixedLengthVector(), getDefaultVLOps(), llvm::MVT::getFixedSizeInBits(), llvm::RISCVSubtarget::getFLen(), llvm::details::FixedOrScalableQuantity< LeafTy, ValueTy >::getKnownMinValue(), llvm::RISCVTargetLowering::getLMUL(), getLMUL1VT(), llvm::SelectionDAG::getNode(), llvm::RISCVSubtarget::getRealMaxVLen(), llvm::RISCVSubtarget::getRealMinVLen(), llvm::SelectionDAG::getSetCC(), llvm::MVT::getSizeInBits(), llvm::SelectionDAG::getSplatBuildVector(), llvm::getSplatValue(), llvm::SelectionDAG::getSplatVector(), llvm::SelectionDAG::getUNDEF(), llvm::MVT::getVectorElementType(), llvm::SelectionDAG::getVectorIdxConstant(), llvm::MVT::getVectorMinNumElements(), llvm::MVT::getVectorNumElements(), llvm::MVT::getVectorVT(), getVSlidedown(), llvm::RISCVSubtarget::getXLenVT(), llvm::ISD::INSERT_SUBVECTOR, llvm::ISD::isBuildVectorOfConstantFPSDNodes(), llvm::ISD::isBuildVectorOfConstantSDNodes(), llvm::MVT::isFixedLengthVector(), llvm::MVT::isFloatingPoint(), llvm::RISCVII::LMUL_2, llvm::RISCVII::LMUL_4, llvm::RISCVII::LMUL_8, lowerBuildVectorOfConstants(), lowerBuildVectorViaDominantValues(), llvm::RISCVII::MASK_AGNOSTIC, matchSplatAsGather(), llvm::Offset, llvm::ISD::SETNE, llvm::ArrayRef< T >::slice(), llvm::Splat, llvm::RISCVII::TAIL_AGNOSTIC, llvm::RISCVISD::VFMV_V_F_VL, llvm::RISCVISD::VFSLIDE1DOWN_VL, llvm::RISCVISD::VMV_V_X_VL, and llvm::RISCVISD::VSLIDE1DOWN_VL.

Referenced by llvm::RISCVTargetLowering::LowerOperation().

lowerBuildVectorOfConstants()

static SDValue lowerBuildVectorOfConstants ( SDValue  Op, SelectionDAG &  DAG, const RISCVSubtarget &  Subtarget  )

static

Definition at line 3405 of file RISCVISelLowering.cpp.

References llvm::ISD::ADD, llvm::ISD::ANY_EXTEND, assert(), llvm::MVT::bitsLE(), llvm::MVT::changeVectorElementType(), llvm::MVT::changeVectorElementTypeToInteger(), llvm::SelectionDAG::ComputeNumSignBits(), convertFromScalableVector(), convertToScalableVector(), llvm::divideCeil(), DL, llvm::enumerate(), llvm::ISD::EXTRACT_SUBVECTOR, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getBuildVector(), llvm::SelectionDAG::getConstant(), getContainerForFixedLengthVector(), getDefaultVLOps(), llvm::RISCVSubtarget::getELen(), llvm::MVT::getIntegerVT(), llvm::SelectionDAG::getNode(), llvm::DWARFExpression::Operation::getNumOperands(), llvm::RISCVSubtarget::getRealMinVLen(), llvm::MVT::getScalarSizeInBits(), llvm::MVT::getSizeInBits(), llvm::getSplatValue(), llvm::SelectionDAG::getUNDEF(), llvm::MVT::getVectorElementType(), llvm::MVT::getVectorNumElements(), llvm::MVT::getVectorVT(), llvm::RISCVSubtarget::getXLen(), llvm::RISCVSubtarget::getXLenVT(), I, llvm::ISD::INSERT_VECTOR_ELT, INT64_MIN, llvm::RISCVSubtarget::is64Bit(), llvm::ISD::isBuildVectorAllOnes(), llvm::ISD::isBuildVectorAllZeros(), llvm::ISD::isBuildVectorOfConstantSDNodes(), llvm::MVT::isFixedLengthVector(), llvm::MVT::isFloatingPoint(), llvm::MVT::isInteger(), llvm::isPowerOf2_32(), llvm::isPowerOf2_64(), isSimpleVIDSequence(), llvm::Log2_64(), lowerBuildVectorViaDominantValues(), llvm::ISD::MUL, llvm::ISD::SHL, llvm::SelectionDAG::shouldOptForSize(), llvm::ISD::SINT_TO_FP, llvm::Splat, llvm::ISD::SRL, llvm::ISD::SUB, llvm::RISCVISD::VFMV_V_F_VL, llvm::RISCVISD::VID_VL, llvm::RISCVISD::VMCLR_VL, llvm::RISCVISD::VMSET_VL, llvm::RISCVISD::VMV_V_X_VL, and llvm::RISCVISD::VSEXT_VL.

Referenced by lowerBUILD_VECTOR().

lowerBuildVectorViaDominantValues()

static SDValue lowerBuildVectorViaDominantValues ( SDValue  Op, SelectionDAG &  DAG, const RISCVSubtarget &  Subtarget  )

static

Try and optimize BUILD_VECTORs with "dominant values" - these are values which constitute a large proportion of the elements.

In such cases we can splat a vector with the dominant element and make up the shortfall with INSERT_VECTOR_ELTs. Returns SDValue if not profitable. Note that this includes vectors of 2 elements by association. The upper-most element is the "dominant" one, allowing us to use a splat to "insert" the upper element, and an insert of the lower element at position 0, which improves codegen.

Definition at line 3298 of file RISCVISelLowering.cpp.

References llvm::ISD::ANY_EXTEND, assert(), llvm::MVT::changeVectorElementType(), convertFromScalableVector(), convertToScalableVector(), llvm::count_if(), DL, llvm::enumerate(), llvm::SelectionDAG::getBuildVector(), llvm::SelectionDAG::getConstant(), getContainerForFixedLengthVector(), getDefaultVLOps(), llvm::SelectionDAG::getNode(), llvm::DWARFExpression::Operation::getNumOperands(), llvm::SelectionDAG::getSplatBuildVector(), llvm::SelectionDAG::getUNDEF(), llvm::RISCVSubtarget::getXLenVT(), llvm::DenseMapBase< DerivedT, KeyT, ValueT, KeyInfoT, BucketT >::insert(), llvm::ISD::INSERT_VECTOR_ELT, llvm::ISD::isBuildVectorOfConstantSDNodes(), llvm::MVT::isFixedLengthVector(), llvm::MVT::isFloatingPoint(), llvm::Log2_32(), llvm::SelectionDAG::shouldOptForSize(), llvm::DenseMapBase< DerivedT, KeyT, ValueT, KeyInfoT, BucketT >::size(), llvm::transform(), llvm::RISCVISD::VFSLIDE1DOWN_VL, llvm::ISD::VSELECT, and llvm::RISCVISD::VSLIDE1DOWN_VL.

Referenced by lowerBUILD_VECTOR(), and lowerBuildVectorOfConstants().

lowerConstant()

static SDValue lowerConstant ( SDValue  Op, SelectionDAG &  DAG, const RISCVSubtarget &  Subtarget  )

static

Definition at line 5147 of file RISCVISelLowering.cpp.

References assert(), llvm::SmallVectorBase< Size_T >::empty(), llvm::RISCVMatInt::generateInstSeq(), llvm::RISCVMatInt::generateTwoRegInstSeq(), llvm::RISCVSubtarget::getMaxBuildIntsCost(), llvm::SelectionDAG::shouldOptForSize(), llvm::SmallVectorBase< Size_T >::size(), and llvm::RISCVSubtarget::useConstantPoolForLargeInts().

Referenced by llvm::RISCVTargetLowering::LowerOperation().

lowerFMAXIMUM_FMINIMUM()

static SDValue lowerFMAXIMUM_FMINIMUM ( SDValue  Op, SelectionDAG &  DAG, const RISCVSubtarget &  Subtarget  )

static

Definition at line 5318 of file RISCVISelLowering.cpp.

References convertFromScalableVector(), convertToScalableVector(), DL, llvm::RISCVISD::FMAX, llvm::ISD::FMAXIMUM, llvm::RISCVISD::FMIN, getContainerForFixedLengthVector(), getDefaultVLOps(), llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getSelect(), llvm::SelectionDAG::getSetCC(), llvm::SelectionDAG::getUNDEF(), llvm::RISCVSubtarget::getXLenVT(), llvm::MVT::isFixedLengthVector(), llvm::SelectionDAG::isKnownNeverNaN(), llvm::MVT::isVector(), llvm::RISCVISD::SETCC_VL, llvm::ISD::SETOEQ, llvm::RISCVISD::VFMAX_VL, llvm::RISCVISD::VFMIN_VL, llvm::RISCVISD::VSELECT_VL, X, and Y.

Referenced by llvm::RISCVTargetLowering::LowerOperation().

lowerFP_TO_INT_SAT()

static SDValue lowerFP_TO_INT_SAT ( SDValue  Op, SelectionDAG &  DAG, const RISCVSubtarget &  Subtarget  )

static

Definition at line 2687 of file RISCVISelLowering.cpp.

References assert(), llvm::MVT::changeVectorElementType(), convertFromScalableVector(), convertToScalableVector(), DL, llvm::RISCVISD::FCVT_W_RV64, llvm::RISCVISD::FCVT_WU_RV64, llvm::RISCVISD::FCVT_X, llvm::RISCVISD::FCVT_XU, llvm::ISD::FP_EXTEND, llvm::RISCVISD::FP_EXTEND_VL, llvm::ISD::FP_TO_SINT_SAT, llvm::SelectionDAG::getConstant(), getContainerForFixedLengthVector(), getDefaultVLOps(), llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getSelectCC(), llvm::MVT::getSizeInBits(), llvm::SelectionDAG::getTargetConstant(), llvm::SelectionDAG::getUNDEF(), llvm::MVT::getVectorElementCount(), llvm::MVT::getVectorElementType(), llvm::RISCVSubtarget::getXLenVT(), llvm::SelectionDAG::getZeroExtendInReg(), llvm::RISCVSubtarget::hasStdExtZfhOrZhinx(), llvm::MVT::isFixedLengthVector(), llvm::MVT::isVector(), llvm::RISCVFPRndMode::RTZ, llvm::RISCVISD::SETCC_VL, llvm::ISD::SETUO, llvm::RISCVISD::VFCVT_RTZ_X_F_VL, llvm::RISCVISD::VFCVT_RTZ_XU_F_VL, llvm::RISCVISD::VMV_V_X_VL, and llvm::RISCVISD::VSELECT_VL.

Referenced by llvm::RISCVTargetLowering::LowerOperation().

lowerFTRUNC_FCEIL_FFLOOR_FROUND()

static SDValue lowerFTRUNC_FCEIL_FFLOOR_FROUND ( SDValue  Op, SelectionDAG &  DAG, const RISCVSubtarget &  Subtarget  )

static

Definition at line 3032 of file RISCVISelLowering.cpp.

References llvm::APFloat::convertFromAPInt(), DL, llvm::SelectionDAG::EVTToAPFloatSemantics(), llvm::RISCVISD::FROUND, llvm::SelectionDAG::getConstantFP(), llvm::SelectionDAG::getNode(), llvm::APInt::getOneBitSet(), llvm::SelectionDAG::getTargetConstant(), llvm::RISCVSubtarget::getXLenVT(), llvm::MVT::isVector(), lowerVectorFTRUNC_FCEIL_FFLOOR_FROUND(), matchRoundingOp(), llvm::APFloatBase::rmNearestTiesToEven, llvm::APFloatBase::semanticsPrecision(), and llvm::SelectionDAG::shouldOptForSize().

Referenced by llvm::RISCVTargetLowering::LowerOperation().

lowerGetVectorLength()

static SDValue lowerGetVectorLength ( SDNode *  N, SelectionDAG &  DAG, const RISCVSubtarget &  Subtarget  )

static

Definition at line 8214 of file RISCVISelLowering.cpp.

References assert(), DL, llvm::RISCVVType::encodeLMUL(), llvm::RISCVVType::encodeSEW(), llvm::RISCVSubtarget::getELen(), llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getTargetConstant(), llvm::RISCVSubtarget::getXLenVT(), llvm::ISD::INTRINSIC_WO_CHAIN, llvm::isPowerOf2_32(), N, llvm::RISCV::RVVBitsPerBlock, llvm::ISD::TRUNCATE, and llvm::ISD::ZERO_EXTEND.

Referenced by llvm::RISCVTargetLowering::ReplaceNodeResults().

lowerReductionSeq()

static SDValue lowerReductionSeq ( unsigned  RVVOpcode, MVT  ResVT, SDValue  StartValue, SDValue  Vec, SDValue  Mask, SDValue  VL, const SDLoc &  DL, SelectionDAG &  DAG, const RISCVSubtarget &  Subtarget  )

static

Helper to lower a reduction sequence of the form: scalar = reduce_op vec, scalar_start.

Definition at line 8973 of file RISCVISelLowering.cpp.

References llvm::MVT::bitsLE(), DL, llvm::ISD::EXTRACT_VECTOR_ELT, llvm::SelectionDAG::getConstant(), getLMUL1VT(), llvm::SelectionDAG::getNode(), llvm::SDValue::getSimpleValueType(), llvm::SelectionDAG::getTargetConstant(), llvm::SelectionDAG::getUNDEF(), llvm::RISCVSubtarget::getXLenVT(), llvm::ISD::INSERT_SUBVECTOR, isNonZeroAVL(), lowerScalarInsert(), Reduction, and llvm::RISCVII::TAIL_AGNOSTIC.

lowerScalarInsert()

static SDValue lowerScalarInsert ( SDValue  Scalar, SDValue  VL, MVT  VT, const SDLoc &  DL, SelectionDAG &  DAG, const RISCVSubtarget &  Subtarget  )

static

Definition at line 3990 of file RISCVISelLowering.cpp.

References llvm::ISD::ANY_EXTEND, assert(), llvm::MVT::bitsLE(), convertToScalableVector(), DL, llvm::ISD::EXTRACT_SUBVECTOR, llvm::ISD::EXTRACT_VECTOR_ELT, llvm::SelectionDAG::getConstant(), getContainerForFixedLengthVector(), llvm::SelectionDAG::getNode(), llvm::SDValue::getSimpleValueType(), llvm::SelectionDAG::getUNDEF(), llvm::RISCVSubtarget::getXLenVT(), llvm::ISD::INSERT_SUBVECTOR, llvm::MVT::isFixedLengthVector(), llvm::MVT::isFloatingPoint(), llvm::isNullConstant(), llvm::MVT::isScalableVector(), lowerScalarSplat(), llvm::ISD::SIGN_EXTEND, llvm::RISCVISD::VFMV_S_F_VL, and llvm::RISCVISD::VMV_S_X_VL.

Referenced by combineBinOpToReduce(), and lowerReductionSeq().

lowerScalarSplat()

static SDValue lowerScalarSplat ( SDValue  Passthru, SDValue  Scalar, SDValue  VL, MVT  VT, const SDLoc &  DL, SelectionDAG &  DAG, const RISCVSubtarget &  Subtarget  )

static

Definition at line 3952 of file RISCVISelLowering.cpp.

References llvm::ISD::ANY_EXTEND, assert(), DL, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getUNDEF(), llvm::RISCVSubtarget::getXLenVT(), llvm::MVT::isFloatingPoint(), llvm::isNullConstant(), llvm::isOneConstant(), llvm::SDValue::isUndef(), llvm::ISD::SIGN_EXTEND, splatSplitI64WithVL(), llvm::RISCVISD::VFMV_V_F_VL, llvm::RISCVISD::VMV_S_X_VL, and llvm::RISCVISD::VMV_V_X_VL.

Referenced by lowerScalarInsert(), and lowerVECTOR_SHUFFLE().

lowerVECTOR_SHUFFLE()

static SDValue lowerVECTOR_SHUFFLE ( SDValue  Op, SelectionDAG &  DAG, const RISCVSubtarget &  Subtarget  )

static

Definition at line 4607 of file RISCVISelLowering.cpp.

References llvm::all_of(), assert(), llvm::DenseMapBase< DerivedT, KeyT, ValueT, KeyInfoT, BucketT >::begin(), llvm::MVT::bitsGT(), llvm::MVT::changeTypeToInteger(), llvm::MVT::changeVectorElementType(), llvm::ISD::CONCAT_VECTORS, convertFromScalableVector(), convertToScalableVector(), DL, llvm::enumerate(), llvm::ISD::EXTRACT_SUBVECTOR, llvm::SelectionDAG::getBuildVector(), llvm::SelectionDAG::getConstant(), getContainerForFixedLengthVector(), getDefaultVLOps(), getDeinterleaveViaVNSRL(), llvm::SelectionDAG::getExtLoad(), llvm::TypeSize::getFixed(), llvm::SDNode::getFlags(), llvm::MVT::getHalfNumVectorElementsVT(), llvm::SelectionDAG::getLoad(), llvm::SelectionDAG::getMachineFunction(), llvm::MachineFunction::getMachineMemOperand(), llvm::ShuffleVectorSDNode::getMask(), llvm::SelectionDAG::getMemBasePlusOffset(), llvm::SelectionDAG::getMemIntrinsicNode(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOperand(), llvm::RISCVSubtarget::getRealMinVLen(), llvm::SelectionDAG::getRegister(), llvm::MVT::getScalarSizeInBits(), llvm::MVT::getScalarType(), llvm::SelectionDAG::getSetCC(), llvm::MVT::getSizeInBits(), llvm::ShuffleVectorSDNode::getSplatIndex(), llvm::SelectionDAG::getSplatValue(), llvm::MVT::getStoreSize(), llvm::SelectionDAG::getTargetConstant(), llvm::SelectionDAG::getUNDEF(), llvm::MVT::getVectorElementCount(), llvm::MVT::getVectorElementType(), llvm::MVT::getVectorNumElements(), llvm::SelectionDAG::getVectorShuffle(), llvm::MVT::getVectorVT(), getVSlidedown(), getVSlideup(), llvm::SelectionDAG::getVTList(), getWideningInterleave(), llvm::RISCVSubtarget::getXLenVT(), llvm::ISD::INTRINSIC_W_CHAIN, isDeinterleaveShuffle(), isElementRotate(), llvm::MVT::isFloatingPoint(), llvm::MVT::isInteger(), isInterleaveShuffle(), llvm::ISD::isNormalLoad(), llvm::ShuffleVectorSDNode::isSplat(), llvm::SelectionDAG::isSplatValue(), llvm::SDValue::isUndef(), lowerBitreverseShuffle(), lowerScalarSplat(), lowerVECTOR_SHUFFLEAsRotate(), lowerVECTOR_SHUFFLEAsVSlide1(), lowerVECTOR_SHUFFLEAsVSlidedown(), lowerVECTOR_SHUFFLEAsVSlideup(), llvm::SelectionDAG::makeEquivalentMemoryOrdering(), llvm::Offset, llvm::SmallVectorTemplateBase< T, bool >::push_back(), llvm::ISD::SETNE, llvm::ISD::SEXTLOAD, llvm::DenseMapBase< DerivedT, KeyT, ValueT, KeyInfoT, BucketT >::size(), llvm::SmallVectorBase< Size_T >::size(), Size, llvm::Splat, std::swap(), llvm::RISCVII::TAIL_AGNOSTIC, llvm::RISCVISD::VFMV_V_F_VL, llvm::RISCVISD::VMV_V_X_VL, llvm::RISCVISD::VRGATHER_VV_VL, llvm::RISCVISD::VRGATHER_VX_VL, llvm::RISCVISD::VRGATHEREI16_VV_VL, llvm::ISD::VSELECT, and llvm::ISD::ZERO_EXTEND.

Referenced by llvm::RISCVTargetLowering::LowerOperation(), and llvm::X86TargetLowering::LowerOperation().

lowerVECTOR_SHUFFLEAsRotate()

static SDValue lowerVECTOR_SHUFFLEAsRotate ( ShuffleVectorSDNode *  SVN, SelectionDAG &  DAG, const RISCVSubtarget &  Subtarget  )

static

Definition at line 4574 of file RISCVISelLowering.cpp.

References llvm::ISD::BSWAP, DL, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getConstant(), llvm::MVT::getIntegerVT(), llvm::ShuffleVectorSDNode::getMask(), llvm::SelectionDAG::getNode(), llvm::SDNode::getOperand(), llvm::EVT::getScalarSizeInBits(), llvm::MVT::getScalarType(), llvm::RISCVSubtarget::getTargetLowering(), llvm::SDNode::getValueType(), llvm::EVT::getVectorNumElements(), llvm::MVT::getVectorVT(), llvm::ShuffleVectorInst::isBitRotateMask(), llvm::TargetLoweringBase::isTypeLegal(), and llvm::ISD::ROTL.

Referenced by lowerVECTOR_SHUFFLE().

lowerVECTOR_SHUFFLEAsVSlide1()

static SDValue lowerVECTOR_SHUFFLEAsVSlide1 ( const SDLoc &  DL, MVT  VT, SDValue  V1, SDValue  V2, ArrayRef< int >  Mask, const RISCVSubtarget &  Subtarget, SelectionDAG &  DAG  )

static

Match v(f)slide1up/down idioms.

These operations involve sliding N-1 elements to make room for an inserted scalar at one end.

Definition at line 4381 of file RISCVISelLowering.cpp.

References llvm::ISD::ANY_EXTEND, llvm::sampleprof::Base, convertFromScalableVector(), convertToScalableVector(), DL, E, getContainerForFixedLengthVector(), getDefaultVLOps(), llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getUNDEF(), llvm::MVT::getVectorNumElements(), llvm::RISCVSubtarget::getXLenVT(), llvm::MVT::isFloatingPoint(), llvm::Offset, llvm::Splat, std::swap(), llvm::RISCVISD::VFSLIDE1DOWN_VL, llvm::RISCVISD::VFSLIDE1UP_VL, llvm::RISCVISD::VSLIDE1DOWN_VL, and llvm::RISCVISD::VSLIDE1UP_VL.

Referenced by lowerVECTOR_SHUFFLE().

lowerVECTOR_SHUFFLEAsVSlidedown()

static SDValue lowerVECTOR_SHUFFLEAsVSlidedown ( const SDLoc &  DL, MVT  VT, SDValue  V1, SDValue  V2, ArrayRef< int >  Mask, const RISCVSubtarget &  Subtarget, SelectionDAG &  DAG  )

static

Definition at line 4260 of file RISCVISelLowering.cpp.

References convertFromScalableVector(), convertToScalableVector(), DL, llvm::ISD::EXTRACT_SUBVECTOR, llvm::SelectionDAG::getConstant(), getContainerForFixedLengthVector(), getDefaultVLOps(), llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getUNDEF(), getVSlidedown(), llvm::RISCVSubtarget::getXLenVT(), llvm::Offset, and llvm::SmallVectorBase< Size_T >::size().

Referenced by lowerVECTOR_SHUFFLE().

lowerVECTOR_SHUFFLEAsVSlideup()

static SDValue lowerVECTOR_SHUFFLEAsVSlideup ( const SDLoc &  DL, MVT  VT, SDValue  V1, SDValue  V2, ArrayRef< int >  Mask, const RISCVSubtarget &  Subtarget, SelectionDAG &  DAG  )

static

Definition at line 4337 of file RISCVISelLowering.cpp.

References convertFromScalableVector(), convertToScalableVector(), DL, llvm::SelectionDAG::getConstant(), getContainerForFixedLengthVector(), getDefaultVLOps(), llvm::SelectionDAG::getNode(), llvm::MVT::getVectorNumElements(), getVSlideup(), llvm::RISCVSubtarget::getXLenVT(), llvm::ShuffleVectorInst::isInsertSubvectorMask(), llvm::RISCVII::MASK_AGNOSTIC, llvm::RISCVII::TAIL_AGNOSTIC, llvm::RISCVII::TAIL_UNDISTURBED_MASK_UNDISTURBED, and llvm::RISCVISD::VMV_V_V_VL.

Referenced by lowerVECTOR_SHUFFLE().

lowerVectorFTRUNC_FCEIL_FFLOOR_FROUND()

static SDValue lowerVectorFTRUNC_FCEIL_FFLOOR_FROUND ( SDValue  Op, SelectionDAG &  DAG, const RISCVSubtarget &  Subtarget  )

static

Definition at line 2822 of file RISCVISelLowering.cpp.

References assert(), llvm::MVT::changeVectorElementTypeToInteger(), llvm::APFloat::convertFromAPInt(), convertFromScalableVector(), convertToScalableVector(), DL, llvm::SelectionDAG::EVTToAPFloatSemantics(), llvm::RISCVISD::FABS_VL, llvm::ISD::FCEIL, llvm::RISCVISD::FCOPYSIGN_VL, llvm::ISD::FFLOOR, llvm::ISD::FNEARBYINT, llvm::ISD::FRINT, llvm::ISD::FROUND, llvm::ISD::FROUNDEVEN, llvm::ISD::FTRUNC, llvm::SelectionDAG::getCondCode(), llvm::SelectionDAG::getConstantFP(), getContainerForFixedLengthVector(), getDefaultVLOps(), llvm::SelectionDAG::getFreeze(), getMaskTypeFor(), llvm::SelectionDAG::getNode(), llvm::APInt::getOneBitSet(), llvm::SDValue::getOpcode(), llvm::SelectionDAG::getTargetConstant(), llvm::SelectionDAG::getUNDEF(), llvm::MVT::getVectorElementCount(), llvm::MVT::getVectorElementType(), llvm::MVT::getVectorVT(), llvm::RISCVSubtarget::getXLenVT(), llvm::RISCVFPRndMode::Invalid, llvm::MVT::isFixedLengthVector(), llvm::MVT::isVector(), llvm_unreachable, matchRoundingOp(), llvm::APFloatBase::rmNearestTiesToEven, llvm::APFloatBase::semanticsPrecision(), llvm::RISCVISD::SETCC_VL, llvm::ISD::SETOLT, llvm::RISCVISD::SINT_TO_FP_VL, llvm::RISCVISD::VFCVT_RM_X_F_VL, llvm::RISCVISD::VFCVT_RTZ_X_F_VL, llvm::RISCVISD::VFCVT_X_F_VL, llvm::RISCVISD::VFMV_V_F_VL, and llvm::RISCVISD::VFROUND_NOEXCEPT_VL.

Referenced by lowerFTRUNC_FCEIL_FFLOOR_FROUND(), and llvm::RISCVTargetLowering::LowerOperation().

lowerVectorIntrinsicScalars()

static SDValue lowerVectorIntrinsicScalars ( SDValue  Op, SelectionDAG &  DAG, const RISCVSubtarget &  Subtarget  )

static

Definition at line 8017 of file RISCVISelLowering.cpp.

References llvm::ISD::ANY_EXTEND, assert(), llvm::MVT::bitsLT(), llvm::SelectionDAG::ComputeNumSignBits(), llvm::RISCVTargetLowering::computeVLMAX(), DL, llvm::RISCVVType::encodeSEW(), getAllOnesMask(), llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getConstant(), llvm::details::FixedOrScalableQuantity< LeafTy, ValueTy >::getKnownMinValue(), llvm::RISCVTargetLowering::getLMUL(), llvm::SelectionDAG::getNode(), llvm::DWARFExpression::Operation::getNumOperands(), llvm::RISCVSubtarget::getRealMaxVLen(), llvm::RISCVSubtarget::getRealMinVLen(), llvm::MVT::getScalarSizeInBits(), llvm::SDValue::getSimpleValueType(), llvm::MVT::getSizeInBits(), llvm::SelectionDAG::getTargetConstant(), llvm::SelectionDAG::getUNDEF(), llvm::SDValue::getValueType(), llvm::MVT::getVectorElementCount(), llvm::MVT::getVectorElementType(), llvm::MVT::getVectorVT(), getVLOperand(), llvm::RISCVSubtarget::getXLenVT(), llvm::RISCVVIntrinsicsTable::RISCVVIntrinsicInfo::hasScalarOperand(), llvm::RISCVSubtarget::hasVInstructions(), llvm::ISD::INTRINSIC_VOID, llvm::ISD::INTRINSIC_W_CHAIN, llvm::ISD::INTRINSIC_WO_CHAIN, llvm::MVT::isScalarInteger(), llvm::SDValue::isUndef(), Operands, llvm::RISCVVIntrinsicsTable::RISCVVIntrinsicInfo::ScalarOperand, SEW, llvm::ISD::SHL, llvm::ISD::SIGN_EXTEND, splatSplitI64WithVL(), llvm::SelectionDAG::SplitScalar(), llvm::RISCVII::TAIL_AGNOSTIC, llvm::ISD::TRUNCATE, llvm::RISCVISD::VP_MERGE_VL, llvm::RISCVISD::VSELECT_VL, llvm::RISCVISD::VSLIDE1DOWN_VL, and llvm::RISCVISD::VSLIDE1UP_VL.

lowerVectorStrictFTRUNC_FCEIL_FFLOOR_FROUND()

static SDValue lowerVectorStrictFTRUNC_FCEIL_FFLOOR_FROUND ( SDValue  Op, SelectionDAG &  DAG, const RISCVSubtarget &  Subtarget  )

static

Definition at line 2931 of file RISCVISelLowering.cpp.

References assert(), llvm::MVT::changeVectorElementTypeToInteger(), llvm::APFloat::convertFromAPInt(), convertFromScalableVector(), convertToScalableVector(), DL, llvm::SelectionDAG::EVTToAPFloatSemantics(), llvm::RISCVISD::FABS_VL, llvm::RISCVISD::FCOPYSIGN_VL, llvm::SelectionDAG::getCondCode(), llvm::SelectionDAG::getConstantFP(), getContainerForFixedLengthVector(), getDefaultVLOps(), llvm::SelectionDAG::getFreeze(), llvm::SelectionDAG::getMergeValues(), llvm::SelectionDAG::getNode(), llvm::APInt::getOneBitSet(), llvm::SelectionDAG::getUNDEF(), llvm::SDValue::getValue(), llvm::MVT::getVectorElementType(), llvm::SelectionDAG::getVTList(), llvm::RISCVSubtarget::getXLenVT(), llvm::RISCVFPRndMode::Invalid, llvm::MVT::isFixedLengthVector(), llvm_unreachable, matchRoundingOp(), llvm::APFloatBase::rmNearestTiesToEven, llvm::APFloatBase::semanticsPrecision(), llvm::RISCVISD::SETCC_VL, llvm::ISD::SETOLT, llvm::RISCVISD::STRICT_FADD_VL, llvm::ISD::STRICT_FCEIL, llvm::ISD::STRICT_FFLOOR, llvm::ISD::STRICT_FNEARBYINT, llvm::ISD::STRICT_FROUND, llvm::ISD::STRICT_FROUNDEVEN, llvm::RISCVISD::STRICT_FSETCC_VL, llvm::ISD::STRICT_FTRUNC, llvm::RISCVISD::STRICT_SINT_TO_FP_VL, llvm::RISCVISD::STRICT_VFCVT_RM_X_F_VL, llvm::RISCVISD::STRICT_VFCVT_RTZ_X_F_VL, llvm::RISCVISD::STRICT_VFROUND_NOEXCEPT_VL, and llvm::RISCVISD::VFMV_V_F_VL.

Referenced by llvm::RISCVTargetLowering::LowerOperation().

lowerVectorXRINT()

static SDValue lowerVectorXRINT ( SDValue  Op, SelectionDAG &  DAG, const RISCVSubtarget &  Subtarget  )

static

Definition at line 3060 of file RISCVISelLowering.cpp.

References assert(), convertFromScalableVector(), convertToScalableVector(), DL, getContainerForFixedLengthVector(), getDefaultVLOps(), llvm::SelectionDAG::getNode(), llvm::MVT::isFixedLengthVector(), llvm::MVT::isVector(), and llvm::RISCVISD::VFCVT_X_F_VL.

Referenced by llvm::RISCVTargetLowering::LowerOperation().

matchIndexAsShuffle()

static bool matchIndexAsShuffle ( EVT  VT, SDValue  Index, SDValue  Mask, SmallVector< int > &  ShuffleMask  )

static

Match the index vector of a scatter or gather node as the shuffle mask which performs the rearrangement if possible.

Will only match if all lanes are touched, and thus replacing the scatter or gather with a unit strided access and shuffle is legal.

Definition at line 14511 of file RISCVISelLowering.cpp.

References llvm::BitVector::all(), assert(), llvm::CallingConv::C, llvm::SmallVectorBase< Size_T >::empty(), llvm::EVT::getScalarStoreSize(), llvm::EVT::getVectorNumElements(), llvm::ISD::isBuildVectorOfConstantSDNodes(), llvm::ISD::isConstantSplatVectorAllOnes(), llvm::SmallVectorTemplateBase< T, bool >::push_back(), and llvm::BitVector::set().

Referenced by llvm::RISCVTargetLowering::PerformDAGCombine().

matchIndexAsWiderOp()

static bool matchIndexAsWiderOp ( EVT  VT, SDValue  Index, SDValue  Mask, Align  BaseAlign, const RISCVSubtarget &  ST  )

static

Match the index of a gather or scatter operation as an operation with twice the element width and half the number of elements.

This is generally profitable (if legal) because these operations are linear in VL, so even if we cause some extract VTYPE/VL toggles, we still come out ahead.

Definition at line 14546 of file RISCVISelLowering.cpp.

References llvm::CallingConv::C, llvm::EVT::getScalarStoreSize(), llvm::EVT::getVectorNumElements(), llvm::ISD::isBuildVectorOfConstantSDNodes(), llvm::ISD::isConstantSplatVectorAllOnes(), and llvm::Last.

Referenced by llvm::RISCVTargetLowering::PerformDAGCombine().

matchRoundingOp()

static RISCVFPRndMode::RoundingMode matchRoundingOp ( unsigned  Opc )

static

Definition at line 2788 of file RISCVISelLowering.cpp.

References llvm::RISCVFPRndMode::DYN, llvm::ISD::FCEIL, llvm::ISD::FFLOOR, llvm::ISD::FRINT, llvm::ISD::FROUND, llvm::ISD::FROUNDEVEN, llvm::ISD::FTRUNC, llvm::RISCVFPRndMode::Invalid, llvm::RISCVFPRndMode::RDN, llvm::RISCVFPRndMode::RMM, llvm::RISCVFPRndMode::RNE, llvm::RISCVFPRndMode::RTZ, llvm::RISCVFPRndMode::RUP, llvm::ISD::STRICT_FCEIL, llvm::ISD::STRICT_FFLOOR, llvm::ISD::STRICT_FROUND, llvm::ISD::STRICT_FROUNDEVEN, and llvm::ISD::STRICT_FTRUNC.

Referenced by lowerFTRUNC_FCEIL_FFLOOR_FROUND(), lowerVectorFTRUNC_FCEIL_FFLOOR_FROUND(), lowerVectorStrictFTRUNC_FCEIL_FFLOOR_FROUND(), performFP_TO_INT_SATCombine(), and performFP_TO_INTCombine().

matchSetCC()

static std::optional< bool > matchSetCC ( SDValue  LHS, SDValue  RHS, ISD::CondCode  CC, SDValue  Val  )

static

Definition at line 6847 of file RISCVISelLowering.cpp.

References assert(), CC, llvm::SDNode::getOpcode(), llvm::SDValue::getOperand(), llvm::ISD::getSetCCInverse(), llvm::ISD::getSetCCSwappedOperands(), llvm::SDValue::getValueType(), LHS, RHS, and llvm::ISD::SETCC.

Referenced by combineSelectToBinOp().

matchSplatAsGather()

static SDValue matchSplatAsGather ( SDValue  SplatVal, MVT  VT, const SDLoc &  DL, SelectionDAG &  DAG, const RISCVSubtarget &  Subtarget  )

static

Definition at line 3256 of file RISCVISelLowering.cpp.

References convertFromScalableVector(), convertToScalableVector(), DL, llvm::ISD::EXTRACT_VECTOR_ELT, getContainerForFixedLengthVector(), getDefaultVLOps(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SelectionDAG::getUNDEF(), llvm::SDValue::getValueType(), llvm::MVT::getVectorElementType(), llvm::RISCVSubtarget::getXLenVT(), Idx, llvm::MVT::isFixedLengthVector(), and llvm::RISCVISD::VRGATHER_VX_VL.

Referenced by lowerBUILD_VECTOR(), and llvm::RISCVTargetLowering::PerformDAGCombine().

narrowIndex()

static bool narrowIndex ( SDValue &  N, ISD::MemIndexType  IndexType, SelectionDAG &  DAG  )

static

According to the property that indexed load/store instructions zero-extend their indices, try to narrow the type of index operand.

Definition at line 12542 of file RISCVISelLowering.cpp.

References llvm::EVT::bitsLT(), llvm::CallingConv::C, llvm::EVT::changeVectorElementType(), llvm::SelectionDAG::computeKnownBits(), llvm::KnownBits::countMaxActiveBits(), DL, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getContext(), llvm::EVT::getIntegerVT(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDNode::getOpcode(), llvm::SDValue::getOperand(), llvm::EVT::getRoundIntegerType(), llvm::EVT::getScalarSizeInBits(), llvm::SDValue::getValueType(), llvm::EVT::getVectorElementType(), llvm::APInt::getZExtValue(), llvm::SDNode::hasOneUse(), llvm::ISD::isBuildVectorOfConstantSDNodes(), llvm::ISD::isConstantSplatVector(), N, llvm::SDNode::ops(), llvm::PowerOf2Ceil(), llvm::ISD::SHL, llvm::ISD::TRUNCATE, llvm::RISCVISD::VZEXT_VL, and llvm::ISD::ZERO_EXTEND.

Referenced by llvm::RISCVTargetLowering::PerformDAGCombine().

negateFMAOpcode()

static unsigned negateFMAOpcode ( unsigned  Opcode, bool  NegMul, bool  NegAcc  )

static

Definition at line 13604 of file RISCVISelLowering.cpp.

References llvm_unreachable, Opcode, llvm::RISCVISD::STRICT_VFMADD_VL, llvm::RISCVISD::STRICT_VFMSUB_VL, llvm::RISCVISD::STRICT_VFNMADD_VL, llvm::RISCVISD::STRICT_VFNMSUB_VL, llvm::RISCVISD::VFMADD_VL, llvm::RISCVISD::VFMSUB_VL, llvm::RISCVISD::VFNMADD_VL, and llvm::RISCVISD::VFNMSUB_VL.

Referenced by combineFMA(), combineFMADDSUB(), combineVFMADD_VLWithVFNEG_VL(), and llvm::X86TargetLowering::getNegatedExpression().

performADDCombine()

static SDValue performADDCombine ( SDNode *  N, SelectionDAG &  DAG, const RISCVSubtarget &  Subtarget  )

static

Definition at line 12186 of file RISCVISelLowering.cpp.

References combineAddOfBooleanXor(), combineBinOpOfExtractToReduceTree(), combineBinOpToReduce(), combineSelectAndUseCommutative(), N, transformAddImmMulImm(), and transformAddShlImm().

performANDCombine()

static SDValue performANDCombine ( SDNode *  N, TargetLowering::DAGCombinerInfo &  DCI, const RISCVSubtarget &  Subtarget  )

static

Definition at line 12343 of file RISCVISelLowering.cpp.

References llvm::And, llvm::ISD::AND, llvm::ISD::ANY_EXTEND, combineBinOpOfExtractToReduceTree(), combineBinOpToReduce(), combineDeMorganOfBoolean(), combineSelectAndUseCommutative(), llvm::TargetLowering::DAGCombinerInfo::DAG, DL, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDValue::hasOneUse(), llvm::RISCVSubtarget::is64Bit(), llvm::TargetLowering::DAGCombinerInfo::isAfterLegalizeDAG(), llvm::isOneConstant(), N, RV64LegalI32, llvm::ISD::SRL, llvm::ISD::TRUNCATE, and llvm::ISD::ZERO_EXTEND.

performBITREVERSECombine()

static SDValue performBITREVERSECombine ( SDNode *  N, SelectionDAG &  DAG, const RISCVSubtarget &  Subtarget  )

static

Definition at line 13584 of file RISCVISelLowering.cpp.

References assert(), llvm::RISCVISD::BREV8, llvm::ISD::BSWAP, DL, llvm::SelectionDAG::getNode(), llvm::EVT::getSizeInBits(), llvm::RISCVSubtarget::getXLen(), llvm::EVT::isScalarInteger(), and N.

Referenced by llvm::RISCVTargetLowering::PerformDAGCombine().

performBUILD_VECTORCombine()

static SDValue performBUILD_VECTORCombine ( SDNode *  N, SelectionDAG &  DAG, const RISCVSubtarget &  Subtarget, const RISCVTargetLowering &  TLI  )

static

If we have a build_vector where each lane is binop X, C, where C is a constant (but not necessarily the same constant on all lanes), form binop (build_vector x1, x2, ...), (build_vector c1, c2, c3, ..).

We assume that materializing a constant build vector will be no more expensive that performing O(n) binops.

Definition at line 14235 of file RISCVISelLowering.cpp.

References assert(), DL, llvm::SelectionDAG::getBuildVector(), llvm::SelectionDAG::getNode(), llvm::EVT::getVectorNumElements(), llvm::TargetLoweringBase::isBinOp(), llvm::TargetLoweringBase::isOperationLegalOrCustom(), llvm::SelectionDAG::isSafeToSpeculativelyExecute(), llvm::EVT::isScalableVector(), llvm::TargetLoweringBase::isTypeLegal(), N, Opcode, and llvm::SmallVectorTemplateBase< T, bool >::push_back().

Referenced by llvm::RISCVTargetLowering::PerformDAGCombine().

performCONCAT_VECTORSCombine()

static SDValue performCONCAT_VECTORSCombine ( SDNode *  N, SelectionDAG &  DAG, const RISCVSubtarget &  Subtarget, const RISCVTargetLowering &  TLI  )

static

Definition at line 14291 of file RISCVISelLowering.cpp.

References llvm::ISD::ADD, llvm::SmallVectorTemplateCommon< T, typename >::begin(), llvm::MVT::changeVectorElementType(), DL, llvm::SmallVectorTemplateCommon< T, typename >::end(), llvm::LoadSDNode::getBasePtr(), llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getConstant(), llvm::MVT::getIntegerVT(), llvm::SelectionDAG::getMachineFunction(), llvm::MachineFunction::getMachineMemOperand(), llvm::SelectionDAG::getMemIntrinsicNode(), llvm::SelectionDAG::getNegative(), llvm::getOffset(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::EVT::getScalarSizeInBits(), llvm::EVT::getSimpleVT(), llvm::MVT::getSizeInBits(), llvm::SelectionDAG::getSplat(), llvm::SelectionDAG::getTargetConstant(), llvm::SelectionDAG::getUNDEF(), llvm::SDValue::getValueType(), llvm::EVT::getVectorNumElements(), llvm::MVT::getVectorVT(), llvm::SelectionDAG::getVTList(), llvm::RISCVSubtarget::getXLenVT(), llvm::ISD::INTRINSIC_W_CHAIN, llvm::RISCVTargetLowering::isLegalStridedLoadStore(), llvm::ISD::isNormalLoad(), llvm::EVT::isScalableVector(), llvm::TargetLoweringBase::isTypeLegal(), llvm::SelectionDAG::makeEquivalentMemoryOrdering(), llvm::BaseIndexOffset::match(), N, llvm::SmallVectorTemplateBase< T, bool >::push_back(), and llvm::MemoryLocation::UnknownSize.

Referenced by llvm::RISCVTargetLowering::PerformDAGCombine().

performFADDSUB_VLCombine()

static SDValue performFADDSUB_VLCombine ( SDNode *  N, SelectionDAG &  DAG, const RISCVSubtarget &  Subtarget  )

static

Definition at line 13782 of file RISCVISelLowering.cpp.

References llvm::RISCVISD::FADD_VL, llvm::RISCVISD::FP_EXTEND_VL, llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDNode::hasNUsesOfValue(), llvm::SDValue::hasOneUse(), llvm::RISCVSubtarget::hasVInstructionsF16(), llvm::RISCVSubtarget::hasVInstructionsF16Minimal(), Merge, N, std::swap(), llvm::RISCVISD::VFWADD_VL, llvm::RISCVISD::VFWADD_W_VL, llvm::RISCVISD::VFWSUB_VL, and llvm::RISCVISD::VFWSUB_W_VL.

Referenced by llvm::RISCVTargetLowering::PerformDAGCombine().

performFP_TO_INT_SATCombine()

static SDValue performFP_TO_INT_SATCombine ( SDNode *  N, TargetLowering::DAGCombinerInfo &  DCI, const RISCVSubtarget &  Subtarget  )

static

Definition at line 13521 of file RISCVISelLowering.cpp.

References llvm::TargetLowering::DAGCombinerInfo::DAG, DL, llvm::RISCVISD::FCVT_W_RV64, llvm::RISCVISD::FCVT_WU_RV64, llvm::RISCVISD::FCVT_X, llvm::RISCVISD::FCVT_XU, llvm::ISD::FP_TO_SINT_SAT, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getSelectCC(), llvm::SelectionDAG::getTargetConstant(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::RISCVSubtarget::getXLenVT(), llvm::SelectionDAG::getZeroExtendInReg(), llvm::RISCVFPRndMode::Invalid, llvm::TargetLoweringBase::isTypeLegal(), matchRoundingOp(), N, and llvm::ISD::SETUO.

Referenced by llvm::RISCVTargetLowering::PerformDAGCombine().

performFP_TO_INTCombine()

static SDValue performFP_TO_INTCombine ( SDNode *  N, TargetLowering::DAGCombinerInfo &  DCI, const RISCVSubtarget &  Subtarget  )

static

Definition at line 13420 of file RISCVISelLowering.cpp.

References convertFromScalableVector(), convertToScalableVector(), llvm::TargetLowering::DAGCombinerInfo::DAG, DL, llvm::RISCVFPRndMode::DYN, llvm::RISCVISD::FCVT_W_RV64, llvm::RISCVISD::FCVT_WU_RV64, llvm::RISCVISD::FCVT_X, llvm::RISCVISD::FCVT_XU, llvm::ISD::FP_TO_SINT, getContainerForFixedLengthVector(), getDefaultVLOps(), llvm::SelectionDAG::getNode(), llvm::MVT::getScalarSizeInBits(), llvm::EVT::getScalarSizeInBits(), llvm::EVT::getSimpleVT(), llvm::SelectionDAG::getTargetConstant(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::RISCVSubtarget::getXLenVT(), llvm::RISCVFPRndMode::Invalid, llvm::MVT::isFixedLengthVector(), llvm::EVT::isFixedLengthVector(), llvm::TargetLoweringBase::isTypeLegal(), llvm::EVT::isVector(), matchRoundingOp(), N, llvm::RISCVFPRndMode::RTZ, llvm::ISD::TRUNCATE, llvm::RISCVISD::VFCVT_RM_X_F_VL, llvm::RISCVISD::VFCVT_RM_XU_F_VL, llvm::RISCVISD::VFCVT_RTZ_X_F_VL, and llvm::RISCVISD::VFCVT_RTZ_XU_F_VL.

Referenced by llvm::RISCVTargetLowering::PerformDAGCombine().

performMemPairCombine()

static SDValue performMemPairCombine ( SDNode *  N, TargetLowering::DAGCombinerInfo &  DCI  )

static

Definition at line 13334 of file RISCVISelLowering.cpp.

References llvm::ISD::ADD, llvm::TargetLowering::DAGCombinerInfo::DAG, getExtensionType(), llvm::SelectionDAG::getMachineFunction(), llvm::MemSDNode::getMemoryVT(), llvm::SDNode::getOpcode(), llvm::SDNode::getOperand(), llvm::MachineFunction::getSubtarget(), llvm::Use::getUser(), llvm::LSBaseSDNode::isIndexed(), llvm::MemSDNode::isSimple(), llvm::ISD::LOAD, N, Ptr, tryMemPairCombine(), llvm::SDNode::use_begin(), and llvm::SDNode::use_end().

Referenced by llvm::RISCVTargetLowering::PerformDAGCombine().

performMULCombine()

static SDValue performMULCombine ( SDNode *  N, SelectionDAG &  DAG  )

static

Definition at line 12499 of file RISCVISelLowering.cpp.

References llvm::ISD::ADD, DL, llvm::SelectionDAG::getNode(), llvm::isOneOrOneSplat(), llvm::EVT::isVector(), llvm::ISD::MUL, N, llvm::ISD::SUB, and std::swap().

performORCombine()

static SDValue performORCombine ( SDNode *  N, TargetLowering::DAGCombinerInfo &  DCI, const RISCVSubtarget &  Subtarget  )

static

Definition at line 12417 of file RISCVISelLowering.cpp.

References combineBinOpOfExtractToReduceTree(), combineBinOpToReduce(), combineDeMorganOfBoolean(), combineOrOfCZERO(), combineSelectAndUseCommutative(), llvm::TargetLowering::DAGCombinerInfo::DAG, llvm::TargetLowering::DAGCombinerInfo::isAfterLegalizeDAG(), and N.

performSELECTCombine()

static SDValue performSELECTCombine ( SDNode *  N, SelectionDAG &  DAG, const RISCVSubtarget &  Subtarget  )

static

Definition at line 14215 of file RISCVISelLowering.cpp.

References foldSelectOfCTTZOrCTLZ(), N, and tryFoldSelectIntoOp().

performSETCCCombine()

static SDValue performSETCCCombine ( SDNode *  N, SelectionDAG &  DAG, const RISCVSubtarget &  Subtarget  )

static

Definition at line 12611 of file RISCVISelLowering.cpp.

References llvm::ISD::AND, Cond, llvm::APInt::getActiveBits(), llvm::SelectionDAG::getBoolConstant(), llvm::SelectionDAG::getConstant(), llvm::SDValue::getConstantOperandVal(), llvm::SelectionDAG::getNode(), llvm::APInt::getOneBitSet(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SelectionDAG::getSetCC(), llvm::SDValue::getValueType(), llvm::SelectionDAG::getValueType(), llvm::SDValue::hasOneUse(), llvm::RISCVSubtarget::is64Bit(), llvm::SelectionDAG::MaskedValueIsZero(), N, llvm::ISD::SETNE, llvm::APInt::sext(), llvm::ISD::SIGN_EXTEND_INREG, and llvm::APInt::trunc().

performSIGN_EXTEND_INREGCombine()

static SDValue performSIGN_EXTEND_INREGCombine ( SDNode *  N, SelectionDAG &  DAG, const RISCVSubtarget &  Subtarget  )

static

Definition at line 12658 of file RISCVISelLowering.cpp.

References llvm::RISCVISD::FMV_X_ANYEXTH, llvm::RISCVISD::FMV_X_SIGNEXTH, llvm::SelectionDAG::getNode(), and N.

Referenced by llvm::RISCVTargetLowering::PerformDAGCombine().

performSRACombine()

static SDValue performSRACombine ( SDNode *  N, SelectionDAG &  DAG, const RISCVSubtarget &  Subtarget  )

static

Definition at line 13842 of file RISCVISelLowering.cpp.

References llvm::ISD::ADD, assert(), llvm::APInt::countr_zero(), DL, llvm::ConstantSDNode::getAPIntValue(), llvm::SelectionDAG::getConstant(), llvm::SDValue::getConstantOperandVal(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SelectionDAG::getValueType(), llvm::SDValue::hasOneUse(), llvm::RISCVSubtarget::is64Bit(), llvm::APInt::lshr(), N, llvm::ISD::SHL, llvm::ISD::SIGN_EXTEND_INREG, llvm::ISD::SRA, llvm::ISD::SUB, and llvm::SDNode::uses().

performSUBCombine()

static SDValue performSUBCombine ( SDNode *  N, SelectionDAG &  DAG, const RISCVSubtarget &  Subtarget  )

static

Definition at line 12245 of file RISCVISelLowering.cpp.

References combineSelectAndUse(), combineSubOfBoolean(), DL, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDValue::getValueSizeInBits(), llvm::SDValue::hasOneUse(), llvm::isNullConstant(), N, llvm::ISD::SETCC, llvm::ISD::SETLT, and llvm::ISD::SRA.

performTRUNCATECombine()

static SDValue performTRUNCATECombine ( SDNode *  N, SelectionDAG &  DAG, const RISCVSubtarget &  Subtarget  )

static

Definition at line 12317 of file RISCVISelLowering.cpp.

References llvm::ISD::ANY_EXTEND, DL, llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDValue::getValueType(), llvm::SDValue::hasOneUse(), llvm::RISCVSubtarget::is64Bit(), N, RV64LegalI32, llvm::ISD::SRL, llvm::ISD::TRUNCATE, and llvm::ISD::ZERO_EXTEND.

Referenced by llvm::RISCVTargetLowering::PerformDAGCombine().

performVFMADD_VLCombine()

static SDValue performVFMADD_VLCombine ( SDNode *  N, SelectionDAG &  DAG, const RISCVSubtarget &  Subtarget  )

static

Definition at line 13679 of file RISCVISelLowering.cpp.

References combineVFMADD_VLWithVFNEG_VL(), llvm::RISCVISD::FP_EXTEND_VL, llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDNode::hasNUsesOfValue(), llvm::SDValue::hasOneUse(), llvm::RISCVSubtarget::hasVInstructionsF16(), llvm::RISCVSubtarget::hasVInstructionsF16Minimal(), llvm_unreachable, N, llvm::RISCVISD::VFMADD_VL, llvm::RISCVISD::VFMSUB_VL, llvm::RISCVISD::VFNMADD_VL, llvm::RISCVISD::VFNMSUB_VL, llvm::RISCVISD::VFWMADD_VL, llvm::RISCVISD::VFWMSUB_VL, llvm::RISCVISD::VFWNMADD_VL, and llvm::RISCVISD::VFWNMSUB_VL.

Referenced by llvm::RISCVTargetLowering::PerformDAGCombine().

performVFMUL_VLCombine()

static SDValue performVFMUL_VLCombine ( SDNode *  N, SelectionDAG &  DAG, const RISCVSubtarget &  Subtarget  )

static

Definition at line 13741 of file RISCVISelLowering.cpp.

References assert(), llvm::RISCVISD::FP_EXTEND_VL, llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDNode::hasNUsesOfValue(), llvm::SDValue::hasOneUse(), llvm::RISCVSubtarget::hasVInstructionsF16(), llvm::RISCVSubtarget::hasVInstructionsF16Minimal(), Merge, N, and llvm::RISCVISD::VFWMUL_VL.

Referenced by llvm::RISCVTargetLowering::PerformDAGCombine().

performXORCombine()

static SDValue performXORCombine ( SDNode *  N, SelectionDAG &  DAG, const RISCVSubtarget &  Subtarget  )

static

Definition at line 12444 of file RISCVISelLowering.cpp.

References llvm::And, llvm::ISD::ANY_EXTEND, CC, combineBinOpOfExtractToReduceTree(), combineBinOpToReduce(), combineSelectAndUseCommutative(), DL, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getNOT(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SelectionDAG::getSetCC(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::SDValue::getValueType(), llvm::SDValue::hasOneUse(), llvm::RISCVSubtarget::is64Bit(), llvm::isAllOnesConstant(), llvm::isOneConstant(), llvm::TargetLoweringBase::isOperationLegal(), N, llvm::RISCVISD::ROLW, llvm::ISD::ROTL, RV64LegalI32, llvm::ISD::SETCC, llvm::ISD::SETLT, llvm::ISD::SHL, llvm::RISCVISD::SLLW, llvm::ISD::TRUNCATE, and llvm::ISD::ZERO_EXTEND.

preAssignMask()

template<typename ArgTy >

static std::optional< unsigned > preAssignMask ( const ArgTy &  Args )

static

Definition at line 17029 of file RISCVISelLowering.cpp.

References llvm::enumerate(), llvm::MVT::getVectorElementType(), and llvm::MVT::isVector().

Referenced by llvm::RISCVTargetLowering::CanLowerReturn().

splatPartsI64WithVL()

static SDValue splatPartsI64WithVL ( const SDLoc &  DL, MVT  VT, SDValue  Passthru, SDValue  Lo, SDValue  Hi, SDValue  VL, SelectionDAG &  DAG  )

static

Definition at line 3882 of file RISCVISelLowering.cpp.

References llvm::ISD::ADD, llvm::ISD::BITCAST, DL, llvm::SelectionDAG::getNode(), getReg(), llvm::SelectionDAG::getRegister(), llvm::SelectionDAG::getUNDEF(), llvm::SDValue::getValueType(), llvm::MVT::getVectorElementCount(), llvm::MVT::getVectorVT(), llvm::Hi, llvm::isAllOnesConstant(), llvm::Lo, llvm::RISCVISD::SPLAT_VECTOR_SPLIT_I64_VL, llvm::ISD::SRA, and llvm::RISCVISD::VMV_V_X_VL.

Referenced by splatSplitI64WithVL().

splatSplitI64WithVL()

static SDValue splatSplitI64WithVL ( const SDLoc &  DL, MVT  VT, SDValue  Passthru, SDValue  Scalar, SDValue  VL, SelectionDAG &  DAG  )

static

Definition at line 3940 of file RISCVISelLowering.cpp.

References assert(), DL, llvm::Hi, llvm::Lo, splatPartsI64WithVL(), and llvm::SelectionDAG::SplitScalar().

Referenced by lowerScalarSplat(), and lowerVectorIntrinsicScalars().

SplitStrictFPVectorOp()

static SDValue SplitStrictFPVectorOp ( SDValue  Op, SelectionDAG &  DAG  )

static

Definition at line 5641 of file RISCVISelLowering.cpp.

References assert(), llvm::ISD::CONCAT_VECTORS, DL, llvm::SelectionDAG::getMergeValues(), llvm::SelectionDAG::getNode(), llvm::DWARFExpression::Operation::getNumOperands(), llvm::SelectionDAG::GetSplitDestVTs(), llvm::SDValue::getValue(), llvm::SelectionDAG::getVTList(), and llvm::SelectionDAG::SplitVector().

Referenced by llvm::RISCVTargetLowering::LowerOperation().

SplitVectorOp()

static SDValue SplitVectorOp ( SDValue  Op, SelectionDAG &  DAG  )

static

Definition at line 5570 of file RISCVISelLowering.cpp.

References llvm::ISD::CONCAT_VECTORS, DL, llvm::SelectionDAG::getNode(), llvm::DWARFExpression::Operation::getNumOperands(), llvm::SelectionDAG::GetSplitDestVTs(), and llvm::SelectionDAG::SplitVector().

Referenced by llvm::RISCVTargetLowering::LowerOperation().

SplitVectorReductionOp()

static SDValue SplitVectorReductionOp ( SDValue  Op, SelectionDAG &  DAG  )

static

Definition at line 5626 of file RISCVISelLowering.cpp.

References DL, llvm::SelectionDAG::getNode(), llvm::Hi, llvm::Lo, llvm::SelectionDAG::SplitEVL(), and llvm::SelectionDAG::SplitVector().

Referenced by llvm::RISCVTargetLowering::LowerOperation().

SplitVPOp()

static SDValue SplitVPOp ( SDValue  Op, SelectionDAG &  DAG  )

static

Definition at line 5595 of file RISCVISelLowering.cpp.

References assert(), llvm::ISD::CONCAT_VECTORS, DL, llvm::SelectionDAG::getNode(), llvm::DWARFExpression::Operation::getNumOperands(), llvm::SelectionDAG::GetSplitDestVTs(), llvm::ISD::getVPExplicitVectorLengthIdx(), llvm::ISD::isVPOpcode(), llvm::SelectionDAG::SplitEVL(), and llvm::SelectionDAG::SplitVector().

Referenced by llvm::RISCVTargetLowering::LowerOperation().

STATISTIC()

STATISTIC	(	NumTailCalls	,
		"Number of tail calls"
	)

transformAddImmMulImm()

static SDValue transformAddImmMulImm ( SDNode *  N, SelectionDAG &  DAG, const RISCVSubtarget &  Subtarget  )

static

Definition at line 12112 of file RISCVISelLowering.cpp.

References llvm::ISD::ADD, DL, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::SDNode::getOpcode(), llvm::SDNode::getOperand(), llvm::EVT::getSizeInBits(), llvm::RISCVSubtarget::getXLen(), llvm::SDNode::hasOneUse(), llvm::EVT::isVector(), llvm::ISD::MUL, and N.

Referenced by performADDCombine().

transformAddShlImm()

static SDValue transformAddShlImm ( SDNode *  N, SelectionDAG &  DAG, const RISCVSubtarget &  Subtarget  )

static

Definition at line 11961 of file RISCVISelLowering.cpp.

References llvm::ISD::ADD, DL, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::SDNode::getOpcode(), llvm::SDNode::getOperand(), llvm::EVT::getSizeInBits(), llvm::RISCVSubtarget::getXLen(), llvm::SDNode::hasOneUse(), llvm::EVT::isVector(), N, NL, and llvm::ISD::SHL.

Referenced by performADDCombine().

translateSetCCForBranch()

static void translateSetCCForBranch ( const SDLoc &  DL, SDValue &  LHS, SDValue &  RHS, ISD::CondCode &  CC, SelectionDAG &  DAG  )

static

Definition at line 2209 of file RISCVISelLowering.cpp.

References llvm::ISD::AND, llvm::bit_width(), llvm::CallingConv::C, CC, DL, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::ISD::getSetCCSwappedOperands(), llvm::Value::hasOneUse(), llvm::isMask_64(), llvm::isNullConstant(), llvm::isPowerOf2_64(), LHS, llvm::Log2_64(), RHS, llvm::ISD::SETEQ, llvm::ISD::SETGE, llvm::ISD::SETGT, llvm::ISD::SETLE, llvm::ISD::SETLT, llvm::ISD::SETUGT, llvm::ISD::SETULE, llvm::ISD::SHL, and std::swap().

Referenced by combine_CC().

tryDemorganOfBooleanCondition()

static SDValue tryDemorganOfBooleanCondition ( SDValue  Cond, SelectionDAG &  DAG  )

static

Definition at line 13954 of file RISCVISelLowering.cpp.

References llvm::ISD::AND, Cond, llvm::APInt::getBitsSetFrom(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SelectionDAG::getSetCC(), llvm::ISD::getSetCCInverse(), llvm::EVT::getSizeInBits(), llvm::SDValue::getValueType(), llvm::SDValue::hasOneUse(), llvm::isAllOnesConstant(), llvm::ISD::isIntEqualitySetCC(), llvm::isNullConstant(), llvm::isOneConstant(), llvm::EVT::isScalarInteger(), llvm::SelectionDAG::MaskedValueIsZero(), llvm::ISD::OR, llvm::ISD::SETCC, llvm::ISD::SETLT, std::swap(), llvm::Xor, and llvm::ISD::XOR.

Referenced by combine_CC().

tryFoldSelectIntoOp()

static SDValue tryFoldSelectIntoOp ( SDNode *  N, SelectionDAG &  DAG, SDValue  TrueVal, SDValue  FalseVal, bool  Swapped  )

static

Definition at line 14106 of file RISCVISelLowering.cpp.

References llvm::ISD::ADD, assert(), DL, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNeutralElement(), llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getSelect(), llvm::SDNode::getValueType(), N, llvm::ISD::OR, llvm::ISD::SHL, llvm::ISD::SRA, llvm::ISD::SRL, llvm::ISD::SUB, std::swap(), and llvm::ISD::XOR.

Referenced by performSELECTCombine().

tryMemPairCombine()

static SDValue tryMemPairCombine ( SelectionDAG &  DAG, LSBaseSDNode *  LSNode1, LSBaseSDNode *  LSNode2, SDValue  BasePtr, uint64_t  Imm  )

static

Definition at line 13276 of file RISCVISelLowering.cpp.

References llvm::MemSDNode::getChain(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getMachineFunction(), llvm::MachineFunction::getMachineMemOperand(), llvm::SelectionDAG::getMemIntrinsicNode(), llvm::MemSDNode::getMemOperand(), llvm::MemSDNode::getMemoryVT(), llvm::SelectionDAG::getMergeValues(), llvm::SDValue::getNode(), llvm::SDNode::getOpcode(), llvm::SDNode::getOperand(), llvm::MachineMemOperand::getPointerInfo(), llvm::MachineFunction::getSubtarget(), llvm::SDValue::getValue(), llvm::SelectionDAG::getVTList(), llvm::RISCVSubtarget::getXLenVT(), llvm::SDNode::hasPredecessorHelper(), llvm::ISD::LOAD, Opcode, llvm::SelectionDAG::ReplaceAllUsesWith(), llvm::RISCVISD::TH_LDD, llvm::RISCVISD::TH_LWD, llvm::RISCVISD::TH_LWUD, llvm::RISCVISD::TH_SDD, llvm::RISCVISD::TH_SWD, and llvm::ISD::ZEXTLOAD.

Referenced by performMemPairCombine().

unpackF64OnRV32DSoftABI()

static SDValue unpackF64OnRV32DSoftABI ( SelectionDAG &  DAG, SDValue  Chain, const CCValAssign &  VA, const CCValAssign &  HiVA, const SDLoc &  DL  )

static

Definition at line 17229 of file RISCVISelLowering.cpp.

References assert(), llvm::RISCVISD::BuildPairF64, llvm::MachineFrameInfo::CreateFixedObject(), DL, llvm::SelectionDAG::getCopyFromReg(), llvm::MachinePointerInfo::getFixedStack(), llvm::SelectionDAG::getFrameIndex(), llvm::MachineFunction::getFrameInfo(), llvm::SelectionDAG::getLoad(), llvm::CCValAssign::getLocMemOffset(), llvm::CCValAssign::getLocReg(), llvm::CCValAssign::getLocVT(), llvm::SelectionDAG::getMachineFunction(), llvm::SelectionDAG::getNode(), llvm::MachineFunction::getRegInfo(), llvm::CCValAssign::getValVT(), llvm::Hi, llvm::CCValAssign::isMemLoc(), llvm::CCValAssign::isRegLoc(), and llvm::Lo.

Referenced by llvm::RISCVTargetLowering::LowerFormalArguments().

unpackFromMemLoc()

static SDValue unpackFromMemLoc ( SelectionDAG &  DAG, SDValue  Chain, const CCValAssign &  VA, const SDLoc &  DL  )

static

Definition at line 17195 of file RISCVISelLowering.cpp.

References llvm::CCValAssign::BCvt, llvm::MachineFrameInfo::CreateFixedObject(), DL, llvm::CCValAssign::Full, llvm::SelectionDAG::getDataLayout(), llvm::SelectionDAG::getExtLoad(), llvm::MachinePointerInfo::getFixedStack(), llvm::SelectionDAG::getFrameIndex(), llvm::MachineFunction::getFrameInfo(), llvm::MVT::getIntegerVT(), llvm::CCValAssign::getLocInfo(), llvm::CCValAssign::getLocMemOffset(), llvm::CCValAssign::getLocVT(), llvm::SelectionDAG::getMachineFunction(), llvm::DataLayout::getPointerSizeInBits(), llvm::EVT::getStoreSize(), llvm::CCValAssign::getValVT(), llvm::CCValAssign::Indirect, llvm::EVT::isScalableVector(), llvm_unreachable, and llvm::ISD::NON_EXTLOAD.

unpackFromRegLoc()

static SDValue unpackFromRegLoc ( SelectionDAG &  DAG, SDValue  Chain, const CCValAssign &  VA, const SDLoc &  DL, const ISD::InputArg &  In, const RISCVTargetLowering &  TLI  )

static

Definition at line 17129 of file RISCVISelLowering.cpp.

References llvm::RISCVMachineFunctionInfo::addSExt32Register(), llvm::BitWidth, convertLocVTToValVT(), DL, llvm::Function::getArg(), llvm::SelectionDAG::getCopyFromReg(), llvm::MachineFunction::getFunction(), llvm::MachineFunction::getInfo(), llvm::Type::getIntegerBitWidth(), llvm::CCValAssign::getLocInfo(), llvm::CCValAssign::getLocReg(), llvm::CCValAssign::getLocVT(), llvm::SelectionDAG::getMachineFunction(), llvm::TargetLoweringBase::getRegClassFor(), llvm::MachineFunction::getRegInfo(), llvm::EVT::getSimpleVT(), llvm::RISCVTargetLowering::getSubtarget(), llvm::Value::getType(), llvm::CCValAssign::Indirect, and llvm::Type::isIntegerTy().

useRVVForFixedLengthVectorVT()

static bool useRVVForFixedLengthVectorVT ( MVT  VT, const RISCVSubtarget &  Subtarget  )

static

Definition at line 2428 of file RISCVISelLowering.cpp.

References assert(), llvm::divideCeil(), llvm::RISCVSubtarget::getELen(), llvm::MVT::getFixedSizeInBits(), llvm::RISCVSubtarget::getMaxLMULForFixedLengthVectors(), llvm::RISCVSubtarget::getRealMinVLen(), llvm::MVT::getSizeInBits(), llvm::MVT::getVectorElementType(), llvm::MVT::getVectorNumElements(), llvm::RISCVSubtarget::hasVInstructionsF16Minimal(), llvm::RISCVSubtarget::hasVInstructionsF32(), llvm::RISCVSubtarget::hasVInstructionsF64(), llvm::RISCVSubtarget::hasVInstructionsI64(), llvm::MVT::isFixedLengthVector(), llvm::MVT::isPow2VectorType(), llvm::MVT::SimpleTy, and llvm::RISCVSubtarget::useRVVForFixedLengthVectors().

Referenced by getContainerForFixedLengthVector().

useTpOffset()

static Value * useTpOffset ( IRBuilderBase &  IRB, unsigned  Offset  )

static

Definition at line 19332 of file RISCVISelLowering.cpp.

References llvm::IRBuilderBase::CreateCall(), llvm::IRBuilderBase::CreateConstGEP1_32(), llvm::Intrinsic::getDeclaration(), llvm::IRBuilderBase::GetInsertBlock(), llvm::IRBuilderBase::getInt8Ty(), llvm::GlobalValue::getParent(), llvm::BasicBlock::getParent(), and llvm::Offset.

Referenced by llvm::RISCVTargetLowering::getIRStackGuard().

widenVectorOpsToi8()

static SDValue widenVectorOpsToi8 ( SDValue  N, const SDLoc &  DL, SelectionDAG &  DAG  )

static

Definition at line 9449 of file RISCVISelLowering.cpp.

References assert(), llvm::MVT::changeVectorElementType(), DL, llvm::SmallVectorTemplateCommon< T, typename >::front(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getMergeValues(), llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getSetCC(), llvm::SDValue::getValue(), llvm::SelectionDAG::getVTList(), I, N, llvm::SmallVectorTemplateBase< T, bool >::push_back(), llvm::ISD::SETNE, llvm::SmallVectorBase< Size_T >::size(), and llvm::ISD::ZERO_EXTEND.

Variable Documentation

AllowSplatInVW_W

cl::opt< bool > AllowSplatInVW_W(DEBUG_TYPE "-form-vw-w-with-splat", cl::Hidden, cl::desc("Allow the formation of VW_W operations (e.g., " "VWADD_W) with splat constants"), cl::init(false)) ( DEBUG_TYPE "-form-vw-w-with-splat"  , cl::Hidden  , cl::desc("Allow the formation of VW_W operations (e.g., " "VWADD_W) with splat constants")  , cl::init(false)    )

static

ArgFPR16s

const MCPhysReg ArgFPR16s[]

static

Initial value:

= {
  RISCV::F10_H, RISCV::F11_H, RISCV::F12_H, RISCV::F13_H,
  RISCV::F14_H, RISCV::F15_H, RISCV::F16_H, RISCV::F17_H
}

Definition at line 16710 of file RISCVISelLowering.cpp.

Referenced by llvm::RISCV::CC_RISCV().

ArgFPR32s

const MCPhysReg ArgFPR32s[]

static

Initial value:

= {
  RISCV::F10_F, RISCV::F11_F, RISCV::F12_F, RISCV::F13_F,
  RISCV::F14_F, RISCV::F15_F, RISCV::F16_F, RISCV::F17_F
}

Definition at line 16714 of file RISCVISelLowering.cpp.

ArgFPR64s

const MCPhysReg ArgFPR64s[]

static

Initial value:

= {
  RISCV::F10_D, RISCV::F11_D, RISCV::F12_D, RISCV::F13_D,
  RISCV::F14_D, RISCV::F15_D, RISCV::F16_D, RISCV::F17_D
}

Definition at line 16718 of file RISCVISelLowering.cpp.

ArgGPRs

const MCPhysReg ArgGPRs[]

static

Initial value:

= {
  RISCV::X10, RISCV::X11, RISCV::X12, RISCV::X13,
  RISCV::X14, RISCV::X15, RISCV::X16, RISCV::X17
}

Definition at line 16706 of file RISCVISelLowering.cpp.

ArgVRM2s

const MCPhysReg ArgVRM2s[]

static

Initial value:

= {RISCV::V8M2,  RISCV::V10M2, RISCV::V12M2,
                                     RISCV::V14M2, RISCV::V16M2, RISCV::V18M2,
                                     RISCV::V20M2, RISCV::V22M2}

Definition at line 16727 of file RISCVISelLowering.cpp.

Referenced by allocateRVVReg().

ArgVRM4s

const MCPhysReg ArgVRM4s[]

static

Initial value:

= {RISCV::V8M4, RISCV::V12M4, RISCV::V16M4,
                                     RISCV::V20M4}

Definition at line 16730 of file RISCVISelLowering.cpp.

Referenced by allocateRVVReg().

ArgVRM8s

const MCPhysReg ArgVRM8s[] = {RISCV::V8M8, RISCV::V16M8}

static

Definition at line 16732 of file RISCVISelLowering.cpp.

Referenced by allocateRVVReg().

ArgVRs

const MCPhysReg ArgVRs[]

static

Initial value:

= {
    RISCV::V8,  RISCV::V9,  RISCV::V10, RISCV::V11, RISCV::V12, RISCV::V13,
    RISCV::V14, RISCV::V15, RISCV::V16, RISCV::V17, RISCV::V18, RISCV::V19,
    RISCV::V20, RISCV::V21, RISCV::V22, RISCV::V23}

Definition at line 16723 of file RISCVISelLowering.cpp.

ExtensionMaxWebSize

cl::opt< unsigned > ExtensionMaxWebSize(DEBUG_TYPE "-ext-max-web-size", cl::Hidden, cl::desc("Give the maximum size (in number of nodes) of the web of " "instructions that we will consider for VW expansion"), cl::init(18)) ( DEBUG_TYPE "-ext-max-web-size"  , cl::Hidden  , cl::desc("Give the maximum size (in number of nodes) of the web of " "instructions that we will consider for VW expansion")  , cl::init(18)    )

static

Referenced by combineBinOp_VLToVWBinOp_VL().

FixedVlsegIntrIds

const Intrinsic::ID FixedVlsegIntrIds[]

static

Initial value:

= {
    Intrinsic::riscv_seg2_load, Intrinsic::riscv_seg3_load,
    Intrinsic::riscv_seg4_load, Intrinsic::riscv_seg5_load,
    Intrinsic::riscv_seg6_load, Intrinsic::riscv_seg7_load,
    Intrinsic::riscv_seg8_load}

Definition at line 19402 of file RISCVISelLowering.cpp.

Referenced by llvm::RISCVTargetLowering::lowerDeinterleaveIntrinsicToLoad(), and llvm::RISCVTargetLowering::lowerInterleavedLoad().

FixedVssegIntrIds

const Intrinsic::ID FixedVssegIntrIds[]

static

Initial value:

= {
    Intrinsic::riscv_seg2_store, Intrinsic::riscv_seg3_store,
    Intrinsic::riscv_seg4_store, Intrinsic::riscv_seg5_store,
    Intrinsic::riscv_seg6_store, Intrinsic::riscv_seg7_store,
    Intrinsic::riscv_seg8_store}

Definition at line 19450 of file RISCVISelLowering.cpp.

Referenced by llvm::RISCVTargetLowering::lowerInterleavedStore(), and llvm::RISCVTargetLowering::lowerInterleaveIntrinsicToStore().

FPImmCost

cl::opt< int > FPImmCost(DEBUG_TYPE "-fpimm-cost", cl::Hidden, cl::desc("Give the maximum number of instructions that we will " "use for creating a floating-point immediate value"), cl::init(2)) ( DEBUG_TYPE "-fpimm-cost"  , cl::Hidden  , cl::desc("Give the maximum number of instructions that we will " "use for creating a floating-point immediate value")  , cl::init(2)    )

static

Referenced by llvm::RISCVTargetLowering::isFPImmLegal().

NumRepeatedDivisors

cl::opt< unsigned > NumRepeatedDivisors(DEBUG_TYPE "-fp-repeated-divisors", cl::Hidden, cl::desc("Set the minimum number of repetitions of a divisor to allow " "transformation to multiplications by the reciprocal"), cl::init(2)) ( DEBUG_TYPE "-fp-repeated-divisors"  , cl::Hidden  , cl::desc("Set the minimum number of repetitions of a divisor to allow " "transformation to multiplications by the reciprocal")  , cl::init(2)    )

static

RV64LegalI32

cl::opt< bool > RV64LegalI32("riscv-experimental-rv64-legal-i32", cl::ReallyHidden, cl::desc("Make i32 a legal type for SelectionDAG on RV64.")) ( "riscv-experimental-rv64-legal-i32"  , cl::ReallyHidden  , cl::desc("Make i32 a legal type for SelectionDAG on RV64.")    )

static

Referenced by convertLocVTToValVT(), convertValVTToLocVT(), llvm::RISCVTargetLowering::getRegisterTypeForCallingConv(), llvm::RISCVTargetLowering::getVectorTypeBreakdownForCallingConv(), llvm::RISCVTargetLowering::LowerOperation(), performANDCombine(), performTRUNCATECombine(), performXORCombine(), and llvm::RISCVTargetLowering::RISCVTargetLowering().