LLVM  12.0.0git
Namespaces | Macros | Enumerations | Functions | Variables
X86ISelLowering.cpp File Reference
#include "X86ISelLowering.h"
#include "MCTargetDesc/X86ShuffleDecode.h"
#include "X86.h"
#include "X86CallingConv.h"
#include "X86FrameLowering.h"
#include "X86InstrBuilder.h"
#include "X86IntrinsicsInfo.h"
#include "X86MachineFunctionInfo.h"
#include "X86TargetMachine.h"
#include "X86TargetObjectFile.h"
#include "llvm/ADT/SmallBitVector.h"
#include "llvm/ADT/SmallSet.h"
#include "llvm/ADT/Statistic.h"
#include "llvm/ADT/StringExtras.h"
#include "llvm/ADT/StringSwitch.h"
#include "llvm/Analysis/BlockFrequencyInfo.h"
#include "llvm/Analysis/EHPersonalities.h"
#include "llvm/Analysis/ProfileSummaryInfo.h"
#include "llvm/Analysis/VectorUtils.h"
#include "llvm/CodeGen/IntrinsicLowering.h"
#include "llvm/CodeGen/MachineFrameInfo.h"
#include "llvm/CodeGen/MachineFunction.h"
#include "llvm/CodeGen/MachineInstrBuilder.h"
#include "llvm/CodeGen/MachineJumpTableInfo.h"
#include "llvm/CodeGen/MachineModuleInfo.h"
#include "llvm/CodeGen/MachineRegisterInfo.h"
#include "llvm/CodeGen/TargetLowering.h"
#include "llvm/CodeGen/WinEHFuncInfo.h"
#include "llvm/IR/CallingConv.h"
#include "llvm/IR/Constants.h"
#include "llvm/IR/DerivedTypes.h"
#include "llvm/IR/DiagnosticInfo.h"
#include "llvm/IR/Function.h"
#include "llvm/IR/GlobalAlias.h"
#include "llvm/IR/GlobalVariable.h"
#include "llvm/IR/Instructions.h"
#include "llvm/IR/Intrinsics.h"
#include "llvm/MC/MCAsmInfo.h"
#include "llvm/MC/MCContext.h"
#include "llvm/MC/MCExpr.h"
#include "llvm/MC/MCSymbol.h"
#include "llvm/Support/CommandLine.h"
#include "llvm/Support/Debug.h"
#include "llvm/Support/ErrorHandling.h"
#include "llvm/Support/KnownBits.h"
#include "llvm/Support/MathExtras.h"
#include "llvm/Target/TargetOptions.h"
#include <algorithm>
#include <bitset>
#include <cctype>
#include <numeric>

Go to the source code of this file.

Namespaces

 llvm
 This class represents lattice values for constants.
 
 llvm::X86
 Define some predicates that are used for node matching.
 

Macros

#define DEBUG_TYPE   "x86-isel"
 
#define NODE_NAME_CASE(NODE)   case X86ISD::NODE: return "X86ISD::" #NODE;
 

Enumerations

enum  StructReturnType { NotStructReturn, RegStructReturn, StackStructReturn }
 CallIsStructReturn - Determines whether a call uses struct return semantics. More...
 
enum  ShrinkMode { ShrinkMode::MULS8, ShrinkMode::MULU8, ShrinkMode::MULS16, ShrinkMode::MULU16 }
 Different mul shrinking modes. More...
 

Functions

 STATISTIC (NumTailCalls, "Number of tail calls")
 
static void errorUnsupported (SelectionDAG &DAG, const SDLoc &dl, const char *Msg)
 Call this when the user attempts to do something unsupported, like returning a double without SSE2 enabled on x86_64. More...
 
static std::pair< MVT, unsignedhandleMaskRegisterForCallingConv (unsigned NumElts, CallingConv::ID CC, const X86Subtarget &Subtarget)
 
static void getMaxByValAlign (Type *Ty, Align &MaxAlign)
 Helper for getByValTypeAlignment to determine the desired ByVal argument alignment. More...
 
static bool hasStackGuardSlotTLS (const Triple &TargetTriple)
 
static ConstantSegmentOffset (IRBuilder<> &IRB, unsigned Offset, unsigned AddressSpace)
 
static SDValue lowerMasksToReg (const SDValue &ValArg, const EVT &ValLoc, const SDLoc &Dl, SelectionDAG &DAG)
 Lowers masks values (v*i1) to the local register values. More...
 
static void Passv64i1ArgInRegs (const SDLoc &Dl, SelectionDAG &DAG, SDValue &Arg, SmallVectorImpl< std::pair< Register, SDValue >> &RegsToPass, CCValAssign &VA, CCValAssign &NextVA, const X86Subtarget &Subtarget)
 Breaks v64i1 value into two registers and adds the new node to the DAG. More...
 
static SDValue getv64i1Argument (CCValAssign &VA, CCValAssign &NextVA, SDValue &Root, SelectionDAG &DAG, const SDLoc &Dl, const X86Subtarget &Subtarget, SDValue *InFlag=nullptr)
 Reads two 32 bit registers and creates a 64 bit mask value. More...
 
static SDValue lowerRegToMasks (const SDValue &ValArg, const EVT &ValVT, const EVT &ValLoc, const SDLoc &Dl, SelectionDAG &DAG)
 The function will lower a register of various sizes (8/16/32/64) to a mask value of the expected size (v8i1/v16i1/v32i1/v64i1) More...
 
static StructReturnType callIsStructReturn (ArrayRef< ISD::OutputArg > Outs, bool IsMCU)
 
static StructReturnType argsAreStructReturn (ArrayRef< ISD::InputArg > Ins, bool IsMCU)
 Determines whether a function uses struct return semantics. More...
 
static SDValue CreateCopyOfByValArgument (SDValue Src, SDValue Dst, SDValue Chain, ISD::ArgFlagsTy Flags, SelectionDAG &DAG, const SDLoc &dl)
 Make a copy of an aggregate at address specified by "Src" to address "Dst" with size and alignment information specified by the specific parameter attribute. More...
 
static bool canGuaranteeTCO (CallingConv::ID CC)
 Return true if the calling convention is one that we can guarantee TCO for. More...
 
static bool mayTailCallThisCC (CallingConv::ID CC)
 Return true if we might ever do TCO for calls with this calling convention. More...
 
static bool shouldGuaranteeTCO (CallingConv::ID CC, bool GuaranteedTailCallOpt)
 Return true if the function is being made into a tailcall target by changing its ABI. More...
 
static ArrayRef< MCPhysReg > get64BitArgumentGPRs (CallingConv::ID CallConv, const X86Subtarget &Subtarget)
 
static ArrayRef< MCPhysReg > get64BitArgumentXMMs (MachineFunction &MF, CallingConv::ID CallConv, const X86Subtarget &Subtarget)
 
static bool isSortedByValueNo (ArrayRef< CCValAssign > ArgLocs)
 
static SDValue EmitTailCallStoreRetAddr (SelectionDAG &DAG, MachineFunction &MF, SDValue Chain, SDValue RetAddrFrIdx, EVT PtrVT, unsigned SlotSize, int FPDiff, const SDLoc &dl)
 Emit a store of the return address if tail call optimization is performed and it is required (FPDiff!=0). More...
 
static SDValue getMOVL (SelectionDAG &DAG, const SDLoc &dl, MVT VT, SDValue V1, SDValue V2)
 Returns a vector_shuffle mask for an movs{s|d}, movd operation of specified width. More...
 
static bool MatchingStackOffset (SDValue Arg, unsigned Offset, ISD::ArgFlagsTy Flags, MachineFrameInfo &MFI, const MachineRegisterInfo *MRI, const X86InstrInfo *TII, const CCValAssign &VA)
 Return true if the given stack call argument is already available in the same position (relatively) of the caller's incoming argument stack. More...
 
static bool MayFoldLoad (SDValue Op)
 
static bool MayFoldIntoStore (SDValue Op)
 
static bool MayFoldIntoZeroExtend (SDValue Op)
 
static bool isTargetShuffle (unsigned Opcode)
 
static bool isTargetShuffleVariableMask (unsigned Opcode)
 
static bool isTargetShuffleSplat (SDValue Op)
 
static bool isX86CCSigned (unsigned X86CC)
 Return true if the condition is an signed comparison operation. More...
 
static X86::CondCode TranslateIntegerX86CC (ISD::CondCode SetCCOpcode)
 
static X86::CondCode TranslateX86CC (ISD::CondCode SetCCOpcode, const SDLoc &DL, bool isFP, SDValue &LHS, SDValue &RHS, SelectionDAG &DAG)
 Do a one-to-one translation of a ISD::CondCode to the X86-specific condition code, returning the condition code and the LHS/RHS of the comparison to make. More...
 
static bool hasFPCMov (unsigned X86CC)
 Is there a floating point cmov for the specific X86 condition code? Current x86 isa includes the following FP cmov instructions: fcmovb, fcomvbe, fcomve, fcmovu, fcmovae, fcmova, fcmovne, fcmovnu. More...
 
static bool isUndefOrEqual (int Val, int CmpVal)
 Val is the undef sentinel value or equal to the specified value. More...
 
static bool isUndefOrZero (int Val)
 Val is either the undef or zero sentinel value. More...
 
static bool isUndefInRange (ArrayRef< int > Mask, unsigned Pos, unsigned Size)
 Return true if every element in Mask, beginning from position Pos and ending in Pos+Size is the undef sentinel value. More...
 
static bool isUndefLowerHalf (ArrayRef< int > Mask)
 Return true if the mask creates a vector whose lower half is undefined. More...
 
static bool isUndefUpperHalf (ArrayRef< int > Mask)
 Return true if the mask creates a vector whose upper half is undefined. More...
 
static bool isInRange (int Val, int Low, int Hi)
 Return true if Val falls within the specified range (L, H]. More...
 
static bool isAnyInRange (ArrayRef< int > Mask, int Low, int Hi)
 Return true if the value of any element in Mask falls within the specified range (L, H]. More...
 
static bool isAnyZero (ArrayRef< int > Mask)
 Return true if the value of any element in Mask is the zero sentinel value. More...
 
static bool isAnyZeroOrUndef (ArrayRef< int > Mask)
 Return true if the value of any element in Mask is the zero or undef sentinel values. More...
 
static bool isUndefOrInRange (int Val, int Low, int Hi)
 Return true if Val is undef or if its value falls within the specified range (L, H]. More...
 
static bool isUndefOrInRange (ArrayRef< int > Mask, int Low, int Hi)
 Return true if every element in Mask is undef or if its value falls within the specified range (L, H]. More...
 
static bool isUndefOrZeroOrInRange (int Val, int Low, int Hi)
 Return true if Val is undef, zero or if its value falls within the specified range (L, H]. More...
 
static bool isUndefOrZeroOrInRange (ArrayRef< int > Mask, int Low, int Hi)
 Return true if every element in Mask is undef, zero or if its value falls within the specified range (L, H]. More...
 
static bool isSequentialOrUndefInRange (ArrayRef< int > Mask, unsigned Pos, unsigned Size, int Low, int Step=1)
 Return true if every element in Mask, beginning from position Pos and ending in Pos + Size, falls within the specified sequence (Low, Low + Step, ..., Low + (Size - 1) * Step) or is undef. More...
 
static bool isSequentialOrUndefOrZeroInRange (ArrayRef< int > Mask, unsigned Pos, unsigned Size, int Low, int Step=1)
 Return true if every element in Mask, beginning from position Pos and ending in Pos+Size, falls within the specified sequential range (Low, Low+Size], or is undef or is zero. More...
 
static bool isUndefOrZeroInRange (ArrayRef< int > Mask, unsigned Pos, unsigned Size)
 Return true if every element in Mask, beginning from position Pos and ending in Pos+Size is undef or is zero. More...
 
static bool canWidenShuffleElements (ArrayRef< int > Mask, SmallVectorImpl< int > &WidenedMask)
 Helper function to test whether a shuffle mask could be simplified by widening the elements being shuffled. More...
 
static bool canWidenShuffleElements (ArrayRef< int > Mask, const APInt &Zeroable, bool V2IsZero, SmallVectorImpl< int > &WidenedMask)
 
static bool canWidenShuffleElements (ArrayRef< int > Mask)
 
static bool scaleShuffleElements (ArrayRef< int > Mask, unsigned NumDstElts, SmallVectorImpl< int > &ScaledMask)
 
static SDValue getConstVector (ArrayRef< int > Values, MVT VT, SelectionDAG &DAG, const SDLoc &dl, bool IsMask=false)
 
static SDValue getConstVector (ArrayRef< APInt > Bits, APInt &Undefs, MVT VT, SelectionDAG &DAG, const SDLoc &dl)
 
static SDValue getZeroVector (MVT VT, const X86Subtarget &Subtarget, SelectionDAG &DAG, const SDLoc &dl)
 Returns a vector of specified type with all zero elements. More...
 
static SDValue extractSubVector (SDValue Vec, unsigned IdxVal, SelectionDAG &DAG, const SDLoc &dl, unsigned vectorWidth)
 
static SDValue extract128BitVector (SDValue Vec, unsigned IdxVal, SelectionDAG &DAG, const SDLoc &dl)
 Generate a DAG to grab 128-bits from a vector > 128 bits. More...
 
static SDValue extract256BitVector (SDValue Vec, unsigned IdxVal, SelectionDAG &DAG, const SDLoc &dl)
 Generate a DAG to grab 256-bits from a 512-bit vector. More...
 
static SDValue insertSubVector (SDValue Result, SDValue Vec, unsigned IdxVal, SelectionDAG &DAG, const SDLoc &dl, unsigned vectorWidth)
 
static SDValue insert128BitVector (SDValue Result, SDValue Vec, unsigned IdxVal, SelectionDAG &DAG, const SDLoc &dl)
 Generate a DAG to put 128-bits into a vector > 128 bits. More...
 
static SDValue widenSubVector (MVT VT, SDValue Vec, bool ZeroNewElements, const X86Subtarget &Subtarget, SelectionDAG &DAG, const SDLoc &dl)
 Widen a vector to a larger size with the same scalar type, with the new elements either zero or undef. More...
 
static SDValue widenSubVector (SDValue Vec, bool ZeroNewElements, const X86Subtarget &Subtarget, SelectionDAG &DAG, const SDLoc &dl, unsigned WideSizeInBits)
 Widen a vector to a larger size with the same scalar type, with the new elements either zero or undef. More...
 
static bool collectConcatOps (SDNode *N, SmallVectorImpl< SDValue > &Ops)
 
static std::pair< SDValue, SDValuesplitVector (SDValue Op, SelectionDAG &DAG, const SDLoc &dl)
 
static SDValue splitVectorIntUnary (SDValue Op, SelectionDAG &DAG)
 
static SDValue splitVectorIntBinary (SDValue Op, SelectionDAG &DAG)
 Break a binary integer operation into 2 half sized ops and then concatenate the result back. More...
 
template<typename F >
SDValue SplitOpsAndApply (SelectionDAG &DAG, const X86Subtarget &Subtarget, const SDLoc &DL, EVT VT, ArrayRef< SDValue > Ops, F Builder, bool CheckBWI=true)
 
static SDValue insert1BitVector (SDValue Op, SelectionDAG &DAG, const X86Subtarget &Subtarget)
 Insert i1-subvector to i1-vector. More...
 
static SDValue concatSubVectors (SDValue V1, SDValue V2, SelectionDAG &DAG, const SDLoc &dl)
 
static SDValue getOnesVector (EVT VT, SelectionDAG &DAG, const SDLoc &dl)
 Returns a vector of specified type with all bits set. More...
 
static unsigned getOpcode_EXTEND_VECTOR_INREG (unsigned Opcode)
 
static SDValue getExtendInVec (unsigned Opcode, const SDLoc &DL, EVT VT, SDValue In, SelectionDAG &DAG)
 
static SDValue IsNOT (SDValue V, SelectionDAG &DAG, bool OneUse=false)
 
static SDValue getUnpackl (SelectionDAG &DAG, const SDLoc &dl, MVT VT, SDValue V1, SDValue V2)
 Returns a vector_shuffle node for an unpackl operation. More...
 
static SDValue getUnpackh (SelectionDAG &DAG, const SDLoc &dl, MVT VT, SDValue V1, SDValue V2)
 Returns a vector_shuffle node for an unpackh operation. More...
 
static SDValue getShuffleVectorZeroOrUndef (SDValue V2, int Idx, bool IsZero, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 Return a vector_shuffle of the specified vector of zero or undef vector. More...
 
static const ConstantgetTargetConstantFromBasePtr (SDValue Ptr)
 
static const ConstantgetTargetConstantFromNode (LoadSDNode *Load)
 
static const ConstantgetTargetConstantFromNode (SDValue Op)
 
static bool getTargetConstantBitsFromNode (SDValue Op, unsigned EltSizeInBits, APInt &UndefElts, SmallVectorImpl< APInt > &EltBits, bool AllowWholeUndefs=true, bool AllowPartialUndefs=true)
 
bool llvm::X86::isConstantSplat (SDValue Op, APInt &SplatVal, bool AllowPartialUndefs=true)
 If Op is a constant whose elements are all the same constant or undefined, return true and return the constant value in SplatVal. More...
 
static bool getTargetShuffleMaskIndices (SDValue MaskNode, unsigned MaskEltSizeInBits, SmallVectorImpl< uint64_t > &RawMask, APInt &UndefElts)
 
static void createPackShuffleMask (MVT VT, SmallVectorImpl< int > &Mask, bool Unary, unsigned NumStages=1)
 Create a shuffle mask that matches the PACKSS/PACKUS truncation. More...
 
static void getPackDemandedElts (EVT VT, const APInt &DemandedElts, APInt &DemandedLHS, APInt &DemandedRHS)
 
static void getHorizDemandedElts (EVT VT, const APInt &DemandedElts, APInt &DemandedLHS, APInt &DemandedRHS)
 
static bool getTargetShuffleMask (SDNode *N, MVT VT, bool AllowSentinelZero, SmallVectorImpl< SDValue > &Ops, SmallVectorImpl< int > &Mask, bool &IsUnary)
 Calculates the shuffle mask corresponding to the target-specific opcode. More...
 
static void computeZeroableShuffleElements (ArrayRef< int > Mask, SDValue V1, SDValue V2, APInt &KnownUndef, APInt &KnownZero)
 Compute whether each element of a shuffle is zeroable. More...
 
static bool getTargetShuffleAndZeroables (SDValue N, SmallVectorImpl< int > &Mask, SmallVectorImpl< SDValue > &Ops, APInt &KnownUndef, APInt &KnownZero)
 Decode a target shuffle mask and inputs and see if any values are known to be undef or zero from their inputs. More...
 
static void resolveTargetShuffleFromZeroables (SmallVectorImpl< int > &Mask, const APInt &KnownUndef, const APInt &KnownZero, bool ResolveKnownZeros=true)
 
static void resolveZeroablesFromTargetShuffle (const SmallVectorImpl< int > &Mask, APInt &KnownUndef, APInt &KnownZero)
 
static bool getTargetShuffleInputs (SDValue Op, SmallVectorImpl< SDValue > &Inputs, SmallVectorImpl< int > &Mask, const SelectionDAG &DAG, unsigned Depth, bool ResolveKnownElts)
 
static bool getFauxShuffleMask (SDValue N, const APInt &DemandedElts, SmallVectorImpl< int > &Mask, SmallVectorImpl< SDValue > &Ops, const SelectionDAG &DAG, unsigned Depth, bool ResolveKnownElts)
 
static void resolveTargetShuffleInputsAndMask (SmallVectorImpl< SDValue > &Inputs, SmallVectorImpl< int > &Mask)
 Removes unused/repeated shuffle source inputs and adjusts the shuffle mask. More...
 
static bool getTargetShuffleInputs (SDValue Op, const APInt &DemandedElts, SmallVectorImpl< SDValue > &Inputs, SmallVectorImpl< int > &Mask, APInt &KnownUndef, APInt &KnownZero, const SelectionDAG &DAG, unsigned Depth, bool ResolveKnownElts)
 Calls getTargetShuffleAndZeroables to resolve a target shuffle mask's inputs and then sets the SM_SentinelUndef and SM_SentinelZero values. More...
 
static SDValue getShuffleScalarElt (SDValue Op, unsigned Index, SelectionDAG &DAG, unsigned Depth)
 Returns the scalar element that will make up the i'th element of the result of the vector shuffle. More...
 
static SDValue LowerBuildVectorAsInsert (SDValue Op, unsigned NonZeros, unsigned NumNonZero, unsigned NumZero, SelectionDAG &DAG, const X86Subtarget &Subtarget)
 
static SDValue LowerBuildVectorv16i8 (SDValue Op, unsigned NonZeros, unsigned NumNonZero, unsigned NumZero, SelectionDAG &DAG, const X86Subtarget &Subtarget)
 Custom lower build_vector of v16i8. More...
 
static SDValue LowerBuildVectorv8i16 (SDValue Op, unsigned NonZeros, unsigned NumNonZero, unsigned NumZero, SelectionDAG &DAG, const X86Subtarget &Subtarget)
 Custom lower build_vector of v8i16. More...
 
static SDValue LowerBuildVectorv4x32 (SDValue Op, SelectionDAG &DAG, const X86Subtarget &Subtarget)
 Custom lower build_vector of v4i32 or v4f32. More...
 
static SDValue getVShift (bool isLeft, EVT VT, SDValue SrcOp, unsigned NumBits, SelectionDAG &DAG, const TargetLowering &TLI, const SDLoc &dl)
 Return a vector logical shift node. More...
 
static SDValue LowerAsSplatVectorLoad (SDValue SrcOp, MVT VT, const SDLoc &dl, SelectionDAG &DAG)
 
static bool findEltLoadSrc (SDValue Elt, LoadSDNode *&Ld, int64_t &ByteOffset)
 
static SDValue EltsFromConsecutiveLoads (EVT VT, ArrayRef< SDValue > Elts, const SDLoc &DL, SelectionDAG &DAG, const X86Subtarget &Subtarget, bool isAfterLegalize)
 Given the initializing elements 'Elts' of a vector of type 'VT', see if the elements can be replaced by a single large load which has the same value as a build_vector or insert_subvector whose loaded operands are 'Elts'. More...
 
static SDValue combineToConsecutiveLoads (EVT VT, SDValue Op, const SDLoc &DL, SelectionDAG &DAG, const X86Subtarget &Subtarget, bool isAfterLegalize)
 
static ConstantgetConstantVector (MVT VT, const APInt &SplatValue, unsigned SplatBitSize, LLVMContext &C)
 
static bool isFoldableUseOfShuffle (SDNode *N)
 
static SDValue isSplatZeroExtended (const BuildVectorSDNode *Op, unsigned &NumElt, MVT &EltType)
 
static SDValue lowerBuildVectorAsBroadcast (BuildVectorSDNode *BVOp, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 Attempt to use the vbroadcast instruction to generate a splat value from a splat BUILD_VECTOR which uses: a. More...
 
static int getUnderlyingExtractedFromVec (SDValue &ExtractedFromVec, SDValue ExtIdx)
 For an EXTRACT_VECTOR_ELT with a constant index return the real underlying vector and index. More...
 
static SDValue buildFromShuffleMostly (SDValue Op, SelectionDAG &DAG)
 
static SDValue LowerBUILD_VECTORvXi1 (SDValue Op, SelectionDAG &DAG, const X86Subtarget &Subtarget)
 
static bool isHorizontalBinOpPart (const BuildVectorSDNode *N, unsigned Opcode, SelectionDAG &DAG, unsigned BaseIdx, unsigned LastIdx, SDValue &V0, SDValue &V1)
 This is a helper function of LowerToHorizontalOp(). More...
 
static SDValue ExpandHorizontalBinOp (const SDValue &V0, const SDValue &V1, const SDLoc &DL, SelectionDAG &DAG, unsigned X86Opcode, bool Mode, bool isUndefLO, bool isUndefHI)
 Emit a sequence of two 128-bit horizontal add/sub followed by a concat_vector. More...
 
static bool isAddSubOrSubAdd (const BuildVectorSDNode *BV, const X86Subtarget &Subtarget, SelectionDAG &DAG, SDValue &Opnd0, SDValue &Opnd1, unsigned &NumExtracts, bool &IsSubAdd)
 Returns true iff BV builds a vector with the result equivalent to the result of ADDSUB/SUBADD operation. More...
 
static bool isFMAddSubOrFMSubAdd (const X86Subtarget &Subtarget, SelectionDAG &DAG, SDValue &Opnd0, SDValue &Opnd1, SDValue &Opnd2, unsigned ExpectedUses)
 Returns true if is possible to fold MUL and an idiom that has already been recognized as ADDSUB/SUBADD(Opnd0, Opnd1) into FMADDSUB/FMSUBADD(x, y, Opnd1). More...
 
static SDValue lowerToAddSubOrFMAddSub (const BuildVectorSDNode *BV, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 Try to fold a build_vector that performs an 'addsub' or 'fmaddsub' or 'fsubadd' operation accordingly to X86ISD::ADDSUB or X86ISD::FMADDSUB or X86ISD::FMSUBADD node. More...
 
static bool isHopBuildVector (const BuildVectorSDNode *BV, SelectionDAG &DAG, unsigned &HOpcode, SDValue &V0, SDValue &V1)
 
static SDValue getHopForBuildVector (const BuildVectorSDNode *BV, SelectionDAG &DAG, unsigned HOpcode, SDValue V0, SDValue V1)
 
static SDValue LowerToHorizontalOp (const BuildVectorSDNode *BV, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 Lower BUILD_VECTOR to a horizontal add/sub operation if possible. More...
 
static SDValue LowerShift (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 
static SDValue lowerBuildVectorToBitOp (BuildVectorSDNode *Op, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 If a BUILD_VECTOR's source elements all apply the same bit operation and one of their operands is constant, lower to a pair of BUILD_VECTOR and just apply the bit to the vectors. More...
 
static SDValue materializeVectorConstant (SDValue Op, SelectionDAG &DAG, const X86Subtarget &Subtarget)
 Create a vector constant without a load. More...
 
static SDValue createVariablePermute (MVT VT, SDValue SrcVec, SDValue IndicesVec, SDLoc &DL, SelectionDAG &DAG, const X86Subtarget &Subtarget)
 Look for opportunities to create a VPERMV/VPERMILPV/PSHUFB variable permute from a vector of source values and a vector of extraction indices. More...
 
static SDValue LowerBUILD_VECTORAsVariablePermute (SDValue V, SelectionDAG &DAG, const X86Subtarget &Subtarget)
 
static SDValue LowerAVXCONCAT_VECTORS (SDValue Op, SelectionDAG &DAG, const X86Subtarget &Subtarget)
 
static SDValue LowerCONCAT_VECTORSvXi1 (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 
static SDValue LowerCONCAT_VECTORS (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 
static bool isNoopShuffleMask (ArrayRef< int > Mask)
 Tiny helper function to identify a no-op mask. More...
 
static bool isLaneCrossingShuffleMask (unsigned LaneSizeInBits, unsigned ScalarSizeInBits, ArrayRef< int > Mask)
 Test whether there are elements crossing LaneSizeInBits lanes in this shuffle mask. More...
 
static bool is128BitLaneCrossingShuffleMask (MVT VT, ArrayRef< int > Mask)
 Test whether there are elements crossing 128-bit lanes in this shuffle mask. More...
 
static bool isRepeatedShuffleMask (unsigned LaneSizeInBits, MVT VT, ArrayRef< int > Mask, SmallVectorImpl< int > &RepeatedMask)
 Test whether a shuffle mask is equivalent within each sub-lane. More...
 
static bool is128BitLaneRepeatedShuffleMask (MVT VT, ArrayRef< int > Mask, SmallVectorImpl< int > &RepeatedMask)
 Test whether a shuffle mask is equivalent within each 128-bit lane. More...
 
static bool is128BitLaneRepeatedShuffleMask (MVT VT, ArrayRef< int > Mask)
 
static bool is256BitLaneRepeatedShuffleMask (MVT VT, ArrayRef< int > Mask, SmallVectorImpl< int > &RepeatedMask)
 Test whether a shuffle mask is equivalent within each 256-bit lane. More...
 
static bool isRepeatedTargetShuffleMask (unsigned LaneSizeInBits, MVT VT, ArrayRef< int > Mask, SmallVectorImpl< int > &RepeatedMask)
 Test whether a target shuffle mask is equivalent within each sub-lane. More...
 
static bool isShuffleEquivalent (SDValue V1, SDValue V2, ArrayRef< int > Mask, ArrayRef< int > ExpectedMask)
 Checks whether a shuffle mask is equivalent to an explicit list of arguments. More...
 
static bool isTargetShuffleEquivalent (ArrayRef< int > Mask, ArrayRef< int > ExpectedMask, SDValue V1=SDValue(), SDValue V2=SDValue())
 Checks whether a target shuffle mask is equivalent to an explicit pattern. More...
 
static bool createShuffleMaskFromVSELECT (SmallVectorImpl< int > &Mask, SDValue Cond)
 
static bool isUnpackWdShuffleMask (ArrayRef< int > Mask, MVT VT)
 
static bool is128BitUnpackShuffleMask (ArrayRef< int > Mask)
 
static bool hasIdenticalHalvesShuffleMask (ArrayRef< int > Mask)
 Return true if a shuffle mask chooses elements identically in its top and bottom halves. More...
 
static unsigned getV4X86ShuffleImm (ArrayRef< int > Mask)
 Get a 4-lane 8-bit shuffle immediate for a mask. More...
 
static SDValue getV4X86ShuffleImm8ForMask (ArrayRef< int > Mask, const SDLoc &DL, SelectionDAG &DAG)
 
static bool isNonZeroElementsInOrder (const APInt &Zeroable, ArrayRef< int > Mask, const EVT &VectorType, bool &IsZeroSideLeft)
 
static SDValue lowerShuffleWithPSHUFB (const SDLoc &DL, MVT VT, ArrayRef< int > Mask, SDValue V1, SDValue V2, const APInt &Zeroable, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 Try to lower a shuffle with a single PSHUFB of V1 or V2. More...
 
static SDValue getMaskNode (SDValue Mask, MVT MaskVT, const X86Subtarget &Subtarget, SelectionDAG &DAG, const SDLoc &dl)
 Return Mask with the necessary casting or extending for Mask according to MaskVT when lowering masking intrinsics. More...
 
static SDValue lowerShuffleToEXPAND (const SDLoc &DL, MVT VT, const APInt &Zeroable, ArrayRef< int > Mask, SDValue &V1, SDValue &V2, SelectionDAG &DAG, const X86Subtarget &Subtarget)
 
static bool matchShuffleWithUNPCK (MVT VT, SDValue &V1, SDValue &V2, unsigned &UnpackOpcode, bool IsUnary, ArrayRef< int > TargetMask, const SDLoc &DL, SelectionDAG &DAG, const X86Subtarget &Subtarget)
 
static SDValue lowerShuffleWithUNPCK (const SDLoc &DL, MVT VT, ArrayRef< int > Mask, SDValue V1, SDValue V2, SelectionDAG &DAG)
 
static SDValue lowerShuffleWithUNPCK256 (const SDLoc &DL, MVT VT, ArrayRef< int > Mask, SDValue V1, SDValue V2, SelectionDAG &DAG)
 Check if the mask can be mapped to a preliminary shuffle (vperm 64-bit) followed by unpack 256-bit. More...
 
static bool matchShuffleAsVTRUNC (MVT &SrcVT, MVT &DstVT, MVT VT, ArrayRef< int > Mask, const APInt &Zeroable, const X86Subtarget &Subtarget)
 
static bool matchShuffleAsVPMOV (ArrayRef< int > Mask, bool SwappedOps, int Delta)
 
static SDValue lowerShuffleWithVPMOV (const SDLoc &DL, ArrayRef< int > Mask, MVT VT, SDValue V1, SDValue V2, SelectionDAG &DAG, const X86Subtarget &Subtarget)
 
static int canLowerByDroppingEvenElements (ArrayRef< int > Mask, bool IsSingleInput)
 Check whether a compaction lowering can be done by dropping even elements and compute how many times even elements must be dropped. More...
 
static bool matchShuffleWithPACK (MVT VT, MVT &SrcVT, SDValue &V1, SDValue &V2, unsigned &PackOpcode, ArrayRef< int > TargetMask, SelectionDAG &DAG, const X86Subtarget &Subtarget, unsigned MaxStages=1)
 
static SDValue lowerShuffleWithPACK (const SDLoc &DL, MVT VT, ArrayRef< int > Mask, SDValue V1, SDValue V2, SelectionDAG &DAG, const X86Subtarget &Subtarget)
 
static SDValue lowerShuffleAsBitMask (const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Mask, const APInt &Zeroable, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 Try to emit a bitmask instruction for a shuffle. More...
 
static SDValue lowerShuffleAsBitBlend (const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Mask, SelectionDAG &DAG)
 Try to emit a blend instruction for a shuffle using bit math. More...
 
static SDValue getVectorMaskingNode (SDValue Op, SDValue Mask, SDValue PreservedSrc, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 Return (and Op, Mask) for compare instructions or (vselect Mask, Op, PreservedSrc) for others along with the necessary casting or extending for Mask when lowering masking intrinsics. More...
 
static bool matchShuffleAsBlend (SDValue V1, SDValue V2, MutableArrayRef< int > Mask, const APInt &Zeroable, bool &ForceV1Zero, bool &ForceV2Zero, uint64_t &BlendMask)
 
static uint64_t scaleVectorShuffleBlendMask (uint64_t BlendMask, int Size, int Scale)
 
static SDValue lowerShuffleAsBlend (const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Original, const APInt &Zeroable, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 Try to emit a blend instruction for a shuffle. More...
 
static SDValue lowerShuffleAsBlendAndPermute (const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Mask, SelectionDAG &DAG, bool ImmBlends=false)
 Try to lower as a blend of elements from two inputs followed by a single-input permutation. More...
 
static SDValue lowerShuffleAsUNPCKAndPermute (const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Mask, SelectionDAG &DAG)
 Try to lower as an unpack of elements from two inputs followed by a single-input permutation. More...
 
static SDValue lowerShuffleAsByteRotateAndPermute (const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Mask, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 Helper to form a PALIGNR-based rotate+permute, merging 2 inputs and then permuting the elements of the result in place. More...
 
static SDValue lowerShuffleAsDecomposedShuffleBlend (const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Mask, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 Generic routine to decompose a shuffle and blend into independent blends and permutes. More...
 
static int matchShuffleAsBitRotate (ArrayRef< int > Mask, int NumSubElts)
 Try to lower a vector shuffle as a bit rotation. More...
 
static int matchShuffleAsBitRotate (MVT &RotateVT, int EltSizeInBits, const X86Subtarget &Subtarget, ArrayRef< int > Mask)
 
static SDValue lowerShuffleAsBitRotate (const SDLoc &DL, MVT VT, SDValue V1, ArrayRef< int > Mask, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 Lower shuffle using X86ISD::VROTLI rotations. More...
 
static int matchShuffleAsElementRotate (SDValue &V1, SDValue &V2, ArrayRef< int > Mask)
 Try to match a vector shuffle as an element rotation. More...
 
static int matchShuffleAsByteRotate (MVT VT, SDValue &V1, SDValue &V2, ArrayRef< int > Mask)
 Try to lower a vector shuffle as a byte rotation. More...
 
static SDValue lowerShuffleAsByteRotate (const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Mask, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 
static SDValue lowerShuffleAsVALIGN (const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Mask, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 Try to lower a vector shuffle as a dword/qword rotation. More...
 
static SDValue lowerShuffleAsByteShiftMask (const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Mask, const APInt &Zeroable, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 Try to lower a vector shuffle as a byte shift sequence. More...
 
static int matchShuffleAsShift (MVT &ShiftVT, unsigned &Opcode, unsigned ScalarSizeInBits, ArrayRef< int > Mask, int MaskOffset, const APInt &Zeroable, const X86Subtarget &Subtarget)
 Try to lower a vector shuffle as a bit shift (shifts in zeros). More...
 
static SDValue lowerShuffleAsShift (const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Mask, const APInt &Zeroable, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 
static bool matchShuffleAsEXTRQ (MVT VT, SDValue &V1, SDValue &V2, ArrayRef< int > Mask, uint64_t &BitLen, uint64_t &BitIdx, const APInt &Zeroable)
 
static bool matchShuffleAsINSERTQ (MVT VT, SDValue &V1, SDValue &V2, ArrayRef< int > Mask, uint64_t &BitLen, uint64_t &BitIdx)
 
static SDValue lowerShuffleWithSSE4A (const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Mask, const APInt &Zeroable, SelectionDAG &DAG)
 Try to lower a vector shuffle using SSE4a EXTRQ/INSERTQ. More...
 
static SDValue lowerShuffleAsSpecificZeroOrAnyExtend (const SDLoc &DL, MVT VT, int Scale, int Offset, bool AnyExt, SDValue InputV, ArrayRef< int > Mask, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 Lower a vector shuffle as a zero or any extension. More...
 
static SDValue lowerShuffleAsZeroOrAnyExtend (const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Mask, const APInt &Zeroable, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 Try to lower a vector shuffle as a zero extension on any microarch. More...
 
static SDValue getScalarValueForVectorElement (SDValue V, int Idx, SelectionDAG &DAG)
 Try to get a scalar value for a specific element of a vector. More...
 
static bool isShuffleFoldableLoad (SDValue V)
 Helper to test for a load that can be folded with x86 shuffles. More...
 
static SDValue lowerShuffleAsElementInsertion (const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Mask, const APInt &Zeroable, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 Try to lower insertion of a single element into a zero vector. More...
 
static SDValue lowerShuffleAsTruncBroadcast (const SDLoc &DL, MVT VT, SDValue V0, int BroadcastIdx, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 Try to lower broadcast of a single - truncated - integer element, coming from a scalar_to_vector/build_vector node V0 with larger elements. More...
 
static bool isSingleSHUFPSMask (ArrayRef< int > Mask)
 Test whether this can be lowered with a single SHUFPS instruction. More...
 
static SDValue lowerShuffleOfExtractsAsVperm (const SDLoc &DL, SDValue N0, SDValue N1, ArrayRef< int > Mask, SelectionDAG &DAG)
 If we are extracting two 128-bit halves of a vector and shuffling the result, match that to a 256-bit AVX2 vperm* instruction to avoid a multi-shuffle lowering. More...
 
static SDValue lowerShuffleAsBroadcast (const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Mask, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 Try to lower broadcast of a single element. More...
 
static bool matchShuffleAsInsertPS (SDValue &V1, SDValue &V2, unsigned &InsertPSMask, const APInt &Zeroable, ArrayRef< int > Mask, SelectionDAG &DAG)
 
static SDValue lowerShuffleAsInsertPS (const SDLoc &DL, SDValue V1, SDValue V2, ArrayRef< int > Mask, const APInt &Zeroable, SelectionDAG &DAG)
 
static SDValue lowerShuffleAsPermuteAndUnpack (const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Mask, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 Try to lower a shuffle as a permute of the inputs followed by an UNPCK instruction. More...
 
static SDValue lowerV2F64Shuffle (const SDLoc &DL, ArrayRef< int > Mask, const APInt &Zeroable, SDValue V1, SDValue V2, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 Handle lowering of 2-lane 64-bit floating point shuffles. More...
 
static SDValue lowerV2I64Shuffle (const SDLoc &DL, ArrayRef< int > Mask, const APInt &Zeroable, SDValue V1, SDValue V2, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 Handle lowering of 2-lane 64-bit integer shuffles. More...
 
static SDValue lowerShuffleWithSHUFPS (const SDLoc &DL, MVT VT, ArrayRef< int > Mask, SDValue V1, SDValue V2, SelectionDAG &DAG)
 Lower a vector shuffle using the SHUFPS instruction. More...
 
static SDValue lowerV4F32Shuffle (const SDLoc &DL, ArrayRef< int > Mask, const APInt &Zeroable, SDValue V1, SDValue V2, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 Lower 4-lane 32-bit floating point shuffles. More...
 
static SDValue lowerV4I32Shuffle (const SDLoc &DL, ArrayRef< int > Mask, const APInt &Zeroable, SDValue V1, SDValue V2, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 Lower 4-lane i32 vector shuffles. More...
 
static SDValue lowerV8I16GeneralSingleInputShuffle (const SDLoc &DL, MVT VT, SDValue V, MutableArrayRef< int > Mask, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 Lowering of single-input v8i16 shuffles is the cornerstone of SSE2 shuffle lowering, and the most complex part. More...
 
static SDValue lowerShuffleAsBlendOfPSHUFBs (const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Mask, const APInt &Zeroable, SelectionDAG &DAG, bool &V1InUse, bool &V2InUse)
 Helper to form a PSHUFB-based shuffle+blend, opportunistically avoiding the blend if only one input is used. More...
 
static SDValue lowerV8I16Shuffle (const SDLoc &DL, ArrayRef< int > Mask, const APInt &Zeroable, SDValue V1, SDValue V2, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 Generic lowering of 8-lane i16 shuffles. More...
 
static SDValue lowerShuffleWithPERMV (const SDLoc &DL, MVT VT, ArrayRef< int > Mask, SDValue V1, SDValue V2, SelectionDAG &DAG)
 
static SDValue lowerV16I8Shuffle (const SDLoc &DL, ArrayRef< int > Mask, const APInt &Zeroable, SDValue V1, SDValue V2, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 Generic lowering of v16i8 shuffles. More...
 
static SDValue lower128BitShuffle (const SDLoc &DL, ArrayRef< int > Mask, MVT VT, SDValue V1, SDValue V2, const APInt &Zeroable, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 Dispatching routine to lower various 128-bit x86 vector shuffles. More...
 
static SDValue splitAndLowerShuffle (const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Mask, SelectionDAG &DAG)
 Generic routine to split vector shuffle into half-sized shuffles. More...
 
static SDValue lowerShuffleAsSplitOrBlend (const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Mask, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 Either split a vector in halves or decompose the shuffles and the blend. More...
 
static SDValue lowerShuffleAsLanePermuteAndSHUFP (const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Mask, SelectionDAG &DAG)
 
static SDValue lowerShuffleAsLanePermuteAndPermute (const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Mask, SelectionDAG &DAG, const X86Subtarget &Subtarget)
 Lower a vector shuffle crossing multiple 128-bit lanes as a lane permutation followed by a per-lane permutation. More...
 
static SDValue lowerShuffleAsLanePermuteAndShuffle (const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Mask, SelectionDAG &DAG, const X86Subtarget &Subtarget)
 Lower a vector shuffle crossing multiple 128-bit lanes by shuffling one source with a lane permutation. More...
 
static SDValue lowerV2X128Shuffle (const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Mask, const APInt &Zeroable, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 Handle lowering 2-lane 128-bit shuffles. More...
 
static SDValue lowerShuffleAsLanePermuteAndRepeatedMask (const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Mask, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 Lower a vector shuffle by first fixing the 128-bit lanes and then shuffling each lane. More...
 
static bool getHalfShuffleMask (ArrayRef< int > Mask, MutableArrayRef< int > HalfMask, int &HalfIdx1, int &HalfIdx2)
 If the input shuffle mask results in a vector that is undefined in all upper or lower half elements and that mask accesses only 2 halves of the shuffle's operands, return true. More...
 
static SDValue getShuffleHalfVectors (const SDLoc &DL, SDValue V1, SDValue V2, ArrayRef< int > HalfMask, int HalfIdx1, int HalfIdx2, bool UndefLower, SelectionDAG &DAG, bool UseConcat=false)
 Given the output values from getHalfShuffleMask(), create a half width shuffle of extracted vectors followed by an insert back to full width. More...
 
static SDValue lowerShuffleWithUndefHalf (const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Mask, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 Lower shuffles where an entire half of a 256 or 512-bit vector is UNDEF. More...
 
static bool isShuffleMaskInputInPlace (int Input, ArrayRef< int > Mask)
 Test whether the specified input (0 or 1) is in-place blended by the given mask. More...
 
static SDValue lowerShuffleAsRepeatedMaskAndLanePermute (const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Mask, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 Handle case where shuffle sources are coming from the same 128-bit lane and every lane can be represented as the same repeating mask - allowing us to shuffle the sources with the repeating shuffle and then permute the result to the destination lanes. More...
 
static bool matchShuffleWithSHUFPD (MVT VT, SDValue &V1, SDValue &V2, bool &ForceV1Zero, bool &ForceV2Zero, unsigned &ShuffleImm, ArrayRef< int > Mask, const APInt &Zeroable)
 
static SDValue lowerShuffleWithSHUFPD (const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Mask, const APInt &Zeroable, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 
static SDValue lowerShuffleAsVTRUNCAndUnpack (const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef< int > Mask, const APInt &Zeroable, SelectionDAG &DAG)
 
static SDValue lowerV4F64Shuffle (const SDLoc &DL, ArrayRef< int > Mask, const APInt &Zeroable, SDValue V1, SDValue V2, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 Handle lowering of 4-lane 64-bit floating point shuffles. More...
 
static SDValue lowerV4I64Shuffle (const SDLoc &DL, ArrayRef< int > Mask, const APInt &Zeroable, SDValue V1, SDValue V2, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 Handle lowering of 4-lane 64-bit integer shuffles. More...
 
static SDValue lowerV8F32Shuffle (const SDLoc &DL, ArrayRef< int > Mask, const APInt &Zeroable, SDValue V1, SDValue V2, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 Handle lowering of 8-lane 32-bit floating point shuffles. More...
 
static SDValue lowerV8I32Shuffle (const SDLoc &DL, ArrayRef< int > Mask, const APInt &Zeroable, SDValue V1, SDValue V2, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 Handle lowering of 8-lane 32-bit integer shuffles. More...
 
static SDValue lowerV16I16Shuffle (const SDLoc &DL, ArrayRef< int > Mask, const APInt &Zeroable, SDValue V1, SDValue V2, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 Handle lowering of 16-lane 16-bit integer shuffles. More...
 
static SDValue lowerV32I8Shuffle (const SDLoc &DL, ArrayRef< int > Mask, const APInt &Zeroable, SDValue V1, SDValue V2, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 Handle lowering of 32-lane 8-bit integer shuffles. More...
 
static SDValue lower256BitShuffle (const SDLoc &DL, ArrayRef< int > Mask, MVT VT, SDValue V1, SDValue V2, const APInt &Zeroable, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 High-level routine to lower various 256-bit x86 vector shuffles. More...
 
static SDValue lowerV4X128Shuffle (const SDLoc &DL, MVT VT, ArrayRef< int > Mask, const APInt &Zeroable, SDValue V1, SDValue V2, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 Try to lower a vector shuffle as a 128-bit shuffles. More...
 
static SDValue lowerV8F64Shuffle (const SDLoc &DL, ArrayRef< int > Mask, const APInt &Zeroable, SDValue V1, SDValue V2, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 Handle lowering of 8-lane 64-bit floating point shuffles. More...
 
static SDValue lowerV16F32Shuffle (const SDLoc &DL, ArrayRef< int > Mask, const APInt &Zeroable, SDValue V1, SDValue V2, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 Handle lowering of 16-lane 32-bit floating point shuffles. More...
 
static SDValue lowerV8I64Shuffle (const SDLoc &DL, ArrayRef< int > Mask, const APInt &Zeroable, SDValue V1, SDValue V2, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 Handle lowering of 8-lane 64-bit integer shuffles. More...
 
static SDValue lowerV16I32Shuffle (const SDLoc &DL, ArrayRef< int > Mask, const APInt &Zeroable, SDValue V1, SDValue V2, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 Handle lowering of 16-lane 32-bit integer shuffles. More...
 
static SDValue lowerV32I16Shuffle (const SDLoc &DL, ArrayRef< int > Mask, const APInt &Zeroable, SDValue V1, SDValue V2, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 Handle lowering of 32-lane 16-bit integer shuffles. More...
 
static SDValue lowerV64I8Shuffle (const SDLoc &DL, ArrayRef< int > Mask, const APInt &Zeroable, SDValue V1, SDValue V2, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 Handle lowering of 64-lane 8-bit integer shuffles. More...
 
static SDValue lower512BitShuffle (const SDLoc &DL, ArrayRef< int > Mask, MVT VT, SDValue V1, SDValue V2, const APInt &Zeroable, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 High-level routine to lower various 512-bit x86 vector shuffles. More...
 
static SDValue lower1BitShuffleAsKSHIFTR (const SDLoc &DL, ArrayRef< int > Mask, MVT VT, SDValue V1, SDValue V2, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 
static int match1BitShuffleAsKSHIFT (unsigned &Opcode, ArrayRef< int > Mask, int MaskOffset, const APInt &Zeroable)
 
static SDValue lower1BitShuffle (const SDLoc &DL, ArrayRef< int > Mask, MVT VT, SDValue V1, SDValue V2, const APInt &Zeroable, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 
static bool canonicalizeShuffleMaskWithCommute (ArrayRef< int > Mask)
 Helper function that returns true if the shuffle mask should be commuted to improve canonicalization. More...
 
static SDValue lowerVECTOR_SHUFFLE (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 Top-level lowering for x86 vector shuffles. More...
 
static SDValue lowerVSELECTtoVectorShuffle (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 Try to lower a VSELECT instruction to a vector shuffle. More...
 
static SDValue LowerEXTRACT_VECTOR_ELT_SSE4 (SDValue Op, SelectionDAG &DAG)
 
static SDValue ExtractBitFromMaskVector (SDValue Op, SelectionDAG &DAG, const X86Subtarget &Subtarget)
 Extract one bit from mask vector, like v16i1 or v8i1. More...
 
static SDValue InsertBitToMaskVector (SDValue Op, SelectionDAG &DAG, const X86Subtarget &Subtarget)
 Insert one bit to mask vector, like v16i1 or v8i1. More...
 
static SDValue LowerSCALAR_TO_VECTOR (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 
static SDValue LowerINSERT_SUBVECTOR (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 
static SDValue LowerEXTRACT_SUBVECTOR (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 
static SDValue GetTLSADDR (SelectionDAG &DAG, SDValue Chain, GlobalAddressSDNode *GA, SDValue *InFlag, const EVT PtrVT, unsigned ReturnReg, unsigned char OperandFlags, bool LocalDynamic=false)
 
static SDValue LowerToTLSGeneralDynamicModel32 (GlobalAddressSDNode *GA, SelectionDAG &DAG, const EVT PtrVT)
 
static SDValue LowerToTLSGeneralDynamicModel64 (GlobalAddressSDNode *GA, SelectionDAG &DAG, const EVT PtrVT)
 
static SDValue LowerToTLSLocalDynamicModel (GlobalAddressSDNode *GA, SelectionDAG &DAG, const EVT PtrVT, bool is64Bit)
 
static SDValue LowerToTLSExecModel (GlobalAddressSDNode *GA, SelectionDAG &DAG, const EVT PtrVT, TLSModel::Model model, bool is64Bit, bool isPIC)
 
static SDValue LowerShiftParts (SDValue Op, SelectionDAG &DAG)
 Lower SRA_PARTS and friends, which return two i32 values and take a 2 x i32 value to shift plus a shift amount. More...
 
static SDValue LowerFunnelShift (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 
static SDValue LowerI64IntToFP_AVX512DQ (SDValue Op, SelectionDAG &DAG, const X86Subtarget &Subtarget)
 
static bool useVectorCast (unsigned Opcode, MVT FromVT, MVT ToVT, const X86Subtarget &Subtarget)
 
static SDValue vectorizeExtractedCast (SDValue Cast, SelectionDAG &DAG, const X86Subtarget &Subtarget)
 Given a scalar cast operation that is extracted from a vector, try to vectorize the cast op followed by extraction. More...
 
static SDValue lowerFPToIntToFP (SDValue CastToFP, SelectionDAG &DAG, const X86Subtarget &Subtarget)
 Given a scalar cast to FP with a cast to integer operand (almost an ftrunc), try to vectorize the cast ops. More...
 
static SDValue lowerINT_TO_FP_vXi64 (SDValue Op, SelectionDAG &DAG, const X86Subtarget &Subtarget)
 
static bool shouldUseHorizontalOp (bool IsSingleSource, SelectionDAG &DAG, const X86Subtarget &Subtarget)
 Horizontal vector math instructions may be slower than normal math with shuffles. More...
 
static SDValue LowerUINT_TO_FP_i64 (SDValue Op, SelectionDAG &DAG, const X86Subtarget &Subtarget)
 64-bit unsigned integer to double expansion. More...
 
static SDValue LowerUINT_TO_FP_i32 (SDValue Op, SelectionDAG &DAG, const X86Subtarget &Subtarget)
 32-bit unsigned integer to float expansion. More...
 
static SDValue lowerUINT_TO_FP_v2i32 (SDValue Op, SelectionDAG &DAG, const X86Subtarget &Subtarget, const SDLoc &DL)
 
static SDValue lowerUINT_TO_FP_vXi32 (SDValue Op, SelectionDAG &DAG, const X86Subtarget &Subtarget)
 
static SDValue lowerUINT_TO_FP_vec (SDValue Op, SelectionDAG &DAG, const X86Subtarget &Subtarget)
 
static SDValue LowerAVXExtend (SDValue Op, SelectionDAG &DAG, const X86Subtarget &Subtarget)
 
static SDValue SplitAndExtendv16i1 (unsigned ExtOpc, MVT VT, SDValue In, const SDLoc &dl, SelectionDAG &DAG)
 
static SDValue LowerZERO_EXTEND_Mask (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 
static SDValue LowerZERO_EXTEND (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 
static SDValue truncateVectorWithPACK (unsigned Opcode, EVT DstVT, SDValue In, const SDLoc &DL, SelectionDAG &DAG, const X86Subtarget &Subtarget)
 Helper to recursively truncate vector elements in half with PACKSS/PACKUS. More...
 
static SDValue LowerTruncateVecI1 (SDValue Op, SelectionDAG &DAG, const X86Subtarget &Subtarget)
 
static SDValue LowerFP16_TO_FP (SDValue Op, SelectionDAG &DAG)
 
static SDValue LowerFP_TO_FP16 (SDValue Op, SelectionDAG &DAG)
 
static SDValue lowerAddSubToHorizontalOp (SDValue Op, SelectionDAG &DAG, const X86Subtarget &Subtarget)
 Depending on uarch and/or optimizing for size, we might prefer to use a vector operation in place of the typical scalar operation. More...
 
static SDValue LowerFROUND (SDValue Op, SelectionDAG &DAG)
 ISD::FROUND is defined to round to nearest with ties rounding away from 0. More...
 
static SDValue LowerFABSorFNEG (SDValue Op, SelectionDAG &DAG)
 The only differences between FABS and FNEG are the mask and the logic op. More...
 
static SDValue LowerFCOPYSIGN (SDValue Op, SelectionDAG &DAG)
 
static SDValue LowerFGETSIGN (SDValue Op, SelectionDAG &DAG)
 
static SDValue getSETCC (X86::CondCode Cond, SDValue EFLAGS, const SDLoc &dl, SelectionDAG &DAG)
 Helper for creating a X86ISD::SETCC node. More...
 
static bool matchScalarReduction (SDValue Op, ISD::NodeType BinOp, SmallVectorImpl< SDValue > &SrcOps, SmallVectorImpl< APInt > *SrcMask=nullptr)
 Helper for matching OR(EXTRACTELT(X,0),OR(EXTRACTELT(X,1),...)) style scalarized (associative) reduction patterns. More...
 
static SDValue LowerVectorAllZero (const SDLoc &DL, SDValue V, ISD::CondCode CC, const APInt &Mask, const X86Subtarget &Subtarget, SelectionDAG &DAG, X86::CondCode &X86CC)
 
static SDValue MatchVectorAllZeroTest (SDValue Op, ISD::CondCode CC, const SDLoc &DL, const X86Subtarget &Subtarget, SelectionDAG &DAG, SDValue &X86CC)
 
static bool hasNonFlagsUse (SDValue Op)
 return true if Op has a use that doesn't just read flags. More...
 
static bool isProfitableToUseFlagOp (SDValue Op)
 
static SDValue EmitTest (SDValue Op, unsigned X86CC, const SDLoc &dl, SelectionDAG &DAG, const X86Subtarget &Subtarget)
 Emit nodes that will be selected as "test Op0,Op0", or something equivalent. More...
 
static SDValue EmitCmp (SDValue Op0, SDValue Op1, unsigned X86CC, const SDLoc &dl, SelectionDAG &DAG, const X86Subtarget &Subtarget)
 Emit nodes that will be selected as "cmp Op0,Op1", or something equivalent. More...
 
static SDValue LowerAndToBT (SDValue And, ISD::CondCode CC, const SDLoc &dl, SelectionDAG &DAG, SDValue &X86CC)
 Result of 'and' is compared against zero. More...
 
static unsigned translateX86FSETCC (ISD::CondCode SetCCOpcode, SDValue &Op0, SDValue &Op1, bool &IsAlwaysSignaling)
 Turns an ISD::CondCode into a value suitable for SSE floating-point mask CMPs. More...
 
static SDValue splitIntVSETCC (SDValue Op, SelectionDAG &DAG)
 Break a VSETCC 256-bit integer VSETCC into two new 128 ones and then concatenate the result back. More...
 
static SDValue LowerIntVSETCC_AVX512 (SDValue Op, SelectionDAG &DAG)
 
static SDValue incDecVectorConstant (SDValue V, SelectionDAG &DAG, bool IsInc)
 Given a buildvector constant, return a new vector constant with each element incremented or decremented. More...
 
static SDValue LowerVSETCCWithSUBUS (SDValue Op0, SDValue Op1, MVT VT, ISD::CondCode Cond, const SDLoc &dl, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 As another special case, use PSUBUS[BW] when it's profitable. More...
 
static SDValue LowerVSETCC (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 
static SDValue EmitAVX512Test (SDValue Op0, SDValue Op1, ISD::CondCode CC, const SDLoc &dl, SelectionDAG &DAG, const X86Subtarget &Subtarget, SDValue &X86CC)
 
static std::pair< SDValue, SDValuegetX86XALUOOp (X86::CondCode &Cond, SDValue Op, SelectionDAG &DAG)
 
static SDValue LowerXALUO (SDValue Op, SelectionDAG &DAG)
 
static bool isX86LogicalCmp (SDValue Op)
 Return true if opcode is a X86 logical comparison. More...
 
static bool isTruncWithZeroHighBitsInput (SDValue V, SelectionDAG &DAG)
 
static SDValue LowerSIGN_EXTEND_Mask (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 
static SDValue LowerANY_EXTEND (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 
static SDValue LowerEXTEND_VECTOR_INREG (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 
static SDValue LowerSIGN_EXTEND (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 
static SDValue splitVectorStore (StoreSDNode *Store, SelectionDAG &DAG)
 Change a vector store into a pair of half-size vector stores. More...
 
static SDValue scalarizeVectorStore (StoreSDNode *Store, MVT StoreVT, SelectionDAG &DAG)
 Scalarize a vector store, bitcasting to TargetVT to determine the scalar type. More...
 
static SDValue LowerStore (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 
static SDValue LowerLoad (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 
static bool isAndOrOfSetCCs (SDValue Op, unsigned &Opc)
 Return true if node is an ISD::AND or ISD::OR of two X86ISD::SETCC nodes each of which has no other use apart from the AND / OR. More...
 
static SDValue LowerVACOPY (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 
static unsigned getTargetVShiftUniformOpcode (unsigned Opc, bool IsVariable)
 
static SDValue getTargetVShiftByConstNode (unsigned Opc, const SDLoc &dl, MVT VT, SDValue SrcOp, uint64_t ShiftAmt, SelectionDAG &DAG)
 Handle vector element shifts where the shift amount is a constant. More...
 
static SDValue getTargetVShiftNode (unsigned Opc, const SDLoc &dl, MVT VT, SDValue SrcOp, SDValue ShAmt, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 Handle vector element shifts where the shift amount may or may not be a constant. More...
 
static SDValue getScalarMaskingNode (SDValue Op, SDValue Mask, SDValue PreservedSrc, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 Creates an SDNode for a predicated scalar operation. More...
 
static int getSEHRegistrationNodeSize (const Function *Fn)
 
static SDValue recoverFramePointer (SelectionDAG &DAG, const Function *Fn, SDValue EntryEBP)
 When the MSVC runtime transfers control to us, either to an outlined function or when returning to a parent frame after catching an exception, we recover the parent frame pointer by doing arithmetic on the incoming EBP. More...
 
static SDValue getAVX2GatherNode (unsigned Opc, SDValue Op, SelectionDAG &DAG, SDValue Src, SDValue Mask, SDValue Base, SDValue Index, SDValue ScaleOp, SDValue Chain, const X86Subtarget &Subtarget)
 
static SDValue getGatherNode (SDValue Op, SelectionDAG &DAG, SDValue Src, SDValue Mask, SDValue Base, SDValue Index, SDValue ScaleOp, SDValue Chain, const X86Subtarget &Subtarget)
 
static SDValue getScatterNode (unsigned Opc, SDValue Op, SelectionDAG &DAG, SDValue Src, SDValue Mask, SDValue Base, SDValue Index, SDValue ScaleOp, SDValue Chain, const X86Subtarget &Subtarget)
 
static SDValue getPrefetchNode (unsigned Opc, SDValue Op, SelectionDAG &DAG, SDValue Mask, SDValue Base, SDValue Index, SDValue ScaleOp, SDValue Chain, const X86Subtarget &Subtarget)
 
static SDValue expandIntrinsicWChainHelper (SDNode *N, const SDLoc &DL, SelectionDAG &DAG, unsigned TargetOpcode, unsigned SrcReg, const X86Subtarget &Subtarget, SmallVectorImpl< SDValue > &Results)
 Handles the lowering of builtin intrinsics with chain that return their value into registers EDX:EAX. More...
 
static void getReadTimeStampCounter (SDNode *N, const SDLoc &DL, unsigned Opcode, SelectionDAG &DAG, const X86Subtarget &Subtarget, SmallVectorImpl< SDValue > &Results)
 Handles the lowering of builtin intrinsics that read the time stamp counter (x86_rdtsc and x86_rdtscp). More...
 
static SDValue LowerREADCYCLECOUNTER (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 
static SDValue MarkEHRegistrationNode (SDValue Op, SelectionDAG &DAG)
 
static SDValue MarkEHGuard (SDValue Op, SelectionDAG &DAG)
 
static SDValue EmitTruncSStore (bool SignedSat, SDValue Chain, const SDLoc &Dl, SDValue Val, SDValue Ptr, EVT MemVT, MachineMemOperand *MMO, SelectionDAG &DAG)
 Emit Truncating Store with signed or unsigned saturation. More...
 
static SDValue EmitMaskedTruncSStore (bool SignedSat, SDValue Chain, const SDLoc &Dl, SDValue Val, SDValue Ptr, SDValue Mask, EVT MemVT, MachineMemOperand *MMO, SelectionDAG &DAG)
 Emit Masked Truncating Store with signed or unsigned saturation. More...
 
static SDValue LowerINTRINSIC_W_CHAIN (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 
static SDValue LowerADJUST_TRAMPOLINE (SDValue Op, SelectionDAG &DAG)
 
static SDValue LowerVectorCTLZ_AVX512CDI (SDValue Op, SelectionDAG &DAG, const X86Subtarget &Subtarget)
 Lower a vector CTLZ using native supported vector CTLZ instruction. More...
 
static SDValue LowerVectorCTLZInRegLUT (SDValue Op, const SDLoc &DL, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 
static SDValue LowerVectorCTLZ (SDValue Op, const SDLoc &DL, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 
static SDValue LowerCTLZ (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 
static SDValue LowerCTTZ (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 
static SDValue lowerAddSub (SDValue Op, SelectionDAG &DAG, const X86Subtarget &Subtarget)
 
static SDValue LowerADDSAT_SUBSAT (SDValue Op, SelectionDAG &DAG, const X86Subtarget &Subtarget)
 
static SDValue LowerABS (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 
static SDValue LowerMINMAX (SDValue Op, SelectionDAG &DAG)
 
static SDValue LowerMUL (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 
static SDValue LowerMULH (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 
static bool SupportedVectorShiftWithImm (MVT VT, const X86Subtarget &Subtarget, unsigned Opcode)
 
static bool SupportedVectorShiftWithBaseAmnt (MVT VT, const X86Subtarget &Subtarget, unsigned Opcode)
 
static bool SupportedVectorVarShift (MVT VT, const X86Subtarget &Subtarget, unsigned Opcode)
 
static SDValue LowerScalarImmediateShift (SDValue Op, SelectionDAG &DAG, const X86Subtarget &Subtarget)
 
static SDValue LowerScalarVariableShift (SDValue Op, SelectionDAG &DAG, const X86Subtarget &Subtarget)
 
static SDValue convertShiftLeftToScale (SDValue Amt, const SDLoc &dl, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 
static SDValue LowerRotate (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 
static SDValue emitLockedStackOp (SelectionDAG &DAG, const X86Subtarget &Subtarget, SDValue Chain, SDLoc DL)
 Emit a locked operation on a stack location which does not change any memory location, but does involve a lock prefix. More...
 
static SDValue LowerATOMIC_FENCE (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 
static SDValue LowerCMP_SWAP (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 
static SDValue getPMOVMSKB (const SDLoc &DL, SDValue V, SelectionDAG &DAG, const X86Subtarget &Subtarget)
 
static SDValue LowerBITCAST (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 
static SDValue LowerHorizontalByteSum (SDValue V, MVT VT, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 Compute the horizontal sum of bytes in V for the elements of VT. More...
 
static SDValue LowerVectorCTPOPInRegLUT (SDValue Op, const SDLoc &DL, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 
static SDValue LowerVectorCTPOP (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 
static SDValue LowerCTPOP (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 
static SDValue LowerBITREVERSE_XOP (SDValue Op, SelectionDAG &DAG)
 
static SDValue LowerBITREVERSE (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 
static SDValue lowerAtomicArithWithLOCK (SDValue N, SelectionDAG &DAG, const X86Subtarget &Subtarget)
 
static SDValue lowerAtomicArith (SDValue N, SelectionDAG &DAG, const X86Subtarget &Subtarget)
 Lower atomic_load_ops into LOCK-prefixed operations. More...
 
static SDValue LowerATOMIC_STORE (SDValue Op, SelectionDAG &DAG, const X86Subtarget &Subtarget)
 
static SDValue LowerADDSUBCARRY (SDValue Op, SelectionDAG &DAG)
 
static SDValue LowerFSINCOS (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 
static SDValue ExtendToType (SDValue InOp, MVT NVT, SelectionDAG &DAG, bool FillWithZeroes=false)
 Widen a vector input to a vector of NVT. More...
 
static SDValue LowerMSCATTER (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 
static SDValue LowerMLOAD (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 
static SDValue LowerMSTORE (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 
static SDValue LowerMGATHER (SDValue Op, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 
static SDValue LowerADDRSPACECAST (SDValue Op, SelectionDAG &DAG)
 
static SDValue LowerCVTPS2PH (SDValue Op, SelectionDAG &DAG)
 
static MachineBasicBlockemitXBegin (MachineInstr &MI, MachineBasicBlock *MBB, const TargetInstrInfo *TII)
 Utility function to emit xbegin specifying the start of an RTM region. More...
 
static bool checkAndUpdateEFLAGSKill (MachineBasicBlock::iterator SelectItr, MachineBasicBlock *BB, const TargetRegisterInfo *TRI)
 
static bool isCMOVPseudo (MachineInstr &MI)
 
static MachineInstrBuilder createPHIsForCMOVsInSinkBB (MachineBasicBlock::iterator MIItBegin, MachineBasicBlock::iterator MIItEnd, MachineBasicBlock *TrueMBB, MachineBasicBlock *FalseMBB, MachineBasicBlock *SinkMBB)
 
static unsigned getSUBriOpcode (bool IsLP64, int64_t Imm)
 
static unsigned getOpcodeForIndirectThunk (unsigned RPOpc)
 
static const chargetIndirectThunkSymbol (const X86Subtarget &Subtarget, unsigned Reg)
 
static SDValue narrowLoadToVZLoad (LoadSDNode *LN, MVT MemVT, MVT VT, SelectionDAG &DAG)
 
static bool matchUnaryShuffle (MVT MaskVT, ArrayRef< int > Mask, bool AllowFloatDomain, bool AllowIntDomain, SDValue &V1, const SDLoc &DL, SelectionDAG &DAG, const X86Subtarget &Subtarget, unsigned &Shuffle, MVT &SrcVT, MVT &DstVT)
 
static bool matchUnaryPermuteShuffle (MVT MaskVT, ArrayRef< int > Mask, const APInt &Zeroable, bool AllowFloatDomain, bool AllowIntDomain, const X86Subtarget &Subtarget, unsigned &Shuffle, MVT &ShuffleVT, unsigned &PermuteImm)
 
static bool matchBinaryShuffle (MVT MaskVT, ArrayRef< int > Mask, bool AllowFloatDomain, bool AllowIntDomain, SDValue &V1, SDValue &V2, const SDLoc &DL, SelectionDAG &DAG, const X86Subtarget &Subtarget, unsigned &Shuffle, MVT &SrcVT, MVT &DstVT, bool IsUnary)
 
static bool matchBinaryPermuteShuffle (MVT MaskVT, ArrayRef< int > Mask, const APInt &Zeroable, bool AllowFloatDomain, bool AllowIntDomain, SDValue &V1, SDValue &V2, const SDLoc &DL, SelectionDAG &DAG, const X86Subtarget &Subtarget, unsigned &Shuffle, MVT &ShuffleVT, unsigned &PermuteImm)
 
static SDValue combineX86ShuffleChainWithExtract (ArrayRef< SDValue > Inputs, SDValue Root, ArrayRef< int > BaseMask, int Depth, bool HasVariableMask, bool AllowVariableMask, SelectionDAG &DAG, const X86Subtarget &Subtarget)
 
static SDValue combineX86ShuffleChain (ArrayRef< SDValue > Inputs, SDValue Root, ArrayRef< int > BaseMask, int Depth, bool HasVariableMask, bool AllowVariableMask, SelectionDAG &DAG, const X86Subtarget &Subtarget)
 Combine an arbitrary chain of shuffles into a single instruction if possible. More...
 
static SDValue combineX86ShufflesConstants (ArrayRef< SDValue > Ops, ArrayRef< int > Mask, SDValue Root, bool HasVariableMask, SelectionDAG &DAG, const X86Subtarget &Subtarget)
 
static SDValue combineX86ShufflesRecursively (ArrayRef< SDValue > SrcOps, int SrcOpIndex, SDValue Root, ArrayRef< int > RootMask, ArrayRef< const SDNode *> SrcNodes, unsigned Depth, bool HasVariableMask, bool AllowVariableMask, SelectionDAG &DAG, const X86Subtarget &Subtarget)
 Fully generic combining of x86 shuffle instructions. More...
 
static SDValue combineX86ShufflesRecursively (SDValue Op, SelectionDAG &DAG, const X86Subtarget &Subtarget)
 Helper entry wrapper to combineX86ShufflesRecursively. More...
 
static SmallVector< int, 4 > getPSHUFShuffleMask (SDValue N)
 Get the PSHUF-style mask from PSHUF node. More...
 
static SDValue combineRedundantDWordShuffle (SDValue N, MutableArrayRef< int > Mask, SelectionDAG &DAG)
 Search for a combinable shuffle across a chain ending in pshufd. More...
 
static SDValue combineCommutableSHUFP (SDValue N, MVT VT, const SDLoc &DL, SelectionDAG &DAG)
 
static SDValue combineTargetShuffle (SDValue N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget)
 Try to combine x86 target specific shuffles. More...
 
static bool isAddSubOrSubAddMask (ArrayRef< int > Mask, bool &Op0Even)
 Checks if the shuffle mask takes subsequent elements alternately from two vectors. More...
 
static bool isAddSubOrSubAdd (SDNode *N, const X86Subtarget &Subtarget, SelectionDAG &DAG, SDValue &Opnd0, SDValue &Opnd1, bool &IsSubAdd)
 Returns true iff the shuffle node N can be replaced with ADDSUB(SUBADD) operation. More...
 
static SDValue combineShuffleToFMAddSub (SDNode *N, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 Combine shuffle of two fma nodes into FMAddSub or FMSubAdd. More...
 
static SDValue combineShuffleToAddSubOrFMAddSub (SDNode *N, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 Try to combine a shuffle into a target-specific add-sub or mul-add-sub node. More...
 
static SDValue combineShuffleOfConcatUndef (SDNode *N, SelectionDAG &DAG, const X86Subtarget &Subtarget)
 
static SDValue foldShuffleOfHorizOp (SDNode *N, SelectionDAG &DAG)
 Eliminate a redundant shuffle of a horizontal math op. More...
 
static SDValue narrowShuffle (ShuffleVectorSDNode *Shuf, SelectionDAG &DAG)
 If we have a shuffle of AVX/AVX512 (256/512 bit) vectors that only uses the low half of each source vector and does not set any high half elements in the destination vector, narrow the shuffle to half its original size. More...
 
static SDValue combineShuffle (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget)
 
static bool checkBitcastSrcVectorSize (SDValue Src, unsigned Size)
 
static unsigned getAltBitOpcode (unsigned Opcode)
 
static SDValue adjustBitcastSrcVectorSSE1 (SelectionDAG &DAG, SDValue Src, const SDLoc &DL)
 
static SDValue signExtendBitcastSrcVector (SelectionDAG &DAG, EVT SExtVT, SDValue Src, const SDLoc &DL)
 
static SDValue combineBitcastvxi1 (SelectionDAG &DAG, EVT VT, SDValue Src, const SDLoc &DL, const X86Subtarget &Subtarget)
 
static SDValue combinevXi1ConstantToInteger (SDValue Op, SelectionDAG &DAG)
 
static SDValue combineCastedMaskArithmetic (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget)
 
static SDValue createMMXBuildVector (BuildVectorSDNode *BV, SelectionDAG &DAG, const X86Subtarget &Subtarget)
 
static SDValue combineBitcastToBoolVector (EVT VT, SDValue V, SDLoc DL, SelectionDAG &DAG, const X86Subtarget &Subtarget)
 
static SDValue combineBitcast (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget)
 
static bool detectZextAbsDiff (const SDValue &Abs, SDValue &Op0, SDValue &Op1)
 
static SDValue createPSADBW (SelectionDAG &DAG, const SDValue &Zext0, const SDValue &Zext1, const SDLoc &DL, const X86Subtarget &Subtarget)
 
static SDValue combineHorizontalMinMaxResult (SDNode *Extract, SelectionDAG &DAG, const X86Subtarget &Subtarget)
 
static SDValue combineHorizontalPredicateResult (SDNode *Extract, SelectionDAG &DAG, const X86Subtarget &Subtarget)
 
static SDValue combineBasicSADPattern (SDNode *Extract, SelectionDAG &DAG, const X86Subtarget &Subtarget)
 
static SDValue combineExtractWithShuffle (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget)
 
static SDValue scalarizeExtEltFP (SDNode *ExtElt, SelectionDAG &DAG)
 Extracting a scalar FP value from vector element 0 is free, so extract each operand first, then perform the math as a scalar op. More...
 
static SDValue combineReductionToHorizontal (SDNode *ExtElt, SelectionDAG &DAG, const X86Subtarget &Subtarget)
 Try to convert a vector reduction sequence composed of binops and shuffles into horizontal ops. More...
 
static SDValue combineExtractVectorElt (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget)
 Detect vector gather/scatter index generation and convert it from being a bunch of shuffles and extracts into a somewhat faster sequence. More...
 
static SDValue combineVSelectWithAllOnesOrZeros (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget)
 If a vector select has an operand that is -1 or 0, try to simplify the select to a bitwise logic operation. More...
 
static SDValue narrowVectorSelect (SDNode *N, SelectionDAG &DAG, const X86Subtarget &Subtarget)
 If both arms of a vector select are concatenated vectors, split the select, and concatenate the result to eliminate a wide (256-bit) vector instruction: vselect Cond, (concat T0, T1), (concat F0, F1) –> concat (vselect (split Cond), T0, F0), (vselect (split Cond), T1, F1) More...
 
static SDValue combineSelectOfTwoConstants (SDNode *N, SelectionDAG &DAG)
 
static SDValue combineVSelectToBLENDV (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget)
 If this is a dynamic select (non-constant condition) and we can match this node with one of the variable blend instructions, restructure the condition so that blends can use the high (sign) bit of each element. More...
 
static SDValue combineLogicBlendIntoConditionalNegate (EVT VT, SDValue Mask, SDValue X, SDValue Y, const SDLoc &DL, SelectionDAG &DAG, const X86Subtarget &Subtarget)
 
static SDValue combineSelect (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget)
 Do target-specific dag combines on SELECT and VSELECT nodes. More...
 
static SDValue combineSetCCAtomicArith (SDValue Cmp, X86::CondCode &CC, SelectionDAG &DAG, const X86Subtarget &Subtarget)
 Combine: (brcond/cmov/setcc .., (cmp (atomic_load_add x, 1), 0), COND_S) to: (brcond/cmov/setcc .., (LADD x, 1), COND_LE) i.e., reusing the EFLAGS produced by the LOCKed instruction. More...
 
static SDValue checkBoolTestSetCCCombine (SDValue Cmp, X86::CondCode &CC)
 
static bool checkBoolTestAndOrSetCCCombine (SDValue Cond, X86::CondCode &CC0, X86::CondCode &CC1, SDValue &Flags, bool &isAnd)
 Check whether Cond is an AND/OR of SETCCs off of the same EFLAGS. More...
 
static SDValue combineCarryThroughADD (SDValue EFLAGS, SelectionDAG &DAG)
 
static SDValue combinePTESTCC (SDValue EFLAGS, X86::CondCode &CC, SelectionDAG &DAG, const X86Subtarget &Subtarget)
 If we are inverting an PTEST/TESTP operand, attempt to adjust the CC to avoid the inversion. More...
 
static SDValue combineSetCCMOVMSK (SDValue EFLAGS, X86::CondCode &CC, SelectionDAG &DAG, const X86Subtarget &Subtarget)
 
static SDValue combineSetCCEFLAGS (SDValue EFLAGS, X86::CondCode &CC, SelectionDAG &DAG, const X86Subtarget &Subtarget)
 Optimize an EFLAGS definition used according to the condition code CC into a simpler EFLAGS value, potentially returning a new CC and replacing uses of chain values. More...
 
static SDValue combineCMov (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget)
 Optimize X86ISD::CMOV [LHS, RHS, CONDCODE (e.g. X86::COND_NE), CONDVAL]. More...
 
static bool canReduceVMulWidth (SDNode *N, SelectionDAG &DAG, ShrinkMode &Mode)
 
static SDValue reduceVMULWidth (SDNode *N, SelectionDAG &DAG, const X86Subtarget &Subtarget)
 When the operands of vector mul are extended from smaller size values, like i8 and i16, the type of mul may be shrinked to generate more efficient code. More...
 
static SDValue combineMulSpecial (uint64_t MulAmt, SDNode *N, SelectionDAG &DAG, EVT VT, const SDLoc &DL)
 
static SDValue combineMulToPMADDWD (SDNode *N, SelectionDAG &DAG, const X86Subtarget &Subtarget)
 
static SDValue combineMulToPMULDQ (SDNode *N, SelectionDAG &DAG, const X86Subtarget &Subtarget)
 
static SDValue combineMul (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget)
 Optimize a single multiply with constant into two operations in order to implement it with two cheaper instructions, e.g. More...
 
static SDValue combineShiftToPMULH (SDNode *N, SelectionDAG &DAG, const X86Subtarget &Subtarget)
 
static SDValue combineShiftLeft (SDNode *N, SelectionDAG &DAG)
 
static SDValue combineShiftRightArithmetic (SDNode *N, SelectionDAG &DAG, const X86Subtarget &Subtarget)
 
static SDValue combineShiftRightLogical (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget)
 
static SDValue combineVectorPackWithShuffle (SDNode *N, SelectionDAG &DAG)
 
static SDValue combineVectorPack (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget)
 
static SDValue combineVectorShiftVar (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget)
 
static SDValue combineVectorShiftImm (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget)
 
static SDValue combineVectorInsert (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget)
 
static SDValue combineCompareEqual (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget)
 Recognize the distinctive (AND (setcc ...) (setcc ..)) where both setccs reference the same FP CMP, and rewrite for CMPEQSS and friends. More...
 
static SDValue combineANDXORWithAllOnesIntoANDNP (SDNode *N, SelectionDAG &DAG)
 Try to fold: (and (xor X, -1), Y) -> (andnp X, Y). More...
 
static SDValue PromoteMaskArithmetic (SDNode *N, EVT VT, SelectionDAG &DAG, unsigned Depth)
 
static SDValue PromoteMaskArithmetic (SDNode *N, SelectionDAG &DAG, const X86Subtarget &Subtarget)
 
static unsigned convertIntLogicToFPLogicOpcode (unsigned Opcode)
 
static SDValue convertIntLogicToFPLogic (SDNode *N, SelectionDAG &DAG, const X86Subtarget &Subtarget)
 If both input operands of a logic op are being cast from floating point types, try to convert this into a floating point logic node to avoid unnecessary moves from SSE to integer registers. More...
 
static SDValue combineBitOpWithMOVMSK (SDNode *N, SelectionDAG &DAG)
 
static SDValue combineAndMaskToShift (SDNode *N, SelectionDAG &DAG, const X86Subtarget &Subtarget)
 If this is a zero/all-bits result that is bitwise-anded with a low bits mask. More...
 
static SDValue getIndexFromUnindexedLoad (LoadSDNode *Ld)
 
static bool hasBZHI (const X86Subtarget &Subtarget, MVT VT)
 
static SDValue combineAndLoadToBZHI (SDNode *Node, SelectionDAG &DAG, const X86Subtarget &Subtarget)
 
static SDValue combineParity (SDNode *N, SelectionDAG &DAG, const X86Subtarget &Subtarget)
 
static SDValue combineScalarAndWithMaskSetcc (SDNode *N, SelectionDAG &DAG, const X86Subtarget &Subtarget)
 
static SDValue combineAnd (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget)
 
static SDValue canonicalizeBitSelect (SDNode *N, SelectionDAG &DAG, const X86Subtarget &Subtarget)
 
static bool matchLogicBlend (SDNode *N, SDValue &X, SDValue &Y, SDValue &Mask)
 
static SDValue combineLogicBlendIntoPBLENDV (SDNode *N, SelectionDAG &DAG, const X86Subtarget &Subtarget)
 
static SDValue lowerX86CmpEqZeroToCtlzSrl (SDValue Op, EVT ExtTy, SelectionDAG &DAG)
 
static SDValue combineOrCmpEqZeroToCtlzSrl (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget)
 
static SDValue combineOr (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget)
 
static SDValue foldXorTruncShiftIntoCmp (SDNode *N, SelectionDAG &DAG)
 Try to turn tests against the signbit in the form of: XOR(TRUNCATE(SRL(X, size(X)-1)), 1) into: SETGT(X, -1) More...
 
static SDValue foldVectorXorShiftIntoCmp (SDNode *N, SelectionDAG &DAG, const X86Subtarget &Subtarget)
 Turn vector tests of the signbit in the form of: xor (sra X, elt_size(X)-1), -1 into: pcmpgt X, -1. More...
 
static SDValue detectUSatPattern (SDValue In, EVT VT, SelectionDAG &DAG, const SDLoc &DL)
 Detect patterns of truncation with unsigned saturation: More...
 
static SDValue detectSSatPattern (SDValue In, EVT VT, bool MatchPackUS=false)
 Detect patterns of truncation with signed saturation: (truncate (smin ((smax (x, signed_min_of_dest_type)), signed_max_of_dest_type)) to dest_type) or: (truncate (smax ((smin (x, signed_max_of_dest_type)), signed_min_of_dest_type)) to dest_type). More...
 
static SDValue combineTruncateWithSat (SDValue In, EVT VT, const SDLoc &DL, SelectionDAG &DAG, const X86Subtarget &Subtarget)
 
static SDValue detectAVGPattern (SDValue In, EVT VT, SelectionDAG &DAG, const X86Subtarget &Subtarget, const SDLoc &DL)
 This function detects the AVG pattern between vectors of unsigned i8/i16, which is c = (a + b + 1) / 2, and replace this operation with the efficient X86ISD::AVG instruction. More...
 
static SDValue combineLoad (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget)
 
static int getOneTrueElt (SDValue V)
 If V is a build vector of boolean constants and exactly one of those constants is true, return the operand index of that true element. More...
 
static bool getParamsForOneTrueMaskedElt (MaskedLoadStoreSDNode *MaskedOp, SelectionDAG &DAG, SDValue &Addr, SDValue &Index, unsigned &Alignment)
 Given a masked memory load/store operation, return true if it has one mask bit set. More...
 
static SDValue reduceMaskedLoadToScalarLoad (MaskedLoadSDNode *ML, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI)
 If exactly one element of the mask is set for a non-extending masked load, it is a scalar load and vector insert. More...
 
static SDValue combineMaskedLoadConstantMask (MaskedLoadSDNode *ML, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI)
 
static SDValue combineMaskedLoad (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget)
 
static SDValue reduceMaskedStoreToScalarStore (MaskedStoreSDNode *MS, SelectionDAG &DAG)
 If exactly one element of the mask is set for a non-truncating masked store, it is a vector extract and scalar store. More...
 
static SDValue combineMaskedStore (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget)
 
static SDValue combineStore (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget)
 
static SDValue combineVEXTRACT_STORE (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget)
 
static bool isHorizontalBinOp (SDValue &LHS, SDValue &RHS, SelectionDAG &DAG, const X86Subtarget &Subtarget, bool IsCommutative)
 Return 'true' if this vector operation is "horizontal" and return the operands for the horizontal operation in LHS and RHS. More...
 
static SDValue combineFaddFsub (SDNode *N, SelectionDAG &DAG, const X86Subtarget &Subtarget)
 Do target-specific dag combines on floating-point adds/subs. More...
 
static SDValue combineTruncatedArithmetic (SDNode *N, SelectionDAG &DAG, const X86Subtarget &Subtarget, const SDLoc &DL)
 Attempt to pre-truncate inputs to arithmetic ops if it will simplify the codegen. More...
 
static SDValue combineVectorTruncationWithPACKUS (SDNode *N, const SDLoc &DL, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 Truncate using ISD::AND mask and X86ISD::PACKUS. More...
 
static SDValue combineVectorTruncationWithPACKSS (SDNode *N, const SDLoc &DL, const X86Subtarget &Subtarget, SelectionDAG &DAG)
 Truncate a group of v4i32 into v8i16 using X86ISD::PACKSS. More...
 
static SDValue combineVectorTruncation (SDNode *N, SelectionDAG &DAG, const X86Subtarget &Subtarget)
 This function transforms truncation from vXi32/vXi64 to vXi8/vXi16 into X86ISD::PACKUS/X86ISD::PACKSS operations. More...
 
static SDValue combineVectorSignBitsTruncation (SDNode *N, const SDLoc &DL, SelectionDAG &DAG, const X86Subtarget &Subtarget)
 This function transforms vector truncation of 'extended sign-bits' or 'extended zero-bits' values. More...
 
static SDValue combinePMULH (SDValue Src, EVT VT, const SDLoc &DL, SelectionDAG &DAG, const X86Subtarget &Subtarget)
 
static SDValue detectPMADDUBSW (SDValue In, EVT VT, SelectionDAG &DAG, const X86Subtarget &Subtarget, const SDLoc &DL)
 
static SDValue combineTruncate (SDNode *N, SelectionDAG &DAG, const X86Subtarget &Subtarget)
 
static SDValue combineVTRUNC (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI)
 
static SDValue isFNEG (SelectionDAG &DAG, SDNode *N, unsigned Depth=0)
 Returns the negated value if the node N flips sign of FP value. More...
 
static unsigned negateFMAOpcode (unsigned Opcode, bool NegMul, bool NegAcc, bool NegRes)
 
static SDValue combineFneg (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget)
 Do target-specific dag combines on floating point negations. More...
 
static SDValue lowerX86FPLogicOp (SDNode *N, SelectionDAG &DAG, const X86Subtarget &Subtarget)
 
static SDValue foldXor1SetCC (SDNode *N, SelectionDAG &DAG)
 Fold a xor(setcc cond, val), 1 –> setcc (inverted(cond), val) More...
 
static SDValue combineXor (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget)
 
static SDValue combineBEXTR (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget)
 
static bool isNullFPScalarOrVectorConst (SDValue V)
 
static SDValue getNullFPConstForNullVal (SDValue V, SelectionDAG &DAG, const X86Subtarget &Subtarget)
 If a value is a scalar FP zero or a vector FP zero (potentially including undefined elements), return a zero constant that may be used to fold away that value. More...
 
static SDValue combineFAndFNotToFAndn (SDNode *N, SelectionDAG &DAG, const X86Subtarget &Subtarget)
 
static SDValue combineFAnd (SDNode *N, SelectionDAG &DAG, const X86Subtarget &Subtarget)
 Do target-specific dag combines on X86ISD::FAND nodes. More...
 
static SDValue combineFAndn (SDNode *N, SelectionDAG &DAG, const X86Subtarget &Subtarget)
 Do target-specific dag combines on X86ISD::FANDN nodes. More...
 
static SDValue combineFOr (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget)
 Do target-specific dag combines on X86ISD::FOR and X86ISD::FXOR nodes. More...
 
static SDValue combineFMinFMax (SDNode *N, SelectionDAG &DAG)
 Do target-specific dag combines on X86ISD::FMIN and X86ISD::FMAX nodes. More...
 
static SDValue combineFMinNumFMaxNum (SDNode *N, SelectionDAG &DAG, const X86Subtarget &Subtarget)
 
static SDValue combineX86INT_TO_FP (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI)
 
static SDValue combineCVTP2I_CVTTP2I (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI)
 
static SDValue combineAndnp (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget)
 Do target-specific dag combines on X86ISD::ANDNP nodes. More...
 
static SDValue combineBT (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI)
 
static SDValue combineCVTPH2PS (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI)
 
static SDValue combineSextInRegCmov (SDNode *N, SelectionDAG &DAG)
 
static SDValue combineSignExtendInReg (SDNode *N, SelectionDAG &DAG, const X86Subtarget &Subtarget)
 
static SDValue promoteExtBeforeAdd (SDNode *Ext, SelectionDAG &DAG, const X86Subtarget &Subtarget)
 sext(add_nsw(x, C)) –> add(sext(x), C_sext) zext(add_nuw(x, C)) –> add(zext(x), C_zext) Promoting a sign/zero extension ahead of a no overflow 'add' exposes opportunities to combine math ops, use an LEA, or use a complex addressing mode. More...
 
static SDValue combineToExtendCMOV (SDNode *Extend, SelectionDAG &DAG)
 
static SDValue combineToExtendBoolVectorInReg (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget)
 
static SDValue combineExtSetcc (SDNode *N, SelectionDAG &DAG, const X86Subtarget &Subtarget)
 
static SDValue combineSext (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget)
 
static SDValue combineFMA (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget)
 
static SDValue combineFMADDSUB (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI)
 
static SDValue combineZext (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget)
 
static bool isOrXorXorTree (SDValue X, bool Root=true)
 Recursive helper for combineVectorSizedSetCCEquality() to see if we have a recognizable memcmp expansion. More...
 
template<typename F >
static SDValue emitOrXorXorTree (SDValue X, SDLoc &DL, SelectionDAG &DAG, EVT VecVT, EVT CmpVT, bool HasPT, F SToV)
 Recursive helper for combineVectorSizedSetCCEquality() to emit the memcmp expansion. More...
 
static SDValue combineVectorSizedSetCCEquality (SDNode *SetCC, SelectionDAG &DAG, const X86Subtarget &Subtarget)
 Try to map a 128-bit or larger integer comparison to vector instructions before type legalization splits it up into chunks. More...
 
static SDValue combineSetCC (SDNode *N, SelectionDAG &DAG, const X86Subtarget &Subtarget)
 
static SDValue combineMOVMSK (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget)
 
static SDValue combineX86GatherScatter (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI)
 
static SDValue rebuildGatherScatter (MaskedGatherScatterSDNode *GorS, SDValue Index, SDValue Base, SDValue Scale, SelectionDAG &DAG)
 
static SDValue combineGatherScatter (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI)
 
static SDValue combineX86SetCC (SDNode *N, SelectionDAG &DAG, const X86Subtarget &Subtarget)
 
static SDValue combineBrCond (SDNode *N, SelectionDAG &DAG, const X86Subtarget &Subtarget)
 Optimize branch condition evaluation. More...
 
static SDValue combineVectorCompareAndMaskUnaryOp (SDNode *N, SelectionDAG &DAG)
 
static SDValue combineToFPTruncExtElt (SDNode *N, SelectionDAG &DAG)
 If we are converting a value to floating-point, try to replace scalar truncate of an extracted vector element with a bitcast. More...
 
static SDValue combineUIntToFP (SDNode *N, SelectionDAG &DAG, const X86Subtarget &Subtarget)
 
static SDValue combineSIntToFP (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget)
 
static bool needCarryOrOverflowFlag (SDValue Flags)
 
static bool onlyZeroFlagUsed (SDValue Flags)
 
static SDValue combineCMP (SDNode *N, SelectionDAG &DAG)
 
static SDValue combineX86AddSub (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI)
 
static SDValue combineSBB (SDNode *N, SelectionDAG &DAG)
 
static SDValue combineADC (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI)
 
static SDValue combineAddOrSubToADCOrSBB (SDNode *N, SelectionDAG &DAG)
 If this is an add or subtract where one operand is produced by a cmp+setcc, then try to convert it to an ADC or SBB. More...
 
static SDValue matchPMADDWD (SelectionDAG &DAG, SDValue Op0, SDValue Op1, const SDLoc &DL, EVT VT, const X86Subtarget &Subtarget)
 
static SDValue matchPMADDWD_2 (SelectionDAG &DAG, SDValue N0, SDValue N1, const SDLoc &DL, EVT VT, const X86Subtarget &Subtarget)
 
static SDValue combineAdd (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget)
 
static SDValue combineSubToSubus (SDNode *N, SelectionDAG &DAG, const X86Subtarget &Subtarget)
 
static SDValue combineSub (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget)
 
static SDValue combineVectorCompare (SDNode *N, SelectionDAG &DAG, const X86Subtarget &Subtarget)
 
static SDValue combineConcatVectorOps (const SDLoc &DL, MVT VT, ArrayRef< SDValue > Ops, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget)
 Helper that combines an array of subvector ops as if they were the operands of a ISD::CONCAT_VECTORS node, but may have come from another source (e.g. More...
 
static SDValue combineConcatVectors (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget)
 
static SDValue combineInsertSubvector (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget)
 
static SDValue narrowExtractedVectorSelect (SDNode *Ext, SelectionDAG &DAG)
 If we are extracting a subvector of a vector select and the select condition is composed of concatenated vectors, try to narrow the select width. More...
 
static SDValue combineExtractSubvector (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget)
 
static SDValue combineScalarToVector (SDNode *N, SelectionDAG &DAG)
 
static SDValue combinePMULDQ (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget)
 
static SDValue combineExtInVec (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const X86Subtarget &Subtarget)
 
static SDValue combineKSHIFT (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI)
 
static SDValue combineFP16_TO_FP (SDNode *N, SelectionDAG &DAG, const X86Subtarget &Subtarget)
 
static SDValue combineFP_EXTEND (SDNode *N, SelectionDAG &DAG, const X86Subtarget &Subtarget)
 
static SDValue combineVBROADCAST_LOAD (SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI)
 
static SDValue combineFP_ROUND (SDNode *N, SelectionDAG &DAG, const X86Subtarget &Subtarget)
 
static SDValue combineMOVDQ2Q (SDNode *N, SelectionDAG &DAG)
 
static bool matchAsm (StringRef S, ArrayRef< const char *> Pieces)
 
static bool clobbersFlagRegisters (const SmallVector< StringRef, 4 > &AsmPieces)
 
static X86::CondCode parseConstraintCode (llvm::StringRef Constraint)
 
static bool isGRClass (const TargetRegisterClass &RC)
 Check if RC is a general purpose register class. More...
 
static bool isFRClass (const TargetRegisterClass &RC)
 Check if RC is a vector register class. More...
 
static bool isVKClass (const TargetRegisterClass &RC)
 Check if RC is a mask register class. More...
 

Variables

static cl::opt< int > ExperimentalPrefLoopAlignment ("x86-experimental-pref-loop-alignment", cl::init(4), cl::desc("Sets the preferable loop alignment for experiments (as log2 bytes)" "(the last x86-experimental-pref-loop-alignment bits" " of the loop header PC will be 0)."), cl::Hidden)
 
static cl::opt< boolMulConstantOptimization ("mul-constant-optimization", cl::init(true), cl::desc("Replace 'mul x, Const' with more effective instructions like " "SHIFT, LEA, etc."), cl::Hidden)
 
static cl::opt< boolExperimentalUnorderedISEL ("x86-experimental-unordered-atomic-isel", cl::init(false), cl::desc("Use LoadSDNode and StoreSDNode instead of " "AtomicSDNode for unordered atomic loads and " "stores respectively."), cl::Hidden)
 

Macro Definition Documentation

◆ DEBUG_TYPE

#define DEBUG_TYPE   "x86-isel"

Definition at line 67 of file X86ISelLowering.cpp.

◆ NODE_NAME_CASE

#define NODE_NAME_CASE (   NODE)    case X86ISD::NODE: return "X86ISD::" #NODE;

Enumeration Type Documentation

◆ ShrinkMode

enum ShrinkMode
strong

Different mul shrinking modes.

Enumerator
MULS8 
MULU8 
MULS16 
MULU16 

Definition at line 41215 of file X86ISelLowering.cpp.

◆ StructReturnType

CallIsStructReturn - Determines whether a call uses struct return semantics.

Enumerator
NotStructReturn 
RegStructReturn 
StackStructReturn 

Definition at line 3080 of file X86ISelLowering.cpp.

Function Documentation

◆ adjustBitcastSrcVectorSSE1()

static SDValue adjustBitcastSrcVectorSSE1 ( SelectionDAG DAG,
SDValue  Src,
const SDLoc DL 
)
static

◆ argsAreStructReturn()

static StructReturnType argsAreStructReturn ( ArrayRef< ISD::InputArg Ins,
bool  IsMCU 
)
static

Determines whether a function uses struct return semantics.

Definition at line 3100 of file X86ISelLowering.cpp.

References llvm::ArrayRef< T >::empty(), llvm::ISD::ArgFlagsTy::isInReg(), llvm::ISD::ArgFlagsTy::isSRet(), NotStructReturn, RegStructReturn, and StackStructReturn.

Referenced by isSortedByValueNo().

◆ buildFromShuffleMostly()

static SDValue buildFromShuffleMostly ( SDValue  Op,
SelectionDAG DAG 
)
static

◆ callIsStructReturn()

static StructReturnType callIsStructReturn ( ArrayRef< ISD::OutputArg Outs,
bool  IsMCU 
)
static

◆ canGuaranteeTCO()

static bool canGuaranteeTCO ( CallingConv::ID  CC)
static

◆ canLowerByDroppingEvenElements()

static int canLowerByDroppingEvenElements ( ArrayRef< int >  Mask,
bool  IsSingleInput 
)
static

Check whether a compaction lowering can be done by dropping even elements and compute how many times even elements must be dropped.

This handles shuffles which take every Nth element where N is a power of two. Example shuffle masks:

N = 1: 0, 2, 4, 6, 8, 10, 12, 14, 0, 2, 4, 6, 8, 10, 12, 14 N = 1: 0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30 N = 2: 0, 4, 8, 12, 0, 4, 8, 12, 0, 4, 8, 12, 0, 4, 8, 12 N = 2: 0, 4, 8, 12, 16, 20, 24, 28, 0, 4, 8, 12, 16, 20, 24, 28 N = 3: 0, 8, 0, 8, 0, 8, 0, 8, 0, 8, 0, 8, 0, 8, 0, 8 N = 3: 0, 8, 16, 24, 0, 8, 16, 24, 0, 8, 16, 24, 0, 8, 16, 24

Any of these lanes can of course be undef.

This routine only supports N <= 3. FIXME: Evaluate whether either AVX or AVX-512 have any opportunities here for larger N.

Returns
N above, or the number of times even elements must be dropped if there is such a number. Otherwise returns zero.

Definition at line 11368 of file X86ISelLowering.cpp.

References llvm::array_lengthof(), assert(), llvm::numbers::e, llvm::isPowerOf2_32(), and llvm::ArrayRef< T >::size().

Referenced by lowerV16I8Shuffle(), and lowerV8I16Shuffle().

◆ canonicalizeBitSelect()

static SDValue canonicalizeBitSelect ( SDNode N,
SelectionDAG DAG,
const X86Subtarget Subtarget 
)
static

◆ canonicalizeShuffleMaskWithCommute()

static bool canonicalizeShuffleMaskWithCommute ( ArrayRef< int >  Mask)
static

Helper function that returns true if the shuffle mask should be commuted to improve canonicalization.

Definition at line 17785 of file X86ISelLowering.cpp.

References assert(), Size, and llvm::ArrayRef< T >::size().

Referenced by combineX86ShufflesRecursively(), and lowerVECTOR_SHUFFLE().

◆ canReduceVMulWidth()

static bool canReduceVMulWidth ( SDNode N,
SelectionDAG DAG,
ShrinkMode Mode 
)
static

◆ canWidenShuffleElements() [1/3]

static bool canWidenShuffleElements ( ArrayRef< int >  Mask,
SmallVectorImpl< int > &  WidenedMask 
)
static

Helper function to test whether a shuffle mask could be simplified by widening the elements being shuffled.

Appends the mask for wider elements in WidenedMask if valid. Otherwise leaves it in an unspecified state.

NOTE: This must handle normal vector shuffle masks and target vector shuffle masks. The latter have the special property of a '-2' representing a zero-ed lane of a vector.

Definition at line 5521 of file X86ISelLowering.cpp.

References assert(), llvm::SmallVectorImpl< T >::assign(), llvm::M0(), llvm::M1(), Size, llvm::SmallVectorBase< SmallVectorSizeType< T > >::size(), llvm::ArrayRef< T >::size(), llvm::SM_SentinelUndef, and llvm::SM_SentinelZero.

Referenced by canWidenShuffleElements(), combineExtractWithShuffle(), combineTargetShuffle(), combineX86ShufflesRecursively(), LowerShift(), lowerShuffleAsBlendAndPermute(), lowerV2X128Shuffle(), lowerV4X128Shuffle(), lowerVECTOR_SHUFFLE(), and scaleShuffleElements().

◆ canWidenShuffleElements() [2/3]

static bool canWidenShuffleElements ( ArrayRef< int >  Mask,
const APInt Zeroable,
bool  V2IsZero,
SmallVectorImpl< int > &  WidenedMask 
)
static

◆ canWidenShuffleElements() [3/3]

static bool canWidenShuffleElements ( ArrayRef< int >  Mask)
static

Definition at line 5587 of file X86ISelLowering.cpp.

References canWidenShuffleElements().

◆ checkAndUpdateEFLAGSKill()

static bool checkAndUpdateEFLAGSKill ( MachineBasicBlock::iterator  SelectItr,
MachineBasicBlock BB,
const TargetRegisterInfo TRI 
)
static

◆ checkBitcastSrcVectorSize()

static bool checkBitcastSrcVectorSize ( SDValue  Src,
unsigned  Size 
)
static

◆ checkBoolTestAndOrSetCCCombine()

static bool checkBoolTestAndOrSetCCCombine ( SDValue  Cond,
X86::CondCode CC0,
X86::CondCode CC1,
SDValue Flags,
bool isAnd 
)
static

Check whether Cond is an AND/OR of SETCCs off of the same EFLAGS.

Match: (X86or (X86setcc) (X86setcc)) (X86cmp (and (X86setcc) (X86setcc)), 0)

Definition at line 40603 of file X86ISelLowering.cpp.

References llvm::X86ISD::AND, llvm::ISD::AND, llvm::X86ISD::CMP, llvm::SDNode::getConstantOperandVal(), llvm::SDValue::getOpcode(), llvm::SDNode::getOpcode(), llvm::SDNode::getOperand(), llvm::isNullConstant(), LLVM_FALLTHROUGH, llvm::X86ISD::OR, llvm::ISD::OR, and llvm::X86ISD::SETCC.

Referenced by combineCMov().

◆ checkBoolTestSetCCCombine()

static SDValue checkBoolTestSetCCCombine ( SDValue  Cmp,
X86::CondCode CC 
)
static

◆ clobbersFlagRegisters()

static bool clobbersFlagRegisters ( const SmallVector< StringRef, 4 > &  AsmPieces)
static

◆ collectConcatOps()

static bool collectConcatOps ( SDNode N,
SmallVectorImpl< SDValue > &  Ops 
)
static

◆ combineADC()

static SDValue combineADC ( SDNode N,
SelectionDAG DAG,
TargetLowering::DAGCombinerInfo DCI 
)
static

◆ combineAdd()

static SDValue combineAdd ( SDNode N,
SelectionDAG DAG,
TargetLowering::DAGCombinerInfo DCI,
const X86Subtarget Subtarget 
)
static

◆ combineAddOrSubToADCOrSBB()

static SDValue combineAddOrSubToADCOrSBB ( SDNode N,
SelectionDAG DAG 
)
static

◆ combineAnd()

static SDValue combineAnd ( SDNode N,
SelectionDAG DAG,
TargetLowering::DAGCombinerInfo DCI,
const X86Subtarget Subtarget 
)
static

Definition at line 42841 of file X86ISelLowering.cpp.

References llvm::all_of(), llvm::ISD::AND, assert(), combineAndLoadToBZHI(), combineAndMaskToShift(), combineANDXORWithAllOnesIntoANDNP(), combineBitcastvxi1(), combineBitOpWithMOVMSK(), combineCompareEqual(), combineParity(), combineScalarAndWithMaskSetcc(), combineX86ShufflesRecursively(), convertIntLogicToFPLogic(), llvm::ISD::EXTRACT_VECTOR_ELT, llvm::X86ISD::FAND, llvm::SelectionDAG::getBitcast(), getBitWidth(), llvm::SelectionDAG::getConstant(), llvm::SDValue::getConstantOperandVal(), llvm::SelectionDAG::getContext(), llvm::APInt::getHighBitsSet(), llvm::EVT::getIntegerVT(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDNode::getOperand(), llvm::EVT::getScalarSizeInBits(), llvm::EVT::getScalarType(), llvm::SelectionDAG::getSetCC(), getTargetConstantBitsFromNode(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::SDValue::getValueType(), llvm::SDNode::getValueType(), llvm::EVT::getVectorNumElements(), llvm::X86Subtarget::hasSSE1(), llvm::X86Subtarget::hasSSE2(), llvm::MVT::i1, llvm::MVT::i32, llvm::MVT::i64, llvm::X86Subtarget::is64Bit(), llvm::APInt::isAllOnesValue(), llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalizeOps(), llvm::APInt::isNullValue(), llvm::SDNode::isOnlyUserOf(), llvm::TargetLoweringBase::isTypeLegal(), llvm::EVT::isVector(), llvm::SelectionDAG::MaskedValueIsZero(), matchScalarReduction(), llvm::ISD::SETEQ, llvm::SmallVectorBase< SmallVectorSizeType< T > >::size(), llvm::SM_SentinelUndef, llvm::SM_SentinelZero, llvm::ISD::TRUNCATE, llvm::MVT::v4f32, llvm::MVT::v4i32, and llvm::ISD::ZERO_EXTEND.

Referenced by llvm::X86TargetLowering::PerformDAGCombine().

◆ combineAndLoadToBZHI()

static SDValue combineAndLoadToBZHI ( SDNode Node,
SelectionDAG DAG,
const X86Subtarget Subtarget 
)
static

Definition at line 42631 of file X86ISelLowering.cpp.

Referenced by combineAnd().

◆ combineAndMaskToShift()

static SDValue combineAndMaskToShift ( SDNode N,
SelectionDAG DAG,
const X86Subtarget Subtarget 
)
static

◆ combineAndnp()

static SDValue combineAndnp ( SDNode N,
SelectionDAG DAG,
TargetLowering::DAGCombinerInfo DCI,
const X86Subtarget Subtarget 
)
static

◆ combineANDXORWithAllOnesIntoANDNP()

static SDValue combineANDXORWithAllOnesIntoANDNP ( SDNode N,
SelectionDAG DAG 
)
static

◆ combineBasicSADPattern()

static SDValue combineBasicSADPattern ( SDNode Extract,
SelectionDAG DAG,
const X86Subtarget Subtarget 
)
static

◆ combineBEXTR()

static SDValue combineBEXTR ( SDNode N,
SelectionDAG DAG,
TargetLowering::DAGCombinerInfo DCI,
const X86Subtarget Subtarget 
)
static

◆ combineBitcast()

static SDValue combineBitcast ( SDNode N,
SelectionDAG DAG,
TargetLowering::DAGCombinerInfo DCI,
const X86Subtarget Subtarget 
)
static

Definition at line 38333 of file X86ISelLowering.cpp.

References llvm::ISD::AND, llvm::ISD::ANY_EXTEND, llvm::ISD::BITCAST, llvm::ISD::BUILD_VECTOR, C, llvm::EVT::changeVectorElementTypeToInteger(), combineBitcastToBoolVector(), combineBitcastvxi1(), combineCastedMaskArithmetic(), combinevXi1ConstantToInteger(), llvm::ISD::CONCAT_VECTORS, llvm::countLeadingZeros(), createMMXBuildVector(), DL, llvm::numbers::e, llvm::ISD::EXTRACT_SUBVECTOR, llvm::ISD::EXTRACT_VECTOR_ELT, llvm::MVT::f32, llvm::MVT::f64, llvm::X86ISD::FAND, llvm::X86ISD::FOR, llvm::ISD::FP_TO_SINT, llvm::X86ISD::FXOR, llvm::SelectionDAG::getAnyExtOrTrunc(), llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getConstantFP(), llvm::MVT::getFloatingPointVT(), llvm::MVT::getIntegerVT(), llvm::SelectionDAG::getIntPtrConstant(), llvm::SelectionDAG::getMemIntrinsicNode(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDValue::getNumOperands(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDNode::getOperand(), llvm::EVT::getScalarSizeInBits(), llvm::EVT::getScalarType(), llvm::SelectionDAG::getSetCC(), llvm::SDValue::getSimpleValueType(), getTargetConstantBitsFromNode(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::SelectionDAG::getUNDEF(), llvm::SDValue::getValue(), llvm::SDValue::getValueType(), llvm::SDNode::getValueType(), llvm::EVT::getVectorElementType(), llvm::MVT::getVectorElementType(), llvm::EVT::getVectorNumElements(), llvm::MVT::getVectorNumElements(), llvm::MVT::getVectorVT(), llvm::SelectionDAG::getVTList(), llvm::SelectionDAG::getZExtOrTrunc(), llvm::X86Subtarget::hasAVX512(), llvm::X86Subtarget::hasBWI(), llvm::X86Subtarget::hasDQI(), llvm::SDValue::hasOneUse(), llvm::X86Subtarget::hasSSE1(), llvm::X86Subtarget::hasSSE2(), llvm::MVT::i1, llvm::MVT::i16, llvm::MVT::i32, llvm::MVT::i8, llvm::APFloatBase::IEEEdouble(), llvm::EVT::is128BitVector(), llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalize(), llvm::ISD::isBuildVectorAllZeros(), llvm::ISD::isBuildVectorOfConstantSDNodes(), llvm::EVT::isFloatingPoint(), llvm::isNullConstant(), llvm::isPowerOf2_32(), llvm::EVT::isScalarInteger(), llvm::TargetLoweringBase::isTypeLegal(), llvm::SDValue::isUndef(), llvm::EVT::isVector(), llvm::X86ISD::MMX_MOVW2D, llvm::X86ISD::MOVDQ2Q, llvm::X86ISD::MOVMSK, llvm::SDNode::op_begin(), llvm::SDNode::op_end(), llvm::ISD::OR, llvm::MVT::Other, llvm::SelectionDAG::ReplaceAllUsesOfValueWith(), llvm::SmallVectorImpl< T >::resize(), llvm::ISD::SETLT, llvm::ISD::TRUNCATE, llvm::MVT::v16i1, llvm::MVT::v2f32, llvm::MVT::v2i1, llvm::MVT::v2i32, llvm::MVT::v2i64, llvm::MVT::v4i1, llvm::MVT::v4i16, llvm::MVT::v4i32, llvm::MVT::v8i1, llvm::MVT::v8i8, llvm::X86ISD::VBROADCAST_LOAD, llvm::MVT::x86mmx, and llvm::ISD::XOR.

Referenced by llvm::X86TargetLowering::PerformDAGCombine().

◆ combineBitcastToBoolVector()

static SDValue combineBitcastToBoolVector ( EVT  VT,
SDValue  V,
SDLoc  DL,
SelectionDAG DAG,
const X86Subtarget Subtarget 
)
static

◆ combineBitcastvxi1()

static SDValue combineBitcastvxi1 ( SelectionDAG DAG,
EVT  VT,
SDValue  Src,
const SDLoc DL,
const X86Subtarget Subtarget 
)
static

◆ combineBitOpWithMOVMSK()

static SDValue combineBitOpWithMOVMSK ( SDNode N,
SelectionDAG DAG 
)
static

◆ combineBrCond()

static SDValue combineBrCond ( SDNode N,
SelectionDAG DAG,
const X86Subtarget Subtarget 
)
static

◆ combineBT()

static SDValue combineBT ( SDNode N,
SelectionDAG DAG,
TargetLowering::DAGCombinerInfo DCI 
)
static

◆ combineCarryThroughADD()

static SDValue combineCarryThroughADD ( SDValue  EFLAGS,
SelectionDAG DAG 
)
static

◆ combineCastedMaskArithmetic()

static SDValue combineCastedMaskArithmetic ( SDNode N,
SelectionDAG DAG,
TargetLowering::DAGCombinerInfo DCI,
const X86Subtarget Subtarget 
)
static

◆ combineCMov()

static SDValue combineCMov ( SDNode N,
SelectionDAG DAG,
TargetLowering::DAGCombinerInfo DCI,
const X86Subtarget Subtarget 
)
static

◆ combineCMP()

static SDValue combineCMP ( SDNode N,
SelectionDAG DAG 
)
static

◆ combineCommutableSHUFP()

static SDValue combineCommutableSHUFP ( SDValue  N,
MVT  VT,
const SDLoc DL,
SelectionDAG DAG 
)
static

◆ combineCompareEqual()

static SDValue combineCompareEqual ( SDNode N,
SelectionDAG DAG,
TargetLowering::DAGCombinerInfo DCI,
const X86Subtarget Subtarget 
)
static

◆ combineConcatVectorOps()

static SDValue combineConcatVectorOps ( const SDLoc DL,
MVT  VT,
ArrayRef< SDValue Ops,
SelectionDAG DAG,
TargetLowering::DAGCombinerInfo DCI,
const X86Subtarget Subtarget 
)
static

Helper that combines an array of subvector ops as if they were the operands of a ISD::CONCAT_VECTORS node, but may have come from another source (e.g.

ISD::INSERT_SUBVECTOR). The ops are assumed to be of the same type.

Definition at line 47841 of file X86ISelLowering.cpp.

References llvm::all_of(), llvm::TargetLoweringBase::allowsMemoryAccess(), assert(), llvm::ISD::CONCAT_VECTORS, DL, EltsFromConsecutiveLoads(), llvm::ISD::EXTRACT_SUBVECTOR, llvm::ISD::EXTRACT_VECTOR_ELT, extractSubVector(), llvm::MVT::f32, llvm::MVT::f64, llvm::CallingConv::Fast, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getContext(), llvm::SelectionDAG::getDataLayout(), llvm::SelectionDAG::getIntPtrConstant(), llvm::SelectionDAG::getMemIntrinsicNode(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::MVT::getScalarSizeInBits(), llvm::EVT::getScalarType(), llvm::MVT::getScalarType(), llvm::SDValue::getSimpleValueType(), llvm::X86Subtarget::getTargetLowering(), llvm::SelectionDAG::getUNDEF(), llvm::SDValue::getValue(), llvm::SDValue::getValueSizeInBits(), llvm::SDValue::getValueType(), llvm::MVT::getVectorNumElements(), llvm::MVT::getVectorVT(), llvm::SelectionDAG::getVTList(), getZeroVector(), llvm::X86Subtarget::hasAVX(), llvm::X86Subtarget::hasAVX2(), llvm::X86Subtarget::hasInt256(), llvm::MVT::is256BitVector(), llvm::MVT::is512BitVector(), llvm::ISD::isBuildVectorAllZeros(), llvm::SDValue::isUndef(), LLVM_FALLTHROUGH, MayFoldLoad(), llvm::X86ISD::MOVDDUP, llvm::MVT::Other, llvm::X86ISD::PACKSS, llvm::X86ISD::PACKUS, llvm::X86ISD::PALIGNR, llvm::peekThroughBitcasts(), llvm::X86ISD::PSHUFD, llvm::X86ISD::PSHUFHW, llvm::X86ISD::PSHUFLW, llvm::SmallVectorTemplateBase< T >::push_back(), llvm::SelectionDAG::ReplaceAllUsesOfValueWith(), llvm::ISD::SCALAR_TO_VECTOR, llvm::X86ISD::SHUFP, llvm::ArrayRef< T >::size(), llvm::X86ISD::SUBV_BROADCAST, llvm::X86Subtarget::useAVX512Regs(), llvm::X86Subtarget::useBWIRegs(), llvm::MVT::v4f32, llvm::MVT::v4f64, llvm::MVT::v8f32, llvm::MVT::v8i32, llvm::X86ISD::VBROADCAST, llvm::X86ISD::VBROADCAST_LOAD, llvm::X86ISD::VPERMI, llvm::X86ISD::VPERMILPI, llvm::X86ISD::VROTLI, llvm::X86ISD::VROTRI, llvm::X86ISD::VSHLI, llvm::X86ISD::VSRAI, and llvm::X86ISD::VSRLI.

Referenced by combineConcatVectors(), and combineInsertSubvector().

◆ combineConcatVectors()

static SDValue combineConcatVectors ( SDNode N,
SelectionDAG DAG,
TargetLowering::DAGCombinerInfo DCI,
const X86Subtarget Subtarget 
)
static

◆ combineCVTP2I_CVTTP2I()

static SDValue combineCVTP2I_CVTTP2I ( SDNode N,
SelectionDAG DAG,
TargetLowering::DAGCombinerInfo DCI 
)
static

◆ combineCVTPH2PS()

static SDValue combineCVTPH2PS ( SDNode N,
SelectionDAG DAG,
TargetLowering::DAGCombinerInfo DCI 
)
static

◆ combineExtInVec()

static SDValue combineExtInVec ( SDNode N,
SelectionDAG DAG,
TargetLowering::DAGCombinerInfo DCI,
const X86Subtarget Subtarget 
)
static

◆ combineExtractSubvector()

static SDValue combineExtractSubvector ( SDNode N,
SelectionDAG DAG,
TargetLowering::DAGCombinerInfo DCI,
const X86Subtarget Subtarget 
)
static

Definition at line 48246 of file X86ISelLowering.cpp.

References llvm::ISD::AND, llvm::ISD::ANY_EXTEND, llvm::ISD::ANY_EXTEND_VECTOR_INREG, llvm::ISD::BUILD_VECTOR, Concat, llvm::ISD::CONCAT_VECTORS, llvm::X86ISD::CVTSI2P, llvm::X86ISD::CVTUI2P, DL, llvm::ISD::EXTRACT_SUBVECTOR, extractSubVector(), llvm::ISD::FP_EXTEND, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getBuildVector(), llvm::SelectionDAG::getConstant(), llvm::SDNode::getConstantOperandVal(), llvm::SelectionDAG::getMemIntrinsicNode(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), getOnesVector(), llvm::SDValue::getOpcode(), getOpcode_EXTEND_VECTOR_INREG(), llvm::SDValue::getOperand(), llvm::SDNode::getOperand(), llvm::MVT::getScalarType(), llvm::SDValue::getSimpleValueType(), llvm::SDNode::getSimpleValueType(), llvm::EVT::getSizeInBits(), llvm::MVT::getSizeInBits(), llvm::SelectionDAG::getTargetLoweringInfo(), getTargetShuffleInputs(), llvm::SelectionDAG::getUNDEF(), llvm::SDValue::getValue(), llvm::SDValue::getValueSizeInBits(), llvm::SDValue::getValueType(), llvm::SDNode::getValueType(), llvm::MVT::getVectorElementType(), llvm::MVT::getVectorNumElements(), llvm::SelectionDAG::getVTList(), getZeroVector(), llvm::X86Subtarget::hasAVX(), llvm::X86Subtarget::hasAVX2(), llvm::SDValue::hasOneUse(), llvm::X86Subtarget::hasVLX(), llvm::MVT::i1, llvm::ISD::INSERT_SUBVECTOR, llvm::MVT::is128BitVector(), llvm::EVT::is256BitVector(), llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalizeOps(), llvm::isBitwiseNot(), llvm::ISD::isBuildVectorAllOnes(), llvm::ISD::isBuildVectorAllZeros(), llvm::isNullConstant(), llvm::EVT::isSimple(), llvm::TargetLoweringBase::isTypeLegal(), narrowExtractedVectorSelect(), llvm::SDNode::ops(), llvm::MVT::Other, llvm::peekThroughBitcasts(), llvm::SelectionDAG::ReplaceAllUsesOfValueWith(), scaleShuffleElements(), llvm::ISD::SIGN_EXTEND, llvm::ISD::SIGN_EXTEND_VECTOR_INREG, llvm::ISD::SINT_TO_FP, llvm::SM_SentinelUndef, llvm::SM_SentinelZero, splitVectorIntBinary(), llvm::X86ISD::SUBV_BROADCAST, llvm::ISD::UINT_TO_FP, llvm::MVT::v2f64, llvm::MVT::v4f32, llvm::MVT::v4f64, llvm::MVT::v4i32, llvm::X86ISD::VBROADCAST, llvm::X86ISD::VBROADCAST_LOAD, llvm::X86ISD::VFPEXT, llvm::ISD::VSELECT, llvm::ISD::ZERO_EXTEND, and llvm::ISD::ZERO_EXTEND_VECTOR_INREG.

Referenced by llvm::X86TargetLowering::PerformDAGCombine().

◆ combineExtractVectorElt()

static SDValue combineExtractVectorElt ( SDNode N,
SelectionDAG DAG,
TargetLowering::DAGCombinerInfo DCI,
const X86Subtarget Subtarget 
)
static

Detect vector gather/scatter index generation and convert it from being a bunch of shuffles and extracts into a somewhat faster sequence.

For i686, the best sequence is apparently storing the value and loading scalars back, while for x64 we should use 64-bit extracts and shifts.

Definition at line 39341 of file X86ISelLowering.cpp.

References llvm::all_of(), llvm::ISD::AND, assert(), llvm::ISD::BITCAST, combineBasicSADPattern(), combineBitcastvxi1(), combineExtractWithShuffle(), combineHorizontalMinMaxResult(), combineHorizontalPredicateResult(), combineReductionToHorizontal(), llvm::TargetLowering::DAGCombinerInfo::CombineTo(), llvm::dyn_cast(), llvm::ISD::EXTRACT_VECTOR_ELT, llvm::APInt::getAllOnesValue(), llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getContext(), llvm::EVT::getIntegerVT(), llvm::SelectionDAG::getNode(), llvm::APInt::getOneBitSet(), llvm::SDValue::getOpcode(), llvm::SDNode::getOpcode(), llvm::SDValue::getOperand(), llvm::SDNode::getOperand(), llvm::SDValue::getResNo(), llvm::EVT::getScalarSizeInBits(), llvm::SelectionDAG::getSetCC(), llvm::EVT::getSizeInBits(), getTargetConstantBitsFromNode(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::SelectionDAG::getUNDEF(), llvm::SDValue::getValueType(), llvm::SDNode::getValueType(), llvm::SelectionDAG::getZExtOrTrunc(), llvm::SDValue::hasOneUse(), llvm::MVT::i1, llvm::MVT::i32, llvm::MVT::i64, llvm::EVT::isInteger(), llvm::isNullConstant(), llvm::X86ISD::MMX_MOVD2W, llvm::X86ISD::PINSRB, llvm::X86ISD::PINSRW, llvm::SmallVectorTemplateBase< T >::push_back(), scalarizeExtEltFP(), llvm::ISD::SETEQ, llvm::TargetLowering::SimplifyDemandedBits(), llvm::SmallVectorBase< SmallVectorSizeType< T > >::size(), llvm::ISD::TRUNCATE, llvm::SDNode::uses(), llvm::MVT::v1i64, llvm::MVT::v2i32, and llvm::MVT::x86mmx.

Referenced by llvm::X86TargetLowering::PerformDAGCombine().

◆ combineExtractWithShuffle()

static SDValue combineExtractWithShuffle ( SDNode N,
SelectionDAG DAG,
TargetLowering::DAGCombinerInfo DCI,
const X86Subtarget Subtarget 
)
static

Definition at line 38960 of file X86ISelLowering.cpp.

References llvm::any_of(), assert(), canWidenShuffleElements(), llvm::numbers::e, extract128BitVector(), llvm::ISD::EXTRACT_VECTOR_ELT, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getConstantFP(), llvm::SDNode::getConstantOperandAPInt(), llvm::SDNode::getConstantOperandVal(), llvm::SelectionDAG::getIntPtrConstant(), llvm::SelectionDAG::getLoad(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDNode::getOpcode(), llvm::SDValue::getOperand(), llvm::SDNode::getOperand(), llvm::EVT::getScalarSizeInBits(), llvm::EVT::getScalarType(), llvm::SDValue::getScalarValueSizeInBits(), llvm::SelectionDAG::getShiftAmountConstant(), llvm::EVT::getSizeInBits(), getTargetShuffleInputs(), llvm::SelectionDAG::getUNDEF(), llvm::SDValue::getValue(), llvm::SDValue::getValueSizeInBits(), llvm::SDValue::getValueType(), llvm::SDNode::getValueType(), llvm::EVT::getVectorElementType(), llvm::EVT::getVectorNumElements(), llvm::SelectionDAG::getZExtOrTrunc(), llvm::APInt::getZExtValue(), llvm::SDValue::hasOneUse(), llvm::X86Subtarget::hasSSE2(), llvm::X86Subtarget::hasSSE41(), llvm::MipsISD::Hi, llvm::MVT::i1, llvm::MVT::i32, llvm::EVT::is128BitVector(), llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalizeOps(), llvm::EVT::isFloatingPoint(), llvm::EVT::isInteger(), llvm::EVT::isScalarInteger(), llvm::MipsISD::Lo, llvm::SPII::Load, llvm::BitmaskEnumDetail::Mask(), llvm::narrowShuffleMaskElts(), llvm::peekThroughBitcasts(), llvm::X86ISD::PEXTRB, llvm::X86ISD::PEXTRW, llvm::SelectionDAG::ReplaceAllUsesOfValueWith(), llvm::ISD::SCALAR_TO_VECTOR, llvm::SmallVectorBase< SmallVectorSizeType< T > >::size(), llvm::SM_SentinelUndef, llvm::SM_SentinelZero, llvm::ISD::SRL, llvm::ISD::TRUNCATE, llvm::APInt::uge(), llvm::APInt::ult(), llvm::APInt::urem(), llvm::MVT::v16i8, llvm::MVT::v2i64, llvm::MVT::v4i32, llvm::MVT::v8i16, llvm::X86ISD::VBROADCAST, and llvm::X86ISD::VBROADCAST_LOAD.

Referenced by combineExtractVectorElt().

◆ combineExtSetcc()

static SDValue combineExtSetcc ( SDNode N,
SelectionDAG DAG,
const X86Subtarget Subtarget 
)
static

◆ combineFaddFsub()

static SDValue combineFaddFsub ( SDNode N,
SelectionDAG DAG,
const X86Subtarget Subtarget 
)
static

◆ combineFAnd()

static SDValue combineFAnd ( SDNode N,
SelectionDAG DAG,
const X86Subtarget Subtarget 
)
static

Do target-specific dag combines on X86ISD::FAND nodes.

Definition at line 45395 of file X86ISelLowering.cpp.

References combineFAndFNotToFAndn(), getNullFPConstForNullVal(), llvm::SDNode::getOperand(), and lowerX86FPLogicOp().

Referenced by llvm::X86TargetLowering::PerformDAGCombine().

◆ combineFAndFNotToFAndn()

static SDValue combineFAndFNotToFAndn ( SDNode N,
SelectionDAG DAG,
const X86Subtarget Subtarget 
)
static

◆ combineFAndn()

static SDValue combineFAndn ( SDNode N,
SelectionDAG DAG,
const X86Subtarget Subtarget 
)
static

Do target-specific dag combines on X86ISD::FANDN nodes.

Definition at line 45412 of file X86ISelLowering.cpp.

References getNullFPConstForNullVal(), llvm::SDNode::getOperand(), isNullFPScalarOrVectorConst(), and lowerX86FPLogicOp().

Referenced by llvm::X86TargetLowering::PerformDAGCombine().

◆ combineFMA()

static SDValue combineFMA ( SDNode N,
SelectionDAG DAG,
TargetLowering::DAGCombinerInfo DCI,
const X86Subtarget Subtarget 
)
static

◆ combineFMADDSUB()

static SDValue combineFMADDSUB ( SDNode N,
SelectionDAG DAG,
TargetLowering::DAGCombinerInfo DCI 
)
static

◆ combineFMinFMax()

static SDValue combineFMinFMax ( SDNode N,
SelectionDAG DAG 
)
static

◆ combineFMinNumFMaxNum()

static SDValue combineFMinNumFMaxNum ( SDNode N,
SelectionDAG DAG,
const X86Subtarget Subtarget 
)
static

◆ combineFneg()

static SDValue combineFneg ( SDNode N,
SelectionDAG DAG,
TargetLowering::DAGCombinerInfo DCI,
const X86Subtarget Subtarget 
)
static

◆ combineFOr()

static SDValue combineFOr ( SDNode N,
SelectionDAG DAG,
TargetLowering::DAGCombinerInfo DCI,
const X86Subtarget Subtarget 
)
static

Do target-specific dag combines on X86ISD::FOR and X86ISD::FXOR nodes.

Definition at line 45426 of file X86ISelLowering.cpp.

References assert(), combineFneg(), llvm::X86ISD::FOR, llvm::X86ISD::FXOR, llvm::SDNode::getOpcode(), llvm::SDNode::getOperand(), isNullFPScalarOrVectorConst(), and lowerX86FPLogicOp().

Referenced by llvm::X86TargetLowering::PerformDAGCombine().

◆ combineFP16_TO_FP()

static SDValue combineFP16_TO_FP ( SDNode N,
SelectionDAG DAG,
const X86Subtarget Subtarget 
)
static

◆ combineFP_EXTEND()

static SDValue combineFP_EXTEND ( SDNode N,
SelectionDAG DAG,
const X86Subtarget Subtarget 
)
static

◆ combineFP_ROUND()

static SDValue combineFP_ROUND ( SDNode N,
SelectionDAG DAG,
const X86Subtarget Subtarget 
)
static

◆ combineGatherScatter()

static SDValue combineGatherScatter ( SDNode N,
SelectionDAG DAG,
TargetLowering::DAGCombinerInfo DCI 
)
static

◆ combineHorizontalMinMaxResult()

static SDValue combineHorizontalMinMaxResult ( SDNode Extract,
SelectionDAG DAG,
const X86Subtarget Subtarget 
)
static

◆ combineHorizontalPredicateResult()

static SDValue combineHorizontalPredicateResult ( SDNode Extract,
SelectionDAG DAG,
const X86Subtarget Subtarget 
)
static

Definition at line 38738 of file X86ISelLowering.cpp.

References llvm::ISD::AND, assert(), llvm::BitWidth, combineBitcastvxi1(), llvm::SelectionDAG::ComputeNumSignBits(), llvm::ISD::CTPOP, DL, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getContext(), llvm::SelectionDAG::getDataLayout(), llvm::MVT::getFloatingPointVT(), llvm::EVT::getIntegerVT(), llvm::APInt::getLowBitsSet(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), getPMOVMSKB(), llvm::EVT::getScalarType(), llvm::SDValue::getScalarValueSizeInBits(), llvm::SelectionDAG::getSetCC(), llvm::TargetLoweringBase::getSetCCResultType(), llvm::EVT::getSizeInBits(), llvm::MVT::getSizeInBits(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::SDValue::getValueSizeInBits(), llvm::SDValue::getValueType(), llvm::SDNode::getValueType(), llvm::EVT::getVectorNumElements(), llvm::MVT::getVectorNumElements(), llvm::EVT::getVectorVT(), llvm::MVT::getVectorVT(), llvm::SelectionDAG::getZExtOrTrunc(), llvm::X86Subtarget::hasAVX(), llvm::X86Subtarget::hasInt256(), llvm::X86Subtarget::hasSSE2(), llvm::X86Subtarget::hasSSE41(), llvm::MipsISD::Hi, llvm::MVT::i1, llvm::MVT::i16, llvm::MVT::i32, llvm::MVT::i64, llvm::MVT::i8, llvm::ISD::isBuildVectorAllZeros(), llvm::isPowerOf2_32(), llvm::TargetLoweringBase::isTypeLegal(), llvm::MipsISD::Lo, llvm::Match, llvm::SelectionDAG::matchBinOpReduction(), llvm::ISD::OR, llvm::ISD::SETCC, llvm::ISD::SETEQ, llvm::ISD::SETNE, llvm::SelectionDAG::SplitVector(), llvm::ISD::SUB, and llvm::ISD::XOR.

Referenced by combineExtractVectorElt().

◆ combineInsertSubvector()

static SDValue combineInsertSubvector ( SDNode N,
SelectionDAG DAG,
TargetLowering::DAGCombinerInfo DCI,
const X86Subtarget Subtarget 
)
static

◆ combineKSHIFT()

static SDValue combineKSHIFT ( SDNode N,
SelectionDAG DAG,
TargetLowering::DAGCombinerInfo DCI 
)
static

◆ combineLoad()

static SDValue combineLoad ( SDNode N,
SelectionDAG DAG,
TargetLowering::DAGCombinerInfo DCI,
const X86Subtarget Subtarget 
)
static

Definition at line 43745 of file X86ISelLowering.cpp.

References llvm::TargetLoweringBase::allowsMemoryAccess(), llvm::TargetLowering::DAGCombinerInfo::CombineTo(), llvm::ISD::CONCAT_VECTORS, llvm::MipsISD::Ext, llvm::CallingConv::Fast, llvm::MemSDNode::getAddressSpace(), llvm::SelectionDAG::getAddrSpaceCast(), llvm::MemSDNode::getAlignment(), llvm::LoadSDNode::getBasePtr(), llvm::SelectionDAG::getBitcast(), llvm::MemSDNode::getChain(), llvm::SelectionDAG::getContext(), llvm::SelectionDAG::getDataLayout(), llvm::LoadSDNode::getExtensionType(), llvm::MachineMemOperand::getFlags(), llvm::EVT::getIntegerVT(), llvm::SelectionDAG::getLoad(), llvm::SelectionDAG::getMemBasePlusOffset(), llvm::MemSDNode::getMemOperand(), llvm::MemSDNode::getMemoryVT(), llvm::SelectionDAG::getNode(), llvm::MemSDNode::getOriginalAlign(), llvm::MemSDNode::getPointerInfo(), llvm::TargetLoweringBase::getPointerTy(), llvm::EVT::getScalarType(), llvm::SDValue::getSimpleValueType(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::SDValue::getValue(), llvm::SDNode::getValueType(), llvm::EVT::getVectorVT(), llvm::MachinePointerInfo::getWithOffset(), llvm::X86Subtarget::hasAVX512(), llvm::X86Subtarget::hasInt256(), llvm::MVT::i1, llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalize(), llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalizeOps(), llvm::MemSDNode::isNonTemporal(), llvm::TargetLoweringBase::isTypeLegal(), N, llvm::ISD::NON_EXTLOAD, llvm::MVT::Other, llvm::X86AS::PTR32_SPTR, llvm::X86AS::PTR32_UPTR, llvm::X86AS::PTR64, and llvm::ISD::TokenFactor.

Referenced by llvm::X86TargetLowering::PerformDAGCombine().

◆ combineLogicBlendIntoConditionalNegate()

static SDValue combineLogicBlendIntoConditionalNegate ( EVT  VT,
SDValue  Mask,
SDValue  X,
SDValue  Y,
const SDLoc DL,
SelectionDAG DAG,
const X86Subtarget Subtarget 
)
static

◆ combineLogicBlendIntoPBLENDV()

static SDValue combineLogicBlendIntoPBLENDV ( SDNode N,
SelectionDAG DAG,
const X86Subtarget Subtarget 
)
static

◆ combineMaskedLoad()

static SDValue combineMaskedLoad ( SDNode N,
SelectionDAG DAG,
TargetLowering::DAGCombinerInfo DCI,
const X86Subtarget Subtarget 
)
static

◆ combineMaskedLoadConstantMask()

static SDValue combineMaskedLoadConstantMask ( MaskedLoadSDNode ML,
SelectionDAG DAG,
TargetLowering::DAGCombinerInfo DCI 
)
static

◆ combineMaskedStore()

static SDValue combineMaskedStore ( SDNode N,
SelectionDAG DAG,
TargetLowering::DAGCombinerInfo DCI,
const X86Subtarget Subtarget 
)
static

◆ combineMOVDQ2Q()

static SDValue combineMOVDQ2Q ( SDNode N,
SelectionDAG DAG 
)
static

◆ combineMOVMSK()

static SDValue combineMOVMSK ( SDNode N,
SelectionDAG DAG,
TargetLowering::DAGCombinerInfo DCI,
const X86Subtarget Subtarget 
)
static

◆ combineMul()

static SDValue combineMul ( SDNode N,
SelectionDAG DAG,
TargetLowering::DAGCombinerInfo DCI,
const X86Subtarget Subtarget 
)
static

◆ combineMulSpecial()

static SDValue combineMulSpecial ( uint64_t  MulAmt,
SDNode N,
SelectionDAG DAG,
EVT  VT,
const SDLoc DL 
)
static

◆ combineMulToPMADDWD()

static SDValue combineMulToPMADDWD ( SDNode N,
SelectionDAG DAG,
const X86Subtarget Subtarget 
)
static

◆ combineMulToPMULDQ()

static SDValue combineMulToPMULDQ ( SDNode N,
SelectionDAG DAG,
const X86Subtarget Subtarget 
)
static

◆ combineOr()

static SDValue combineOr ( SDNode N,
SelectionDAG DAG,
TargetLowering::DAGCombinerInfo DCI,
const X86Subtarget Subtarget 
)
static

◆ combineOrCmpEqZeroToCtlzSrl()

static SDValue combineOrCmpEqZeroToCtlzSrl ( SDNode N,
SelectionDAG DAG,
TargetLowering::DAGCombinerInfo DCI,
const X86Subtarget Subtarget 
)
static

◆ combineParity()

static SDValue combineParity ( SDNode N,
SelectionDAG DAG,
const X86Subtarget Subtarget 
)
static

◆ combinePMULDQ()

static SDValue combinePMULDQ ( SDNode N,
SelectionDAG DAG,
TargetLowering::DAGCombinerInfo DCI,
const X86Subtarget Subtarget 
)
static

◆ combinePMULH()

static SDValue combinePMULH ( SDValue  Src,
EVT  VT,
const SDLoc DL,
SelectionDAG DAG,
const X86Subtarget Subtarget 
)
static

◆ combinePTESTCC()

static SDValue combinePTESTCC ( SDValue  EFLAGS,
X86::CondCode CC,
SelectionDAG DAG,
const X86Subtarget Subtarget 
)
static

◆ combineReductionToHorizontal()

static SDValue combineReductionToHorizontal ( SDNode ExtElt,
SelectionDAG DAG,
const X86Subtarget Subtarget 
)
static

◆ combineRedundantDWordShuffle()

static SDValue combineRedundantDWordShuffle ( SDValue  N,
MutableArrayRef< int >  Mask,
SelectionDAG DAG 
)
static

◆ combineSBB()

static SDValue combineSBB ( SDNode N,
SelectionDAG DAG 
)
static

◆ combineScalarAndWithMaskSetcc()

static SDValue combineScalarAndWithMaskSetcc ( SDNode N,
SelectionDAG DAG,
const X86Subtarget Subtarget 
)
static

◆ combineScalarToVector()

static SDValue combineScalarToVector ( SDNode N,
SelectionDAG DAG 
)
static

◆ combineSelect()

static SDValue combineSelect ( SDNode N,
SelectionDAG DAG,
TargetLowering::DAGCombinerInfo DCI,
const X86Subtarget Subtarget 
)
static

Do target-specific dag combines on SELECT and VSELECT nodes.

Definition at line 39839 of file X86ISelLowering.cpp.

References llvm::ISD::ADD, llvm::ISD::AND, llvm::ISD::BITCAST, C, combineLogicBlendIntoConditionalNegate(), combineSelectOfTwoConstants(), combineVSelectToBLENDV(), combineVSelectWithAllOnesOrZeros(), combinevXi1ConstantToInteger(), llvm::SelectionDAG::ComputeNumSignBits(), Cond, createShuffleMaskFromVSELECT(), DL, llvm::numbers::e, llvm::ISD::EXTRACT_SUBVECTOR, extractSubVector(), llvm::MVT::f128, llvm::MVT::f32, llvm::MVT::f64, llvm::MVT::f80, llvm::X86ISD::FMAX, llvm::X86ISD::FMIN, llvm::ConstantSDNode::getAPIntValue(), llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getConstant(), getConstVector(), llvm::SelectionDAG::getContext(), llvm::EVT::getIntegerVT(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getNOT(), llvm::SDNode::getNumOperands(), llvm::SDValue::getOpcode(), llvm::SDNode::getOpcode(), llvm::SDValue::getOperand(), llvm::SDNode::getOperand(), llvm::EVT::getScalarSizeInBits(), llvm::EVT::getScalarType(), llvm::SelectionDAG::getSelect(), llvm::SelectionDAG::getSetCC(), llvm::ISD::getSetCCInverse(), llvm::SelectionDAG::getTarget(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::SelectionDAG::getUNDEF(), llvm::SDValue::getValueType(), llvm::SDNode::getValueType(), llvm::EVT::getVectorElementType(), llvm::EVT::getVectorNumElements(), llvm::SelectionDAG::getVectorShuffle(), llvm::EVT::getVectorVT(), llvm::SelectionDAG::getZExtOrTrunc(), llvm::X86Subtarget::hasAVX(), llvm::X86Subtarget::hasAVX2(), llvm::X86Subtarget::hasAVX512(), llvm::X86Subtarget::hasBWI(), llvm::SDValue::hasOneUse(), llvm::X86Subtarget::hasSSE1(), llvm::X86Subtarget::hasSSE2(), llvm::X86Subtarget::hasXOP(), llvm::MVT::i1, llvm::MVT::i16, llvm::MVT::i8, llvm::ISD::INSERT_SUBVECTOR, insertSubVector(), llvm::X86Subtarget::is64Bit(), llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalize(), llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalizeOps(), llvm::ISD::isBuildVectorAllOnes(), llvm::ISD::isBuildVectorAllZeros(), llvm::ISD::isBuildVectorOfConstantSDNodes(), llvm::isConstOrConstSplat(), llvm::SelectionDAG::isEqualTo(), llvm::EVT::isFloatingPoint(), llvm::EVT::isInteger(), llvm::SelectionDAG::isKnownNeverNaN(), llvm::SelectionDAG::isKnownNeverZeroFloat(), IsNOT(), llvm::isNullConstant(), llvm::isNullOrNullSplat(), llvm::isOneConstant(), llvm::isPowerOf2_32(), isTargetShuffle(), llvm::TargetLoweringBase::isTypeLegal(), llvm::EVT::isVector(), LLVM_FALLTHROUGH, llvm::BitmaskEnumDetail::Mask(), llvm::ISD::matchBinaryPredicate(), llvm::ISD::matchUnaryPredicate(), narrowVectorSelect(), llvm::TargetOptions::NoSignedZerosFPMath, llvm::TargetMachine::Options, llvm::Other, llvm::SmallVectorTemplateBase< T >::push_back(), llvm::MCID::Select, llvm::ISD::SELECT, llvm::ISD::SETCC, llvm::ISD::SETEQ, llvm::ISD::SETGE, llvm::ISD::SETGT, llvm::ISD::SETLE, llvm::ISD::SETLT, llvm::ISD::SETNE, llvm::ISD::SETOGE, llvm::ISD::SETOGT, llvm::ISD::SETOLE, llvm::ISD::SETOLT, llvm::ISD::SETUGE, llvm::ISD::SETUGT, llvm::ISD::SETULE, llvm::ISD::SETULT, llvm::ISD::SHL, llvm::ISD::SIGN_EXTEND, llvm::SelectionDAG::simplifySelect(), llvm::ISD::SUB, std::swap(), llvm::ISD::UADDSAT, llvm::ISD::USUBSAT, llvm::MVT::v2f32, llvm::MVT::v64i1, llvm::ISD::VSELECT, and llvm::ISD::XOR.

Referenced by llvm::X86TargetLowering::PerformDAGCombine().

◆ combineSelectOfTwoConstants()

static SDValue combineSelectOfTwoConstants ( SDNode N,
SelectionDAG DAG 
)
static

◆ combineSetCC()

static SDValue combineSetCC ( SDNode N,
SelectionDAG DAG,
const X86Subtarget Subtarget 
)
static

◆ combineSetCCAtomicArith()

static SDValue combineSetCCAtomicArith ( SDValue  Cmp,
X86::CondCode CC,
SelectionDAG DAG,
const X86Subtarget Subtarget 
)
static

◆ combineSetCCEFLAGS()

static SDValue combineSetCCEFLAGS ( SDValue  EFLAGS,
X86::CondCode CC,
SelectionDAG DAG,
const X86Subtarget Subtarget 
)
static

Optimize an EFLAGS definition used according to the condition code CC into a simpler EFLAGS value, potentially returning a new CC and replacing uses of chain values.

Definition at line 40971 of file X86ISelLowering.cpp.

References checkBoolTestSetCCCombine(), combineCarryThroughADD(), combinePTESTCC(), combineSetCCAtomicArith(), combineSetCCMOVMSK(), and llvm::X86::COND_B.

Referenced by combineBrCond(), combineCMov(), and combineX86SetCC().

◆ combineSetCCMOVMSK()

static SDValue combineSetCCMOVMSK ( SDValue  EFLAGS,
X86::CondCode CC,
SelectionDAG DAG,
const X86Subtarget Subtarget 
)
static

◆ combineSext()

static SDValue combineSext ( SDNode N,
SelectionDAG DAG,
TargetLowering::DAGCombinerInfo DCI,
const X86Subtarget Subtarget 
)
static

◆ combineSextInRegCmov()

static SDValue combineSextInRegCmov ( SDNode N,
SelectionDAG DAG 
)
static

◆ combineShiftLeft()

static SDValue combineShiftLeft ( SDNode N,
SelectionDAG DAG 
)
static

◆ combineShiftRightArithmetic()

static SDValue combineShiftRightArithmetic ( SDNode N,
SelectionDAG DAG,
const X86Subtarget Subtarget 
)
static

◆ combineShiftRightLogical()

static SDValue combineShiftRightLogical ( SDNode N,
SelectionDAG DAG,
TargetLowering::DAGCombinerInfo DCI,
const X86Subtarget Subtarget 
)
static

◆ combineShiftToPMULH()

static SDValue combineShiftToPMULH ( SDNode N,
SelectionDAG DAG,
const X86Subtarget Subtarget 
)
static

◆ combineShuffle()

static SDValue combineShuffle ( SDNode N,
SelectionDAG DAG,
TargetLowering::DAGCombinerInfo DCI,
const X86Subtarget Subtarget 
)
static

◆ combineShuffleOfConcatUndef()

static SDValue combineShuffleOfConcatUndef ( SDNode N,
SelectionDAG DAG,
const X86Subtarget Subtarget 
)
static

◆ combineShuffleToAddSubOrFMAddSub()

static SDValue combineShuffleToAddSubOrFMAddSub ( SDNode N,
const X86Subtarget Subtarget,
SelectionDAG DAG 
)
static

◆ combineShuffleToFMAddSub()

static SDValue combineShuffleToFMAddSub ( SDNode N,
const X86Subtarget Subtarget,
SelectionDAG DAG 
)
static

◆ combineSignExtendInReg()

static SDValue combineSignExtendInReg ( SDNode N,
SelectionDAG DAG,
const X86Subtarget Subtarget 
)
static