#include "AArch64ISelLowering.h"#include "AArch64CallingConvention.h"#include "AArch64ExpandImm.h"#include "AArch64MachineFunctionInfo.h"#include "AArch64PerfectShuffle.h"#include "AArch64RegisterInfo.h"#include "AArch64SMEAttributes.h"#include "AArch64Subtarget.h"#include "AArch64TargetMachine.h"#include "MCTargetDesc/AArch64AddressingModes.h"#include "Utils/AArch64BaseInfo.h"#include "llvm/ADT/APFloat.h"#include "llvm/ADT/APInt.h"#include "llvm/ADT/ArrayRef.h"#include "llvm/ADT/STLExtras.h"#include "llvm/ADT/SmallSet.h"#include "llvm/ADT/SmallVector.h"#include "llvm/ADT/SmallVectorExtras.h"#include "llvm/ADT/Statistic.h"#include "llvm/ADT/StringRef.h"#include "llvm/ADT/StringSwitch.h"#include "llvm/ADT/Twine.h"#include "llvm/Analysis/LoopInfo.h"#include "llvm/Analysis/MemoryLocation.h"#include "llvm/Analysis/ObjCARCUtil.h"#include "llvm/Analysis/OptimizationRemarkEmitter.h"#include "llvm/Analysis/TargetTransformInfo.h"#include "llvm/Analysis/ValueTracking.h"#include "llvm/Analysis/VectorUtils.h"#include "llvm/CodeGen/Analysis.h"#include "llvm/CodeGen/CallingConvLower.h"#include "llvm/CodeGen/ComplexDeinterleavingPass.h"#include "llvm/CodeGen/GlobalISel/GISelValueTracking.h"#include "llvm/CodeGen/GlobalISel/Utils.h"#include "llvm/CodeGen/ISDOpcodes.h"#include "llvm/CodeGen/MachineBasicBlock.h"#include "llvm/CodeGen/MachineFrameInfo.h"#include "llvm/CodeGen/MachineFunction.h"#include "llvm/CodeGen/MachineInstr.h"#include "llvm/CodeGen/MachineInstrBuilder.h"#include "llvm/CodeGen/MachineMemOperand.h"#include "llvm/CodeGen/MachineRegisterInfo.h"#include "llvm/CodeGen/SDPatternMatch.h"#include "llvm/CodeGen/SelectionDAG.h"#include "llvm/CodeGen/SelectionDAGNodes.h"#include "llvm/CodeGen/TargetCallingConv.h"#include "llvm/CodeGen/TargetInstrInfo.h"#include "llvm/CodeGen/TargetOpcodes.h"#include "llvm/CodeGen/ValueTypes.h"#include "llvm/CodeGenTypes/MachineValueType.h"#include "llvm/IR/Attributes.h"#include "llvm/IR/Constants.h"#include "llvm/IR/DataLayout.h"#include "llvm/IR/DebugLoc.h"#include "llvm/IR/DerivedTypes.h"#include "llvm/IR/Function.h"#include "llvm/IR/GetElementPtrTypeIterator.h"#include "llvm/IR/GlobalValue.h"#include "llvm/IR/IRBuilder.h"#include "llvm/IR/Instruction.h"#include "llvm/IR/Instructions.h"#include "llvm/IR/IntrinsicInst.h"#include "llvm/IR/Intrinsics.h"#include "llvm/IR/IntrinsicsAArch64.h"#include "llvm/IR/Module.h"#include "llvm/IR/PatternMatch.h"#include "llvm/IR/Type.h"#include "llvm/IR/Use.h"#include "llvm/IR/Value.h"#include "llvm/Support/AtomicOrdering.h"#include "llvm/Support/Casting.h"#include "llvm/Support/CodeGen.h"#include "llvm/Support/CommandLine.h"#include "llvm/Support/Debug.h"#include "llvm/Support/ErrorHandling.h"#include "llvm/Support/InstructionCost.h"#include "llvm/Support/KnownBits.h"#include "llvm/Support/MathExtras.h"#include "llvm/Support/SipHash.h"#include "llvm/Support/raw_ostream.h"#include "llvm/Target/TargetMachine.h"#include "llvm/Target/TargetOptions.h"#include "llvm/TargetParser/Triple.h"#include <algorithm>#include <bitset>#include <cassert>#include <cctype>#include <cstdint>#include <cstdlib>#include <iterator>#include <limits>#include <optional>#include <tuple>#include <utility>#include <vector>#include "AArch64GenAsmMatcher.inc"Go to the source code of this file.
Classes | |
| struct | SVEMultiVectorInfo |
| struct | GenericSetCCInfo |
| Helper structure to keep track of ISD::SET_CC operands. More... | |
| struct | AArch64SetCCInfo |
| Helper structure to keep track of a SET_CC lowered into AArch64 code. More... | |
| union | SetCCInfo |
| Helper structure to keep track of SetCC information. More... | |
| struct | SetCCInfoAndKind |
| Helper structure to be able to read SetCC information. More... | |
Macros | |
| #define | DEBUG_TYPE "aarch64-lower" |
| #define | GET_REGISTER_MATCHER |
Enumerations | |
| enum class | PredicateConstraint { Uph , Upl , Upa } |
| enum class | ReducedGprConstraint { Uci , Ucj } |
Variables | |
| cl::opt< bool > | EnableAArch64ELFLocalDynamicTLSGeneration ("aarch64-elf-ldtls-generation", cl::Hidden, cl::desc("Allow AArch64 Local Dynamic TLS code generation"), cl::init(false)) |
| static cl::opt< bool > | EnableOptimizeLogicalImm ("aarch64-enable-logical-imm", cl::Hidden, cl::desc("Enable AArch64 logical imm instruction " "optimization"), cl::init(true)) |
| static cl::opt< bool > | EnableCombineMGatherIntrinsics ("aarch64-enable-mgather-combine", cl::Hidden, cl::desc("Combine extends of AArch64 masked " "gather intrinsics"), cl::init(true)) |
| static cl::opt< bool > | EnableExtToTBL ("aarch64-enable-ext-to-tbl", cl::Hidden, cl::desc("Combine ext and trunc to TBL"), cl::init(true)) |
| static cl::opt< unsigned > | MaxXors ("aarch64-max-xors", cl::init(16), cl::Hidden, cl::desc("Maximum of xors")) |
| cl::opt< bool > | EnableSVEGISel ("aarch64-enable-gisel-sve", cl::Hidden, cl::desc("Enable / disable SVE scalable vectors in Global ISel"), cl::init(false)) |
| static cl::opt< int > | BrMergingBaseCostThresh ("aarch64-br-merging-base-cost", cl::init(2), cl::desc("Cost threshold for merging multiple conditionals into one branch " "versus splitting into multiple branches: conditionals are merged when " "their instruction cost is below this limit and split above it. Set to " "-1 to never merge branches."), cl::Hidden) |
| static cl::opt< int > | BrMergingCcmpBias ("aarch64-br-merging-ccmp-bias", cl::init(6), cl::desc("Increases 'aarch64-br-merging-base-cost' to account for the " "CCMP instruction, which is always available on AArch64 and " "makes merging branch conditions cheaper."), cl::Hidden) |
| static cl::opt< int > | BrMergingCbzTbnzBias ("aarch64-br-merging-cbz-tbnz-bias", cl::init(6), cl::desc("Decreases 'aarch64-br-merging-base-cost' when a condition can " "lower to a single CBZ/CBNZ or TBZ/TBNZ compare-and-branch, to " "bias toward splitting. Set to 0 to disable."), cl::Hidden) |
| static cl::opt< int > | BrMergingLikelyBias ("aarch64-br-merging-likely-bias", cl::init(0), cl::desc("Increases 'aarch64-br-merging-base-cost' when all conditionals " "are likely to be executed, biasing toward merging. Set to -1 to " "never merge likely branches."), cl::Hidden) |
| static cl::opt< int > | BrMergingUnlikelyBias ("aarch64-br-merging-unlikely-bias", cl::init(-1), cl::desc("Decreases 'aarch64-br-merging-base-cost' when all conditionals are " "unlikely to be executed, biasing toward splitting. Set to -1 to never " "merge unlikely branches."), cl::Hidden) |
| static cl::opt< bool > | UseFEATCPACodegen ("aarch64-use-featcpa-codegen", cl::Hidden, cl::desc("Generate ISD::PTRADD nodes for pointer arithmetic in " "SelectionDAG for FEAT_CPA"), cl::init(false)) |
| static cl::opt< bool > | UseConditionalFPMRWrite ("aarch64-use-conditional-fpmr-write", cl::Hidden, cl::desc("Only write FPMR when the requested value differs from the " "current value"), cl::init(false)) |
| static cl::opt< bool > | EnableSVEFixedLengthBfloatSupport ("aarch64-sve-vls-bfloat-support", cl::Hidden, cl::desc("Use SVE for fixed-length vector bfloat operations"), cl::init(false)) |
| constexpr MVT | CondCodeVT = MVT::i32 |
| Value type used for condition codes. | |
| constexpr MVT | FlagsVT = MVT::i32 |
| Value type used for NZCV flags. | |
| static const MCPhysReg | GPRArgRegs [] |
| static const MCPhysReg | FPRArgRegs [] |
| false | |
| Simplify Addr given that the top byte of it is ignored by HW during / address translation. | |
Macro Definition Documentation
◆ DEBUG_TYPE
| #define DEBUG_TYPE "aarch64-lower" |
Definition at line 111 of file AArch64ISelLowering.cpp.
◆ GET_REGISTER_MATCHER
| #define GET_REGISTER_MATCHER |
Definition at line 13896 of file AArch64ISelLowering.cpp.
Enumeration Type Documentation
◆ PredicateConstraint
|
strong |
| Enumerator | |
|---|---|
| Uph | |
| Upl | |
| Upa | |
Definition at line 14219 of file AArch64ISelLowering.cpp.
◆ ReducedGprConstraint
|
strong |
| Enumerator | |
|---|---|
| Uci | |
| Ucj | |
Definition at line 14282 of file AArch64ISelLowering.cpp.
Function Documentation
◆ analyzeCallOperands()
|
static |
Definition at line 10059 of file AArch64ISelLowering.cpp.
References llvm::CCState::AllocateStack(), llvm::TargetLowering::CallLoweringInfo::Args, llvm::CallingConv::ARM64EC_Thunk_X64, assert(), llvm::TargetLowering::CallLoweringInfo::CallConv, llvm::TargetLowering::CallLoweringInfo::CB, llvm::AArch64TargetLowering::CCAssignFnForCall(), llvm::TargetLowering::CallLoweringInfo::DAG, llvm::CCValAssign::Full, llvm::SelectionDAG::getDataLayout(), llvm::EVT::getSimpleVT(), llvm::TargetLoweringBase::getValueType(), llvm::AArch64Subtarget::isCallingConvWin64(), llvm::CallBase::isMustTailCall(), llvm::EVT::isSimple(), llvm::TargetLowering::CallLoweringInfo::IsVarArg, llvm::ISD::ArgFlagsTy::isVarArg(), llvm::TargetLowering::CallLoweringInfo::Outs, llvm::report_fatal_error(), and llvm::SmallVectorTemplateCommon< T, typename >::size().
◆ areLoadedOffsetButOtherwiseSame()
|
static |
Definition at line 24316 of file AArch64ISelLowering.cpp.
References llvm::ISD::ADD, llvm::all_of(), llvm::ISD::ANY_EXTEND, areLoadedOffsetButOtherwiseSame(), llvm::SelectionDAG::areNonVolatileConsecutiveLoads(), llvm::get(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::EVT::getScalarSizeInBits(), llvm::SDValue::getValueType(), llvm::SDValue::hasOneUse(), isLoadOrMultipleLoads(), llvm::ISD::SIGN_EXTEND, Size, llvm::SmallVectorTemplateCommon< T, typename >::size(), llvm::ISD::SUB, llvm::ISD::ZERO_EXTEND, and llvm::zip().
Referenced by areLoadedOffsetButOtherwiseSame(), and performExtBinopLoadFold().
◆ calculatePreExtendType()
Calculates what the pre-extend type is, based on the extension operation node provided by Extend.
In the case that Extend is a SIGN_EXTEND or a ZERO_EXTEND, the pre-extend type is pulled directly from the operand, while other extend operations need a bit more inspection to get this information.
- Parameters
-
Extend The SDNode from the DAG that represents the extend operation
- Returns
- The type representing the
Extendsource type, orMVT::Otherif no valid type can be determined
Definition at line 21213 of file AArch64ISelLowering.cpp.
References llvm::ISD::AND, llvm::ISD::ANY_EXTEND, llvm::ISD::AssertSext, llvm::ISD::AssertZext, llvm::dyn_cast(), llvm::SDValue::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDValue::getValueType(), llvm::VTSDNode::getVT(), llvm::ISD::SIGN_EXTEND, llvm::ISD::SIGN_EXTEND_INREG, uint64_t, and llvm::ISD::ZERO_EXTEND.
Referenced by performBuildShuffleExtendCombine().
◆ callConvSupportsVarArgs()
|
static |
Return true if the call convention supports varargs Currently only those that pass varargs like the C calling convention does are eligible Calling conventions listed in this function must also be properly handled in AArch64Subtarget::isCallingConvWin64.
Definition at line 10045 of file AArch64ISelLowering.cpp.
References llvm::CallingConv::AArch64_SVE_VectorCall, llvm::CallingConv::C, and llvm::CallingConv::PreserveNone.
◆ canEmitConjunction()
|
static |
Returns true if Val is a tree of AND/OR/SETCC operations that can be expressed as a conjunction.
See CMP;CCMP matching.
- Parameters
-
CanNegate Set to true if we can negate the whole sub-tree just by changing the conditions on the SETCC tests. (this means we can call emitConjunctionRec() with Negate==true on this sub-tree) MustBeFirst Set to true if this subtree needs to be negated and we cannot do the negation naturally. We are required to emit the subtree first in this case. PreferFirst Set to true if processing this subtree first may result in more efficient code. WillNegate Is true if are called when the result of this subexpression must be negated. This happens when the outer expression is an OR. We can use this fact to know that we have a double negation (or (or ...) ...) that can be implemented for free.
Definition at line 4202 of file AArch64ISelLowering.cpp.
References llvm::ISD::AND, assert(), canEmitConjunction(), llvm::Depth, llvm::SelectionDAG::doesNodeExist(), llvm::SDNode::getOpcode(), llvm::SDNode::getOperand(), llvm::SDValue::getValueType(), llvm::SelectionDAG::getVTList(), llvm::SDValue::hasOneUse(), llvm::O0, llvm::O1, llvm::ISD::OR, llvm::ISD::SETCC, and llvm::ISD::SUB.
Referenced by canEmitConjunction(), canEmitConjunction(), emitConjunction(), and emitConjunctionRec().
◆ canGuaranteeTCO()
|
static |
Return true if the calling convention is one that we can guarantee TCO for.
Definition at line 10017 of file AArch64ISelLowering.cpp.
References llvm::CallingConv::Fast, llvm::CallingConv::SwiftTail, and llvm::CallingConv::Tail.
Referenced by llvm::AArch64CallLowering::isEligibleForTailCallOptimization(), mayTailCallThisCC(), mayTailCallThisCC(), shouldGuaranteeTCO(), and shouldGuaranteeTCO().
◆ canLowerSRLToRoundingShiftForVT()
|
static |
Definition at line 18128 of file AArch64ISelLowering.cpp.
References llvm::ISD::ADD, llvm::Add, assert(), llvm::dyn_cast_or_null(), llvm::SDNode::getOpcode(), llvm::SDNode::getOperand(), llvm::EVT::getScalarSizeInBits(), llvm::SelectionDAG::getSplatValue(), llvm::SDValue::getValueType(), llvm::EVT::isScalableVT(), llvm::ISD::SRL, and uint64_t.
Referenced by trySimplifySrlAddToRshrnb().
◆ carryFlagToValue()
|
static |
Definition at line 4803 of file AArch64ISelLowering.cpp.
References assert(), Cond, DL, getCondCode(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::SDValue::getResNo(), llvm::AArch64CC::HS, and llvm::AArch64CC::LO.
Referenced by LowerADDSUBO_CARRY(), and lowerADDSUBO_CARRY().
◆ changeFPCCToAArch64CC()
|
static |
changeFPCCToAArch64CC - Convert a DAG fp condition code to an AArch64 CC.
Definition at line 3842 of file AArch64ISelLowering.cpp.
References llvm::AArch64CC::AL, llvm::AArch64CC::EQ, llvm::AArch64CC::GE, llvm::AArch64CC::GT, llvm::AArch64CC::HI, llvm::AArch64CC::LE, llvm_unreachable, llvm::AArch64CC::LS, llvm::AArch64CC::LT, llvm::AArch64CC::MI, llvm::AArch64CC::NE, llvm::AArch64CC::PL, llvm::ISD::SETEQ, llvm::ISD::SETGE, llvm::ISD::SETGT, llvm::ISD::SETLE, llvm::ISD::SETLT, llvm::ISD::SETNE, llvm::ISD::SETO, llvm::ISD::SETOEQ, llvm::ISD::SETOGE, llvm::ISD::SETOGT, llvm::ISD::SETOLE, llvm::ISD::SETOLT, llvm::ISD::SETONE, llvm::ISD::SETUEQ, llvm::ISD::SETUGE, llvm::ISD::SETUGT, llvm::ISD::SETULE, llvm::ISD::SETULT, llvm::ISD::SETUNE, llvm::ISD::SETUO, llvm::AArch64CC::VC, and llvm::AArch64CC::VS.
Referenced by changeFPCCToANDAArch64CC(), and changeVectorFPCCToAArch64CC().
◆ changeFPCCToANDAArch64CC()
|
static |
Convert a DAG fp condition code to an AArch64 CC.
This differs from changeFPCCToAArch64CC in that it returns cond codes that should be AND'ed instead of OR'ed.
Definition at line 3905 of file AArch64ISelLowering.cpp.
References llvm::AArch64CC::AL, assert(), changeFPCCToAArch64CC(), llvm::AArch64CC::LE, llvm::AArch64CC::NE, llvm::AArch64CC::PL, llvm::ISD::SETONE, llvm::ISD::SETUEQ, and llvm::AArch64CC::VC.
Referenced by emitConjunctionRec().
◆ changeIntCCToAArch64CC()
|
static |
changeIntCCToAArch64CC - Convert a DAG integer condition code to an AArch64 CC
Definition at line 3813 of file AArch64ISelLowering.cpp.
References RHS.
Referenced by emitConjunctionRec(), and getAArch64Cmp().
◆ changeVectorFPCCToAArch64CC()
|
static |
changeVectorFPCCToAArch64CC - Convert a DAG fp condition code to an AArch64 CC usable with the vector instructions.
Fewer operations are available without a real NZCV register, so we have to use less efficient combinations to get the same effect.
Definition at line 3935 of file AArch64ISelLowering.cpp.
References llvm::AArch64CC::AL, changeFPCCToAArch64CC(), llvm::AArch64CC::GE, llvm::AArch64CC::LS, llvm::AArch64CC::MI, llvm::ISD::SETLE, llvm::ISD::SETLT, llvm::ISD::SETO, llvm::ISD::SETUEQ, llvm::ISD::SETUGE, llvm::ISD::SETUGT, llvm::ISD::SETULE, llvm::ISD::SETULT, and llvm::ISD::SETUO.
Referenced by emitFloatCompareMask().
◆ checkLaneSlide()
|
static |
Flag slide shuffle patterns where one operand is zeros.
Left slide: shufflevector v, zeros, <1,2,3,...> -> ushr Right slide: shufflevector zeros, v, <N-1,N,N+1,...> -> shl Check if a single 64-bit lane has a valid slide pattern. LaneStart: first element index of this lane in the full vector LaneElts: number of elements in the lane Returns slide amount in elements, or 0 if not a valid slide.
Definition at line 15392 of file AArch64ISelLowering.cpp.
Referenced by isSlideWithZerosMask().
◆ checkValueWidth()
|
static |
Definition at line 28736 of file AArch64ISelLowering.cpp.
References llvm::ISD::AssertSext, llvm::ISD::AssertZext, llvm::cast(), llvm::ISD::Constant, llvm::LoadSDNode::getExtensionType(), llvm::MemSDNode::getMemoryVT(), llvm::VTSDNode::getVT(), llvm::ISD::LOAD, llvm::ISD::NON_EXTLOAD, llvm::ISD::SEXTLOAD, llvm::ISD::TargetConstant, and llvm::ISD::ZEXTLOAD.
Referenced by performCONDCombine(), and performSETCCCombine().
◆ checkZExtBool()
|
static |
Definition at line 10341 of file AArch64ISelLowering.cpp.
References llvm::SelectionDAG::computeKnownBits(), llvm::EVT::getSizeInBits(), and llvm::SDValue::getValueType().
◆ combineAcrossLanesIntrinsic()
|
static |
Definition at line 24888 of file AArch64ISelLowering.cpp.
References DL, llvm::SelectionDAG::getExtractVectorElt(), llvm::SelectionDAG::getNode(), N, and Opc.
Referenced by performIntrinsicCombine().
◆ combineSVEBitSel()
|
static |
Definition at line 25147 of file AArch64ISelLowering.cpp.
References DL, llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getNOT(), llvm_unreachable, and N.
Referenced by performIntrinsicCombine().
◆ combineSVEPrefetchVecBaseImmOff()
|
static |
Combines a node carrying the intrinsic aarch64_sve_prf<T>_gather_scalar_offset into a node that uses aarch64_sve_prfb_gather_uxtw_index when the scalar offset passed to aarch64_sve_prf<T>_gather_scalar_offset is not a valid immediate for the sve gather prefetch instruction with vector plus immediate addressing mode.
Definition at line 30921 of file AArch64ISelLowering.cpp.
References AbstractManglingParser< Derived, Alloc >::Ops, DL, llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getTargetConstant(), llvm::SelectionDAG::getVTList(), isValidImmForSVEVecImmAddrMode(), N, and std::swap().
Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().
◆ combineSVEReductionFP()
|
static |
Definition at line 25077 of file AArch64ISelLowering.cpp.
References DL, llvm::SelectionDAG::getExtractVectorElt(), llvm::SelectionDAG::getNode(), llvm::SDValue::getValueType(), N, and Opc.
Referenced by performIntrinsicCombine().
◆ combineSVEReductionInt()
|
static |
Definition at line 25060 of file AArch64ISelLowering.cpp.
References DL, llvm::SelectionDAG::getExtractVectorElt(), llvm::SelectionDAG::getNode(), getPackedSVEVectorVT(), N, and Opc.
Referenced by performIntrinsicCombine().
◆ combineSVEReductionOrderedFP()
|
static |
Definition at line 25092 of file AArch64ISelLowering.cpp.
References DL, llvm::SelectionDAG::getExtractVectorElt(), llvm::SelectionDAG::getInsertVectorElt(), llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getPOISON(), llvm::SDValue::getValueType(), N, and Opc.
Referenced by performIntrinsicCombine().
◆ combineToExtendBoolVectorInReg()
|
static |
Definition at line 25975 of file AArch64ISelLowering.cpp.
References llvm::ISD::AND, llvm::ISD::ANY_EXTEND, assert(), llvm::ISD::BITCAST, DL, llvm::SelectionDAG::getAnyExtOrTrunc(), llvm::SelectionDAG::getBitcast(), llvm::APInt::getBitsSet(), llvm::SelectionDAG::getBuildVector(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getContext(), llvm::SelectionDAG::getDataLayout(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::EVT::getScalarType(), llvm::SelectionDAG::getSetCC(), llvm::SelectionDAG::getSExtOrTrunc(), llvm::EVT::getSizeInBits(), llvm::SelectionDAG::getSplat(), llvm::SDValue::getValueType(), llvm::EVT::getVectorNumElements(), llvm::SelectionDAG::getVectorShuffle(), llvm::EVT::getVectorVT(), I, llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalizeOps(), llvm::DataLayout::isBigEndian(), llvm::EVT::isScalarInteger(), llvm::EVT::isVector(), llvm::ISD::SCALAR_TO_VECTOR, llvm::ISD::SETEQ, llvm::ISD::SIGN_EXTEND, llvm::ISD::SRL, and llvm::ISD::ZERO_EXTEND.
Referenced by canonicalizeBoolMask(), combineSelect(), combineSext(), combineZext(), performExtendCombine(), performExtendToBoolVectorLoadCombine(), and performVSelectCombine().
◆ ConstantBuildVector()
|
static |
Definition at line 17009 of file AArch64ISelLowering.cpp.
References assert(), llvm::cast(), DL, llvm::ISD::FNEG, llvm::APInt::getHighBitsSet(), llvm::SelectionDAG::getNode(), llvm::EVT::getSizeInBits(), llvm::MVT::getVectorVT(), resolveBuildVector(), tryAdvSIMDModImm16(), tryAdvSIMDModImm32(), tryAdvSIMDModImm321s(), tryAdvSIMDModImm64(), tryAdvSIMDModImm8(), tryAdvSIMDModImmFP(), trySVESplat64(), and llvm::APInt::zext().
◆ constructDup()
|
static |
Definition at line 15910 of file AArch64ISelLowering.cpp.
References llvm::ISD::BITCAST, llvm::ISD::CONCAT_VECTORS, DL, llvm::ISD::EXTRACT_SUBVECTOR, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getConstant(), llvm::SDValue::getConstantOperandVal(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::MVT::getScalarType(), llvm::SDValue::getScalarValueSizeInBits(), llvm::SDValue::getSimpleValueType(), llvm::EVT::getSizeInBits(), llvm::SDValue::getValueSizeInBits(), llvm::SDValue::getValueType(), llvm::EVT::getVectorNumElements(), llvm::MVT::getVectorVT(), llvm::EVT::is128BitVector(), and WidenVector().
◆ convertFixedMaskToScalableVector()
|
static |
Definition at line 34208 of file AArch64ISelLowering.cpp.
References llvm::cast(), convertToScalableVector(), DL, llvm::SelectionDAG::getConstant(), getContainerForFixedLengthVector(), llvm::SelectionDAG::getNode(), getPredicateForFixedLengthVector(), llvm::SDValue::getValueType(), llvm::isBitwiseNot(), llvm::ISD::isBuildVectorAllOnes(), llvm::ISD::SETCC, and llvm::ISD::SETNE.
Referenced by llvm::AArch64TargetLowering::LowerOperation(), and LowerVectorMatch().
◆ convertFromScalableVector()
|
static |
Definition at line 34160 of file AArch64ISelLowering.cpp.
References assert(), llvm::SelectionDAG::getExtractSubvector(), and llvm::EVT::isFixedLengthVector().
Referenced by convertLocVTToValVT(), GenerateFixedLengthSVETBL(), getVCIXISDNodeWCHAIN(), getWideningInterleave(), llvm::RISCVTargetLowering::joinRegisterPartsIntoValue(), lowerBUILD_VECTOR(), lowerBuildVectorOfConstants(), lowerBuildVectorViaDominantValues(), lowerBuildVectorViaVID(), lowerCONVERT_FROM_ARBITRARY_FP(), lowerFixedVectorSegLoadIntrinsics(), lowerFMAXIMUM_FMINIMUM(), lowerFP_TO_INT_SAT(), llvm::RISCVTargetLowering::LowerOperation(), lowerVECTOR_SHUFFLEAsVSlide1(), lowerVECTOR_SHUFFLEAsVSlidedown(), lowerVECTOR_SHUFFLEAsVSlideup(), lowerVectorFTRUNC_FCEIL_FFLOOR_FROUND(), LowerVectorMatch(), lowerVectorStrictFTRUNC_FCEIL_FFLOOR_FROUND(), lowerVectorXRINT_XROUND(), lowerZvzipVPAIR(), lowerZvzipVUNZIP(), lowerZvzipVZIP(), matchSplatAsGather(), performDUPCombine(), performFP_TO_INTCombine(), and performSVEMulAddSubCombine().
◆ convertMergedOpToPredOp()
|
static |
Definition at line 25126 of file AArch64ISelLowering.cpp.
References assert(), llvm::SelectionDAG::getNode(), llvm::ISD::INTRINSIC_WO_CHAIN, isAllActivePredicate(), N, and Opc.
Referenced by performIntrinsicCombine().
◆ convertMulToShlAdd()
|
static |
Definition at line 6146 of file AArch64ISelLowering.cpp.
References llvm::APInt::abs(), llvm::ISD::ADD, llvm::APInt::countr_zero(), llvm::ISD::DELETED_NODE, DL, llvm::APInt::getBoolValue(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNegative(), llvm::SelectionDAG::getNode(), llvm::ISD::isConstantSplatVector(), llvm::APInt::isNegative(), LHS, llvm::APInt::lshrInPlace(), N, llvm::ISD::SHL, and llvm::ISD::SUB.
◆ convertToScalableVector()
|
static |
Definition at line 34151 of file AArch64ISelLowering.cpp.
References assert(), llvm::SelectionDAG::getInsertSubvector(), llvm::SelectionDAG::getPOISON(), and llvm::EVT::isScalableVector().
Referenced by combineToVCPOP(), convertFixedMaskToScalableVector(), convertValVTToLocVT(), GenerateFixedLengthSVETBL(), getWideningInterleave(), lowerBUILD_VECTOR(), lowerBuildVectorOfConstants(), lowerBuildVectorViaDominantValues(), lowerCONVERT_FROM_ARBITRARY_FP(), lowerCttzElts(), lowerFixedVectorSegLoadIntrinsics(), lowerFixedVectorSegStoreIntrinsics(), lowerFMAXIMUM_FMINIMUM(), lowerFP_TO_INT_SAT(), llvm::RISCVTargetLowering::LowerOperation(), lowerScalarInsert(), lowerShuffleViaVRegSplitting(), lowerVECTOR_SHUFFLEAsVSlide1(), lowerVECTOR_SHUFFLEAsVSlidedown(), lowerVECTOR_SHUFFLEAsVSlideup(), lowerVectorFTRUNC_FCEIL_FFLOOR_FROUND(), LowerVectorMatch(), lowerVectorStrictFTRUNC_FCEIL_FFLOOR_FROUND(), lowerVectorXRINT_XROUND(), lowerZvzipVPAIR(), lowerZvzipVUNZIP(), lowerZvzipVZIP(), matchSplatAsGather(), llvm::RISCVTargetLowering::PerformDAGCombine(), performFP_TO_INTCombine(), performSVEMulAddSubCombine(), processVCIXOperands(), llvm::RISCVTargetLowering::ReplaceNodeResults(), and llvm::RISCVTargetLowering::splitValueIntoRegisterParts().
◆ createGPRPairNode()
|
static |
Definition at line 32572 of file AArch64ISelLowering.cpp.
References AbstractManglingParser< Derived, Alloc >::Ops, DL, llvm::SelectionDAG::getDataLayout(), llvm::SelectionDAG::getMachineNode(), llvm::SelectionDAG::getTargetConstant(), llvm::DataLayout::isBigEndian(), llvm::SelectionDAG::SplitScalar(), and std::swap().
Referenced by ReplaceCMP_SWAP_128Results().
◆ createTblForTrunc()
Definition at line 19123 of file AArch64ISelLowering.cpp.
References assert(), llvm::SmallVectorTemplateCommon< T, typename >::back(), llvm::SmallVectorTemplateCommon< T, typename >::begin(), llvm::cast(), llvm::SmallVectorImpl< T >::clear(), llvm::SmallVectorTemplateCommon< T, typename >::empty(), llvm::SmallVectorTemplateCommon< T, typename >::end(), llvm::Instruction::eraseFromParent(), llvm::ConstantVector::get(), llvm::FixedVectorType::get(), llvm::User::getOperand(), llvm::Value::getType(), llvm::SmallVectorTemplateBase< T, bool >::push_back(), llvm::Value::replaceAllUsesWith(), Results, and llvm::SmallVectorTemplateCommon< T, typename >::size().
Referenced by llvm::AArch64TargetLowering::optimizeExtendOrTruncateConversion().
◆ createTblShuffleForSExt()
|
static |
Definition at line 19105 of file AArch64ISelLowering.cpp.
References llvm::cast(), createTblShuffleMask(), llvm::PoisonValue::get(), llvm::VectorType::getElementType(), and uint64_t.
Referenced by llvm::AArch64TargetLowering::optimizeExtendOrTruncateConversion().
◆ createTblShuffleForZExt()
|
static |
Definition at line 19083 of file AArch64ISelLowering.cpp.
References llvm::cast(), createTblShuffleMask(), llvm::PoisonValue::get(), llvm::VectorType::getElementType(), and uint64_t.
Referenced by llvm::AArch64TargetLowering::optimizeExtendOrTruncateConversion().
◆ createTblShuffleMask()
|
static |
Definition at line 19060 of file AArch64ISelLowering.cpp.
Referenced by createTblShuffleForSExt(), and createTblShuffleForZExt().
◆ CustomNonLegalBITCASTResults()
|
static |
Definition at line 32355 of file AArch64ISelLowering.cpp.
References llvm::ISD::BITCAST, DL, llvm::SelectionDAG::getExtractSubvector(), llvm::SelectionDAG::getNode(), N, Results, and llvm::ISD::SCALAR_TO_VECTOR.
◆ DoCrossByteCLMUL()
Definition at line 8611 of file AArch64ISelLowering.cpp.
References llvm::ISD::CLMUL, DL, llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getTargetConstant(), and llvm::ISD::INTRINSIC_WO_CHAIN.
◆ emitComparison()
|
static |
Definition at line 4040 of file AArch64ISelLowering.cpp.
References llvm::ISD::AND, assert(), DL, FlagsVT, llvm::ISD::FP_EXTEND, llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getSubtarget(), llvm::SDValue::getValue(), llvm::SelectionDAG::getVTList(), isCMN(), llvm::EVT::isFloatingPoint(), llvm::isNullConstant(), LHS, llvm::SelectionDAG::ReplaceAllUsesWith(), RHS, and llvm::ISD::SUB.
Referenced by emitConjunctionRec(), and getAArch64Cmp().
◆ emitConditionalComparison()
|
static |
can be transformed to: not (and (not (and (setCC (cmp C)) (setCD (cmp D)))) (and (not (setCA (cmp A)) (not (setCB (cmp B))))))" which can be implemented as: cmp C ccmp D, inv(CD), CC ccmp A, CA, inv(CD) ccmp B, CB, inv(CA) check for CB flags
A counterexample is "or (and A B) (and C D)" which translates to not (and (not (and (not A) (not B))) (not (and (not C) (not D)))), we can only implement 1 of the inner (not) operations, but not both! Create a conditional comparison; Use CCMP, CCMN or FCCMP as appropriate.
Definition at line 4144 of file AArch64ISelLowering.cpp.
References assert(), DL, llvm::dyn_cast(), FlagsVT, llvm::ISD::FP_EXTEND, getCondCode(), llvm::SelectionDAG::getConstant(), llvm::AArch64CC::getInvertedCondCode(), llvm::SelectionDAG::getNode(), llvm::AArch64CC::getNZCVToSatisfyCondCode(), llvm::SelectionDAG::getSubtarget(), Imm, isCMN(), llvm::isNullConstant(), LHS, RHS, and llvm::ISD::SUB.
Referenced by emitConjunctionRec().
◆ emitConjunction()
|
static |
Emit expression as a conjunction (a series of CCMP/CFCMP ops).
In some cases this is even possible with OR operations in the expression. See CMP;CCMP matching.
- See also
- emitConjunctionRec().
Definition at line 4393 of file AArch64ISelLowering.cpp.
References llvm::AArch64CC::AL, canEmitConjunction(), and emitConjunctionRec().
Referenced by getAArch64Cmp(), LowerBRCOND(), and performANDSETCCCombine().
◆ emitConjunctionRec()
|
static |
Emit conjunction or disjunction tree with the CMP/FCMP followed by a chain of CCMP/CFCMP ops.
See CMP;CCMP matching. Tries to transform the given i1 producing node Val to a series compare and conditional compare operations.
- Returns
- an NZCV flags producing node and sets
OutCCto the flags that should be tested or returns SDValue() if transformation was not possible.Negateis true if we want this sub-tree being negated just by changing SETCC conditions.
Definition at line 4275 of file AArch64ISelLowering.cpp.
References llvm::AArch64CC::AL, llvm::ISD::AND, assert(), canEmitConjunction(), llvm::cast(), changeFPCCToANDAArch64CC(), changeIntCCToAArch64CC(), DL, emitComparison(), emitConditionalComparison(), emitConjunctionRec(), llvm::AArch64CC::getInvertedCondCode(), llvm::SDValue::getNode(), llvm::SDNode::getOpcode(), llvm::SDNode::getOperand(), llvm::SDNode::hasOneUse(), LHS, llvm::ISD::OR, RHS, llvm::ISD::SETCC, and std::swap().
Referenced by emitConjunction(), and emitConjunctionRec().
◆ emitFloatCompareMask()
|
static |
For SELECT_CC, when the true/false values are (-1, 0) and the compared values are scalars, try to emit a mask generating vector instruction.
Definition at line 13024 of file AArch64ISelLowering.cpp.
References llvm::AArch64CC::AL, assert(), changeVectorFPCCToAArch64CC(), DL, llvm::dyn_cast(), emitVectorComparison(), llvm::SelectionDAG::getContext(), llvm::SelectionDAG::getExtractVectorElt(), llvm::SelectionDAG::getInsertVectorElt(), llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getNOT(), llvm::SelectionDAG::getPOISON(), llvm::ISD::getSetCCInverse(), llvm::EVT::getSizeInBits(), llvm::SDValue::getValueType(), llvm::EVT::getVectorVT(), llvm::ConstantSDNode::isAllOnes(), llvm::SelectionDAG::isKnownNeverNaN(), llvm::ConstantSDNode::isZero(), LHS, llvm::ISD::OR, Poison, RHS, llvm::ISD::SETO, llvm::ISD::SETOEQ, llvm::ISD::SETUNE, and llvm::ISD::SETUO.
◆ emitStrictFPComparison()
|
static |
Definition at line 4020 of file AArch64ISelLowering.cpp.
References assert(), DL, FlagsVT, llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getSubtarget(), LHS, RHS, and llvm::ISD::STRICT_FP_EXTEND.
◆ emitVectorComparison()
|
static |
Emit vector comparison for floating-point values, producing a mask.
Definition at line 12976 of file AArch64ISelLowering.cpp.
References assert(), DL, llvm::AArch64CC::EQ, llvm::AArch64CC::GE, llvm::SelectionDAG::getContext(), llvm::SelectionDAG::getExtractVectorElt(), llvm::SelectionDAG::getInsertVectorElt(), llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getNOT(), llvm::SelectionDAG::getPOISON(), llvm::EVT::getSizeInBits(), llvm::EVT::getVectorVT(), llvm::AArch64CC::GT, llvm::AArch64CC::LE, LHS, llvm::AArch64CC::LS, llvm::AArch64CC::LT, llvm::AArch64CC::MI, llvm::AArch64CC::NE, and RHS.
Referenced by emitFloatCompareMask().
◆ extractPtrauthBlendDiscriminators()
|
static |
Definition at line 447 of file AArch64ISelLowering.cpp.
References DL, llvm::dyn_cast(), llvm::SDNode::getConstantOperandVal(), llvm::SDNode::getOpcode(), llvm::SDNode::getOperand(), llvm::SelectionDAG::getRegister(), llvm::SelectionDAG::getTargetConstant(), llvm::ISD::INTRINSIC_WO_CHAIN, llvm::isUInt(), and Register.
◆ findMoreOptimalIndexType()
|
static |
Definition at line 28486 of file AArch64ISelLowering.cpp.
References llvm::cast(), Changed, llvm::EVT::changeVectorElementType(), foldIndexIntoBase(), llvm::SelectionDAG::getContext(), llvm::AArch64Subtarget::getMaxSVEVectorSizeInBits(), llvm::SelectionDAG::getNode(), llvm::EVT::getScalarSizeInBits(), llvm::SelectionDAG::getStepVector(), llvm::SelectionDAG::getSubtarget(), llvm::EVT::getVectorElementType(), llvm::EVT::getVectorMinNumElements(), llvm::EVT::isFixedLengthVector(), llvm::ISD::isVectorShrinkable(), N, llvm::ISD::STEP_VECTOR, llvm::AArch64::SVEBitsPerBlock, and llvm::ISD::TRUNCATE.
Referenced by performMaskedGatherScatterCombine().
◆ foldADCToCINC()
|
static |
Definition at line 23683 of file AArch64ISelLowering.cpp.
References Cond, DL, getCondCode(), llvm::SelectionDAG::getNode(), llvm::isNullConstant(), LHS, llvm::AArch64CC::LO, N, and RHS.
Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().
◆ foldCSELOfCSEL()
|
static |
Definition at line 29194 of file AArch64ISelLowering.cpp.
References llvm::cast(), Cond, DL, llvm::AArch64CC::EQ, llvm::ConstantSDNode::getAPIntValue(), getCondCode(), llvm::SDNode::getConstantOperandVal(), llvm::AArch64CC::getInvertedCondCode(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDNode::getOperand(), llvm::SDValue::getOperand(), llvm::isa(), isCMP(), llvm::AArch64CC::NE, std::swap(), X, and Y.
Referenced by performCSELCombine().
◆ foldCSELofCTTZ()
|
static |
Definition at line 29146 of file AArch64ISelLowering.cpp.
References llvm::ISD::AND, assert(), llvm::BitWidth, llvm::ISD::CTTZ, llvm::AArch64CC::EQ, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::isNullConstant(), N, llvm::AArch64CC::NE, llvm::ISD::TRUNCATE, and X.
Referenced by performCSELCombine().
◆ foldCSELofLASTB()
|
static |
Definition at line 29389 of file AArch64ISelLowering.cpp.
References Default, llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), isAllActivePredicate(), and llvm::AArch64CC::NE.
Referenced by performCSELCombine().
◆ foldIndexIntoBase()
|
static |
- Returns
- true if part of the index was folded into the Base.
Definition at line 28436 of file AArch64ISelLowering.cpp.
References llvm::ISD::ADD, llvm::Add, DL, llvm::SelectionDAG::getNode(), llvm::SDValue::getOperand(), llvm::SelectionDAG::getSplatValue(), llvm::EVT::isVectorOf(), llvm::ISD::MUL, llvm::Offset, and llvm::ISD::SHL.
Referenced by findMoreOptimalIndexType().
◆ foldMaskedShiftToUSHL()
|
static |
Definition at line 30086 of file AArch64ISelLowering.cpp.
References llvm::SelectionDAG::computeKnownBits(), Cond, DL, llvm::SelectionDAG::getConstant(), llvm::KnownBits::getMaxValue(), llvm::SelectionDAG::getNode(), llvm::EVT::getScalarSizeInBits(), llvm::SelectionDAG::getTargetConstant(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::ISD::INTRINSIC_WO_CHAIN, llvm::EVT::is128BitVector(), llvm::EVT::is64BitVector(), llvm::EVT::isFixedLengthVector(), llvm::EVT::isInteger(), llvm::AArch64Subtarget::isNeonAvailable(), llvm::AArch64Subtarget::isSVEAvailable(), llvm::TargetLoweringBase::isTypeLegal(), llvm::SDPatternMatch::m_SetCC(), llvm::PatternMatch::m_Specific(), llvm::SDPatternMatch::m_SpecificCondCode(), llvm::PatternMatch::m_SpecificInt(), N, llvm::SDPatternMatch::sd_match(), llvm::ISD::SUB, llvm::APInt::ule(), llvm::ISD::UMIN, and X.
Referenced by performVSelectMaskedShiftCombine().
◆ foldOverflowCheck()
|
static |
Definition at line 23659 of file AArch64ISelLowering.cpp.
References getCSETCondCode(), llvm::SelectionDAG::getNode(), llvm::SDNode::getOperand(), llvm::SDValue::getOperand(), llvm::AArch64CC::HS, isCMP(), llvm::isNullConstant(), llvm::isOneConstant(), and llvm::AArch64CC::LO.
Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().
◆ foldVectorXorShiftIntoCmp()
|
static |
Turn vector tests of the signbit in the form of: xor (sra X, elt_size(X)-1), -1 into: cmge X, X, #0.
Definition at line 20559 of file AArch64ISelLowering.cpp.
References DL, llvm::dyn_cast(), llvm::SelectionDAG::getConstant(), llvm::SDValue::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SelectionDAG::getSetCC(), llvm::EVT::getSizeInBits(), llvm::SDValue::getValueType(), llvm::EVT::getVectorElementType(), llvm::SDValue::hasOneUse(), llvm::ISD::isBuildVectorAllOnes(), llvm::EVT::isVector(), N, and llvm::ISD::SETGE.
Referenced by combineXor(), and performXorCombine().
◆ GenerateFixedLengthSVETBL()
|
static |
Definition at line 35714 of file AArch64ISelLowering.cpp.
References llvm::ISD::ADD, llvm::ArrayRef(), assert(), llvm::ISD::BITCAST, llvm::EVT::changeTypeToInteger(), convertFromScalableVector(), convertToScalableVector(), llvm::SmallVectorTemplateCommon< T, typename >::data(), DL, llvm::SelectionDAG::getBuildVector(), llvm::SelectionDAG::getConstant(), getContainerForFixedLengthVector(), llvm::SelectionDAG::getContext(), llvm::AArch64Subtarget::getMaxSVEVectorSizeInBits(), llvm::AArch64Subtarget::getMinSVEVectorSizeInBits(), llvm::SelectionDAG::getNode(), llvm::EVT::getSizeInBits(), llvm::SelectionDAG::getSubtarget(), llvm::SelectionDAG::getTargetConstant(), llvm::EVT::getVectorElementType(), llvm::EVT::getVectorNumElements(), llvm::EVT::getVectorVT(), llvm::SelectionDAG::getVScale(), llvm::ISD::INTRINSIC_WO_CHAIN, llvm::AArch64Subtarget::isNeonAvailable(), llvm::ShuffleVectorInst::isSingleSourceMask(), llvm::maxUIntN(), llvm::ISD::MUL, llvm::SmallVectorTemplateBase< T, bool >::push_back(), llvm::ISD::SPLAT_VECTOR, llvm::AArch64::SVEBitsPerBlock, and uint64_t.
◆ GeneratePerfectShuffle()
|
static |
GeneratePerfectShuffle - Given an entry in the perfect-shuffle table, emit the specified operations to build the shuffle.
ID is the perfect-shuffle
Definition at line 15658 of file AArch64ISelLowering.cpp.
References assert(), llvm::ISD::BSWAP, DL, GeneratePerfectShuffle(), llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getConstant(), getExtFactor(), llvm::SelectionDAG::getExtractVectorElt(), llvm::SelectionDAG::getInsertVectorElt(), llvm::SelectionDAG::getNode(), llvm::EVT::getScalarSizeInBits(), llvm::EVT::getSizeInBits(), llvm::SDValue::getValueType(), llvm::EVT::getVectorElementType(), Imm, LHS, llvm_unreachable, OP_COPY, OP_VDUP0, OP_VDUP1, OP_VDUP2, OP_VDUP3, OP_VEXT1, OP_VEXT2, OP_VEXT3, OP_VREV, OP_VTRNL, OP_VTRNR, OP_VUZPL, OP_VUZPR, OP_VZIPL, OP_VZIPR, llvm::PerfectShuffleTable, RHS, llvm::V1, and WidenVector().
Referenced by GeneratePerfectShuffle(), GeneratePerfectShuffle(), GeneratePerfectShuffle(), and LowerVECTOR_SHUFFLE().
◆ GenerateTBL()
|
static |
Definition at line 15821 of file AArch64ISelLowering.cpp.
References llvm::ISD::BITCAST, llvm::ISD::CONCAT_VECTORS, llvm::SmallVectorTemplateCommon< T, typename >::data(), DL, llvm::SelectionDAG::getBuildVector(), llvm::SelectionDAG::getConstant(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::EVT::getSizeInBits(), llvm::SelectionDAG::getTargetConstant(), llvm::ISD::INTRINSIC_WO_CHAIN, llvm::SDValue::isUndef(), isZerosVector(), llvm::Offset, llvm::SmallVectorTemplateBase< T, bool >::push_back(), Swap, std::swap(), and llvm::V1.
◆ getAArch64Cmp()
|
static |
Definition at line 4471 of file AArch64ISelLowering.cpp.
References assert(), C(), llvm::cast(), changeIntCCToAArch64CC(), DL, llvm::dyn_cast(), emitComparison(), emitConjunction(), getCmpOperandFoldingProfit(), getCmpOrCmnOperandFoldingProfit(), getCondCode(), llvm::SelectionDAG::getConstant(), llvm::AArch64CC::getInvertedCondCode(), llvm::SelectionDAG::getNode(), llvm::ISD::getSetCCSwappedOperands(), llvm::SelectionDAG::getSignedConstant(), llvm::SelectionDAG::getValueType(), llvm::ConstantSDNode::getZExtValue(), llvm::isa(), llvm::AArch64_AM::isLegalArithImmed(), llvm::AArch64_AM::isLegalCmpImmed(), llvm::ConstantSDNode::isOne(), llvm::ConstantSDNode::isZero(), LHS, numberOfInstrToLoadImm(), RHS, llvm::ISD::SETGE, llvm::ISD::SETGT, llvm::ISD::SETLE, llvm::ISD::SETLT, llvm::ISD::SETNE, llvm::ISD::SETUGE, llvm::ISD::SETUGT, llvm::ISD::SETULE, llvm::ISD::SETULT, shouldBeAdjustedToZero(), llvm::ISD::SIGN_EXTEND_INREG, std::swap(), and llvm::ISD::ZEXTLOAD.
Referenced by performSetccAddFolding().
◆ getAArch64XALUOOp()
|
static |
Definition at line 4615 of file AArch64ISelLowering.cpp.
References assert(), DL, FlagsVT, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::SDValue::getValue(), llvm::SelectionDAG::getVTList(), llvm::AArch64CC::HS, LHS, llvm_unreachable, llvm::AArch64CC::LO, llvm::ISD::MUL, Mul, llvm::ISD::MULHS, llvm::ISD::MULHU, llvm::AArch64CC::NE, Opc, RHS, llvm::ISD::SADDO, llvm::ISD::SIGN_EXTEND, llvm::ISD::SMULO, llvm::ISD::SRA, llvm::ISD::SSUBO, llvm::ISD::TRUNCATE, llvm::ISD::UADDO, llvm::ISD::UMULO, llvm::ISD::USUBO, llvm::AArch64CC::VS, and llvm::ISD::ZERO_EXTEND.
Referenced by LowerXALUO().
◆ getAtomicLoad128Opcode()
|
static |
Definition at line 32674 of file AArch64ISelLowering.cpp.
References llvm::Acquire, llvm::AcquireRelease, assert(), llvm::ISD::ATOMIC_LOAD_ADD, llvm::ISD::ATOMIC_LOAD_AND, llvm::ISD::ATOMIC_LOAD_CLR, llvm::ISD::ATOMIC_LOAD_OR, llvm::ISD::ATOMIC_LOAD_SUB, llvm::ISD::ATOMIC_SWAP, llvm_unreachable, llvm::Monotonic, llvm::Release, and llvm::SequentiallyConsistent.
Referenced by ReplaceATOMIC_LOAD_128Results().
◆ getCmpOperandFoldingProfit()
Returns how profitable it is to fold a comparison's operand's shift and/or extension operations.
Definition at line 4421 of file AArch64ISelLowering.cpp.
References llvm::dyn_cast(), isSupportedExtend(), Opc, llvm::ISD::SHL, llvm::ISD::SRA, llvm::ISD::SRL, and uint64_t.
Referenced by getAArch64Cmp(), and getCmpOrCmnOperandFoldingProfit().
◆ getCmpOrCmnOperandFoldingProfit()
|
static |
Definition at line 4442 of file AArch64ISelLowering.cpp.
References getCmpOperandFoldingProfit(), and isCMN().
Referenced by getAArch64Cmp().
◆ getCondCode()
|
static |
Like SelectionDAG::getCondCode(), but for AArch64 condition codes.
Definition at line 3975 of file AArch64ISelLowering.cpp.
References CondCodeVT, and llvm::SelectionDAG::getConstant().
Referenced by carryFlagToValue(), emitConditionalComparison(), foldADCToCINC(), foldCSELOfCSEL(), getAArch64Cmp(), getPTest(), getSETCC(), LowerBRCOND(), LowerXALUO(), overflowFlagToValue(), performAddCSelIntoCSinc(), performANDORCSELCombine(), performANDSETCCCombine(), performBRCONDCombine(), performCSELCombine(), performRNDRCombine(), performSETCCCombine(), and reassociateCSELOperandsForCSE().
◆ getConstantLaneNumOfExtractHalfOperand()
Definition at line 5864 of file AArch64ISelLowering.cpp.
References C(), llvm::dyn_cast(), llvm::ISD::EXTRACT_VECTOR_ELT, llvm::SDNode::getOpcode(), llvm::SDNode::getOperand(), llvm::SDValue::getValueType(), llvm::EVT::getVectorNumElements(), and llvm::EVT::isFixedLengthVector().
◆ getContainerForFixedLengthVector()
|
static |
Definition at line 34073 of file AArch64ISelLowering.cpp.
References assert(), llvm::EVT::getSimpleVT(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::EVT::getVectorElementType(), llvm::EVT::isFixedLengthVector(), llvm::TargetLoweringBase::isTypeLegal(), llvm_unreachable, and llvm::MVT::SimpleTy.
Referenced by combineToVCPOP(), convertFixedMaskToScalableVector(), GenerateFixedLengthSVETBL(), getVCIXISDNodeWCHAIN(), getWideningInterleave(), isLegalVTForZvzipDeinterleavedOperand(), isLegalVTForZvzipInterleavedOperand(), lowerBUILD_VECTOR(), lowerBuildVectorOfConstants(), lowerBuildVectorViaDominantValues(), lowerBuildVectorViaVID(), lowerCONVERT_FROM_ARBITRARY_FP(), lowerCttzElts(), lowerFixedVectorSegLoadIntrinsics(), lowerFixedVectorSegStoreIntrinsics(), lowerFMAXIMUM_FMINIMUM(), lowerFP_TO_INT_SAT(), lowerScalarInsert(), lowerShuffleViaVRegSplitting(), lowerVECTOR_SHUFFLEAsVSlide1(), lowerVECTOR_SHUFFLEAsVSlidedown(), lowerVECTOR_SHUFFLEAsVSlideup(), lowerVectorFTRUNC_FCEIL_FFLOOR_FROUND(), LowerVectorMatch(), lowerVectorStrictFTRUNC_FCEIL_FFLOOR_FROUND(), lowerVectorXRINT_XROUND(), lowerZvzipVPAIR(), lowerZvzipVUNZIP(), lowerZvzipVZIP(), matchSplatAsGather(), performDUPCombine(), performFP_TO_INTCombine(), and processVCIXOperands().
◆ getCSETCondCode()
|
static |
Definition at line 23641 of file AArch64ISelLowering.cpp.
References llvm::AArch64CC::AL, llvm::isNullConstant(), llvm::isOneConstant(), and llvm::AArch64CC::NV.
Referenced by foldOverflowCheck(), performBRCONDCombine(), performExtendCombine(), and performSubWithBorrowCombine().
◆ getDUPLANEOp()
Definition at line 15897 of file AArch64ISelLowering.cpp.
References llvm_unreachable.
Referenced by performDUPCombine().
◆ getEstimate()
|
static |
Definition at line 14064 of file AArch64ISelLowering.cpp.
References llvm::EVT::getFltSemantics(), llvm::SelectionDAG::getNode(), llvm::SDValue::getValueType(), llvm::Log2_64_Ceil(), llvm::APFloatBase::semanticsPrecision(), and llvm::TargetLoweringBase::Unspecified.
◆ getExtFactor()
getExtFactor - Determine the adjustment factor for the position when generating an "extract from vector registers" instruction.
Definition at line 14791 of file AArch64ISelLowering.cpp.
References llvm::EVT::getSizeInBits(), and llvm::EVT::getVectorElementType().
Referenced by GeneratePerfectShuffle(), and llvm::AArch64TargetLowering::ReconstructShuffle().
◆ getGatherVecOpcode()
Definition at line 7490 of file AArch64ISelLowering.cpp.
References llvm::InnerAnalysisManagerProxy< AnalysisManagerT, IRUnitT, ExtraArgTs... >::Key.
Referenced by performGLD1Combine().
◆ getIntrinsicID()
Definition at line 9248 of file AArch64ISelLowering.cpp.
References llvm::ISD::INTRINSIC_WO_CHAIN, N, and llvm::Intrinsic::not_intrinsic.
Referenced by llvm::AMDGPURegisterBankInfo::applyMappingImpl(), collectBitParts(), FindPreallocatedCall(), llvm::AMDGPURegisterBankInfo::getInstrAlternativeMappingsIntrinsic(), llvm::AMDGPURegisterBankInfo::getInstrAlternativeMappingsIntrinsicWSideEffects(), llvm::AArch64RegisterBankInfo::getInstrMapping(), llvm::AMDGPURegisterBankInfo::getInstrMapping(), getMaxValueForSVECntIntrinsic(), llvm::VPRecipeBuilder::handleReplication(), llvm::SITargetLowering::isCanonicalized(), isFPIntrinsic(), isFPIntrinsic(), llvm::AArch64TargetLowering::isReassocProfitable(), IsSVECntIntrinsic(), llvm::MipsLegalizerInfo::legalizeIntrinsic(), performIntrinsicCombine(), llvm::SubsumingPositionIterator::SubsumingPositionIterator(), tryConvertSVEWideCompare(), llvm::SPIRVGlobalRegistry::updateAssignType(), and valueIsKnownNeverF32Denorm().
◆ getMaxValueForSVECntIntrinsic()
|
static |
Definition at line 21184 of file AArch64ISelLowering.cpp.
References getIntrinsicID(), and llvm::ElementCount::getScalable().
◆ getPackedSVEVectorVT() [1/2]
|
inlinestatic |
Definition at line 263 of file AArch64ISelLowering.cpp.
References llvm_unreachable.
◆ getPackedSVEVectorVT() [2/2]
Definition at line 238 of file AArch64ISelLowering.cpp.
References llvm::EVT::getSimpleVT(), llvm_unreachable, and llvm::MVT::SimpleTy.
Referenced by combineSVEReductionInt(), getSVEMultiVectorInfo(), lowerPredicateInterleaveWithPackedVectors(), LowerVectorMatch(), performDupLane128Combine(), and tryToReplaceScalarFPConversionWithSVE().
◆ getPairwiseOpForReduction()
|
static |
Returns the pairwise SVE2 op that could be used for a v2<ty> reduction.
Definition at line 34749 of file AArch64ISelLowering.cpp.
References llvm::ISD::VECREDUCE_SMAX, llvm::ISD::VECREDUCE_SMIN, llvm::ISD::VECREDUCE_UMAX, and llvm::ISD::VECREDUCE_UMIN.
Referenced by llvm::AArch64TargetLowering::shouldLowerReductionToSVE().
◆ getPredCounterWhileElementSize()
|
static |
Returns the element size of a predicate-as-counter while intrinsic.
Definition at line 25289 of file AArch64ISelLowering.cpp.
References llvm_unreachable.
Referenced by lowerPredCounterWhile().
◆ getPredCounterWhileOpcode()
|
static |
Returns the ISD opcode for a predicate-as-counter while intrinsic.
Definition at line 25333 of file AArch64ISelLowering.cpp.
References llvm_unreachable.
Referenced by lowerPredCounterWhile().
◆ getPredicatedReductionOpcode()
Returns the corresponding predicated SVE reduction opcode for a VECREDUCE_*.
Definition at line 34765 of file AArch64ISelLowering.cpp.
References llvm_unreachable, llvm::ISD::VECREDUCE_ADD, llvm::ISD::VECREDUCE_AND, llvm::ISD::VECREDUCE_FADD, llvm::ISD::VECREDUCE_FMAX, llvm::ISD::VECREDUCE_FMAXIMUM, llvm::ISD::VECREDUCE_FMIN, llvm::ISD::VECREDUCE_FMINIMUM, llvm::ISD::VECREDUCE_OR, llvm::ISD::VECREDUCE_SMAX, llvm::ISD::VECREDUCE_SMIN, llvm::ISD::VECREDUCE_UMAX, llvm::ISD::VECREDUCE_UMIN, and llvm::ISD::VECREDUCE_XOR.
◆ getPredicateForFixedLengthVector()
|
static |
Definition at line 34100 of file AArch64ISelLowering.cpp.
References assert(), DL, getPTrue(), llvm::EVT::getSimpleVT(), llvm::getSVEPredPatternFromNumElements(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::EVT::getVectorElementType(), llvm::EVT::getVectorNumElements(), llvm::EVT::isFixedLengthVector(), llvm::TargetLoweringBase::isTypeLegal(), llvm_unreachable, and llvm::MVT::SimpleTy.
Referenced by convertFixedMaskToScalableVector(), and getPredicateForVector().
◆ getPredicateForScalableVector()
|
static |
Definition at line 34135 of file AArch64ISelLowering.cpp.
References assert(), llvm::EVT::changeVectorElementType(), DL, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getContext(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::EVT::isScalableVector(), and llvm::TargetLoweringBase::isTypeLegal().
Referenced by getPredicateForVector().
◆ getPredicateForVector()
|
static |
Definition at line 34143 of file AArch64ISelLowering.cpp.
References DL, getPredicateForFixedLengthVector(), getPredicateForScalableVector(), and llvm::EVT::isFixedLengthVector().
Referenced by llvm::AArch64TargetLowering::LowerOperation(), and tryCombineExtendRShTrunc().
◆ getPredicateRegisterClass()
|
static |
Definition at line 14263 of file AArch64ISelLowering.cpp.
References llvm::EVT::isScalableVectorOf(), llvm_unreachable, Upa, Uph, and Upl.
◆ getPromotedVTForPredicate()
Definition at line 278 of file AArch64ISelLowering.cpp.
References assert(), llvm::EVT::getVectorMinNumElements(), llvm::EVT::isScalableVectorOf(), and llvm_unreachable.
◆ getPTest()
|
static |
Definition at line 25018 of file AArch64ISelLowering.cpp.
References llvm::AArch64CC::ANY_ACTIVE, assert(), Cond, DL, llvm::AArch64CC::FIRST_ACTIVE, getCondCode(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getContext(), llvm::SelectionDAG::getNode(), getSVEPredicateBitCast(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::TargetLoweringBase::getTypeToTransformTo(), llvm::SDValue::getValueType(), llvm::SelectionDAG::getZExtOrTrunc(), llvm::TargetLoweringBase::isTypeLegal(), isZeroingInactiveLanes(), llvm::AArch64CC::NONE_ACTIVE, and llvm::Test.
Referenced by performFirstTrueTestVectorCombine(), performIntrinsicCombine(), and performLastTrueTestVectorCombine().
◆ getPTrue()
|
inlinestatic |
Definition at line 6280 of file AArch64ISelLowering.cpp.
References DL, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::getNumElementsFromSVEPredPattern(), llvm::SelectionDAG::getSubtarget(), llvm::AArch64Subtarget::getSVEVectorSizeInBits(), llvm::SelectionDAG::getTargetConstant(), and llvm::EVT::getVectorMinNumElements().
Referenced by getPredicateForFixedLengthVector(), optimizeIncrementingWhile(), performMSTORECombine(), and performUnpackCombine().
◆ getPTrueAsCounter()
|
inlinestatic |
Definition at line 6299 of file AArch64ISelLowering.cpp.
References DL, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::EVT::getScalarSizeInBits(), llvm::ISD::INTRINSIC_WO_CHAIN, and llvm_unreachable.
Referenced by tryLowerMultiVectorLoad(), and tryLowerMultiVectorStore().
◆ getReducedGprRegisterClass()
|
static |
Definition at line 14293 of file AArch64ISelLowering.cpp.
References llvm::EVT::getFixedSizeInBits(), llvm::EVT::isScalarInteger(), llvm_unreachable, Uci, and Ucj.
◆ getReductionForOpcode()
Definition at line 31694 of file AArch64ISelLowering.cpp.
References llvm_unreachable, llvm::ISD::SMAX, llvm::ISD::SMIN, llvm::ISD::UMAX, llvm::ISD::UMIN, llvm::ISD::VECREDUCE_SMAX, llvm::ISD::VECREDUCE_SMIN, llvm::ISD::VECREDUCE_UMAX, and llvm::ISD::VECREDUCE_UMIN.
Referenced by performMINMAXCombine().
◆ getReductionSDNode()
|
static |
Definition at line 18309 of file AArch64ISelLowering.cpp.
References DL, llvm::SelectionDAG::getExtractVectorElt(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOperand(), llvm::SDValue::getSimpleValueType(), and llvm::SDValue::getValueType().
◆ getScaledOffsetForBitWidth()
|
static |
Definition at line 30481 of file AArch64ISelLowering.cpp.
References assert(), llvm::BitWidth, DL, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::Log2_32(), llvm::Offset, llvm::ISD::SHL, and llvm::ISD::SPLAT_VECTOR.
Referenced by performGatherLoadCombine(), and performScatterStoreCombine().
◆ getSETCC()
|
static |
Helper function to create 'CSET', which is equivalent to 'CSINC <Wd>, WZR, WZR, invert(<cond>)'.
Definition at line 14333 of file AArch64ISelLowering.cpp.
References DL, getCondCode(), llvm::SelectionDAG::getConstant(), and llvm::SelectionDAG::getNode().
Referenced by combineAnd(), combineCMov(), combineM68kSetCC(), combineOr(), combineSetCC(), combineSubSetcc(), combineVECREDUCE_LOGIC(), combineVectorSizedSetCCEquality(), combineX86SetCC(), foldXor1SetCC(), LowerADDSUBO_CARRY(), llvm::X86TargetLowering::LowerAsmOutputForConstraint(), LowerCMP_SWAP(), LowerINTRINSIC_W_CHAIN(), LowerPARITY(), LowerXALUO(), perfomPextFirstTrueVectorCombine(), llvm::X86TargetLowering::ReplaceNodeResults(), and llvm::X86TargetLowering::SimplifyDemandedBitsForTargetNode().
◆ getSignExtendedGatherOpcode()
Definition at line 7514 of file AArch64ISelLowering.cpp.
References llvm_unreachable.
Referenced by performGLD1Combine().
◆ getSMECallAttrs()
|
static |
Definition at line 10128 of file AArch64ISelLowering.cpp.
References llvm::TargetLowering::CallLoweringInfo::Callee, llvm::TargetLowering::CallLoweringInfo::CB, llvm::dyn_cast(), and llvm::SMEAttrs::Normal.
◆ getSMToggleCondition()
|
static |
Definition at line 10458 of file AArch64ISelLowering.cpp.
References llvm::AArch64SME::Always, CallAttrs, llvm::AArch64SME::IfCallerIsNonStreaming, llvm::AArch64SME::IfCallerIsStreaming, and llvm_unreachable.
◆ getStructuredLoadFunction()
|
static |
Definition at line 19488 of file AArch64ISelLowering.cpp.
References assert(), and llvm::Intrinsic::getOrInsertDeclaration().
Referenced by llvm::AArch64TargetLowering::lowerDeinterleaveIntrinsicToLoad(), and llvm::AArch64TargetLowering::lowerInterleavedLoad().
◆ getStructuredStoreFunction()
|
static |
Definition at line 19506 of file AArch64ISelLowering.cpp.
References assert(), and llvm::Intrinsic::getOrInsertDeclaration().
Referenced by llvm::AArch64TargetLowering::lowerInterleavedStore(), and llvm::AArch64TargetLowering::lowerInterleaveIntrinsicToStore().
◆ getSVEContainerIRType()
|
static |
Definition at line 19460 of file AArch64ISelLowering.cpp.
References llvm::ScalableVectorType::get(), llvm::Type::getBFloatTy(), llvm::Type::getContext(), llvm::Type::getDoubleTy(), llvm::VectorType::getElementType(), llvm::Type::getFloatTy(), llvm::Type::getHalfTy(), llvm::Type::getInt16Ty(), llvm::Type::getInt32Ty(), llvm::Type::getInt64Ty(), llvm::Type::getInt8Ty(), and llvm_unreachable.
Referenced by llvm::AArch64TargetLowering::lowerInterleavedLoad(), and llvm::AArch64TargetLowering::lowerInterleavedStore().
◆ getSVEContainerType()
Definition at line 26285 of file AArch64ISelLowering.cpp.
References assert(), llvm::EVT::getSimpleVT(), llvm::EVT::isSimple(), llvm_unreachable, and llvm::MVT::SimpleTy.
Referenced by performGatherLoadCombine(), performLD1Combine(), performScatterStoreCombine(), and performST1Combine().
◆ getSVEMultiVectorInfo()
|
static |
Definition at line 7939 of file AArch64ISelLowering.cpp.
References assert(), getPackedSVEVectorVT(), llvm::TypeSize::getScalable(), llvm::EVT::getSimpleVT(), llvm::MVT::getSizeInBits(), llvm::MVT::getVectorElementType(), and Size.
Referenced by tryLowerMultiVectorLoad(), and tryLowerMultiVectorStore().
◆ getSVEOpcodeForFPToInt()
Definition at line 35595 of file AArch64ISelLowering.cpp.
References llvm::ISD::FP_TO_SINT, llvm::ISD::FP_TO_SINT_SAT, llvm::ISD::FP_TO_UINT, llvm::ISD::FP_TO_UINT_SAT, llvm_unreachable, Opc, llvm::ISD::STRICT_FP_TO_SINT, and llvm::ISD::STRICT_FP_TO_UINT.
◆ getSVEPredicateBitCast()
|
static |
Definition at line 6413 of file AArch64ISelLowering.cpp.
References llvm::ISD::AND, assert(), llvm::EVT::bitsGT(), DL, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::EVT::getVectorElementType(), llvm::ISD::INTRINSIC_WO_CHAIN, llvm::EVT::isScalableVector(), llvm::TargetLoweringBase::isTypeLegal(), and isZeroingInactiveLanes().
Referenced by getPTest(), and optimizeBrk().
◆ getTestBitOperand()
|
static |
Definition at line 29874 of file AArch64ISelLowering.cpp.
References llvm::ISD::AND, llvm::ISD::ANY_EXTEND, C(), llvm::dyn_cast(), getTestBitOperand(), llvm::ISD::SHL, llvm::ISD::SRA, llvm::ISD::SRL, llvm::ISD::TRUNCATE, and llvm::ISD::XOR.
Referenced by getTestBitOperand(), and performTBZCombine().
◆ getVectorBitwiseReduce()
|
static |
Definition at line 18316 of file AArch64ISelLowering.cpp.
References llvm::ISD::AND, llvm::ISD::ANY_EXTEND, assert(), llvm::EVT::changeVectorElementType(), DL, llvm::SelectionDAG::getAnyExtOrTrunc(), llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getContext(), llvm::EVT::getIntegerVT(), llvm::MVT::getIntegerVT(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::EVT::getSizeInBits(), llvm::SDValue::getValueSizeInBits(), llvm::SDValue::getValueType(), getVectorBitwiseReduce(), llvm::EVT::getVectorElementType(), llvm::EVT::getVectorNumElements(), llvm::Hi, llvm::EVT::isFixedLengthVector(), llvm::EVT::isPow2VectorType(), llvm_unreachable, llvm::Lo, llvm::ISD::OR, llvm::ISD::SETCC, llvm::ISD::SIGN_EXTEND, llvm::SelectionDAG::SplitVector(), llvm::ISD::SRL, llvm::ISD::VECREDUCE_ADD, llvm::ISD::VECREDUCE_AND, llvm::ISD::VECREDUCE_OR, llvm::ISD::VECREDUCE_UMAX, llvm::ISD::VECREDUCE_UMIN, llvm::ISD::VECREDUCE_XOR, and llvm::ISD::XOR.
Referenced by getVectorBitwiseReduce(), and performVecReduceBitwiseCombine().
◆ getVShiftImm()
getVShiftImm - Check if this is a valid build_vector for the immediate operand of a vector shift operation, where all the elements of the build_vector must have the same constant integer value.
Definition at line 18058 of file AArch64ISelLowering.cpp.
References llvm::ISD::BITCAST, llvm::dyn_cast(), llvm::APInt::getSExtValue(), and llvm::BuildVectorSDNode::isConstantSplat().
Referenced by isVShiftLImm(), isVShiftLImm(), isVShiftRImm(), and isVShiftRImm().
◆ getZAMarkerForCall()
|
static |
Returns the ZA state save/use marker node necessary for CallAttrs.
Definition at line 10511 of file AArch64ISelLowering.cpp.
References CallAttrs.
◆ hasCheapXorImmediateThatNeedsCondCmpReg()
Definition at line 12713 of file AArch64ISelLowering.cpp.
References llvm::BitWidth, C(), llvm::dyn_cast(), Imm, isLegalCondCmpImmediate(), and llvm::AArch64_AM::isLogicalImmediate().
Referenced by performOrXorChainCombine().
◆ hasLegalCmpImmediate()
Definition at line 12688 of file AArch64ISelLowering.cpp.
References C(), llvm::dyn_cast(), llvm::AArch64_AM::isLegalCmpImmed(), LHS, and RHS.
Referenced by isOrXorChain(), and performOrXorChainCombine().
◆ hasLegalCondCmpImmediate()
Definition at line 12699 of file AArch64ISelLowering.cpp.
References C(), llvm::dyn_cast(), isLegalCondCmpImmediate(), LHS, and RHS.
Referenced by performOrXorChainCombine().
◆ hasNearbyPairedStore()
| bool hasNearbyPairedStore | ( | Iter | It, |
| Iter | End, | ||
| Value * | Ptr, | ||
| const DataLayout & | DL ) |
Definition at line 19667 of file AArch64ISelLowering.cpp.
References DL, llvm::dyn_cast(), llvm::APInt::sextOrTrunc(), and llvm::Value::stripAndAccumulateConstantOffsets().
Referenced by llvm::AArch64TargetLowering::lowerInterleavedStore().
◆ hasPairwiseAdd()
Definition at line 22437 of file AArch64ISelLowering.cpp.
References llvm::ISD::ADD, llvm::ISD::FADD, and llvm::ISD::STRICT_FADD.
Referenced by performExtractVectorEltCombine().
◆ hasSVEMultiVectorOps()
|
static |
Definition at line 22564 of file AArch64ISelLowering.cpp.
References llvm::AArch64Subtarget::isStreaming(), and llvm::AArch64Subtarget::isSVEorStreamingSVEAvailable().
Referenced by perfomPextFirstTrueVectorCombine().
◆ isAddSubSExt()
|
static |
Definition at line 5925 of file AArch64ISelLowering.cpp.
References llvm::ISD::ADD, llvm::SDNode::hasOneUse(), isSignExtended(), N, and llvm::ISD::SUB.
Referenced by LowerMUL(), and selectUmullSmull().
◆ isAddSubZExt()
|
static |
Definition at line 5936 of file AArch64ISelLowering.cpp.
References llvm::ISD::ADD, llvm::SDNode::hasOneUse(), isZeroExtended(), N, and llvm::ISD::SUB.
Referenced by LowerMUL(), and selectUmullSmull().
◆ isAllActivePredicate()
|
static |
Definition at line 16686 of file AArch64ISelLowering.cpp.
References llvm::ISD::isConstantSplatVectorAllOnes(), and N.
Referenced by convertMergedOpToPredOp(), foldCSELofLASTB(), instCombineSVECmpNE(), instCombineSVEDup(), instCombineSVELD1(), instCombineSVESel(), instCombineSVESrshl(), instCombineSVEST1(), instCombineSVEUxt(), instCombineSVEVectorBinOp(), instCombineZExtSVECmpNE(), performSetccMergeZeroCombine(), performSVEAndCombine(), performVSelectCombine(), simplifySVEIntrinsic(), and tryLowerToSLI().
◆ isAllConstantBuildVector()
Definition at line 16661 of file AArch64ISelLowering.cpp.
References llvm::dyn_cast(), llvm::SDNode::getOperand(), llvm::SDNode::getValueType(), llvm::EVT::getVectorNumElements(), llvm::ConstantSDNode::getZExtValue(), and uint64_t.
Referenced by isPow2Splat().
◆ isAllInactivePredicate()
Definition at line 16678 of file AArch64ISelLowering.cpp.
References llvm::ISD::isConstantSplatVectorAllZeros(), and N.
Referenced by performVSelectCombine().
◆ isCheapToExtend()
Definition at line 25796 of file AArch64ISelLowering.cpp.
References llvm::ISD::isConstantSplatVectorAllZeros(), llvm::ISD::LOAD, llvm::ISD::MLOAD, and N.
Referenced by performSignExtendSetCCCombine().
◆ isCMN()
|
static |
Definition at line 4013 of file AArch64ISelLowering.cpp.
References llvm::SelectionDAG::isKnownNeverZero(), llvm::isNullConstant(), isSafeSignedCMN(), and llvm::ISD::SUB.
Referenced by emitComparison(), emitConditionalComparison(), and getCmpOrCmnOperandFoldingProfit().
◆ isCMP()
Definition at line 23635 of file AArch64ISelLowering.cpp.
Referenced by foldCSELOfCSEL(), foldOverflowCheck(), and performBRCONDCombine().
◆ isConcatMask()
Definition at line 15607 of file AArch64ISelLowering.cpp.
References E(), llvm::EVT::getSizeInBits(), llvm::EVT::getVectorNumElements(), I, and llvm::Offset.
Referenced by llvm::AArch64TargetLowering::isShuffleMaskLegal(), and tryFormConcatFromShuffle().
◆ isConstantSplatVectorMaskForType()
Definition at line 22145 of file AArch64ISelLowering.cpp.
References llvm::dyn_cast(), llvm::EVT::getSimpleVT(), llvm::EVT::getVectorElementType(), llvm::EVT::isSimple(), N, llvm::MVT::SimpleTy, llvm::ISD::SPLAT_VECTOR, and uint64_t.
◆ isDeinterleave4ForWideningUToFP()
Definition at line 31367 of file AArch64ISelLowering.cpp.
References llvm::all_of(), llvm::Use::getUser(), N, llvm::ISD::UINT_TO_FP, and Users.
Referenced by performVectorDeinterleaveCombine().
◆ isEligibleForSmallVectorLoadOpt()
|
static |
Helper function to check if a small vector load can be optimized.
Definition at line 7432 of file AArch64ISelLowering.cpp.
References llvm::details::FixedOrScalableQuantity< LeafTy, ValueTy >::getFixedValue(), llvm::EVT::getStoreSize(), and llvm::AArch64Subtarget::isNeonAvailable().
Referenced by tryLowerSmallVectorExtLoad().
◆ isEquivalentMaskless()
|
static |
Definition at line 28842 of file AArch64ISelLowering.cpp.
References llvm::AArch64CC::AL, llvm::AArch64CC::EQ, llvm::AArch64CC::GE, llvm::AArch64CC::GT, llvm::AArch64CC::HI, llvm::AArch64CC::HS, llvm::AArch64CC::Invalid, llvm::AArch64CC::LE, llvm::AArch64CC::LO, llvm::AArch64CC::LS, llvm::AArch64CC::LT, llvm::AArch64CC::MI, llvm::AArch64CC::NE, llvm::AArch64CC::NV, llvm::AArch64CC::PL, llvm::ISD::SEXTLOAD, llvm::AArch64CC::VC, and llvm::AArch64CC::VS.
Referenced by performCONDCombine().
◆ isEssentiallyExtractHighSubvector()
Definition at line 23212 of file AArch64ISelLowering.cpp.
References llvm::ISD::BITCAST, llvm::ISD::EXTRACT_SUBVECTOR, and N.
Referenced by performAddSubLongCombine(), reassociateAddSubLong(), tryCombineLongOpWithDup(), and tryCombineMULLWithUZP1().
◆ isExtendedBUILD_VECTOR()
|
static |
Definition at line 5877 of file AArch64ISelLowering.cpp.
References llvm::ISD::BUILD_VECTOR, C(), llvm::dyn_cast(), llvm::EVT::getScalarSizeInBits(), llvm::isIntN(), isSigned(), llvm::isUIntN(), and N.
Referenced by isSignExtended(), isSignExtended(), isZeroExtended(), and isZeroExtended().
◆ isExtendOrShiftOperand()
Definition at line 23998 of file AArch64ISelLowering.cpp.
References llvm::ISD::AND, llvm::cast(), llvm::dyn_cast(), llvm::ConstantSDNode::getZExtValue(), llvm::isa(), llvm::ISD::isExtOpcode(), N, llvm::ISD::SHL, llvm::ISD::SIGN_EXTEND_INREG, llvm::ISD::SRA, llvm::ISD::SRL, and uint64_t.
Referenced by performAddCombineSubShift().
◆ isEXTMask()
Definition at line 15345 of file AArch64ISelLowering.cpp.
References llvm::find_if(), llvm::EVT::getVectorNumElements(), llvm::APInt::getZExtValue(), Imm, and llvm::APInt::logBase2().
Referenced by llvm::AArch64TargetLowering::isShuffleMaskLegal().
◆ isEXTMaskWithSplat()
|
static |
Definition at line 15513 of file AArch64ISelLowering.cpp.
References llvm::EVT::getVectorNumElements(), I, and Imm.
◆ isHalvingTruncateAndConcatOfLegalIntScalableType()
Definition at line 26693 of file AArch64ISelLowering.cpp.
References llvm::SDNode::getValueType(), and N.
Referenced by performUzpCombine().
◆ isINSMask()
Definition at line 15568 of file AArch64ISelLowering.cpp.
Referenced by llvm::AArch64TargetLowering::isShuffleMaskLegal().
◆ isIntImmediate()
Definition at line 2770 of file AArch64ISelLowering.cpp.
References C(), llvm::dyn_cast(), Imm, N, and uint64_t.
◆ isLane0KnownActive()
Definition at line 30967 of file AArch64ISelLowering.cpp.
References isLane0KnownActive(), llvm::isOneConstant(), and llvm::ISD::SPLAT_VECTOR.
Referenced by isLane0KnownActive(), and performPTestFirstCombine().
◆ isLanes1toNKnownZero()
Definition at line 30942 of file AArch64ISelLowering.cpp.
Referenced by removeRedundantInsertVectorElt().
◆ isLegalCondCmpImmediate()
Definition at line 12681 of file AArch64ISelLowering.cpp.
References Imm.
Referenced by hasCheapXorImmediateThatNeedsCondCmpReg(), and hasLegalCondCmpImmediate().
◆ isLegalNTStore()
|
static |
Definition at line 7846 of file AArch64ISelLowering.cpp.
References DL, llvm::dyn_cast(), and llvm::isPowerOf2_64().
Referenced by llvm::AArch64TargetLowering::lowerInterleavedStore(), and LowerNTStore().
◆ isLoadOrMultipleLoads()
|
static |
Definition at line 24245 of file AArch64ISelLowering.cpp.
References B(), llvm::ISD::BUILD_VECTOR, llvm::cast(), llvm::ISD::CONCAT_VECTORS, llvm::dyn_cast(), llvm::SDValue::getNumOperands(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDNode::hasOneUse(), llvm::SDValue::hasOneUse(), I, llvm::peekThroughOneUseBitcasts(), llvm::SmallVectorTemplateBase< T, bool >::push_back(), and llvm::ISD::VECTOR_SHUFFLE.
Referenced by areLoadedOffsetButOtherwiseSame(), and performExtBinopLoadFold().
◆ isMergePassthruOpcode()
Definition at line 329 of file AArch64ISelLowering.cpp.
References Opc.
Referenced by performVSelectCombine().
◆ isNegatedInteger()
Definition at line 23493 of file AArch64ISelLowering.cpp.
References llvm::isNullConstant(), and llvm::ISD::SUB.
Referenced by performNegCSelCombine().
◆ isNVCastToHalfWidthElements()
|
static |
Definition at line 26774 of file AArch64ISelLowering.cpp.
References llvm::ISD::BITCAST, and llvm::AArch64Subtarget::isLittleEndian().
Referenced by performUzpCombine().
◆ isOneVector()
Definition at line 3806 of file AArch64ISelLowering.cpp.
References llvm::isOneConstant(), and llvm::isOneOrOneSplat().
Referenced by performSubNegAndOneCombine().
◆ isOnlyUsedAsMemoryAddress()
Definition at line 21489 of file AArch64ISelLowering.cpp.
References llvm::ISD::ADD, assert(), llvm::cast(), llvm::Use::getOperandNo(), llvm::Use::getUser(), llvm::ISD::LOAD, llvm::Load, N, llvm::ISD::STORE, llvm::Store, and llvm::ISD::SUB.
Referenced by performMulCombine().
◆ isOpcWithIntImmediate()
Definition at line 2790 of file AArch64ISelLowering.cpp.
References Imm, isIntImmediate(), N, Opc, and uint64_t.
◆ isOrXorChain()
|
static |
Definition at line 12730 of file AArch64ISelLowering.cpp.
References llvm::SelectionDAG::getConstant(), hasLegalCmpImmediate(), isOrXorChain(), llvm::EVT::isScalarInteger(), MaxXors, N, llvm::ISD::OR, llvm::ISD::XOR, and llvm::ISD::ZERO_EXTEND.
Referenced by isOrXorChain(), and performOrXorChainCombine().
◆ isPackedPredicateType()
|
inlinestatic |
Definition at line 307 of file AArch64ISelLowering.cpp.
References assert(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::TargetLoweringBase::isTypeLegal(), and llvm::EVT::isVector().
Referenced by isUnpackedType().
◆ isPackedVectorType()
|
inlinestatic |
Returns true if VT's elements occupy the lowest bit positions of its associated register class without any intervening space.
For example, nxv2f16, nxv4f16 and nxv8f16 are legal types that belong to the same register class, but only nxv8f16 can be treated as a packed vector.
Definition at line 300 of file AArch64ISelLowering.cpp.
References assert(), llvm::details::FixedOrScalableQuantity< LeafTy, ValueTy >::getKnownMinValue(), llvm::EVT::getSizeInBits(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::EVT::isFixedLengthVector(), llvm::TargetLoweringBase::isTypeLegal(), llvm::EVT::isVector(), and llvm::AArch64::SVEBitsPerBlock.
Referenced by isUnpackedType().
◆ isPassedInFPR()
Definition at line 9355 of file AArch64ISelLowering.cpp.
References llvm::EVT::isFixedLengthVector(), llvm::EVT::isFloatingPoint(), and llvm::EVT::isScalableVector().
◆ isPow2Splat()
Definition at line 17921 of file AArch64ISelLowering.cpp.
References llvm::ISD::BUILD_VECTOR, llvm::isa(), isAllConstantBuildVector(), llvm::isPowerOf2_64(), llvm::ISD::SPLAT_VECTOR, and uint64_t.
◆ isPredicateCCSettingOp()
Definition at line 22452 of file AArch64ISelLowering.cpp.
References llvm::ISD::GET_ACTIVE_LANE_MASK, llvm::ISD::INTRINSIC_WO_CHAIN, N, and llvm::ISD::SETCC.
Referenced by performFirstTrueTestVectorCombine().
◆ isSafeSignedCMN()
|
static |
Definition at line 3987 of file AArch64ISelLowering.cpp.
References llvm::SelectionDAG::computeKnownBits(), llvm::KnownBits::getSignedMinValue(), and llvm::APInt::isMinSignedValue().
◆ isSetCC()
|
static |
Check whether or not Op is a SET_CC operation, either a generic or an AArch64 lowered one.
SetCCInfo is filled accordingly.
- Postcondition
- SetCCInfo is meanginfull only when this function returns true.
- Returns
- True when Op is a kind of SET_CC operation.
Definition at line 23256 of file AArch64ISelLowering.cpp.
References SetCCInfo::AArch64, llvm::cast(), AArch64SetCCInfo::CC, GenericSetCCInfo::CC, AArch64SetCCInfo::Cmp, llvm::dyn_cast(), SetCCInfo::Generic, llvm::AArch64CC::getInvertedCondCode(), llvm::ConstantSDNode::isOne(), llvm::ConstantSDNode::isZero(), GenericSetCCInfo::Opnd0, GenericSetCCInfo::Opnd1, llvm::ISD::SETCC, and std::swap().
Referenced by isSetCCOrZExtSetCC().
◆ isSetCCOrZExtSetCC()
|
static |
Definition at line 23299 of file AArch64ISelLowering.cpp.
References isSetCC(), and llvm::ISD::ZERO_EXTEND.
Referenced by performSetccAddFolding().
◆ isSignExtended()
|
static |
Definition at line 5913 of file AArch64ISelLowering.cpp.
References llvm::ISD::ANY_EXTEND, isExtendedBUILD_VECTOR(), N, and llvm::ISD::SIGN_EXTEND.
Referenced by isAddSubSExt(), isAddSubSExt(), LowerMUL(), performMulCombine(), selectUmullSmull(), and signExtendOperandIfUnknown().
◆ isSignExtInReg()
Definition at line 29788 of file AArch64ISelLowering.cpp.
References llvm::BitWidth.
Referenced by performSetccMergeZeroCombine().
◆ isSimpleScalableLoadOrStore()
|
static |
Definition at line 7963 of file AArch64ISelLowering.cpp.
References llvm::MemSDNode::getMemoryVT(), llvm::LSBaseSDNode::getOffset(), llvm::EVT::isScalableVector(), llvm::EVT::isSimple(), llvm::MemSDNode::isSimple(), llvm::SDValue::isUndef(), and llvm::LSBaseSDNode::isUnindexed().
Referenced by tryLowerMultiVectorLoad(), and tryLowerMultiVectorStore().
◆ isSingletonEXTMask()
Definition at line 15189 of file AArch64ISelLowering.cpp.
References llvm::EVT::getVectorNumElements(), and Imm.
Referenced by llvm::AArch64TargetLowering::isShuffleMaskLegal().
◆ isSlideWithZerosMask()
|
static |
Definition at line 15445 of file AArch64ISelLowering.cpp.
References checkLaneSlide(), llvm::ShuffleVectorSDNode::commuteMask(), llvm::SDValue::getNode(), llvm::EVT::getScalarSizeInBits(), llvm::EVT::getSizeInBits(), llvm::EVT::getVectorNumElements(), llvm::ISD::isBuildVectorAllZeros(), and llvm::V1.
◆ isSupportedExtend()
Definition at line 4407 of file AArch64ISelLowering.cpp.
References llvm::ISD::AND, llvm::dyn_cast(), llvm::ISD::SIGN_EXTEND_INREG, and uint64_t.
Referenced by getCmpOperandFoldingProfit().
◆ IsSVECntIntrinsic()
Definition at line 21169 of file AArch64ISelLowering.cpp.
References getIntrinsicID(), and llvm::SDValue::getNode().
Referenced by performMulCombine().
◆ isUnpackedType()
|
inlinestatic |
Returns true if the conceptual representation for VT does not map directly to its physical register representation, meaning there are gaps between elements in the register.
In practice, the vector elements will be strided by a power of two and placed starting from lane 0. For example, nxv8i1 or nxv2f32 are unpacked types.
- Precondition
- VT is a legal type.
Definition at line 320 of file AArch64ISelLowering.cpp.
References assert(), isPackedPredicateType(), isPackedVectorType(), and llvm::EVT::isScalableVector().
Referenced by performConcatVectorsCombine().
◆ isValidImmForSVEVecImmAddrMode() [1/2]
Check if the value of Offset represents a valid immediate for the SVE gather load/prefetch and scatter store instructiona with vector base and immediate offset addressing mode:
[<Zn>.[S|D]{, #<imm>}]
where <imm> = sizeof(<T>) * k, for k = 0, 1, ..., 31.
Definition at line 30519 of file AArch64ISelLowering.cpp.
References llvm::dyn_cast(), llvm::ConstantSDNode::getZExtValue(), isValidImmForSVEVecImmAddrMode(), and llvm::Offset.
◆ isValidImmForSVEVecImmAddrMode() [2/2]
Check if the value of OffsetInBytes can be used as an immediate for the gather load/prefetch and scatter store instructions with vector base and immediate offset addressing mode:
[<Zn>.[S|D]{, #<imm>}]
where <imm> = sizeof(<T>) * k, for k = 0, 1, ..., 31.
Definition at line 30499 of file AArch64ISelLowering.cpp.
Referenced by combineSVEPrefetchVecBaseImmOff(), isValidImmForSVEVecImmAddrMode(), performGatherLoadCombine(), and performScatterStoreCombine().
◆ isVectorizedBinOp()
Definition at line 2778 of file AArch64ISelLowering.cpp.
Referenced by performConcatVectorsCombine().
◆ isVShiftLImm()
isVShiftLImm - Check if this is a valid build_vector for the immediate operand of a vector shift left operation.
That value must be in the range: 0 <= Value < ElementBits for a left shift; or 0 <= Value <= ElementBits for a long left shift.
Definition at line 18078 of file AArch64ISelLowering.cpp.
References assert(), llvm::EVT::getScalarSizeInBits(), getVShiftImm(), and llvm::EVT::isVector().
Referenced by LowerShift(), llvm::ARMTargetLowering::PerformIntrinsicCombine(), and PerformShiftCombine().
◆ isVShiftRImm()
isVShiftRImm - Check if this is a valid build_vector for the immediate operand of a vector shift right operation.
The value must be in the range: 1 <= Value <= ElementBits for a right shift; or
Definition at line 18089 of file AArch64ISelLowering.cpp.
References assert(), llvm::EVT::getScalarSizeInBits(), getVShiftImm(), and llvm::EVT::isVector().
Referenced by LowerShift(), llvm::ARMTargetLowering::PerformIntrinsicCombine(), and PerformShiftCombine().
◆ isWideDUPMask()
Check if a vector shuffle corresponds to a DUP instructions with a larger element width than the vector lane type.
If that is the case the function returns true and writes the value of the DUP instruction lane operand into DupLaneOp
Definition at line 15272 of file AArch64ISelLowering.cpp.
References assert(), llvm::SmallVectorTemplateCommon< T, typename >::begin(), BlockSize, llvm::SmallVectorTemplateCommon< T, typename >::end(), llvm::find_if(), llvm::EVT::getScalarSizeInBits(), llvm::EVT::getSizeInBits(), llvm::EVT::getVectorNumElements(), and I.
◆ isZeroExtended()
|
static |
Definition at line 5919 of file AArch64ISelLowering.cpp.
References llvm::ISD::ANY_EXTEND, isExtendedBUILD_VECTOR(), N, and llvm::ISD::ZERO_EXTEND.
Referenced by isAddSubZExt(), isAddSubZExt(), LowerMUL(), performMulCombine(), and selectUmullSmull().
◆ isZeroingInactiveLanes()
Definition at line 384 of file AArch64ISelLowering.cpp.
References llvm::ISD::GET_ACTIVE_LANE_MASK, llvm::ISD::INTRINSIC_WO_CHAIN, and llvm::ISD::SPLAT_VECTOR.
Referenced by getPTest(), and getSVEPredicateBitCast().
◆ isZerosVector()
isZerosVector - Check whether SDNode N is a zero-filled vector.
Definition at line 3791 of file AArch64ISelLowering.cpp.
References llvm::ISD::BITCAST, llvm::ISD::isConstantSplatVectorAllZeros(), llvm::isNullConstant(), llvm::isNullFPConstant(), and N.
Referenced by GenerateTBL(), performAddDotCombine(), performSETCCCombine(), performSetccMergeZeroCombine(), performSetCCPunpkCombine(), performSubNegAndOneCombine(), performUnpackCombine(), and performVectorDeinterleaveCombine().
◆ legalizeSVEGatherPrefetchOffsVec()
|
static |
Legalize the gather prefetch (scalar + vector addressing mode) when the offset vector is an unpacked 32-bit scalable vector. The other cases (Offset != nxv2i32) do not need legalization.
Definition at line 30898 of file AArch64ISelLowering.cpp.
References AbstractManglingParser< Derived, Alloc >::Ops, llvm::ISD::ANY_EXTEND, DL, llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getVTList(), N, and llvm::Offset.
Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().
◆ lookThroughSignExtension()
Definition at line 12112 of file AArch64ISelLowering.cpp.
References llvm::cast(), llvm::EVT::getFixedSizeInBits(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDValue::getValueSizeInBits(), llvm::SDNode::getValueType(), llvm::ISD::SIGN_EXTEND, and llvm::ISD::SIGN_EXTEND_INREG.
◆ LowerADDRSPACECAST()
|
static |
Definition at line 7811 of file AArch64ISelLowering.cpp.
References assert(), llvm::cast(), DL, llvm::SelectionDAG::getAnyExtOrTrunc(), llvm::SelectionDAG::getNode(), llvm::TargetMachine::getPointerSize(), llvm::SelectionDAG::getTargetConstant(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::TargetLoweringBase::getTargetMachine(), llvm::SelectionDAG::getZeroExtendInReg(), N, llvm::ARM64AS::PTR32_SPTR, llvm::ARM64AS::PTR32_UPTR, llvm::ISD::SIGN_EXTEND, and llvm::ISD::ZERO_EXTEND.
Referenced by llvm::AArch64TargetLowering::LowerOperation(), llvm::X86TargetLowering::LowerOperation(), and llvm::X86TargetLowering::ReplaceNodeResults().
◆ lowerADDSUBO_CARRY()
|
static |
Definition at line 4826 of file AArch64ISelLowering.cpp.
References carryFlagToValue(), DL, FlagsVT, llvm::SelectionDAG::getMergeValues(), llvm::SelectionDAG::getNode(), llvm::SDValue::getValue(), llvm::SelectionDAG::getVTList(), overflowFlagToValue(), and valueToCarryFlag().
Referenced by llvm::AArch64TargetLowering::LowerOperation().
◆ LowerBRCOND()
|
static |
Definition at line 8321 of file AArch64ISelLowering.cpp.
References Cond, DL, emitConjunction(), getCondCode(), and llvm::SelectionDAG::getNode().
Referenced by llvm::AArch64TargetLowering::LowerOperation().
◆ LowerFLDEXP()
|
static |
Definition at line 8375 of file AArch64ISelLowering.cpp.
References llvm::EVT::changeVectorElementType(), DL, llvm::ISD::FP_EXTEND, llvm::ISD::FP_ROUND, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getContext(), llvm::SelectionDAG::getExtractVectorElt(), llvm::SelectionDAG::getInsertVectorElt(), llvm::SelectionDAG::getIntPtrConstant(), llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getPOISON(), llvm::SelectionDAG::getTargetConstant(), llvm::ISD::INTRINSIC_WO_CHAIN, llvm::ISD::SIGN_EXTEND, and X.
Referenced by llvm::AArch64TargetLowering::LowerOperation(), and llvm::X86TargetLowering::LowerOperation().
◆ LowerFunnelShift()
|
static |
Definition at line 8339 of file AArch64ISelLowering.cpp.
References DL, llvm::dyn_cast(), llvm::ISD::FSHL, llvm::ISD::FSHR, llvm::SelectionDAG::getConstant(), llvm::MVT::getFixedSizeInBits(), llvm::SelectionDAG::getNode(), and llvm::SDValue::getValueType().
Referenced by llvm::AArch64TargetLowering::LowerOperation(), and llvm::X86TargetLowering::LowerOperation().
◆ lowerIntNeonIntrinsic()
|
static |
Definition at line 4851 of file AArch64ISelLowering.cpp.
References assert(), DL, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getNode(), I, and llvm::SmallVectorTemplateBase< T, bool >::push_back().
◆ LowerNTStore()
|
static |
Definition at line 7888 of file AArch64ISelLowering.cpp.
References assert(), DL, llvm::MemSDNode::getAlign(), llvm::SelectionDAG::getContext(), llvm::SelectionDAG::getDataLayout(), llvm::SelectionDAG::getExtractSubvector(), llvm::EVT::getHalfNumVectorElementsVT(), llvm::SelectionDAG::getMemIntrinsicNode(), llvm::MemSDNode::getMemOperand(), llvm::MemSDNode::getMemoryVT(), llvm::EVT::getScalarSizeInBits(), llvm::EVT::getSizeInBits(), llvm::EVT::getTypeForEVT(), llvm::StoreSDNode::getValue(), llvm::EVT::getVectorElementCount(), llvm::SelectionDAG::getVTList(), llvm::Hi, isLegalNTStore(), llvm::DataLayout::isLittleEndian(), llvm::MemSDNode::isNonTemporal(), llvm::EVT::isVector(), llvm::Lo, and llvm::SelectionDAG::shouldOptForSize().
◆ lowerPredCounterWhile()
|
static |
Lower a predicate-as-counter while intrinsic to an AArch64ISD node.
Definition at line 25381 of file AArch64ISelLowering.cpp.
References DL, FlagsVT, llvm::SelectionDAG::getNode(), getPredCounterWhileElementSize(), getPredCounterWhileOpcode(), llvm::SelectionDAG::getTargetConstant(), llvm::SelectionDAG::getVTList(), and N.
Referenced by performIntrinsicCombine().
◆ lowerPredicateInterleaveWithPackedVectors()
|
static |
Definition at line 35095 of file AArch64ISelLowering.cpp.
References assert(), DL, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getMergeValues(), llvm::SelectionDAG::getNode(), getPackedSVEVectorVT(), llvm::SelectionDAG::getSetCC(), llvm::SDValue::getValue(), llvm::EVT::getVectorElementCount(), llvm::EVT::getVectorElementType(), llvm::SelectionDAG::getVTList(), I, llvm::EVT::isScalableVector(), llvm::SmallVectorTemplateBase< T, bool >::push_back(), Results, llvm::ISD::SETNE, and llvm::ISD::ZERO_EXTEND.
◆ LowerPREFETCH()
|
static |
Definition at line 4909 of file AArch64ISelLowering.cpp.
References assert(), DL, llvm::SelectionDAG::getNode(), and llvm::SelectionDAG::getTargetConstant().
Referenced by llvm::AArch64TargetLowering::LowerOperation(), llvm::ARMTargetLowering::LowerOperation(), llvm::RISCVTargetLowering::LowerOperation(), and llvm::X86TargetLowering::LowerOperation().
◆ LowerPtrAuthGlobalAddressStatically()
|
static |
Definition at line 12012 of file AArch64ISelLowering.cpp.
References assert(), llvm::cast(), DL, llvm::SelectionDAG::getMachineNode(), llvm::SDValue::getNode(), llvm::SelectionDAG::getTargetConstant(), llvm::InnerAnalysisManagerProxy< AnalysisManagerT, IRUnitT, ExtraArgTs... >::Key, llvm::isNullConstant(), and llvm::report_fatal_error().
◆ LowerSMELdrStr()
|
static |
Definition at line 6514 of file AArch64ISelLowering.cpp.
References llvm::ISD::ADD, llvm::sampleprof::Base, C(), llvm::cast(), DL, llvm::dyn_cast(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SelectionDAG::getTargetConstant(), llvm::isa(), llvm::ISD::MUL, Mul, N, and llvm::ISD::SIGN_EXTEND.
◆ LowerSVEIntrinsicDUP()
|
static |
Definition at line 24914 of file AArch64ISelLowering.cpp.
References llvm::ISD::ANY_EXTEND, DL, llvm::SelectionDAG::getNode(), and N.
Referenced by performIntrinsicCombine().
◆ LowerSVEIntrinsicEXT()
|
static |
Definition at line 24928 of file AArch64ISelLowering.cpp.
References assert(), llvm::ISD::BITCAST, DL, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getContext(), llvm::details::FixedOrScalableQuantity< LeafTy, ValueTy >::getKnownMinValue(), llvm::SelectionDAG::getNode(), llvm::ElementCount::getScalable(), llvm::EVT::getSizeInBits(), llvm::EVT::getVectorElementType(), llvm::EVT::getVectorVT(), llvm::EVT::isScalableVector(), llvm::ISD::MUL, N, and llvm::AArch64::SVEBitsPerBlock.
Referenced by performIntrinsicCombine().
◆ LowerSVEIntrinsicIndex()
|
static |
Definition at line 24898 of file AArch64ISelLowering.cpp.
References llvm::ISD::ADD, llvm::sampleprof::Base, DL, llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getStepVector(), llvm::SDValue::getValueType(), llvm::ISD::MUL, Mul, N, and llvm::ISD::SPLAT_VECTOR.
Referenced by performIntrinsicCombine().
◆ LowerTruncateVectorStore()
|
static |
Definition at line 7781 of file AArch64ISelLowering.cpp.
References assert(), llvm::ISD::BITCAST, llvm::ISD::CONCAT_VECTORS, DL, llvm::SelectionDAG::getBuildVector(), llvm::SelectionDAG::getExtractVectorElt(), llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getPOISON(), llvm::SelectionDAG::getStore(), llvm::EVT::isVector(), Poison, and llvm::ISD::TRUNCATE.
◆ LowerVectorMatch()
|
static |
Definition at line 6561 of file AArch64ISelLowering.cpp.
References convertFixedMaskToScalableVector(), convertFromScalableVector(), convertToScalableVector(), DL, llvm::SelectionDAG::getBitcast(), getContainerForFixedLengthVector(), llvm::SelectionDAG::getExtractVectorElt(), llvm::EVT::getFixedSizeInBits(), llvm::SelectionDAG::getInsertSubvector(), llvm::SelectionDAG::getInsertVectorElt(), llvm::MVT::getIntegerVT(), llvm::SelectionDAG::getNode(), getPackedSVEVectorVT(), llvm::SelectionDAG::getPOISON(), llvm::SelectionDAG::getSplatVector(), llvm::SelectionDAG::getTargetConstant(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::SDValue::getValueType(), llvm::EVT::getVectorElementType(), llvm::MVT::getVectorVT(), llvm::ISD::INTRINSIC_WO_CHAIN, llvm::EVT::is128BitVector(), llvm::EVT::isScalableVector(), llvm::TargetLoweringBase::isTypeLegal(), llvm::ISD::OR, llvm::ISD::SIGN_EXTEND, llvm::SelectionDAG::SplitVector(), llvm::ISD::TRUNCATE, and llvm::ISD::VECTOR_MATCH.
Referenced by llvm::AArch64TargetLowering::LowerOperation().
◆ LowerXALUO()
|
static |
Definition at line 4879 of file AArch64ISelLowering.cpp.
References DL, getAArch64XALUOOp(), getCondCode(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getMergeValues(), llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getTargetLoweringInfo(), and llvm::TargetLoweringBase::isTypeLegal().
Referenced by LowerMULO(), llvm::AArch64TargetLowering::LowerOperation(), and llvm::X86TargetLowering::LowerOperation().
◆ m_PredicateAsCounterWhile()
|
static |
Definition at line 22570 of file AArch64ISelLowering.cpp.
References llvm::SDPatternMatch::m_AnyOf(), and llvm::SDPatternMatch::m_SpecificOpc().
Referenced by perfomPextFirstTrueVectorCombine().
◆ matchZeroSelectArm()
|
static |
Definition at line 30068 of file AArch64ISelLowering.cpp.
References llvm::SDValue::getNode(), llvm::ISD::isConstantSplatVectorAllZeros(), llvm::ISD::SETUGE, and llvm::ISD::SETULT.
Referenced by performVSelectMaskedShiftCombine().
◆ mayTailCallThisCC()
|
static |
Return true if we might ever do TCO for calls with this calling convention.
Definition at line 10023 of file AArch64ISelLowering.cpp.
References llvm::CallingConv::AArch64_SVE_VectorCall, llvm::CallingConv::C, llvm::CallingConv::Fast, llvm::CallingConv::PreserveAll, llvm::CallingConv::PreserveMost, llvm::CallingConv::PreserveNone, llvm::CallingConv::Swift, llvm::CallingConv::SwiftTail, and llvm::CallingConv::Tail.
Referenced by llvm::AArch64CallLowering::isEligibleForTailCallOptimization().
◆ NormalizeBuildVector()
|
static |
Definition at line 16954 of file AArch64ISelLowering.cpp.
References AbstractManglingParser< Derived, Alloc >::Ops, assert(), llvm::ISD::BUILD_VECTOR, DL, llvm::dyn_cast(), llvm::SelectionDAG::getBuildVector(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getPOISON(), llvm::EVT::getSizeInBits(), llvm::SelectionDAG::getUNDEF(), llvm::EVT::getVectorElementType(), llvm::EVT::isFloatingPoint(), llvm::ISD::POISON, and llvm::ISD::UNDEF.
◆ numberOfInstrToLoadImm()
Definition at line 3980 of file AArch64ISelLowering.cpp.
References C(), llvm::AArch64_IMM::expandMOVImm(), Imm, llvm::SmallVectorTemplateCommon< T, typename >::size(), and uint64_t.
Referenced by getAArch64Cmp().
◆ optimizeBitTest()
|
static |
Definition at line 12127 of file AArch64ISelLowering.cpp.
References llvm::ISD::AND, DL, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::isa(), llvm::isPowerOf2_64(), llvm::Log2_64(), llvm::ISD::SHL, llvm::ISD::SRL, llvm::Test, and uint64_t.
◆ optimizeBrk()
|
static |
Definition at line 6375 of file AArch64ISelLowering.cpp.
References llvm::ISD::ADD, DL, llvm::SelectionDAG::getNode(), getSVEPredicateBitCast(), llvm::SelectionDAG::getTargetConstant(), llvm::ISD::INTRINSIC_WO_CHAIN, llvm::isOneOrOneSplat(), llvm::EVT::isScalableVector(), llvm::isZeroOrZeroSplat(), N, and llvm::Upper.
Referenced by performActiveLaneMaskCombine().
◆ optimizeIncrementingWhile()
|
static |
Definition at line 6323 of file AArch64ISelLowering.cpp.
References DL, llvm::APInt::getBitWidth(), llvm::SelectionDAG::getConstant(), llvm::AArch64Subtarget::getMinSVEVectorSizeInBits(), getPTrue(), llvm::SelectionDAG::getSubtarget(), llvm::getSVEPredPatternFromNumElements(), llvm::APInt::getZExtValue(), llvm::ISD::INTRINSIC_WO_CHAIN, llvm::isa(), N, llvm::APInt::sadd_ov(), llvm::APInt::uadd_ov(), X, and Y.
Referenced by performActiveLaneMaskCombine().
◆ optimizeLogicalImm()
|
static |
Definition at line 2796 of file AArch64ISelLowering.cpp.
References assert(), llvm::TargetLowering::TargetLoweringOpt::CombineTo(), llvm::TargetLowering::TargetLoweringOpt::DAG, DL, llvm::AArch64_AM::encodeLogicalImmediate(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getMachineNode(), llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getTargetConstant(), llvm::APInt::getZExtValue(), llvm::Hi, Imm, llvm::AArch64_AM::isLogicalImmediate(), llvm::isShiftedMask_64(), Size, and uint64_t.
Referenced by llvm::AArch64TargetLowering::targetShrinkDemandedConstant(), and llvm::ARMTargetLowering::targetShrinkDemandedConstant().
◆ overflowFlagToValue()
|
static |
Definition at line 4815 of file AArch64ISelLowering.cpp.
References assert(), DL, getCondCode(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::SDValue::getResNo(), and llvm::AArch64CC::VS.
Referenced by LowerADDSUBO_CARRY(), and lowerADDSUBO_CARRY().
◆ parseConstraintCode()
|
static |
Definition at line 14309 of file AArch64ISelLowering.cpp.
References llvm::StringSwitch< T, R >::Case(), Cond, llvm::StringSwitch< T, R >::Default(), llvm::AArch64CC::EQ, llvm::AArch64CC::GE, llvm::AArch64CC::GT, llvm::AArch64CC::HI, llvm::AArch64CC::HS, llvm::AArch64CC::Invalid, llvm::AArch64CC::LE, llvm::AArch64CC::LO, llvm::AArch64CC::LS, llvm::AArch64CC::LT, llvm::AArch64CC::MI, llvm::AArch64CC::NE, llvm::AArch64CC::PL, llvm::AArch64CC::VC, and llvm::AArch64CC::VS.
Referenced by llvm::X86TargetLowering::getConstraintType(), llvm::X86TargetLowering::getRegForInlineAsmConstraint(), and llvm::X86TargetLowering::LowerAsmOutputForConstraint().
◆ parsePredicateConstraint()
|
static |
Definition at line 14254 of file AArch64ISelLowering.cpp.
References llvm::StringSwitch< T, R >::Case(), llvm::StringSwitch< T, R >::Default(), Upa, Uph, and Upl.
◆ parseReducedGprConstraint()
|
static |
Definition at line 14285 of file AArch64ISelLowering.cpp.
References llvm::StringSwitch< T, R >::Case(), llvm::StringSwitch< T, R >::Default(), Uci, and Ucj.
◆ parseSVERegAsConstraint()
|
static |
Definition at line 14231 of file AArch64ISelLowering.cpp.
References llvm::StringRef::drop_front(), llvm::StringRef::ends_with(), llvm::StringRef::getAsInteger(), llvm::StringRef::size(), llvm::StringRef::starts_with(), and llvm::StringRef::substr().
◆ perfomPextFirstTrueVectorCombine()
|
static |
Folds extracting the first lane from the first segment of a predicate-as-counter while to a conditional set (CSET) based on the "FIRST_ACTIVE" status flag from the while.
while, flags = WHILE_*_PRED_COUNTER .. ; predicate-as-counter while first.segment = pext(while, 0) ; predicate extract of segment 0 first.active = extract_elt(first.segment, 0) ; extract first lane
->
while, flags = WHILE_*_PRED_COUNTER .. ; predicate-as-counter while first.active = cset(flags, FIRST_ACTIVE)
Definition at line 22595 of file AArch64ISelLowering.cpp.
References assert(), llvm::TargetLowering::DAGCombinerInfo::DAG, llvm::ISD::EXTRACT_VECTOR_ELT, llvm::AArch64CC::FIRST_ACTIVE, llvm::SDValue::getNode(), llvm::SDNode::getOperand(), getSETCC(), llvm::SDValue::getValueType(), hasSVEMultiVectorOps(), llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalize(), llvm::isNullConstant(), llvm::EVT::isScalableVectorOf(), llvm::SDPatternMatch::m_IntrinsicWOChain(), m_PredicateAsCounterWhile(), llvm::PatternMatch::m_Zero(), N, and llvm::SDPatternMatch::sd_match().
Referenced by performExtractVectorEltCombine().
◆ performActiveLaneMaskCombine()
|
static |
Definition at line 20692 of file AArch64ISelLowering.cpp.
References assert(), llvm::TargetLowering::DAGCombinerInfo::CombineTo(), llvm::ISD::CONCAT_VECTORS, llvm::count_if(), llvm::TargetLowering::DAGCombinerInfo::DAG, llvm::details::FixedOrScalableQuantity< LeafTy, ValueTy >::divideCoefficientBy(), DL, llvm::ISD::EXTRACT_SUBVECTOR, llvm::ISD::GET_ACTIVE_LANE_MASK, llvm::SelectionDAG::getContext(), llvm::EVT::getDoubleNumVectorElementsVT(), llvm::SelectionDAG::getElementCount(), llvm::details::FixedOrScalableQuantity< LeafTy, ValueTy >::getKnownMinValue(), llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getTargetConstant(), llvm::SDValue::getValueType(), llvm::EVT::getVectorElementCount(), I, llvm::ISD::INTRINSIC_WO_CHAIN, llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalize(), N, llvm::Offset, optimizeBrk(), optimizeIncrementingWhile(), llvm::SmallVectorTemplateBase< T, bool >::push_back(), llvm::ISD::UADDSAT, and llvm::ISD::ZERO_EXTEND.
Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().
◆ performAddAddCombine()
|
static |
Definition at line 24611 of file AArch64ISelLowering.cpp.
References A(), llvm::ISD::ADD, B(), DL, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDValue::hasOneUse(), llvm::EVT::isScalarInteger(), N, and llvm::ISD::SHL.
Referenced by performAddSubCombine().
◆ performAddCombineForShiftedOperands()
|
static |
Definition at line 24051 of file AArch64ISelLowering.cpp.
References llvm::ISD::ADD, DL, llvm::SelectionDAG::getNode(), isOpcWithIntImmediate(), LHS, N, performAddCombineSubShift(), RHS, llvm::ISD::SHL, and uint64_t.
Referenced by performAddSubCombine().
◆ performAddCombineSubShift()
|
static |
Definition at line 24023 of file AArch64ISelLowering.cpp.
References llvm::ISD::ADD, DL, llvm::SelectionDAG::getNode(), llvm::isa(), isExtendOrShiftOperand(), N, llvm::ISD::SUB, and Y.
Referenced by performAddCombineForShiftedOperands().
◆ performAddCSelIntoCSinc()
|
static |
Perform the scalar expression combine in the form of: CSEL(c, 1, cc) + b => CSINC(b+c, b, cc) CSNEG(c, -1, cc) + b => CSINC(b+c, b, cc)
Definition at line 23397 of file AArch64ISelLowering.cpp.
References llvm::ISD::ADD, assert(), C(), llvm::cast(), DL, llvm::dyn_cast(), llvm::SelectionDAG::getAllOnesConstant(), llvm::ConstantSDNode::getAPIntValue(), getCondCode(), llvm::SelectionDAG::getConstant(), llvm::AArch64CC::getInvertedCondCode(), llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::ConstantSDNode::isAllOnes(), llvm::TargetLoweringBase::isLegalAddImmediate(), llvm::ConstantSDNode::isOne(), llvm::EVT::isScalarInteger(), LHS, N, RHS, and std::swap().
Referenced by performAddSubCombine().
◆ performAddDotCombine()
|
static |
Definition at line 23470 of file AArch64ISelLowering.cpp.
References A(), llvm::ISD::ADD, llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), isZerosVector(), N, and std::swap().
Referenced by performAddSubCombine().
◆ performAddMulZextCombine()
|
static |
Definition at line 24487 of file AArch64ISelLowering.cpp.
References llvm::ISD::ADD, C(), llvm::TargetLowering::DAGCombinerInfo::DAG, DL, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::isa(), llvm::TargetLowering::DAGCombinerInfo::isAfterLegalizeDAG(), llvm::ISD::MUL, Mul, N, llvm::Other, llvm::ISD::SUB, std::swap(), and llvm::ISD::ZERO_EXTEND.
Referenced by performAddSubCombine().
◆ performAddSubCarryCombine()
|
static |
Takes advantage of the negation of the second operand by the with-carry-in instructions to fold:
AddWithCarry(X, -1, C) => SubWithCarry(X, 0, C) SubWithCarry(X, -1, C) => AddWithCarry(X, 0, C)
This allows using the zero register (xzr/wzr) to encode the immediate operand (rather than a MOV to a GPR).
Definition at line 23707 of file AArch64ISelLowering.cpp.
References DL, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::isAllOnesConstant(), N, and RHS.
Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().
◆ performAddSubCombine()
|
static |
Definition at line 24700 of file AArch64ISelLowering.cpp.
References llvm::TargetLowering::DAGCombinerInfo::DAG, N, performAddAddCombine(), performAddCombineForShiftedOperands(), performAddCSelIntoCSinc(), performAddDotCombine(), performAddMulZextCombine(), performAddSubIntoVectorOp(), performAddSubLongCombine(), performAddTruncShiftCombine(), performAddUADDVCombine(), performAddWithSBCCombine(), performConditionalSubCombine(), performExtBinopLoadFold(), performNegCSelCombine(), performNegSignMaskedAndCombine(), performSubAddMULCombine(), performSubNegAndOneCombine(), performSubWithBorrowCombine(), performSVEMulAddSubCombine(), and performVectorExtCombine().
Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().
◆ performAddSubIntoVectorOp()
|
static |
Definition at line 24212 of file AArch64ISelLowering.cpp.
References DL, llvm::ISD::EXTRACT_VECTOR_ELT, llvm::SelectionDAG::getExtractVectorElt(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::SDValue::getValueType(), llvm::TargetLoweringBase::isOperationExpand(), llvm::ISD::LOAD, N, and llvm::ISD::SCALAR_TO_VECTOR.
Referenced by performAddSubCombine().
◆ performAddSubLongCombine()
|
static |
Definition at line 23590 of file AArch64ISelLowering.cpp.
References llvm::ISD::ADD, llvm::TargetLowering::DAGCombinerInfo::DAG, llvm::SelectionDAG::getNode(), llvm::MVT::is128BitVector(), llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalizeOps(), isEssentiallyExtractHighSubvector(), LHS, N, performSetccAddFolding(), reassociateAddSubLong(), RHS, llvm::ISD::SIGN_EXTEND, tryExtendDUPToExtractHigh(), and llvm::ISD::ZERO_EXTEND.
Referenced by performAddSubCombine().
◆ performAddTruncShiftCombine()
|
static |
Definition at line 24568 of file AArch64ISelLowering.cpp.
References llvm::ISD::ADD, DL, llvm::SDValue::getConstantOperandVal(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::EVT::getScalarSizeInBits(), llvm::SelectionDAG::getTargetConstant(), llvm::PatternMatch::m_Add(), llvm::PatternMatch::m_Trunc(), llvm::PatternMatch::m_Value(), N, llvm::SDPatternMatch::sd_match(), std::swap(), and llvm::ISD::TRUNCATE.
Referenced by performAddSubCombine().
◆ performAddUADDVCombine()
|
static |
Definition at line 23359 of file AArch64ISelLowering.cpp.
References llvm::ISD::ADD, DL, llvm::dyn_cast(), llvm::ISD::EXTRACT_VECTOR_ELT, llvm::SelectionDAG::getExtractVectorElt(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDNode::getValueType(), llvm::SDValue::getValueType(), llvm::EVT::getVectorElementType(), llvm::EVT::isScalarInteger(), LHS, N, and RHS.
Referenced by performAddSubCombine().
◆ performAddWithSBCCombine()
|
static |
Definition at line 24643 of file AArch64ISelLowering.cpp.
References llvm::ISD::ADD, C(), DL, llvm::SelectionDAG::getNegative(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDValue::hasOneUse(), llvm::isNullConstant(), N, and std::swap().
Referenced by performAddSubCombine().
◆ performANDCombine()
|
static |
Definition at line 22335 of file AArch64ISelLowering.cpp.
References llvm::SelectionDAG::computeKnownBits(), llvm::TargetLowering::DAGCombinerInfo::DAG, llvm::dyn_cast(), llvm::EVT::getSizeInBits(), llvm::SelectionDAG::getTargetLoweringInfo(), I, llvm::EVT::is128BitVector(), llvm::EVT::is64BitVector(), llvm::EVT::isScalableVector(), llvm::TargetLoweringBase::isTypeLegal(), llvm::Known, LHS, N, performANDORCSELCombine(), performANDORDUPNOTCombine(), performANDSETCCCombine(), performSVEAndCombine(), resolveBuildVector(), RHS, tryAdvSIMDModImm16(), and tryAdvSIMDModImm32().
Referenced by llvm::AArch64TargetLowering::PerformDAGCombine(), llvm::LoongArchTargetLowering::PerformDAGCombine(), llvm::MipsSETargetLowering::PerformDAGCombine(), llvm::MipsTargetLowering::PerformDAGCombine(), and llvm::RISCVTargetLowering::PerformDAGCombine().
◆ performANDORCSELCombine()
|
static |
Definition at line 21946 of file AArch64ISelLowering.cpp.
References llvm::ISD::AND, DL, llvm::dyn_cast(), FlagsVT, getCondCode(), llvm::SelectionDAG::getConstant(), llvm::SDValue::getConstantOperandVal(), llvm::AArch64CC::getInvertedCondCode(), llvm::SelectionDAG::getNode(), llvm::AArch64CC::getNZCVToSatisfyCondCode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDNode::getValueType(), llvm::SDNode::hasOneUse(), llvm::isNullConstant(), llvm::isOneConstant(), N, and std::swap().
Referenced by performANDCombine(), performANDSCombine(), and performORCombine().
◆ performANDORDUPNOTCombine()
|
static |
Definition at line 22112 of file AArch64ISelLowering.cpp.
References llvm::ISD::AND, assert(), DL, llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getNOT(), llvm::SelectionDAG::getSplat(), llvm::PatternMatch::m_c_BinOp(), llvm::PatternMatch::m_InsertElt(), llvm::MIPatternMatch::m_Not(), llvm::PatternMatch::m_Poison(), llvm::PatternMatch::m_Shuffle(), llvm::PatternMatch::m_Value(), llvm::PatternMatch::m_Zero(), N, Opc, llvm::ISD::OR, llvm::SDPatternMatch::sd_match(), X, and Y.
Referenced by performANDCombine(), and performORCombine().
◆ performANDSCombine()
|
static |
Definition at line 29732 of file AArch64ISelLowering.cpp.
References llvm::ISD::AND, llvm::TargetLowering::DAGCombinerInfo::DAG, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), N, performANDORCSELCombine(), and performFlagSettingCombine().
Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().
◆ performANDSETCCCombine()
|
static |
Definition at line 22296 of file AArch64ISelLowering.cpp.
References llvm::TargetLowering::DAGCombinerInfo::DAG, DL, emitConjunction(), getCondCode(), llvm::SelectionDAG::getConstant(), llvm::AArch64CC::getInvertedCondCode(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDValue::getValueType(), llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalize(), N, llvm::ISD::SELECT, and llvm::ISD::SETCC.
Referenced by performANDCombine().
◆ performBICiCombine()
|
static |
Definition at line 21063 of file AArch64ISelLowering.cpp.
References llvm::APInt::getAllOnes(), llvm::SelectionDAG::getTargetLoweringInfo(), N, and llvm::TargetLowering::SimplifyDemandedBits().
Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().
◆ performBRCONDCombine()
|
static |
Definition at line 29072 of file AArch64ISelLowering.cpp.
References assert(), llvm::TargetLowering::DAGCombinerInfo::CombineTo(), DL, llvm::AArch64CC::EQ, llvm::SDNode::getAsZExtVal(), getCondCode(), getCSETCondCode(), llvm::MachineFunction::getFunction(), llvm::SelectionDAG::getMachineFunction(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOperand(), llvm::Function::hasFnAttribute(), llvm::isa(), isCMP(), llvm::isNullConstant(), llvm::isOneConstant(), LHS, N, llvm::AArch64CC::NE, performCONDCombine(), RHS, llvm::ISD::SHL, llvm::ISD::SRA, llvm::ISD::SRL, and std::swap().
Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().
◆ performBSPExpandForSVE()
|
static |
Definition at line 31073 of file AArch64ISelLowering.cpp.
References llvm::ISD::AND, DL, llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getNOT(), llvm::EVT::isScalableVector(), N, and llvm::ISD::OR.
Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().
◆ performBuildShuffleExtendCombine()
|
static |
Combines a buildvector(sext/zext) or shuffle(sext/zext, undef) node pattern into sext/zext(buildvector) or sext/zext(shuffle) making use of the vector SExt/ZExt rather than the scalar SExt/ZExt.
Definition at line 21252 of file AArch64ISelLowering.cpp.
References llvm::ISD::AND, llvm::ISD::ANY_EXTEND, llvm::ISD::AssertSext, llvm::ISD::AssertZext, llvm::ISD::BUILD_VECTOR, calculatePreExtendType(), llvm::cast(), llvm::EVT::changeVectorElementType(), DL, llvm::drop_begin(), llvm::SelectionDAG::getAnyExtOrTrunc(), llvm::SelectionDAG::getContext(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDNode::getOperand(), llvm::SDValue::getOperand(), llvm::SelectionDAG::getPOISON(), llvm::EVT::getScalarSizeInBits(), llvm::EVT::getScalarType(), llvm::SDValue::getValueType(), llvm::SelectionDAG::getVectorShuffle(), llvm::SDValue::isUndef(), Opc, llvm::SDNode::ops(), llvm::SmallVectorTemplateBase< T, bool >::push_back(), llvm::ISD::SIGN_EXTEND, llvm::ISD::SIGN_EXTEND_INREG, llvm::ISD::VECTOR_SHUFFLE, and llvm::ISD::ZERO_EXTEND.
Referenced by performMulVectorExtendCombine().
◆ performBuildVectorCombine()
|
static |
Definition at line 23718 of file AArch64ISelLowering.cpp.
References llvm::ISD::ANY_EXTEND, llvm::EVT::changeVectorElementType(), Concat, llvm::ISD::CONCAT_VECTORS, DL, llvm::ISD::EXTRACT_SUBVECTOR, llvm::ISD::EXTRACT_VECTOR_ELT, llvm::ISD::FP_EXTEND, llvm::ISD::FP_ROUND, llvm::SDNode::getConstantOperandVal(), llvm::SelectionDAG::getContext(), llvm::SelectionDAG::getExtractSubvector(), llvm::SelectionDAG::getNode(), llvm::SDNode::getOpcode(), llvm::SDNode::getOperand(), llvm::SelectionDAG::getPOISON(), llvm::SelectionDAG::getSubtarget(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::SDValue::getValueType(), llvm::EVT::getVectorMinNumElements(), llvm::isa(), llvm::AArch64Subtarget::isNeonAvailable(), llvm::TargetLoweringBase::isTypeLegal(), and N.
Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().
◆ performConcatVectorsCombine()
|
static |
Definition at line 22755 of file AArch64ISelLowering.cpp.
References AbstractManglingParser< Derived, Alloc >::Ops, llvm::ISD::ADD, llvm::Add, llvm::all_of(), assert(), llvm::ISD::BITCAST, llvm::cast(), llvm::ISD::CONCAT_VECTORS, llvm::TargetLowering::DAGCombinerInfo::DAG, llvm::dbgs(), DL, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getBuildVector(), llvm::SelectionDAG::getConstant(), llvm::SDNode::getConstantOperandVal(), llvm::SDValue::getConstantOperandVal(), llvm::SelectionDAG::getContext(), llvm::SelectionDAG::getLoad(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getNOT(), llvm::SDNode::getOpcode(), llvm::SDValue::getOpcode(), llvm::SDNode::getOperand(), llvm::SDValue::getOperand(), llvm::SelectionDAG::getPOISON(), llvm::EVT::getScalarSizeInBits(), llvm::SelectionDAG::getTargetConstant(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::SelectionDAG::getUNDEF(), llvm::SDValue::getValue(), llvm::SDValue::getValueType(), llvm::MVT::getVectorElementType(), llvm::EVT::getVectorNumElements(), llvm::MVT::getVectorNumElements(), llvm::SelectionDAG::getVectorShuffle(), llvm::EVT::getVectorVT(), llvm::MVT::getVectorVT(), llvm::SDNode::hasOneUse(), Imm, llvm::EVT::is128BitVector(), llvm::isa(), llvm::TargetLowering::DAGCombinerInfo::isAfterLegalizeDAG(), llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalizeOps(), llvm::TargetLoweringBase::isBinOp(), llvm::ISD::isConstantSplatVectorAllOnes(), llvm::SDNode::isOnlyUserOf(), llvm::EVT::isScalableVector(), llvm::SDValue::isUndef(), isUnpackedType(), llvm::MVT::isVector(), isVectorizedBinOp(), LLVM_DEBUG, N, llvm::SelectionDAG::ReplaceAllUsesOfValueWith(), RHS, llvm::ISD::TRUNCATE, uint64_t, WidenVector(), X, llvm::ISD::XOR, and Y.
Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().
◆ performCONDCombine()
|
static |
Definition at line 28966 of file AArch64ISelLowering.cpp.
References AbstractManglingParser< Derived, Alloc >::Ops, llvm::ISD::ADD, llvm::ISD::AND, C(), llvm::cast(), checkValueWidth(), llvm::TargetLowering::DAGCombinerInfo::CombineTo(), DL, llvm::dyn_cast(), llvm::AArch64CC::EQ, llvm::SelectionDAG::getConstant(), llvm::SDNode::getConstantOperandAPInt(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDNode::getOpcode(), llvm::SDValue::getOpcode(), llvm::SDNode::getOperand(), llvm::SDValue::getOperand(), llvm::SelectionDAG::getShiftAmountConstant(), llvm::SDValue::getValue(), llvm::SDNode::getValueType(), llvm::SDNode::getVTList(), llvm::SelectionDAG::getVTList(), llvm::SDNode::hasAnyUseOfValue(), llvm::SDNode::hasOneUse(), llvm::isa(), isEquivalentMaskless(), llvm::AArch64_AM::isLegalArithImmed(), N, llvm::AArch64CC::NE, performSubsToAndsCombine(), llvm::SelectionDAG::ReplaceAllUsesWith(), llvm::ISD::SHL, and X.
Referenced by performBRCONDCombine(), and performCSELCombine().
◆ performConditionalSubCombine()
|
static |
Definition at line 24670 of file AArch64ISelLowering.cpp.
References llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDValue::getValue(), llvm::SDValue::hasOneUse(), llvm::isNullConstant(), N, llvm::ISD::SUB, and X.
Referenced by performAddSubCombine().
◆ performCSELCombine()
|
static |
Definition at line 29423 of file AArch64ISelLowering.cpp.
References llvm::AArch64CC::AL, Cond, DL, llvm::SelectionDAG::doesNodeExist(), foldCSELOfCSEL(), foldCSELofCTTZ(), foldCSELofLASTB(), getCondCode(), llvm::SelectionDAG::getNode(), getSwappedCondition(), llvm::TargetLowering::DAGCombinerInfo::isAfterLegalizeDAG(), llvm::isNullConstant(), N, performCONDCombine(), reassociateCSELOperandsForCSE(), llvm::ISD::SUB, and llvm::Sub.
Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().
◆ performCTLZCombine()
|
static |
Definition at line 30468 of file AArch64ISelLowering.cpp.
References llvm::ISD::BITREVERSE, llvm::ISD::CTTZ, DL, llvm::SelectionDAG::getNode(), and N.
Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().
◆ performCTPOPCombine()
|
static |
Definition at line 31653 of file AArch64ISelLowering.cpp.
References llvm::EVT::changeVectorElementTypeToInteger(), DL, llvm::SelectionDAG::getContext(), llvm::SelectionDAG::getNegative(), llvm::SelectionDAG::getNode(), llvm::EVT::getScalarSizeInBits(), llvm::EVT::getScalarType(), llvm::SelectionDAG::getSExtOrTrunc(), llvm::EVT::getSizeInBits(), llvm::EVT::getVectorElementCount(), llvm::EVT::getVectorElementType(), llvm::EVT::getVectorVT(), llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalize(), llvm::EVT::isFixedLengthVector(), llvm::EVT::isVector(), llvm::Log2_64_Ceil(), llvm::PatternMatch::m_BitCast(), llvm::SDPatternMatch::m_SetCC(), llvm::PatternMatch::m_Value(), llvm::SDPatternMatch::m_VT(), llvm::PatternMatch::m_ZExtOrSelf(), N, llvm::SDPatternMatch::sd_match(), llvm::ISD::SIGN_EXTEND, and llvm::ISD::VECREDUCE_ADD.
Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().
◆ performDUPCombine()
|
static |
Definition at line 30295 of file AArch64ISelLowering.cpp.
References AbstractManglingParser< Derived, Alloc >::Ops, llvm::ISD::BITCAST, convertFromScalableVector(), llvm::TargetLowering::DAGCombinerInfo::DAG, DL, llvm::dyn_cast(), llvm::ISD::EXTLOAD, llvm::ISD::EXTRACT_VECTOR_ELT, llvm::SelectionDAG::getConstant(), getContainerForFixedLengthVector(), llvm::SelectionDAG::getContext(), llvm::EVT::getDoubleNumVectorElementsVT(), getDUPLANEOp(), llvm::SelectionDAG::getExtractSubvector(), llvm::SelectionDAG::getInsertVectorElt(), llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getNodeIfExists(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SelectionDAG::getPOISON(), llvm::EVT::getScalarType(), llvm::EVT::getSizeInBits(), llvm::SDValue::getValueType(), llvm::EVT::getVectorElementType(), llvm::EVT::getVectorVT(), llvm::SelectionDAG::getVTList(), llvm::EVT::is64BitVector(), llvm::TargetLowering::DAGCombinerInfo::isAfterLegalizeDAG(), llvm::EVT::isInteger(), llvm::AArch64Subtarget::isSVEorStreamingSVEAvailable(), N, performPostLD1Combine(), llvm::ISD::SCALAR_TO_VECTOR, llvm::ISD::SPLAT_VECTOR, and llvm::ISD::ZEXTLOAD.
Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().
◆ performDupLane128Combine()
|
static |
Definition at line 31093 of file AArch64ISelLowering.cpp.
References llvm::ISD::BITCAST, DL, llvm::SelectionDAG::getNode(), getPackedSVEVectorVT(), llvm::SelectionDAG::getPOISON(), llvm::SDValue::getValueType(), llvm::EVT::getVectorElementType(), llvm::ISD::INSERT_SUBVECTOR, llvm::EVT::is128BitVector(), N, llvm::ISD::SPLAT_VECTOR, and uint64_t.
Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().
◆ performExtBinopLoadFold()
|
static |
Definition at line 24371 of file AArch64ISelLowering.cpp.
References AbstractManglingParser< Derived, Alloc >::Ops, areLoadedOffsetButOtherwiseSame(), llvm::ISD::CONCAT_VECTORS, DL, llvm::SelectionDAG::getContext(), llvm::EVT::getDoubleNumVectorElementsVT(), llvm::SelectionDAG::getExtractSubvector(), llvm::SelectionDAG::getLoad(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::EVT::getScalarSizeInBits(), llvm::EVT::getScalarType(), llvm::EVT::getSizeInBits(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::SDValue::getValueType(), llvm::EVT::getVectorNumElements(), llvm::SelectionDAG::getVectorShuffle(), llvm::EVT::getVectorVT(), llvm::SDValue::hasOneUse(), llvm::Hi, llvm::ISD::isConstantSplatVector(), llvm::ISD::isExtOpcode(), llvm::EVT::isFixedLengthVector(), isLoadOrMultipleLoads(), llvm::TargetLoweringBase::isTypeLegal(), llvm::Lo, llvm::Load, llvm::SelectionDAG::makeEquivalentMemoryOrdering(), N, llvm::O0, llvm::O1, llvm::SDNode::op_values(), llvm::Other, llvm::SmallVectorTemplateBase< T, bool >::push_back(), llvm::ISD::SHL, llvm::SmallVectorTemplateCommon< T, typename >::size(), llvm::ISD::SUB, std::swap(), and llvm::zip().
Referenced by performAddSubCombine().
◆ performExtendCombine()
|
static |
Definition at line 26163 of file AArch64ISelLowering.cpp.
References llvm::ISD::ABDS, llvm::ISD::ABDU, llvm::ISD::ANY_EXTEND, llvm::ISD::BSWAP, combineToExtendBoolVectorInReg(), DL, getCSETCondCode(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDNode::getOperand(), llvm::SDValue::getOperand(), llvm::SDValue::getValueType(), llvm::SDValue::hasOneUse(), llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalizeOps(), N, llvm::Intrinsic::not_intrinsic, performExtendDuplaneTruncCombine(), performExtendToBoolVectorLoadCombine(), performSignExtendSetCCCombine(), performZExtDeinterleaveShuffleCombine(), performZExtUZPCombine(), llvm::ISD::SETCC, llvm::ISD::SIGN_EXTEND, tryCombineLongOpWithDup(), and llvm::ISD::ZERO_EXTEND.
Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().
◆ performExtendDuplaneTruncCombine()
|
static |
Definition at line 26062 of file AArch64ISelLowering.cpp.
References llvm::ISD::ANY_EXTEND, assert(), llvm::SelectionDAG::ComputeNumSignBits(), llvm::APInt::getHighBitsSet(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::EVT::getScalarSizeInBits(), llvm::SDValue::getValueType(), llvm::SDNode::hasOneUse(), llvm::SDValue::hasOneUse(), llvm::ISD::INSERT_SUBVECTOR, llvm::EVT::is128BitVector(), llvm::isNullConstant(), llvm::SelectionDAG::MaskedValueIsZero(), N, llvm::ISD::SIGN_EXTEND, llvm::ISD::TRUNCATE, and llvm::ISD::ZERO_EXTEND.
Referenced by performExtendCombine().
◆ performExtendToBoolVectorLoadCombine()
|
static |
Definition at line 26120 of file AArch64ISelLowering.cpp.
References combineToExtendBoolVectorInReg(), DL, llvm::dyn_cast(), llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getContext(), llvm::EVT::getIntegerVT(), llvm::SelectionDAG::getLoad(), llvm::EVT::getScalarSizeInBits(), llvm::EVT::getScalarType(), llvm::SDValue::getValue(), llvm::SDValue::getValueType(), llvm::EVT::getVectorNumElements(), llvm::SDValue::hasOneUse(), llvm::EVT::isFixedLengthVector(), llvm::AArch64Subtarget::isNeonAvailable(), llvm::ISD::isNormalLoad(), N, and llvm::SelectionDAG::ReplaceAllUsesOfValueWith().
Referenced by performExtendCombine().
◆ performExtractLastActiveCombine()
|
static |
Definition at line 22538 of file AArch64ISelLowering.cpp.
References llvm::ArrayRef(), assert(), llvm::TargetLowering::DAGCombinerInfo::DAG, llvm::ISD::EXTRACT_VECTOR_ELT, llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::EVT::getSimpleVT(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::SDValue::getValueType(), llvm::EVT::getVectorElementType(), llvm::is_contained(), llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalize(), llvm::TargetLoweringBase::isOperationLegal(), N, and llvm::ISD::VECTOR_FIND_LAST_ACTIVE.
Referenced by performExtractVectorEltCombine().
◆ performExtractSubvectorCombine()
|
static |
Definition at line 23033 of file AArch64ISelLowering.cpp.
References llvm::SelectionDAG::getNode(), llvm::isa(), llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalizeOps(), llvm::EVT::isScalableVector(), N, and llvm::ISD::SPLAT_VECTOR.
Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().
◆ performExtractVectorEltCombine()
|
static |
Definition at line 22621 of file AArch64ISelLowering.cpp.
References llvm::all_of(), llvm::ISD::ANY_EXTEND, assert(), llvm::commonAlignment(), llvm::TargetLowering::DAGCombinerInfo::DAG, DL, llvm::dyn_cast(), llvm::ISD::EXTLOAD, llvm::ISD::EXTRACT_SUBVECTOR, llvm::ISD::EXTRACT_VECTOR_ELT, llvm::SelectionDAG::getConstant(), llvm::SDValue::getConstantOperandVal(), llvm::SelectionDAG::getExtLoad(), llvm::SelectionDAG::getExtractVectorElt(), llvm::ShuffleVectorSDNode::getMaskElt(), llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getObjectPtrOffset(), llvm::SDNode::getOpcode(), llvm::SDValue::getOpcode(), llvm::SDNode::getOperand(), llvm::SDValue::getOperand(), llvm::SDValue::getValue(), llvm::SelectionDAG::getZExtOrTrunc(), llvm::SDValue::hasOneUse(), hasPairwiseAdd(), llvm::ISD::INSERT_SUBVECTOR, llvm::isa(), llvm::TargetLowering::DAGCombinerInfo::isAfterLegalizeDAG(), llvm::EVT::isInteger(), llvm::AArch64Subtarget::isLittleEndian(), llvm::ISD::isNormalLoad(), llvm::isNullConstant(), llvm::SDNode::isStrictFPOpcode(), llvm::SDValue::isUndef(), llvm::Load, llvm::SelectionDAG::makeEquivalentMemoryOrdering(), N, llvm::Offset, llvm::Other, perfomPextFirstTrueVectorCombine(), performExtractLastActiveCombine(), performFirstTrueTestVectorCombine(), performLastTrueTestVectorCombine(), llvm::SelectionDAG::ReplaceAllUsesOfValueWith(), llvm::ISD::SEXTLOAD, llvm::ISD::SIGN_EXTEND, llvm::SDNode::uses(), llvm::ISD::ZERO_EXTEND, and llvm::ISD::ZEXTLOAD.
Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().
◆ performFADDCombine()
|
static |
Definition at line 22401 of file AArch64ISelLowering.cpp.
References A(), B(), llvm::TargetLowering::DAGCombinerInfo::DAG, DL, llvm::ISD::FADD, llvm::SelectionDAG::getNode(), llvm::ISD::INTRINSIC_WO_CHAIN, LHS, N, Opc, RHS, and llvm::SDNode::setFlags().
Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().
◆ performFirstTrueTestVectorCombine()
|
static |
Definition at line 22481 of file AArch64ISelLowering.cpp.
References assert(), llvm::TargetLowering::DAGCombinerInfo::DAG, llvm::ISD::EXTRACT_VECTOR_ELT, llvm::AArch64CC::FIRST_ACTIVE, llvm::SelectionDAG::getConstant(), getPTest(), llvm::SDValue::getResNo(), llvm::SDValue::getValueType(), llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalize(), llvm::isNullConstant(), isPredicateCCSettingOp(), llvm::EVT::isScalableVectorOf(), and N.
Referenced by performExtractVectorEltCombine().
◆ performFlagSettingCombine()
|
static |
Definition at line 29709 of file AArch64ISelLowering.cpp.
References llvm::TargetLowering::DAGCombinerInfo::CombineTo(), llvm::TargetLowering::DAGCombinerInfo::DAG, DL, Generic, llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getNodeIfExists(), llvm::SelectionDAG::getVTList(), LHS, N, and RHS.
Referenced by performANDSCombine(), and llvm::AArch64TargetLowering::PerformDAGCombine().
◆ performFPExtendCombine()
|
static |
Definition at line 31035 of file AArch64ISelLowering.cpp.
References llvm::cast(), llvm::TargetLowering::DAGCombinerInfo::CombineTo(), llvm::ISD::EXTLOAD, llvm::ISD::FP_ROUND, llvm::LoadSDNode::getBasePtr(), llvm::MemSDNode::getChain(), llvm::SelectionDAG::getExtLoad(), llvm::EVT::getFixedSizeInBits(), llvm::SelectionDAG::getIntPtrConstant(), llvm::MemSDNode::getMemOperand(), llvm::AArch64Subtarget::getMinSVEVectorSizeInBits(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDValue::getValue(), llvm::SDValue::getValueType(), llvm::EVT::getVectorElementType(), llvm::SDValue::hasOneUse(), llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalizeOps(), llvm::EVT::isFixedLengthVector(), llvm::ISD::isNormalLoad(), N, and llvm::AArch64Subtarget::useSVEForFixedLengthVectors().
Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().
◆ performFpToIntCombine()
|
static |
Fold a floating-point multiply by power of two into floating-point to fixed-point conversion.
Definition at line 21926 of file AArch64ISelLowering.cpp.
References N, and tryToReplaceScalarFPConversionWithSVE().
Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().
◆ performGatherLoadCombine()
|
static |
Definition at line 30639 of file AArch64ISelLowering.cpp.
References AbstractManglingParser< Derived, Alloc >::Ops, llvm::ISD::ANY_EXTEND, assert(), llvm::sampleprof::Base, llvm::ISD::BITCAST, DL, llvm::details::FixedOrScalableQuantity< LeafTy, ValueTy >::getKnownMinValue(), llvm::SelectionDAG::getMergeValues(), llvm::SelectionDAG::getNode(), llvm::EVT::getScalarSizeInBits(), getScaledOffsetForBitWidth(), llvm::EVT::getSizeInBits(), getSVEContainerType(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::SDValue::getValue(), llvm::SelectionDAG::getValueType(), llvm::SelectionDAG::getVTList(), llvm::EVT::isFloatingPoint(), llvm::EVT::isInteger(), llvm::EVT::isScalableVector(), llvm::TargetLoweringBase::isTypeLegal(), isValidImmForSVEVecImmAddrMode(), llvm::Load, N, llvm::Offset, llvm::AArch64::SVEBitsPerBlock, std::swap(), and llvm::ISD::TRUNCATE.
Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().
◆ performGLD1Combine()
|
static |
Definition at line 26996 of file AArch64ISelLowering.cpp.
References llvm::sampleprof::Base, llvm::cast(), DL, getGatherVecOpcode(), llvm::SelectionDAG::getNode(), getSignExtendedGatherOpcode(), llvm::EVT::getVectorElementType(), llvm::VTSDNode::getVT(), N, llvm::Offset, Opc, Scaled, and Signed.
Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().
◆ performGlobalAddressCombine()
|
static |
Definition at line 30417 of file AArch64ISelLowering.cpp.
References llvm::ISD::ADD, C(), llvm::cast(), llvm::AArch64Subtarget::ClassifyGlobalReference(), DL, llvm::dyn_cast(), llvm::SelectionDAG::getConstant(), llvm::GlobalValue::getDataLayout(), llvm::GlobalAddressSDNode::getGlobal(), llvm::SelectionDAG::getGlobalAddress(), llvm::SelectionDAG::getNode(), llvm::GlobalAddressSDNode::getOffset(), llvm::DataLayout::getTypeAllocSize(), llvm::GlobalValue::getValueType(), llvm::AArch64II::MO_NO_FLAG, N, llvm::Offset, llvm::ISD::SUB, T, uint64_t, and llvm::SDNode::users().
Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().
◆ performInsertSubvectorCombine()
|
static |
Definition at line 23059 of file AArch64ISelLowering.cpp.
References C(), llvm::ISD::CONCAT_VECTORS, DL, llvm::dyn_cast_or_null(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getConstantFP(), llvm::SelectionDAG::getExtractSubvector(), llvm::SelectionDAG::getNode(), llvm::EVT::getSizeInBits(), llvm::SelectionDAG::getSplatValue(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::SDValue::getValueType(), llvm::EVT::getVectorNumElements(), llvm::Hi, llvm::EVT::isFixedLengthVector(), llvm::EVT::isScalableVector(), llvm::TargetLoweringBase::isTypeLegal(), llvm::SDValue::isUndef(), llvm::Lo, N, and uint64_t.
Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().
◆ performInsertVectorEltCombine()
|
static |
Definition at line 31017 of file AArch64ISelLowering.cpp.
References llvm::TargetLowering::DAGCombinerInfo::DAG, DL, llvm::SelectionDAG::getNode(), llvm::SDValue::getValueType(), llvm::isNullConstant(), llvm::EVT::isScalableVector(), llvm::SDNode::isUndef(), N, performPostLD1Combine(), removeRedundantInsertVectorElt(), and llvm::ISD::SPLAT_VECTOR.
Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().
◆ performIntrinsicCombine()
|
static |
Definition at line 25391 of file AArch64ISelLowering.cpp.
References llvm::ISD::ABDS, llvm::ISD::ABDU, AbstractManglingParser< Derived, Alloc >::Ops, llvm::ISD::ADD, llvm::ISD::AND, llvm::AArch64CC::ANY_ACTIVE, combineAcrossLanesIntrinsic(), combineSVEBitSel(), combineSVEReductionFP(), combineSVEReductionInt(), combineSVEReductionOrderedFP(), convertMergedOpToPredOp(), llvm::TargetLowering::DAGCombinerInfo::DAG, DL, llvm::AArch64CC::FIRST_ACTIVE, llvm::ISD::FMAXIMUM, llvm::ISD::FMAXNUM, llvm::ISD::FMINIMUM, llvm::ISD::FMINNUM, llvm::ISD::GET_ACTIVE_LANE_MASK, llvm::SelectionDAG::getCondCode(), llvm::SelectionDAG::getConstant(), getIntrinsicID(), llvm::SelectionDAG::getNode(), getPTest(), I, llvm::ISD::INTRINSIC_WO_CHAIN, llvm::ISD::isBuildVectorAllZeros(), llvm::AArch64CC::LAST_ACTIVE, llvm_unreachable, lowerPredCounterWhile(), LowerSVEIntrinsicDUP(), LowerSVEIntrinsicEXT(), LowerSVEIntrinsicIndex(), N, llvm::ISD::OR, llvm::ISD::SADDSAT, llvm::ISD::SETEQ, llvm::ISD::SETGE, llvm::ISD::SETGT, llvm::ISD::SETLE, llvm::ISD::SETLT, llvm::ISD::SETNE, llvm::ISD::SETOEQ, llvm::ISD::SETOGE, llvm::ISD::SETOGT, llvm::ISD::SETUGE, llvm::ISD::SETUGT, llvm::ISD::SETULE, llvm::ISD::SETULT, llvm::ISD::SETUNE, llvm::ISD::SETUO, llvm::ISD::SPLAT_VECTOR, llvm::ISD::SSUBSAT, llvm::ISD::SUB, tryCombineCRC32(), tryCombineFADDReductionWithZero(), tryCombineFixedPointConvert(), tryCombineLongOpWithDup(), tryCombineNeonFcvtFP16ToI16(), tryCombineShiftImm(), tryConvertSVEWideCompare(), llvm::ISD::UADDSAT, llvm::ISD::USUBSAT, llvm::ISD::VSELECT, and llvm::ISD::XOR.
Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().
◆ performIntToFpCombine()
|
static |
Definition at line 21877 of file AArch64ISelLowering.cpp.
References llvm::cast(), llvm::MemSDNode::getAlign(), llvm::LoadSDNode::getBasePtr(), llvm::MemSDNode::getChain(), llvm::MachineMemOperand::getFlags(), llvm::SelectionDAG::getLoad(), llvm::MemSDNode::getMemOperand(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::MemSDNode::getPointerInfo(), llvm::EVT::getSizeInBits(), llvm::SDValue::hasOneUse(), llvm::AArch64Subtarget::isNeonAvailable(), llvm::ISD::isNormalLoad(), llvm::Load, N, performVectorCompareAndMaskUnaryOpCombine(), llvm::SelectionDAG::ReplaceAllUsesOfValueWith(), llvm::ISD::SINT_TO_FP, and tryToReplaceScalarFPConversionWithSVE().
Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().
◆ performLastTrueTestVectorCombine()
|
static |
Definition at line 22510 of file AArch64ISelLowering.cpp.
References assert(), llvm::TargetLowering::DAGCombinerInfo::DAG, llvm::ISD::EXTRACT_VECTOR_ELT, llvm::SelectionDAG::getConstant(), llvm::details::FixedOrScalableQuantity< LeafTy, ValueTy >::getKnownMinValue(), getPTest(), llvm::SDValue::getValueType(), llvm::EVT::getVectorElementCount(), llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalize(), llvm::EVT::isScalableVectorOf(), llvm::AArch64CC::LAST_ACTIVE, llvm::PatternMatch::m_Add(), llvm::MIPatternMatch::m_AllOnes(), llvm::PatternMatch::m_SpecificInt(), llvm::PatternMatch::m_VScale(), llvm::PatternMatch::m_ZExtOrSelf(), N, and llvm::SDPatternMatch::sd_match().
Referenced by performExtractVectorEltCombine().
◆ performLD1Combine()
|
static |
Definition at line 26313 of file AArch64ISelLowering.cpp.
References AbstractManglingParser< Derived, Alloc >::Ops, DL, llvm::details::FixedOrScalableQuantity< LeafTy, ValueTy >::getKnownMinValue(), llvm::SelectionDAG::getMergeValues(), llvm::SelectionDAG::getNode(), llvm::EVT::getSizeInBits(), getSVEContainerType(), llvm::SelectionDAG::getValueType(), llvm::SelectionDAG::getVTList(), llvm::EVT::isInteger(), llvm::Load, N, Opc, llvm::AArch64::SVEBitsPerBlock, and llvm::ISD::TRUNCATE.
Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().
◆ performLD1ReplicateCombine()
|
static |
Definition at line 26365 of file AArch64ISelLowering.cpp.
References AbstractManglingParser< Derived, Alloc >::Ops, llvm::ISD::BITCAST, llvm::EVT::changeTypeToInteger(), DL, llvm::SelectionDAG::getMergeValues(), llvm::SelectionDAG::getNode(), llvm::EVT::isFloatingPoint(), llvm::Load, and N.
Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().
◆ performLDNT1Combine()
|
static |
Definition at line 26339 of file AArch64ISelLowering.cpp.
References AbstractManglingParser< Derived, Alloc >::Ops, llvm::ISD::BITCAST, llvm::cast(), llvm::EVT::changeTypeToInteger(), DL, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getMaskedLoad(), llvm::SelectionDAG::getMergeValues(), llvm::SelectionDAG::getNode(), llvm::SDNode::getOperand(), llvm::SelectionDAG::getPOISON(), llvm::EVT::isFloatingPoint(), N, llvm::ISD::NON_EXTLOAD, and llvm::ISD::UNINDEXED.
Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().
◆ performLegalizedVectorDeinterleaveCombine()
|
static |
Definition at line 31391 of file AArch64ISelLowering.cpp.
References llvm::AfterLegalizeTypes, llvm::SelectionDAG::areNonVolatileConsecutiveLoads(), llvm::TargetLowering::DAGCombinerInfo::CombineTo(), DL, llvm::dyn_cast(), llvm::enumerate(), llvm::SelectionDAG::getContext(), llvm::TargetLowering::DAGCombinerInfo::getDAGCombineLevel(), llvm::details::FixedOrScalableQuantity< LeafTy, ValueTy >::getFixedValue(), llvm::SelectionDAG::getMachineFunction(), llvm::MachineFunction::getMachineMemOperand(), llvm::SelectionDAG::getMemIntrinsicNode(), llvm::MemSDNode::getMemOperand(), llvm::EVT::getStoreSize(), llvm::SelectionDAG::getSubtarget(), llvm::SDValue::getValue(), llvm::EVT::getVectorElementCount(), llvm::EVT::getVectorElementType(), llvm::EVT::getVectorVT(), llvm::SelectionDAG::getVTList(), llvm::SDValue::hasOneUse(), I, llvm::ISD::INTRINSIC_W_CHAIN, llvm::EVT::is128BitVector(), llvm::EVT::is64BitVector(), llvm::AArch64Subtarget::isNeonAvailable(), llvm::Load, N, llvm::SmallVectorTemplateBase< T, bool >::push_back(), llvm::SelectionDAG::ReplaceAllUsesOfValueWith(), and llvm::ISD::VECTOR_DEINTERLEAVE.
Referenced by performVectorDeinterleaveCombine().
◆ performMaskedGatherScatterCombine()
|
static |
Definition at line 28546 of file AArch64ISelLowering.cpp.
References AbstractManglingParser< Derived, Alloc >::Ops, llvm::cast(), llvm::Data, DL, llvm::dyn_cast(), findMoreOptimalIndexType(), llvm::MaskedGatherScatterSDNode::getBasePtr(), llvm::MemSDNode::getChain(), llvm::MaskedGatherSDNode::getExtensionType(), llvm::MaskedGatherScatterSDNode::getIndex(), llvm::MaskedGatherScatterSDNode::getIndexType(), llvm::MaskedGatherScatterSDNode::getMask(), llvm::SelectionDAG::getMaskedGather(), llvm::SelectionDAG::getMaskedHistogram(), llvm::SelectionDAG::getMaskedScatter(), llvm::MemSDNode::getMemOperand(), llvm::MemSDNode::getMemoryVT(), llvm::MaskedGatherSDNode::getPassThru(), llvm::MaskedGatherScatterSDNode::getScale(), llvm::MaskedScatterSDNode::getValue(), llvm::SelectionDAG::getVTList(), llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalize(), llvm::MaskedScatterSDNode::isTruncatingStore(), and N.
Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().
◆ performMINMAXCombine()
|
static |
Definition at line 31709 of file AArch64ISelLowering.cpp.
References llvm::SelectionDAG::getNode(), getReductionForOpcode(), llvm::TargetLoweringBase::isOperationLegalOrCustom(), llvm::SDPatternMatch::m_ConstInt(), llvm::PatternMatch::m_ExtractElt(), llvm::MIPatternMatch::m_OneUse(), llvm::PatternMatch::m_Specific(), llvm::PatternMatch::m_SpecificInt(), llvm::SDPatternMatch::m_SpecificVT(), llvm::PatternMatch::m_Value(), N, llvm::SDPatternMatch::sd_match(), and trySQDMULHCombine().
Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().
◆ performMSTORECombine()
|
static |
Definition at line 28365 of file AArch64ISelLowering.cpp.
References llvm::ISD::BITCAST, llvm::cast(), llvm::EVT::changeVectorElementType(), DL, llvm::MaskedLoadStoreSDNode::getAddressingMode(), llvm::MaskedStoreSDNode::getBasePtr(), llvm::MemSDNode::getChain(), llvm::SelectionDAG::getContext(), llvm::MaskedStoreSDNode::getMask(), llvm::SelectionDAG::getMaskedStore(), llvm::MemSDNode::getMemOperand(), llvm::MemSDNode::getMemoryVT(), llvm::AArch64Subtarget::getMinSVEVectorSizeInBits(), llvm::getNumElementsFromSVEPredPattern(), llvm::MaskedStoreSDNode::getOffset(), getPTrue(), llvm::EVT::getSizeInBits(), llvm::MaskedStoreSDNode::getValue(), llvm::SDValue::getValueType(), llvm::EVT::getVectorElementType(), llvm::Value::hasOneUse(), llvm::MaskedStoreSDNode::isCompressingStore(), llvm::TargetLoweringBase::isOperationLegalOrCustomOrPromote(), llvm::EVT::isScalableVector(), llvm::MaskedStoreSDNode::isTruncatingStore(), llvm::MaskedLoadStoreSDNode::isUnindexed(), N, trySimplifySrlAddToRshrnb(), llvm::ISD::VECTOR_COMPRESS, and llvm::EVT::widenIntegerVectorElementType().
Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().
◆ performMulCombine()
|
static |
Definition at line 21520 of file AArch64ISelLowering.cpp.
References llvm::ISD::ADD, llvm::Add, llvm::APInt::ashr(), llvm::BitWidth, C(), llvm::cast(), llvm::APInt::countr_zero(), DL, llvm::dyn_cast(), llvm::APInt::getBitWidth(), llvm::SelectionDAG::getConstant(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDNode::getOpcode(), llvm::SDNode::getOperand(), llvm::SDValue::getValueSizeInBits(), llvm::APInt::getZExtValue(), llvm::SDNode::hasOneUse(), llvm::isa(), llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalizeOps(), llvm::APInt::isNegative(), llvm::APInt::isNonNegative(), isOnlyUsedAsMemoryAddress(), llvm::APInt::isPowerOf2(), isSignExtended(), IsSVECntIntrinsic(), isZeroExtended(), llvm::APInt::logBase2(), llvm::ISD::MUL, N, performMulRdsvlCombine(), performMulVectorCmpZeroCombine(), performMulVectorExtendCombine(), performVectorExtCombine(), RHS, llvm::APInt::sdivrem(), llvm::APInt::sge(), llvm::ISD::SHL, llvm::APInt::sle(), llvm::ISD::SUB, llvm::Sub, std::swap(), llvm::ISD::TRUNCATE, and X.
Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().
◆ performMULLCombine()
|
static |
Definition at line 31265 of file AArch64ISelLowering.cpp.
References N, llvm::Intrinsic::not_intrinsic, tryCombineLongOpWithDup(), and tryCombineMULLWithUZP1().
Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().
◆ performMulRdsvlCombine()
|
static |
Definition at line 21367 of file AArch64ISelLowering.cpp.
References llvm::abs(), B(), llvm::cast(), llvm::countr_zero(), DL, llvm::SDNodeFlags::Exact, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::SDNode::getOpcode(), llvm::SDValue::getOpcode(), llvm::SDNode::getOperand(), llvm::SelectionDAG::getSignedConstant(), llvm::Log2_32_Ceil(), Mul, llvm::ISD::SHL, and llvm::ISD::SRL.
Referenced by performMulCombine().
◆ performMulVectorCmpZeroCombine()
|
static |
Definition at line 21414 of file AArch64ISelLowering.cpp.
References llvm::ISD::AND, DL, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getContext(), llvm::EVT::getIntegerVT(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOperand(), llvm::EVT::getScalarSizeInBits(), llvm::SelectionDAG::getSetCC(), llvm::EVT::getVectorElementCount(), llvm::EVT::getVectorVT(), llvm::ISD::isConstantSplatVector(), N, llvm::ISD::SETGT, llvm::ISD::SRL, llvm::V1, and llvm::V3.
Referenced by performMulCombine().
◆ performMulVectorExtendCombine()
|
static |
Combines a mul(dup(sext/zext)) node pattern into mul(sext/zext(dup)) making use of the vector SExt/ZExt rather than the scalar SExt/ZExt.
Definition at line 21339 of file AArch64ISelLowering.cpp.
References DL, llvm::SelectionDAG::getNode(), Mul, and performBuildShuffleExtendCombine().
Referenced by performMulCombine().
◆ performNegCSelCombine()
|
static |
Definition at line 23504 of file AArch64ISelLowering.cpp.
References DL, llvm::SelectionDAG::getNegative(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDValue::getValueType(), llvm::SDNode::hasOneUse(), isNegatedInteger(), and N.
Referenced by performAddSubCombine().
◆ performNegSignMaskedAndCombine()
|
static |
Definition at line 24092 of file AArch64ISelLowering.cpp.
References llvm::ISD::AND, llvm::any_of(), llvm::SelectionDAG::ComputeNumSignBits(), DL, llvm::SelectionDAG::getNegative(), llvm::SelectionDAG::getNode(), llvm::EVT::getScalarSizeInBits(), llvm::isNullConstant(), N, llvm::ISD::SUB, std::swap(), and X.
Referenced by performAddSubCombine().
◆ performNEONPostLDSTCombine()
|
static |
Target-specific DAG combine function for NEON load/store intrinsics to merge base address updates.
Definition at line 28599 of file AArch64ISelLowering.cpp.
References AbstractManglingParser< Derived, Alloc >::Ops, llvm::ISD::ADD, llvm::ArrayRef(), llvm::cast(), llvm::TargetLowering::DAGCombinerInfo::CombineTo(), llvm::dyn_cast(), llvm::SelectionDAG::getMemIntrinsicNode(), llvm::MemSDNode::getMemOperand(), llvm::MemSDNode::getMemoryVT(), llvm::SDValue::getNode(), llvm::User::getOperand(), llvm::SelectionDAG::getRegister(), llvm::SDValue::getResNo(), llvm::EVT::getSizeInBits(), llvm::Use::getUser(), llvm::EVT::getVectorNumElements(), llvm::SelectionDAG::getVTList(), llvm::SDNode::hasPredecessorHelper(), llvm::SmallPtrSetImpl< PtrType >::insert(), llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalize(), llvm::TargetLowering::DAGCombinerInfo::isCalledByLegalizer(), llvm_unreachable, N, llvm::SmallVectorTemplateBase< T, bool >::push_back(), uint64_t, and llvm::SDNode::uses().
Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().
◆ performNVCASTCombine()
|
static |
Get rid of unnecessary NVCASTs (that don't change the type). Under LE a NVCAST is converted to a bitcast.
Definition at line 30394 of file AArch64ISelLowering.cpp.
References llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getUNDEF(), llvm::AArch64Subtarget::isLittleEndian(), and N.
Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().
◆ performORCombine()
|
static |
Definition at line 22129 of file AArch64ISelLowering.cpp.
References llvm::TargetLowering::DAGCombinerInfo::DAG, N, performANDORCSELCombine(), performANDORDUPNOTCombine(), and tryCombineToREV().
Referenced by llvm::AArch64TargetLowering::PerformDAGCombine(), llvm::LoongArchTargetLowering::PerformDAGCombine(), llvm::MipsSETargetLowering::PerformDAGCombine(), llvm::MipsTargetLowering::PerformDAGCombine(), and llvm::RISCVTargetLowering::PerformDAGCombine().
◆ performOrXorChainCombine()
|
static |
Definition at line 12774 of file AArch64ISelLowering.cpp.
References llvm::ISD::AND, llvm::any_of(), llvm::SmallVectorTemplateCommon< T, typename >::begin(), llvm::cast(), Cond, DL, llvm::SmallVectorTemplateCommon< T, typename >::end(), F, llvm::find_if(), llvm::First, llvm::MachineFunction::getFunction(), llvm::SelectionDAG::getMachineFunction(), llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getSetCC(), hasCheapXorImmediateThatNeedsCondCmpReg(), hasLegalCmpImmediate(), hasLegalCondCmpImmediate(), I, llvm::isNullConstant(), isOrXorChain(), LHS, MaxXors, N, llvm::ISD::OR, RHS, llvm::ISD::SETCC, llvm::ISD::SETEQ, llvm::ISD::SETNE, and llvm::SmallVectorTemplateCommon< T, typename >::size().
Referenced by performSETCCCombine().
◆ performPostLD1Combine()
|
static |
Target-specific DAG combine function for post-increment LD1 (lane) and post-increment LD1R.
Definition at line 27099 of file AArch64ISelLowering.cpp.
References AbstractManglingParser< Derived, Alloc >::Ops, llvm::ISD::ADD, llvm::cast(), llvm::TargetLowering::DAGCombinerInfo::CombineTo(), llvm::TargetLowering::DAGCombinerInfo::DAG, llvm::dyn_cast(), llvm::ISD::FMA, llvm::ISD::FMUL, llvm::SelectionDAG::getMemIntrinsicNode(), llvm::MemSDNode::getMemOperand(), llvm::MemSDNode::getMemoryVT(), llvm::SDValue::getNode(), llvm::User::getOperand(), llvm::SelectionDAG::getRegister(), llvm::SDValue::getResNo(), llvm::EVT::getScalarSizeInBits(), llvm::Use::getUser(), llvm::EVT::getVectorElementType(), llvm::EVT::getVectorNumElements(), llvm::SelectionDAG::getVTList(), llvm::SDNode::hasPredecessorHelper(), llvm::SmallPtrSetImpl< PtrType >::insert(), llvm::EVT::is128BitVector(), llvm::EVT::is64BitVector(), llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalizeOps(), llvm::isNullOrNullSplat(), N, llvm::SmallVectorTemplateBase< T, bool >::push_back(), uint64_t, llvm::SDNode::uses(), and llvm::Vector.
Referenced by performDUPCombine(), and performInsertVectorEltCombine().
◆ performPredicateLoadCombine()
|
static |
Definition at line 31739 of file AArch64ISelLowering.cpp.
References llvm::cast(), DL, llvm::SelectionDAG::getLoad(), llvm::SDValue::getNode(), llvm::EVT::getSizeInBits(), llvm::SelectionDAG::getSubtarget(), llvm::AArch64Subtarget::getSVEPredicateSizeInBits(), llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalize(), llvm::EVT::isFixedLengthVectorOf(), llvm::ISD::isNormalLoad(), llvm::AArch64Subtarget::isSVEorStreamingSVEAvailable(), llvm::Load, llvm::PatternMatch::m_BitCast(), llvm::SDPatternMatch::m_InsertSubvector(), llvm::PatternMatch::m_SpecificInt(), llvm::SDPatternMatch::m_SpecificVT(), llvm::PatternMatch::m_Undef(), llvm::PatternMatch::m_Value(), llvm::SelectionDAG::makeEquivalentMemoryOrdering(), N, and llvm::SDPatternMatch::sd_match().
Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().
◆ performPTestFirstCombine()
|
static |
Definition at line 31278 of file AArch64ISelLowering.cpp.
References llvm::ISD::CONCAT_VECTORS, DL, llvm::SelectionDAG::getNode(), llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalize(), isLane0KnownActive(), and N.
Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().
◆ performReinterpretCastCombine()
Definition at line 22172 of file AArch64ISelLowering.cpp.
References llvm::SDValue::getValueType(), and N.
Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().
◆ performRNDRCombine()
|
static |
Definition at line 31636 of file AArch64ISelLowering.cpp.
References A(), B(), DL, FlagsVT, getCondCode(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getMergeValues(), llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getVTList(), llvm::SelectionDAG::getZExtOrTrunc(), N, and llvm::AArch64CC::NE.
Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().
◆ performScalarToVectorCombine()
|
static |
Definition at line 31305 of file AArch64ISelLowering.cpp.
References AbstractManglingParser< Derived, Alloc >::Ops, llvm::TargetLowering::DAGCombinerInfo::DAG, DL, llvm::ISD::EXTRACT_VECTOR_ELT, llvm::SelectionDAG::getBuildVector(), llvm::SelectionDAG::getExtractSubvector(), llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getNodeIfExists(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SelectionDAG::getPOISON(), llvm::SDValue::getValueType(), llvm::EVT::getVectorNumElements(), llvm::TargetLowering::DAGCombinerInfo::isAfterLegalizeDAG(), llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalizeOps(), llvm::EVT::isFixedLengthVector(), llvm::isIntOrFPConstant(), llvm::isNullConstant(), N, llvm::Undef, and llvm::ISD::ZERO_EXTEND.
Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().
◆ performScatterStoreCombine()
|
static |
Definition at line 30526 of file AArch64ISelLowering.cpp.
References AbstractManglingParser< Derived, Alloc >::Ops, llvm::ISD::ANY_EXTEND, assert(), llvm::sampleprof::Base, llvm::ISD::BITCAST, DL, llvm::details::FixedOrScalableQuantity< LeafTy, ValueTy >::getKnownMinValue(), llvm::SelectionDAG::getNode(), llvm::EVT::getScalarSizeInBits(), getScaledOffsetForBitWidth(), llvm::EVT::getSimpleVT(), llvm::EVT::getSizeInBits(), llvm::MVT::getSizeInBits(), getSVEContainerType(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::SDValue::getValue(), llvm::SelectionDAG::getValueType(), llvm::EVT::getVectorElementType(), llvm::SelectionDAG::getVTList(), llvm::EVT::isFloatingPoint(), llvm::MVT::isFloatingPoint(), llvm::EVT::isScalableVector(), llvm::TargetLoweringBase::isTypeLegal(), isValidImmForSVEVecImmAddrMode(), N, llvm::Offset, llvm::AArch64::SVEBitsPerBlock, and std::swap().
Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().
◆ performSELECT_CCCombine()
|
static |
Definition at line 29670 of file AArch64ISelLowering.cpp.
References assert(), llvm::ISD::BITCAST, llvm::cast(), Cond, DL, llvm::SelectionDAG::getNode(), llvm::isAllOnesConstant(), llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalize(), llvm::EVT::isFixedLengthVectorOf(), llvm::isNullConstant(), LHS, N, RHS, llvm::ISD::SELECT_CC, llvm::ISD::SETEQ, llvm::ISD::SETNE, and std::swap().
Referenced by llvm::AArch64TargetLowering::PerformDAGCombine(), and llvm::LoongArchTargetLowering::PerformDAGCombine().
◆ performSelectCombine()
|
static |
A vector select: "(select vL, vR, (setcc LHS, RHS))" is best performed with the compare-mask instructions rather than going via NZCV, even if LHS and RHS are really scalar. This replaces any scalar setcc in the above pattern with a vector one followed by a DUP shuffle on the result.
Definition at line 30227 of file AArch64ISelLowering.cpp.
References assert(), llvm::ISD::BITCAST, llvm::EVT::changeVectorElementTypeToInteger(), llvm::TargetLowering::DAGCombinerInfo::DAG, DL, llvm::SelectionDAG::getContext(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SelectionDAG::getSelect(), llvm::EVT::getSizeInBits(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::SDValue::getValueType(), llvm::EVT::getVectorNumElements(), llvm::SelectionDAG::getVectorShuffle(), llvm::EVT::getVectorVT(), llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalize(), llvm::EVT::isFloatingPoint(), llvm::EVT::isScalableVT(), llvm::TargetLoweringBase::isTypeLegal(), llvm::EVT::isVector(), LHS, N, RHS, llvm::ISD::SCALAR_TO_VECTOR, and llvm::ISD::SETCC.
Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().
◆ performSetccAddFolding()
|
static |
Definition at line 23312 of file AArch64ISelLowering.cpp.
References SetCCInfo::AArch64, llvm::ISD::ADD, assert(), AArch64SetCCInfo::CC, GenericSetCCInfo::CC, AArch64SetCCInfo::Cmp, DL, SetCCInfo::Generic, getAArch64Cmp(), llvm::SelectionDAG::getConstant(), llvm::AArch64CC::getInvertedCondCode(), llvm::SelectionDAG::getNode(), llvm::SDNode::getOperand(), llvm::ISD::getSetCCInverse(), llvm::SDNode::getValueType(), llvm::SDValue::getValueType(), SetCCInfoAndKind::Info, SetCCInfoAndKind::IsAArch64, isSetCCOrZExtSetCC(), LHS, GenericSetCCInfo::Opnd0, GenericSetCCInfo::Opnd1, RHS, and std::swap().
Referenced by performAddSubLongCombine().
◆ performSETCCCARRYCombine()
|
static |
Definition at line 29636 of file AArch64ISelLowering.cpp.
References llvm::ISD::AND, assert(), llvm::cast(), Cond, DL, llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDValue::getResNo(), llvm::SelectionDAG::getSetCC(), llvm::isa(), llvm::isNullConstant(), N, llvm::ISD::OR, llvm::ISD::SETCCCARRY, llvm::ISD::SETEQ, llvm::ISD::SETNE, llvm::ISD::SETULT, and llvm::ISD::USUBO.
Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().
◆ performSETCCCombine()
|
static |
Definition at line 29538 of file AArch64ISelLowering.cpp.
References llvm::ISD::AND, assert(), llvm::ISD::BITCAST, llvm::cast(), Cond, DL, getCondCode(), llvm::SelectionDAG::getConstant(), llvm::EVT::getFixedSizeInBits(), llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getNOT(), llvm::SelectionDAG::getSetCC(), llvm::SelectionDAG::getSignedConstant(), llvm::EVT::getVectorElementType(), llvm::SelectionDAG::getZExtOrTrunc(), llvm::isa(), llvm::TargetLowering::DAGCombinerInfo::isAfterLegalizeDAG(), llvm::isAllOnesConstant(), llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalize(), llvm::ISD::isConstantSplatVector(), llvm::EVT::isFixedLengthVector(), llvm::EVT::isInteger(), llvm::isNullConstant(), llvm::APInt::isOne(), llvm::isOneConstant(), llvm::EVT::isScalarInteger(), isZerosVector(), LHS, N, performOrXorChainCombine(), RHS, llvm::ISD::SETCC, llvm::ISD::SETEQ, llvm::ISD::SETGE, llvm::ISD::SETGT, llvm::ISD::SETNE, llvm::ISD::SIGN_EXTEND, llvm::ISD::SRL, tryToWidenSetCCOperands(), uint64_t, llvm::ISD::VECREDUCE_AND, llvm::ISD::VECREDUCE_OR, and llvm::ISD::ZERO_EXTEND.
Referenced by llvm::AArch64TargetLowering::PerformDAGCombine(), llvm::LoongArchTargetLowering::PerformDAGCombine(), llvm::MipsSETargetLowering::PerformDAGCombine(), and llvm::RISCVTargetLowering::PerformDAGCombine().
◆ performSetccMergeZeroCombine()
|
static |
Definition at line 29800 of file AArch64ISelLowering.cpp.
References llvm::ISD::AND, assert(), llvm::cast(), llvm::SelectionDAG::ComputeNumSignBits(), Cond, llvm::TargetLowering::DAGCombinerInfo::DAG, DL, llvm::SDValue::getConstantOperandVal(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOperand(), llvm::ISD::INSERT_SUBVECTOR, llvm::TargetLowering::DAGCombinerInfo::isAfterLegalizeDAG(), isAllActivePredicate(), llvm::isNullConstant(), isSignExtInReg(), llvm::SDValue::isUndef(), isZerosVector(), LHS, N, performSetCCPunpkCombine(), RHS, llvm::ISD::SETNE, and llvm::ISD::SIGN_EXTEND.
Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().
◆ performSetCCPunpkCombine()
|
static |
Definition at line 29750 of file AArch64ISelLowering.cpp.
References llvm::cast(), Cond, llvm::ISD::EXTRACT_SUBVECTOR, llvm::SDNode::getConstantOperandVal(), llvm::SDValue::getConstantOperandVal(), llvm::SDNode::getOpcode(), llvm::SDValue::getOpcode(), llvm::SDNode::getOperand(), llvm::SDValue::getOperand(), llvm::SDNode::getValueType(), isZerosVector(), LHS, N, RHS, llvm::ISD::SETNE, and llvm::ISD::SIGN_EXTEND.
Referenced by performSetccMergeZeroCombine().
◆ performSHLCombine()
|
static |
If the operand is a bitwise AND with a constant RHS, and the shift has a constant RHS and is the only use, we can pull it out of the shift, i.e.
(shl (and X, C1), C2) -> (and (shl X, C2), (shl C1, C2))
We prefer this canonical form to match existing isel patterns.
Definition at line 31598 of file AArch64ISelLowering.cpp.
References llvm::ISD::ADD, llvm::ISD::AND, DL, llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDNode::getOperand(), llvm::SDValue::hasOneUse(), llvm::isa(), llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalizeOps(), N, llvm::ISD::SETCC, llvm::ISD::SHL, llvm::ISD::SUB, and X.
Referenced by llvm::AArch64TargetLowering::PerformDAGCombine(), llvm::LoongArchTargetLowering::PerformDAGCombine(), llvm::MipsSETargetLowering::PerformDAGCombine(), llvm::MipsTargetLowering::PerformDAGCombine(), and llvm::RISCVTargetLowering::PerformDAGCombine().
◆ performSignExtendInRegCombine()
|
static |
Definition at line 30746 of file AArch64ISelLowering.cpp.
References AbstractManglingParser< Derived, Alloc >::Ops, llvm::cast(), llvm::TargetLowering::DAGCombinerInfo::CombineTo(), DL, EnableCombineMGatherIntrinsics, llvm::SelectionDAG::getAllOnesConstant(), llvm::SelectionDAG::getContext(), llvm::EVT::getDoubleNumVectorElementsVT(), llvm::SelectionDAG::getNode(), llvm::SDNode::getOperand(), llvm::SDValue::getOperand(), llvm::EVT::getScalarType(), llvm::EVT::getSizeInBits(), llvm::SDValue::getValue(), llvm::SDValue::getValueType(), llvm::SelectionDAG::getValueType(), llvm::EVT::getVectorElementType(), llvm::SelectionDAG::getVTList(), I, llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalizeOps(), llvm::isNullConstant(), llvm::isOneConstant(), N, Opc, and llvm::ISD::SIGN_EXTEND_INREG.
Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().
◆ performSignExtendSetCCCombine()
|
static |
Definition at line 25803 of file AArch64ISelLowering.cpp.
References assert(), llvm::cast(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDNode::getOperand(), llvm::SDValue::getOperand(), llvm::SelectionDAG::getSetCC(), llvm::SDNode::getValueType(), isCheapToExtend(), llvm::EVT::isInteger(), N, llvm::ISD::SETCC, llvm::ISD::SIGN_EXTEND, and llvm::ISD::ZERO_EXTEND.
Referenced by performExtendCombine().
◆ performSpliceCombine()
|
static |
Definition at line 26633 of file AArch64ISelLowering.cpp.
Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().
◆ performST1Combine()
|
static |
Definition at line 26386 of file AArch64ISelLowering.cpp.
References AbstractManglingParser< Derived, Alloc >::Ops, llvm::ISD::ANY_EXTEND, llvm::ISD::BITCAST, llvm::Data, DL, llvm::SelectionDAG::getNode(), getSVEContainerType(), llvm::SelectionDAG::getValueType(), llvm::EVT::isFloatingPoint(), and N.
Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().
◆ performSTNT1Combine()
|
static |
Definition at line 26412 of file AArch64ISelLowering.cpp.
References llvm::ISD::BITCAST, llvm::cast(), llvm::EVT::changeTypeToInteger(), llvm::Data, DL, llvm::SelectionDAG::getMaskedStore(), llvm::SelectionDAG::getNode(), llvm::SDNode::getOperand(), llvm::SelectionDAG::getPOISON(), llvm::EVT::isFloatingPoint(), N, and llvm::ISD::UNINDEXED.
Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().
◆ performSubAddMULCombine()
|
static |
Definition at line 24127 of file AArch64ISelLowering.cpp.
References llvm::ISD::ADD, llvm::Add, llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SelectionDAG::isConstantIntBuildVectorOrConstantInt(), llvm::M1(), llvm::ISD::MUL, N, llvm::ISD::SUB, llvm::Sub, and X.
Referenced by performAddSubCombine().
◆ performSubNegAndOneCombine()
|
static |
Definition at line 24594 of file AArch64ISelLowering.cpp.
References llvm::ISD::AND, DL, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getSetCC(), llvm::EVT::isFixedLengthVector(), isOneVector(), isZerosVector(), N, llvm::ISD::SETNE, and llvm::ISD::SUB.
Referenced by performAddSubCombine().
◆ performSubsToAndsCombine()
|
static |
Definition at line 28919 of file AArch64ISelLowering.cpp.
References AbstractManglingParser< Derived, Alloc >::Ops, assert(), DL, llvm::dyn_cast(), llvm::AArch64CC::EQ, llvm::ConstantSDNode::getAPIntValue(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::SDNode::getOperand(), llvm::SDValue::getValue(), llvm::SDNode::getValueType(), llvm::SDNode::getVTList(), llvm::AArch64CC::HI, llvm::APInt::isMask(), llvm::APInt::isPowerOf2(), llvm::AArch64CC::LO, N, and llvm::AArch64CC::NE.
Referenced by performCONDCombine().
◆ performSubWithBorrowCombine()
|
static |
Definition at line 24518 of file AArch64ISelLowering.cpp.
References DL, llvm::dyn_cast(), llvm::ConstantSDNode::getAPIntValue(), llvm::SelectionDAG::getConstant(), getCSETCondCode(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDValue::getValue(), llvm::SDValue::hasOneUse(), llvm::AArch64CC::HI, llvm::AArch64_AM::isLegalCmpImmed(), llvm::AArch64CC::LO, N, llvm::ISD::SUB, and llvm::ISD::ZERO_EXTEND.
Referenced by performAddSubCombine().
◆ performSunpkloCombine()
|
static |
Definition at line 27081 of file AArch64ISelLowering.cpp.
References llvm::SelectionDAG::getContext(), llvm::SelectionDAG::getExtractSubvector(), llvm::EVT::getHalfNumVectorElementsVT(), llvm::SelectionDAG::getNode(), llvm::SDNode::getValueType(), N, and llvm::ISD::SIGN_EXTEND.
Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().
◆ performSVEAndCombine()
|
static |
Definition at line 22183 of file AArch64ISelLowering.cpp.
References llvm::ISD::AND, C(), llvm::cast(), llvm::TargetLowering::DAGCombinerInfo::DAG, DL, llvm::dyn_cast(), EnableCombineMGatherIntrinsics, llvm::ISD::EXTLOAD, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDNode::getOperand(), llvm::EVT::getSizeInBits(), llvm::SDNode::getValueType(), llvm::EVT::getVectorElementType(), isAllActivePredicate(), llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalizeOps(), isConstantSplatVectorMaskForType(), N, Opc, llvm::ISD::SPLAT_VECTOR, uint64_t, and llvm::ISD::ZEXTLOAD.
Referenced by performANDCombine().
◆ performSVEMulAddSubCombine()
|
static |
Definition at line 24163 of file AArch64ISelLowering.cpp.
References llvm::ISD::ADD, llvm::cast(), convertFromScalableVector(), convertToScalableVector(), llvm::TargetLowering::DAGCombinerInfo::DAG, llvm::ISD::EXTRACT_SUBVECTOR, llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDNode::getOperand(), llvm::SelectionDAG::getSubtarget(), llvm::SDValue::getValueType(), llvm::SDValue::hasOneUse(), llvm::TargetLowering::DAGCombinerInfo::isAfterLegalizeDAG(), llvm::EVT::isScalableVector(), N, and llvm::ISD::SUB.
Referenced by performAddSubCombine().
◆ performTBZCombine()
|
static |
Definition at line 29947 of file AArch64ISelLowering.cpp.
References assert(), DL, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), getTestBitOperand(), and N.
Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().
◆ performTruncateCombine()
|
static |
Definition at line 23937 of file AArch64ISelLowering.cpp.
References assert(), llvm::cast(), llvm::SelectionDAG::ComputeNumSignBits(), DL, llvm::ISD::EXTRACT_VECTOR_ELT, llvm::SelectionDAG::getExtractVectorElt(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::EVT::getScalarSizeInBits(), llvm::EVT::getScalarType(), llvm::SDValue::getValueType(), llvm::SDValue::hasOneUse(), llvm::EVT::is64BitVector(), llvm::isa(), llvm::TargetLowering::DAGCombinerInfo::isAfterLegalizeDAG(), llvm::EVT::isFixedLengthVector(), llvm::EVT::isScalableVector(), N, llvm::ISD::OR, and llvm::ISD::TRUNCATE.
Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().
◆ performUADDVAddCombine()
|
static |
Definition at line 20953 of file AArch64ISelLowering.cpp.
References A(), llvm::ISD::ADD, assert(), llvm::ISD::EXTRACT_SUBVECTOR, llvm::SDValue::getConstantOperandVal(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDValue::getValueType(), llvm::EVT::getVectorNumElements(), llvm::EVT::isScalableVector(), performUADDVAddCombine(), llvm::ISD::SIGN_EXTEND, and llvm::ISD::ZERO_EXTEND.
Referenced by performUADDVAddCombine(), and performUADDVCombine().
◆ performUADDVCombine()
|
static |
Definition at line 21041 of file AArch64ISelLowering.cpp.
References A(), llvm::ISD::ADD, assert(), llvm::SelectionDAG::computeKnownBits(), llvm::APInt::getAllOnes(), llvm::SelectionDAG::getNode(), llvm::MVT::getVectorNumElements(), llvm::KnownBits::isZero(), N, performUADDVAddCombine(), and performUADDVZextCombine().
Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().
◆ performUADDVZextCombine()
|
static |
Definition at line 21003 of file AArch64ISelLowering.cpp.
References A(), llvm::ISD::ADD, assert(), Concat, llvm::ISD::CONCAT_VECTORS, llvm::SelectionDAG::getContext(), llvm::EVT::getDoubleNumVectorElementsVT(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::EVT::getScalarSizeInBits(), llvm::EVT::getSimpleVT(), llvm::SDValue::getValueType(), llvm_unreachable, llvm::MVT::SimpleTy, and llvm::ISD::ZERO_EXTEND.
Referenced by performUADDVCombine().
◆ performUnpackCombine()
|
static |
Definition at line 26643 of file AArch64ISelLowering.cpp.
References assert(), llvm::cast(), llvm::EVT::changeVectorElementType(), DL, llvm::MaskedLoadStoreSDNode::getAddressingMode(), llvm::MaskedLoadSDNode::getBasePtr(), llvm::MemSDNode::getChain(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getContext(), llvm::MaskedLoadSDNode::getExtensionType(), llvm::MaskedLoadSDNode::getMask(), llvm::SelectionDAG::getMaskedLoad(), llvm::MemSDNode::getMemOperand(), llvm::MemSDNode::getMemoryVT(), llvm::AArch64Subtarget::getMinSVEVectorSizeInBits(), llvm::SDValue::getNode(), llvm::getNumElementsFromSVEPredPattern(), llvm::MaskedLoadSDNode::getOffset(), llvm::MaskedLoadSDNode::getPassThru(), getPTrue(), llvm::EVT::getSizeInBits(), llvm::SelectionDAG::getUNDEF(), llvm::SDValue::getValue(), llvm::EVT::getVectorElementType(), llvm::SDNode::isUndef(), llvm::MaskedLoadStoreSDNode::isUnindexed(), isZerosVector(), llvm::ISD::MLOAD, N, llvm::SelectionDAG::ReplaceAllUsesOfValueWith(), llvm::ISD::SEXTLOAD, and llvm::ISD::ZEXTLOAD.
Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().
◆ performUzpCombine()
|
static |
Definition at line 26789 of file AArch64ISelLowering.cpp.
References llvm::ISD::BITCAST, Concat, llvm::ISD::CONCAT_VECTORS, DL, llvm::ISD::EXTRACT_SUBVECTOR, llvm::SDNode::getAsZExtVal(), llvm::SelectionDAG::getBitcast(), llvm::SDValue::getConstantOperandVal(), llvm::SelectionDAG::getContext(), llvm::SelectionDAG::getDataLayout(), llvm::EVT::getDoubleNumVectorElementsVT(), llvm::SelectionDAG::getExtractSubvector(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SelectionDAG::getPOISON(), llvm::EVT::getScalarSizeInBits(), llvm::SDValue::getSimpleValueType(), llvm::EVT::getSimpleVT(), llvm::SelectionDAG::getUNDEF(), llvm::SDValue::getValueType(), llvm::EVT::getVectorMinNumElements(), llvm::isConstOrConstSplat(), isHalvingTruncateAndConcatOfLegalIntScalableType(), llvm::DataLayout::isLittleEndian(), isNVCastToHalfWidthElements(), llvm::EVT::isSimple(), llvm::SDValue::isUndef(), llvm_unreachable, N, llvm::peekThroughBitcasts(), llvm::MVT::SimpleTy, llvm::ISD::SRL, llvm::ISD::TRUNCATE, tryCombineExtendRShTrunc(), trySimplifySrlAddToRshrnb(), uint64_t, and X.
Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().
◆ performVecReduceAddCntpCombine()
|
static |
Definition at line 20802 of file AArch64ISelLowering.cpp.
References DL, llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getTargetConstant(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::SDValue::getValueType(), llvm::SelectionDAG::getZExtOrTrunc(), llvm::ISD::INTRINSIC_WO_CHAIN, llvm::EVT::isScalableVectorOf(), llvm::TargetLoweringBase::isTypeLegal(), N, and llvm::ISD::ZERO_EXTEND.
Referenced by performVecReduceAddCombine().
◆ performVecReduceAddCombine()
|
static |
Definition at line 20822 of file AArch64ISelLowering.cpp.
References A(), llvm::ISD::ADD, B(), llvm::ISD::CONCAT_VECTORS, DL, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getContext(), llvm::SelectionDAG::getExtractSubvector(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::EVT::getScalarSizeInBits(), llvm::SDValue::getValueType(), llvm::EVT::getVectorElementType(), llvm::EVT::getVectorNumElements(), llvm::EVT::getVectorVT(), I, llvm::EVT::isScalableVT(), llvm::ISD::MUL, N, performVecReduceAddCntpCombine(), performVecReduceAddCombineWithUADDLP(), llvm::SmallVectorTemplateBase< T, bool >::push_back(), llvm::ISD::SIGN_EXTEND, std::swap(), llvm::ISD::VECREDUCE_ADD, and llvm::ISD::ZERO_EXTEND.
Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().
◆ performVecReduceAddCombineWithUADDLP()
|
static |
Definition at line 20608 of file AArch64ISelLowering.cpp.
References llvm::ISD::ABDS, llvm::ISD::ABDU, llvm::ISD::ADD, DL, llvm::SelectionDAG::getExtractSubvector(), llvm::SelectionDAG::getNode(), llvm::SDNode::getOpcode(), llvm::SDValue::getOpcode(), llvm::SDNode::getOperand(), llvm::SDValue::getOperand(), llvm::SDNode::getValueType(), llvm::ISD::isAbsOpcode(), N, llvm::ISD::SIGN_EXTEND, llvm::ISD::SUB, llvm::ISD::VECREDUCE_ADD, and llvm::ISD::ZERO_EXTEND.
Referenced by performVecReduceAddCombine().
◆ performVecReduceBitwiseCombine()
|
static |
Definition at line 29523 of file AArch64ISelLowering.cpp.
References DL, llvm::SDValue::getValueType(), getVectorBitwiseReduce(), llvm::EVT::getVectorElementType(), llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalize(), llvm::EVT::isFixedLengthVector(), llvm::EVT::isPow2VectorType(), and N.
Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().
◆ performVectorCompareAndMaskUnaryOpCombine()
|
static |
Definition at line 21775 of file AArch64ISelLowering.cpp.
References llvm::ISD::AND, llvm::ISD::BITCAST, DL, llvm::dyn_cast(), llvm::SelectionDAG::getNode(), llvm::EVT::getSizeInBits(), llvm::EVT::isVector(), N, and llvm::ISD::SETCC.
Referenced by performIntToFpCombine().
◆ performVectorDeinterleaveCombine()
|
static |
Definition at line 31458 of file AArch64ISelLowering.cpp.
References llvm::EVT::changeVectorElementType(), llvm::TargetLowering::DAGCombinerInfo::CombineTo(), DL, llvm::dyn_cast(), llvm::ISD::EXTRACT_SUBVECTOR, llvm::SelectionDAG::getConstant(), llvm::SDNode::getConstantOperandVal(), llvm::SelectionDAG::getContext(), llvm::details::FixedOrScalableQuantity< LeafTy, ValueTy >::getKnownMinValue(), llvm::SelectionDAG::getMemIntrinsicNode(), llvm::MemSDNode::getMemOperand(), llvm::MemSDNode::getMemoryVT(), llvm::SDValue::getNode(), llvm::SDNode::getOpcode(), llvm::SDNode::getOperand(), llvm::EVT::getSizeInBits(), llvm::SelectionDAG::getTargetConstant(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::SDValue::getValue(), llvm::EVT::getVectorMinNumElements(), llvm::SelectionDAG::getVTList(), I, llvm::ISD::INTRINSIC_W_CHAIN, llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalize(), isDeinterleave4ForWideningUToFP(), llvm::ISD::isNormalLoad(), llvm::ISD::isNormalMaskedLoad(), llvm::EVT::isScalableVector(), llvm::TargetLoweringBase::isTypeLegal(), isZerosVector(), llvm::Load, N, performLegalizedVectorDeinterleaveCombine(), llvm::SmallVectorTemplateBase< T, bool >::push_back(), and llvm::SelectionDAG::ReplaceAllUsesOfValueWith().
Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().
◆ performVectorExtCombine()
|
static |
Definition at line 21452 of file AArch64ISelLowering.cpp.
References DL, llvm::SelectionDAG::getContext(), llvm::EVT::getHalfSizedIntegerVT(), llvm::SelectionDAG::getNode(), llvm::EVT::getScalarType(), llvm::EVT::getSizeInBits(), llvm::SDValue::getValueType(), llvm::EVT::getVectorElementCount(), llvm::EVT::getVectorVT(), llvm::EVT::isFixedLengthVector(), llvm::ISD::MUL, N, S1, llvm::ISD::SIGN_EXTEND, and llvm::ISD::ZERO_EXTEND.
Referenced by performAddSubCombine(), and performMulCombine().
◆ performVectorShiftCombine()
|
static |
Optimize a vector shift instruction and its operand if shifted out bits are not used.
Definition at line 27049 of file AArch64ISelLowering.cpp.
References assert(), llvm::SelectionDAG::ComputeNumSignBits(), llvm::TargetLowering::DAGCombinerInfo::DAG, llvm::APInt::getLowBitsSet(), N, and llvm::TargetLowering::SimplifyDemandedBits().
Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().
◆ performVSelectCombine()
|
static |
Definition at line 30146 of file AArch64ISelLowering.cpp.
References AbstractManglingParser< Derived, Alloc >::Ops, llvm::ISD::BITCAST, llvm::cast(), llvm::EVT::changeVectorElementTypeToInteger(), combineToExtendBoolVectorInReg(), llvm::TargetLowering::DAGCombinerInfo::DAG, DL, llvm::ElementCount::getFixed(), llvm::SelectionDAG::getNode(), llvm::SDValue::getNumOperands(), llvm::SDValue::getOpcode(), llvm::SDNode::getOperand(), llvm::SDValue::getOperand(), llvm::SelectionDAG::getSelect(), llvm::SelectionDAG::getSetCC(), llvm::EVT::getSizeInBits(), llvm::SDValue::getValueType(), llvm::EVT::getVectorElementCount(), llvm::EVT::getVectorElementType(), llvm::SDValue::hasOneUse(), isAllActivePredicate(), isAllInactivePredicate(), llvm::EVT::isFloatingPoint(), isMergePassthruOpcode(), llvm::SDValue::isUndef(), llvm::EVT::isVector(), N, llvm::SDNode::op_values(), performVSelectMaskedShiftCombine(), performVselectPowCombine(), llvm::ISD::SETCC, llvm::ISD::SIGN_EXTEND, llvm::ISD::TRUNCATE, trySwapVSelectOperands(), and llvm::ISD::VSELECT.
Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().
◆ performVSelectMaskedShiftCombine()
|
static |
Definition at line 30127 of file AArch64ISelLowering.cpp.
References foldMaskedShiftToUSHL(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), matchZeroSelectArm(), N, llvm::ISD::SHL, and llvm::ISD::SRL.
Referenced by performVSelectCombine().
◆ performVselectPowCombine()
|
static |
Definition at line 30011 of file AArch64ISelLowering.cpp.
References assert(), Cond, llvm::TargetLowering::DAGCombinerInfo::DAG, DL, llvm::ISD::FPOW, llvm::SelectionDAG::getConstantFP(), llvm::SDNode::getFlags(), llvm::LibcallLoweringInfo::getLibcallImpl(), llvm::TargetLoweringBase::getLibcallLoweringInfo(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDNode::getOperand(), llvm::SelectionDAG::getTargetLoweringInfo(), N, and llvm::ISD::VSELECT.
Referenced by performVSelectCombine().
◆ performXorCombine()
|
static |
Definition at line 21076 of file AArch64ISelLowering.cpp.
References foldVectorXorShiftIntoCmp(), llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalizeOps(), and N.
Referenced by llvm::AArch64TargetLowering::PerformDAGCombine().
◆ performZExtDeinterleaveShuffleCombine()
|
static |
Definition at line 25845 of file AArch64ISelLowering.cpp.
References llvm::all_of(), llvm::ISD::AND, assert(), DL, llvm::dyn_cast(), llvm::ISD::EXTRACT_SUBVECTOR, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::EVT::getScalarSizeInBits(), llvm::EVT::getVectorNumElements(), llvm::ShuffleVectorInst::isDeInterleaveMaskOfFactor(), llvm::EVT::isFixedLengthVector(), N, llvm::ISD::SRL, and llvm::ISD::ZERO_EXTEND.
Referenced by performExtendCombine().
◆ performZExtUZPCombine()
|
static |
Definition at line 25911 of file AArch64ISelLowering.cpp.
References llvm::ISD::AND, DL, llvm::ISD::EXTRACT_SUBVECTOR, llvm::SelectionDAG::getConstant(), llvm::APInt::getLowBitsSet(), llvm::SelectionDAG::getNode(), llvm::EVT::getScalarSizeInBits(), llvm::SelectionDAG::getTargetConstant(), llvm::EVT::getVectorNumElements(), llvm::ISD::isConstantSplatVector(), N, and llvm::ISD::ZERO_EXTEND.
Referenced by performExtendCombine().
◆ reassociateAddSubLong()
|
static |
Definition at line 23539 of file AArch64ISelLowering.cpp.
References A(), llvm::ISD::ADD, B(), DL, llvm::MVT::getIntegerVT(), llvm::SelectionDAG::getNode(), llvm::EVT::getScalarSizeInBits(), isEssentiallyExtractHighSubvector(), llvm::PatternMatch::m_Add(), llvm::SDPatternMatch::m_Node(), llvm::MIPatternMatch::m_OneUse(), llvm::SDPatternMatch::m_SpecificVectorElementVT(), llvm::PatternMatch::m_Sub(), llvm::PatternMatchHelpers::m_Unless(), llvm::PatternMatch::m_Value(), N, llvm::SDPatternMatch::sd_match(), llvm::ISD::SIGN_EXTEND, llvm::ISD::SUB, X, and llvm::ISD::ZERO_EXTEND.
Referenced by performAddSubLongCombine().
◆ reassociateCSELOperandsForCSE()
|
static |
Definition at line 29252 of file AArch64ISelLowering.cpp.
References llvm::ISD::ADD, Check, llvm::dyn_cast(), llvm::AArch64CC::EQ, FlagsVT, llvm::AArch64CC::GE, getCondCode(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), getSwappedCondition(), llvm::SDValue::getValue(), llvm::SelectionDAG::getVTList(), llvm::AArch64CC::GT, llvm::SDValue::hasOneUse(), llvm::AArch64CC::HI, llvm::AArch64CC::HS, llvm::AArch64CC::LE, llvm::AArch64CC::LO, llvm::AArch64CC::LS, llvm::AArch64CC::LT, N, llvm::AArch64CC::NE, llvm::SelectionDAG::ReplaceAllUsesWith(), llvm::ISD::SUB, std::swap(), X, and Y.
Referenced by performCSELCombine().
◆ ReconstructShuffleWithRuntimeMask()
| SDValue ReconstructShuffleWithRuntimeMask | ( | SDValue | Op, |
| SelectionDAG & | DAG ) |
Definition at line 14799 of file AArch64ISelLowering.cpp.
References llvm::ISD::AND, llvm::ISD::ANY_EXTEND, assert(), llvm::ISD::BUILD_VECTOR, llvm::cast(), llvm::ISD::CONCAT_VECTORS, DL, llvm::SmallVectorTemplateCommon< T, typename >::empty(), llvm::ISD::EXTRACT_VECTOR_ELT, llvm::SelectionDAG::getBuildVector(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDNode::getOperand(), llvm::SDValue::getOperand(), llvm::SelectionDAG::getPOISON(), llvm::SelectionDAG::getTargetConstant(), llvm::SDValue::getValueType(), llvm::EVT::getVectorNumElements(), llvm::ISD::INTRINSIC_WO_CHAIN, llvm::isa(), llvm::EVT::isScalableVector(), and llvm::SmallVectorTemplateBase< T, bool >::push_back().
◆ ReconstructTruncateFromBuildVector()
|
static |
Definition at line 15221 of file AArch64ISelLowering.cpp.
References assert(), llvm::sampleprof::Base, llvm::ISD::CONCAT_VECTORS, DL, llvm::ISD::EXTRACT_VECTOR_ELT, llvm::SDValue::getConstantOperandVal(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDValue::getValueType(), llvm::isa(), llvm::ISD::TRUNCATE, X, and Y.
◆ removeRedundantInsertVectorElt()
Definition at line 30980 of file AArch64ISelLowering.cpp.
References assert(), llvm::ISD::EXTRACT_VECTOR_ELT, llvm::SDValue::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDValue::getValueType(), llvm::ISD::INSERT_VECTOR_ELT, llvm::ISD::isConstantSplatVectorAllZeros(), isLanes1toNKnownZero(), llvm::isNullConstant(), and N.
Referenced by performInsertVectorEltCombine().
◆ ReplaceAddWithADDP()
|
static |
Definition at line 32420 of file AArch64ISelLowering.cpp.
References assert(), llvm::ISD::CONCAT_VECTORS, DL, llvm::dyn_cast(), E(), llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getPOISON(), llvm::EVT::getScalarType(), llvm::EVT::getVectorNumElements(), llvm::SelectionDAG::getVectorShuffle(), I, llvm::EVT::is256BitVector(), llvm::EVT::isFloatingPoint(), N, llvm::SmallVectorTemplateBase< T, bool >::push_back(), Results, llvm::SelectionDAG::SplitVector(), and X.
◆ ReplaceATOMIC_LOAD_128Results()
|
static |
Definition at line 32749 of file AArch64ISelLowering.cpp.
References AbstractManglingParser< Derived, Alloc >::Ops, assert(), llvm::ISD::ATOMIC_LOAD_AND, llvm::ISD::BUILD_PAIR, llvm::cast(), DL, llvm::SelectionDAG::getAllOnesConstant(), getAtomicLoad128Opcode(), llvm::SelectionDAG::getDataLayout(), llvm::SelectionDAG::getMachineNode(), llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getVTList(), llvm::Hi, llvm::DataLayout::isBigEndian(), llvm::Lo, N, Results, llvm::SelectionDAG::setNodeMemRefs(), llvm::SelectionDAG::SplitScalar(), std::swap(), and llvm::ISD::XOR.
◆ replaceBoolVectorBitcast()
|
static |
Definition at line 32329 of file AArch64ISelLowering.cpp.
References assert(), llvm::ISD::CONCAT_VECTORS, DL, llvm::SelectionDAG::getZExtOrTrunc(), I, llvm::EVT::isVectorOf(), N, and Results.
◆ ReplaceCMP_SWAP_128Results()
|
static |
Definition at line 32586 of file AArch64ISelLowering.cpp.
References AbstractManglingParser< Derived, Alloc >::Ops, llvm::Acquire, llvm::AcquireRelease, assert(), llvm::ISD::BUILD_PAIR, llvm::cast(), createGPRPairNode(), DL, llvm::SelectionDAG::getDataLayout(), llvm::SelectionDAG::getMachineNode(), llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getTargetExtractSubreg(), llvm::SelectionDAG::getVTList(), llvm::Hi, llvm::DataLayout::isBigEndian(), llvm_unreachable, llvm::Lo, llvm::Monotonic, N, llvm::Release, Results, llvm::SequentiallyConsistent, llvm::SelectionDAG::setNodeMemRefs(), llvm::SelectionDAG::SplitScalar(), and std::swap().
◆ ReplaceReductionResults()
|
static |
Definition at line 32468 of file AArch64ISelLowering.cpp.
References DL, llvm::SelectionDAG::getNode(), llvm::SelectionDAG::GetSplitDestVTs(), llvm::Hi, llvm::Lo, N, Results, and llvm::SelectionDAG::SplitVectorOperand().
◆ replaceSplatVectorStore()
|
static |
Replace a splat of a scalar to a vector store by scalar stores of the scalar value.
The load store optimizer pass will merge them to store pair stores. This has better performance than a splat of the scalar followed by a split vector store. Even if the stores are not merged it is four stores vs a dup, followed by an ext.b and two stores.
Definition at line 26511 of file AArch64ISelLowering.cpp.
References llvm::dyn_cast(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDValue::getValue(), llvm::SDValue::getValueType(), llvm::EVT::getVectorNumElements(), llvm::ConstantSDNode::getZExtValue(), I, llvm::ISD::INSERT_VECTOR_ELT, llvm::EVT::isFloatingPoint(), splitStoreSplat(), and uint64_t.
Referenced by splitStores().
◆ replaceZeroVectorStore()
|
static |
Replace a splat of zeros to a vector store by scalar stores of WZR/XZR.
The load store optimizer pass will merge them to store pair stores. This should be better than a movi to create the vector zero followed by a vector store if the zero constant is not re-used, since one instructions and one register live range will be removed.
For example, the final generated code should be:
stp xzr, xzr, [x0]
instead of:
movi v0.2d, #0 str q0, [x0]
Definition at line 26444 of file AArch64ISelLowering.cpp.
References llvm::ISD::BUILD_VECTOR, DL, llvm::SelectionDAG::getCopyFromReg(), llvm::SelectionDAG::getEntryNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::EVT::getSizeInBits(), llvm::SDValue::getValue(), llvm::SDValue::getValueType(), llvm::EVT::getVectorElementType(), llvm::EVT::getVectorNumElements(), llvm::SDValue::hasOneUse(), I, llvm::SelectionDAG::isBaseWithConstantOffset(), llvm::isNullConstant(), llvm::isNullFPConstant(), llvm::EVT::isScalableVector(), llvm::Offset, and splitStoreSplat().
Referenced by splitStores().
◆ resolveBuildVector()
|
static |
Definition at line 16434 of file AArch64ISelLowering.cpp.
References llvm::EVT::getSizeInBits(), llvm::SDNode::getValueType(), llvm::BuildVectorSDNode::isConstantSplat(), and llvm::APInt::zextOrTrunc().
Referenced by ConstantBuildVector(), and performANDCombine().
◆ rmwOpMayLowerToLibcall()
|
static |
Definition at line 33266 of file AArch64ISelLowering.cpp.
References llvm::Type::BFloatTyID, llvm::Type::DoubleTyID, llvm::Type::FloatTyID, llvm::Type::getScalarType(), llvm::Value::getType(), llvm::Type::getTypeID(), llvm::Type::HalfTyID, llvm::AtomicRMWInst::isFloatingPointOperation(), and llvm_unreachable.
Referenced by llvm::AArch64TargetLowering::shouldExpandAtomicRMWInIR().
◆ selectUmullSmull()
|
static |
Definition at line 6075 of file AArch64ISelLowering.cpp.
References llvm::SelectionDAG::ComputeNumSignBits(), DL, llvm::APInt::getHighBitsSet(), llvm::EVT::getScalarSizeInBits(), llvm::SDValue::getValueType(), isAddSubSExt(), isAddSubZExt(), isSignExtended(), isZeroExtended(), llvm::SelectionDAG::MaskedValueIsZero(), and std::swap().
◆ setInfoSVEStN()
|
static |
Set the IntrinsicInfo for the aarch64_sve_st<N> intrinsics.
Definition at line 18653 of file AArch64ISelLowering.cpp.
References llvm::CallBase::arg_size(), assert(), DL, llvm::dyn_cast(), llvm::CallBase::getArgOperand(), llvm::Type::getContext(), llvm::TargetLoweringBase::getMemValueType(), llvm::EVT::getScalarType(), llvm::Value::getType(), llvm::EVT::getVectorElementCount(), llvm::EVT::getVectorVT(), I, llvm::ISD::INTRINSIC_VOID, llvm::MachineMemOperand::MOStore, and llvm::MemoryLocation::UnknownSize.
Referenced by llvm::AArch64TargetLowering::getTgtMemIntrinsic().
◆ shouldBeAdjustedToZero()
|
static |
Definition at line 4452 of file AArch64ISelLowering.cpp.
References llvm::ISD::AND, C(), LHS, llvm::ISD::SETGE, llvm::ISD::SETGT, llvm::ISD::SETLE, and llvm::ISD::SETLT.
Referenced by getAArch64Cmp().
◆ shouldLowerTailCallStackArg()
|
static |
Check whether a stack argument requires lowering in a tail call.
Definition at line 10471 of file AArch64ISelLowering.cpp.
References llvm::ISD::ANY_EXTEND, llvm::ISD::BITCAST, llvm::dyn_cast(), llvm::LoadSDNode::getBasePtr(), llvm::EVT::getFixedSizeInBits(), llvm::MachineFunction::getFrameInfo(), llvm::MemSDNode::getMemoryVT(), llvm::MachineFrameInfo::getObjectOffset(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::MVT::getSizeInBits(), llvm::CCValAssign::getValVT(), llvm::SDNode::isAssert(), llvm::MachineFrameInfo::isImmutableObjectIndex(), uint64_t, and llvm::ISD::ZERO_EXTEND.
◆ simplifySetCCIntoEq()
|
static |
Definition at line 4939 of file AArch64ISelLowering.cpp.
References llvm::ISD::AND, DL, llvm::dyn_cast(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::ConstantSDNode::getZExtValue(), llvm::isPowerOf2_64(), LHS, RHS, llvm::ISD::SETEQ, llvm::ISD::SETULT, and uint64_t.
◆ skipExtensionForVectorMULL()
|
static |
Definition at line 5903 of file AArch64ISelLowering.cpp.
References assert(), llvm::SelectionDAG::getContext(), llvm::EVT::getHalfSizedIntegerVT(), llvm::SelectionDAG::getNode(), llvm::EVT::getScalarType(), llvm::EVT::getVectorElementCount(), llvm::EVT::getVectorVT(), llvm::EVT::is128BitVector(), N, and llvm::ISD::TRUNCATE.
◆ splitStores()
|
static |
Definition at line 26564 of file AArch64ISelLowering.cpp.
References llvm::ISD::ADD, llvm::cast(), DL, llvm::MemSDNode::getAlign(), llvm::StoreSDNode::getBasePtr(), llvm::MemSDNode::getChain(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getContext(), llvm::SelectionDAG::getExtractSubvector(), llvm::MachineMemOperand::getFlags(), llvm::MachineFunction::getFunction(), llvm::EVT::getHalfNumVectorElementsVT(), llvm::SelectionDAG::getMachineFunction(), llvm::MemSDNode::getMemOperand(), llvm::SelectionDAG::getNode(), llvm::MemSDNode::getPointerInfo(), llvm::EVT::getSizeInBits(), llvm::SelectionDAG::getStore(), llvm::SDValue::getValue(), llvm::StoreSDNode::getValue(), llvm::SDValue::getValueType(), llvm::EVT::getVectorNumElements(), llvm::Function::hasMinSize(), llvm::EVT::isFixedLengthVector(), llvm::LSBaseSDNode::isIndexed(), llvm::MemSDNode::isVolatile(), N, replaceSplatVectorStore(), and replaceZeroVectorStore().
◆ splitStoreSplat()
|
static |
Definition at line 26243 of file AArch64ISelLowering.cpp.
References llvm::ISD::ADD, assert(), llvm::cast(), llvm::commonAlignment(), DL, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::EVT::getSizeInBits(), llvm::SelectionDAG::getStore(), llvm::SDValue::getValue(), llvm::SDValue::getValueType(), llvm::MachinePointerInfo::getWithOffset(), llvm::isa(), llvm::Offset, and uint64_t.
Referenced by replaceSplatVectorStore(), and replaceZeroVectorStore().
◆ STATISTIC() [1/2]
| STATISTIC | ( | NumOptimizedImms | , |
| "Number of times immediates were optimized" | ) |
◆ STATISTIC() [2/2]
| STATISTIC | ( | NumTailCalls | , |
| "Number of tail calls" | ) |
◆ stripVRegCopies()
|
static |
Definition at line 3586 of file AArch64ISelLowering.cpp.
References assert(), DefMI, llvm::MachineRegisterInfo::getVRegDef(), and Reg.
Referenced by llvm::AArch64TargetLowering::fixupPtrauthDiscriminator().
◆ tryAdvSIMDModImm16()
|
static |
Definition at line 16531 of file AArch64ISelLowering.cpp.
References DL, llvm::AArch64_AM::encodeAdvSIMDModImmType5(), llvm::AArch64_AM::encodeAdvSIMDModImmType6(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::EVT::getSizeInBits(), llvm::SelectionDAG::getSubtarget(), llvm::AArch64_AM::isAdvSIMDModImmType5(), llvm::AArch64_AM::isAdvSIMDModImmType6(), llvm::EVT::isFixedLengthVector(), llvm::AArch64Subtarget::isNeonAvailable(), LHS, and uint64_t.
Referenced by ConstantBuildVector(), and performANDCombine().
◆ tryAdvSIMDModImm32()
|
static |
Definition at line 16478 of file AArch64ISelLowering.cpp.
References DL, llvm::AArch64_AM::encodeAdvSIMDModImmType1(), llvm::AArch64_AM::encodeAdvSIMDModImmType2(), llvm::AArch64_AM::encodeAdvSIMDModImmType3(), llvm::AArch64_AM::encodeAdvSIMDModImmType4(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::EVT::getSizeInBits(), llvm::SelectionDAG::getSubtarget(), llvm::AArch64_AM::isAdvSIMDModImmType1(), llvm::AArch64_AM::isAdvSIMDModImmType2(), llvm::AArch64_AM::isAdvSIMDModImmType3(), llvm::AArch64_AM::isAdvSIMDModImmType4(), llvm::EVT::isFixedLengthVector(), llvm::AArch64Subtarget::isNeonAvailable(), LHS, and uint64_t.
Referenced by ConstantBuildVector(), and performANDCombine().
◆ tryAdvSIMDModImm321s()
|
static |
Definition at line 16576 of file AArch64ISelLowering.cpp.
References DL, llvm::AArch64_AM::encodeAdvSIMDModImmType7(), llvm::AArch64_AM::encodeAdvSIMDModImmType8(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::EVT::getSizeInBits(), llvm::AArch64_AM::isAdvSIMDModImmType7(), llvm::AArch64_AM::isAdvSIMDModImmType8(), and uint64_t.
Referenced by ConstantBuildVector().
◆ tryAdvSIMDModImm64()
|
static |
Definition at line 16457 of file AArch64ISelLowering.cpp.
References DL, llvm::AArch64_AM::encodeAdvSIMDModImmType10(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::EVT::getSizeInBits(), llvm::AArch64_AM::isAdvSIMDModImmType10(), and uint64_t.
Referenced by ConstantBuildVector().
◆ tryAdvSIMDModImm8()
|
static |
Definition at line 16607 of file AArch64ISelLowering.cpp.
References DL, llvm::AArch64_AM::encodeAdvSIMDModImmType9(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::EVT::getSizeInBits(), llvm::AArch64_AM::isAdvSIMDModImmType9(), and uint64_t.
Referenced by ConstantBuildVector().
◆ tryAdvSIMDModImmFP()
|
static |
Definition at line 16628 of file AArch64ISelLowering.cpp.
References DL, llvm::AArch64_AM::encodeAdvSIMDModImmType11(), llvm::AArch64_AM::encodeAdvSIMDModImmType12(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::EVT::getSizeInBits(), llvm::AArch64_AM::isAdvSIMDModImmType11(), llvm::AArch64_AM::isAdvSIMDModImmType12(), and uint64_t.
Referenced by ConstantBuildVector().
◆ tryCombineCRC32()
|
static |
Definition at line 24875 of file AArch64ISelLowering.cpp.
References llvm::ISD::AND, llvm::dyn_cast(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::ConstantSDNode::getZExtValue(), llvm::ISD::INTRINSIC_WO_CHAIN, and N.
Referenced by performIntrinsicCombine().
◆ tryCombineExtendRShTrunc()
|
static |
Definition at line 26707 of file AArch64ISelLowering.cpp.
References assert(), DL, llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), getPredicateForVector(), llvm::Hi, llvm::Lo, and N.
Referenced by performUzpCombine().
◆ tryCombineFADDReductionWithZero()
|
static |
Optimize patterns where we insert zeros into vector lanes before a floating-point add reduction.
This handles both the aarch64.neon.faddv intrinsic and ISD::VECREDUCE_FADD/VECREDUCE_SEQ_FADD. When Start is set, the reduction is sequential (left-to-right), otherwise it is pairwise (tree).
Definition at line 25177 of file AArch64ISelLowering.cpp.
References llvm::SelectionDAG::canIgnoreSignBitOfZero(), llvm::SelectionDAG::computeKnownBits(), DL, llvm::ISD::FADD, llvm::APInt::getAllOnes(), llvm::SelectionDAG::getConstantFP(), llvm::SelectionDAG::getExtractVectorElt(), llvm::SelectionDAG::getNode(), llvm::APInt::getOneBitSet(), llvm::APInt::getSignedMaxValue(), llvm::EVT::getSizeInBits(), llvm::SDValue::getValueType(), llvm::EVT::getVectorElementType(), llvm::EVT::getVectorNumElements(), llvm::APInt::getZero(), I, llvm::EVT::isFixedLengthVector(), llvm::isPowerOf2_32(), llvm::APInt::isSubsetOf(), N, llvm::APInt::popcount(), llvm::SmallVectorTemplateBase< T, bool >::push_back(), llvm::APInt::setBit(), llvm::SmallVectorTemplateCommon< T, typename >::size(), and llvm::KnownBits::Zero.
Referenced by llvm::AArch64TargetLowering::PerformDAGCombine(), and performIntrinsicCombine().
◆ tryCombineFixedPointConvert()
|
static |
Definition at line 23109 of file AArch64ISelLowering.cpp.
References DL, llvm::ISD::EXTRACT_VECTOR_ELT, llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SDValue::getValueSizeInBits(), llvm::SDValue::getValueType(), llvm::ISD::INTRINSIC_WO_CHAIN, llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalizeOps(), and N.
Referenced by performIntrinsicCombine().
◆ tryCombineLongOpWithDup()
|
static |
Definition at line 24750 of file AArch64ISelLowering.cpp.
References assert(), llvm::SelectionDAG::getNode(), llvm::ISD::INTRINSIC_WO_CHAIN, llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalizeOps(), isEssentiallyExtractHighSubvector(), LHS, N, llvm::Intrinsic::not_intrinsic, RHS, and tryExtendDUPToExtractHigh().
Referenced by performExtendCombine(), performIntrinsicCombine(), and performMULLCombine().
◆ tryCombineMULLWithUZP1()
|
static |
Definition at line 31133 of file AArch64ISelLowering.cpp.
References llvm::ISD::BITCAST, DL, llvm::ISD::EXTRACT_SUBVECTOR, llvm::SelectionDAG::getContext(), llvm::EVT::getDoubleNumVectorElementsVT(), llvm::SelectionDAG::getExtractSubvector(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDNode::getOpcode(), llvm::SDValue::getOpcode(), llvm::SDNode::getOperand(), llvm::SDValue::getOperand(), llvm::User::getOperand(), llvm::SelectionDAG::getPOISON(), llvm::SDValue::getValueType(), llvm::EVT::getVectorNumElements(), llvm::SDNode::hasOneUse(), llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalizeOps(), isEssentiallyExtractHighSubvector(), llvm::isNullConstant(), llvm::SelectionDAG::isSplatValue(), LHS, N, llvm::SelectionDAG::ReplaceAllUsesWith(), RHS, llvm::SDValue::setNode(), llvm::ISD::TRUNCATE, llvm::SDNode::use_size(), llvm::SDNode::user_begin(), and llvm::SDNode::users().
Referenced by performMULLCombine().
◆ tryCombineNeonFcvtFP16ToI16()
|
static |
Definition at line 25112 of file AArch64ISelLowering.cpp.
References DL, llvm::SelectionDAG::getBitcast(), llvm::SelectionDAG::getNode(), N, and llvm::ISD::TRUNCATE.
Referenced by performIntrinsicCombine().
◆ tryCombineShiftImm()
|
static |
Definition at line 24783 of file AArch64ISelLowering.cpp.
References DL, llvm::dyn_cast(), llvm::SelectionDAG::getExtractVectorElt(), llvm::SelectionDAG::getNode(), llvm::APInt::getSExtValue(), llvm::SelectionDAG::getSignedConstant(), llvm::SelectionDAG::getTargetConstant(), llvm::BuildVectorSDNode::isConstantSplat(), llvm_unreachable, N, and llvm::ISD::SCALAR_TO_VECTOR.
Referenced by performIntrinsicCombine().
◆ tryCombineToREV()
|
static |
Definition at line 22024 of file AArch64ISelLowering.cpp.
References assert(), DL, llvm::SDValue::getConstantOperandVal(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDNode::getOperand(), llvm::SDValue::getOperand(), llvm::SelectionDAG::getPOISON(), llvm::EVT::getScalarSizeInBits(), llvm::EVT::is64BitVector(), llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalizeOps(), llvm::ISD::isConstantSplatVector(), llvm::EVT::isFixedLengthVector(), llvm::EVT::isScalableVector(), llvm::EVT::isVector(), N, llvm::ISD::OR, and std::swap().
Referenced by performORCombine().
◆ tryConvertSVEWideCompare()
|
static |
Definition at line 24954 of file AArch64ISelLowering.cpp.
References DL, llvm::dyn_cast(), llvm::SelectionDAG::getCondCode(), llvm::SelectionDAG::getConstant(), getIntrinsicID(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SelectionDAG::getSignedConstant(), Imm, llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalize(), llvm_unreachable, N, llvm::Splat, llvm::ISD::SPLAT_VECTOR, and uint64_t.
Referenced by performIntrinsicCombine().
◆ tryExtendDUPToExtractHigh()
|
static |
Definition at line 23172 of file AArch64ISelLowering.cpp.
References DL, llvm::ISD::EXTRACT_SUBVECTOR, llvm::SelectionDAG::getExtractSubvector(), llvm::SelectionDAG::getNode(), llvm::MVT::getVectorElementType(), llvm::MVT::getVectorNumElements(), llvm::MVT::getVectorVT(), llvm::MVT::is64BitVector(), and N.
Referenced by performAddSubLongCombine(), and tryCombineLongOpWithDup().
◆ tryFormConcatFromShuffle()
|
static |
Definition at line 15627 of file AArch64ISelLowering.cpp.
References llvm::cast(), llvm::ISD::CONCAT_VECTORS, DL, llvm::SelectionDAG::getContext(), llvm::SelectionDAG::getExtractSubvector(), llvm::EVT::getHalfNumVectorElementsVT(), llvm::SelectionDAG::getNode(), llvm::EVT::getVectorElementType(), isConcatMask(), and llvm::V1.
◆ tryLowerMultiVectorLoad()
|
static |
Definition at line 8005 of file AArch64ISelLowering.cpp.
References llvm::ISD::CONCAT_VECTORS, DL, llvm::LoadSDNode::getBasePtr(), llvm::MemSDNode::getChain(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), getPTrueAsCounter(), llvm::EVT::getSimpleVT(), getSVEMultiVectorInfo(), llvm::SDValue::getValue(), llvm::SDNode::getValueType(), I, llvm::ISD::INTRINSIC_W_CHAIN, isSimpleScalableLoadOrStore(), SVEMultiVectorInfo::LoadIntID, SVEMultiVectorInfo::NumVecs, llvm::SmallVectorTemplateBase< T, bool >::push_back(), SVEMultiVectorInfo::RegVT, and Results.
◆ tryLowerMultiVectorStore()
|
static |
Definition at line 7971 of file AArch64ISelLowering.cpp.
References AbstractManglingParser< Derived, Alloc >::Ops, DL, llvm::StoreSDNode::getBasePtr(), llvm::MemSDNode::getChain(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getExtractSubvector(), llvm::SelectionDAG::getMemIntrinsicNode(), llvm::MemSDNode::getMemOperand(), llvm::MemSDNode::getMemoryVT(), getPTrueAsCounter(), llvm::EVT::getSimpleVT(), getSVEMultiVectorInfo(), llvm::StoreSDNode::getValue(), llvm::MVT::getVectorMinNumElements(), llvm::SelectionDAG::getVTList(), llvm::ISD::INTRINSIC_VOID, isSimpleScalableLoadOrStore(), SVEMultiVectorInfo::NumVecs, SVEMultiVectorInfo::RegVT, and SVEMultiVectorInfo::StoreIntID.
◆ tryLowerSmallVectorExtLoad()
|
static |
Helper function to optimize loads of extended small vectors.
These patterns would otherwise get scalarized into inefficient sequences.
Definition at line 8133 of file AArch64ISelLowering.cpp.
References llvm::ISD::BITCAST, DL, llvm::ISD::EXTLOAD, llvm::SelectionDAG::getExtractSubvector(), llvm::MVT::getFloatingPointVT(), llvm::MVT::getIntegerVT(), llvm::SelectionDAG::getLoad(), llvm::SelectionDAG::getMergeValues(), llvm::SelectionDAG::getNode(), llvm::EVT::getScalarSizeInBits(), llvm::EVT::getStoreSizeInBits(), llvm::SelectionDAG::getSubtarget(), llvm::SDValue::getValue(), llvm::SDValue::getValueSizeInBits(), llvm::SDValue::getValueType(), llvm::EVT::getVectorNumElements(), llvm::MVT::getVectorVT(), isEligibleForSmallVectorLoadOpt(), llvm::Load, llvm::ISD::NON_EXTLOAD, llvm::ISD::SCALAR_TO_VECTOR, llvm::ISD::SEXTLOAD, llvm::ISD::SIGN_EXTEND, llvm::ISD::ZERO_EXTEND, and llvm::ISD::ZEXTLOAD.
◆ tryLowerToBSL()
|
static |
Definition at line 16810 of file AArch64ISelLowering.cpp.
References llvm::ISD::ADD, llvm::Add, llvm::ISD::AND, assert(), DL, llvm::dyn_cast(), llvm::ConstantSDNode::getAPIntValue(), llvm::SDNode::getAsAPIntVal(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDNode::getOperand(), llvm::EVT::getScalarSizeInBits(), llvm::SelectionDAG::getSubtarget(), llvm::EVT::getVectorNumElements(), llvm::ISD::isConstantSplatVector(), llvm::ISD::isConstantSplatVectorAllOnes(), llvm::ISD::isConstantSplatVectorAllZeros(), llvm::EVT::isScalableVector(), llvm::EVT::isVector(), N, llvm::O0, llvm::O1, llvm::ISD::SUB, llvm::Sub, and llvm::APInt::trunc().
◆ tryLowerToSLI()
|
static |
Definition at line 16717 of file AArch64ISelLowering.cpp.
References llvm::ISD::AND, assert(), C(), DL, llvm::dyn_cast(), llvm::ConstantSDNode::getAPIntValue(), llvm::APInt::getHighBitsSet(), llvm::APInt::getLowBitsSet(), llvm::SDValue::getNode(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::EVT::getScalarSizeInBits(), llvm::SelectionDAG::getTargetConstant(), Imm, isAllActivePredicate(), llvm::ISD::isConstantSplatVector(), llvm::EVT::isVector(), N, uint64_t, X, Y, and llvm::APInt::zextOrTrunc().
◆ trySimplifySrlAddToRshrnb()
|
static |
Definition at line 26747 of file AArch64ISelLowering.cpp.
References canLowerSRLToRoundingShiftForVT(), DL, llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getTargetConstant(), llvm::SDNode::getValueType(), and llvm::EVT::isScalableVector().
Referenced by performMSTORECombine(), and performUzpCombine().
◆ trySQDMULHCombine()
|
static |
Definition at line 23856 of file AArch64ISelLowering.cpp.
References DL, llvm::EVT::getFixedSizeInBits(), llvm::MVT::getIntegerVT(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::EVT::getScalarSizeInBits(), llvm::EVT::getScalarType(), llvm::ConstantSDNode::getSExtValue(), llvm::SDValue::getValueType(), llvm::EVT::getVectorNumElements(), llvm::MVT::getVectorVT(), llvm::SDValue::hasOneUse(), llvm::isConstOrConstSplat(), llvm::EVT::isPow2VectorType(), llvm::EVT::isScalableVector(), llvm::EVT::isVector(), llvm::ISD::MUL, Mul, N, llvm::ISD::SIGN_EXTEND, llvm::ISD::SMIN, llvm::ISD::SRA, and llvm::V1.
Referenced by performMINMAXCombine().
◆ trySVESplat64()
|
static |
Definition at line 16987 of file AArch64ISelLowering.cpp.
References DL, llvm::SelectionDAG::getConstant(), llvm::APInt::getHiBits(), llvm::APInt::getLoBits(), llvm::SelectionDAG::getNode(), llvm::APInt::getSExtValue(), llvm::APInt::getZExtValue(), llvm::EVT::is128BitVector(), llvm::AArch64_AM::isSVECpyDupImm(), llvm::AArch64_AM::isSVELogicalImm(), llvm::APInt::trunc(), and uint64_t.
Referenced by ConstantBuildVector().
◆ trySwapVSelectOperands()
|
static |
Definition at line 29978 of file AArch64ISelLowering.cpp.
References llvm::cast(), llvm::ISD::FADD, llvm::ISD::FMUL, llvm::ISD::FSUB, llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDValue::getOperand(), llvm::SelectionDAG::getSetCC(), llvm::ISD::getSetCCInverse(), llvm::SDValue::getValueType(), llvm::SDValue::hasOneUse(), N, llvm::ISD::SETCC, and llvm::ISD::VSELECT.
Referenced by performVSelectCombine().
◆ tryToConvertShuffleOfTbl2ToTbl4()
|
static |
Definition at line 16012 of file AArch64ISelLowering.cpp.
References llvm::ISD::BUILD_VECTOR, C(), DL, llvm::dyn_cast(), llvm::SelectionDAG::getBuildVector(), llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), llvm::SDValue::getOpcode(), llvm::SDNode::getOperand(), llvm::SDValue::getOperand(), llvm::SelectionDAG::getTargetConstant(), llvm::SDValue::getValueType(), I, and llvm::ISD::INTRINSIC_WO_CHAIN.
◆ tryToReplaceScalarFPConversionWithSVE()
|
static |
Tries to replace scalar FP <-> INT conversions with SVE in streaming functions, this can help to reduce the number of fmovs to/from GPRs.
Definition at line 21824 of file AArch64ISelLowering.cpp.
References llvm::EVT::bitsGT(), llvm::EVT::changeVectorElementType(), DL, llvm::ISD::FP_TO_SINT_SAT, llvm::ISD::FP_TO_UINT_SAT, llvm::SelectionDAG::getContext(), llvm::SelectionDAG::getExtractVectorElt(), llvm::SelectionDAG::getInsertVectorElt(), llvm::SelectionDAG::getNode(), getPackedSVEVectorVT(), llvm::SelectionDAG::getPOISON(), llvm::SDValue::getValueType(), llvm::SelectionDAG::getValueType(), llvm::EVT::getVectorElementType(), llvm::TargetLowering::DAGCombinerInfo::isBeforeLegalizeOps(), llvm::AArch64Subtarget::isStreaming(), llvm::AArch64Subtarget::isStreamingCompatible(), isSupportedType(), llvm::AArch64Subtarget::isSVEorStreamingSVEAvailable(), llvm::EVT::isVector(), N, and llvm::SmallVectorTemplateBase< T, bool >::push_back().
Referenced by performFpToIntCombine(), and performIntToFpCombine().
◆ tryToWidenSetCCOperands()
|
static |
Definition at line 29476 of file AArch64ISelLowering.cpp.
References llvm::any_of(), llvm::cast(), llvm::EVT::changeVectorElementType(), DL, llvm::SelectionDAG::getContext(), llvm::SelectionDAG::getNode(), llvm::SelectionDAG::getNodeIfExists(), llvm::SDNode::getOpcode(), llvm::EVT::getScalarSizeInBits(), llvm::SDNode::getValueType(), llvm::SelectionDAG::getVTList(), llvm::ISD::isConstantSplatVector(), llvm::EVT::isVector(), N, llvm::ISD::SETCC, llvm::ISD::SIGN_EXTEND, llvm::ISD::VSELECT, and llvm::ISD::ZERO_EXTEND.
Referenced by performSETCCCombine().
◆ tryWidenMaskForShuffle()
|
static |
Definition at line 15979 of file AArch64ISelLowering.cpp.
References llvm::cast(), DL, llvm::SelectionDAG::getBitcast(), llvm::EVT::getFixedSizeInBits(), llvm::MVT::getFloatingPointVT(), llvm::MVT::getIntegerVT(), llvm::SelectionDAG::getTargetLoweringInfo(), llvm::EVT::getVectorElementType(), llvm::EVT::getVectorNumElements(), llvm::SelectionDAG::getVectorShuffle(), llvm::MVT::getVectorVT(), llvm::EVT::isFloatingPoint(), llvm::TargetLoweringBase::isTypeLegal(), llvm::V1, and llvm::widenShuffleMaskElts().
◆ UseTlsOffset()
|
static |
Definition at line 33554 of file AArch64ISelLowering.cpp.
References llvm::IRBuilderBase::CreateCall(), llvm::IRBuilderBase::CreateConstGEP1_32(), llvm::IRBuilderBase::CreatePointerCast(), llvm::IRBuilderBase::GetInsertBlock(), llvm::IRBuilderBase::getInt8Ty(), llvm::Intrinsic::getOrInsertDeclaration(), llvm::BasicBlock::getParent(), llvm::GlobalValue::getParent(), llvm::IRBuilderBase::getPtrTy(), and llvm::Offset.
Referenced by llvm::AArch64TargetLowering::getIRStackGuard(), and llvm::AArch64TargetLowering::getSafeStackPointerLocation().
◆ valueToCarryFlag()
|
static |
Definition at line 4791 of file AArch64ISelLowering.cpp.
References DL, FlagsVT, llvm::SelectionDAG::getConstant(), llvm::SelectionDAG::getNode(), and llvm::SelectionDAG::getVTList().
Referenced by LowerADDSUBO_CARRY(), lowerADDSUBO_CARRY(), and LowerSETCCCARRY().
◆ WidenVector()
|
static |
WidenVector - Given a value in the V64 register class, produce the equivalent value in the V128 register class.
Definition at line 14779 of file AArch64ISelLowering.cpp.
References DL, llvm::SelectionDAG::getInsertSubvector(), llvm::SelectionDAG::getPOISON(), llvm::EVT::getSimpleVT(), llvm::SDValue::getValueType(), llvm::EVT::getVectorElementType(), llvm::EVT::getVectorNumElements(), and llvm::MVT::getVectorVT().
Referenced by constructDup(), GeneratePerfectShuffle(), and performConcatVectorsCombine().
Variable Documentation
◆ BrMergingBaseCostThresh
|
static |
◆ BrMergingCbzTbnzBias
|
static |
Referenced by llvm::AArch64TargetLowering::getJumpConditionMergingParams().
◆ BrMergingCcmpBias
|
static |
◆ BrMergingLikelyBias
|
static |
◆ BrMergingUnlikelyBias
|
static |
◆ CondCodeVT
|
constexpr |
Value type used for condition codes.
Definition at line 222 of file AArch64ISelLowering.cpp.
Referenced by getCondCode().
◆ EnableAArch64ELFLocalDynamicTLSGeneration
| cl::opt< bool > EnableAArch64ELFLocalDynamicTLSGeneration("aarch64-elf-ldtls-generation", cl::Hidden, cl::desc("Allow AArch64 Local Dynamic TLS code generation"), cl::init(false)) | ( | "aarch64-elf-ldtls-generation" | , |
| cl::Hidden | , | ||
| cl::desc("Allow AArch64 Local Dynamic TLS code generation") | , | ||
| cl::init(false) | ) |
Referenced by llvm::AArch64::getELFTLSModel(), and llvm::AArch64MCInstLower::lowerSymbolOperandELF().
◆ EnableCombineMGatherIntrinsics
|
static |
Referenced by performSignExtendInRegCombine(), and performSVEAndCombine().
◆ EnableExtToTBL
◆ EnableOptimizeLogicalImm
|
static |
Referenced by llvm::AArch64TargetLowering::targetShrinkDemandedConstant().
◆ EnableSVEFixedLengthBfloatSupport
|
static |
Referenced by llvm::AArch64TargetLowering::useSVEForFixedLengthVectorVT().
◆ EnableSVEGISel
◆ false
| false |
Simplify Addr given that the top byte of it is ignored by HW during / address translation.
static bool performTBISimplification(SDValue Addr, TargetLowering::DAGCombinerInfo &DCI, SelectionDAG &DAG) { const auto &Subtarget = DAG.getSubtarget<AArch64Subtarget>(); If MTE is enabled, TBI only applies to the top 4 bits. Both arm64 and arm64e processes on Darwin may run with MTE enabled. unsigned NumIgnoreBits = Subtarget.hasMTE() || Subtarget.isTargetDarwin() ? 4 : 8; APInt DemandedMask = APInt::getLowBitsSet(64, 64 - NumIgnoreBits); KnownBits Known; TargetLowering::TargetLoweringOpt TLO(DAG, !DCI.isBeforeLegalize(), !DCI.isBeforeLegalizeOps()); const TargetLowering &TLI = DAG.getTargetLoweringInfo(); if (TLI.SimplifyDemandedBits(Addr, DemandedMask, Known, TLO)) { DCI.CommitTargetLoweringOpt(TLO); return true; } return false; }
static SDValue foldTruncStoreOfExt(SelectionDAG &DAG, SDNode *N) { assert((N->getOpcode() == ISD::STORE || N->getOpcode() == ISD::MSTORE) && "Expected STORE dag node in input!");
if (auto Store = dyn_cast<StoreSDNode>(N)) { if (!Store->isTruncatingStore() || Store->isIndexed()) return SDValue(); SDValue Ext = Store->getValue(); auto ExtOpCode = Ext.getOpcode(); if (ExtOpCode != ISD::ZERO_EXTEND && ExtOpCode != ISD::SIGN_EXTEND && ExtOpCode != ISD::ANY_EXTEND) return SDValue(); SDValue Orig = Ext->getOperand(0); if (Store->getMemoryVT() != Orig.getValueType()) return SDValue(); return DAG.getStore(Store->getChain(), SDLoc(Store), Orig, Store->getBasePtr(), Store->getMemOperand()); }
return SDValue(); }
A custom combine to lower load <3 x i8> as the more efficient sequence below: ldrb wX, [x0, #2] ldrh wY, [x0] orr wX, wY, wX, lsl #16 fmov s0, wX
Note that an alternative sequence with even fewer (although usually more complex/expensive) instructions would be: ld1r.4h { v0 }, [x0], #2 ld1.b { v0 }[2], [x0]
Generating this sequence unfortunately results in noticeably worse codegen for code that extends the loaded v3i8, due to legalization breaking vector shuffle detection in a way that is very difficult to work around. TODO: Revisit once v3i8 legalization has been improved in general. static SDValue combineV3I8LoadExt(LoadSDNode *LD, SelectionDAG &DAG) { EVT MemVT = LD->getMemoryVT(); if (MemVT != EVT::getVectorVT(*DAG.getContext(), MVT::i8, 3) || LD->getBaseAlign() >= 4) return SDValue();
SDLoc DL(LD); MachineFunction &MF = DAG.getMachineFunction(); SDValue Chain = LD->getChain(); SDValue BasePtr = LD->getBasePtr(); MachineMemOperand *MMO = LD->getMemOperand(); assert(LD->getOffset().isUndef() && "undef offset expected");
Load 2 x i8, then 1 x i8. SDValue L16 = DAG.getLoad(MVT::i16, DL, Chain, BasePtr, MMO); TypeSize Offset2 = TypeSize::getFixed(2); SDValue L8 = DAG.getLoad(MVT::i8, DL, Chain, DAG.getMemBasePlusOffset(BasePtr, Offset2, DL), MF.getMachineMemOperand(MMO, 2, 1));
Extend to i32. SDValue Ext16 = DAG.getNode(ISD::ZERO_EXTEND, DL, MVT::i32, L16); SDValue Ext8 = DAG.getNode(ISD::ZERO_EXTEND, DL, MVT::i32, L8);
Pack 2 x i8 and 1 x i8 in an i32 and convert to v4i8. SDValue Shl = DAG.getNode(ISD::SHL, DL, MVT::i32, Ext8, DAG.getConstant(16, DL, MVT::i32)); SDValue Or = DAG.getNode(ISD::OR, DL, MVT::i32, Ext16, Shl); SDValue Cast = DAG.getNode(ISD::BITCAST, DL, MVT::v4i8, Or);
Extract v3i8 again. SDValue Extract = DAG.getExtractSubvector(DL, MemVT, Cast, 0); SDValue TokenFactor = DAG.getNode( ISD::TokenFactor, DL, MVT::Other, {SDValue(cast<SDNode>(L16), 1), SDValue(cast<SDNode>(L8), 1)}); return DAG.getMergeValues({Extract, TokenFactor}, DL); }
Perform TBI simplification if supported by the target and try to break up nontemporal loads larger than 256-bits loads for odd types so LDNPQ 256-bit load instructions can be selected. static SDValue performLOADCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, SelectionDAG &DAG, const AArch64Subtarget *Subtarget) { if (Subtarget->supportsAddressTopByteIgnored()) performTBISimplification(N->getOperand(1), DCI, DAG);
LoadSDNode *LD = cast<LoadSDNode>(N); EVT RegVT = LD->getValueType(0); EVT MemVT = LD->getMemoryVT(); const TargetLowering &TLI = DAG.getTargetLoweringInfo(); SDLoc DL(LD);
Cast ptr32 and ptr64 pointers to the default address space before a load. unsigned AddrSpace = LD->getAddressSpace(); if (AddrSpace == ARM64AS::PTR64 || AddrSpace == ARM64AS::PTR32_SPTR || AddrSpace == ARM64AS::PTR32_UPTR) { MVT PtrVT = TLI.getPointerTy(DAG.getDataLayout()); if (PtrVT != LD->getBasePtr().getSimpleValueType()) { SDValue Cast = DAG.getAddrSpaceCast(DL, PtrVT, LD->getBasePtr(), AddrSpace, 0); return DAG.getExtLoad(LD->getExtensionType(), DL, RegVT, LD->getChain(), Cast, LD->getPointerInfo(), MemVT, LD->getBaseAlign(), LD->getMemOperand()->getFlags()); } }
if (LD->isVolatile() || !Subtarget->isLittleEndian()) return SDValue(N, 0);
if (SDValue Res = combineV3I8LoadExt(LD, DAG)) return Res;
if (!LD->isNonTemporal()) return SDValue(N, 0);
if (MemVT.isScalableVector() || MemVT.getSizeInBits() <= 256 || MemVT.getSizeInBits() % 256 == 0 || 256 % MemVT.getScalarSizeInBits() != 0) return SDValue(N, 0);
SDValue Chain = LD->getChain(); SDValue BasePtr = LD->getBasePtr(); SDNodeFlags Flags = LD->getFlags(); SmallVector<SDValue, 4> LoadOps; SmallVector<SDValue, 4> LoadOpsChain; Replace any non temporal load over 256-bit with a series of 256 bit loads and a scalar/vector load less than 256. This way we can utilize 256-bit loads and reduce the amount of load instructions generated. MVT NewVT = MVT::getVectorVT(MemVT.getVectorElementType().getSimpleVT(), 256 / MemVT.getVectorElementType().getSizeInBits()); unsigned Num256Loads = MemVT.getSizeInBits() / 256; Create all 256-bit loads starting from offset 0 and up to Num256Loads-1*32. for (unsigned I = 0; I < Num256Loads; I++) { unsigned PtrOffset = I * 32; SDValue NewPtr = DAG.getMemBasePlusOffset( BasePtr, TypeSize::getFixed(PtrOffset), DL, Flags); Align NewAlign = commonAlignment(LD->getAlign(), PtrOffset); SDValue NewLoad = DAG.getLoad( NewVT, DL, Chain, NewPtr, LD->getPointerInfo().getWithOffset(PtrOffset), NewAlign, LD->getMemOperand()->getFlags(), LD->getAAInfo()); LoadOps.push_back(NewLoad); LoadOpsChain.push_back(SDValue(cast<SDNode>(NewLoad), 1)); }
Process remaining bits of the load operation. This is done by creating an UNDEF vector to match the size of the 256-bit loads and inserting the remaining load to it. We extract the original load type at the end using EXTRACT_SUBVECTOR instruction. unsigned BitsRemaining = MemVT.getSizeInBits() % 256; unsigned PtrOffset = (MemVT.getSizeInBits() - BitsRemaining) / 8; MVT RemainingVT = MVT::getVectorVT( MemVT.getVectorElementType().getSimpleVT(), BitsRemaining / MemVT.getVectorElementType().getSizeInBits()); SDValue NewPtr = DAG.getMemBasePlusOffset( BasePtr, TypeSize::getFixed(PtrOffset), DL, Flags); Align NewAlign = commonAlignment(LD->getAlign(), PtrOffset); SDValue RemainingLoad = DAG.getLoad(RemainingVT, DL, Chain, NewPtr, LD->getPointerInfo().getWithOffset(PtrOffset), NewAlign, LD->getMemOperand()->getFlags(), LD->getAAInfo()); SDValue PoisonVector = DAG.getPOISON(NewVT); SDValue ExtendedRemainingLoad = DAG.getInsertSubvector(DL, PoisonVector, RemainingLoad, 0); LoadOps.push_back(ExtendedRemainingLoad); LoadOpsChain.push_back(SDValue(cast<SDNode>(RemainingLoad), 1)); EVT ConcatVT = EVT::getVectorVT(*DAG.getContext(), MemVT.getScalarType(), LoadOps.size() * NewVT.getVectorNumElements()); SDValue ConcatVectors = DAG.getNode(ISD::CONCAT_VECTORS, DL, ConcatVT, LoadOps); Extract the original vector type size. SDValue ExtractSubVector = DAG.getExtractSubvector(DL, MemVT, ConcatVectors, 0); SDValue TokenFactor = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, LoadOpsChain); return DAG.getMergeValues({ExtractSubVector, TokenFactor}, DL); }
static EVT tryGetOriginalBoolVectorType(SDValue Op, int Depth = 0) { EVT VecVT = Op.getValueType(); assert(VecVT.isVectorOf(MVT::i1) && "Need boolean vector type.");
if (Depth > 3) return MVT::INVALID_SIMPLE_VALUE_TYPE;
We can get the base type from a vector compare or truncate. if (Op.getOpcode() == ISD::SETCC || Op.getOpcode() == ISD::TRUNCATE) return Op.getOperand(0).getValueType();
If an operand is a bool vector, continue looking. EVT BaseVT = MVT::INVALID_SIMPLE_VALUE_TYPE; for (SDValue Operand : Op->op_values()) { if (Operand.getValueType() != VecVT) continue;
EVT OperandVT = tryGetOriginalBoolVectorType(Operand, Depth + 1); if (!BaseVT.isSimple()) BaseVT = OperandVT; else if (OperandVT != BaseVT) return MVT::INVALID_SIMPLE_VALUE_TYPE; }
return BaseVT; }
static bool getBoolVectorBitcastCompare(SDValue Vec, SDValue RHS, const SDLoc &DL, SelectionDAG &DAG, SDValue &CompareLHS, SDValue &CompareRHS) { if (DAG.getDataLayout().isBigEndian()) return false;
EVT VecVT = Vec.getValueType(); assert(VecVT.isFixedLengthVector() && VecVT.getVectorElementType() == MVT::i1 && "Expected a fixed-length bool vector");
unsigned NumElts = VecVT.getVectorNumElements(); if (NumElts != 2 && NumElts != 4 && NumElts != 8 && NumElts != 16) return false;
auto getCanonicalCompareVecVT = [&]() { unsigned BitsPerElement = std::max(64 / NumElts, 8u); return MVT::getVectorVT(MVT::getIntegerVT(BitsPerElement), NumElts); };
EVT CompareVecVT = tryGetOriginalBoolVectorType(Vec); if (!CompareVecVT.isSimple() || CompareVecVT.getSizeInBits() > 128) { CompareVecVT = getCanonicalCompareVecVT(); } CompareVecVT = CompareVecVT.changeVectorElementTypeToInteger();
if (CompareVecVT.getSizeInBits() > 128) return false;
SDValue CompareBits = DAG.getSExtOrTrunc(Vec, DL, CompareVecVT); unsigned CompareBitsSize = CompareBits.getValueSizeInBits();
Use a canonical 64/128-bit vector representation before bitcasting to a scalar view. Some legal original vector types are smaller than 64-bit, which would make the direct scalar bitcasts below invalid. if (CompareBitsSize != 64 && CompareBitsSize != 128) { CompareVecVT = getCanonicalCompareVecVT(); CompareBits = DAG.getSExtOrTrunc(Vec, DL, CompareVecVT); CompareBitsSize = CompareBits.getValueSizeInBits(); }
if (CompareBitsSize != 64 && CompareBitsSize != 128) return false;
bool IsNull = isNullConstant(RHS); if (CompareBitsSize == 64) { CompareLHS = DAG.getBitcast(MVT::i64, CompareBits); CompareRHS = IsNull ? DAG.getConstant(0, DL, MVT::i64) : DAG.getAllOnesConstant(DL, MVT::i64); } else { SDValue PairwiseBits = DAG.getBitcast(MVT::v2i64, CompareBits); SDValue Lo = DAG.getExtractVectorElt(DL, MVT::i64, PairwiseBits, 0); SDValue Hi = DAG.getExtractVectorElt(DL, MVT::i64, PairwiseBits, 1); CompareLHS = DAG.getNode(ISD::ADD, DL, MVT::i64, Lo, Hi); CompareRHS = IsNull ? DAG.getConstant(0, DL, MVT::i64) : DAG.getSignedConstant(-2, DL, MVT::i64); }
return true; }
When converting a <N x iX> vector to <N x i1> to store or use as a scalar iN, we can use a trick that extracts the i^th bit from the i^th element and then performs a vector add to get a scalar bitmask. This requires that each element's bits are either all 1 or all 0. static SDValue vectorToScalarBitmask(SDValue ComparisonResult, SelectionDAG &DAG) { SDLoc DL(ComparisonResult); EVT VecVT = ComparisonResult.getValueType(); assert(VecVT.isVector() && "Must be a vector type");
unsigned NumElts = VecVT.getVectorNumElements(); if (NumElts != 2 && NumElts != 4 && NumElts != 8 && NumElts != 16) return SDValue();
if (VecVT.getVectorElementType() != MVT::i1 && !DAG.getTargetLoweringInfo().isTypeLegal(VecVT)) return SDValue();
If we can find the original types to work on instead of a vector of i1, we can avoid extend/extract conversion instructions. if (VecVT.getVectorElementType() == MVT::i1) { VecVT = tryGetOriginalBoolVectorType(ComparisonResult); if (!VecVT.isSimple()) { unsigned BitsPerElement = std::max(64 / NumElts, 8u); // >= 64-bit vector VecVT = MVT::getVectorVT(MVT::getIntegerVT(BitsPerElement), NumElts); } } VecVT = VecVT.changeVectorElementTypeToInteger();
The bitmask depends only on the lane count, not the lane width so when the preferred type exceeds 128 bits narrow the lanes to fill one 128-bit register instead of spilling through the stack. if (VecVT.getSizeInBits() > 128) { assert(128 % NumElts == 0 && "Expected NumElts to evenly divide 128 bits"); unsigned NarrowedEltBits = 128 / NumElts; if (NarrowedEltBits < 8) return SDValue(); VecVT = MVT::getVectorVT(MVT::getIntegerVT(NarrowedEltBits), NumElts); }
Ensure that all elements' bits are either 0s or 1s. ComparisonResult = DAG.getSExtOrTrunc(ComparisonResult, DL, VecVT);
bool IsLE = DAG.getDataLayout().isLittleEndian(); SmallVector<SDValue, 16> MaskConstants; if (DAG.getSubtarget<AArch64Subtarget>().isNeonAvailable() && VecVT == MVT::v16i8) { v16i8 is a special case: we have 16 entries but only 8 positional bits per i16 of bitmask. ANDing with a mask whose two 8-byte halves both hold the powers-of-two 1,2,...,128 leaves at most one bit set per byte lane within each 8-byte group, so pairwise addition equals OR and cannot carry. Three levels of ADDP then losslessly pack the 16 per-lane bits into the low i16 of the result vector. for (unsigned Half = 0; Half < 2; ++Half) { for (unsigned I = 0; I < 8; ++I) { On big-endian targets, the lane order in sub-byte vector elements gets reversed, so we need to flip the bit index. unsigned MaskBit = IsLE ? (1u << I) : (1u << (7 - I)); MaskConstants.push_back(DAG.getConstant(MaskBit, DL, MVT::i32)); } } SDValue Mask = DAG.getNode(ISD::BUILD_VECTOR, DL, VecVT, MaskConstants); SDValue RepresentativeBits = DAG.getNode(ISD::AND, DL, VecVT, ComparisonResult, Mask);
SDValue V = RepresentativeBits; V = DAG.getNode(AArch64ISD::ADDP, DL, VecVT, V, V); V = DAG.getNode(AArch64ISD::ADDP, DL, VecVT, V, V); V = DAG.getNode(AArch64ISD::ADDP, DL, VecVT, V, V); V = DAG.getNode(ISD::BITCAST, DL, MVT::v8i16, V); return DAG.getExtractVectorElt(DL, MVT::i32, V, 0); }
All other vector sizes. unsigned NumEl = VecVT.getVectorNumElements(); for (unsigned I = 0; I < NumEl; ++I) { unsigned MaskBit = IsLE ? (1u << I) : (1u << (NumEl - 1 - I)); MaskConstants.push_back(DAG.getConstant(MaskBit, DL, MVT::i64)); }
SDValue Mask = DAG.getNode(ISD::BUILD_VECTOR, DL, VecVT, MaskConstants); SDValue RepresentativeBits = DAG.getNode(ISD::AND, DL, VecVT, ComparisonResult, Mask); EVT ResultVT = MVT::getIntegerVT(std::max<unsigned>( NumElts, VecVT.getVectorElementType().getSizeInBits())); return DAG.getNode(ISD::VECREDUCE_ADD, DL, ResultVT, RepresentativeBits); }
When taking the trailing-zero count of a bitcast from <N x i1> to scalar iN, we can pack each lane into a small bit-slot of a 64-bit scalar with shrn (for byte lanes) or xtn (for wider lanes) then run a scalar cttz on the result. static SDValue performCTTZCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, SelectionDAG &DAG) { Run before vectorToScalarBitmask() expands it. if (!DCI.isBeforeLegalize()) return SDValue();
EVT VT = N->getValueType(0); if (!VT.isScalarInteger()) return SDValue(); SDValue BitcastNode = N->getOperand(0); if (BitcastNode.getOpcode() != ISD::BITCAST) return SDValue(); SDValue Predicate = BitcastNode.getOperand(0); EVT PredVT = Predicate.getValueType(); if (!PredVT.isFixedLengthVector() || PredVT.getVectorElementType() != MVT::i1) return SDValue();
unsigned NumLanes = PredVT.getVectorNumElements(); if (!isPowerOf2_32(NumLanes) || NumLanes < 2 || NumLanes > 32) return SDValue();
constexpr unsigned DRegBits = 64; constexpr unsigned QRegBits = 128; constexpr unsigned MinLaneBits = 8;
Pick a lane width to sext the predicate to. Prefer the original compare input width if we can find it and it fits in at most two NEON registers but otherwise pick a minimum-width type that fits. EVT MaskVT = tryGetOriginalBoolVectorType(Predicate); if (!MaskVT.isSimple() || MaskVT.changeVectorElementTypeToInteger().getSizeInBits() > 2 * QRegBits || MaskVT.changeVectorElementTypeToInteger().getSizeInBits() < DRegBits) { unsigned LaneBits = std::max(DRegBits / NumLanes, MinLaneBits); MaskVT = MVT::getVectorVT(MVT::getIntegerVT(LaneBits), NumLanes); } MaskVT = MaskVT.changeVectorElementTypeToInteger();
unsigned MaskBits = MaskVT.getSizeInBits(); if (MaskBits != DRegBits && MaskBits != QRegBits && MaskBits != 2 * QRegBits) return SDValue();
Target 64 bits when lanes fit, otherwise 128. unsigned TargetBits = DRegBits; if (MaskBits == 2 * QRegBits && NumLanes > 16) TargetBits = QRegBits; unsigned CompressedBits = std::min(MaskBits, TargetBits);
SDLoc DL(N); SDValue Mask = DAG.getSExtOrTrunc(Predicate, DL, MaskVT);
LLVMContext &Ctx = *DAG.getContext();
NEON's smallest vector lane type is i8 so we can't narrow further. Instead, bitcast the mask to i16 lanes and shrn by 4 to pack each input byte into 4 bits of the output auto ShrnPair = [&](SDValue V) -> SDValue { EVT InVT = V.getValueType(); EVT OutVT = InVT.getHalfNumVectorElementsVT(Ctx); EVT PairedVT = OutVT.widenIntegerVectorElementType(Ctx); SDValue Paired = DAG.getNode(AArch64ISD::NVCAST, DL, PairedVT, V); SDValue ShAmt = DAG.getConstant(MinLaneBits / 2, DL, PairedVT); SDValue Shifted = DAG.getNode(ISD::SRL, DL, PairedVT, Paired, ShAmt); return DAG.getNode(ISD::TRUNCATE, DL, OutVT, Shifted); };
Trailing-zero count of the compressed result as an i64. auto CompressedCttz = [&](SDValue V) -> SDValue { if (V.getValueType().getSizeInBits() == DRegBits) { SDValue Nv = DAG.getNode(AArch64ISD::NVCAST, DL, MVT::v1i64, V); SDValue Scalar = DAG.getExtractVectorElt(DL, MVT::i64, Nv, 0); return DAG.getNode(ISD::CTTZ, DL, MVT::i64, Scalar); } SDValue Compressed = DAG.getBitcast(MVT::i128, V); BITCAST puts lane 0 at the high end of the scalar on BE (unlike NVCAST above), so use clz instead of cttz to find the first match unsigned Op = DAG.getDataLayout().isLittleEndian() ? ISD::CTTZ : ISD::CTLZ; SDValue Ctz = DAG.getNode(Op, DL, MVT::i128, Compressed); return DAG.getNode(ISD::TRUNCATE, DL, MVT::i64, Ctz); };
SDValue Cur = Mask; Narrow wide lanes toward CompressedBits with xtn. while (Cur.getValueType().getScalarSizeInBits() > MinLaneBits && Cur.getValueType().getSizeInBits() > CompressedBits) { EVT CurVT = Cur.getValueType(); unsigned CurLaneBits = CurVT.getScalarSizeInBits(); EVT NarrowVT = CurVT.changeVectorElementType( Ctx, EVT::getIntegerVT(Ctx, CurLaneBits / 2)); Cur = DAG.getNode(ISD::TRUNCATE, DL, NarrowVT, Cur); }
If still too wide, pack two lanes per byte with shrn. if (Cur.getValueType().getSizeInBits() > CompressedBits) { EVT CurVT = Cur.getValueType(); if (CurVT.getSizeInBits() <= QRegBits) { Cur = ShrnPair(Cur); } else { Split into Q-reg-sized halves so each ShrnPair stays in range. EVT HalfVT = CurVT.getHalfNumVectorElementsVT(Ctx); SDValue Lo = DAG.getExtractSubvector(DL, HalfVT, Cur, 0); SDValue Hi = DAG.getNode( ISD::EXTRACT_SUBVECTOR, DL, HalfVT, Cur, DAG.getConstant(HalfVT.getVectorNumElements(), DL, MVT::i64)); Cur = DAG.getNode(ISD::CONCAT_VECTORS, DL, HalfVT, ShrnPair(Lo), ShrnPair(Hi)); } }
Cur has been narrowed to CompressedBits. assert(Cur.getValueType().getSizeInBits() == CompressedBits && "Unexpected mask width!");
SDValue Ctz = CompressedCttz(Cur); unsigned LaneIndexShift = Log2_32(CompressedBits / NumLanes); if (LaneIndexShift) Ctz = DAG.getNode(ISD::SRL, DL, MVT::i64, Ctz, DAG.getShiftAmountConstant(LaneIndexShift, MVT::i64, DL)); return DAG.getSExtOrTrunc(Ctz, DL, VT); }
static SDValue combineBoolVectorAndTruncateStore(SelectionDAG &DAG, StoreSDNode *Store) { if (!Store->isTruncatingStore()) return SDValue();
SDLoc DL(Store); SDValue VecOp = Store->getValue(); EVT VT = VecOp.getValueType(); EVT MemVT = Store->getMemoryVT();
if (!MemVT.isVector() || !VT.isVector() || MemVT.getVectorElementType() != MVT::i1) return SDValue();
If we are storing a vector that we are currently building, let scalarizeVectorStore() handle this more efficiently. if (VecOp.getOpcode() == ISD::BUILD_VECTOR) return SDValue();
VecOp = DAG.getNode(ISD::TRUNCATE, DL, MemVT, VecOp); SDValue VectorBits = vectorToScalarBitmask(VecOp, DAG); if (!VectorBits) return SDValue();
EVT StoreVT = EVT::getIntegerVT(*DAG.getContext(), MemVT.getStoreSizeInBits()); SDValue ExtendedBits = DAG.getZExtOrTrunc(VectorBits, DL, StoreVT); return DAG.getStore(Store->getChain(), DL, ExtendedBits, Store->getBasePtr(), Store->getMemOperand()); }
Combine store (fp_to_int X) to use vector semantics around the conversion when NEON is available. This allows us to store the in-vector result directly without transferring the result into a GPR in the process. static SDValue combineStoreValueFPToInt(StoreSDNode *ST, TargetLowering::DAGCombinerInfo &DCI, SelectionDAG &DAG, const AArch64Subtarget *Subtarget) { if (!Subtarget->isNeonAvailable()) return SDValue(); Source operand is already a vector. SDValue Value = ST->getValue(); if (Value.getValueType().isVector()) return SDValue();
Look through potential assertions. while (Value->isAssert()) Value = Value.getOperand(0);
if (Value.getOpcode() != ISD::FP_TO_SINT && Value.getOpcode() != ISD::FP_TO_UINT) return SDValue(); if (!Value->hasOneUse()) return SDValue();
SDValue FPSrc = Value.getOperand(0); EVT SrcVT = FPSrc.getValueType(); if (SrcVT != MVT::f32 && SrcVT != MVT::f64) return SDValue();
No support for assignments such as i64 = fp_to_sint i32 EVT VT = Value.getSimpleValueType(); if (VT != SrcVT.changeTypeToInteger()) return SDValue();
Create a 128-bit element vector to avoid widening. The floating point conversion is transformed into a single element conversion via a pattern. unsigned NumElements = 128 / SrcVT.getFixedSizeInBits(); EVT VecSrcVT = EVT::getVectorVT(*DAG.getContext(), SrcVT, NumElements); EVT VecDstVT = VecSrcVT.changeTypeToInteger(); SDLoc DL(ST); SDValue VecFP = DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, VecSrcVT, FPSrc); SDValue VecConv = DAG.getNode(Value.getOpcode(), DL, VecDstVT, VecFP); SDValue Extracted = DAG.getExtractVectorElt(DL, VT, VecConv, 0);
DCI.CombineTo(ST->getValue().getNode(), Extracted); return SDValue(ST, 0); }
static bool isHalvingTruncateOfLegalScalableType(EVT SrcVT, EVT DstVT) { return (SrcVT == MVT::nxv8i16 && DstVT == MVT::nxv8i8) || (SrcVT == MVT::nxv4i32 && DstVT == MVT::nxv4i16) || (SrcVT == MVT::nxv2i64 && DstVT == MVT::nxv2i32); }
Combine store (trunc X to <3 x i8>) to sequence of ST1.b. static SDValue combineI8TruncStore(StoreSDNode *ST, SelectionDAG &DAG, const AArch64Subtarget *Subtarget) { SDValue Value = ST->getValue(); EVT ValueVT = Value.getValueType();
if (ST->isVolatile() || !Subtarget->isLittleEndian() || Value.getOpcode() != ISD::TRUNCATE || ValueVT != EVT::getVectorVT(*DAG.getContext(), MVT::i8, 3)) return SDValue();
assert(ST->getOffset().isUndef() && "undef offset expected"); SDLoc DL(ST); EVT WideVT = EVT::getVectorVT( DAG.getContext(), Value->getOperand(0).getValueType().getVectorElementType(), 4); if (!DAG.getTargetLoweringInfo().isTypeLegal(WideVT)) return SDValue(); SDValue PoisonVector = DAG.getPOISON(WideVT); SDValue WideTrunc = DAG.getNode( ISD::INSERT_SUBVECTOR, DL, WideVT, {PoisonVector, Value->getOperand(0), DAG.getVectorIdxConstant(0, DL)}); SDValue Cast = DAG.getNode( ISD::BITCAST, DL, WideVT.getSizeInBits() == 64 ? MVT::v8i8 : MVT::v16i8, WideTrunc);
MachineFunction &MF = DAG.getMachineFunction(); SDValue Chain = ST->getChain(); MachineMemOperand *MMO = ST->getMemOperand(); unsigned IdxScale = WideVT.getScalarSizeInBits() / 8; SDValue E2 = DAG.getExtractVectorElt(DL, MVT::i8, Cast, IdxScale * 2); TypeSize Offset2 = TypeSize::getFixed(2); SDValue Ptr2 = DAG.getMemBasePlusOffset(ST->getBasePtr(), Offset2, DL); Chain = DAG.getStore(Chain, DL, E2, Ptr2, MF.getMachineMemOperand(MMO, 2, 1));
SDValue E1 = DAG.getExtractVectorElt(DL, MVT::i8, Cast, IdxScale); TypeSize Offset1 = TypeSize::getFixed(1); SDValue Ptr1 = DAG.getMemBasePlusOffset(ST->getBasePtr(), Offset1, DL); Chain = DAG.getStore(Chain, DL, E1, Ptr1, MF.getMachineMemOperand(MMO, 1, 1));
SDValue E0 = DAG.getExtractVectorElt(DL, MVT::i8, Cast, 0); Chain = DAG.getStore(Chain, DL, E0, ST->getBasePtr(), MF.getMachineMemOperand(MMO, 0, 1)); return Chain; }
static unsigned getFPSubregForVT(EVT VT) { assert(VT.isSimple() && "Expected simple VT"); switch (VT.getSimpleVT().SimpleTy) { case MVT::aarch64mfp8: return AArch64::bsub; case MVT::f16: return AArch64::hsub; case MVT::f32: return AArch64::ssub; case MVT::f64: return AArch64::dsub; default: llvm_unreachable("Unexpected VT!"); } }
static SDValue performInterleavedStoreCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, SelectionDAG &DAG);
static SDValue performLegalizedInterleavedStoreCombine( StoreSDNode *ST, TargetLowering::DAGCombinerInfo &DCI, SelectionDAG &DAG);
static SDValue performSTORECombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, SelectionDAG &DAG, const AArch64Subtarget *Subtarget) { using namespace llvm::SDPatternMatch; StoreSDNode *ST = cast<StoreSDNode>(N); SDValue Chain = ST->getChain(); SDValue Value = ST->getValue(); SDValue Ptr = ST->getBasePtr(); EVT ValueVT = Value.getValueType(); EVT MemVT = ST->getMemoryVT(); const TargetLowering &TLI = DAG.getTargetLoweringInfo(); SDLoc DL(ST);
Fixed length svbool_t store operations use an i8 vector as the underlying memory type which is cast from a scalable boolean vector. This combine replaces such sequences with a direct store of an SVE predicate. SDValue SVBool; if (DCI.isBeforeLegalize() && Subtarget->isSVEorStreamingSVEAvailable() && ISD::isNormalStore(N) && ValueVT.isFixedLengthVectorOf(MVT::i8) && ValueVT.getSizeInBits() == Subtarget->getSVEPredicateSizeInBits() && sd_match(Value, m_ExtractSubvector(m_BitCast(m_SpecificVT( MVT::nxv16i1, m_Value(SVBool))), m_SpecificInt(0)))) return DAG.getStore(Chain, DL, SVBool, Ptr, ST->getPointerInfo(), ST->getBaseAlign(), ST->getMemOperand()->getFlags(), ST->getAAInfo());
if (SDValue Res = combineStoreValueFPToInt(ST, DCI, DAG, Subtarget)) return Res;
if (SDValue Res = performInterleavedStoreCombine(N, DCI, DAG)) return Res;
if (SDValue Res = performLegalizedInterleavedStoreCombine(ST, DCI, DAG)) return Res;
Cast ptr32 and ptr64 pointers to the default address space before a store. unsigned AddrSpace = ST->getAddressSpace(); if (AddrSpace == ARM64AS::PTR64 || AddrSpace == ARM64AS::PTR32_SPTR || AddrSpace == ARM64AS::PTR32_UPTR) { MVT PtrVT = TLI.getPointerTy(DAG.getDataLayout()); if (PtrVT != Ptr.getSimpleValueType()) { SDValue Cast = DAG.getAddrSpaceCast(DL, PtrVT, Ptr, AddrSpace, 0); return DAG.getStore(Chain, DL, Value, Cast, ST->getPointerInfo(), ST->getBaseAlign(), ST->getMemOperand()->getFlags(), ST->getAAInfo()); } }
if (SDValue Res = combineI8TruncStore(ST, DAG, Subtarget)) return Res;
auto hasValidElementTypeForFPTruncStore = [](EVT DstVT, EVT SrcVT) { if (DstVT.getScalarType() == MVT::bf16) return false; EVT EltVT = SrcVT.getVectorElementType(); return EltVT == MVT::f32 || EltVT == MVT::f64; };
If this is an FP_ROUND followed by a store, fold this into a truncating store. We can do this even if this is already a truncstore. We purposefully don't care about legality of the nodes here as we know they can be split down into something legal. if (DCI.isBeforeLegalizeOps() && Value.getOpcode() == ISD::FP_ROUND && Value->hasOneUse() && ST->isUnindexed() && Subtarget->useSVEForFixedLengthVectors() && ValueVT.isFixedLengthVector() && ValueVT.getFixedSizeInBits() >= Subtarget->getMinSVEVectorSizeInBits() && hasValidElementTypeForFPTruncStore(Value.getValueType(), Value.getOperand(0).getValueType())) return DAG.getTruncStore(Chain, DL, Value.getOperand(0), Ptr, MemVT, ST->getMemOperand());
if (SDValue Split = splitStores(N, DCI, DAG, Subtarget)) return Split;
if (Subtarget->supportsAddressTopByteIgnored() && performTBISimplification(N->getOperand(2), DCI, DAG)) return SDValue(N, 0);
if (SDValue Store = foldTruncStoreOfExt(DAG, N)) return Store;
if (SDValue Store = combineBoolVectorAndTruncateStore(DAG, ST)) return Store;
if (ST->isTruncatingStore() && isHalvingTruncateOfLegalScalableType(ValueVT, MemVT)) { if (SDValue Rshrnb = trySimplifySrlAddToRshrnb(ST->getOperand(1), DAG, Subtarget)) { return DAG.getTruncStore(ST->getChain(), ST, Rshrnb, ST->getBasePtr(), MemVT, ST->getMemOperand()); } }
This is an integer vector_extract_elt followed by a (possibly truncating) store. We may be able to replace this with a store of an FP subregister. if (DCI.isAfterLegalizeDAG() && ST->isUnindexed() && Value.getOpcode() == ISD::EXTRACT_VECTOR_ELT) {
SDValue Vector = Value.getOperand(0); SDValue ExtIdx = Value.getOperand(1); EVT VectorVT = Vector.getValueType(); EVT ElemVT = VectorVT.getVectorElementType();
if (!ValueVT.isInteger()) return SDValue();
Propagate zero constants (applying this fold may miss optimizations). if (ISD::isConstantSplatVectorAllZeros(Vector.getNode())) { SDValue ZeroElt = DAG.getConstant(0, DL, ValueVT); DAG.ReplaceAllUsesWith(Value, ZeroElt); return SDValue(); }
if (ValueVT != MemVT && !ST->isTruncatingStore()) return SDValue();
This could generate an additional extract if the index is non-zero and the extracted value has multiple uses. auto *ExtCst = dyn_cast<ConstantSDNode>(ExtIdx); if ((!ExtCst || !ExtCst->isZero()) && !Value.hasOneUse()) return SDValue();
These can lower to st1, which is preferable if we're unlikely to fold the addressing into the store. if (Subtarget->isNeonAvailable() && ElemVT == MemVT && (VectorVT.is64BitVector() || VectorVT.is128BitVector()) && ExtCst && !ExtCst->isZero() && ST->getBasePtr().getOpcode() != ISD::ADD) return SDValue();
if (MemVT == MVT::i64 || MemVT == MVT::i32) { Heuristic: If there are other users of w/x integer scalars extracted from this vector that won't fold into the store – abandon folding. Applying this fold may disrupt paired stores. for (const auto &Use : Vector->uses()) { if (Use.getResNo() != Vector.getResNo()) continue; const SDNode *User = Use.getUser(); if (User->getOpcode() == ISD::EXTRACT_VECTOR_ELT && (!User->hasOneUse() || (*User->user_begin())->getOpcode() != ISD::STORE)) return SDValue(); } }
SDValue ExtVector = Vector; if (!ExtCst || !ExtCst->isZero()) { Handle extracting from lanes != 0. SDValue Ext = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, Value.getValueType(), Vector, ExtIdx); ExtVector = DAG.getInsertVectorElt(DL, DAG.getPOISON(VectorVT), Ext, 0); }
EVT FPMemVT = MemVT == MVT::i8 ? MVT::aarch64mfp8 : EVT::getFloatingPointVT(MemVT.getSizeInBits()); SDValue FPSubreg = DAG.getTargetExtractSubreg(getFPSubregForVT(FPMemVT), DL, FPMemVT, ExtVector);
return DAG.getStore(ST->getChain(), DL, FPSubreg, ST->getBasePtr(), ST->getMemOperand()); }
return SDValue(); }
static bool isSequentialConcatOfVectorInterleave(SDNode *N, SmallVectorImpl<SDValue> &Ops) { if (N->getOpcode() != ISD::CONCAT_VECTORS) return false;
unsigned NumParts = N->getNumOperands();
We should be concatenating each sequential result from a VECTOR_INTERLEAVE. SDNode *InterleaveOp = N->getOperand(0).getNode(); if (InterleaveOp->getOpcode() != ISD::VECTOR_INTERLEAVE || InterleaveOp->getNumOperands() != NumParts) return false;
for (unsigned I = 0; I < NumParts; I++) if (N->getOperand(I) != SDValue(InterleaveOp, I)) return false;
Ops.append(InterleaveOp->op_begin(), InterleaveOp->op_end()); return true; }
static bool isSplatVectorInterleaveOps(SelectionDAG &DAG, SDLoc DL, SDValue WideValue, SmallVectorImpl<SDValue> &Ops) { EVT WideVT = WideValue.getValueType(); if (!WideVT.isScalableVector()) return false;
SDValue SplatValue = DAG.getSplatValue(WideValue); if (!SplatValue) return false;
TypeSize WideSize = WideVT.getSizeInBits(); if (!WideSize.isKnownMultipleOf(128)) return false;
unsigned NumParts = WideSize.getKnownMinValue() / 128; if (NumParts != 2 && NumParts != 3 && NumParts != 4) return false;
ElementCount WideEC = WideVT.getVectorElementCount(); if (!WideEC.isKnownMultipleOf(NumParts)) return false;
EVT SubVecTy = EVT::getVectorVT(*DAG.getContext(), WideVT.getVectorElementType(), WideEC.divideCoefficientBy(NumParts));
if (SubVecTy.getSizeInBits().getKnownMinValue() != 128 || !DAG.getTargetLoweringInfo().isTypeLegal(SubVecTy)) return false;
SDValue NarrowSplat = DAG.getNode(ISD::SPLAT_VECTOR, DL, SubVecTy, SplatValue); Ops.append(NumParts, NarrowSplat); return true; }
static SDValue getNarrowMaskForInterleavedOps(SelectionDAG &DAG, SDLoc &DL, SDValue WideMask, unsigned RequiredNumParts) { if (WideMask->getOpcode() == ISD::CONCAT_VECTORS) { SmallVector<SDValue, 4> MaskInterleaveOps; if (!isSequentialConcatOfVectorInterleave(WideMask.getNode(), MaskInterleaveOps)) return SDValue();
if (MaskInterleaveOps.size() != RequiredNumParts) return SDValue();
Make sure the inputs to the vector interleave are identical. if (!llvm::all_equal(MaskInterleaveOps)) return SDValue();
return MaskInterleaveOps[0]; }
if (WideMask->getOpcode() != ISD::SPLAT_VECTOR) return SDValue();
ElementCount EC = WideMask.getValueType().getVectorElementCount(); assert(EC.isKnownMultipleOf(RequiredNumParts) && "Expected element count divisible by number of parts"); EC = EC.divideCoefficientBy(RequiredNumParts); return DAG.getNode(ISD::SPLAT_VECTOR, DL, MVT::getVectorVT(MVT::i1, EC), WideMask->getOperand(0)); }
static SDValue performInterleavedStoreCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, SelectionDAG &DAG) { if (!DCI.isBeforeLegalize()) return SDValue();
SDValue WideValue, Chain, BasePtr, Mask; bool IsMasked = false; if (auto *MST = dyn_cast<MaskedStoreSDNode>(N)) { if (!ISD::isNormalMaskedStore(MST) || !MST->isSimple() || !MST->getOffset().isUndef()) return SDValue(); WideValue = MST->getValue(); Chain = MST->getChain(); BasePtr = MST->getBasePtr(); Mask = MST->getMask(); IsMasked = true; } else { auto *ST = cast<StoreSDNode>(N); if (!ISD::isNormalStore(ST) || !ST->isSimple() || !ST->getOffset().isUndef()) return SDValue(); WideValue = ST->getValue(); Chain = ST->getChain(); BasePtr = ST->getBasePtr(); }
SDLoc DL(N); SmallVector<SDValue, 4> ValueInterleaveOps; if (isSequentialConcatOfVectorInterleave(WideValue.getNode(), ValueInterleaveOps)) { Bail out if the stored value has an unexpected number of uses, since we'll have to perform manual interleaving and may as well just use normal masked stores. if (!WideValue.hasOneUse()) return SDValue(); } else if (!isSplatVectorInterleaveOps(DAG, DL, WideValue, ValueInterleaveOps)) { return SDValue(); }
unsigned NumParts = ValueInterleaveOps.size(); if (NumParts != 2 && NumParts != 3 && NumParts != 4) return SDValue();
EVT SubVecTy = ValueInterleaveOps[0].getValueType(); const TargetLowering &TLI = DAG.getTargetLoweringInfo(); if (!TLI.isTypeLegal(SubVecTy)) return SDValue(); bool IsScalable = SubVecTy.isScalableVector(); unsigned SubBits = SubVecTy.getSizeInBits().getKnownMinValue(); if (IsScalable && SubBits != 128) return SDValue();
auto *MemN = cast<MemSDNode>(N); if (IsScalable) { SDValue Pred; if (IsMasked) { Pred = getNarrowMaskForInterleavedOps(DAG, DL, Mask, NumParts); if (!Pred) return SDValue(); } else { EVT PredVT = SubVecTy.changeVectorElementType(*DAG.getContext(), MVT::i1); Pred = DAG.getConstant(1, DL, PredVT); }
Intrinsic::ID IID; switch (NumParts) { case 2: IID = Intrinsic::aarch64_sve_st2; break; case 3: IID = Intrinsic::aarch64_sve_st3; break; case 4: IID = Intrinsic::aarch64_sve_st4; break; } SmallVector<SDValue, 8> Ops; Ops.append({Chain, DAG.getConstant(IID, DL, MVT::i32)}); Ops.append(ValueInterleaveOps); Ops.append({Pred, BasePtr}); return DAG.getMemIntrinsicNode(ISD::INTRINSIC_VOID, DL, DAG.getVTList(MVT::Other), Ops, MemN->getMemoryVT(), MemN->getMemOperand()); }
const AArch64Subtarget &Subtarget = DAG.getSubtarget<AArch64Subtarget>(); Fixed length vector using NEON if (!IsMasked && (SubBits == 64 || SubBits == 128) && Subtarget.isNeonAvailable()) { static constexpr Intrinsic::ID NEONStores[] = {Intrinsic::aarch64_neon_st2, Intrinsic::aarch64_neon_st3, Intrinsic::aarch64_neon_st4}; SmallVector<SDValue, 8> Ops; Ops.push_back(Chain); Ops.push_back( DAG.getTargetConstant(NEONStores[NumParts - 2], DL, MVT::i64)); Ops.append(ValueInterleaveOps); Ops.push_back(BasePtr); return DAG.getMemIntrinsicNode(ISD::INTRINSIC_VOID, DL, DAG.getVTList(MVT::Other), Ops, MemN->getMemoryVT(), MemN->getMemOperand()); }
Fixed-length vectors using SVE if (!Subtarget.isSVEorStreamingSVEAvailable()) return SDValue();
EVT ContainerVT = getContainerForFixedLengthVector(DAG, SubVecTy); SDValue Pred; if (IsMasked) { Pred = getNarrowMaskForInterleavedOps(DAG, DL, Mask, NumParts); if (!Pred) return SDValue(); EVT MaskVT = SubVecTy.changeTypeToInteger(); Widen the i1 mask if (Pred.getValueType() != MaskVT) Pred = DAG.getNode(ISD::SIGN_EXTEND, DL, MaskVT, Pred); Pred = convertFixedMaskToScalableVector(Pred, DAG); } else { Pred = getPredicateForFixedLengthVector(DAG, DL, SubVecTy); }
static constexpr Intrinsic::ID SVEStores[] = {Intrinsic::aarch64_sve_st2, Intrinsic::aarch64_sve_st3, Intrinsic::aarch64_sve_st4}; SmallVector<SDValue, 8> Ops; Ops.append({Chain, DAG.getConstant(SVEStores[NumParts - 2], DL, MVT::i32)}); for (SDValue V : ValueInterleaveOps) Ops.push_back(convertToScalableVector(DAG, ContainerVT, V)); Ops.append({Pred, BasePtr}); return DAG.getMemIntrinsicNode(ISD::INTRINSIC_VOID, DL, DAG.getVTList(MVT::Other), Ops, MemN->getMemoryVT(), MemN->getMemOperand()); }
static SDValue performLegalizedInterleavedStoreCombine( StoreSDNode *ST, TargetLowering::DAGCombinerInfo &DCI, SelectionDAG &DAG) { Type legalization splits a wide interleaved store into consecutive legal stores. Combine each group that directly stores all results of a legal VECTOR_INTERLEAVE into a structured store. if (DCI.getDAGCombineLevel() < AfterLegalizeTypes || !DAG.getSubtarget<AArch64Subtarget>().isNeonAvailable()) return SDValue();
SDValue StoredValue = ST->getValue(); SDNode *Interleave = StoredValue.getNode(); if (Interleave->getOpcode() != ISD::VECTOR_INTERLEAVE) return SDValue();
unsigned NumParts = Interleave->getNumOperands(); if (NumParts < 2 || NumParts > 4) return SDValue();
EVT SubVecTy = Interleave->getValueType(0); if (!SubVecTy.is64BitVector() && !SubVecTy.is128BitVector()) return SDValue();
SmallVector<StoreSDNode *, 4> Stores(NumParts, nullptr); for (SDUse &Use : Interleave->uses()) { unsigned ResNo = Use.getResNo(); auto *Store = dyn_cast<StoreSDNode>(Use.getUser()); if (Stores[ResNo] || !Store) return SDValue(); Stores[ResNo] = Store; }
if (llvm::is_contained(Stores, nullptr)) return SDValue();
StoreSDNode *BaseStore = Stores[0]; unsigned Bytes = SubVecTy.getStoreSize().getFixedValue(); for (auto [Idx, Store] : enumerate(Stores)) { if (!DAG.areNonVolatileConsecutiveStores(Store, BaseStore, Bytes, Idx)) return SDValue(); }
static constexpr Intrinsic::ID NEONStores[] = {Intrinsic::aarch64_neon_st2, Intrinsic::aarch64_neon_st3, Intrinsic::aarch64_neon_st4}; SDLoc DL(Interleave); SmallVector<SDValue, 8> Ops = { BaseStore->getChain(), DAG.getTargetConstant(NEONStores[NumParts - 2], DL, MVT::i64)}; Ops.append(Interleave->op_begin(), Interleave->op_end()); Ops.push_back(BaseStore->getBasePtr());
EVT MemVT = EVT::getVectorVT(*DAG.getContext(), SubVecTy.getVectorElementType(), SubVecTy.getVectorElementCount() * NumParts); MachineFunction &MF = DAG.getMachineFunction(); MachineMemOperand *MMO = MF.getMachineMemOperand(BaseStore->getMemOperand(), 0, NumParts * Bytes); SDValue NewStore = DAG.getMemIntrinsicNode( ISD::INTRINSIC_VOID, DL, DAG.getVTList(MVT::Other), Ops, MemVT, MMO);
for (StoreSDNode *Store : Stores) if (Store != ST) DCI.CombineTo(Store, NewStore); return NewStore; }
/ Expand a scalable compressing store to a VECTOR_COMPRESS + a masked store. static SDValue expandScalableCompressingStore(MaskedStoreSDNode *Store, SelectionDAG &DAG) { SDLoc DL(Store); EVT VT = Store->getValue().getValueType(); assert(VT.isScalableVector() && Store->isCompressingStore() && "Expected a scalable compressing store");
EVT MaskVT = Store->getMask().getValueType(); SDValue Zero = DAG.getConstant(0, DL, MVT::i64); SDValue CntActive = DAG.getNode( ISD::INTRINSIC_WO_CHAIN, DL, MVT::i64, DAG.getTargetConstant(Intrinsic::aarch64_sve_cntp, DL, MVT::i64), Store->getMask(), Store->getMask());
SDValue CompressedValue = DAG.getNode(ISD::VECTOR_COMPRESS, DL, VT, Store->getValue(), Store->getMask(), DAG.getPOISON(VT)); SDValue CompressedMask = DAG.getNode(ISD::GET_ACTIVE_LANE_MASK, DL, MaskVT, Zero, CntActive);
return DAG.getMaskedStore(Store->getChain(), DL, CompressedValue, Store->getBasePtr(), Store->getOffset(), CompressedMask, Store->getMemoryVT(), Store->getMemOperand(), Store->getAddressingMode(), Store->isTruncatingStore(), /*isCompressing=
Definition at line 28362 of file AArch64ISelLowering.cpp.
◆ FlagsVT
|
constexpr |
Value type used for NZCV flags.
Definition at line 225 of file AArch64ISelLowering.cpp.
Referenced by emitComparison(), emitConditionalComparison(), emitStrictFPComparison(), Expand64BitShift(), getAArch64XALUOOp(), lowerADDSUBO_CARRY(), lowerPredCounterWhile(), performANDORCSELCombine(), performRNDRCombine(), reassociateCSELOperandsForCSE(), and valueToCarryFlag().
◆ FPRArgRegs
Definition at line 230 of file AArch64ISelLowering.cpp.
Referenced by llvm::AArch64::getFPRArgRegs().
◆ GPRArgRegs
Definition at line 227 of file AArch64ISelLowering.cpp.
Referenced by f64AssignAAPCS(), and llvm::AArch64::getGPRArgRegs().
◆ MaxXors
|
static |
Referenced by isOrXorChain(), and performOrXorChainCombine().
◆ UseConditionalFPMRWrite
|
static |
Referenced by llvm::AArch64TargetLowering::EmitLoweredSetFpmr().
◆ UseFEATCPACodegen
|
static |
Referenced by llvm::AArch64TargetLowering::shouldPreservePtrArith().
Generated on for LLVM by
1.14.0