|
LLVM 24.0.0git
|
#include "llvm/CodeGen/GlobalISel/LegalizerHelper.h"#include "llvm/CodeGen/GlobalISel/CallLowering.h"#include "llvm/CodeGen/GlobalISel/GISelChangeObserver.h"#include "llvm/CodeGen/GlobalISel/GISelValueTracking.h"#include "llvm/CodeGen/GlobalISel/GenericMachineInstrs.h"#include "llvm/CodeGen/GlobalISel/LegalizerInfo.h"#include "llvm/CodeGen/GlobalISel/LostDebugLocObserver.h"#include "llvm/CodeGen/GlobalISel/MIPatternMatch.h"#include "llvm/CodeGen/GlobalISel/MachineIRBuilder.h"#include "llvm/CodeGen/GlobalISel/Utils.h"#include "llvm/CodeGen/LowLevelTypeUtils.h"#include "llvm/CodeGen/MachineConstantPool.h"#include "llvm/CodeGen/MachineFrameInfo.h"#include "llvm/CodeGen/MachineRegisterInfo.h"#include "llvm/CodeGen/RuntimeLibcallUtil.h"#include "llvm/CodeGen/TargetFrameLowering.h"#include "llvm/CodeGen/TargetInstrInfo.h"#include "llvm/CodeGen/TargetLowering.h"#include "llvm/CodeGen/TargetOpcodes.h"#include "llvm/CodeGen/TargetSubtargetInfo.h"#include "llvm/IR/Instructions.h"#include "llvm/Support/Debug.h"#include "llvm/Support/MathExtras.h"#include "llvm/Support/raw_ostream.h"#include "llvm/Target/TargetMachine.h"#include <cassert>#include <numeric>#include <optional>Go to the source code of this file.
Macros | |
| #define | DEBUG_TYPE "legalizer" |
| #define | RTLIBCASE_INT(LibcallPrefix) |
| #define | RTLIBCASE(LibcallPrefix) |
| #define | RTLIBCASE_CMP(LibcallPrefix, ICmpPred) |
Functions | |
| static MachineBasicBlock::iterator | getPHIIncomingInsertPoint (MachineBasicBlock &MBB, Register Reg, const MachineRegisterInfo &MRI) |
| Choose an insertion point for a conversion of an incoming PHI value. | |
| static std::pair< int, int > | getNarrowTypeBreakDown (LLT OrigTy, LLT NarrowTy, LLT &LeftoverTy) |
Try to break down OrigTy into NarrowTy sized pieces. | |
| static Type * | getFloatTypeForLLT (LLVMContext &Ctx, LLT Ty) |
| static void | getUnmergeResults (SmallVectorImpl< Register > &Regs, const MachineInstr &MI) |
Append the result registers of G_UNMERGE_VALUES MI to Regs. | |
| static RTLIB::Libcall | getRTLibDesc (unsigned Opcode, unsigned Size) |
| static bool | hasSwiftErrorArg (MachineFunction &MF) |
| static bool | isLibCallInTailPosition (const CallLowering::ArgInfo &Result, MachineInstr &MI, const TargetInstrInfo &TII, MachineRegisterInfo &MRI) |
| True if an instruction is in tail position in its caller. | |
| static RTLIB::Libcall | getConvRTLibDesc (unsigned Opcode, Type *ToType, Type *FromType) |
| static unsigned | getAtomicLibcallISDOpcode (unsigned Opc) |
| static RTLIB::Libcall | getStateLibraryFunctionFor (MachineInstr &MI, const TargetLowering &TLI) |
| static std::pair< RTLIB::Libcall, CmpInst::Predicate > | getFCMPLibcallDesc (const CmpInst::Predicate Pred, unsigned Size) |
| Returns the corresponding libcall for the given Pred and the ICMP predicate that should be generated to compare with #0 after the libcall. | |
| static void | getUnmergePieces (SmallVectorImpl< Register > &Pieces, MachineIRBuilder &B, Register Src, LLT Ty) |
| static void | emitLoadFromConstantPool (Register DstReg, const Constant *ConstVal, MachineIRBuilder &MIRBuilder) |
| static Register | getBitcastWiderVectorElementOffset (MachineIRBuilder &B, Register Idx, unsigned NewEltSize, unsigned OldEltSize) |
| Figure out the bit offset into a register when coercing a vector index for the wide element type. | |
| static Register | buildBitFieldInsert (MachineIRBuilder &B, Register TargetReg, Register InsertReg, Register OffsetBits) |
Emit code to insert InsertReg into TargetRet at OffsetBits in TargetReg, while preserving other bits in TargetReg. | |
| static Register | clampVectorIndex (MachineIRBuilder &B, Register IdxReg, LLT VecTy) |
| static bool | hasSameNumEltsOnAllVectorOperands (GenericMachineInstr &MI, MachineRegisterInfo &MRI, std::initializer_list< unsigned > NonVecOpIndices) |
| Check that all vector operands have same number of elements. | |
| static void | makeDstOps (SmallVectorImpl< DstOp > &DstOps, LLT Ty, unsigned NumElts) |
Fill DstOps with DstOps that have same number of elements combined as the Ty. | |
| static void | broadcastSrcOp (SmallVectorImpl< SrcOp > &Ops, unsigned N, MachineOperand &Op) |
Operand Op is used on N sub-instructions. | |
| static bool | isNonZeroModBitWidthOrUndef (const MachineRegisterInfo &MRI, Register Reg, unsigned BW) |
| static bool | isNonZeroModBitWidth (const MachineRegisterInfo &MRI, Register Reg, unsigned BW) |
| static LegalizerHelper::LegalizeResult | loweri64tof16ITOFP (MachineInstr &MI, Register Dst, LLT DstTy, Register Src, LLT SrcTy, MachineIRBuilder &MIRBuilder) |
| i64->fp16 itofp can be lowered to i64->f64,f64->f32,f32->f16. | |
| static CmpInst::Predicate | minMaxToCompare (unsigned Opc) |
| static MachineInstrBuilder | SwapN (unsigned N, DstOp Dst, MachineIRBuilder &B, MachineInstrBuilder Src, const APInt &Mask) |
| static Register | getMemsetValue (Register Val, LLT Ty, MachineIRBuilder &MIB) |
| #define DEBUG_TYPE "legalizer" |
Definition at line 44 of file LegalizerHelper.cpp.
| #define RTLIBCASE | ( | LibcallPrefix | ) |
Referenced by getRTLibDesc().
| #define RTLIBCASE_CMP | ( | LibcallPrefix, | |
| ICmpPred ) |
Referenced by getFCMPLibcallDesc().
| #define RTLIBCASE_INT | ( | LibcallPrefix | ) |
Referenced by getRTLibDesc().
|
static |
Operand Op is used on N sub-instructions.
Fill Ops with N SrcOps made from Op depending on operand type.
Definition at line 5358 of file LegalizerHelper.cpp.
References AbstractManglingParser< Derived, Alloc >::Ops, llvm_unreachable, and N.
Referenced by llvm::LegalizerHelper::fewerElementsVectorMultiEltType().
|
static |
Emit code to insert InsertReg into TargetRet at OffsetBits in TargetReg, while preserving other bits in TargetReg.
(InsertReg << Offset) | (TargetReg & ~(-1 >> InsertReg.size()) << Offset)
Definition at line 4027 of file LegalizerHelper.cpp.
References B(), llvm::APInt::getLowBitsSet(), and llvm::LLT::getSizeInBits().
Referenced by llvm::LegalizerHelper::bitcastInsertVectorElt().
|
static |
Definition at line 5248 of file LegalizerHelper.cpp.
References B(), llvm::APInt::getLowBitsSet(), llvm::LLT::getNumElements(), llvm::LLT::getSizeInBits(), Imm, llvm::isPowerOf2_32(), llvm::Log2_32(), llvm::MIPatternMatch::m_ICst(), and llvm::MIPatternMatch::mi_match().
Referenced by llvm::LegalizerHelper::getVectorElementPointer().
|
static |
Definition at line 3734 of file LegalizerHelper.cpp.
References llvm::MachineIRBuilder::buildConstantPool(), llvm::MachineIRBuilder::buildLoadInstr(), DL, llvm::MachineFunction::getConstantPool(), llvm::MachinePointerInfo::getConstantPool(), llvm::MachineConstantPool::getConstantPoolIndex(), llvm::MachineIRBuilder::getDataLayout(), llvm::MachineFunction::getMachineMemOperand(), llvm::MachineIRBuilder::getMF(), llvm::MachineIRBuilder::getMRI(), llvm::Value::getType(), llvm::MachineMemOperand::MOLoad, MRI, and llvm::LLT::pointer().
Referenced by llvm::LegalizerHelper::lowerConstant(), and llvm::LegalizerHelper::lowerFConstant().
Definition at line 921 of file LegalizerHelper.cpp.
References llvm::ISD::ATOMIC_CMP_SWAP, llvm::ISD::ATOMIC_LOAD_ADD, llvm::ISD::ATOMIC_LOAD_CLR, llvm::ISD::ATOMIC_LOAD_OR, llvm::ISD::ATOMIC_LOAD_XOR, llvm::ISD::ATOMIC_SWAP, llvm_unreachable, and Opc.
Referenced by llvm::LegalizerHelper::createAtomicLibcall().
|
static |
Figure out the bit offset into a register when coercing a vector index for the wide element type.
This is only for the case when promoting vector to one with larger elements. offset_idx = G_AND idx, ~(-1 << Log2(DstEltSize / SrcEltSize)) offset_bits = G_SHL offset_idx, Log2(SrcEltSize)
Definition at line 3899 of file LegalizerHelper.cpp.
References B(), llvm::APInt::getAllOnes(), llvm::LLT::getSizeInBits(), and llvm::Log2_32().
Referenced by llvm::LegalizerHelper::bitcastExtractVectorElt(), and llvm::LegalizerHelper::bitcastInsertVectorElt().
Definition at line 785 of file LegalizerHelper.cpp.
References llvm::RTLIB::getFPEXT(), llvm::RTLIB::getFPROUND(), llvm::RTLIB::getFPTOSINT(), llvm::RTLIB::getFPTOUINT(), llvm::RTLIB::getSINTTOFP(), llvm::RTLIB::getUINTTOFP(), llvm::MVT::getVT(), and llvm_unreachable.
Referenced by llvm::LegalizerHelper::conversionLibcall().
|
static |
Returns the corresponding libcall for the given Pred and the ICMP predicate that should be generated to compare with #0 after the libcall.
Definition at line 1144 of file LegalizerHelper.cpp.
References llvm::CmpInst::BAD_ICMP_PREDICATE, llvm::CmpInst::FCMP_OEQ, llvm::CmpInst::FCMP_OGE, llvm::CmpInst::FCMP_OGT, llvm::CmpInst::FCMP_OLE, llvm::CmpInst::FCMP_OLT, llvm::CmpInst::FCMP_UNE, llvm::CmpInst::FCMP_UNO, llvm::CmpInst::ICMP_EQ, llvm::CmpInst::ICMP_NE, llvm::CmpInst::ICMP_SGE, llvm::CmpInst::ICMP_SGT, llvm::CmpInst::ICMP_SLE, llvm::CmpInst::ICMP_SLT, Pred, RTLIBCASE_CMP, and llvm::Size.
|
static |
Definition at line 105 of file LegalizerHelper.cpp.
References llvm::Type::getDoubleTy(), llvm::Type::getFloatTy(), llvm::Type::getFP128Ty(), llvm::Type::getHalfTy(), and llvm::Type::getX86_FP80Ty().
Referenced by llvm::LegalizerHelper::libcall().
|
static |
Definition at line 11122 of file LegalizerHelper.cpp.
References llvm::MachineIRBuilder::buildConstant(), llvm::MachineIRBuilder::buildMul(), llvm::MachineIRBuilder::buildSplatBuildVector(), llvm::MachineIRBuilder::buildZExtOrTrunc(), llvm::getIConstantVRegValWithLookThrough(), llvm::MachineIRBuilder::getMRI(), llvm::MachineInstrBuilder::getReg(), llvm::LLT::getScalarType(), llvm::APInt::getSplat(), MRI, and Scalar.
Referenced by getMemsetStores().
Try to break down OrigTy into NarrowTy sized pieces.
Returns the number of NarrowTy elements needed to reconstruct OrigTy, with any leftover piece as type LeftoverTy
Returns -1 in the first element of the pair if the breakdown is not satisfiable.
Definition at line 79 of file LegalizerHelper.cpp.
References assert(), llvm::LLT::changeElementCount(), llvm::ElementCount::getFixed(), llvm::LLT::getScalarSizeInBits(), llvm::LLT::getSizeInBits(), llvm::LLT::integer(), llvm::LLT::isValid(), llvm::LLT::isVector(), NumParts, and llvm::Size.
Referenced by makeDstOps(), and llvm::LegalizerHelper::reduceLoadStoreWidth().
|
static |
Choose an insertion point for a conversion of an incoming PHI value.
Values available before an invoke must be converted before it, so the conversion also executes on the unwind edge. An invoke result is defined after G_INVOKE_REGION_START and is only available on the normal edge.
Definition at line 55 of file LegalizerHelper.cpp.
References E(), I, MBB, MRI, and Reg.
Referenced by llvm::LegalizerHelper::fewerElementsVectorPhi(), llvm::LegalizerHelper::moreElementsVectorPhi(), llvm::LegalizerHelper::narrowScalar(), and llvm::LegalizerHelper::widenScalar().
Definition at line 406 of file LegalizerHelper.cpp.
References llvm_unreachable, RTLIBCASE, RTLIBCASE_INT, and llvm::Size.
Referenced by llvm::LegalizerHelper::libcall(), and llvm::LegalizerHelper::simpleLibcall().
|
static |
Definition at line 1034 of file LegalizerHelper.cpp.
References llvm_unreachable, MI, and TLI.
|
static |
Definition at line 3727 of file LegalizerHelper.cpp.
References B(), E(), I, and llvm::SmallVectorTemplateBase< T, bool >::push_back().
Referenced by llvm::LegalizerHelper::lowerBitcast().
|
static |
Append the result registers of G_UNMERGE_VALUES MI to Regs.
Definition at line 249 of file LegalizerHelper.cpp.
References assert(), I, MI, llvm::SmallVectorImpl< T >::resize(), and llvm::SmallVectorTemplateCommon< T, typename >::size().
Referenced by llvm::LegalizerHelper::fewerElementsBitcast().
|
static |
Check that all vector operands have same number of elements.
Other operands should be listed in NonVecOp.
Definition at line 5298 of file LegalizerHelper.cpp.
References llvm::LLT::getNumElements(), llvm::is_contained(), llvm::LLT::isVector(), MI, and MRI.
Referenced by llvm::LegalizerHelper::fewerElementsVectorMultiEltType().
|
static |
Definition at line 542 of file LegalizerHelper.cpp.
References llvm::Function::getAttributes(), llvm::MachineFunction::getFunction(), llvm::MachineFunction::getSubtarget(), llvm::TargetSubtargetInfo::getTargetLowering(), and TLI.
Referenced by llvm::LegalizerHelper::createLibcall(), and llvm::LegalizerHelper::createMemLibcall().
|
static |
True if an instruction is in tail position in its caller.
Intended for legalizing libcalls as tail calls when possible.
Definition at line 551 of file LegalizerHelper.cpp.
References F, llvm::Register::isPhysical(), llvm::Register::isVirtual(), MBB, MI, MRI, llvm::Next, llvm::next_nodbg(), and TII.
Referenced by llvm::LegalizerHelper::createLibcall(), and llvm::LegalizerHelper::createMemLibcall().
|
static |
Definition at line 8260 of file LegalizerHelper.cpp.
References C(), llvm::cast(), llvm::matchUnaryPredicate(), MRI, and Reg.
Referenced by llvm::LegalizerHelper::lowerFunnelShiftAsShifts().
|
static |
Definition at line 8246 of file LegalizerHelper.cpp.
References C(), llvm::dyn_cast_or_null(), llvm::ConstantInt::getValue(), llvm::matchUnaryPredicate(), MRI, Reg, and llvm::APInt::urem().
Referenced by llvm::TargetLowering::expandFunnelShift(), llvm::LegalizerHelper::lowerFunnelShiftAsShifts(), and llvm::LegalizerHelper::lowerFunnelShiftWithInverse().
|
static |
i64->fp16 itofp can be lowered to i64->f64,f64->f32,f32->f16.
We cannot convert fpround f64->f16 without double-rounding, so we manually perform the lowering here where we know it is valid.
Definition at line 8650 of file LegalizerHelper.cpp.
References llvm::MachineIRBuilder::buildFPTrunc(), llvm::MachineIRBuilder::buildSITOFP(), llvm::MachineIRBuilder::buildUITOFP(), llvm::LLT::changeElementSize(), llvm::LLT::floatIEEE(), llvm::LegalizerHelper::Legalized, llvm::M1(), and MI.
Referenced by llvm::LegalizerHelper::lowerSITOFP(), and llvm::LegalizerHelper::lowerUITOFP().
|
static |
Fill DstOps with DstOps that have same number of elements combined as the Ty.
These DstOps have either scalar type when NumElts = 1 or are vectors with NumElts elements. When Ty.getNumElements() is not multiple of NumElts last DstOp (leftover) has fewer then NumElts elements.
Definition at line 5336 of file LegalizerHelper.cpp.
References assert(), llvm::LLT::changeElementCount(), llvm::ElementCount::getFixed(), getNarrowTypeBreakDown(), llvm::LLT::isValid(), NumParts, and llvm::SmallVectorTemplateBase< T, bool >::push_back().
Referenced by llvm::LegalizerHelper::fewerElementsVectorMultiEltType(), and llvm::LegalizerHelper::fewerElementsVectorPhi().
|
static |
Definition at line 9287 of file LegalizerHelper.cpp.
References llvm::CmpInst::ICMP_SGT, llvm::CmpInst::ICMP_SLT, llvm::CmpInst::ICMP_UGT, llvm::CmpInst::ICMP_ULT, llvm_unreachable, and Opc.
Referenced by llvm::LegalizerHelper::lowerMinMax().
|
static |
Definition at line 10469 of file LegalizerHelper.cpp.
References B(), LHS, N, and RHS.
Referenced by llvm::LegalizerHelper::lowerBitreverse().