22#ifndef LLVM_CODEGEN_TARGETLOWERING_H
23#define LLVM_CODEGEN_TARGETLOWERING_H
70class FunctionLoweringInfo;
78class MachineBasicBlock;
81class MachineJumpTableInfo;
83class MachineRegisterInfo;
87class ProfileSummaryInfo;
88class TargetLibraryInfo;
90class TargetRegisterClass;
91class TargetRegisterInfo;
92class TargetTransformInfo;
115 bool DstAlignCanChange;
130 Align SrcAlign,
bool IsVolatile,
131 bool MemcpyStrSrc =
false) {
134 Op.DstAlignCanChange = DstAlignCanChange;
135 Op.DstAlign = DstAlign;
136 Op.AllowOverlap = !IsVolatile;
138 Op.ZeroMemset =
false;
139 Op.MemcpyStrSrc = MemcpyStrSrc;
140 Op.SrcAlign = SrcAlign;
145 bool IsZeroMemset,
bool IsVolatile) {
148 Op.DstAlignCanChange = DstAlignCanChange;
149 Op.DstAlign = DstAlign;
150 Op.AllowOverlap = !IsVolatile;
152 Op.ZeroMemset = IsZeroMemset;
153 Op.MemcpyStrSrc =
false;
159 assert(!DstAlignCanChange);
167 return isMemcpy() && !DstAlignCanChange;
407 bool LegalTypes =
true)
const;
463 bool IsScalable)
const {
483 return HasMultipleConditionRegisters;
519 unsigned DefinedValues)
const {
520 return DefinedValues < 3;
577 return BypassSlowDivWidths;
621 unsigned AddrSpace)
const {
770 unsigned KeptBits)
const {
784 unsigned OldShiftOpcode,
unsigned NewShiftOpcode,
792 if (OldShiftOpcode ==
ISD::SHL &&
CC->isOne())
796 if (XC && NewShiftOpcode ==
ISD::SHL && XC->isOne())
835 unsigned &
Cost)
const {
880 return BooleanVectorContents;
881 return isFloat ? BooleanFloatContents : BooleanContents;
904 return SchedPreferenceInfo;
919 assert(RC &&
"This value type is not natively supported!");
927 const Value *)
const {
946 return RepRegClassCostForVT[VT.
SimpleTy];
958 unsigned ExpansionFactor)
const {
959 if (ExpansionFactor == 1)
980 std::fill(std::begin(ValueTypeActions), std::end(ValueTypeActions),
985 return ValueTypeActions[VT.
SimpleTy];
989 ValueTypeActions[VT.
SimpleTy] = Action;
994 return ValueTypeActions;
1062 EVT &IntermediateVT,
1063 unsigned &NumIntermediates,
1064 MVT &RegisterVT)
const;
1071 unsigned &NumIntermediates,
MVT &RegisterVT)
const {
1110 bool ForCodeSize =
false)
const {
1148 if (Op >= std::size(OpActions[0]))
1157 unsigned Scale)
const {
1166 unsigned Scale)
const {
1168 if (Action !=
Legal)
1189 return Supported ? Action :
Expand;
1198#define DAG_INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC, DAGN) \
1199 case ISD::STRICT_##DAGN: EqOpc = ISD::DAGN; break;
1200#define CMP_INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC, DAGN) \
1201 case ISD::STRICT_##DAGN: EqOpc = ISD::SETCC; break;
1202#include "llvm/IR/ConstrainedOps.def"
1213 bool LegalOnly =
false)
const {
1227 bool LegalOnly =
false)
const {
1241 bool LegalOnly =
false)
const {
1283 EVT ConditionVT)
const;
1308 return (NumDests == 1 && NumCmps >= 3) || (NumDests == 2 && NumCmps >= 5) ||
1309 (NumDests == 3 && NumCmps >= 6);
1335 unsigned Shift = 4 * ExtType;
1336 return (
LegalizeAction)((LoadExtActions[ValI][MemI] >> Shift) & 0xf);
1359 "Table isn't big enough!");
1360 return TruncStoreActions[ValI][MemI];
1378 bool LegalOnly)
const {
1389 return getIndexedModeAction(IdxMode, VT, IMAB_Load);
1403 return getIndexedModeAction(IdxMode, VT, IMAB_Store);
1417 return getIndexedModeAction(IdxMode, VT, IMAB_MaskedLoad);
1431 return getIndexedModeAction(IdxMode, VT, IMAB_MaskedStore);
1458 if (Scale != ElemSize && Scale != 1)
1468 assert((
unsigned)
CC < std::size(CondCodeActions) &&
1469 ((
unsigned)VT.
SimpleTy >> 3) < std::size(CondCodeActions[0]) &&
1470 "Table isn't big enough!");
1475 assert(Action !=
Promote &&
"Can't promote condition code!");
1495 "This operation isn't promoted!");
1498 std::map<std::pair<unsigned, MVT::SimpleValueType>,
1500 PromoteToType.find(std::make_pair(Op, VT.
SimpleTy));
1501 if (PTTI != PromoteToType.end())
return PTTI->second;
1504 "Cannot autopromote this type, add it with AddPromotedToType.");
1510 "Didn't find type to promote to!");
1517 bool AllowUnknown =
false)
const {
1526 bool AllowUnknown =
false)
const {
1528 if (
auto *PTy = dyn_cast<PointerType>(Ty))
1531 if (
auto *VTy = dyn_cast<VectorType>(Ty)) {
1532 Type *EltTy = VTy->getElementType();
1534 if (
auto *PTy = dyn_cast<PointerType>(EltTy)) {
1539 VTy->getElementCount());
1546 bool AllowUnknown =
false)
const {
1548 if (
auto *PTy = dyn_cast<PointerType>(Ty))
1551 if (
auto *VTy = dyn_cast<VectorType>(Ty)) {
1552 Type *EltTy = VTy->getElementType();
1553 if (
auto *PTy = dyn_cast<PointerType>(EltTy)) {
1558 VTy->getElementCount());
1567 bool AllowUnknown =
false)
const {
1579 return RegisterTypeForVT[VT.
SimpleTy];
1589 unsigned NumIntermediates;
1591 NumIntermediates, RegisterVT);
1613 std::optional<MVT> RegisterVT = std::nullopt)
const {
1616 std::size(NumRegistersForVT));
1622 unsigned NumIntermediates;
1628 return (
BitWidth + RegWidth - 1) / RegWidth;
1654 return DL.getABITypeAlign(ArgTy);
1668 if (NewVT.
isVector() && !Load->hasOneUse())
1682 return DL.isBigEndian() || VT == MVT::ppcf128;
1688 assert(
unsigned(NT >> 3) < std::size(TargetDAGCombineArray));
1689 return TargetDAGCombineArray[NT >> 3] & (1 << (NT&7));
1763 unsigned * =
nullptr)
const {
1771 unsigned * =
nullptr)
const {
1781 unsigned AddrSpace = 0,
Align Alignment =
Align(1),
1783 unsigned *
Fast =
nullptr)
const;
1792 unsigned *
Fast =
nullptr)
const;
1800 unsigned AddrSpace = 0,
Align Alignment =
Align(1),
1802 unsigned *
Fast =
nullptr)
const;
1810 unsigned *
Fast =
nullptr)
const;
1815 unsigned *
Fast =
nullptr)
const;
1858 return StackPointerRegisterToSaveRestore;
1881 return MinStackArgumentAlignment;
1985 return MaxAtomicSizeInBitsSupported;
1991 return MaxDivRemBitWidthSupported;
1997 return MaxLargeFPConvertBitWidthSupported;
2051 llvm_unreachable(
"Masked atomicrmw expansion unimplemented on this target");
2059 "Generic atomicrmw expansion unimplemented on this target");
2067 "Bit test atomicrmw expansion unimplemented on this target");
2075 "Compare arith atomicrmw expansion unimplemented on this target");
2185 if (
SI->getValueOperand()->getType()->isFloatingPointTy())
2309 bool IsSigned)
const {
2333 BooleanContents = Ty;
2334 BooleanFloatContents = Ty;
2340 BooleanContents = IntTy;
2341 BooleanFloatContents = FloatTy;
2347 BooleanVectorContents = Ty;
2352 SchedPreferenceInfo = Pref;
2365 StackPointerRegisterToSaveRestore = R;
2374 HasMultipleConditionRegisters = hasManyRegs;
2382 HasExtractBitsInsn = hasExtractInsn;
2392 BypassSlowDivWidths[SlowBitWidth] = FastBitWidth;
2405 virtual std::pair<const TargetRegisterClass *, uint8_t>
2416 assert(Op < std::size(OpActions[0]) &&
"Table isn't big enough!");
2435 MemVT.
isValid() &&
"Table isn't big enough!");
2436 assert((
unsigned)Action < 0x10 &&
"too many bits for bitfield array");
2437 unsigned Shift = 4 * ExtType;
2443 for (
auto ExtType : ExtTypes)
2448 for (
auto MemVT : MemVTs)
2466 for (
auto IdxMode : IdxModes)
2467 setIndexedModeAction(IdxMode, VT, IMAB_Load, Action);
2483 for (
auto IdxMode : IdxModes)
2484 setIndexedModeAction(IdxMode, VT, IMAB_Store, Action);
2500 setIndexedModeAction(IdxMode, VT, IMAB_MaskedLoad, Action);
2510 setIndexedModeAction(IdxMode, VT, IMAB_MaskedStore, Action);
2517 for (
auto CC : CCs) {
2519 "Table isn't big enough!");
2520 assert((
unsigned)Action < 0x10 &&
"too many bits for bitfield array");
2554 for (
auto NT : NTs) {
2555 assert(
unsigned(NT >> 3) < std::size(TargetDAGCombineArray));
2556 TargetDAGCombineArray[NT >> 3] |= 1 << (NT & 7);
2562 MinFunctionAlignment = Alignment;
2568 PrefFunctionAlignment = Alignment;
2576 MaxBytesForAlignment = MaxBytes;
2581 MinStackArgumentAlignment = Alignment;
2589 MaxAtomicSizeInBitsSupported = SizeInBits;
2595 MaxDivRemBitWidthSupported = SizeInBits;
2601 MaxLargeFPConvertBitWidthSupported = SizeInBits;
2606 MinCmpXchgSizeInBits = SizeInBits;
2611 SupportsUnalignedAtomics = UnalignedSupported;
2656 Type *Ty,
unsigned AddrSpace,
2703 return (
From->isIntegerTy() ||
From->isFloatingPointTy()) &&
2745 default:
return false;
2813 switch (
I->getOpcode()) {
2814 case Instruction::FPExt:
2819 case Instruction::ZExt:
2820 if (
isZExtFree(
I->getOperand(0)->getType(),
I->getType()))
2823 case Instruction::SExt:
2850 if (isa<ZExtInst>(Ext))
2853 assert(isa<SExtInst>(Ext) &&
"Unexpected ext type!");
2959 unsigned Factor)
const {
2970 unsigned Factor)
const {
2979 "invalid fpext types");
2987 LLT DestTy,
LLT SrcTy)
const {
2995 EVT DestVT,
EVT SrcVT)
const {
2997 "invalid fpext types");
3055 assert((
MI.getOpcode() == TargetOpcode::G_FADD ||
3056 MI.getOpcode() == TargetOpcode::G_FSUB ||
3057 MI.getOpcode() == TargetOpcode::G_FMUL) &&
3058 "unexpected node in FMAD forming combine");
3079 "unexpected node in FMAD forming combine");
3120 unsigned Index)
const {
3143 bool MathUsed)
const {
3212 LibcallRoutineNames[Call] =
Name;
3215 for (
auto Call : Calls)
3221 return LibcallRoutineNames[Call];
3227 CmpLibcallCCs[Call] =
CC;
3233 return CmpLibcallCCs[Call];
3238 LibcallCallingConvs[Call] =
CC;
3243 return LibcallCallingConvs[Call];
3267 bool HasMultipleConditionRegisters;
3273 bool HasExtractBitsInsn;
3279 DenseMap <unsigned int, unsigned int> BypassSlowDivWidths;
3284 bool JumpIsExpensive;
3303 Align MinStackArgumentAlignment;
3307 Align MinFunctionAlignment;
3311 Align PrefFunctionAlignment;
3314 Align PrefLoopAlignment;
3316 unsigned MaxBytesForAlignment;
3320 unsigned MaxAtomicSizeInBitsSupported;
3324 unsigned MaxDivRemBitWidthSupported;
3328 unsigned MaxLargeFPConvertBitWidthSupported;
3332 unsigned MinCmpXchgSizeInBits;
3335 bool SupportsUnalignedAtomics;
3339 Register StackPointerRegisterToSaveRestore;
3400 ValueTypeActionImpl ValueTypeActions;
3419 const char *LibcallRoutineNames[RTLIB::UNKNOWN_LIBCALL + 1];
3429 void InitLibcalls(
const Triple &TT);
3433 enum IndexedModeActionsBits {
3436 IMAB_MaskedStore = 8,
3437 IMAB_MaskedLoad = 12
3440 void setIndexedModeAction(
unsigned IdxMode, MVT VT,
unsigned Shift,
3443 (
unsigned)Action < 0xf &&
"Table isn't big enough!");
3444 unsigned Ty = (
unsigned)VT.SimpleTy;
3445 IndexedModeActions[Ty][IdxMode] &= ~(0xf << Shift);
3446 IndexedModeActions[Ty][IdxMode] |= ((
uint16_t)Action) << Shift;
3450 unsigned Shift)
const {
3452 "Table isn't big enough!");
3453 unsigned Ty = (
unsigned)VT.SimpleTy;
3454 return (
LegalizeAction)((IndexedModeActions[Ty][IdxMode] >> Shift) & 0xf);
3654 unsigned OpNo)
const {
3670 bool IsSignaling =
false)
const;
3676 MakeLibCallOptions CallOptions,
3684 const uint32_t *CallerPreservedMask,
3723 const MemOp &Op,
unsigned DstAS,
unsigned SrcAS,
3731 const APInt &DemandedElts,
3732 TargetLoweringOpt &TLO)
const;
3736 TargetLoweringOpt &TLO)
const;
3743 const APInt &DemandedElts,
3753 TargetLoweringOpt &TLO)
const;
3770 TargetLoweringOpt &TLO,
unsigned Depth = 0,
3771 bool AssumeSingleUse =
false)
const;
3776 KnownBits &Known, TargetLoweringOpt &TLO,
3778 bool AssumeSingleUse =
false)
const;
3783 DAGCombinerInfo &DCI)
const;
3788 const APInt &DemandedElts,
3789 DAGCombinerInfo &DCI)
const;
3795 const APInt &DemandedElts,
3797 unsigned Depth = 0)
const;
3803 unsigned Depth = 0)
const;
3808 const APInt &DemandedElts,
3810 unsigned Depth = 0)
const;
3828 TargetLoweringOpt &TLO,
unsigned Depth = 0,
3829 bool AssumeSingleUse =
false)
const;
3834 DAGCombinerInfo &DCI)
const;
3850 const APInt &DemandedElts,
3852 unsigned Depth = 0)
const;
3860 const APInt &DemandedElts,
3862 unsigned Depth = 0)
const;
3871 unsigned Depth = 0)
const;
3885 const APInt &DemandedElts,
3887 unsigned Depth = 0)
const;
3895 const APInt &DemandedElts,
3897 unsigned Depth = 0)
const;
3906 APInt &KnownZero, TargetLoweringOpt &TLO,
unsigned Depth = 0)
const;
3915 const APInt &DemandedElts,
3917 TargetLoweringOpt &TLO,
3918 unsigned Depth = 0)
const;
3932 bool PoisonOnly,
unsigned Depth)
const;
3940 bool ConsiderFlags,
unsigned Depth)
const;
3960 unsigned Depth = 0)
const;
3967 unsigned Depth = 0)
const;
4016 bool foldBooleans, DAGCombinerInfo &DCI,
4017 const SDLoc &dl)
const;
4064 bool IsAfterLegal)
const {
4159 bool LegalOps,
bool OptForSize,
4161 unsigned Depth = 0)
const;
4166 unsigned Depth = 0)
const {
4185 bool LegalOps,
bool OptForSize,
4186 unsigned Depth = 0)
const {
4194 bool OptForSize,
unsigned Depth = 0)
const {
4208 unsigned NumParts,
MVT PartVT, std::optional<CallingConv::ID>
CC)
const {
4225 unsigned &PhysReg,
int &
Cost)
const {
4232 const SDValue *Parts,
unsigned NumParts,
4234 std::optional<CallingConv::ID>
CC)
const {
4310 Args = std::move(ArgsList);
4323 Args = std::move(ArgsList);
4332 IsInReg = Call.hasRetAttr(Attribute::InReg);
4334 Call.doesNotReturn() ||
4335 (!isa<InvokeInst>(Call) && isa<UnreachableInst>(Call.getNextNode()));
4338 RetSExt = Call.hasRetAttr(Attribute::SExt);
4339 RetZExt = Call.hasRetAttr(Attribute::ZExt);
4340 NoMerge = Call.hasFnAttr(Attribute::NoMerge);
4346 Args = std::move(ArgsList);