22#ifndef LLVM_CODEGEN_TARGETLOWERING_H
23#define LLVM_CODEGEN_TARGETLOWERING_H
70class FunctionLoweringInfo;
77class LegacyDivergenceAnalysis;
79class MachineBasicBlock;
82class MachineJumpTableInfo;
84class MachineRegisterInfo;
88class ProfileSummaryInfo;
89class TargetLibraryInfo;
91class TargetRegisterClass;
92class TargetRegisterInfo;
93class TargetTransformInfo;
116 bool DstAlignCanChange;
131 Align SrcAlign,
bool IsVolatile,
132 bool MemcpyStrSrc =
false) {
135 Op.DstAlignCanChange = DstAlignCanChange;
136 Op.DstAlign = DstAlign;
137 Op.AllowOverlap = !IsVolatile;
139 Op.ZeroMemset =
false;
140 Op.MemcpyStrSrc = MemcpyStrSrc;
141 Op.SrcAlign = SrcAlign;
146 bool IsZeroMemset,
bool IsVolatile) {
149 Op.DstAlignCanChange = DstAlignCanChange;
150 Op.DstAlign = DstAlign;
151 Op.AllowOverlap = !IsVolatile;
153 Op.ZeroMemset = IsZeroMemset;
154 Op.MemcpyStrSrc =
false;
160 assert(!DstAlignCanChange);
168 return isMemcpy() && !DstAlignCanChange;
399 bool LegalTypes =
true)
const;
457 return HasMultipleConditionRegisters;
493 unsigned DefinedValues)
const {
494 return DefinedValues < 3;
551 return BypassSlowDivWidths;
595 unsigned AddrSpace)
const {
745 unsigned KeptBits)
const {
759 unsigned OldShiftOpcode,
unsigned NewShiftOpcode,
767 if (OldShiftOpcode ==
ISD::SHL &&
CC->isOne())
771 if (XC && NewShiftOpcode ==
ISD::SHL && XC->isOne())
804 unsigned &Cost)
const {
849 return BooleanVectorContents;
850 return isFloat ? BooleanFloatContents : BooleanContents;
873 return SchedPreferenceInfo;
888 assert(RC &&
"This value type is not natively supported!");
896 const Value *)
const {
915 return RepRegClassCostForVT[VT.
SimpleTy];
927 unsigned ExpansionFactor)
const {
928 if (ExpansionFactor == 1)
949 std::fill(std::begin(ValueTypeActions), std::end(ValueTypeActions),
954 return ValueTypeActions[VT.
SimpleTy];
958 ValueTypeActions[VT.
SimpleTy] = Action;
963 return ValueTypeActions;
1031 EVT &IntermediateVT,
1032 unsigned &NumIntermediates,
1033 MVT &RegisterVT)
const;
1040 unsigned &NumIntermediates,
MVT &RegisterVT)
const {
1079 bool ForCodeSize =
false)
const {
1117 if (Op >= std::size(OpActions[0]))
1126 unsigned Scale)
const {
1135 unsigned Scale)
const {
1137 if (Action !=
Legal)
1158 return Supported ? Action :
Expand;
1167#define DAG_INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC, DAGN) \
1168 case ISD::STRICT_##DAGN: EqOpc = ISD::DAGN; break;
1169#define CMP_INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC, DAGN) \
1170 case ISD::STRICT_##DAGN: EqOpc = ISD::SETCC; break;
1171#include "llvm/IR/ConstrainedOps.def"
1182 bool LegalOnly =
false)
const {
1196 bool LegalOnly =
false)
const {
1210 bool LegalOnly =
false)
const {
1252 EVT ConditionVT)
const;
1277 return (NumDests == 1 && NumCmps >= 3) || (NumDests == 2 && NumCmps >= 5) ||
1278 (NumDests == 3 && NumCmps >= 6);
1304 unsigned Shift = 4 * ExtType;
1305 return (
LegalizeAction)((LoadExtActions[ValI][MemI] >> Shift) & 0xf);
1328 "Table isn't big enough!");
1329 return TruncStoreActions[ValI][MemI];
1347 bool LegalOnly)
const {
1358 return getIndexedModeAction(IdxMode, VT, IMAB_Load);
1372 return getIndexedModeAction(IdxMode, VT, IMAB_Store);
1386 return getIndexedModeAction(IdxMode, VT, IMAB_MaskedLoad);
1400 return getIndexedModeAction(IdxMode, VT, IMAB_MaskedStore);
1427 if (Scale != ElemSize && Scale != 1)
1437 assert((
unsigned)
CC < std::size(CondCodeActions) &&
1438 ((
unsigned)VT.
SimpleTy >> 3) < std::size(CondCodeActions[0]) &&
1439 "Table isn't big enough!");
1444 assert(Action !=
Promote &&
"Can't promote condition code!");
1464 "This operation isn't promoted!");
1467 std::map<std::pair<unsigned, MVT::SimpleValueType>,
1469 PromoteToType.find(std::make_pair(Op, VT.
SimpleTy));
1470 if (PTTI != PromoteToType.end())
return PTTI->second;
1473 "Cannot autopromote this type, add it with AddPromotedToType.");
1479 "Didn't find type to promote to!");
1486 bool AllowUnknown =
false)
const {
1495 bool AllowUnknown =
false)
const {
1497 if (
auto *PTy = dyn_cast<PointerType>(Ty))
1500 if (
auto *VTy = dyn_cast<VectorType>(Ty)) {
1501 Type *EltTy = VTy->getElementType();
1503 if (
auto *PTy = dyn_cast<PointerType>(EltTy)) {
1508 VTy->getElementCount());
1515 bool AllowUnknown =
false)
const {
1519 else if (
VectorType *VTy = dyn_cast<VectorType>(Ty)) {
1520 Type *Elm = VTy->getElementType();
1521 if (
PointerType *PT = dyn_cast<PointerType>(Elm)) {
1526 VTy->getElementCount());
1535 bool AllowUnknown =
false)
const {
1547 return RegisterTypeForVT[VT.
SimpleTy];
1554 std::size(RegisterTypeForVT));
1560 unsigned NumIntermediates;
1562 NumIntermediates, RegisterVT);
1584 std::optional<MVT> RegisterVT = std::nullopt)
const {
1587 std::size(NumRegistersForVT));
1593 unsigned NumIntermediates;
1599 return (
BitWidth + RegWidth - 1) / RegWidth;
1625 return DL.getABITypeAlign(ArgTy);
1639 if (NewVT.
isVector() && !Load->hasOneUse())
1655 assert(
unsigned(NT >> 3) < std::size(TargetDAGCombineArray));
1656 return TargetDAGCombineArray[NT >> 3] & (1 << (NT&7));
1730 unsigned * =
nullptr)
const {
1738 unsigned * =
nullptr)
const {
1748 unsigned AddrSpace = 0,
Align Alignment =
Align(1),
1750 unsigned *
Fast =
nullptr)
const;
1759 unsigned *
Fast =
nullptr)
const;
1767 unsigned AddrSpace = 0,
Align Alignment =
Align(1),
1769 unsigned *
Fast =
nullptr)
const;
1777 unsigned *
Fast =
nullptr)
const;
1782 unsigned *
Fast =
nullptr)
const;
1825 return StackPointerRegisterToSaveRestore;
1848 return MinStackArgumentAlignment;
1952 return MaxAtomicSizeInBitsSupported;
1958 return MaxDivRemBitWidthSupported;
1964 return MaxLargeFPConvertBitWidthSupported;
2018 llvm_unreachable(
"Masked atomicrmw expansion unimplemented on this target");
2026 "Generic atomicrmw expansion unimplemented on this target");
2034 "Bit test atomicrmw expansion unimplemented on this target");
2042 "Compare arith atomicrmw expansion unimplemented on this target");
2140 if (
SI->getValueOperand()->getType()->isFloatingPointTy())
2264 bool IsSigned)
const {
2276 return !OptForSize ||
2288 BooleanContents = Ty;
2289 BooleanFloatContents = Ty;
2295 BooleanContents = IntTy;
2296 BooleanFloatContents = FloatTy;
2302 BooleanVectorContents = Ty;
2307 SchedPreferenceInfo = Pref;
2320 StackPointerRegisterToSaveRestore = R;
2329 HasMultipleConditionRegisters = hasManyRegs;
2337 HasExtractBitsInsn = hasExtractInsn;
2347 BypassSlowDivWidths[SlowBitWidth] = FastBitWidth;
2360 virtual std::pair<const TargetRegisterClass *, uint8_t>
2371 assert(Op < std::size(OpActions[0]) &&
"Table isn't big enough!");
2390 MemVT.
isValid() &&
"Table isn't big enough!");
2391 assert((
unsigned)Action < 0x10 &&
"too many bits for bitfield array");
2392 unsigned Shift = 4 * ExtType;
2398 for (
auto ExtType : ExtTypes)
2403 for (
auto MemVT : MemVTs)
2421 for (
auto IdxMode : IdxModes)
2422 setIndexedModeAction(IdxMode, VT, IMAB_Load, Action);
2438 for (
auto IdxMode : IdxModes)
2439 setIndexedModeAction(IdxMode, VT, IMAB_Store, Action);
2455 setIndexedModeAction(IdxMode, VT, IMAB_MaskedLoad, Action);
2465 setIndexedModeAction(IdxMode, VT, IMAB_MaskedStore, Action);
2472 for (
auto CC : CCs) {
2474 "Table isn't big enough!");
2475 assert((
unsigned)Action < 0x10 &&
"too many bits for bitfield array");
2509 for (
auto NT : NTs) {
2510 assert(
unsigned(NT >> 3) < std::size(TargetDAGCombineArray));
2511 TargetDAGCombineArray[NT >> 3] |= 1 << (NT & 7);
2517 MinFunctionAlignment = Alignment;
2523 PrefFunctionAlignment = Alignment;
2531 MaxBytesForAlignment = MaxBytes;
2536 MinStackArgumentAlignment = Alignment;
2544 MaxAtomicSizeInBitsSupported = SizeInBits;
2550 MaxDivRemBitWidthSupported = SizeInBits;
2556 MaxLargeFPConvertBitWidthSupported = SizeInBits;
2561 MinCmpXchgSizeInBits = SizeInBits;
2566 SupportsUnalignedAtomics = UnalignedSupported;
2611 Type *Ty,
unsigned AddrSpace,
2658 return (
From->isIntegerTy() ||
From->isFloatingPointTy()) &&
2698 default:
return false;
2766 switch (
I->getOpcode()) {
2767 case Instruction::FPExt:
2772 case Instruction::ZExt:
2773 if (
isZExtFree(
I->getOperand(0)->getType(),
I->getType()))
2776 case Instruction::SExt:
2803 if (isa<ZExtInst>(Ext))
2806 assert(isa<SExtInst>(Ext) &&
"Unexpected ext type!");
2904 unsigned Factor)
const {
2915 unsigned Factor)
const {
2931 "invalid fpext types");
2939 LLT DestTy,
LLT SrcTy)
const {
2947 EVT DestVT,
EVT SrcVT)
const {
2949 "invalid fpext types");
3007 assert((
MI.getOpcode() == TargetOpcode::G_FADD ||
3008 MI.getOpcode() == TargetOpcode::G_FSUB ||
3009 MI.getOpcode() == TargetOpcode::G_FMUL) &&
3010 "unexpected node in FMAD forming combine");
3031 "unexpected node in FMAD forming combine");
3072 unsigned Index)
const {
3095 bool MathUsed)
const {
3164 LibcallRoutineNames[Call] =
Name;
3167 for (
auto Call : Calls)
3173 return LibcallRoutineNames[Call];
3179 CmpLibcallCCs[Call] =
CC;
3185 return CmpLibcallCCs[Call];
3190 LibcallCallingConvs[Call] =
CC;
3195 return LibcallCallingConvs[Call];
3219 bool HasMultipleConditionRegisters;
3225 bool HasExtractBitsInsn;
3231 DenseMap <unsigned int, unsigned int> BypassSlowDivWidths;
3236 bool JumpIsExpensive;
3255 Align MinStackArgumentAlignment;
3259 Align MinFunctionAlignment;
3263 Align PrefFunctionAlignment;
3266 Align PrefLoopAlignment;
3268 unsigned MaxBytesForAlignment;
3272 unsigned MaxAtomicSizeInBitsSupported;
3276 unsigned MaxDivRemBitWidthSupported;
3280 unsigned MaxLargeFPConvertBitWidthSupported;
3284 unsigned MinCmpXchgSizeInBits;
3287 bool SupportsUnalignedAtomics;
3291 Register StackPointerRegisterToSaveRestore;
3352 ValueTypeActionImpl ValueTypeActions;
3371 const char *LibcallRoutineNames[RTLIB::UNKNOWN_LIBCALL + 1];
3381 void InitLibcalls(
const Triple &TT);
3385 enum IndexedModeActionsBits {
3388 IMAB_MaskedStore = 8,
3389 IMAB_MaskedLoad = 12
3392 void setIndexedModeAction(
unsigned IdxMode, MVT VT,
unsigned Shift,
3395 (
unsigned)Action < 0xf &&
"Table isn't big enough!");
3396 unsigned Ty = (
unsigned)VT.SimpleTy;
3397 IndexedModeActions[Ty][IdxMode] &= ~(0xf << Shift);
3398 IndexedModeActions[Ty][IdxMode] |= ((
uint16_t)Action) << Shift;
3402 unsigned Shift)
const {
3404 "Table isn't big enough!");
3405 unsigned Ty = (
unsigned)VT.SimpleTy;
3406 return (
LegalizeAction)((IndexedModeActions[Ty][IdxMode] >> Shift) & 0xf);
3608 unsigned OpNo)
const {
3624 bool IsSignaling =
false)
const;
3630 MakeLibCallOptions CallOptions,
3638 const uint32_t *CallerPreservedMask,
3677 const MemOp &Op,
unsigned DstAS,
unsigned SrcAS,
3685 const APInt &DemandedElts,
3686 TargetLoweringOpt &TLO)
const;
3690 TargetLoweringOpt &TLO)
const;
3697 const APInt &DemandedElts,
3706 TargetLoweringOpt &TLO)
const;
3723 TargetLoweringOpt &TLO,
unsigned Depth = 0,
3724 bool AssumeSingleUse =
false)
const;
3729 KnownBits &Known, TargetLoweringOpt &TLO,
3731 bool AssumeSingleUse =
false)
const;
3736 DAGCombinerInfo &DCI)
const;
3741 const APInt &DemandedElts,
3742 DAGCombinerInfo &DCI)
const;
3748 const APInt &DemandedElts,
3750 unsigned Depth = 0)
const;
3756 unsigned Depth = 0)
const;
3761 const APInt &DemandedElts,
3763 unsigned Depth = 0)
const;
3781 TargetLoweringOpt &TLO,
unsigned Depth = 0,
3782 bool AssumeSingleUse =
false)
const;
3787 DAGCombinerInfo &DCI)
const;
3803 const APInt &DemandedElts,
3805 unsigned Depth = 0)
const;
3813 const APInt &DemandedElts,
3815 unsigned Depth = 0)
const;
3824 unsigned Depth = 0)
const;
3838 const APInt &DemandedElts,
3840 unsigned Depth = 0)
const;
3848 const APInt &DemandedElts,
3850 unsigned Depth = 0)
const;
3859 APInt &KnownZero, TargetLoweringOpt &TLO,
unsigned Depth = 0)
const;
3868 const APInt &DemandedElts,
3870 TargetLoweringOpt &TLO,
3871 unsigned Depth = 0)
const;
3885 bool PoisonOnly,
unsigned Depth)
const;
3893 bool ConsiderFlags,
unsigned Depth)
const;
3913 unsigned Depth = 0)
const;
3920 unsigned Depth = 0)
const;
3969 bool foldBooleans, DAGCombinerInfo &DCI,
3970 const SDLoc &dl)
const;
4076 bool LegalOps,
bool OptForSize,
4078 unsigned Depth = 0)
const;
4083 unsigned Depth = 0)
const {
4102 bool LegalOps,
bool OptForSize,
4103 unsigned Depth = 0)
const {
4111 bool OptForSize,
unsigned Depth = 0)
const {
4125 unsigned NumParts,
MVT PartVT, std::optional<CallingConv::ID>
CC)
const {
4142 unsigned &PhysReg,
int &Cost)
const {
4149 const SDValue *Parts,
unsigned NumParts,
4151 std::optional<CallingConv::ID>
CC)
const {
4227 Args = std::move(ArgsList);
4240 Args = std::move(ArgsList);
4249 IsInReg = Call.hasRetAttr(Attribute::InReg);
4251 Call.doesNotReturn() ||
4252 (!isa<InvokeInst>(Call) && isa<UnreachableInst>(Call.getNextNode()));
4255 RetSExt = Call.hasRetAttr(Attribute::SExt);
4256 RetZExt = Call.hasRetAttr(Attribute::ZExt);
4257 NoMerge = Call.hasFnAttr(Attribute::NoMerge);
4263 Args = std::move(ArgsList);
4372 bool Value =
true) {
4384 std::pair<SDValue, SDValue>
LowerCallTo(CallLoweringInfo &CLI)
const;