22 #ifndef LLVM_CODEGEN_TARGETLOWERING_H
23 #define LLVM_CODEGEN_TARGETLOWERING_H
65 class BranchProbability;
70 class FunctionLoweringInfo;
75 class LegacyDivergenceAnalysis;
77 class MachineBasicBlock;
78 class MachineFunction;
80 class MachineJumpTableInfo;
82 class MachineRegisterInfo;
86 class ProfileSummaryInfo;
87 class TargetLibraryInfo;
89 class TargetRegisterClass;
90 class TargetRegisterInfo;
91 class TargetTransformInfo;
112 bool DstAlignCanChange;
128 bool MemcpyStrSrc =
false) {
131 Op.DstAlignCanChange = DstAlignCanChange;
132 Op.DstAlign = DstAlign;
135 Op.ZeroMemset =
false;
136 Op.MemcpyStrSrc = MemcpyStrSrc;
137 Op.SrcAlign = SrcAlign;
145 Op.DstAlignCanChange = DstAlignCanChange;
146 Op.DstAlign = DstAlign;
149 Op.ZeroMemset = IsZeroMemset;
150 Op.MemcpyStrSrc =
false;
154 uint64_t
size()
const {
return Size; }
156 assert(!DstAlignCanChange);
164 return isMemcpy() && !DstAlignCanChange;
376 bool LegalTypes =
true)
const;
420 return HasMultipleConditionRegisters;
456 unsigned DefinedValues)
const {
457 return DefinedValues < 3;
514 return BypassSlowDivWidths;
572 unsigned AddrSpace)
const {
718 unsigned KeptBits)
const {
732 unsigned OldShiftOpcode,
unsigned NewShiftOpcode,
774 unsigned &Cost)
const {
816 return BooleanVectorContents;
817 return isFloat ? BooleanFloatContents : BooleanContents;
826 return SchedPreferenceInfo;
841 assert(RC &&
"This value type is not natively supported!");
849 const Value *)
const {
868 return RepRegClassCostForVT[VT.
SimpleTy];
899 return ValueTypeActions[VT.
SimpleTy];
903 ValueTypeActions[VT.
SimpleTy] = Action;
908 return ValueTypeActions;
916 return getTypeConversion(
Context, VT).first;
929 return getTypeConversion(
Context, VT).second;
961 unsigned &NumIntermediates,
962 MVT &RegisterVT)
const;
969 unsigned &NumIntermediates,
MVT &RegisterVT)
const {
1004 bool ForCodeSize =
false)
const {
1045 unsigned Scale)
const {
1054 unsigned Scale)
const {
1056 if (Action !=
Legal)
1077 return Supported ? Action :
Expand;
1086 #define DAG_INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC, DAGN) \
1087 case ISD::STRICT_##DAGN: EqOpc = ISD::DAGN; break;
1088 #define CMP_INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC, DAGN) \
1089 case ISD::STRICT_##DAGN: EqOpc = ISD::SETCC; break;
1090 #include "llvm/IR/ConstrainedOps.def"
1101 bool LegalOnly =
false)
const {
1115 bool LegalOnly =
false)
const {
1129 bool LegalOnly =
false)
const {
1158 uint64_t BW =
DL.getIndexSizeInBits(0u);
1159 uint64_t Range = (
High - Low).getLimitedValue(
UINT64_MAX - 1) + 1;
1192 return (NumDests == 1 && NumCmps >= 3) || (NumDests == 2 && NumCmps >= 5) ||
1193 (NumDests == 3 && NumCmps >= 6);
1219 unsigned Shift = 4 * ExtType;
1243 "Table isn't big enough!");
1244 return TruncStoreActions[ValI][MemI];
1265 return getIndexedModeAction(IdxMode, VT, IMAB_Load);
1279 return getIndexedModeAction(IdxMode, VT, IMAB_Store);
1293 return getIndexedModeAction(IdxMode, VT, IMAB_MaskedLoad);
1307 return getIndexedModeAction(IdxMode, VT, IMAB_MaskedStore);
1332 "Table isn't big enough!");
1337 assert(Action !=
Promote &&
"Can't promote condition code!");
1357 "This operation isn't promoted!");
1360 std::map<std::pair<unsigned, MVT::SimpleValueType>,
1362 PromoteToType.find(std::make_pair(
Op, VT.
SimpleTy));
1363 if (PTTI != PromoteToType.end())
return PTTI->second;
1366 "Cannot autopromote this type, add it with AddPromotedToType.");
1372 "Didn't find type to promote to!");
1383 bool AllowUnknown =
false)
const {
1385 if (
auto *PTy = dyn_cast<PointerType>(Ty))
1388 if (
auto *VTy = dyn_cast<VectorType>(Ty)) {
1389 Type *EltTy = VTy->getElementType();
1391 if (
auto *PTy = dyn_cast<PointerType>(EltTy)) {
1396 VTy->getElementCount());
1403 bool AllowUnknown =
false)
const {
1407 else if (
VectorType *VTy = dyn_cast<VectorType>(Ty)) {
1408 Type *Elm = VTy->getElementType();
1409 if (
PointerType *PT = dyn_cast<PointerType>(Elm)) {
1414 VTy->getElementCount());
1423 bool AllowUnknown =
false)
const {
1435 return RegisterTypeForVT[VT.
SimpleTy];
1448 unsigned NumIntermediates;
1450 NumIntermediates, RegisterVT);
1476 unsigned NumIntermediates;
1482 return (
BitWidth + RegWidth - 1) / RegWidth;
1508 return DL.getABITypeAlign(ArgTy);
1539 return TargetDAGCombineArray[NT >> 3] & (1 << (NT&7));
1612 bool * =
nullptr)
const {
1620 bool * =
nullptr)
const {
1630 unsigned AddrSpace = 0,
Align Alignment =
Align(1),
1632 bool *Fast =
nullptr)
const;
1641 bool *Fast =
nullptr)
const;
1649 unsigned AddrSpace = 0,
Align Alignment =
Align(1),
1651 bool *Fast =
nullptr)
const;
1659 bool *Fast =
nullptr)
const;
1664 bool *Fast =
nullptr)
const;
1707 return StackPointerRegisterToSaveRestore;
1730 return MinStackArgumentAlignment;
1741 return PrefLoopAlignment;
1805 unsigned & )
const {
1831 return MaxAtomicSizeInBitsSupported;
1877 llvm_unreachable(
"Masked atomicrmw expansion unimplemented on this target");
1925 return Builder.CreateFence(Ord);
1934 return Builder.CreateFence(Ord);
2063 bool IsSigned)
const {
2075 BooleanContents = Ty;
2076 BooleanFloatContents = Ty;
2082 BooleanContents = IntTy;
2083 BooleanFloatContents = FloatTy;
2089 BooleanVectorContents = Ty;
2094 SchedPreferenceInfo = Pref;
2107 StackPointerRegisterToSaveRestore = R;
2116 HasMultipleConditionRegisters = hasManyRegs;
2124 HasExtractBitsInsn = hasExtractInsn;
2134 BypassSlowDivWidths[SlowBitWidth] = FastBitWidth;
2147 virtual std::pair<const TargetRegisterClass *, uint8_t>
2160 OpActions[(unsigned)VT.
SimpleTy][
Op] = Action;
2168 MemVT.
isValid() &&
"Table isn't big enough!");
2169 assert((
unsigned)Action < 0x10 &&
"too many bits for bitfield array");
2170 unsigned Shift = 4 * ExtType;
2189 setIndexedModeAction(IdxMode, VT, IMAB_Load, Action);
2198 setIndexedModeAction(IdxMode, VT, IMAB_Store, Action);
2208 setIndexedModeAction(IdxMode, VT, IMAB_MaskedLoad, Action);
2218 setIndexedModeAction(IdxMode, VT, IMAB_MaskedStore, Action);
2226 "Table isn't big enough!");
2227 assert((
unsigned)Action < 0x10 &&
"too many bits for bitfield array");
2256 TargetDAGCombineArray[NT >> 3] |= 1 << (NT&7);
2261 MinFunctionAlignment = Alignment;
2267 PrefFunctionAlignment = Alignment;
2277 MinStackArgumentAlignment = Alignment;
2285 MaxAtomicSizeInBitsSupported = SizeInBits;
2290 MinCmpXchgSizeInBits = SizeInBits;
2295 SupportsUnalignedAtomics = UnalignedSupported;
2340 Type *Ty,
unsigned AddrSpace,
2351 Type *Ty,
unsigned AS = 0)
const {
2402 return (
From->isIntegerTy() ||
From->isFloatingPointTy()) &&
2438 default:
return false;
2501 switch (
I->getOpcode()) {
2502 case Instruction::FPExt:
2507 case Instruction::ZExt:
2508 if (
isZExtFree(
I->getOperand(0)->getType(),
I->getType()))
2511 case Instruction::SExt:
2538 if (isa<ZExtInst>(
Ext))
2541 assert(isa<SExtInst>(
Ext) &&
"Unexpected ext type!");
2625 unsigned Factor)
const {
2636 unsigned Factor)
const {
2652 "invalid fpext types");
2660 EVT DestVT,
EVT SrcVT)
const {
2662 "invalid fpext types");
2708 "unexpected node in FMAD forming combine");
2741 unsigned Index)
const {
2764 bool MathUsed)
const {
2807 LibcallRoutineNames[Call] =
Name;
2812 return LibcallRoutineNames[Call];
2818 CmpLibcallCCs[Call] = CC;
2824 return CmpLibcallCCs[Call];
2829 LibcallCallingConvs[Call] = CC;
2834 return LibcallCallingConvs[Call];
2858 bool HasMultipleConditionRegisters;
2864 bool HasExtractBitsInsn;
2875 bool JumpIsExpensive;
2894 Align MinStackArgumentAlignment;
2898 Align MinFunctionAlignment;
2902 Align PrefFunctionAlignment;
2905 Align PrefLoopAlignment;
2909 unsigned MaxAtomicSizeInBitsSupported;
2913 unsigned MinCmpXchgSizeInBits;
2916 bool SupportsUnalignedAtomics;
2920 Register StackPointerRegisterToSaveRestore;
2981 ValueTypeActionImpl ValueTypeActions;
3002 const char *LibcallRoutineNames[RTLIB::UNKNOWN_LIBCALL + 1];
3012 void InitLibcalls(
const Triple &TT);
3016 enum IndexedModeActionsBits {
3019 IMAB_MaskedStore = 8,
3020 IMAB_MaskedLoad = 12
3023 void setIndexedModeAction(
unsigned IdxMode, MVT VT,
unsigned Shift,
3026 (
unsigned)Action < 0xf &&
"Table isn't big enough!");
3027 unsigned Ty = (unsigned)VT.SimpleTy;
3028 IndexedModeActions[Ty][IdxMode] &= ~(0xf <<
Shift);
3029 IndexedModeActions[Ty][IdxMode] |= ((
uint16_t)Action) <<
Shift;
3033 unsigned Shift)
const {
3035 "Table isn't big enough!");
3036 unsigned Ty = (unsigned)VT.SimpleTy;
3229 bool IsSignaling =
false)
const;
3235 MakeLibCallOptions CallOptions,
3243 const uint32_t *CallerPreservedMask,
3280 const MemOp &
Op,
unsigned DstAS,
unsigned SrcAS,
3288 const APInt &DemandedElts,
3289 TargetLoweringOpt &TLO)
const;
3293 TargetLoweringOpt &TLO)
const;
3300 const APInt &DemandedElts,
3309 TargetLoweringOpt &TLO)
const;
3326 TargetLoweringOpt &TLO,
unsigned Depth = 0,
3327 bool AssumeSingleUse =
false)
const;
3332 KnownBits &Known, TargetLoweringOpt &TLO,
3334 bool AssumeSingleUse =
false)
const;
3339 DAGCombinerInfo &DCI)
const;
3345 const APInt &DemandedElts,
3347 unsigned Depth)
const;
3353 unsigned Depth = 0)
const;
3358 const APInt &DemandedElts,
3360 unsigned Depth = 0)
const;
3378 TargetLoweringOpt &TLO,
unsigned Depth = 0,
3379 bool AssumeSingleUse =
false)
const;
3385 DAGCombinerInfo &DCI)
const;
3393 const APInt &DemandedElts,
3395 unsigned Depth = 0)
const;
3403 const APInt &DemandedElts,
3405 unsigned Depth = 0)
const;
3414 unsigned Depth = 0)
const;
3428 const APInt &DemandedElts,
3430 unsigned Depth = 0)
const;
3438 const APInt &DemandedElts,
3440 unsigned Depth = 0)
const;
3449 APInt &KnownZero, TargetLoweringOpt &TLO,
unsigned Depth = 0)
const;
3458 const APInt &DemandedElts,
3460 TargetLoweringOpt &TLO,
3461 unsigned Depth = 0)
const;
3488 unsigned Depth = 0)
const;
3530 bool foldBooleans, DAGCombinerInfo &DCI,
3531 const SDLoc &dl)
const;
3626 bool LegalOps,
bool OptForSize,
3628 unsigned Depth = 0)
const;
3633 bool LegalOps,
bool OptForSize,
3634 unsigned Depth = 0)
const {
3649 bool OptForSize,
unsigned Depth = 0)
const {
3663 unsigned NumParts,
MVT PartVT,
3671 const SDValue *Parts,
unsigned NumParts,
3770 IsInReg = Call.hasRetAttr(Attribute::InReg);
3772 Call.doesNotReturn() ||
3773 (!isa<InvokeInst>(Call) && isa<UnreachableInst>(Call.getNextNode()));
3776 RetSExt = Call.hasRetAttr(Attribute::SExt);
3777 RetZExt = Call.hasRetAttr(Attribute::ZExt);
3778 NoMerge = Call.hasFnAttr(Attribute::NoMerge);
3888 bool Value =
true) {
3900 std::pair<SDValue, SDValue>
LowerCallTo(CallLoweringInfo &CLI)
const;
3959 return "__clear_cache";
3979 return VT.
bitsLT(MinVT) ? MinVT : VT;
3987 bool isVarArg)
const {
3997 return DL.isLittleEndian();
4023 assert(
SI.isAtomic() &&
"violated precondition");
4198 virtual std::pair<unsigned, const TargetRegisterClass *>
4203 if (ConstraintCode ==
"m")
4216 std::vector<SDValue> &Ops,
4222 const AsmOperandInfo &OpInfo,
4268 int Enabled,
int &RefinementSteps,
4269 bool &UseOneConstNR,
bool Reciprocal)
const {
4284 int Enabled,
int &RefinementSteps)
const {
4410 bool IsNegative =
false)
const;
4453 bool IsCompressedMemory)
const;
4537 bool IsSignaling =
false)
const;