30#include "llvm/IR/IntrinsicsAArch64.h"
33#include <initializer_list>
35#define DEBUG_TYPE "aarch64-legalinfo"
38using namespace LegalizeActions;
39using namespace LegalizeMutations;
40using namespace LegalityPredicates;
41using namespace MIPatternMatch;
45 using namespace TargetOpcode;
63 std::initializer_list<LLT> PackedVectorAllTypeList = {
69 std::initializer_list<LLT> ScalarAndPtrTypesList = {s8, s16, s32, s64, p0};
76 if (!ST.hasNEON() || !ST.hasFPARMv8()) {
83 const bool HasFP16 = ST.hasFullFP16();
84 const LLT &MinFPScalar = HasFP16 ? s16 : s32;
86 const bool HasCSSC = ST.hasCSSC();
87 const bool HasRCPC3 = ST.hasRCPC3();
90 {G_IMPLICIT_DEF, G_FREEZE, G_CONSTANT_FOLD_BARRIER})
91 .legalFor({p0, s8, s16, s32, s64})
92 .legalFor(PackedVectorAllTypeList)
97 return Query.
Types[0].isVector() &&
98 (Query.
Types[0].getElementType() != s64 ||
99 Query.
Types[0].getNumElements() != 2);
102 LLT EltTy = Query.
Types[0].getElementType();
105 return std::make_pair(0, EltTy);
110 .legalFor(PackedVectorAllTypeList)
121 .
legalFor({s32, s64, v4s32, v2s32, v2s64})
122 .widenScalarToNextPow2(0)
126 .legalFor({s32, s64, v2s32, v2s64, v4s32, v4s16, v8s16, v16s8, v8s8})
127 .widenScalarToNextPow2(0)
135 return Query.
Types[0].getNumElements() <= 2;
140 return Query.
Types[0].getNumElements() <= 4;
145 return Query.
Types[0].getNumElements() <= 16;
152 const auto &SrcTy = Query.
Types[0];
153 const auto &AmtTy = Query.
Types[1];
154 return !SrcTy.isVector() && SrcTy.getSizeInBits() == 32 &&
155 AmtTy.getSizeInBits() == 32;
169 .widenScalarToNextPow2(0)
178 .
legalFor({{p0, s64}, {v2p0, v2s64}})
179 .clampScalar(1, s64, s64);
184 .legalFor({s32, s64})
186 .clampScalar(0, s32, s64)
191 .lowerFor({s8, s16, s32, s64, v2s64, v4s32, v2s32})
193 .clampScalarOrElt(0, s32, s64)
194 .clampNumElements(0, v2s32, v4s32)
195 .clampNumElements(0, v2s64, v2s64)
196 .moreElementsToNextPow2(0);
200 .widenScalarToNextPow2(0, 32)
205 .legalFor({s64, v8s16, v16s8, v4s32})
209 {G_SMIN, G_SMAX, G_UMIN, G_UMAX});
212 .
legalFor({s32, s64, v8s8, v16s8, v4s16, v8s16, v2s32, v4s32})
219 .
legalFor({v8s8, v16s8, v4s16, v8s16, v2s32, v4s32});
230 {G_SADDE, G_SSUBE, G_UADDE, G_USUBE, G_SADDO, G_SSUBO, G_UADDO, G_USUBO})
231 .legalFor({{s32, s32}, {s64, s32}})
232 .clampScalar(0, s32, s64)
237 G_FABS, G_FSQRT, G_FMAXNUM, G_FMINNUM,
238 G_FMAXIMUM, G_FMINIMUM, G_FCEIL, G_FFLOOR,
239 G_FRINT, G_FNEARBYINT, G_INTRINSIC_TRUNC,
240 G_INTRINSIC_ROUND, G_INTRINSIC_ROUNDEVEN})
241 .legalFor({MinFPScalar, s32, s64, v2s32, v4s32, v2s64})
243 const auto &Ty = Query.
Types[0];
244 return (Ty == v8s16 || Ty == v4s16) && HasFP16;
247 .minScalarOrElt(0, MinFPScalar)
263 const auto &Ty = Query.
Types[0];
264 return Ty.isVector() && Ty.getElementType() == s16 &&
267 [=](
const LegalityQuery &Query) {
return std::make_pair(0, s16); })
272 return Query.
Types[0] == s16 && !ST.hasFullFP16();
274 [=](
const LegalityQuery &Query) {
return std::make_pair(0, s32); })
275 .legalFor({s16, s32, s64, v2s32, v4s32, v2s64, v2s16, v4s16, v8s16});
278 {G_FCOS, G_FSIN, G_FPOW, G_FLOG, G_FLOG2, G_FLOG10,
279 G_FEXP, G_FEXP2, G_FEXP10})
284 .libcallFor({s32, s64});
308 for (
unsigned Op : {G_SEXTLOAD, G_ZEXTLOAD}) {
311 if (
Op == G_SEXTLOAD)
316 .legalForTypesWithMemDesc({{s32, p0, s8, 8},
324 {v2s32, p0, s64, 8}})
325 .widenScalarToNextPow2(0)
326 .clampScalar(0, s32, s64)
329 .unsupportedIfMemSizeNotPow2()
344 return HasRCPC3 && Query.
Types[0] == s128 &&
348 return Query.
Types[0] == s128 &&
351 .legalForTypesWithMemDesc({{s8, p0, s8, 8},
358 {v16s8, p0, s128, 8},
360 {v8s16, p0, s128, 8},
362 {v4s32, p0, s128, 8},
363 {v2s64, p0, s128, 8}})
365 .legalForTypesWithMemDesc({{s32, p0, s8, 8}, {s32, p0, s16, 8}})
366 .widenScalarToNextPow2(0, 8)
372 return Query.
Types[0].isScalar() &&
374 Query.
Types[0].getSizeInBits() > 32;
377 .clampMaxNumElements(0, s8, 16)
387 return HasRCPC3 && Query.
Types[0] == s128 &&
391 return Query.
Types[0] == s128 &&
394 .legalForTypesWithMemDesc(
395 {{s8, p0, s8, 8}, {s16, p0, s8, 8},
398 {s16, p0, s16, 8}, {s32, p0, s16, 8},
400 {s32, p0, s8, 8}, {s32, p0, s16, 8}, {s32, p0, s32, 8},
401 {s64, p0, s64, 8}, {s64, p0, s32, 8},
402 {p0, p0, s64, 8}, {s128, p0, s128, 8}, {v16s8, p0, s128, 8},
403 {v8s8, p0, s64, 8}, {v4s16, p0, s64, 8}, {v8s16, p0, s128, 8},
404 {v2s32, p0, s64, 8}, {v4s32, p0, s128, 8}, {v2s64, p0, s128, 8}})
405 .clampScalar(0, s8, s64)
407 return Query.
Types[0].isScalar() &&
411 .clampMaxNumElements(0, s8, 16)
433 {p0, v16s8, v16s8, 8},
434 {p0, v4s16, v4s16, 8},
435 {p0, v8s16, v8s16, 8},
436 {p0, v2s32, v2s32, 8},
437 {p0, v4s32, v4s32, 8},
438 {p0, v2s64, v2s64, 8},
444 auto IndexedLoadBasicPred = [=](
const LegalityQuery &Query) {
447 if (
llvm::find(PackedVectorAllTypesVec, LdTy) ==
448 PackedVectorAllTypesVec.
end() &&
449 llvm::find(ScalarAndPtrTypesVec, LdTy) == ScalarAndPtrTypesVec.
end() &&
474 return MemTy == s8 || MemTy == s16;
476 return MemTy == s8 || MemTy == s16 || MemTy == s32;
484 .widenScalarToNextPow2(0)
488 const auto &Ty = Query.
Types[0];
489 if (HasFP16 && Ty == s16)
491 return Ty == s32 || Ty == s64 || Ty == s128;
493 .clampScalar(0, MinFPScalar, s128);
508 .clampScalar(1, s32, s64)
509 .clampScalar(0, s32, s32)
510 .minScalarEltSameAsIf(
524 .clampNumElements(0, v2s32, v4s32);
531 const auto &Ty = Query.
Types[0];
532 return Ty.isVector() && Ty.getElementType() == s16 && !HasFP16;
534 [=](
const LegalityQuery &Query) {
return std::make_pair(0, s16); })
539 return Query.
Types[0] == s16 && !HasFP16;
541 [=](
const LegalityQuery &Query) {
return std::make_pair(0, s32); })
542 .legalFor({{s16, s16},
551 .clampScalar(1, s32, s64)
552 .clampScalar(0, s32, s32)
553 .minScalarEltSameAsIf(
561 .clampNumElements(0, v2s32, v4s32)
562 .clampMaxNumElements(1, s64, 2);
566 unsigned DstSize = Query.
Types[0].getSizeInBits();
569 if (Query.
Types[0].isVector())
572 if (DstSize < 8 || DstSize >= 128 || !
isPowerOf2_32(DstSize))
587 .legalIf(ExtLegalFunc)
588 .
legalFor({{v2s64, v2s32}, {v4s32, v4s16}, {v8s16, v8s8}})
589 .clampScalar(0, s64, s64)
596 return (Query.
Types[0].getScalarSizeInBits() >
597 Query.
Types[1].getScalarSizeInBits() * 2) &&
598 Query.
Types[0].isVector() &&
599 (Query.
Types[1].getScalarSizeInBits() == 8 ||
600 Query.
Types[1].getScalarSizeInBits() == 16);
604 .
legalFor({{v2s32, v2s64}, {v4s16, v4s32}, {v8s8, v8s16}})
606 .clampMaxNumElements(0, s8, 8)
607 .clampMaxNumElements(0, s16, 4)
608 .clampMaxNumElements(0, s32, 2)
624 .legalFor(PackedVectorAllTypeList)
631 {{s16, s32}, {s16, s64}, {s32, s64}, {v4s16, v4s32}, {v2s32, v2s64}})
632 .clampNumElements(0, v4s16, v4s16)
638 {{s32, s16}, {s64, s16}, {s64, s32}, {v4s32, v4s16}, {v2s64, v2s32}})
639 .clampNumElements(0, v4s32, v4s32)
645 .legalForCartesianProduct({s32, s64, v2s64, v4s32, v2s32})
648 (Query.
Types[1] == s16 || Query.
Types[1] == v4s16 ||
649 Query.
Types[1] == v8s16) &&
650 (Query.
Types[0] == s32 || Query.
Types[0] == s64 ||
651 Query.
Types[0] == v4s16 || Query.
Types[0] == v8s16);
653 .widenScalarToNextPow2(0)
660 return Query.
Types[0].getScalarSizeInBits() >
661 Query.
Types[1].getScalarSizeInBits();
666 return Query.
Types[0].getScalarSizeInBits() <
667 Query.
Types[1].getScalarSizeInBits();
670 .clampNumElements(0, v4s16, v8s16)
675 .legalForCartesianProduct({s32, s64, v2s64, v4s32, v2s32})
678 (Query.
Types[0] == s16 || Query.
Types[0] == v4s16 ||
679 Query.
Types[0] == v8s16) &&
680 (Query.
Types[1] == s32 || Query.
Types[1] == s64 ||
681 Query.
Types[1] == v4s16 || Query.
Types[1] == v8s16);
683 .widenScalarToNextPow2(1)
690 return Query.
Types[0].getScalarSizeInBits() <
691 Query.
Types[1].getScalarSizeInBits();
696 return Query.
Types[0].getScalarSizeInBits() >
697 Query.
Types[1].getScalarSizeInBits();
700 .clampNumElements(0, v4s16, v8s16)
707 .clampScalar(0, s32, s32);
711 .
legalFor({{s32, s32}, {s64, s32}, {p0, s32}})
712 .widenScalarToNextPow2(0)
727 .
legalFor({{s64, p0}, {v2s64, v2p0}})
728 .widenScalarToNextPow2(0, 64)
733 return Query.
Types[0].getSizeInBits() != Query.
Types[1].getSizeInBits();
735 .legalFor({{p0, s64}, {v2p0, v2s64}});
744 v8s16, v4s16, v2s16, v4s32, v2s32, v2s64,
753 .clampScalar(0, s8, s64)
762 return Query.
Types[0].getSizeInBits() == 128;
764 .clampScalar(0, s32, s64)
768 {G_ATOMICRMW_XCHG, G_ATOMICRMW_ADD, G_ATOMICRMW_SUB, G_ATOMICRMW_AND,
769 G_ATOMICRMW_OR, G_ATOMICRMW_XOR, G_ATOMICRMW_MIN, G_ATOMICRMW_MAX,
770 G_ATOMICRMW_UMIN, G_ATOMICRMW_UMAX})
771 .clampScalar(0, s32, s64)
777 for (
unsigned Op : {G_MERGE_VALUES, G_UNMERGE_VALUES}) {
778 unsigned BigTyIdx =
Op == G_MERGE_VALUES ? 0 : 1;
779 unsigned LitTyIdx =
Op == G_MERGE_VALUES ? 1 : 0;
786 switch (Q.
Types[BigTyIdx].getSizeInBits()) {
794 switch (Q.
Types[LitTyIdx].getSizeInBits()) {
808 const LLT &EltTy = Query.
Types[1].getElementType();
809 return Query.
Types[0] != EltTy;
814 return VecTy == v2s16 || VecTy == v4s16 || VecTy == v8s16 ||
815 VecTy == v4s32 || VecTy == v2s64 || VecTy == v2s32 ||
816 VecTy == v8s8 || VecTy == v16s8 || VecTy == v2p0;
822 return Query.
Types[1].getNumElements() <= 2;
827 return Query.
Types[1].getNumElements() <= 4;
832 return Query.
Types[1].getNumElements() <= 8;
837 return Query.
Types[1].getNumElements() <= 16;
840 .minScalarOrElt(0, s8)
859 .clampNumElements(0, v4s32, v4s32)
869 {s32, s64, v8s8, v16s8, v4s16, v8s16, v2s32, v4s32})
871 .widenScalarToNextPow2(1, 32)
872 .clampScalar(1, s32, s64)
873 .scalarSameSizeAs(0, 1);
879 .widenScalarToNextPow2(0, 32)
890 return (HasCSSC &&
typeInSet(0, {s32, s64})(Query));
893 return (!HasCSSC &&
typeInSet(0, {s32, s64})(Query));
905 {v2s64, v2p0, v2s32, v4s32, v4s16, v16s8, v8s8, v8s16}, DstTy);
910 return !Query.
Types[1].isVector();
914 return Query.
Types[0].isVector() && Query.
Types[1].isVector() &&
915 Query.
Types[0].getNumElements() >
916 Query.
Types[1].getNumElements();
920 .clampNumElements(0, v4s32, v4s32)
921 .clampNumElements(0, v2s64, v2s64)
924 return Query.
Types[0].isVector() && Query.
Types[1].isVector() &&
925 Query.
Types[0].getNumElements() <
926 Query.
Types[1].getNumElements();
931 .
legalFor({{v4s32, v2s32}, {v8s16, v4s16}, {v16s8, v8s8}});
939 G_STACKRESTORE}).lower();
948 .customForCartesianProduct({p0}, {s8}, {s64})
952 .legalForCartesianProduct({p0}, {p0}, {s64})
968 .legalFor({s32, s64});
970 .legalFor(PackedVectorAllTypeList)
977 .
legalFor({{s32, v2s32}, {s32, v4s32}, {s64, v2s64}})
979 const auto &Ty = Query.
Types[1];
980 return (Ty == v4s16 || Ty == v8s16) && HasFP16;
982 .minScalarOrElt(0, MinFPScalar)
996 .clampMaxNumElements(1, s64, 2)
1003 G_VECREDUCE_FMINIMUM, G_VECREDUCE_FMAXIMUM})
1004 .legalFor({{s32, v4s32}, {s32, v2s32}, {s64, v2s64}})
1006 const auto &Ty = Query.
Types[1];
1007 return Query.
Types[0] == s16 && (Ty == v8s16 || Ty == v4s16) && HasFP16;
1009 .minScalarOrElt(0, MinFPScalar)
1023 {G_VECREDUCE_SMIN, G_VECREDUCE_SMAX, G_VECREDUCE_UMIN, G_VECREDUCE_UMAX})
1024 .legalFor({{s8, v8s8},
1030 .clampMaxNumElements(1, s64, 2)
1038 {G_VECREDUCE_OR, G_VECREDUCE_AND, G_VECREDUCE_XOR})
1054 return std::make_pair(1, SrcTy.
divide(2));
1063 .customFor({{s32, s32}, {s32, s64}, {s64, s64}})
1067 .
legalFor({{s32, s64}, {s64, s64}})
1069 return Q.
Types[0].isScalar() && Q.
Types[1].getScalarSizeInBits() < 64;
1075 .customFor({{s32, s32}, {s64, s64}});
1081 .legalFor({{s32, s32},
1085 .customFor({{s128, s128},
1093 .legalFor({{v8s8, v8s8},
1095 .customFor({{s32, s32},
1104 .clampScalar(0, s32, s128)
1105 .widenScalarToNextPow2(0)
1106 .minScalarEltSameAsIf(always, 1, 0)
1107 .maxScalarEltSameAsIf(always, 1, 0);
1115 .legalFor({{s64, s32}, {s64, s64}});
1131 verify(*ST.getInstrInfo());
1139 switch (
MI.getOpcode()) {
1143 case TargetOpcode::G_VAARG:
1144 return legalizeVaArg(
MI,
MRI, MIRBuilder);
1145 case TargetOpcode::G_LOAD:
1146 case TargetOpcode::G_STORE:
1147 return legalizeLoadStore(
MI,
MRI, MIRBuilder, Observer);
1148 case TargetOpcode::G_SHL:
1149 case TargetOpcode::G_ASHR:
1150 case TargetOpcode::G_LSHR:
1151 return legalizeShlAshrLshr(
MI,
MRI, MIRBuilder, Observer);
1152 case TargetOpcode::G_GLOBAL_VALUE:
1153 return legalizeSmallCMGlobalValue(
MI,
MRI, MIRBuilder, Observer);
1154 case TargetOpcode::G_SBFX:
1155 case TargetOpcode::G_UBFX:
1156 return legalizeBitfieldExtract(
MI,
MRI, Helper);
1157 case TargetOpcode::G_FSHL:
1158 case TargetOpcode::G_FSHR:
1159 return legalizeFunnelShift(
MI,
MRI, MIRBuilder, Observer, Helper);
1160 case TargetOpcode::G_ROTR:
1161 return legalizeRotate(
MI,
MRI, Helper);
1162 case TargetOpcode::G_CTPOP:
1163 return legalizeCTPOP(
MI,
MRI, Helper);
1164 case TargetOpcode::G_ATOMIC_CMPXCHG:
1165 return legalizeAtomicCmpxchg128(
MI,
MRI, Helper);
1166 case TargetOpcode::G_CTTZ:
1167 return legalizeCTTZ(
MI, Helper);
1168 case TargetOpcode::G_BZERO:
1169 case TargetOpcode::G_MEMCPY:
1170 case TargetOpcode::G_MEMMOVE:
1171 case TargetOpcode::G_MEMSET:
1172 return legalizeMemOps(
MI, Helper);
1173 case TargetOpcode::G_FCOPYSIGN:
1174 return legalizeFCopySign(
MI, Helper);
1175 case TargetOpcode::G_EXTRACT_VECTOR_ELT:
1176 return legalizeExtractVectorElt(
MI,
MRI, Helper);
1187 assert(
MI.getOpcode() == TargetOpcode::G_FSHL ||
1188 MI.getOpcode() == TargetOpcode::G_FSHR);
1192 Register ShiftNo =
MI.getOperand(3).getReg();
1193 LLT ShiftTy =
MRI.getType(ShiftNo);
1198 LLT OperationTy =
MRI.getType(
MI.getOperand(0).getReg());
1202 if (!VRegAndVal || VRegAndVal->Value.urem(
BitWidth) == 0)
1208 Amount =
MI.getOpcode() == TargetOpcode::G_FSHL ?
BitWidth - Amount : Amount;
1212 if (ShiftTy.
getSizeInBits() == 64 &&
MI.getOpcode() == TargetOpcode::G_FSHR &&
1219 if (
MI.getOpcode() == TargetOpcode::G_FSHR) {
1221 MI.getOperand(3).setReg(Cast64.getReg(0));
1226 else if (
MI.getOpcode() == TargetOpcode::G_FSHL) {
1228 {
MI.getOperand(1).
getReg(),
MI.getOperand(2).getReg(),
1230 MI.eraseFromParent();
1241 LLT AmtTy =
MRI.getType(AmtReg);
1247 MI.getOperand(2).setReg(NewAmt.getReg(0));
1252bool AArch64LegalizerInfo::legalizeSmallCMGlobalValue(
1255 assert(
MI.getOpcode() == TargetOpcode::G_GLOBAL_VALUE);
1260 auto &GlobalOp =
MI.getOperand(1);
1261 const auto* GV = GlobalOp.getGlobal();
1262 if (GV->isThreadLocal())
1271 auto Offset = GlobalOp.getOffset();
1276 MRI.setRegClass(
ADRP.getReg(0), &AArch64::GPR64RegClass);
1293 "Should not have folded in an offset for a tagged global!");
1295 .addGlobalAddress(GV, 0x100000000,
1298 MRI.setRegClass(
ADRP.getReg(0), &AArch64::GPR64RegClass);
1302 .addGlobalAddress(GV,
Offset,
1304 MI.eraseFromParent();
1311 switch (IntrinsicID) {
1312 case Intrinsic::vacopy: {
1314 unsigned VaListSize =
1326 VaListSize,
Align(PtrSize)));
1330 VaListSize,
Align(PtrSize)));
1331 MI.eraseFromParent();
1334 case Intrinsic::get_dynamic_area_offset: {
1337 MI.eraseFromParent();
1340 case Intrinsic::aarch64_mops_memset_tag: {
1341 assert(
MI.getOpcode() == TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS);
1345 auto &
Value =
MI.getOperand(3);
1347 Value.setReg(ExtValueReg);
1350 case Intrinsic::prefetch: {
1352 auto &AddrVal =
MI.getOperand(1);
1354 int64_t IsWrite =
MI.getOperand(2).getImm();
1355 int64_t Locality =
MI.getOperand(3).getImm();
1356 int64_t IsData =
MI.getOperand(4).getImm();
1358 bool IsStream = Locality == 0;
1359 if (Locality != 0) {
1360 assert(Locality <= 3 &&
"Prefetch locality out-of-range");
1364 Locality = 3 - Locality;
1368 (IsWrite << 4) | (!IsData << 3) | (Locality << 1) | IsStream;
1371 MI.eraseFromParent();
1374 case Intrinsic::aarch64_prefetch: {
1376 auto &AddrVal =
MI.getOperand(1);
1378 int64_t IsWrite =
MI.getOperand(2).getImm();
1379 int64_t
Target =
MI.getOperand(3).getImm();
1380 int64_t IsStream =
MI.getOperand(4).getImm();
1381 int64_t IsData =
MI.getOperand(5).getImm();
1383 unsigned PrfOp = (IsWrite << 4) |
1389 MI.eraseFromParent();
1392 case Intrinsic::aarch64_neon_uaddv:
1393 case Intrinsic::aarch64_neon_saddv:
1394 case Intrinsic::aarch64_neon_umaxv:
1395 case Intrinsic::aarch64_neon_smaxv:
1396 case Intrinsic::aarch64_neon_uminv:
1397 case Intrinsic::aarch64_neon_sminv: {
1400 bool IsSigned = IntrinsicID == Intrinsic::aarch64_neon_saddv ||
1401 IntrinsicID == Intrinsic::aarch64_neon_smaxv ||
1402 IntrinsicID == Intrinsic::aarch64_neon_sminv;
1404 auto OldDst =
MI.getOperand(0).getReg();
1405 auto OldDstTy =
MRI.getType(OldDst);
1406 LLT NewDstTy =
MRI.getType(
MI.getOperand(2).getReg()).getElementType();
1407 if (OldDstTy == NewDstTy)
1410 auto NewDst =
MRI.createGenericVirtualRegister(NewDstTy);
1413 MI.getOperand(0).setReg(NewDst);
1417 MIB.
buildExtOrTrunc(IsSigned ? TargetOpcode::G_SEXT : TargetOpcode::G_ZEXT,
1422 case Intrinsic::aarch64_neon_smax:
1423 case Intrinsic::aarch64_neon_smin:
1424 case Intrinsic::aarch64_neon_umax:
1425 case Intrinsic::aarch64_neon_umin:
1426 case Intrinsic::aarch64_neon_fmax:
1427 case Intrinsic::aarch64_neon_fmin: {
1429 if (IntrinsicID == Intrinsic::aarch64_neon_smax)
1431 else if (IntrinsicID == Intrinsic::aarch64_neon_smin)
1433 else if (IntrinsicID == Intrinsic::aarch64_neon_umax)
1435 else if (IntrinsicID == Intrinsic::aarch64_neon_umin)
1437 else if (IntrinsicID == Intrinsic::aarch64_neon_fmax)
1438 MIB.
buildInstr(TargetOpcode::G_FMAXIMUM, {
MI.getOperand(0)},
1439 {
MI.getOperand(2),
MI.getOperand(3)});
1440 else if (IntrinsicID == Intrinsic::aarch64_neon_fmin)
1441 MIB.
buildInstr(TargetOpcode::G_FMINIMUM, {
MI.getOperand(0)},
1442 {
MI.getOperand(2),
MI.getOperand(3)});
1443 MI.eraseFromParent();
1446 case Intrinsic::experimental_vector_reverse:
1454bool AArch64LegalizerInfo::legalizeShlAshrLshr(
1457 assert(
MI.getOpcode() == TargetOpcode::G_ASHR ||
1458 MI.getOpcode() == TargetOpcode::G_LSHR ||
1459 MI.getOpcode() == TargetOpcode::G_SHL);
1472 MI.getOperand(2).setReg(ExtCst.getReg(0));
1485 isShiftedInt<7, 3>(NewOffset)) {
1493bool AArch64LegalizerInfo::legalizeLoadStore(
1496 assert(
MI.getOpcode() == TargetOpcode::G_STORE ||
1497 MI.getOpcode() == TargetOpcode::G_LOAD);
1508 const LLT ValTy =
MRI.getType(ValReg);
1513 bool IsLoad =
MI.getOpcode() == TargetOpcode::G_LOAD;
1517 ST->hasLSE2() && ST->hasRCPC3() && (IsLoadAcquire || IsStoreRelease);
1523 Opcode = IsLoad ? AArch64::LDIAPPX : AArch64::STILPX;
1529 assert(ST->hasLSE2() &&
"ldp/stp not single copy atomic without +lse2");
1531 Opcode = IsLoad ? AArch64::LDPXi : AArch64::STPXi;
1546 NewI.
addUse(
MI.getOperand(1).getReg());
1557 *
MRI.getTargetRegisterInfo(),
1559 MI.eraseFromParent();
1565 LLVM_DEBUG(
dbgs() <<
"Tried to do custom legalization on wrong load/store");
1571 auto &MMO = **
MI.memoperands_begin();
1574 if (
MI.getOpcode() == TargetOpcode::G_STORE) {
1578 auto NewLoad = MIRBuilder.
buildLoad(NewTy,
MI.getOperand(1), MMO);
1581 MI.eraseFromParent();
1589 Align Alignment(
MI.getOperand(2).getImm());
1591 Register ListPtr =
MI.getOperand(1).getReg();
1593 LLT PtrTy =
MRI.getType(ListPtr);
1604 if (Alignment > PtrAlign) {
1608 auto ListTmp = MIRBuilder.
buildPtrAdd(PtrTy,
List, AlignMinus1.getReg(0));
1613 LLT ValTy =
MRI.getType(Dst);
1618 ValTy, std::max(Alignment, PtrAlign)));
1629 MI.eraseFromParent();
1633bool AArch64LegalizerInfo::legalizeBitfieldExtract(
1667 LLT Ty =
MRI.getType(Val);
1671 "Expected src and dst to have the same type!");
1679 auto Add = MIRBuilder.
buildAdd(s64, CTPOP1, CTPOP2);
1682 MI.eraseFromParent();
1686 if (!ST->hasNEON() ||
1687 MI.getMF()->getFunction().hasFnAttribute(Attribute::NoImplicitFloat)) {
1699 assert((
Size == 32 ||
Size == 64 ||
Size == 128) &&
"Expected only 32, 64, or 128 bit scalars!");
1714 Opc = Intrinsic::aarch64_neon_uaddlv;
1717 Opc = Intrinsic::aarch64_neon_uaddlp;
1720 Opc = Intrinsic::aarch64_neon_uaddlp;
1724 Opc = Intrinsic::aarch64_neon_uaddlp;
1729 Opc = Intrinsic::aarch64_neon_uaddlp;
1732 Opc = Intrinsic::aarch64_neon_uaddlp;
1738 for (
LLT HTy : HAddTys) {
1748 MI.eraseFromParent();
1752bool AArch64LegalizerInfo::legalizeAtomicCmpxchg128(
1756 auto Addr =
MI.getOperand(1).getReg();
1757 auto DesiredI = MIRBuilder.
buildUnmerge({s64, s64},
MI.getOperand(2));
1758 auto NewI = MIRBuilder.
buildUnmerge({s64, s64},
MI.getOperand(3));
1759 auto DstLo =
MRI.createGenericVirtualRegister(s64);
1760 auto DstHi =
MRI.createGenericVirtualRegister(s64);
1773 auto Ordering = (*
MI.memoperands_begin())->getMergedOrdering();
1777 Opcode = AArch64::CASPAX;
1780 Opcode = AArch64::CASPLX;
1784 Opcode = AArch64::CASPALX;
1792 auto CASDst =
MRI.createGenericVirtualRegister(s128);
1793 auto CASDesired =
MRI.createGenericVirtualRegister(s128);
1794 auto CASNew =
MRI.createGenericVirtualRegister(s128);
1795 MIRBuilder.
buildInstr(TargetOpcode::REG_SEQUENCE, {CASDesired}, {})
1796 .addUse(DesiredI->getOperand(0).getReg())
1798 .
addUse(DesiredI->getOperand(1).getReg())
1799 .
addImm(AArch64::subo64);
1800 MIRBuilder.
buildInstr(TargetOpcode::REG_SEQUENCE, {CASNew}, {})
1804 .
addImm(AArch64::subo64);
1814 auto Ordering = (*
MI.memoperands_begin())->getMergedOrdering();
1818 Opcode = AArch64::CMP_SWAP_128_ACQUIRE;
1821 Opcode = AArch64::CMP_SWAP_128_RELEASE;
1825 Opcode = AArch64::CMP_SWAP_128;
1828 Opcode = AArch64::CMP_SWAP_128_MONOTONIC;
1832 auto Scratch =
MRI.createVirtualRegister(&AArch64::GPR64RegClass);
1834 {
Addr, DesiredI->getOperand(0),
1835 DesiredI->getOperand(1), NewI->
getOperand(0),
1841 *
MRI.getTargetRegisterInfo(),
1845 MI.eraseFromParent();
1853 LLT Ty =
MRI.getType(
MI.getOperand(1).getReg());
1855 MIRBuilder.
buildCTLZ(
MI.getOperand(0).getReg(), BitReverse);
1856 MI.eraseFromParent();
1865 if (
MI.getOpcode() == TargetOpcode::G_MEMSET) {
1868 auto &
Value =
MI.getOperand(1);
1871 Value.setReg(ExtValueReg);
1883 LLT DstTy =
MRI.getType(Dst);
1886 assert((DstSize == 32 || DstSize == 64) &&
"Unexpected dst type!");
1887 assert(
MRI.getType(
MI.getOperand(2).getReg()) == DstTy &&
1888 "Expected homogeneous types!");
1907 EltMask = 0x80000000ULL;
1917 VecTy, Undef,
MI.getOperand(1).getReg(), Zero);
1919 VecTy, Undef,
MI.getOperand(2).getReg(), Zero);
1926 auto Sel = MIRBuilder.
buildInstr(AArch64::G_BSP, {VecTy}, {
Mask, Ins2, Ins1});
1932 DstRegs.push_back(
MRI.createGenericVirtualRegister(DstTy));
1934 MI.eraseFromParent();
1938bool AArch64LegalizerInfo::legalizeExtractVectorElt(
1940 assert(
MI.getOpcode() == TargetOpcode::G_EXTRACT_VECTOR_ELT);
unsigned const MachineRegisterInfo * MRI
static void matchLDPSTPAddrMode(Register Root, Register &Base, int &Offset, MachineRegisterInfo &MRI)
This file declares the targeting of the Machinelegalizer class for AArch64.
This file declares the targeting of the RegisterBankInfo class for AArch64.
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
Declares convenience wrapper classes for interpreting MachineInstr instances as specific generic oper...
Interface for Targets to specify which operations they can successfully select and how the others sho...
Contains matchers for matching SSA Machine Instructions.
This file declares the MachineIRBuilder class.
static unsigned getReg(const MCDisassembler *D, unsigned RC, unsigned RegNo)
const char LLVMTargetMachineRef TM
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
static constexpr uint32_t Opcode
bool legalizeCustom(LegalizerHelper &Helper, MachineInstr &MI) const override
Called for instructions with the Custom LegalizationAction.
bool legalizeIntrinsic(LegalizerHelper &Helper, MachineInstr &MI) const override
AArch64LegalizerInfo(const AArch64Subtarget &ST)
bool isTargetWindows() const
const AArch64InstrInfo * getInstrInfo() const override
bool isTargetDarwin() const
bool isTargetILP32() const
const AArch64TargetLowering * getTargetLowering() const override
unsigned ClassifyGlobalReference(const GlobalValue *GV, const TargetMachine &TM) const
ClassifyGlobalReference - Find the target operand flags that describe how a global value should be re...
const RegisterBankInfo * getRegBankInfo() const override
Class for arbitrary precision integers.
APInt zext(unsigned width) const
Zero extend to a new width.
APInt urem(const APInt &RHS) const
Unsigned remainder operation.
int64_t getSExtValue() const
Get sign extended value.
This class represents an Operation in the Expression.
Abstract class that contains various methods for clients to notify about changes.
virtual void changingInstr(MachineInstr &MI)=0
This instruction is about to be mutated in some way.
virtual void changedInstr(MachineInstr &MI)=0
This instruction was mutated in some way.
constexpr unsigned getScalarSizeInBits() const
constexpr bool isScalar() const
static constexpr LLT vector(ElementCount EC, unsigned ScalarSizeInBits)
Get a low-level vector of some number of elements and element width.
static constexpr LLT scalar(unsigned SizeInBits)
Get a low-level scalar or aggregate "bag of bits".
constexpr uint16_t getNumElements() const
Returns the number of elements in a vector LLT.
constexpr bool isVector() const
static constexpr LLT pointer(unsigned AddressSpace, unsigned SizeInBits)
Get a low-level pointer in the given address space.
constexpr TypeSize getSizeInBits() const
Returns the total size of the type. Must only be called on sized types.
constexpr bool isPointer() const
constexpr LLT getElementType() const
Returns the vector's element type. Only valid for vector types.
constexpr ElementCount getElementCount() const
constexpr unsigned getAddressSpace() const
static constexpr LLT fixed_vector(unsigned NumElements, unsigned ScalarSizeInBits)
Get a low-level fixed-width vector of some number of elements and element width.
constexpr LLT divide(int Factor) const
Return a type that is Factor times smaller.
void computeTables()
Compute any ancillary tables needed to quickly decide how an operation should be handled.
LegalizeRuleSet & minScalar(unsigned TypeIdx, const LLT Ty)
Ensure the scalar is at least as wide as Ty.
LegalizeRuleSet & legalFor(std::initializer_list< LLT > Types)
The instruction is legal when type index 0 is any type in the given list.
LegalizeRuleSet & unsupported()
The instruction is unsupported.
LegalizeRuleSet & scalarSameSizeAs(unsigned TypeIdx, unsigned SameSizeIdx)
Change the type TypeIdx to have the same scalar size as type SameSizeIdx.
LegalizeRuleSet & fewerElementsIf(LegalityPredicate Predicate, LegalizeMutation Mutation)
Remove elements to reach the type selected by the mutation if the predicate is true.
LegalizeRuleSet & clampScalarOrElt(unsigned TypeIdx, const LLT MinTy, const LLT MaxTy)
Limit the range of scalar sizes to MinTy and MaxTy.
LegalizeRuleSet & libcallFor(std::initializer_list< LLT > Types)
LegalizeRuleSet & maxScalar(unsigned TypeIdx, const LLT Ty)
Ensure the scalar is at most as wide as Ty.
LegalizeRuleSet & minScalarOrElt(unsigned TypeIdx, const LLT Ty)
Ensure the scalar or element is at least as wide as Ty.
LegalizeRuleSet & clampMaxNumElements(unsigned TypeIdx, const LLT EltTy, unsigned MaxElements)
Limit the number of elements in EltTy vectors to at most MaxElements.
LegalizeRuleSet & widenVectorEltsToVectorMinSize(unsigned TypeIdx, unsigned VectorSize)
Ensure the vector size is at least as wide as VectorSize by promoting the element.
LegalizeRuleSet & lowerIfMemSizeNotPow2()
Lower a memory operation if the memory size, rounded to bytes, is not a power of 2.
LegalizeRuleSet & minScalarEltSameAsIf(LegalityPredicate Predicate, unsigned TypeIdx, unsigned LargeTypeIdx)
Conditionally widen the scalar or elt to match the size of another.
LegalizeRuleSet & customForCartesianProduct(std::initializer_list< LLT > Types)
LegalizeRuleSet & lowerIfMemSizeNotByteSizePow2()
Lower a memory operation if the memory access size is not a round power of 2 byte size.
LegalizeRuleSet & moreElementsToNextPow2(unsigned TypeIdx)
Add more elements to the vector to reach the next power of two.
LegalizeRuleSet & narrowScalarIf(LegalityPredicate Predicate, LegalizeMutation Mutation)
Narrow the scalar to the one selected by the mutation if the predicate is true.
LegalizeRuleSet & lower()
The instruction is lowered.
LegalizeRuleSet & scalarizeIf(LegalityPredicate Predicate, unsigned TypeIdx)
LegalizeRuleSet & lowerIf(LegalityPredicate Predicate)
The instruction is lowered if predicate is true.
LegalizeRuleSet & clampScalar(unsigned TypeIdx, const LLT MinTy, const LLT MaxTy)
Limit the range of scalar sizes to MinTy and MaxTy.
LegalizeRuleSet & custom()
Unconditionally custom lower.
LegalizeRuleSet & minScalarSameAs(unsigned TypeIdx, unsigned LargeTypeIdx)
Widen the scalar to match the size of another.
LegalizeRuleSet & unsupportedIf(LegalityPredicate Predicate)
LegalizeRuleSet & minScalarOrEltIf(LegalityPredicate Predicate, unsigned TypeIdx, const LLT Ty)
Ensure the scalar or element is at least as wide as Ty.
LegalizeRuleSet & widenScalarIf(LegalityPredicate Predicate, LegalizeMutation Mutation)
Widen the scalar to the one selected by the mutation if the predicate is true.
LegalizeRuleSet & clampNumElements(unsigned TypeIdx, const LLT MinTy, const LLT MaxTy)
Limit the number of elements for the given vectors to at least MinTy's number of elements and at most...
LegalizeRuleSet & maxScalarIf(LegalityPredicate Predicate, unsigned TypeIdx, const LLT Ty)
Conditionally limit the maximum size of the scalar.
LegalizeRuleSet & customIf(LegalityPredicate Predicate)
LegalizeRuleSet & widenScalarToNextPow2(unsigned TypeIdx, unsigned MinSize=0)
Widen the scalar to the next power of two that is at least MinSize.
LegalizeRuleSet & scalarize(unsigned TypeIdx)
LegalizeRuleSet & legalForCartesianProduct(std::initializer_list< LLT > Types)
The instruction is legal when type indexes 0 and 1 are both in the given list.
LegalizeRuleSet & legalForTypesWithMemDesc(std::initializer_list< LegalityPredicates::TypePairAndMemDesc > TypesAndMemDesc)
The instruction is legal when type indexes 0 and 1 along with the memory size and minimum alignment i...
LegalizeRuleSet & legalIf(LegalityPredicate Predicate)
The instruction is legal if predicate is true.
LegalizeRuleSet & customFor(std::initializer_list< LLT > Types)
LegalizeResult lowerBitCount(MachineInstr &MI)
LegalizeResult lowerExtractInsertVectorElt(MachineInstr &MI)
Lower a vector extract or insert by writing the vector to a stack temporary and reloading the element...
LegalizeResult lowerFunnelShiftAsShifts(MachineInstr &MI)
@ Legalized
Instruction has been legalized and the MachineFunction changed.
@ UnableToLegalize
Some kind of error has occurred and we could not legalize this instruction.
GISelChangeObserver & Observer
To keep track of changes made by the LegalizerHelper.
MachineIRBuilder & MIRBuilder
Expose MIRBuilder so clients can set their own RecordInsertInstruction functions.
LegalizeRuleSet & getActionDefinitionsBuilder(unsigned Opcode)
Get the action definition builder for the given opcode.
const LegacyLegalizerInfo & getLegacyLegalizerInfo() const
MachineMemOperand * getMachineMemOperand(MachinePointerInfo PtrInfo, MachineMemOperand::Flags f, uint64_t s, Align base_alignment, const AAMDNodes &AAInfo=AAMDNodes(), const MDNode *Ranges=nullptr, SyncScope::ID SSID=SyncScope::System, AtomicOrdering Ordering=AtomicOrdering::NotAtomic, AtomicOrdering FailureOrdering=AtomicOrdering::NotAtomic)
getMachineMemOperand - Allocate a new MachineMemOperand.
MachineRegisterInfo & getRegInfo()
getRegInfo - Return information about the registers currently in use.
Helper class to build MachineInstr.
void setInsertPt(MachineBasicBlock &MBB, MachineBasicBlock::iterator II)
Set the insertion point before the specified position.
MachineInstrBuilder buildAdd(const DstOp &Dst, const SrcOp &Src0, const SrcOp &Src1, std::optional< unsigned > Flags=std::nullopt)
Build and insert Res = G_ADD Op0, Op1.
MachineInstrBuilder buildUndef(const DstOp &Res)
Build and insert Res = IMPLICIT_DEF.
MachineInstrBuilder buildUnmerge(ArrayRef< LLT > Res, const SrcOp &Op)
Build and insert Res0, ... = G_UNMERGE_VALUES Op.
MachineInstrBuilder buildExtract(const DstOp &Res, const SrcOp &Src, uint64_t Index)
Build and insert Res0, ... = G_EXTRACT Src, Idx0.
MachineBasicBlock::iterator getInsertPt()
Current insertion point for new instructions.
MachineInstrBuilder buildIntrinsic(Intrinsic::ID ID, ArrayRef< Register > Res, bool HasSideEffects, bool isConvergent)
Build and insert a G_INTRINSIC instruction.
MachineInstrBuilder buildCTLZ(const DstOp &Dst, const SrcOp &Src0)
Build and insert Res = G_CTLZ Op0, Src0.
MachineInstrBuilder buildSMax(const DstOp &Dst, const SrcOp &Src0, const SrcOp &Src1)
Build and insert Res = G_SMAX Op0, Op1.
MachineInstrBuilder buildMergeLikeInstr(const DstOp &Res, ArrayRef< Register > Ops)
Build and insert Res = G_MERGE_VALUES Op0, ... or Res = G_BUILD_VECTOR Op0, ... or Res = G_CONCAT_VEC...
MachineInstrBuilder buildLoad(const DstOp &Res, const SrcOp &Addr, MachineMemOperand &MMO)
Build and insert Res = G_LOAD Addr, MMO.
MachineInstrBuilder buildPtrAdd(const DstOp &Res, const SrcOp &Op0, const SrcOp &Op1, std::optional< unsigned > Flags=std::nullopt)
Build and insert Res = G_PTR_ADD Op0, Op1.
MachineInstrBuilder buildBitReverse(const DstOp &Dst, const SrcOp &Src)
Build and insert Dst = G_BITREVERSE Src.
MachineInstrBuilder buildStore(const SrcOp &Val, const SrcOp &Addr, MachineMemOperand &MMO)
Build and insert G_STORE Val, Addr, MMO.
MachineInstrBuilder buildInstr(unsigned Opcode)
Build and insert <empty> = Opcode <empty>.
MachineInstrBuilder buildCTPOP(const DstOp &Dst, const SrcOp &Src0)
Build and insert Res = G_CTPOP Op0, Src0.
MachineInstrBuilder buildZExt(const DstOp &Res, const SrcOp &Op)
Build and insert Res = G_ZEXT Op.
MachineFunction & getMF()
Getter for the function we currently build.
MachineInstrBuilder buildSMin(const DstOp &Dst, const SrcOp &Src0, const SrcOp &Src1)
Build and insert Res = G_SMIN Op0, Op1.
MachineInstrBuilder buildExtOrTrunc(unsigned ExtOpc, const DstOp &Res, const SrcOp &Op)
Build and insert Res = ExtOpc, Res = G_TRUNC Op, or Res = COPY Op depending on the differing sizes of...
const MachineBasicBlock & getMBB() const
Getter for the basic block we currently build.
MachineInstrBuilder buildFNeg(const DstOp &Dst, const SrcOp &Src0, std::optional< unsigned > Flags=std::nullopt)
Build and insert Res = G_FNEG Op0.
MachineInstrBuilder buildInsertVectorElement(const DstOp &Res, const SrcOp &Val, const SrcOp &Elt, const SrcOp &Idx)
Build and insert Res = G_INSERT_VECTOR_ELT Val, Elt, Idx.
MachineInstrBuilder buildAnyExt(const DstOp &Res, const SrcOp &Op)
Build and insert Res = G_ANYEXT Op0.
MachineInstrBuilder buildBitcast(const DstOp &Dst, const SrcOp &Src)
Build and insert Dst = G_BITCAST Src.
MachineRegisterInfo * getMRI()
Getter for MRI.
MachineInstrBuilder buildMaskLowPtrBits(const DstOp &Res, const SrcOp &Op0, uint32_t NumBits)
Build and insert Res = G_PTRMASK Op0, G_CONSTANT (1 << NumBits) - 1.
virtual MachineInstrBuilder buildConstant(const DstOp &Res, const ConstantInt &Val)
Build and insert Res = G_CONSTANT Val.
MachineInstrBuilder buildUMin(const DstOp &Dst, const SrcOp &Src0, const SrcOp &Src1)
Build and insert Res = G_UMIN Op0, Op1.
MachineInstrBuilder buildUMax(const DstOp &Dst, const SrcOp &Src0, const SrcOp &Src1)
Build and insert Res = G_UMAX Op0, Op1.
Register getReg(unsigned Idx) const
Get the register for the operand index.
const MachineInstrBuilder & addImm(int64_t Val) const
Add a new immediate operand.
const MachineInstrBuilder & add(const MachineOperand &MO) const
const MachineInstrBuilder & cloneMemRefs(const MachineInstr &OtherMI) const
const MachineInstrBuilder & addUse(Register RegNo, unsigned Flags=0, unsigned SubReg=0) const
Add a virtual register use operand.
Representation of each machine instruction.
const MachineOperand & getOperand(unsigned i) const
@ MOLoad
The memory access reads data.
@ MOStore
The memory access writes data.
void setReg(Register Reg)
Change the register this operand corresponds to.
Register getReg() const
getReg - Returns the register number.
MachineRegisterInfo - Keep track of information for virtual and physical registers,...
Register createGenericVirtualRegister(LLT Ty, StringRef Name="")
Create and return a new generic virtual register with low-level type Ty.
Wrapper class representing virtual and physical registers.
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
const TargetMachine & getTargetMachine() const
Primary interface to the complete machine description for the target machine.
Target - Wrapper for Target specific information.
LLVM Value Representation.
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
@ MO_NC
MO_NC - Indicates whether the linker is expected to check the symbol reference for overflow.
@ MO_PAGEOFF
MO_PAGEOFF - A symbol operand with this flag represents the offset of that symbol within a 4K page.
@ MO_GOT
MO_GOT - This flag indicates that a symbol operand represents the address of the GOT entry for the sy...
@ MO_PREL
MO_PREL - Indicates that the bits of the symbol operand represented by MO_G0 etc are PC relative.
@ MO_PAGE
MO_PAGE - A symbol operand with this flag represents the pc-relative offset of the 4K page containing...
@ MO_TAGGED
MO_TAGGED - With MO_PAGE, indicates that the page includes a memory tag in bits 56-63.
@ MO_G3
MO_G3 - A symbol operand with this flag (granule 3) represents the high 16-bits of a 64-bit address,...
constexpr std::underlying_type_t< E > Mask()
Get a bitmask with 1s in all places up to the high-order bit of E's largest value.
LegalityPredicate isScalar(unsigned TypeIdx)
True iff the specified type index is a scalar.
LegalityPredicate typeInSet(unsigned TypeIdx, std::initializer_list< LLT > TypesInit)
True iff the given type index is one of the specified types.
LegalityPredicate smallerThan(unsigned TypeIdx0, unsigned TypeIdx1)
True iff the first type index has a smaller total bit size than second type index.
LegalityPredicate atomicOrderingAtLeastOrStrongerThan(unsigned MMOIdx, AtomicOrdering Ordering)
True iff the specified MMO index has at an atomic ordering of at Ordering or stronger.
LegalityPredicate isVector(unsigned TypeIdx)
True iff the specified type index is a vector.
Predicate all(Predicate P0, Predicate P1)
True iff P0 and P1 are true.
LegalityPredicate typeIs(unsigned TypeIdx, LLT TypesInit)
True iff the given type index is the specified type.
@ Bitcast
Perform the operation on a different, but equivalently sized type.
LegalizeMutation moreElementsToNextPow2(unsigned TypeIdx, unsigned Min=0)
Add more elements to the type for the given type index to the next power of.
LegalizeMutation scalarize(unsigned TypeIdx)
Break up the vector type for the given type index into the element type.
LegalizeMutation widenScalarOrEltToNextPow2(unsigned TypeIdx, unsigned Min=0)
Widen the scalar type or vector element type for the given type index to the next power of 2.
LegalizeMutation changeTo(unsigned TypeIdx, LLT Ty)
Select this specific type for the given type index.
LegalizeMutation changeElementSizeTo(unsigned TypeIdx, unsigned FromTypeIdx)
Change the scalar size or element size to have the same scalar size as type index FromIndex.
operand_type_match m_Reg()
ConstantMatch< APInt > m_ICst(APInt &Cst)
bool mi_match(Reg R, const MachineRegisterInfo &MRI, Pattern &&P)
BinaryOp_match< LHS, RHS, TargetOpcode::G_PTR_ADD, false > m_GPtrAdd(const LHS &L, const RHS &R)
@ Undef
Value of the register doesn't matter.
This is an optimization pass for GlobalISel generic memory operations.
auto find(R &&Range, const T &Val)
Provide wrappers to std::find which take ranges instead of having to pass begin/end explicitly.
bool constrainSelectedInstRegOperands(MachineInstr &I, const TargetInstrInfo &TII, const TargetRegisterInfo &TRI, const RegisterBankInfo &RBI)
Mutate the newly-selected instruction I to constrain its (possibly generic) virtual register operands...
constexpr bool isPowerOf2_32(uint32_t Value)
Return true if the argument is a power of two > 0.
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
AtomicOrdering
Atomic ordering for LLVM's memory model.
uint64_t alignTo(uint64_t Size, Align A)
Returns a multiple of A needed to store Size bytes.
constexpr unsigned BitWidth
std::optional< ValueAndVReg > getIConstantVRegValWithLookThrough(Register VReg, const MachineRegisterInfo &MRI, bool LookThroughInstrs=true)
If VReg is defined by a statically evaluable chain of instructions rooted on a G_CONSTANT returns its...
bool is_contained(R &&Range, const E &Element)
Returns true if Element is found in Range.
unsigned Log2(Align A)
Returns the log2 of the alignment.
std::function< bool(const LegalityQuery &)> LegalityPredicate
This struct is a compact representation of a valid (non-zero power of two) alignment.
The LegalityQuery object bundles together all the information that's needed to decide whether a given...
ArrayRef< MemDesc > MMODescrs
Operations which require memory can use this to place requirements on the memory type for each MMO.
This class contains a discriminated union of information about pointers in memory operands,...