30#include "llvm/IR/IntrinsicsAArch64.h"
33#include <initializer_list>
35#define DEBUG_TYPE "aarch64-legalinfo"
38using namespace LegalizeActions;
39using namespace LegalizeMutations;
40using namespace LegalityPredicates;
41using namespace MIPatternMatch;
45 using namespace TargetOpcode;
64 std::initializer_list<LLT> PackedVectorAllTypeList = {
70 std::initializer_list<LLT> ScalarAndPtrTypesList = {s8, s16, s32, s64, p0};
77 if (!ST.hasNEON() || !ST.hasFPARMv8()) {
84 const bool HasFP16 = ST.hasFullFP16();
85 const LLT &MinFPScalar = HasFP16 ? s16 : s32;
87 const bool HasCSSC = ST.hasCSSC();
88 const bool HasRCPC3 = ST.hasRCPC3();
91 {G_IMPLICIT_DEF, G_FREEZE, G_CONSTANT_FOLD_BARRIER})
92 .legalFor({p0, s8, s16, s32, s64})
93 .legalFor(PackedVectorAllTypeList)
105 .legalFor(PackedVectorAllTypeList)
116 .
legalFor({s32, s64, v4s16, v8s16, v2s32, v4s32, v2s64})
118 .clampScalar(0, s32, s64)
119 .clampNumElements(0, v4s16, v8s16)
120 .clampNumElements(0, v2s32, v4s32)
121 .clampNumElements(0, v2s64, v2s64)
122 .moreElementsToNextPow2(0);
125 .legalFor({s32, s64, v2s32, v2s64, v4s32, v4s16, v8s16, v16s8, v8s8})
126 .widenScalarToNextPow2(0)
134 return Query.
Types[0].getNumElements() <= 2;
139 return Query.
Types[0].getNumElements() <= 4;
144 return Query.
Types[0].getNumElements() <= 16;
151 const auto &SrcTy = Query.
Types[0];
152 const auto &AmtTy = Query.
Types[1];
153 return !SrcTy.isVector() && SrcTy.getSizeInBits() == 32 &&
154 AmtTy.getSizeInBits() == 32;
168 .widenScalarToNextPow2(0)
179 .
legalFor({{p0, s64}, {v2p0, v2s64}})
180 .clampScalar(1, s64, s64);
185 .legalFor({s32, s64})
187 .clampScalar(0, s32, s64)
192 .lowerFor({s8, s16, s32, s64, v2s64, v4s32, v2s32})
194 .clampScalarOrElt(0, s32, s64)
195 .clampNumElements(0, v2s32, v4s32)
196 .clampNumElements(0, v2s64, v2s64)
197 .moreElementsToNextPow2(0);
201 .widenScalarToNextPow2(0, 32)
206 .legalFor({s64, v8s16, v16s8, v4s32})
210 {G_SMIN, G_SMAX, G_UMIN, G_UMAX});
213 .
legalFor({s32, s64, v8s8, v16s8, v4s16, v8s16, v2s32, v4s32})
220 .
legalFor({v8s8, v16s8, v4s16, v8s16, v2s32, v4s32});
231 {G_SADDE, G_SSUBE, G_UADDE, G_USUBE, G_SADDO, G_SSUBO, G_UADDO, G_USUBO})
232 .legalFor({{s32, s32}, {s64, s32}})
233 .clampScalar(0, s32, s64)
238 G_FABS, G_FSQRT, G_FMAXNUM, G_FMINNUM,
239 G_FMAXIMUM, G_FMINIMUM, G_FCEIL, G_FFLOOR,
240 G_FRINT, G_FNEARBYINT, G_INTRINSIC_TRUNC,
241 G_INTRINSIC_ROUND, G_INTRINSIC_ROUNDEVEN})
242 .legalFor({MinFPScalar, s32, s64, v2s32, v4s32, v2s64})
244 const auto &Ty = Query.
Types[0];
245 return (Ty == v8s16 || Ty == v4s16) && HasFP16;
248 .minScalarOrElt(0, MinFPScalar)
260 .legalFor({{s64, MinFPScalar}, {s64, s32}, {s64, s64}})
261 .libcallFor({{s64, s128}})
262 .minScalarOrElt(1, MinFPScalar);
265 {G_FCOS, G_FSIN, G_FPOW, G_FLOG, G_FLOG2, G_FLOG10,
266 G_FEXP, G_FEXP2, G_FEXP10})
271 .libcallFor({s32, s64});
299 for (
unsigned Op : {G_SEXTLOAD, G_ZEXTLOAD}) {
302 if (
Op == G_SEXTLOAD)
307 .legalForTypesWithMemDesc({{s32, p0, s8, 8},
315 {v2s32, p0, s64, 8}})
316 .widenScalarToNextPow2(0)
317 .clampScalar(0, s32, s64)
320 .unsupportedIfMemSizeNotPow2()
332 return HasRCPC3 && Query.
Types[0] == s128 &&
336 return Query.
Types[0] == s128 &&
339 .legalForTypesWithMemDesc({{s8, p0, s8, 8},
346 {v16s8, p0, s128, 8},
348 {v8s16, p0, s128, 8},
350 {v4s32, p0, s128, 8},
351 {v2s64, p0, s128, 8}})
353 .legalForTypesWithMemDesc(
354 {{s32, p0, s8, 8}, {s32, p0, s16, 8}, {s64, p0, s32, 8}})
355 .widenScalarToNextPow2(0, 8)
366 return Query.
Types[0].isScalar() &&
368 Query.
Types[0].getSizeInBits() > 32;
377 .customIf(IsPtrVecPred)
382 return HasRCPC3 && Query.
Types[0] == s128 &&
386 return Query.
Types[0] == s128 &&
389 .legalForTypesWithMemDesc(
390 {{s8, p0, s8, 8}, {s16, p0, s8, 8},
393 {s16, p0, s16, 8}, {s32, p0, s16, 8},
395 {s32, p0, s8, 8}, {s32, p0, s16, 8}, {s32, p0, s32, 8},
396 {s64, p0, s64, 8}, {s64, p0, s32, 8},
397 {p0, p0, s64, 8}, {s128, p0, s128, 8}, {v16s8, p0, s128, 8},
398 {v8s8, p0, s64, 8}, {v4s16, p0, s64, 8}, {v8s16, p0, s128, 8},
399 {v2s32, p0, s64, 8}, {v4s32, p0, s128, 8}, {v2s64, p0, s128, 8}})
400 .clampScalar(0, s8, s64)
402 return Query.
Types[0].isScalar() &&
406 .clampMaxNumElements(0, s8, 16)
418 .customIf(IsPtrVecPred)
434 {p0, v16s8, v16s8, 8},
435 {p0, v4s16, v4s16, 8},
436 {p0, v8s16, v8s16, 8},
437 {p0, v2s32, v2s32, 8},
438 {p0, v4s32, v4s32, 8},
439 {p0, v2s64, v2s64, 8},
445 auto IndexedLoadBasicPred = [=](
const LegalityQuery &Query) {
473 return MemTy == s8 || MemTy == s16;
475 return MemTy == s8 || MemTy == s16 || MemTy == s32;
483 .widenScalarToNextPow2(0)
487 const auto &Ty = Query.
Types[0];
488 if (HasFP16 && Ty == s16)
490 return Ty == s32 || Ty == s64 || Ty == s128;
492 .clampScalar(0, MinFPScalar, s128);
508 .clampScalar(1, s32, s64)
509 .clampScalar(0, s32, s32)
510 .minScalarEltSameAsIf(
525 .clampNumElements(1, v8s8, v16s8)
526 .clampNumElements(1, v4s16, v8s16)
527 .clampNumElements(1, v2s32, v4s32)
528 .clampNumElements(1, v2s64, v2s64);
538 const auto &Ty = Query.
Types[1];
539 return (Ty == v8s16 || Ty == v4s16) && Ty == Query.
Types[0] && HasFP16;
542 .clampScalar(0, s32, s32)
543 .clampScalarOrElt(1, MinFPScalar, s64)
544 .minScalarEltSameAsIf(
552 .clampNumElements(1, v4s16, v8s16)
553 .clampNumElements(1, v2s32, v4s32)
554 .clampMaxNumElements(1, s64, 2)
555 .moreElementsToNextPow2(1);
559 unsigned DstSize = Query.
Types[0].getSizeInBits();
562 if (Query.
Types[0].isVector())
565 if (DstSize < 8 || DstSize >= 128 || !
isPowerOf2_32(DstSize))
580 .legalIf(ExtLegalFunc)
581 .
legalFor({{v2s64, v2s32}, {v4s32, v4s16}, {v8s16, v8s8}})
582 .clampScalar(0, s64, s64)
589 return (Query.
Types[0].getScalarSizeInBits() >
590 Query.
Types[1].getScalarSizeInBits() * 2) &&
591 Query.
Types[0].isVector() &&
592 (Query.
Types[1].getScalarSizeInBits() == 8 ||
593 Query.
Types[1].getScalarSizeInBits() == 16);
595 .clampMinNumElements(1, s8, 8)
599 .
legalFor({{v2s32, v2s64}, {v4s16, v4s32}, {v8s8, v8s16}})
601 .clampMaxNumElements(0, s8, 8)
602 .clampMaxNumElements(0, s16, 4)
603 .clampMaxNumElements(0, s32, 2)
613 .clampMinNumElements(0, s8, 8)
614 .clampMinNumElements(0, s16, 4)
619 .legalFor(PackedVectorAllTypeList)
630 {{s16, s32}, {s16, s64}, {s32, s64}, {v4s16, v4s32}, {v2s32, v2s64}})
631 .clampNumElements(0, v4s16, v4s16)
637 {{s32, s16}, {s64, s16}, {s64, s32}, {v4s32, v4s16}, {v2s64, v2s32}})
638 .clampNumElements(0, v4s32, v4s32)
644 .legalForCartesianProduct({s32, s64, v2s64, v4s32, v2s32})
647 (Query.
Types[1] == s16 || Query.
Types[1] == v4s16 ||
648 Query.
Types[1] == v8s16) &&
649 (Query.
Types[0] == s32 || Query.
Types[0] == s64 ||
650 Query.
Types[0] == v4s16 || Query.
Types[0] == v8s16);
652 .widenScalarToNextPow2(0)
659 return Query.
Types[0].getScalarSizeInBits() >
660 Query.
Types[1].getScalarSizeInBits();
665 return Query.
Types[0].getScalarSizeInBits() <
666 Query.
Types[1].getScalarSizeInBits();
669 .clampNumElements(0, v4s16, v8s16)
674 .legalForCartesianProduct({s32, s64, v2s64, v4s32, v2s32})
677 (Query.
Types[0] == s16 || Query.
Types[0] == v4s16 ||
678 Query.
Types[0] == v8s16) &&
679 (Query.
Types[1] == s32 || Query.
Types[1] == s64 ||
680 Query.
Types[1] == v4s16 || Query.
Types[1] == v8s16);
682 .widenScalarToNextPow2(1)
689 return Query.
Types[0].getScalarSizeInBits() <
690 Query.
Types[1].getScalarSizeInBits();
695 return Query.
Types[0].getScalarSizeInBits() >
696 Query.
Types[1].getScalarSizeInBits();
699 .clampNumElements(0, v4s16, v8s16)
706 .clampScalar(0, s32, s32);
710 .
legalFor({{s32, s32}, {s64, s32}, {p0, s32}})
711 .widenScalarToNextPow2(0)
726 .
legalFor({{s64, p0}, {v2s64, v2p0}})
727 .widenScalarToNextPow2(0, 64)
732 return Query.
Types[0].getSizeInBits() != Query.
Types[1].getSizeInBits();
734 .legalFor({{p0, s64}, {v2p0, v2s64}});
741 .legalForCartesianProduct({s64, v8s8, v4s16, v2s32})
742 .legalForCartesianProduct({s128, v16s8, v8s16, v4s32, v2s64, v2p0})
744 return Query.
Types[0].isVector() != Query.
Types[1].isVector();
747 .clampNumElements(0, v8s8, v16s8)
748 .clampNumElements(0, v4s16, v8s16)
749 .clampNumElements(0, v2s32, v4s32)
758 .clampScalar(0, s8, s64)
766 return ST.outlineAtomics() && !ST.hasLSE();
774 return Query.
Types[0].getSizeInBits() == 128 &&
775 !UseOutlineAtomics(Query);
782 G_ATOMICRMW_SUB, G_ATOMICRMW_AND, G_ATOMICRMW_OR,
793 {G_ATOMICRMW_MIN, G_ATOMICRMW_MAX, G_ATOMICRMW_UMIN, G_ATOMICRMW_UMAX})
800 for (
unsigned Op : {G_MERGE_VALUES, G_UNMERGE_VALUES}) {
801 unsigned BigTyIdx =
Op == G_MERGE_VALUES ? 0 : 1;
802 unsigned LitTyIdx =
Op == G_MERGE_VALUES ? 1 : 0;
809 switch (Q.
Types[BigTyIdx].getSizeInBits()) {
817 switch (Q.
Types[LitTyIdx].getSizeInBits()) {
831 const LLT &EltTy = Query.
Types[1].getElementType();
832 return Query.
Types[0] != EltTy;
837 return VecTy == v2s16 || VecTy == v4s16 || VecTy == v8s16 ||
838 VecTy == v4s32 || VecTy == v2s64 || VecTy == v2s32 ||
839 VecTy == v8s8 || VecTy == v16s8 || VecTy == v2p0;
845 return Query.
Types[1].getNumElements() <= 2;
850 return Query.
Types[1].getNumElements() <= 4;
855 return Query.
Types[1].getNumElements() <= 8;
860 return Query.
Types[1].getNumElements() <= 16;
863 .minScalarOrElt(0, s8)
873 typeInSet(0, {v16s8, v8s8, v8s16, v4s16, v4s32, v2s32, v2s64, v2p0}))
891 .clampNumElements(0, v4s32, v4s32)
901 {s32, s64, v8s8, v16s8, v4s16, v8s16, v2s32, v4s32})
903 .widenScalarToNextPow2(1, 32)
904 .clampScalar(1, s32, s64)
905 .scalarSameSizeAs(0, 1);
911 .widenScalarToNextPow2(0, 32)
922 return (HasCSSC &&
typeInSet(0, {s32, s64})(Query));
925 return (!HasCSSC &&
typeInSet(0, {s32, s64})(Query));
937 {v2s64, v2p0, v2s32, v4s32, v4s16, v16s8, v8s8, v8s16}, DstTy);
942 return !Query.
Types[1].isVector();
946 return Query.
Types[0].isVector() && Query.
Types[1].isVector() &&
947 Query.
Types[0].getNumElements() >
948 Query.
Types[1].getNumElements();
954 return Query.
Types[0].isVector() && Query.
Types[1].isVector() &&
955 Query.
Types[0].getNumElements() <
956 Query.
Types[1].getNumElements();
959 .widenScalarOrEltToNextPow2OrMinSize(0, 8)
960 .clampNumElements(0, v8s8, v16s8)
961 .clampNumElements(0, v4s16, v8s16)
962 .clampNumElements(0, v4s32, v4s32)
963 .clampNumElements(0, v2s64, v2s64);
966 .
legalFor({{v4s32, v2s32}, {v8s16, v4s16}, {v16s8, v8s8}});
983 .customForCartesianProduct({p0}, {s8}, {s64})
987 .legalForCartesianProduct({p0}, {p0}, {s64})
1003 .legalFor({s32, s64});
1004 ABSActions.legalFor(PackedVectorAllTypeList)
1012 [=](
const LegalityQuery &Query) {
return std::make_pair(0, v4s16); })
1015 [=](
const LegalityQuery &Query) {
return std::make_pair(0, v2s32); })
1016 .clampNumElements(0, v8s8, v16s8)
1017 .clampNumElements(0, v4s16, v8s16)
1018 .clampNumElements(0, v2s32, v4s32)
1019 .clampNumElements(0, v2s64, v2s64)
1020 .moreElementsToNextPow2(0)
1027 .
legalFor({{s32, v2s32}, {s32, v4s32}, {s64, v2s64}})
1029 const auto &Ty = Query.
Types[1];
1030 return (Ty == v4s16 || Ty == v8s16) && HasFP16;
1032 .minScalarOrElt(0, MinFPScalar)
1063 .clampMaxNumElements(1, s64, 2)
1070 G_VECREDUCE_FMINIMUM, G_VECREDUCE_FMAXIMUM})
1071 .legalFor({{s32, v4s32}, {s32, v2s32}, {s64, v2s64}})
1073 const auto &Ty = Query.
Types[1];
1074 return Query.
Types[0] == s16 && (Ty == v8s16 || Ty == v4s16) && HasFP16;
1076 .minScalarOrElt(0, MinFPScalar)
1090 {G_VECREDUCE_SMIN, G_VECREDUCE_SMAX, G_VECREDUCE_UMIN, G_VECREDUCE_UMAX})
1091 .legalFor({{s8, v8s8},
1099 return Query.
Types[1].isVector() &&
1100 Query.
Types[1].getElementType() != s8 &&
1101 Query.
Types[1].getNumElements() & 1;
1104 .clampMaxNumElements(1, s64, 2)
1112 {G_VECREDUCE_OR, G_VECREDUCE_AND, G_VECREDUCE_XOR})
1128 return std::make_pair(1, SrcTy.
divide(2));
1134 .customFor({{s32, s32}, {s32, s64}, {s64, s64}})
1138 .
legalFor({{s32, s64}, {s64, s64}})
1140 return Q.
Types[0].isScalar() && Q.
Types[1].getScalarSizeInBits() < 64;
1146 .customFor({{s32, s32}, {s64, s64}});
1152 .legalFor({{s32, s32},
1156 .customFor({{s128, s128},
1164 .legalFor({{v8s8, v8s8},
1166 .customFor({{s32, s32},
1175 .clampScalar(0, s32, s128)
1176 .widenScalarToNextPow2(0)
1177 .minScalarEltSameAsIf(always, 1, 0)
1178 .maxScalarEltSameAsIf(always, 1, 0);
1181 .legalFor({v2s64, v2s32, v4s32, v4s16, v8s16, v8s8, v16s8})
1182 .clampNumElements(0, v8s8, v16s8)
1192 .legalFor({{s64, s32}, {s64, s64}});
1208 G_GET_FPMODE, G_SET_FPMODE, G_RESET_FPMODE})
1216 verify(*ST.getInstrInfo());
1225 switch (
MI.getOpcode()) {
1229 case TargetOpcode::G_VAARG:
1230 return legalizeVaArg(
MI,
MRI, MIRBuilder);
1231 case TargetOpcode::G_LOAD:
1232 case TargetOpcode::G_STORE:
1233 return legalizeLoadStore(
MI,
MRI, MIRBuilder, Observer);
1234 case TargetOpcode::G_SHL:
1235 case TargetOpcode::G_ASHR:
1236 case TargetOpcode::G_LSHR:
1237 return legalizeShlAshrLshr(
MI,
MRI, MIRBuilder, Observer);
1238 case TargetOpcode::G_GLOBAL_VALUE:
1239 return legalizeSmallCMGlobalValue(
MI,
MRI, MIRBuilder, Observer);
1240 case TargetOpcode::G_SBFX:
1241 case TargetOpcode::G_UBFX:
1242 return legalizeBitfieldExtract(
MI,
MRI, Helper);
1243 case TargetOpcode::G_FSHL:
1244 case TargetOpcode::G_FSHR:
1245 return legalizeFunnelShift(
MI,
MRI, MIRBuilder, Observer, Helper);
1246 case TargetOpcode::G_ROTR:
1247 return legalizeRotate(
MI,
MRI, Helper);
1248 case TargetOpcode::G_CTPOP:
1249 return legalizeCTPOP(
MI,
MRI, Helper);
1250 case TargetOpcode::G_ATOMIC_CMPXCHG:
1251 return legalizeAtomicCmpxchg128(
MI,
MRI, Helper);
1252 case TargetOpcode::G_CTTZ:
1253 return legalizeCTTZ(
MI, Helper);
1254 case TargetOpcode::G_BZERO:
1255 case TargetOpcode::G_MEMCPY:
1256 case TargetOpcode::G_MEMMOVE:
1257 case TargetOpcode::G_MEMSET:
1258 return legalizeMemOps(
MI, Helper);
1259 case TargetOpcode::G_EXTRACT_VECTOR_ELT:
1260 return legalizeExtractVectorElt(
MI,
MRI, Helper);
1261 case TargetOpcode::G_DYN_STACKALLOC:
1262 return legalizeDynStackAlloc(
MI, Helper);
1263 case TargetOpcode::G_PREFETCH:
1264 return legalizePrefetch(
MI, Helper);
1265 case TargetOpcode::G_ABS:
1277 assert(
MI.getOpcode() == TargetOpcode::G_FSHL ||
1278 MI.getOpcode() == TargetOpcode::G_FSHR);
1282 Register ShiftNo =
MI.getOperand(3).getReg();
1283 LLT ShiftTy =
MRI.getType(ShiftNo);
1288 LLT OperationTy =
MRI.getType(
MI.getOperand(0).getReg());
1292 if (!VRegAndVal || VRegAndVal->Value.urem(
BitWidth) == 0)
1298 Amount =
MI.getOpcode() == TargetOpcode::G_FSHL ?
BitWidth - Amount : Amount;
1302 if (ShiftTy.
getSizeInBits() == 64 &&
MI.getOpcode() == TargetOpcode::G_FSHR &&
1309 if (
MI.getOpcode() == TargetOpcode::G_FSHR) {
1311 MI.getOperand(3).setReg(Cast64.getReg(0));
1316 else if (
MI.getOpcode() == TargetOpcode::G_FSHL) {
1318 {
MI.getOperand(1).
getReg(),
MI.getOperand(2).getReg(),
1320 MI.eraseFromParent();
1331 LLT AmtTy =
MRI.getType(AmtReg);
1337 MI.getOperand(2).setReg(NewAmt.getReg(0));
1342bool AArch64LegalizerInfo::legalizeSmallCMGlobalValue(
1345 assert(
MI.getOpcode() == TargetOpcode::G_GLOBAL_VALUE);
1350 auto &GlobalOp =
MI.getOperand(1);
1352 if (GlobalOp.isSymbol())
1354 const auto* GV = GlobalOp.getGlobal();
1355 if (GV->isThreadLocal())
1364 auto Offset = GlobalOp.getOffset();
1369 MRI.setRegClass(
ADRP.getReg(0), &AArch64::GPR64RegClass);
1386 "Should not have folded in an offset for a tagged global!");
1388 .addGlobalAddress(GV, 0x100000000,
1391 MRI.setRegClass(
ADRP.getReg(0), &AArch64::GPR64RegClass);
1395 .addGlobalAddress(GV,
Offset,
1397 MI.eraseFromParent();
1404 switch (IntrinsicID) {
1405 case Intrinsic::vacopy: {
1407 unsigned VaListSize =
1419 VaListSize,
Align(PtrSize)));
1423 VaListSize,
Align(PtrSize)));
1424 MI.eraseFromParent();
1427 case Intrinsic::get_dynamic_area_offset: {
1430 MI.eraseFromParent();
1433 case Intrinsic::aarch64_mops_memset_tag: {
1434 assert(
MI.getOpcode() == TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS);
1438 auto &
Value =
MI.getOperand(3);
1440 Value.setReg(ExtValueReg);
1443 case Intrinsic::aarch64_prefetch: {
1445 auto &AddrVal =
MI.getOperand(1);
1447 int64_t IsWrite =
MI.getOperand(2).getImm();
1448 int64_t
Target =
MI.getOperand(3).getImm();
1449 int64_t IsStream =
MI.getOperand(4).getImm();
1450 int64_t IsData =
MI.getOperand(5).getImm();
1452 unsigned PrfOp = (IsWrite << 4) |
1458 MI.eraseFromParent();
1461 case Intrinsic::aarch64_neon_uaddv:
1462 case Intrinsic::aarch64_neon_saddv:
1463 case Intrinsic::aarch64_neon_umaxv:
1464 case Intrinsic::aarch64_neon_smaxv:
1465 case Intrinsic::aarch64_neon_uminv:
1466 case Intrinsic::aarch64_neon_sminv: {
1469 bool IsSigned = IntrinsicID == Intrinsic::aarch64_neon_saddv ||
1470 IntrinsicID == Intrinsic::aarch64_neon_smaxv ||
1471 IntrinsicID == Intrinsic::aarch64_neon_sminv;
1473 auto OldDst =
MI.getOperand(0).getReg();
1474 auto OldDstTy =
MRI.getType(OldDst);
1475 LLT NewDstTy =
MRI.getType(
MI.getOperand(2).getReg()).getElementType();
1476 if (OldDstTy == NewDstTy)
1479 auto NewDst =
MRI.createGenericVirtualRegister(NewDstTy);
1482 MI.getOperand(0).setReg(NewDst);
1486 MIB.
buildExtOrTrunc(IsSigned ? TargetOpcode::G_SEXT : TargetOpcode::G_ZEXT,
1491 case Intrinsic::aarch64_neon_uaddlp:
1492 case Intrinsic::aarch64_neon_saddlp: {
1495 unsigned Opc = IntrinsicID == Intrinsic::aarch64_neon_uaddlp
1497 : AArch64::G_SADDLP;
1499 MI.eraseFromParent();
1503 case Intrinsic::aarch64_neon_uaddlv:
1504 case Intrinsic::aarch64_neon_saddlv: {
1508 unsigned Opc = IntrinsicID == Intrinsic::aarch64_neon_uaddlv
1510 : AArch64::G_SADDLV;
1513 LLT DstTy =
MRI.getType(DstReg);
1537 MI.eraseFromParent();
1541 case Intrinsic::aarch64_neon_smax:
1542 case Intrinsic::aarch64_neon_smin:
1543 case Intrinsic::aarch64_neon_umax:
1544 case Intrinsic::aarch64_neon_umin:
1545 case Intrinsic::aarch64_neon_fmax:
1546 case Intrinsic::aarch64_neon_fmin:
1547 case Intrinsic::aarch64_neon_fmaxnm:
1548 case Intrinsic::aarch64_neon_fminnm: {
1550 if (IntrinsicID == Intrinsic::aarch64_neon_smax)
1552 else if (IntrinsicID == Intrinsic::aarch64_neon_smin)
1554 else if (IntrinsicID == Intrinsic::aarch64_neon_umax)
1556 else if (IntrinsicID == Intrinsic::aarch64_neon_umin)
1558 else if (IntrinsicID == Intrinsic::aarch64_neon_fmax)
1559 MIB.
buildInstr(TargetOpcode::G_FMAXIMUM, {
MI.getOperand(0)},
1560 {
MI.getOperand(2),
MI.getOperand(3)});
1561 else if (IntrinsicID == Intrinsic::aarch64_neon_fmin)
1562 MIB.
buildInstr(TargetOpcode::G_FMINIMUM, {
MI.getOperand(0)},
1563 {
MI.getOperand(2),
MI.getOperand(3)});
1564 else if (IntrinsicID == Intrinsic::aarch64_neon_fmaxnm)
1565 MIB.
buildInstr(TargetOpcode::G_FMAXNUM, {
MI.getOperand(0)},
1566 {
MI.getOperand(2),
MI.getOperand(3)});
1567 else if (IntrinsicID == Intrinsic::aarch64_neon_fminnm)
1568 MIB.
buildInstr(TargetOpcode::G_FMINNUM, {
MI.getOperand(0)},
1569 {
MI.getOperand(2),
MI.getOperand(3)});
1570 MI.eraseFromParent();
1573 case Intrinsic::vector_reverse:
1581bool AArch64LegalizerInfo::legalizeShlAshrLshr(
1584 assert(
MI.getOpcode() == TargetOpcode::G_ASHR ||
1585 MI.getOpcode() == TargetOpcode::G_LSHR ||
1586 MI.getOpcode() == TargetOpcode::G_SHL);
1599 MI.getOperand(2).setReg(ExtCst.getReg(0));
1612 isShiftedInt<7, 3>(NewOffset)) {
1620bool AArch64LegalizerInfo::legalizeLoadStore(
1623 assert(
MI.getOpcode() == TargetOpcode::G_STORE ||
1624 MI.getOpcode() == TargetOpcode::G_LOAD);
1635 const LLT ValTy =
MRI.getType(ValReg);
1640 bool IsLoad =
MI.getOpcode() == TargetOpcode::G_LOAD;
1644 ST->hasLSE2() && ST->hasRCPC3() && (IsLoadAcquire || IsStoreRelease);
1650 Opcode = IsLoad ? AArch64::LDIAPPX : AArch64::STILPX;
1656 assert(ST->hasLSE2() &&
"ldp/stp not single copy atomic without +lse2");
1658 Opcode = IsLoad ? AArch64::LDPXi : AArch64::STPXi;
1663 NewI = MIRBuilder.
buildInstr(Opcode, {s64, s64}, {});
1669 Opcode, {}, {
Split->getOperand(0),
Split->getOperand(1)});
1673 NewI.
addUse(
MI.getOperand(1).getReg());
1684 *
MRI.getTargetRegisterInfo(),
1686 MI.eraseFromParent();
1692 LLVM_DEBUG(
dbgs() <<
"Tried to do custom legalization on wrong load/store");
1698 auto &MMO = **
MI.memoperands_begin();
1701 if (
MI.getOpcode() == TargetOpcode::G_STORE) {
1705 auto NewLoad = MIRBuilder.
buildLoad(NewTy,
MI.getOperand(1), MMO);
1708 MI.eraseFromParent();
1716 Align Alignment(
MI.getOperand(2).getImm());
1718 Register ListPtr =
MI.getOperand(1).getReg();
1720 LLT PtrTy =
MRI.getType(ListPtr);
1731 if (Alignment > PtrAlign) {
1735 auto ListTmp = MIRBuilder.
buildPtrAdd(PtrTy,
List, AlignMinus1.getReg(0));
1740 LLT ValTy =
MRI.getType(Dst);
1745 ValTy, std::max(Alignment, PtrAlign)));
1756 MI.eraseFromParent();
1760bool AArch64LegalizerInfo::legalizeBitfieldExtract(
1794 LLT Ty =
MRI.getType(Val);
1798 "Expected src and dst to have the same type!");
1806 auto Add = MIRBuilder.
buildAdd(s64, CTPOP1, CTPOP2);
1809 MI.eraseFromParent();
1813 if (!ST->hasNEON() ||
1814 MI.getMF()->getFunction().hasFnAttribute(Attribute::NoImplicitFloat)) {
1826 assert((
Size == 32 ||
Size == 64 ||
Size == 128) &&
"Expected only 32, 64, or 128 bit scalars!");
1841 Opc = Intrinsic::aarch64_neon_uaddlv;
1844 Opc = Intrinsic::aarch64_neon_uaddlp;
1847 Opc = Intrinsic::aarch64_neon_uaddlp;
1851 Opc = Intrinsic::aarch64_neon_uaddlp;
1856 Opc = Intrinsic::aarch64_neon_uaddlp;
1859 Opc = Intrinsic::aarch64_neon_uaddlp;
1865 for (
LLT HTy : HAddTys) {
1875 MI.eraseFromParent();
1879bool AArch64LegalizerInfo::legalizeAtomicCmpxchg128(
1883 auto Addr =
MI.getOperand(1).getReg();
1884 auto DesiredI = MIRBuilder.
buildUnmerge({s64, s64},
MI.getOperand(2));
1885 auto NewI = MIRBuilder.
buildUnmerge({s64, s64},
MI.getOperand(3));
1886 auto DstLo =
MRI.createGenericVirtualRegister(s64);
1887 auto DstHi =
MRI.createGenericVirtualRegister(s64);
1900 auto Ordering = (*
MI.memoperands_begin())->getMergedOrdering();
1904 Opcode = AArch64::CASPAX;
1907 Opcode = AArch64::CASPLX;
1911 Opcode = AArch64::CASPALX;
1914 Opcode = AArch64::CASPX;
1919 auto CASDst =
MRI.createGenericVirtualRegister(s128);
1920 auto CASDesired =
MRI.createGenericVirtualRegister(s128);
1921 auto CASNew =
MRI.createGenericVirtualRegister(s128);
1922 MIRBuilder.
buildInstr(TargetOpcode::REG_SEQUENCE, {CASDesired}, {})
1923 .addUse(DesiredI->getOperand(0).getReg())
1925 .
addUse(DesiredI->getOperand(1).getReg())
1926 .
addImm(AArch64::subo64);
1927 MIRBuilder.
buildInstr(TargetOpcode::REG_SEQUENCE, {CASNew}, {})
1931 .
addImm(AArch64::subo64);
1933 CAS = MIRBuilder.
buildInstr(Opcode, {CASDst}, {CASDesired, CASNew,
Addr});
1941 auto Ordering = (*
MI.memoperands_begin())->getMergedOrdering();
1945 Opcode = AArch64::CMP_SWAP_128_ACQUIRE;
1948 Opcode = AArch64::CMP_SWAP_128_RELEASE;
1952 Opcode = AArch64::CMP_SWAP_128;
1955 Opcode = AArch64::CMP_SWAP_128_MONOTONIC;
1959 auto Scratch =
MRI.createVirtualRegister(&AArch64::GPR64RegClass);
1960 CAS = MIRBuilder.
buildInstr(Opcode, {DstLo, DstHi, Scratch},
1961 {
Addr, DesiredI->getOperand(0),
1962 DesiredI->getOperand(1), NewI->
getOperand(0),
1968 *
MRI.getTargetRegisterInfo(),
1972 MI.eraseFromParent();
1980 LLT Ty =
MRI.getType(
MI.getOperand(1).getReg());
1982 MIRBuilder.
buildCTLZ(
MI.getOperand(0).getReg(), BitReverse);
1983 MI.eraseFromParent();
1992 if (
MI.getOpcode() == TargetOpcode::G_MEMSET) {
1995 auto &
Value =
MI.getOperand(1);
1998 Value.setReg(ExtValueReg);
2005bool AArch64LegalizerInfo::legalizeExtractVectorElt(
2007 assert(
MI.getOpcode() == TargetOpcode::G_EXTRACT_VECTOR_ELT);
2016bool AArch64LegalizerInfo::legalizeDynStackAlloc(
2032 Register AllocSize =
MI.getOperand(1).getReg();
2036 "Unexpected type for dynamic alloca");
2038 "Unexpected type for dynamic alloca");
2040 LLT PtrTy =
MRI.getType(Dst);
2046 MIRBuilder.
buildInstr(AArch64::PROBED_STACKALLOC_DYN, {}, {SPTmp});
2047 MRI.setRegClass(NewMI.getReg(0), &AArch64::GPR64commonRegClass);
2048 MIRBuilder.
setInsertPt(*NewMI->getParent(), NewMI);
2051 MI.eraseFromParent();
2058 auto &AddrVal =
MI.getOperand(0);
2060 int64_t IsWrite =
MI.getOperand(1).getImm();
2061 int64_t Locality =
MI.getOperand(2).getImm();
2062 int64_t
IsData =
MI.getOperand(3).getImm();
2064 bool IsStream = Locality == 0;
2065 if (Locality != 0) {
2066 assert(Locality <= 3 &&
"Prefetch locality out-of-range");
2070 Locality = 3 - Locality;
2073 unsigned PrfOp = (IsWrite << 4) | (!IsData << 3) | (Locality << 1) | IsStream;
2076 MI.eraseFromParent();
unsigned const MachineRegisterInfo * MRI
static void matchLDPSTPAddrMode(Register Root, Register &Base, int &Offset, MachineRegisterInfo &MRI)
This file declares the targeting of the Machinelegalizer class for AArch64.
This file declares the targeting of the RegisterBankInfo class for AArch64.
Declares convenience wrapper classes for interpreting MachineInstr instances as specific generic oper...
Interface for Targets to specify which operations they can successfully select and how the others sho...
Contains matchers for matching SSA Machine Instructions.
This file declares the MachineIRBuilder class.
static unsigned getReg(const MCDisassembler *D, unsigned RC, unsigned RegNo)
const char LLVMTargetMachineRef TM
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
bool legalizeCustom(LegalizerHelper &Helper, MachineInstr &MI, LostDebugLocObserver &LocObserver) const override
Called for instructions with the Custom LegalizationAction.
bool legalizeIntrinsic(LegalizerHelper &Helper, MachineInstr &MI) const override
AArch64LegalizerInfo(const AArch64Subtarget &ST)
bool isTargetWindows() const
const AArch64InstrInfo * getInstrInfo() const override
bool isTargetDarwin() const
bool isTargetILP32() const
const AArch64TargetLowering * getTargetLowering() const override
unsigned ClassifyGlobalReference(const GlobalValue *GV, const TargetMachine &TM) const
ClassifyGlobalReference - Find the target operand flags that describe how a global value should be re...
const RegisterBankInfo * getRegBankInfo() const override
Class for arbitrary precision integers.
APInt zext(unsigned width) const
Zero extend to a new width.
APInt urem(const APInt &RHS) const
Unsigned remainder operation.
int64_t getSExtValue() const
Get sign extended value.
StringRef getValueAsString() const
Return the attribute's value as a string.
This class represents an Operation in the Expression.
Attribute getFnAttribute(Attribute::AttrKind Kind) const
Return the attribute for the given attribute kind.
bool hasFnAttribute(Attribute::AttrKind Kind) const
Return true if the function has the attribute.
Abstract class that contains various methods for clients to notify about changes.
virtual void changingInstr(MachineInstr &MI)=0
This instruction is about to be mutated in some way.
virtual void changedInstr(MachineInstr &MI)=0
This instruction was mutated in some way.
constexpr unsigned getScalarSizeInBits() const
constexpr bool isScalar() const
static constexpr LLT vector(ElementCount EC, unsigned ScalarSizeInBits)
Get a low-level vector of some number of elements and element width.
constexpr bool isPointerVector() const
static constexpr LLT scalar(unsigned SizeInBits)
Get a low-level scalar or aggregate "bag of bits".
constexpr uint16_t getNumElements() const
Returns the number of elements in a vector LLT.
constexpr bool isVector() const
static constexpr LLT pointer(unsigned AddressSpace, unsigned SizeInBits)
Get a low-level pointer in the given address space.
constexpr TypeSize getSizeInBits() const
Returns the total size of the type. Must only be called on sized types.
constexpr LLT getElementType() const
Returns the vector's element type. Only valid for vector types.
constexpr ElementCount getElementCount() const
constexpr unsigned getAddressSpace() const
static constexpr LLT fixed_vector(unsigned NumElements, unsigned ScalarSizeInBits)
Get a low-level fixed-width vector of some number of elements and element width.
constexpr LLT divide(int Factor) const
Return a type that is Factor times smaller.
void computeTables()
Compute any ancillary tables needed to quickly decide how an operation should be handled.
LegalizeRuleSet & minScalar(unsigned TypeIdx, const LLT Ty)
Ensure the scalar is at least as wide as Ty.
LegalizeRuleSet & legalFor(std::initializer_list< LLT > Types)
The instruction is legal when type index 0 is any type in the given list.
LegalizeRuleSet & unsupported()
The instruction is unsupported.
LegalizeRuleSet & scalarSameSizeAs(unsigned TypeIdx, unsigned SameSizeIdx)
Change the type TypeIdx to have the same scalar size as type SameSizeIdx.
LegalizeRuleSet & clampScalarOrElt(unsigned TypeIdx, const LLT MinTy, const LLT MaxTy)
Limit the range of scalar sizes to MinTy and MaxTy.
LegalizeRuleSet & bitcastIf(LegalityPredicate Predicate, LegalizeMutation Mutation)
The specified type index is coerced if predicate is true.
LegalizeRuleSet & libcallFor(std::initializer_list< LLT > Types)
LegalizeRuleSet & maxScalar(unsigned TypeIdx, const LLT Ty)
Ensure the scalar is at most as wide as Ty.
LegalizeRuleSet & minScalarOrElt(unsigned TypeIdx, const LLT Ty)
Ensure the scalar or element is at least as wide as Ty.
LegalizeRuleSet & clampMaxNumElements(unsigned TypeIdx, const LLT EltTy, unsigned MaxElements)
Limit the number of elements in EltTy vectors to at most MaxElements.
LegalizeRuleSet & clampMinNumElements(unsigned TypeIdx, const LLT EltTy, unsigned MinElements)
Limit the number of elements in EltTy vectors to at least MinElements.
LegalizeRuleSet & widenVectorEltsToVectorMinSize(unsigned TypeIdx, unsigned VectorSize)
Ensure the vector size is at least as wide as VectorSize by promoting the element.
LegalizeRuleSet & lowerIfMemSizeNotPow2()
Lower a memory operation if the memory size, rounded to bytes, is not a power of 2.
LegalizeRuleSet & minScalarEltSameAsIf(LegalityPredicate Predicate, unsigned TypeIdx, unsigned LargeTypeIdx)
Conditionally widen the scalar or elt to match the size of another.
LegalizeRuleSet & customForCartesianProduct(std::initializer_list< LLT > Types)
LegalizeRuleSet & lowerIfMemSizeNotByteSizePow2()
Lower a memory operation if the memory access size is not a round power of 2 byte size.
LegalizeRuleSet & moreElementsToNextPow2(unsigned TypeIdx)
Add more elements to the vector to reach the next power of two.
LegalizeRuleSet & narrowScalarIf(LegalityPredicate Predicate, LegalizeMutation Mutation)
Narrow the scalar to the one selected by the mutation if the predicate is true.
LegalizeRuleSet & lower()
The instruction is lowered.
LegalizeRuleSet & moreElementsIf(LegalityPredicate Predicate, LegalizeMutation Mutation)
Add more elements to reach the type selected by the mutation if the predicate is true.
LegalizeRuleSet & scalarizeIf(LegalityPredicate Predicate, unsigned TypeIdx)
LegalizeRuleSet & lowerIf(LegalityPredicate Predicate)
The instruction is lowered if predicate is true.
LegalizeRuleSet & clampScalar(unsigned TypeIdx, const LLT MinTy, const LLT MaxTy)
Limit the range of scalar sizes to MinTy and MaxTy.
LegalizeRuleSet & custom()
Unconditionally custom lower.
LegalizeRuleSet & minScalarSameAs(unsigned TypeIdx, unsigned LargeTypeIdx)
Widen the scalar to match the size of another.
LegalizeRuleSet & unsupportedIf(LegalityPredicate Predicate)
LegalizeRuleSet & minScalarOrEltIf(LegalityPredicate Predicate, unsigned TypeIdx, const LLT Ty)
Ensure the scalar or element is at least as wide as Ty.
LegalizeRuleSet & widenScalarIf(LegalityPredicate Predicate, LegalizeMutation Mutation)
Widen the scalar to the one selected by the mutation if the predicate is true.
LegalizeRuleSet & clampNumElements(unsigned TypeIdx, const LLT MinTy, const LLT MaxTy)
Limit the number of elements for the given vectors to at least MinTy's number of elements and at most...
LegalizeRuleSet & maxScalarIf(LegalityPredicate Predicate, unsigned TypeIdx, const LLT Ty)
Conditionally limit the maximum size of the scalar.
LegalizeRuleSet & customIf(LegalityPredicate Predicate)
LegalizeRuleSet & widenScalarToNextPow2(unsigned TypeIdx, unsigned MinSize=0)
Widen the scalar to the next power of two that is at least MinSize.
LegalizeRuleSet & scalarize(unsigned TypeIdx)
LegalizeRuleSet & legalForCartesianProduct(std::initializer_list< LLT > Types)
The instruction is legal when type indexes 0 and 1 are both in the given list.
LegalizeRuleSet & legalForTypesWithMemDesc(std::initializer_list< LegalityPredicates::TypePairAndMemDesc > TypesAndMemDesc)
The instruction is legal when type indexes 0 and 1 along with the memory size and minimum alignment i...
LegalizeRuleSet & libcallIf(LegalityPredicate Predicate)
Like legalIf, but for the Libcall action.
LegalizeRuleSet & legalIf(LegalityPredicate Predicate)
The instruction is legal if predicate is true.
LegalizeResult lowerDynStackAlloc(MachineInstr &MI)
LegalizeResult lowerBitCount(MachineInstr &MI)
LegalizeResult lowerExtractInsertVectorElt(MachineInstr &MI)
Lower a vector extract or insert by writing the vector to a stack temporary and reloading the element...
LegalizeResult lowerAbsToCNeg(MachineInstr &MI)
const TargetLowering & getTargetLowering() const
LegalizeResult lowerFunnelShiftAsShifts(MachineInstr &MI)
@ Legalized
Instruction has been legalized and the MachineFunction changed.
@ UnableToLegalize
Some kind of error has occurred and we could not legalize this instruction.
GISelChangeObserver & Observer
To keep track of changes made by the LegalizerHelper.
Register getDynStackAllocTargetPtr(Register SPReg, Register AllocSize, Align Alignment, LLT PtrTy)
MachineIRBuilder & MIRBuilder
Expose MIRBuilder so clients can set their own RecordInsertInstruction functions.
LegalizeRuleSet & getActionDefinitionsBuilder(unsigned Opcode)
Get the action definition builder for the given opcode.
const LegacyLegalizerInfo & getLegacyLegalizerInfo() const
MachineMemOperand * getMachineMemOperand(MachinePointerInfo PtrInfo, MachineMemOperand::Flags f, LLT MemTy, Align base_alignment, const AAMDNodes &AAInfo=AAMDNodes(), const MDNode *Ranges=nullptr, SyncScope::ID SSID=SyncScope::System, AtomicOrdering Ordering=AtomicOrdering::NotAtomic, AtomicOrdering FailureOrdering=AtomicOrdering::NotAtomic)
getMachineMemOperand - Allocate a new MachineMemOperand.
MachineRegisterInfo & getRegInfo()
getRegInfo - Return information about the registers currently in use.
Function & getFunction()
Return the LLVM function that this machine code represents.
Helper class to build MachineInstr.
void setInsertPt(MachineBasicBlock &MBB, MachineBasicBlock::iterator II)
Set the insertion point before the specified position.
MachineInstrBuilder buildAdd(const DstOp &Dst, const SrcOp &Src0, const SrcOp &Src1, std::optional< unsigned > Flags=std::nullopt)
Build and insert Res = G_ADD Op0, Op1.
MachineInstrBuilder buildUnmerge(ArrayRef< LLT > Res, const SrcOp &Op)
Build and insert Res0, ... = G_UNMERGE_VALUES Op.
MachineInstrBuilder buildExtract(const DstOp &Res, const SrcOp &Src, uint64_t Index)
Build and insert Res0, ... = G_EXTRACT Src, Idx0.
MachineBasicBlock::iterator getInsertPt()
Current insertion point for new instructions.
MachineInstrBuilder buildIntrinsic(Intrinsic::ID ID, ArrayRef< Register > Res, bool HasSideEffects, bool isConvergent)
Build and insert a G_INTRINSIC instruction.
MachineInstrBuilder buildCTLZ(const DstOp &Dst, const SrcOp &Src0)
Build and insert Res = G_CTLZ Op0, Src0.
MachineInstrBuilder buildSMax(const DstOp &Dst, const SrcOp &Src0, const SrcOp &Src1)
Build and insert Res = G_SMAX Op0, Op1.
MachineInstrBuilder buildMergeLikeInstr(const DstOp &Res, ArrayRef< Register > Ops)
Build and insert Res = G_MERGE_VALUES Op0, ... or Res = G_BUILD_VECTOR Op0, ... or Res = G_CONCAT_VEC...
MachineInstrBuilder buildLoad(const DstOp &Res, const SrcOp &Addr, MachineMemOperand &MMO)
Build and insert Res = G_LOAD Addr, MMO.
MachineInstrBuilder buildPtrAdd(const DstOp &Res, const SrcOp &Op0, const SrcOp &Op1, std::optional< unsigned > Flags=std::nullopt)
Build and insert Res = G_PTR_ADD Op0, Op1.
MachineInstrBuilder buildBitReverse(const DstOp &Dst, const SrcOp &Src)
Build and insert Dst = G_BITREVERSE Src.
MachineInstrBuilder buildStore(const SrcOp &Val, const SrcOp &Addr, MachineMemOperand &MMO)
Build and insert G_STORE Val, Addr, MMO.
MachineInstrBuilder buildInstr(unsigned Opcode)
Build and insert <empty> = Opcode <empty>.
MachineInstrBuilder buildCTPOP(const DstOp &Dst, const SrcOp &Src0)
Build and insert Res = G_CTPOP Op0, Src0.
MachineInstrBuilder buildZExt(const DstOp &Res, const SrcOp &Op)
Build and insert Res = G_ZEXT Op.
MachineFunction & getMF()
Getter for the function we currently build.
MachineInstrBuilder buildSMin(const DstOp &Dst, const SrcOp &Src0, const SrcOp &Src1)
Build and insert Res = G_SMIN Op0, Op1.
MachineInstrBuilder buildExtOrTrunc(unsigned ExtOpc, const DstOp &Res, const SrcOp &Op)
Build and insert Res = ExtOpc, Res = G_TRUNC Op, or Res = COPY Op depending on the differing sizes of...
const MachineBasicBlock & getMBB() const
Getter for the basic block we currently build.
MachineInstrBuilder buildAnyExt(const DstOp &Res, const SrcOp &Op)
Build and insert Res = G_ANYEXT Op0.
MachineInstrBuilder buildBitcast(const DstOp &Dst, const SrcOp &Src)
Build and insert Dst = G_BITCAST Src.
MachineInstrBuilder buildTrunc(const DstOp &Res, const SrcOp &Op)
Build and insert Res = G_TRUNC Op.
MachineRegisterInfo * getMRI()
Getter for MRI.
MachineInstrBuilder buildCopy(const DstOp &Res, const SrcOp &Op)
Build and insert Res = COPY Op.
MachineInstrBuilder buildMaskLowPtrBits(const DstOp &Res, const SrcOp &Op0, uint32_t NumBits)
Build and insert Res = G_PTRMASK Op0, G_CONSTANT (1 << NumBits) - 1.
virtual MachineInstrBuilder buildConstant(const DstOp &Res, const ConstantInt &Val)
Build and insert Res = G_CONSTANT Val.
MachineInstrBuilder buildUMin(const DstOp &Dst, const SrcOp &Src0, const SrcOp &Src1)
Build and insert Res = G_UMIN Op0, Op1.
MachineInstrBuilder buildUMax(const DstOp &Dst, const SrcOp &Src0, const SrcOp &Src1)
Build and insert Res = G_UMAX Op0, Op1.
Register getReg(unsigned Idx) const
Get the register for the operand index.
const MachineInstrBuilder & addImm(int64_t Val) const
Add a new immediate operand.
const MachineInstrBuilder & add(const MachineOperand &MO) const
const MachineInstrBuilder & cloneMemRefs(const MachineInstr &OtherMI) const
const MachineInstrBuilder & addUse(Register RegNo, unsigned Flags=0, unsigned SubReg=0) const
Add a virtual register use operand.
Representation of each machine instruction.
const MachineOperand & getOperand(unsigned i) const
@ MOLoad
The memory access reads data.
@ MOStore
The memory access writes data.
void setReg(Register Reg)
Change the register this operand corresponds to.
Register getReg() const
getReg - Returns the register number.
MachineRegisterInfo - Keep track of information for virtual and physical registers,...
Register createGenericVirtualRegister(LLT Ty, StringRef Name="")
Create and return a new generic virtual register with low-level type Ty.
Wrapper class representing virtual and physical registers.
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
const TargetMachine & getTargetMachine() const
Register getStackPointerRegisterToSaveRestore() const
If a physical register, this specifies the register that llvm.savestack/llvm.restorestack should save...
Primary interface to the complete machine description for the target machine.
Target - Wrapper for Target specific information.
LLVM Value Representation.
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
@ MO_NC
MO_NC - Indicates whether the linker is expected to check the symbol reference for overflow.
@ MO_PAGEOFF
MO_PAGEOFF - A symbol operand with this flag represents the offset of that symbol within a 4K page.
@ MO_GOT
MO_GOT - This flag indicates that a symbol operand represents the address of the GOT entry for the sy...
@ MO_PREL
MO_PREL - Indicates that the bits of the symbol operand represented by MO_G0 etc are PC relative.
@ MO_PAGE
MO_PAGE - A symbol operand with this flag represents the pc-relative offset of the 4K page containing...
@ MO_TAGGED
MO_TAGGED - With MO_PAGE, indicates that the page includes a memory tag in bits 56-63.
@ MO_G3
MO_G3 - A symbol operand with this flag (granule 3) represents the high 16-bits of a 64-bit address,...
LegalityPredicate typeInSet(unsigned TypeIdx, std::initializer_list< LLT > TypesInit)
True iff the given type index is one of the specified types.
LegalityPredicate smallerThan(unsigned TypeIdx0, unsigned TypeIdx1)
True iff the first type index has a smaller total bit size than second type index.
LegalityPredicate atomicOrderingAtLeastOrStrongerThan(unsigned MMOIdx, AtomicOrdering Ordering)
True iff the specified MMO index has at an atomic ordering of at Ordering or stronger.
LegalityPredicate isVector(unsigned TypeIdx)
True iff the specified type index is a vector.
Predicate all(Predicate P0, Predicate P1)
True iff P0 and P1 are true.
LegalityPredicate typeIs(unsigned TypeIdx, LLT TypesInit)
True iff the given type index is the specified type.
Predicate predNot(Predicate P)
True iff P is false.
@ Bitcast
Perform the operation on a different, but equivalently sized type.
LegalizeMutation moreElementsToNextPow2(unsigned TypeIdx, unsigned Min=0)
Add more elements to the type for the given type index to the next power of.
LegalizeMutation scalarize(unsigned TypeIdx)
Break up the vector type for the given type index into the element type.
LegalizeMutation widenScalarOrEltToNextPow2(unsigned TypeIdx, unsigned Min=0)
Widen the scalar type or vector element type for the given type index to the next power of 2.
LegalizeMutation changeTo(unsigned TypeIdx, LLT Ty)
Select this specific type for the given type index.
LegalizeMutation changeElementSizeTo(unsigned TypeIdx, unsigned FromTypeIdx)
Change the scalar size or element size to have the same scalar size as type index FromIndex.
operand_type_match m_Reg()
ConstantMatch< APInt > m_ICst(APInt &Cst)
bool mi_match(Reg R, const MachineRegisterInfo &MRI, Pattern &&P)
BinaryOp_match< LHS, RHS, TargetOpcode::G_PTR_ADD, false > m_GPtrAdd(const LHS &L, const RHS &R)
This is an optimization pass for GlobalISel generic memory operations.
bool constrainSelectedInstRegOperands(MachineInstr &I, const TargetInstrInfo &TII, const TargetRegisterInfo &TRI, const RegisterBankInfo &RBI)
Mutate the newly-selected instruction I to constrain its (possibly generic) virtual register operands...
constexpr bool isPowerOf2_32(uint32_t Value)
Return true if the argument is a power of two > 0.
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
AtomicOrdering
Atomic ordering for LLVM's memory model.
uint64_t alignTo(uint64_t Size, Align A)
Returns a multiple of A needed to store Size bytes.
constexpr unsigned BitWidth
std::optional< ValueAndVReg > getIConstantVRegValWithLookThrough(Register VReg, const MachineRegisterInfo &MRI, bool LookThroughInstrs=true)
If VReg is defined by a statically evaluable chain of instructions rooted on a G_CONSTANT returns its...
bool is_contained(R &&Range, const E &Element)
Returns true if Element is found in Range.
Align assumeAligned(uint64_t Value)
Treats the value 0 as a 1, so Align is always at least 1.
unsigned Log2(Align A)
Returns the log2 of the alignment.
std::function< bool(const LegalityQuery &)> LegalityPredicate
This struct is a compact representation of a valid (non-zero power of two) alignment.
The LegalityQuery object bundles together all the information that's needed to decide whether a given...
ArrayRef< MemDesc > MMODescrs
Operations which require memory can use this to place requirements on the memory type for each MMO.
This class contains a discriminated union of information about pointers in memory operands,...