42#define DEBUG_TYPE "legalizer"
45using namespace LegalizeActions;
46using namespace MIPatternMatch;
55static std::pair<int, int>
61 unsigned NumParts =
Size / NarrowSize;
62 unsigned LeftoverSize =
Size - NumParts * NarrowSize;
65 if (LeftoverSize == 0)
70 if (LeftoverSize % EltSize != 0)
79 return std::make_pair(NumParts, NumLeftover);
106 : MIRBuilder(Builder), Observer(Observer),
MRI(MF.getRegInfo()),
107 LI(*MF.getSubtarget().getLegalizerInfo()),
108 TLI(*MF.getSubtarget().getTargetLowering()), KB(nullptr) {}
113 : MIRBuilder(
B), Observer(Observer),
MRI(MF.getRegInfo()), LI(LI),
114 TLI(*MF.getSubtarget().getTargetLowering()), KB(KB) {}
123 if (isa<GIntrinsic>(
MI))
126 switch (Step.Action) {
141 return bitcast(
MI, Step.TypeIdx, Step.NewType);
144 return lower(
MI, Step.TypeIdx, Step.NewType);
161void LegalizerHelper::insertParts(
Register DstReg,
183 assert(LeftoverRegs.
size() == 1 &&
"Expected one leftover register");
185 for (
auto Reg : concat<const Register>(PartRegs, LeftoverRegs))
187 return mergeMixedSubvectors(DstReg, AllRegs);
192 for (
auto PartReg : concat<const Register>(PartRegs, LeftoverRegs))
193 extractGCDType(GCDRegs, GCDTy, PartReg);
194 LLT ResultLCMTy = buildLCMMergePieces(ResultTy, LeftoverTy, GCDTy, GCDRegs);
195 buildWidenedRemergeToDst(DstReg, ResultLCMTy, GCDRegs);
208void LegalizerHelper::mergeMixedSubvectors(
Register DstReg,
211 for (
unsigned i = 0; i < PartRegs.
size() - 1; ++i)
212 appendVectorElts(AllElts, PartRegs[i]);
218 appendVectorElts(AllElts, Leftover);
226 assert(
MI.getOpcode() == TargetOpcode::G_UNMERGE_VALUES);
228 const int StartIdx = Regs.
size();
229 const int NumResults =
MI.getNumOperands() - 1;
231 for (
int I = 0;
I != NumResults; ++
I)
232 Regs[StartIdx +
I] =
MI.getOperand(
I).getReg();
238 if (SrcTy == GCDTy) {
253 extractGCDType(Parts, GCDTy, SrcReg);
257LLT LegalizerHelper::buildLCMMergePieces(
LLT DstTy,
LLT NarrowTy,
LLT GCDTy,
259 unsigned PadStrategy) {
264 int NumOrigSrc = VRegs.
size();
270 if (NumOrigSrc < NumParts * NumSubParts) {
271 if (PadStrategy == TargetOpcode::G_ZEXT)
273 else if (PadStrategy == TargetOpcode::G_ANYEXT)
276 assert(PadStrategy == TargetOpcode::G_SEXT);
297 for (
int I = 0;
I != NumParts; ++
I) {
298 bool AllMergePartsArePadding =
true;
301 for (
int J = 0; J != NumSubParts; ++J) {
302 int Idx =
I * NumSubParts + J;
303 if (
Idx >= NumOrigSrc) {
304 SubMerge[J] = PadReg;
308 SubMerge[J] = VRegs[
Idx];
311 AllMergePartsArePadding =
false;
317 if (AllMergePartsArePadding && !AllPadReg) {
318 if (PadStrategy == TargetOpcode::G_ANYEXT)
320 else if (PadStrategy == TargetOpcode::G_ZEXT)
330 Remerge[
I] = AllPadReg;
334 if (NumSubParts == 1)
335 Remerge[
I] = SubMerge[0];
340 if (AllMergePartsArePadding && !AllPadReg)
341 AllPadReg = Remerge[
I];
344 VRegs = std::move(Remerge);
348void LegalizerHelper::buildWidenedRemergeToDst(
Register DstReg,
LLT LCMTy,
355 if (DstTy == LCMTy) {
369 UnmergeDefs[0] = DstReg;
370 for (
unsigned I = 1;
I != NumDefs; ++
I)
382#define RTLIBCASE_INT(LibcallPrefix) \
386 return RTLIB::LibcallPrefix##32; \
388 return RTLIB::LibcallPrefix##64; \
390 return RTLIB::LibcallPrefix##128; \
392 llvm_unreachable("unexpected size"); \
396#define RTLIBCASE(LibcallPrefix) \
400 return RTLIB::LibcallPrefix##32; \
402 return RTLIB::LibcallPrefix##64; \
404 return RTLIB::LibcallPrefix##80; \
406 return RTLIB::LibcallPrefix##128; \
408 llvm_unreachable("unexpected size"); \
413 case TargetOpcode::G_MUL:
415 case TargetOpcode::G_SDIV:
417 case TargetOpcode::G_UDIV:
419 case TargetOpcode::G_SREM:
421 case TargetOpcode::G_UREM:
423 case TargetOpcode::G_CTLZ_ZERO_UNDEF:
425 case TargetOpcode::G_FADD:
427 case TargetOpcode::G_FSUB:
429 case TargetOpcode::G_FMUL:
431 case TargetOpcode::G_FDIV:
433 case TargetOpcode::G_FEXP:
435 case TargetOpcode::G_FEXP2:
437 case TargetOpcode::G_FEXP10:
439 case TargetOpcode::G_FREM:
441 case TargetOpcode::G_FPOW:
443 case TargetOpcode::G_FPOWI:
445 case TargetOpcode::G_FMA:
447 case TargetOpcode::G_FSIN:
449 case TargetOpcode::G_FCOS:
451 case TargetOpcode::G_FLOG10:
453 case TargetOpcode::G_FLOG:
455 case TargetOpcode::G_FLOG2:
457 case TargetOpcode::G_FLDEXP:
459 case TargetOpcode::G_FCEIL:
461 case TargetOpcode::G_FFLOOR:
463 case TargetOpcode::G_FMINNUM:
465 case TargetOpcode::G_FMAXNUM:
467 case TargetOpcode::G_FSQRT:
469 case TargetOpcode::G_FRINT:
471 case TargetOpcode::G_FNEARBYINT:
473 case TargetOpcode::G_INTRINSIC_ROUNDEVEN:
499 if (CallerAttrs.
hasRetAttr(Attribute::ZExt) ||
511 if (
MI.getOpcode() == TargetOpcode::G_BZERO)
518 if (!VReg.
isVirtual() || VReg != Next->getOperand(1).getReg())
521 Register PReg = Next->getOperand(0).getReg();
529 if (Ret->getNumImplicitOperands() != 1)
532 if (!Ret->getOperand(0).isReg() || PReg != Ret->getOperand(0).getReg())
556 Info.OrigRet = Result;
559 (Result.Ty->isVoidTy() ||
564 std::copy(Args.begin(), Args.end(), std::back_inserter(
Info.OrigArgs));
565 if (!CLI.lowerCall(MIRBuilder,
Info))
568 if (
MI &&
Info.LoweredTailCall) {
569 assert(
Info.IsTailCall &&
"Lowered tail call when it wasn't a tail call?");
580 "Expected instr following MI to be return or debug inst?");
584 }
while (
MI->getNextNode());
614 Args.push_back({MO.getReg(), OpType, 0});
616 {
MI.getOperand(0).
getReg(), OpType, 0}, Args,
627 for (
unsigned i = 0; i <
MI.getNumOperands() - 1; ++i) {
631 LLT OpLLT =
MRI.getType(Reg);
632 Type *OpTy =
nullptr;
637 Args.push_back({Reg, OpTy, 0});
643 unsigned Opc =
MI.getOpcode();
645 case TargetOpcode::G_BZERO:
646 RTLibcall = RTLIB::BZERO;
648 case TargetOpcode::G_MEMCPY:
649 RTLibcall = RTLIB::MEMCPY;
650 Args[0].Flags[0].setReturned();
652 case TargetOpcode::G_MEMMOVE:
653 RTLibcall = RTLIB::MEMMOVE;
654 Args[0].Flags[0].setReturned();
656 case TargetOpcode::G_MEMSET:
657 RTLibcall = RTLIB::MEMSET;
658 Args[0].Flags[0].setReturned();
663 const char *
Name = TLI.getLibcallName(RTLibcall);
673 Info.CallConv = TLI.getLibcallCallingConv(RTLibcall);
677 MI.getOperand(
MI.getNumOperands() - 1).getImm() &&
680 std::copy(Args.begin(), Args.end(), std::back_inserter(
Info.OrigArgs));
681 if (!CLI.lowerCall(MIRBuilder,
Info))
684 if (
Info.LoweredTailCall) {
685 assert(
Info.IsTailCall &&
"Lowered tail call when it wasn't a tail call?");
696 "Expected instr following MI to be return or debug inst?");
700 }
while (
MI.getNextNode());
710 unsigned Opc =
MI.getOpcode();
711 auto &AtomicMI = cast<GMemOperation>(
MI);
712 auto &MMO = AtomicMI.getMMO();
713 auto Ordering = MMO.getMergedOrdering();
714 LLT MemType = MMO.getMemoryType();
717 return RTLIB::UNKNOWN_LIBCALL;
719#define LCALLS(A, B) \
720 { A##B##_RELAX, A##B##_ACQ, A##B##_REL, A##B##_ACQ_REL }
722 LCALLS(A, 1), LCALLS(A, 2), LCALLS(A, 4), LCALLS(A, 8), LCALLS(A, 16)
724 case TargetOpcode::G_ATOMIC_CMPXCHG:
725 case TargetOpcode::G_ATOMIC_CMPXCHG_WITH_SUCCESS: {
727 return getOutlineAtomicHelper(LC, Ordering, MemSize);
729 case TargetOpcode::G_ATOMICRMW_XCHG: {
731 return getOutlineAtomicHelper(LC, Ordering, MemSize);
733 case TargetOpcode::G_ATOMICRMW_ADD:
734 case TargetOpcode::G_ATOMICRMW_SUB: {
736 return getOutlineAtomicHelper(LC, Ordering, MemSize);
738 case TargetOpcode::G_ATOMICRMW_AND: {
740 return getOutlineAtomicHelper(LC, Ordering, MemSize);
742 case TargetOpcode::G_ATOMICRMW_OR: {
744 return getOutlineAtomicHelper(LC, Ordering, MemSize);
746 case TargetOpcode::G_ATOMICRMW_XOR: {
748 return getOutlineAtomicHelper(LC, Ordering, MemSize);
751 return RTLIB::UNKNOWN_LIBCALL;
764 unsigned Opc =
MI.getOpcode();
766 case TargetOpcode::G_ATOMIC_CMPXCHG:
767 case TargetOpcode::G_ATOMIC_CMPXCHG_WITH_SUCCESS: {
770 auto [Ret, RetLLT, Mem, MemLLT, Cmp, CmpLLT, New, NewLLT] =
771 MI.getFirst4RegLLTs();
774 if (Opc == TargetOpcode::G_ATOMIC_CMPXCHG_WITH_SUCCESS) {
775 std::tie(Ret, RetLLT,
Success, SuccessLLT, Mem, MemLLT, Cmp, CmpLLT, New,
776 NewLLT) =
MI.getFirst5RegLLTs();
786 case TargetOpcode::G_ATOMICRMW_XCHG:
787 case TargetOpcode::G_ATOMICRMW_ADD:
788 case TargetOpcode::G_ATOMICRMW_SUB:
789 case TargetOpcode::G_ATOMICRMW_AND:
790 case TargetOpcode::G_ATOMICRMW_OR:
791 case TargetOpcode::G_ATOMICRMW_XOR: {
792 auto [Ret, RetLLT, Mem, MemLLT, Val, ValLLT] =
MI.getFirst3RegLLTs();
795 if (Opc == TargetOpcode::G_ATOMICRMW_AND)
799 else if (Opc == TargetOpcode::G_ATOMICRMW_SUB)
814 const char *
Name = TLI.getLibcallName(RTLibcall);
824 Info.CallConv = TLI.getLibcallCallingConv(RTLibcall);
828 std::copy(Args.begin(), Args.end(), std::back_inserter(
Info.OrigArgs));
829 if (!CLI.lowerCall(MIRBuilder,
Info))
841 case TargetOpcode::G_FPEXT:
843 case TargetOpcode::G_FPTRUNC:
845 case TargetOpcode::G_FPTOSI:
847 case TargetOpcode::G_FPTOUI:
849 case TargetOpcode::G_SITOFP:
851 case TargetOpcode::G_UITOFP:
863 {{
MI.getOperand(1).
getReg(), FromType, 0}}, LocObserver, &
MI);
869 switch (
MI.getOpcode()) {
870 case TargetOpcode::G_GET_FPENV:
871 RTLibcall = RTLIB::FEGETENV;
873 case TargetOpcode::G_SET_FPENV:
874 case TargetOpcode::G_RESET_FPENV:
875 RTLibcall = RTLIB::FESETENV;
877 case TargetOpcode::G_GET_FPMODE:
878 RTLibcall = RTLIB::FEGETMODE;
880 case TargetOpcode::G_SET_FPMODE:
881 case TargetOpcode::G_RESET_FPMODE:
882 RTLibcall = RTLIB::FESETMODE;
911 auto &Ctx = MF.getFunction().getContext();
922 unsigned TempAddrSpace =
DL.getAllocaAddrSpace();
929 LocObserver,
nullptr);
951 auto &Ctx = MF.getFunction().getContext();
967 unsigned TempAddrSpace =
DL.getAllocaAddrSpace();
973 LocObserver,
nullptr);
990 unsigned AddrSpace =
DL.getDefaultGlobalsAddressSpace();
992 unsigned PtrSize =
DL.getPointerSizeInBits(AddrSpace);
1009 switch (
MI.getOpcode()) {
1012 case TargetOpcode::G_MUL:
1013 case TargetOpcode::G_SDIV:
1014 case TargetOpcode::G_UDIV:
1015 case TargetOpcode::G_SREM:
1016 case TargetOpcode::G_UREM:
1017 case TargetOpcode::G_CTLZ_ZERO_UNDEF: {
1026 case TargetOpcode::G_FADD:
1027 case TargetOpcode::G_FSUB:
1028 case TargetOpcode::G_FMUL:
1029 case TargetOpcode::G_FDIV:
1030 case TargetOpcode::G_FMA:
1031 case TargetOpcode::G_FPOW:
1032 case TargetOpcode::G_FREM:
1033 case TargetOpcode::G_FCOS:
1034 case TargetOpcode::G_FSIN:
1035 case TargetOpcode::G_FLOG10:
1036 case TargetOpcode::G_FLOG:
1037 case TargetOpcode::G_FLOG2:
1038 case TargetOpcode::G_FLDEXP:
1039 case TargetOpcode::G_FEXP:
1040 case TargetOpcode::G_FEXP2:
1041 case TargetOpcode::G_FEXP10:
1042 case TargetOpcode::G_FCEIL:
1043 case TargetOpcode::G_FFLOOR:
1044 case TargetOpcode::G_FMINNUM:
1045 case TargetOpcode::G_FMAXNUM:
1046 case TargetOpcode::G_FSQRT:
1047 case TargetOpcode::G_FRINT:
1048 case TargetOpcode::G_FNEARBYINT:
1049 case TargetOpcode::G_INTRINSIC_ROUNDEVEN: {
1054 LLVM_DEBUG(
dbgs() <<
"No libcall available for type " << LLTy <<
".\n");
1062 case TargetOpcode::G_FPOWI: {
1069 LLVM_DEBUG(
dbgs() <<
"No libcall available for type " << LLTy <<
".\n");
1073 std::initializer_list<CallLowering::ArgInfo> Args = {
1074 {
MI.getOperand(1).getReg(), HLTy, 0},
1075 {
MI.getOperand(2).getReg(), ITy, 1}};
1078 Args, LocObserver, &
MI);
1083 case TargetOpcode::G_FPEXT:
1084 case TargetOpcode::G_FPTRUNC: {
1087 if (!FromTy || !ToTy)
1095 case TargetOpcode::G_FPTOSI:
1096 case TargetOpcode::G_FPTOUI: {
1100 if ((ToSize != 32 && ToSize != 64) || (FromSize != 32 && FromSize != 64))
1111 case TargetOpcode::G_SITOFP:
1112 case TargetOpcode::G_UITOFP: {
1116 if ((FromSize != 32 && FromSize != 64) || (ToSize != 32 && ToSize != 64))
1127 case TargetOpcode::G_ATOMICRMW_XCHG:
1128 case TargetOpcode::G_ATOMICRMW_ADD:
1129 case TargetOpcode::G_ATOMICRMW_SUB:
1130 case TargetOpcode::G_ATOMICRMW_AND:
1131 case TargetOpcode::G_ATOMICRMW_OR:
1132 case TargetOpcode::G_ATOMICRMW_XOR:
1133 case TargetOpcode::G_ATOMIC_CMPXCHG:
1134 case TargetOpcode::G_ATOMIC_CMPXCHG_WITH_SUCCESS: {
1140 case TargetOpcode::G_BZERO:
1141 case TargetOpcode::G_MEMCPY:
1142 case TargetOpcode::G_MEMMOVE:
1143 case TargetOpcode::G_MEMSET: {
1148 MI.eraseFromParent();
1151 case TargetOpcode::G_GET_FPENV:
1152 case TargetOpcode::G_GET_FPMODE: {
1158 case TargetOpcode::G_SET_FPENV:
1159 case TargetOpcode::G_SET_FPMODE: {
1165 case TargetOpcode::G_RESET_FPENV:
1166 case TargetOpcode::G_RESET_FPMODE: {
1175 MI.eraseFromParent();
1185 switch (
MI.getOpcode()) {
1188 case TargetOpcode::G_IMPLICIT_DEF: {
1198 if (SizeOp0 % NarrowSize != 0) {
1199 LLT ImplicitTy = NarrowTy;
1206 MI.eraseFromParent();
1210 int NumParts = SizeOp0 / NarrowSize;
1213 for (
int i = 0; i < NumParts; ++i)
1220 MI.eraseFromParent();
1223 case TargetOpcode::G_CONSTANT: {
1225 const APInt &Val =
MI.getOperand(1).getCImm()->getValue();
1228 int NumParts = TotalSize / NarrowSize;
1231 for (
int I = 0;
I != NumParts; ++
I) {
1232 unsigned Offset =
I * NarrowSize;
1239 unsigned LeftoverBits = TotalSize - NumParts * NarrowSize;
1241 if (LeftoverBits != 0) {
1245 Val.
lshr(NumParts * NarrowSize).
trunc(LeftoverBits));
1249 insertParts(
MI.getOperand(0).getReg(),
1250 Ty, NarrowTy, PartRegs, LeftoverTy, LeftoverRegs);
1252 MI.eraseFromParent();
1255 case TargetOpcode::G_SEXT:
1256 case TargetOpcode::G_ZEXT:
1257 case TargetOpcode::G_ANYEXT:
1259 case TargetOpcode::G_TRUNC: {
1265 LLVM_DEBUG(
dbgs() <<
"Can't narrow trunc to type " << NarrowTy <<
"\n");
1271 MI.eraseFromParent();
1275 case TargetOpcode::G_FREEZE: {
1286 for (
unsigned i = 0; i < Unmerge->getNumDefs(); ++i) {
1292 MI.eraseFromParent();
1295 case TargetOpcode::G_ADD:
1296 case TargetOpcode::G_SUB:
1297 case TargetOpcode::G_SADDO:
1298 case TargetOpcode::G_SSUBO:
1299 case TargetOpcode::G_SADDE:
1300 case TargetOpcode::G_SSUBE:
1301 case TargetOpcode::G_UADDO:
1302 case TargetOpcode::G_USUBO:
1303 case TargetOpcode::G_UADDE:
1304 case TargetOpcode::G_USUBE:
1306 case TargetOpcode::G_MUL:
1307 case TargetOpcode::G_UMULH:
1309 case TargetOpcode::G_EXTRACT:
1311 case TargetOpcode::G_INSERT:
1313 case TargetOpcode::G_LOAD: {
1314 auto &LoadMI = cast<GLoad>(
MI);
1315 Register DstReg = LoadMI.getDstReg();
1320 if (8 * LoadMI.getMemSize().getValue() != DstTy.
getSizeInBits()) {
1324 LoadMI.eraseFromParent();
1330 case TargetOpcode::G_ZEXTLOAD:
1331 case TargetOpcode::G_SEXTLOAD: {
1332 auto &LoadMI = cast<GExtLoad>(
MI);
1333 Register DstReg = LoadMI.getDstReg();
1334 Register PtrReg = LoadMI.getPointerReg();
1337 auto &MMO = LoadMI.getMMO();
1340 if (MemSize == NarrowSize) {
1342 }
else if (MemSize < NarrowSize) {
1344 }
else if (MemSize > NarrowSize) {
1349 if (isa<GZExtLoad>(LoadMI))
1354 LoadMI.eraseFromParent();
1357 case TargetOpcode::G_STORE: {
1358 auto &StoreMI = cast<GStore>(
MI);
1360 Register SrcReg = StoreMI.getValueReg();
1365 int NumParts = SizeOp0 / NarrowSize;
1367 unsigned LeftoverBits = SrcTy.
getSizeInBits() - HandledSize;
1368 if (SrcTy.
isVector() && LeftoverBits != 0)
1371 if (8 * StoreMI.getMemSize().getValue() != SrcTy.
getSizeInBits()) {
1375 StoreMI.eraseFromParent();
1381 case TargetOpcode::G_SELECT:
1383 case TargetOpcode::G_AND:
1384 case TargetOpcode::G_OR:
1385 case TargetOpcode::G_XOR: {
1397 case TargetOpcode::G_SHL:
1398 case TargetOpcode::G_LSHR:
1399 case TargetOpcode::G_ASHR:
1401 case TargetOpcode::G_CTLZ:
1402 case TargetOpcode::G_CTLZ_ZERO_UNDEF:
1403 case TargetOpcode::G_CTTZ:
1404 case TargetOpcode::G_CTTZ_ZERO_UNDEF:
1405 case TargetOpcode::G_CTPOP:
1407 switch (
MI.getOpcode()) {
1408 case TargetOpcode::G_CTLZ:
1409 case TargetOpcode::G_CTLZ_ZERO_UNDEF:
1411 case TargetOpcode::G_CTTZ:
1412 case TargetOpcode::G_CTTZ_ZERO_UNDEF:
1414 case TargetOpcode::G_CTPOP:
1424 case TargetOpcode::G_INTTOPTR:
1432 case TargetOpcode::G_PTRTOINT:
1440 case TargetOpcode::G_PHI: {
1443 if (SizeOp0 % NarrowSize != 0)
1446 unsigned NumParts = SizeOp0 / NarrowSize;
1450 for (
unsigned i = 1; i <
MI.getNumOperands(); i += 2) {
1458 for (
unsigned i = 0; i < NumParts; ++i) {
1462 for (
unsigned j = 1; j <
MI.getNumOperands(); j += 2)
1463 MIB.
addUse(SrcRegs[j / 2][i]).
add(
MI.getOperand(j + 1));
1468 MI.eraseFromParent();
1471 case TargetOpcode::G_EXTRACT_VECTOR_ELT:
1472 case TargetOpcode::G_INSERT_VECTOR_ELT: {
1476 int OpIdx =
MI.getOpcode() == TargetOpcode::G_EXTRACT_VECTOR_ELT ? 2 : 3;
1482 case TargetOpcode::G_ICMP: {
1501 if (!
extractParts(
MI.getOperand(3).getReg(), SrcTy, NarrowTy, Unused,
1502 RHSPartRegs, RHSLeftoverRegs,
MIRBuilder, MRI))
1515 for (
auto LHSAndRHS :
zip(LHSPartRegs, RHSPartRegs)) {
1516 auto LHS = std::get<0>(LHSAndRHS);
1517 auto RHS = std::get<1>(LHSAndRHS);
1525 for (
auto LHSAndRHS :
zip(LHSLeftoverRegs, RHSLeftoverRegs)) {
1526 auto LHS = std::get<0>(LHSAndRHS);
1527 auto RHS = std::get<1>(LHSAndRHS);
1529 LLT GCDTy = extractGCDType(WidenedXors, NarrowTy, LeftoverTy,
Xor);
1530 buildLCMMergePieces(LeftoverTy, NarrowTy, GCDTy, WidenedXors,
1531 TargetOpcode::G_ZEXT);
1538 assert(Xors.
size() >= 2 &&
"Should have gotten at least two Xors?");
1540 for (
unsigned I = 2, E = Xors.
size();
I < E; ++
I)
1545 assert(LHSPartRegs.
size() == 2 &&
"Expected exactly 2 LHS part regs?");
1546 assert(RHSPartRegs.
size() == 2 &&
"Expected exactly 2 RHS part regs?");
1558 MI.eraseFromParent();
1561 case TargetOpcode::G_FCMP:
1570 case TargetOpcode::G_SEXT_INREG: {
1574 int64_t SizeInBits =
MI.getOperand(2).getImm();
1584 MO1.
setReg(TruncMIB.getReg(0));
1599 if (SizeOp0 % NarrowSize != 0)
1601 int NumParts = SizeOp0 / NarrowSize;
1609 for (
int i = 0; i < NumParts; ++i) {
1625 for (
int i = 0; i < NumParts; ++i) {
1628 PartialExtensionReg = DstRegs.
back();
1630 assert(PartialExtensionReg &&
1631 "Expected to visit partial extension before full");
1632 if (FullExtensionReg) {
1639 FullExtensionReg = DstRegs.
back();
1644 TargetOpcode::G_SEXT_INREG, {NarrowTy},
1647 PartialExtensionReg = DstRegs.
back();
1654 MI.eraseFromParent();
1657 case TargetOpcode::G_BSWAP:
1658 case TargetOpcode::G_BITREVERSE: {
1659 if (SizeOp0 % NarrowSize != 0)
1664 unsigned NumParts = SizeOp0 / NarrowSize;
1665 extractParts(
MI.getOperand(1).getReg(), NarrowTy, NumParts, SrcRegs,
1668 for (
unsigned i = 0; i < NumParts; ++i) {
1670 {SrcRegs[NumParts - 1 - i]});
1677 MI.eraseFromParent();
1680 case TargetOpcode::G_PTR_ADD:
1681 case TargetOpcode::G_PTRMASK: {
1689 case TargetOpcode::G_FPTOUI:
1690 case TargetOpcode::G_FPTOSI:
1692 case TargetOpcode::G_FPEXT:
1699 case TargetOpcode::G_FLDEXP:
1700 case TargetOpcode::G_STRICT_FLDEXP:
1702 case TargetOpcode::G_VSCALE: {
1713 MI.eraseFromParent();
1741 unsigned OpIdx,
unsigned ExtOpcode) {
1744 MO.
setReg(ExtB.getReg(0));
1751 MO.
setReg(ExtB.getReg(0));
1755 unsigned OpIdx,
unsigned TruncOpcode) {
1764 unsigned OpIdx,
unsigned ExtOpcode) {
1803LegalizerHelper::widenScalarMergeValues(
MachineInstr &
MI,
unsigned TypeIdx,
1808 auto [DstReg, DstTy, Src1Reg, Src1Ty] =
MI.getFirst2RegLLTs();
1809 if (DstTy.isVector())
1816 const int NumMerge = (DstSize + WideSize - 1) / WideSize;
1818 unsigned NumOps =
MI.getNumOperands();
1819 unsigned NumSrc =
MI.getNumOperands() - 1;
1820 unsigned PartSize = DstTy.getSizeInBits() / NumSrc;
1822 if (WideSize >= DstSize) {
1826 for (
unsigned I = 2;
I != NumOps; ++
I) {
1827 const unsigned Offset = (
I - 1) * PartSize;
1834 Register NextResult =
I + 1 == NumOps && WideTy == DstTy ? DstReg :
1840 ResultReg = NextResult;
1843 if (WideSize > DstSize)
1845 else if (DstTy.isPointer())
1848 MI.eraseFromParent();
1873 const int GCD = std::gcd(SrcSize, WideSize);
1884 if (GCD == SrcSize) {
1888 for (
int J = 0, JE = Unmerge->getNumOperands() - 1; J != JE; ++J)
1894 if (
static_cast<int>(Unmerges.
size()) != NumMerge * WideSize) {
1896 for (
int I = Unmerges.
size();
I != NumMerge * WideSize; ++
I)
1900 const int PartsPerGCD = WideSize / GCD;
1904 for (
int I = 0;
I != NumMerge; ++
I, Slicer = Slicer.drop_front(PartsPerGCD)) {
1919 MI.eraseFromParent();
1924LegalizerHelper::widenScalarUnmergeValues(
MachineInstr &
MI,
unsigned TypeIdx,
1929 int NumDst =
MI.getNumOperands() - 1;
1930 Register SrcReg =
MI.getOperand(NumDst).getReg();
1935 Register Dst0Reg =
MI.getOperand(0).getReg();
1945 dbgs() <<
"Not casting non-integral address space integer\n");
1966 for (
int I = 1;
I != NumDst; ++
I) {
1972 MI.eraseFromParent();
1983 LLVM_DEBUG(
dbgs() <<
"Widening pointer source types not implemented\n");
2008 const int NumUnmerge = Unmerge->getNumOperands() - 1;
2013 if (PartsPerRemerge == 1) {
2016 for (
int I = 0;
I != NumUnmerge; ++
I) {
2019 for (
int J = 0; J != PartsPerUnmerge; ++J) {
2020 int Idx =
I * PartsPerUnmerge + J;
2022 MIB.addDef(
MI.getOperand(
Idx).getReg());
2029 MIB.addUse(Unmerge.getReg(
I));
2033 for (
int J = 0; J != NumUnmerge; ++J)
2034 extractGCDType(Parts, GCDTy, Unmerge.getReg(J));
2037 for (
int I = 0;
I != NumDst; ++
I) {
2038 for (
int J = 0; J < PartsPerRemerge; ++J) {
2039 const int Idx =
I * PartsPerRemerge + J;
2044 RemergeParts.
clear();
2048 MI.eraseFromParent();
2053LegalizerHelper::widenScalarExtract(
MachineInstr &
MI,
unsigned TypeIdx,
2055 auto [DstReg, DstTy, SrcReg, SrcTy] =
MI.getFirst2RegLLTs();
2056 unsigned Offset =
MI.getOperand(2).getImm();
2059 if (SrcTy.
isVector() || DstTy.isVector())
2075 if (DstTy.isPointer())
2082 MI.eraseFromParent();
2087 LLT ShiftTy = SrcTy;
2096 MI.eraseFromParent();
2127LegalizerHelper::widenScalarInsert(
MachineInstr &
MI,
unsigned TypeIdx,
2129 if (TypeIdx != 0 || WideTy.
isVector())
2139LegalizerHelper::widenScalarAddSubOverflow(
MachineInstr &
MI,
unsigned TypeIdx,
2143 std::optional<Register> CarryIn;
2144 switch (
MI.getOpcode()) {
2147 case TargetOpcode::G_SADDO:
2148 Opcode = TargetOpcode::G_ADD;
2149 ExtOpcode = TargetOpcode::G_SEXT;
2151 case TargetOpcode::G_SSUBO:
2152 Opcode = TargetOpcode::G_SUB;
2153 ExtOpcode = TargetOpcode::G_SEXT;
2155 case TargetOpcode::G_UADDO:
2156 Opcode = TargetOpcode::G_ADD;
2157 ExtOpcode = TargetOpcode::G_ZEXT;
2159 case TargetOpcode::G_USUBO:
2160 Opcode = TargetOpcode::G_SUB;
2161 ExtOpcode = TargetOpcode::G_ZEXT;
2163 case TargetOpcode::G_SADDE:
2164 Opcode = TargetOpcode::G_UADDE;
2165 ExtOpcode = TargetOpcode::G_SEXT;
2166 CarryIn =
MI.getOperand(4).getReg();
2168 case TargetOpcode::G_SSUBE:
2169 Opcode = TargetOpcode::G_USUBE;
2170 ExtOpcode = TargetOpcode::G_SEXT;
2171 CarryIn =
MI.getOperand(4).getReg();
2173 case TargetOpcode::G_UADDE:
2174 Opcode = TargetOpcode::G_UADDE;
2175 ExtOpcode = TargetOpcode::G_ZEXT;
2176 CarryIn =
MI.getOperand(4).getReg();
2178 case TargetOpcode::G_USUBE:
2179 Opcode = TargetOpcode::G_USUBE;
2180 ExtOpcode = TargetOpcode::G_ZEXT;
2181 CarryIn =
MI.getOperand(4).getReg();
2202 LLT CarryOutTy = MRI.
getType(
MI.getOperand(1).getReg());
2205 {LHSExt, RHSExt, *CarryIn})
2217 MI.eraseFromParent();
2222LegalizerHelper::widenScalarAddSubShlSat(
MachineInstr &
MI,
unsigned TypeIdx,
2224 bool IsSigned =
MI.getOpcode() == TargetOpcode::G_SADDSAT ||
2225 MI.getOpcode() == TargetOpcode::G_SSUBSAT ||
2226 MI.getOpcode() == TargetOpcode::G_SSHLSAT;
2227 bool IsShift =
MI.getOpcode() == TargetOpcode::G_SSHLSAT ||
2228 MI.getOpcode() == TargetOpcode::G_USHLSAT;
2253 {ShiftL, ShiftR},
MI.getFlags());
2261 MI.eraseFromParent();
2266LegalizerHelper::widenScalarMulo(
MachineInstr &
MI,
unsigned TypeIdx,
2275 bool IsSigned =
MI.getOpcode() == TargetOpcode::G_SMULO;
2278 LLT OverflowTy = MRI.
getType(OriginalOverflow);
2285 unsigned ExtOp = IsSigned ? TargetOpcode::G_SEXT : TargetOpcode::G_ZEXT;
2294 WideMulCanOverflow ?
MI.getOpcode() : (
unsigned)TargetOpcode::G_MUL;
2297 if (WideMulCanOverflow)
2299 {LeftOperand, RightOperand});
2320 if (WideMulCanOverflow) {
2328 MI.eraseFromParent();
2334 switch (
MI.getOpcode()) {
2337 case TargetOpcode::G_ATOMICRMW_XCHG:
2338 case TargetOpcode::G_ATOMICRMW_ADD:
2339 case TargetOpcode::G_ATOMICRMW_SUB:
2340 case TargetOpcode::G_ATOMICRMW_AND:
2341 case TargetOpcode::G_ATOMICRMW_OR:
2342 case TargetOpcode::G_ATOMICRMW_XOR:
2343 case TargetOpcode::G_ATOMICRMW_MIN:
2344 case TargetOpcode::G_ATOMICRMW_MAX:
2345 case TargetOpcode::G_ATOMICRMW_UMIN:
2346 case TargetOpcode::G_ATOMICRMW_UMAX:
2347 assert(TypeIdx == 0 &&
"atomicrmw with second scalar type");
2353 case TargetOpcode::G_ATOMIC_CMPXCHG:
2354 assert(TypeIdx == 0 &&
"G_ATOMIC_CMPXCHG with second scalar type");
2361 case TargetOpcode::G_ATOMIC_CMPXCHG_WITH_SUCCESS:
2371 "G_ATOMIC_CMPXCHG_WITH_SUCCESS with third scalar type");
2376 case TargetOpcode::G_EXTRACT:
2377 return widenScalarExtract(
MI, TypeIdx, WideTy);
2378 case TargetOpcode::G_INSERT:
2379 return widenScalarInsert(
MI, TypeIdx, WideTy);
2380 case TargetOpcode::G_MERGE_VALUES:
2381 return widenScalarMergeValues(
MI, TypeIdx, WideTy);
2382 case TargetOpcode::G_UNMERGE_VALUES:
2383 return widenScalarUnmergeValues(
MI, TypeIdx, WideTy);
2384 case TargetOpcode::G_SADDO:
2385 case TargetOpcode::G_SSUBO:
2386 case TargetOpcode::G_UADDO:
2387 case TargetOpcode::G_USUBO:
2388 case TargetOpcode::G_SADDE:
2389 case TargetOpcode::G_SSUBE:
2390 case TargetOpcode::G_UADDE:
2391 case TargetOpcode::G_USUBE:
2392 return widenScalarAddSubOverflow(
MI, TypeIdx, WideTy);
2393 case TargetOpcode::G_UMULO:
2394 case TargetOpcode::G_SMULO:
2395 return widenScalarMulo(
MI, TypeIdx, WideTy);
2396 case TargetOpcode::G_SADDSAT:
2397 case TargetOpcode::G_SSUBSAT:
2398 case TargetOpcode::G_SSHLSAT:
2399 case TargetOpcode::G_UADDSAT:
2400 case TargetOpcode::G_USUBSAT:
2401 case TargetOpcode::G_USHLSAT:
2402 return widenScalarAddSubShlSat(
MI, TypeIdx, WideTy);
2403 case TargetOpcode::G_CTTZ:
2404 case TargetOpcode::G_CTTZ_ZERO_UNDEF:
2405 case TargetOpcode::G_CTLZ:
2406 case TargetOpcode::G_CTLZ_ZERO_UNDEF:
2407 case TargetOpcode::G_CTPOP: {
2418 unsigned ExtOpc =
MI.getOpcode() == TargetOpcode::G_CTTZ ||
2419 MI.getOpcode() == TargetOpcode::G_CTTZ_ZERO_UNDEF
2420 ? TargetOpcode::G_ANYEXT
2421 : TargetOpcode::G_ZEXT;
2424 unsigned NewOpc =
MI.getOpcode();
2425 if (NewOpc == TargetOpcode::G_CTTZ) {
2434 NewOpc = TargetOpcode::G_CTTZ_ZERO_UNDEF;
2440 if (
MI.getOpcode() == TargetOpcode::G_CTLZ ||
2441 MI.getOpcode() == TargetOpcode::G_CTLZ_ZERO_UNDEF) {
2449 MI.eraseFromParent();
2452 case TargetOpcode::G_BSWAP: {
2461 MI.getOperand(0).setReg(DstExt);
2474 case TargetOpcode::G_BITREVERSE: {
2483 MI.getOperand(0).setReg(DstExt);
2492 case TargetOpcode::G_FREEZE:
2499 case TargetOpcode::G_ABS:
2506 case TargetOpcode::G_ADD:
2507 case TargetOpcode::G_AND:
2508 case TargetOpcode::G_MUL:
2509 case TargetOpcode::G_OR:
2510 case TargetOpcode::G_XOR:
2511 case TargetOpcode::G_SUB:
2512 case TargetOpcode::G_SHUFFLE_VECTOR:
2523 case TargetOpcode::G_SBFX:
2524 case TargetOpcode::G_UBFX:
2538 case TargetOpcode::G_SHL:
2554 case TargetOpcode::G_ROTR:
2555 case TargetOpcode::G_ROTL:
2564 case TargetOpcode::G_SDIV:
2565 case TargetOpcode::G_SREM:
2566 case TargetOpcode::G_SMIN:
2567 case TargetOpcode::G_SMAX:
2575 case TargetOpcode::G_SDIVREM:
2584 case TargetOpcode::G_ASHR:
2585 case TargetOpcode::G_LSHR:
2589 unsigned CvtOp =
MI.getOpcode() == TargetOpcode::G_ASHR ?
2590 TargetOpcode::G_SEXT : TargetOpcode::G_ZEXT;
2603 case TargetOpcode::G_UDIV:
2604 case TargetOpcode::G_UREM:
2605 case TargetOpcode::G_UMIN:
2606 case TargetOpcode::G_UMAX:
2614 case TargetOpcode::G_UDIVREM:
2623 case TargetOpcode::G_SELECT:
2640 case TargetOpcode::G_FPTOSI:
2641 case TargetOpcode::G_FPTOUI:
2642 case TargetOpcode::G_IS_FPCLASS:
2652 case TargetOpcode::G_SITOFP:
2662 case TargetOpcode::G_UITOFP:
2672 case TargetOpcode::G_LOAD:
2673 case TargetOpcode::G_SEXTLOAD:
2674 case TargetOpcode::G_ZEXTLOAD:
2680 case TargetOpcode::G_STORE: {
2691 TargetOpcode::G_ZEXT : TargetOpcode::G_ANYEXT;
2697 case TargetOpcode::G_CONSTANT: {
2701 MRI.
getType(
MI.getOperand(0).getReg()));
2702 assert((ExtOpc == TargetOpcode::G_ZEXT || ExtOpc == TargetOpcode::G_SEXT ||
2703 ExtOpc == TargetOpcode::G_ANYEXT) &&
2706 const APInt &Val = (ExtOpc == TargetOpcode::G_SEXT)
2710 SrcMO.
setCImm(ConstantInt::get(Ctx, Val));
2716 case TargetOpcode::G_FCONSTANT: {
2724 MI.eraseFromParent();
2727 case TargetOpcode::G_IMPLICIT_DEF: {
2733 case TargetOpcode::G_BRCOND:
2739 case TargetOpcode::G_FCMP:
2750 case TargetOpcode::G_ICMP:
2756 MI.getOperand(1).getPredicate()))
2757 ? TargetOpcode::G_SEXT
2758 : TargetOpcode::G_ZEXT;
2765 case TargetOpcode::G_PTR_ADD:
2766 assert(TypeIdx == 1 &&
"unable to legalize pointer of G_PTR_ADD");
2772 case TargetOpcode::G_PHI: {
2773 assert(TypeIdx == 0 &&
"Expecting only Idx 0");
2776 for (
unsigned I = 1;
I <
MI.getNumOperands();
I += 2) {
2788 case TargetOpcode::G_EXTRACT_VECTOR_ELT: {
2796 TargetOpcode::G_ANYEXT);
2811 case TargetOpcode::G_INSERT_VECTOR_ELT: {
2847 case TargetOpcode::G_FADD:
2848 case TargetOpcode::G_FMUL:
2849 case TargetOpcode::G_FSUB:
2850 case TargetOpcode::G_FMA:
2851 case TargetOpcode::G_FMAD:
2852 case TargetOpcode::G_FNEG:
2853 case TargetOpcode::G_FABS:
2854 case TargetOpcode::G_FCANONICALIZE:
2855 case TargetOpcode::G_FMINNUM:
2856 case TargetOpcode::G_FMAXNUM:
2857 case TargetOpcode::G_FMINNUM_IEEE:
2858 case TargetOpcode::G_FMAXNUM_IEEE:
2859 case TargetOpcode::G_FMINIMUM:
2860 case TargetOpcode::G_FMAXIMUM:
2861 case TargetOpcode::G_FDIV:
2862 case TargetOpcode::G_FREM:
2863 case TargetOpcode::G_FCEIL:
2864 case TargetOpcode::G_FFLOOR:
2865 case TargetOpcode::G_FCOS:
2866 case TargetOpcode::G_FSIN:
2867 case TargetOpcode::G_FLOG10:
2868 case TargetOpcode::G_FLOG:
2869 case TargetOpcode::G_FLOG2:
2870 case TargetOpcode::G_FRINT:
2871 case TargetOpcode::G_FNEARBYINT:
2872 case TargetOpcode::G_FSQRT:
2873 case TargetOpcode::G_FEXP:
2874 case TargetOpcode::G_FEXP2:
2875 case TargetOpcode::G_FEXP10:
2876 case TargetOpcode::G_FPOW:
2877 case TargetOpcode::G_INTRINSIC_TRUNC:
2878 case TargetOpcode::G_INTRINSIC_ROUND:
2879 case TargetOpcode::G_INTRINSIC_ROUNDEVEN:
2883 for (
unsigned I = 1, E =
MI.getNumOperands();
I != E; ++
I)
2889 case TargetOpcode::G_FPOWI:
2890 case TargetOpcode::G_FLDEXP:
2891 case TargetOpcode::G_STRICT_FLDEXP: {
2893 if (
MI.getOpcode() == TargetOpcode::G_STRICT_FLDEXP)
2914 case TargetOpcode::G_FFREXP: {
2927 case TargetOpcode::G_INTTOPTR:
2935 case TargetOpcode::G_PTRTOINT:
2943 case TargetOpcode::G_BUILD_VECTOR: {
2947 for (
int I = 1, E =
MI.getNumOperands();
I != E; ++
I)
2961 case TargetOpcode::G_SEXT_INREG:
2970 case TargetOpcode::G_PTRMASK: {
2978 case TargetOpcode::G_VECREDUCE_FADD:
2979 case TargetOpcode::G_VECREDUCE_FMUL:
2980 case TargetOpcode::G_VECREDUCE_FMIN:
2981 case TargetOpcode::G_VECREDUCE_FMAX:
2982 case TargetOpcode::G_VECREDUCE_FMINIMUM:
2983 case TargetOpcode::G_VECREDUCE_FMAXIMUM: {
2997 case TargetOpcode::G_VSCALE: {
3004 SrcMO.
setCImm(ConstantInt::get(Ctx, Val));
3014 auto Unmerge =
B.buildUnmerge(Ty, Src);
3015 for (
int I = 0, E = Unmerge->getNumOperands() - 1;
I != E; ++
I)
3024 unsigned AddrSpace =
DL.getDefaultGlobalsAddressSpace();
3026 LLT DstLLT =
MRI.getType(DstReg);
3047 MI.eraseFromParent();
3058 MI.eraseFromParent();
3065 auto [Dst, DstTy, Src, SrcTy] =
MI.getFirst2RegLLTs();
3070 if (DstTy.isVector()) {
3071 int NumDstElt = DstTy.getNumElements();
3075 LLT DstCastTy = DstEltTy;
3076 LLT SrcPartTy = SrcEltTy;
3080 if (NumSrcElt < NumDstElt) {
3090 SrcPartTy = SrcEltTy;
3091 }
else if (NumSrcElt > NumDstElt) {
3102 DstCastTy = DstEltTy;
3112 MI.eraseFromParent();
3116 if (DstTy.isVector()) {
3120 MI.eraseFromParent();
3136 unsigned NewEltSize,
3137 unsigned OldEltSize) {
3138 const unsigned Log2EltRatio =
Log2_32(NewEltSize / OldEltSize);
3139 LLT IdxTy =
B.getMRI()->getType(
Idx);
3142 auto OffsetMask =
B.buildConstant(
3144 auto OffsetIdx =
B.buildAnd(IdxTy,
Idx, OffsetMask);
3145 return B.buildShl(IdxTy, OffsetIdx,
3146 B.buildConstant(IdxTy,
Log2_32(OldEltSize))).getReg(0);
3161 auto [Dst, DstTy, SrcVec, SrcVecTy,
Idx, IdxTy] =
MI.getFirst3RegLLTs();
3165 unsigned OldNumElts = SrcVecTy.getNumElements();
3172 if (NewNumElts > OldNumElts) {
3183 if (NewNumElts % OldNumElts != 0)
3187 const unsigned NewEltsPerOldElt = NewNumElts / OldNumElts;
3196 for (
unsigned I = 0;
I < NewEltsPerOldElt; ++
I) {
3200 NewOps[
I] = Elt.getReg(0);
3205 MI.eraseFromParent();
3209 if (NewNumElts < OldNumElts) {
3210 if (NewEltSize % OldEltSize != 0)
3232 const unsigned Log2EltRatio =
Log2_32(NewEltSize / OldEltSize);
3251 MI.eraseFromParent();
3265 LLT TargetTy =
B.getMRI()->getType(TargetReg);
3266 LLT InsertTy =
B.getMRI()->getType(InsertReg);
3267 auto ZextVal =
B.buildZExt(TargetTy, InsertReg);
3268 auto ShiftedInsertVal =
B.buildShl(TargetTy, ZextVal, OffsetBits);
3271 auto EltMask =
B.buildConstant(
3275 auto ShiftedMask =
B.buildShl(TargetTy, EltMask, OffsetBits);
3276 auto InvShiftedMask =
B.buildNot(TargetTy, ShiftedMask);
3279 auto MaskedOldElt =
B.buildAnd(TargetTy, TargetReg, InvShiftedMask);
3283 return B.buildOr(TargetTy, MaskedOldElt, ShiftedInsertVal).getReg(0);
3297 auto [Dst, DstTy, SrcVec, SrcVecTy, Val, ValTy,
Idx, IdxTy] =
3298 MI.getFirst4RegLLTs();
3310 if (NewNumElts < OldNumElts) {
3311 if (NewEltSize % OldEltSize != 0)
3320 const unsigned Log2EltRatio =
Log2_32(NewEltSize / OldEltSize);
3340 CastTy, CastVec, InsertedElt, ScaledIdx).
getReg(0);
3344 MI.eraseFromParent();
3363 if (MemSizeInBits != MemStoreSizeInBits) {
3383 if (isa<GSExtLoad>(LoadMI)) {
3386 }
else if (isa<GZExtLoad>(LoadMI) || WideMemTy == LoadTy) {
3395 if (DstTy != LoadTy)
3421 uint64_t LargeSplitSize, SmallSplitSize;
3426 SmallSplitSize = MemSizeInBits - LargeSplitSize;
3436 SmallSplitSize = LargeSplitSize = MemSizeInBits / 2;
3461 LargeSplitSize / 8);
3465 SmallPtr, *SmallMMO);
3470 if (AnyExtTy == DstTy)
3505 if (StoreWidth != StoreSizeInBits) {
3540 uint64_t LargeSplitSize, SmallSplitSize;
3543 LargeSplitSize = llvm::bit_floor<uint64_t>(MemTy.
getSizeInBits());
3550 SmallSplitSize = LargeSplitSize = MemSizeInBits / 2;
3589 switch (
MI.getOpcode()) {
3590 case TargetOpcode::G_LOAD: {
3605 case TargetOpcode::G_STORE: {
3621 case TargetOpcode::G_SELECT: {
3627 dbgs() <<
"bitcast action not implemented for vector select\n");
3638 case TargetOpcode::G_AND:
3639 case TargetOpcode::G_OR:
3640 case TargetOpcode::G_XOR: {
3648 case TargetOpcode::G_EXTRACT_VECTOR_ELT:
3650 case TargetOpcode::G_INSERT_VECTOR_ELT:
3658void LegalizerHelper::changeOpcode(
MachineInstr &
MI,
unsigned NewOpcode) {
3666 using namespace TargetOpcode;
3668 switch(
MI.getOpcode()) {
3671 case TargetOpcode::G_FCONSTANT:
3673 case TargetOpcode::G_BITCAST:
3675 case TargetOpcode::G_SREM:
3676 case TargetOpcode::G_UREM: {
3680 {MI.getOperand(1), MI.getOperand(2)});
3684 MI.eraseFromParent();
3687 case TargetOpcode::G_SADDO:
3688 case TargetOpcode::G_SSUBO:
3690 case TargetOpcode::G_UMULH:
3691 case TargetOpcode::G_SMULH:
3693 case TargetOpcode::G_SMULO:
3694 case TargetOpcode::G_UMULO: {
3697 auto [Res, Overflow,
LHS,
RHS] =
MI.getFirst4Regs();
3700 unsigned Opcode =
MI.getOpcode() == TargetOpcode::G_SMULO
3701 ? TargetOpcode::G_SMULH
3702 : TargetOpcode::G_UMULH;
3706 MI.setDesc(
TII.get(TargetOpcode::G_MUL));
3707 MI.removeOperand(1);
3718 if (Opcode == TargetOpcode::G_SMULH) {
3727 case TargetOpcode::G_FNEG: {
3728 auto [Res, SubByReg] =
MI.getFirst2Regs();
3738 MI.eraseFromParent();
3741 case TargetOpcode::G_FSUB:
3742 case TargetOpcode::G_STRICT_FSUB: {
3743 auto [Res,
LHS,
RHS] =
MI.getFirst3Regs();
3749 if (
MI.getOpcode() == TargetOpcode::G_STRICT_FSUB)
3754 MI.eraseFromParent();
3757 case TargetOpcode::G_FMAD:
3759 case TargetOpcode::G_FFLOOR:
3761 case TargetOpcode::G_INTRINSIC_ROUND:
3763 case TargetOpcode::G_FRINT: {
3766 changeOpcode(
MI, TargetOpcode::G_INTRINSIC_ROUNDEVEN);
3769 case TargetOpcode::G_ATOMIC_CMPXCHG_WITH_SUCCESS: {
3770 auto [OldValRes, SuccessRes,
Addr, CmpVal, NewVal] =
MI.getFirst5Regs();
3772 **
MI.memoperands_begin());
3774 MI.eraseFromParent();
3777 case TargetOpcode::G_LOAD:
3778 case TargetOpcode::G_SEXTLOAD:
3779 case TargetOpcode::G_ZEXTLOAD:
3781 case TargetOpcode::G_STORE:
3783 case TargetOpcode::G_CTLZ_ZERO_UNDEF:
3784 case TargetOpcode::G_CTTZ_ZERO_UNDEF:
3785 case TargetOpcode::G_CTLZ:
3786 case TargetOpcode::G_CTTZ:
3787 case TargetOpcode::G_CTPOP:
3790 auto [Res, CarryOut,
LHS,
RHS] =
MI.getFirst4Regs();
3795 MI.eraseFromParent();
3799 auto [Res, CarryOut,
LHS,
RHS, CarryIn] =
MI.getFirst5Regs();
3820 MI.eraseFromParent();
3824 auto [Res, BorrowOut,
LHS,
RHS] =
MI.getFirst4Regs();
3829 MI.eraseFromParent();
3833 auto [Res, BorrowOut,
LHS,
RHS, BorrowIn] =
MI.getFirst5Regs();
3855 MI.eraseFromParent();
3880 case G_MERGE_VALUES:
3882 case G_UNMERGE_VALUES:
3884 case TargetOpcode::G_SEXT_INREG: {
3885 assert(
MI.getOperand(2).isImm() &&
"Expected immediate");
3886 int64_t SizeInBits =
MI.getOperand(2).getImm();
3888 auto [DstReg, SrcReg] =
MI.getFirst2Regs();
3895 MI.eraseFromParent();
3898 case G_EXTRACT_VECTOR_ELT:
3899 case G_INSERT_VECTOR_ELT:
3901 case G_SHUFFLE_VECTOR:
3903 case G_DYN_STACKALLOC:
3907 case G_STACKRESTORE:
3917 case G_READ_REGISTER:
3918 case G_WRITE_REGISTER:
3963 case G_MEMCPY_INLINE:
3964 return lowerMemcpyInline(
MI);
3995 unsigned AddrSpace =
DL.getAllocaAddrSpace();
4004 LLT IdxTy =
B.getMRI()->getType(IdxReg);
4016 return B.buildAnd(IdxTy, IdxReg,
B.buildConstant(IdxTy, Imm)).getReg(0);
4019 return B.buildUMin(IdxTy, IdxReg,
B.buildConstant(IdxTy, NElts - 1))
4030 "Converting bits to bytes lost precision");
4037 unsigned IndexSizeInBits =
DL.getIndexSize(AS) * 8;
4054 std::initializer_list<unsigned> NonVecOpIndices) {
4055 if (
MI.getNumMemOperands() != 0)
4058 LLT VecTy =
MRI.getType(
MI.getReg(0));
4063 for (
unsigned OpIdx = 1; OpIdx <
MI.getNumOperands(); ++OpIdx) {
4096 int NumParts, NumLeftover;
4097 std::tie(NumParts, NumLeftover) =
4100 assert(NumParts > 0 &&
"Error in getNarrowTypeBreakDown");
4101 for (
int i = 0; i < NumParts; ++i) {
4106 assert(NumLeftover == 1 &&
"expected exactly one leftover");
4115 for (
unsigned i = 0; i <
N; ++i) {
4118 else if (
Op.isImm())
4120 else if (
Op.isPredicate())
4142 std::initializer_list<unsigned> NonVecOpIndices) {
4144 "Non-compatible opcode or not specified non-vector operands");
4147 unsigned NumInputs =
MI.getNumOperands() -
MI.getNumDefs();
4148 unsigned NumDefs =
MI.getNumDefs();
4156 for (
unsigned i = 0; i < NumDefs; ++i) {
4165 for (
unsigned UseIdx = NumDefs, UseNo = 0; UseIdx <
MI.getNumOperands();
4166 ++UseIdx, ++UseNo) {
4169 MI.getOperand(UseIdx));
4174 for (
auto Reg : SplitPieces)
4179 unsigned NumLeftovers = OrigNumElts % NumElts ? 1 : 0;
4183 for (
unsigned i = 0; i < OrigNumElts / NumElts + NumLeftovers; ++i) {
4185 for (
unsigned DstNo = 0; DstNo < NumDefs; ++DstNo)
4186 Defs.
push_back(OutputOpsPieces[DstNo][i]);
4189 for (
unsigned InputNo = 0; InputNo < NumInputs; ++InputNo)
4190 Uses.push_back(InputOpsPieces[InputNo][i]);
4193 for (
unsigned DstNo = 0; DstNo < NumDefs; ++DstNo)
4194 OutputRegs[DstNo].push_back(
I.getReg(DstNo));
4199 for (
unsigned i = 0; i < NumDefs; ++i)
4200 mergeMixedSubvectors(
MI.getReg(i), OutputRegs[i]);
4202 for (
unsigned i = 0; i < NumDefs; ++i)
4206 MI.eraseFromParent();
4215 unsigned NumInputs =
MI.getNumOperands() -
MI.getNumDefs();
4216 unsigned NumDefs =
MI.getNumDefs();
4225 for (
unsigned UseIdx = NumDefs, UseNo = 0; UseIdx <
MI.getNumOperands();
4226 UseIdx += 2, ++UseNo) {
4234 unsigned NumLeftovers = OrigNumElts % NumElts ? 1 : 0;
4236 for (
unsigned i = 0; i < OrigNumElts / NumElts + NumLeftovers; ++i) {
4242 for (
unsigned j = 0; j < NumInputs / 2; ++j) {
4243 Phi.addUse(InputOpsPieces[j][i]);
4244 Phi.add(
MI.getOperand(1 + j * 2 + 1));
4250 mergeMixedSubvectors(
MI.getReg(0), OutputRegs);
4255 MI.eraseFromParent();
4263 const int NumDst =
MI.getNumOperands() - 1;
4264 const Register SrcReg =
MI.getOperand(NumDst).getReg();
4268 if (TypeIdx != 1 || NarrowTy == DstTy)
4294 const int PartsPerUnmerge = NumDst / NumUnmerge;
4296 for (
int I = 0;
I != NumUnmerge; ++
I) {
4299 for (
int J = 0; J != PartsPerUnmerge; ++J)
4300 MIB.
addDef(
MI.getOperand(
I * PartsPerUnmerge + J).getReg());
4301 MIB.
addUse(Unmerge.getReg(
I));
4304 MI.eraseFromParent();
4311 auto [DstReg, DstTy, SrcReg, SrcTy] =
MI.getFirst2RegLLTs();
4315 assert(DstTy.isVector() && NarrowTy.
isVector() &&
"Expected vector types");
4317 if (NarrowTy == SrcTy)
4327 if ((DstTy.getSizeInBits() % NarrowTy.
getSizeInBits() != 0) ||
4341 for (
unsigned i = 1; i <
MI.getNumOperands(); ++i) {
4343 for (
unsigned j = 0; j < Unmerge->getNumDefs(); ++j)
4349 unsigned NumNarrowTyPieces = DstTy.getNumElements() / NumNarrowTyElts;
4350 for (
unsigned i = 0,
Offset = 0; i < NumNarrowTyPieces;
4351 ++i,
Offset += NumNarrowTyElts) {
4358 MI.eraseFromParent();
4362 assert(TypeIdx == 0 &&
"Bad type index");
4378 unsigned NumParts = DstTy.getNumElements() / NarrowTy.
getNumElements();
4381 for (
unsigned i = 0; i < NumParts; ++i) {
4383 for (
unsigned j = 0; j < NumElts; ++j)
4384 Sources.
push_back(
MI.getOperand(1 + i * NumElts + j).getReg());
4390 MI.eraseFromParent();
4398 auto [DstReg, SrcVec] =
MI.getFirst2Regs();
4400 bool IsInsert =
MI.getOpcode() == TargetOpcode::G_INSERT_VECTOR_ELT;
4402 assert((IsInsert ? TypeIdx == 0 : TypeIdx == 1) &&
"not a vector type index");
4404 InsertVal =
MI.getOperand(2).getReg();
4419 IdxVal = MaybeCst->Value.getSExtValue();
4423 MI.eraseFromParent();
4428 LLT GCDTy = extractGCDType(VecParts, VecTy, NarrowVecTy, SrcVec);
4431 LLT LCMTy = buildLCMMergePieces(VecTy, NarrowVecTy, GCDTy, VecParts,
4432 TargetOpcode::G_ANYEXT);
4437 int64_t PartIdx = IdxVal / NewNumElts;
4446 PartTy, VecParts[PartIdx], InsertVal, NewIdx);
4447 VecParts[PartIdx] = InsertPart.getReg(0);
4451 buildWidenedRemergeToDst(DstReg, LCMTy, VecParts);
4456 MI.eraseFromParent();
4480 bool IsLoad = isa<GLoad>(LdStMI);
4492 int NumLeftover = -1;
4498 if (
extractParts(ValReg, ValTy, NarrowTy, LeftoverTy, NarrowRegs,
4500 NumParts = NarrowRegs.
size();
4501 NumLeftover = NarrowLeftoverRegs.
size();
4518 auto MMO = LdStMI.
getMMO();
4520 unsigned NumParts,
unsigned Offset) ->
unsigned {
4523 for (
unsigned Idx = 0, E = NumParts;
Idx != E &&
Offset < TotalSize;
4525 unsigned ByteOffset =
Offset / 8;
4535 ValRegs.push_back(Dst);
4547 unsigned HandledOffset =
4548 splitTypePieces(NarrowTy, NarrowRegs, NumParts,
Offset);
4552 splitTypePieces(LeftoverTy, NarrowLeftoverRegs, NumLeftover, HandledOffset);
4555 insertParts(ValReg, ValTy, NarrowTy, NarrowRegs,
4556 LeftoverTy, NarrowLeftoverRegs);
4566 using namespace TargetOpcode;
4570 switch (
MI.getOpcode()) {
4571 case G_IMPLICIT_DEF:
4587 case G_FCANONICALIZE:
4604 case G_INTRINSIC_ROUND:
4605 case G_INTRINSIC_ROUNDEVEN:
4606 case G_INTRINSIC_TRUNC:
4625 case G_FMINNUM_IEEE:
4626 case G_FMAXNUM_IEEE:
4646 case G_CTLZ_ZERO_UNDEF:
4648 case G_CTTZ_ZERO_UNDEF:
4662 case G_ADDRSPACE_CAST:
4675 case G_STRICT_FLDEXP:
4689 case G_UNMERGE_VALUES:
4691 case G_BUILD_VECTOR:
4692 assert(TypeIdx == 0 &&
"not a vector type index");
4694 case G_CONCAT_VECTORS:
4698 case G_EXTRACT_VECTOR_ELT:
4699 case G_INSERT_VECTOR_ELT:
4708 case TargetOpcode::G_VECREDUCE_SEQ_FADD:
4709 case TargetOpcode::G_VECREDUCE_SEQ_FMUL:
4711 case G_SHUFFLE_VECTOR:
4725 assert(
MI.getOpcode() == TargetOpcode::G_BITCAST &&
4726 "Not a bitcast operation");
4731 auto [DstReg, DstTy, SrcReg, SrcTy] =
MI.getFirst2RegLLTs();
4739 if (extractGCDType(SrcVRegs, DstTy, SrcNarrowTy, SrcReg) != SrcNarrowTy)
4744 for (
unsigned i = 0; i < SrcVRegs.
size(); i++)
4749 MI.eraseFromParent();
4755 assert(
MI.getOpcode() == TargetOpcode::G_SHUFFLE_VECTOR);
4759 auto [DstReg, DstTy, Src1Reg, Src1Ty, Src2Reg, Src2Ty] =
4760 MI.getFirst3RegLLTs();
4763 if (DstTy != Src1Ty)
4765 if (DstTy != Src2Ty)
4780 Register Inputs[4] = {SplitSrc1Regs[0], SplitSrc1Regs[1], SplitSrc2Regs[0],
4796 unsigned InputUsed[2] = {-1U, -1U};
4797 unsigned FirstMaskIdx =
High * NewElts;
4798 bool UseBuildVector =
false;
4799 for (
unsigned MaskOffset = 0; MaskOffset < NewElts; ++MaskOffset) {
4801 int Idx = Mask[FirstMaskIdx + MaskOffset];
4806 if (Input >= std::size(Inputs)) {
4813 Idx -= Input * NewElts;
4817 for (OpNo = 0; OpNo < std::size(InputUsed); ++OpNo) {
4818 if (InputUsed[OpNo] == Input) {
4821 }
else if (InputUsed[OpNo] == -1U) {
4823 InputUsed[OpNo] = Input;
4828 if (OpNo >= std::size(InputUsed)) {
4831 UseBuildVector =
true;
4839 if (UseBuildVector) {
4844 for (
unsigned MaskOffset = 0; MaskOffset < NewElts; ++MaskOffset) {
4846 int Idx = Mask[FirstMaskIdx + MaskOffset];
4851 if (Input >= std::size(Inputs)) {
4858 Idx -= Input * NewElts;
4862 .buildExtractVectorElement(
4863 EltTy, Inputs[Input],
4870 }
else if (InputUsed[0] == -1U) {
4874 Register Op0 = Inputs[InputUsed[0]];
4878 : Inputs[InputUsed[1]];
4887 MI.eraseFromParent();
4893 auto &RdxMI = cast<GVecReduce>(
MI);
4900 auto [DstReg, DstTy, SrcReg, SrcTy] = RdxMI.getFirst2RegLLTs();
4906 unsigned ScalarOpc = RdxMI.getScalarOpcForReduction();
4909 const unsigned NumParts =
4915 if (DstTy != NarrowTy)
4921 unsigned NumPartsLeft = NumParts;
4922 while (NumPartsLeft > 1) {
4923 for (
unsigned Idx = 0;
Idx < NumPartsLeft - 1;
Idx += 2) {
4926 .buildInstr(ScalarOpc, {NarrowTy},
4927 {SplitSrcs[
Idx], SplitSrcs[
Idx + 1]})
4930 SplitSrcs = PartialResults;
4931 PartialResults.
clear();
4932 NumPartsLeft = SplitSrcs.
size();
4936 MI.eraseFromParent();
4941 for (
unsigned Idx = 1;
Idx < NumParts; ++
Idx)
4945 MI.eraseFromParent();
4949 for (
unsigned Part = 0; Part < NumParts; ++Part) {
4959 return tryNarrowPow2Reduction(
MI, SrcReg, SrcTy, NarrowTy, ScalarOpc);
4962 Register Acc = PartialReductions[0];
4963 for (
unsigned Part = 1; Part < NumParts; ++Part) {
4964 if (Part == NumParts - 1) {
4966 {Acc, PartialReductions[Part]});
4969 .
buildInstr(ScalarOpc, {DstTy}, {Acc, PartialReductions[Part]})
4973 MI.eraseFromParent();
4979 unsigned int TypeIdx,
4981 auto [DstReg, DstTy, ScalarReg, ScalarTy, SrcReg, SrcTy] =
4982 MI.getFirst3RegLLTs();
4983 if (!NarrowTy.
isScalar() || TypeIdx != 2 || DstTy != ScalarTy ||
4987 assert((
MI.getOpcode() == TargetOpcode::G_VECREDUCE_SEQ_FADD ||
4988 MI.getOpcode() == TargetOpcode::G_VECREDUCE_SEQ_FMUL) &&
4989 "Unexpected vecreduce opcode");
4990 unsigned ScalarOpc =
MI.getOpcode() == TargetOpcode::G_VECREDUCE_SEQ_FADD
4991 ? TargetOpcode::G_FADD
4992 : TargetOpcode::G_FMUL;
4998 for (
unsigned i = 0; i < NumParts; i++)
5003 MI.eraseFromParent();
5010 unsigned ScalarOpc) {
5018 while (SplitSrcs.
size() > 1) {
5020 for (
unsigned Idx = 0;
Idx < SplitSrcs.
size()-1;
Idx += 2) {
5028 SplitSrcs = std::move(PartialRdxs);
5032 MI.getOperand(1).setReg(SplitSrcs[0]);
5039 const LLT HalfTy,
const LLT AmtTy) {
5047 MI.eraseFromParent();
5053 unsigned VTBits = 2 * NVTBits;
5056 if (
MI.getOpcode() == TargetOpcode::G_SHL) {
5057 if (Amt.
ugt(VTBits)) {
5059 }
else if (Amt.
ugt(NVTBits)) {
5063 }
else if (Amt == NVTBits) {
5074 }
else if (
MI.getOpcode() == TargetOpcode::G_LSHR) {
5075 if (Amt.
ugt(VTBits)) {
5077 }
else if (Amt.
ugt(NVTBits)) {
5081 }
else if (Amt == NVTBits) {
5095 if (Amt.
ugt(VTBits)) {
5098 }
else if (Amt.
ugt(NVTBits)) {
5103 }
else if (Amt == NVTBits) {
5120 MI.eraseFromParent();
5144 if (DstEltSize % 2 != 0)
5150 const unsigned NewBitSize = DstEltSize / 2;
5176 switch (
MI.getOpcode()) {
5177 case TargetOpcode::G_SHL: {
5193 ResultRegs[0] =
Lo.getReg(0);
5194 ResultRegs[1] =
Hi.getReg(0);
5197 case TargetOpcode::G_LSHR:
5198 case TargetOpcode::G_ASHR: {
5208 if (
MI.getOpcode() == TargetOpcode::G_LSHR) {
5222 ResultRegs[0] =
Lo.getReg(0);
5223 ResultRegs[1] =
Hi.getReg(0);
5231 MI.eraseFromParent();
5238 assert(TypeIdx == 0 &&
"Expecting only Idx 0");
5241 for (
unsigned I = 1, E =
MI.getNumOperands();
I != E;
I += 2) {
5256 assert(Ty.
isScalar() &&
"Expected scalar type to make neutral element for");
5261 "getNeutralElementForVecReduce called with invalid opcode!");
5262 case TargetOpcode::G_VECREDUCE_ADD:
5263 case TargetOpcode::G_VECREDUCE_OR:
5264 case TargetOpcode::G_VECREDUCE_XOR:
5265 case TargetOpcode::G_VECREDUCE_UMAX:
5267 case TargetOpcode::G_VECREDUCE_MUL:
5269 case TargetOpcode::G_VECREDUCE_AND:
5270 case TargetOpcode::G_VECREDUCE_UMIN:
5273 case TargetOpcode::G_VECREDUCE_SMAX:
5276 case TargetOpcode::G_VECREDUCE_SMIN:
5279 case TargetOpcode::G_VECREDUCE_FADD:
5281 case TargetOpcode::G_VECREDUCE_FMUL:
5283 case TargetOpcode::G_VECREDUCE_FMINIMUM:
5284 case TargetOpcode::G_VECREDUCE_FMAXIMUM:
5285 assert(
false &&
"getNeutralElementForVecReduce unimplemented for "
5286 "G_VECREDUCE_FMINIMUM and G_VECREDUCE_FMAXIMUM!");
5294 unsigned Opc =
MI.getOpcode();
5296 case TargetOpcode::G_IMPLICIT_DEF:
5297 case TargetOpcode::G_LOAD: {
5305 case TargetOpcode::G_STORE:
5312 case TargetOpcode::G_AND:
5313 case TargetOpcode::G_OR:
5314 case TargetOpcode::G_XOR:
5315 case TargetOpcode::G_ADD:
5316 case TargetOpcode::G_SUB:
5317 case TargetOpcode::G_MUL:
5318 case TargetOpcode::G_FADD:
5319 case TargetOpcode::G_FSUB:
5320 case TargetOpcode::G_FMUL:
5321 case TargetOpcode::G_FDIV:
5322 case TargetOpcode::G_FCOPYSIGN:
5323 case TargetOpcode::G_UADDSAT:
5324 case TargetOpcode::G_USUBSAT:
5325 case TargetOpcode::G_SADDSAT:
5326 case TargetOpcode::G_SSUBSAT:
5327 case TargetOpcode::G_SMIN:
5328 case TargetOpcode::G_SMAX:
5329 case TargetOpcode::G_UMIN:
5330 case TargetOpcode::G_UMAX:
5331 case TargetOpcode::G_FMINNUM:
5332 case TargetOpcode::G_FMAXNUM:
5333 case TargetOpcode::G_FMINNUM_IEEE:
5334 case TargetOpcode::G_FMAXNUM_IEEE:
5335 case TargetOpcode::G_FMINIMUM:
5336 case TargetOpcode::G_FMAXIMUM:
5337 case TargetOpcode::G_STRICT_FADD:
5338 case TargetOpcode::G_STRICT_FSUB:
5339 case TargetOpcode::G_STRICT_FMUL:
5340 case TargetOpcode::G_SHL:
5341 case TargetOpcode::G_ASHR:
5342 case TargetOpcode::G_LSHR: {
5350 case TargetOpcode::G_FMA:
5351 case TargetOpcode::G_STRICT_FMA:
5352 case TargetOpcode::G_FSHR:
5353 case TargetOpcode::G_FSHL: {
5362 case TargetOpcode::G_EXTRACT_VECTOR_ELT:
5363 case TargetOpcode::G_EXTRACT:
5370 case TargetOpcode::G_INSERT:
5371 case TargetOpcode::G_INSERT_VECTOR_ELT:
5372 case TargetOpcode::G_FREEZE:
5373 case TargetOpcode::G_FNEG:
5374 case TargetOpcode::G_FABS:
5375 case TargetOpcode::G_FSQRT:
5376 case TargetOpcode::G_FCEIL:
5377 case TargetOpcode::G_FFLOOR:
5378 case TargetOpcode::G_FNEARBYINT:
5379 case TargetOpcode::G_FRINT:
5380 case TargetOpcode::G_INTRINSIC_ROUND:
5381 case TargetOpcode::G_INTRINSIC_ROUNDEVEN:
5382 case TargetOpcode::G_INTRINSIC_TRUNC:
5383 case TargetOpcode::G_BSWAP:
5384 case TargetOpcode::G_FCANONICALIZE:
5385 case TargetOpcode::G_SEXT_INREG:
5386 case TargetOpcode::G_ABS:
5394 case TargetOpcode::G_SELECT: {
5395 auto [DstReg, DstTy, CondReg, CondTy] =
MI.getFirst2RegLLTs();
5397 if (!CondTy.isScalar() ||
5405 MI.getOperand(1).setReg(ShufSplat.getReg(0));
5410 if (CondTy.isVector())
5420 case TargetOpcode::G_UNMERGE_VALUES:
5422 case TargetOpcode::G_PHI:
5424 case TargetOpcode::G_SHUFFLE_VECTOR:
5426 case TargetOpcode::G_BUILD_VECTOR: {
5428 for (
auto Op :
MI.uses()) {
5438 MI.eraseFromParent();
5441 case TargetOpcode::G_SEXT:
5442 case TargetOpcode::G_ZEXT:
5443 case TargetOpcode::G_ANYEXT:
5444 case TargetOpcode::G_TRUNC:
5445 case TargetOpcode::G_FPTRUNC:
5446 case TargetOpcode::G_FPEXT:
5447 case TargetOpcode::G_FPTOSI:
5448 case TargetOpcode::G_FPTOUI:
5449 case TargetOpcode::G_SITOFP:
5450 case TargetOpcode::G_UITOFP: {
5470 case TargetOpcode::G_ICMP:
5471 case TargetOpcode::G_FCMP: {
5485 case TargetOpcode::G_BITCAST: {
5506 case TargetOpcode::G_VECREDUCE_FADD:
5507 case TargetOpcode::G_VECREDUCE_FMUL:
5508 case TargetOpcode::G_VECREDUCE_ADD:
5509 case TargetOpcode::G_VECREDUCE_MUL:
5510 case TargetOpcode::G_VECREDUCE_AND:
5511 case TargetOpcode::G_VECREDUCE_OR:
5512 case TargetOpcode::G_VECREDUCE_XOR:
5513 case TargetOpcode::G_VECREDUCE_SMAX:
5514 case TargetOpcode::G_VECREDUCE_SMIN:
5515 case TargetOpcode::G_VECREDUCE_UMAX:
5516 case TargetOpcode::G_VECREDUCE_UMIN: {
5520 auto NeutralElement = getNeutralElementForVecReduce(
5528 NeutralElement,
Idx);
5532 MO.
setReg(NewVec.getReg(0));
5544 auto [DstReg, DstTy, SrcReg, SrcTy] =
MI.getFirst2RegLLTs();
5546 unsigned MaskNumElts = Mask.size();
5550 if (MaskNumElts == SrcNumElts)
5553 if (MaskNumElts < SrcNumElts) {
5557 for (
unsigned I = MaskNumElts;
I < SrcNumElts; ++
I)
5563 MI.getOperand(1).getReg(),
5564 MI.getOperand(2).getReg(), NewMask);
5565 MI.eraseFromParent();
5570 unsigned PaddedMaskNumElts =
alignTo(MaskNumElts, SrcNumElts);
5571 unsigned NumConcat = PaddedMaskNumElts / SrcNumElts;
5579 MOps1[0] =
MI.getOperand(1).getReg();
5580 MOps2[0] =
MI.getOperand(2).getReg();
5587 for (
unsigned I = 0;
I != MaskNumElts; ++
I) {
5589 if (
Idx >=
static_cast<int>(SrcNumElts))
5590 Idx += PaddedMaskNumElts - SrcNumElts;
5595 if (MaskNumElts != PaddedMaskNumElts) {
5600 for (
unsigned I = 0;
I < MaskNumElts; ++
I) {
5610 MI.eraseFromParent();
5616 unsigned int TypeIdx,
LLT MoreTy) {
5617 auto [DstTy, Src1Ty, Src2Ty] =
MI.getFirst3LLTs();
5619 unsigned NumElts = DstTy.getNumElements();
5622 if (DstTy.isVector() && Src1Ty.isVector() &&
5623 DstTy.getNumElements() != Src1Ty.getNumElements()) {
5631 if (DstTy != Src1Ty || DstTy != Src2Ty)
5639 for (
unsigned I = 0;
I != NumElts; ++
I) {
5641 if (
Idx <
static_cast<int>(NumElts))
5646 for (
unsigned I = NumElts;
I != WidenNumElts; ++
I)
5651 MI.getOperand(1).getReg(),
5652 MI.getOperand(2).getReg(), NewMask);
5653 MI.eraseFromParent();
5662 unsigned SrcParts = Src1Regs.
size();
5663 unsigned DstParts = DstRegs.
size();
5665 unsigned DstIdx = 0;
5667 B.buildMul(NarrowTy, Src1Regs[DstIdx], Src2Regs[DstIdx]).getReg(0);
5668 DstRegs[DstIdx] = FactorSum;
5670 unsigned CarrySumPrevDstIdx;
5673 for (DstIdx = 1; DstIdx < DstParts; DstIdx++) {
5675 for (
unsigned i = DstIdx + 1 < SrcParts ? 0 : DstIdx - SrcParts + 1;
5676 i <= std::min(DstIdx, SrcParts - 1); ++i) {
5678 B.buildMul(NarrowTy, Src1Regs[DstIdx - i], Src2Regs[i]);
5682 for (
unsigned i = DstIdx < SrcParts ? 0 : DstIdx - SrcParts;
5683 i <= std::min(DstIdx - 1, SrcParts - 1); ++i) {
5685 B.buildUMulH(NarrowTy, Src1Regs[DstIdx - 1 - i], Src2Regs[i]);
5695 if (DstIdx != DstParts - 1) {
5697 B.buildUAddo(NarrowTy,
LLT::scalar(1), Factors[0], Factors[1]);
5698 FactorSum = Uaddo.
getReg(0);
5699 CarrySum =
B.buildZExt(NarrowTy, Uaddo.
getReg(1)).
getReg(0);
5700 for (
unsigned i = 2; i < Factors.
size(); ++i) {
5702 B.buildUAddo(NarrowTy,
LLT::scalar(1), FactorSum, Factors[i]);
5703 FactorSum = Uaddo.
getReg(0);
5705 CarrySum =
B.buildAdd(NarrowTy, CarrySum, Carry).getReg(0);
5709 FactorSum =
B.buildAdd(NarrowTy, Factors[0], Factors[1]).getReg(0);
5710 for (
unsigned i = 2; i < Factors.
size(); ++i)
5711 FactorSum =
B.buildAdd(NarrowTy, FactorSum, Factors[i]).getReg(0);
5714 CarrySumPrevDstIdx = CarrySum;
5715 DstRegs[DstIdx] = FactorSum;
5732 unsigned Opcode =
MI.getOpcode();
5733 unsigned OpO, OpE, OpF;
5735 case TargetOpcode::G_SADDO:
5736 case TargetOpcode::G_SADDE:
5737 case TargetOpcode::G_UADDO:
5738 case TargetOpcode::G_UADDE:
5739 case TargetOpcode::G_ADD:
5740 OpO = TargetOpcode::G_UADDO;
5741 OpE = TargetOpcode::G_UADDE;
5742 OpF = TargetOpcode::G_UADDE;
5743 if (Opcode == TargetOpcode::G_SADDO || Opcode == TargetOpcode::G_SADDE)
5744 OpF = TargetOpcode::G_SADDE;
5746 case TargetOpcode::G_SSUBO:
5747 case TargetOpcode::G_SSUBE:
5748 case TargetOpcode::G_USUBO:
5749 case TargetOpcode::G_USUBE:
5750 case TargetOpcode::G_SUB:
5751 OpO = TargetOpcode::G_USUBO;
5752 OpE = TargetOpcode::G_USUBE;
5753 OpF = TargetOpcode::G_USUBE;
5754 if (Opcode == TargetOpcode::G_SSUBO || Opcode == TargetOpcode::G_SSUBE)
5755 OpF = TargetOpcode::G_SSUBE;
5762 unsigned NumDefs =
MI.getNumExplicitDefs();
5763 Register Src1 =
MI.getOperand(NumDefs).getReg();
5764 Register Src2 =
MI.getOperand(NumDefs + 1).getReg();
5767 CarryDst =
MI.getOperand(1).getReg();
5768 if (
MI.getNumOperands() == NumDefs + 3)
5769 CarryIn =
MI.getOperand(NumDefs + 2).getReg();
5772 LLT LeftoverTy, DummyTy;
5774 extractParts(Src1, RegTy, NarrowTy, LeftoverTy, Src1Regs, Src1Left,
5779 int NarrowParts = Src1Regs.
size();
5780 for (
int I = 0, E = Src1Left.
size();
I != E; ++
I) {
5786 for (
int i = 0, e = Src1Regs.
size(); i != e; ++i) {
5791 if (i == e - 1 && CarryDst)
5792 CarryOut = CarryDst;
5796 {Src1Regs[i], Src2Regs[i]});
5797 }
else if (i == e - 1) {
5799 {Src1Regs[i], Src2Regs[i], CarryIn});
5802 {Src1Regs[i], Src2Regs[i], CarryIn});
5808 insertParts(
MI.getOperand(0).getReg(), RegTy, NarrowTy,
5809 ArrayRef(DstRegs).take_front(NarrowParts), LeftoverTy,
5810 ArrayRef(DstRegs).drop_front(NarrowParts));
5812 MI.eraseFromParent();
5818 auto [DstReg, Src1, Src2] =
MI.getFirst3Regs();
5826 if (
Size % NarrowSize != 0)
5829 unsigned NumParts =
Size / NarrowSize;
5830 bool IsMulHigh =
MI.getOpcode() == TargetOpcode::G_UMULH;
5831 unsigned DstTmpParts = NumParts * (IsMulHigh ? 2 : 1);
5837 multiplyRegisters(DstTmpRegs, Src1Parts, Src2Parts, NarrowTy);
5842 MI.eraseFromParent();
5852 bool IsSigned =
MI.getOpcode() == TargetOpcode::G_FPTOSI;
5866 IsSigned ? TargetOpcode::G_SEXT : TargetOpcode::G_ZEXT);
5882 if (SizeOp1 % NarrowSize != 0)
5884 int NumParts = SizeOp1 / NarrowSize;
5888 extractParts(
MI.getOperand(1).getReg(), NarrowTy, NumParts, SrcRegs,
5892 uint64_t OpStart =
MI.getOperand(2).getImm();
5894 for (
int i = 0; i < NumParts; ++i) {
5895 unsigned SrcStart = i * NarrowSize;
5897 if (SrcStart + NarrowSize <= OpStart || SrcStart >= OpStart + OpSize) {
5900 }
else if (SrcStart == OpStart && NarrowTy == MRI.
getType(OpReg)) {
5908 int64_t ExtractOffset;
5910 if (OpStart < SrcStart) {
5912 SegSize = std::min(NarrowSize, OpStart + OpSize - SrcStart);
5914 ExtractOffset = OpStart - SrcStart;
5915 SegSize = std::min(SrcStart + NarrowSize - OpStart, OpSize);
5919 if (ExtractOffset != 0 || SegSize != NarrowSize) {
5931 else if (DstRegs.
size() > 1)
5935 MI.eraseFromParent();
5950 extractParts(
MI.getOperand(1).getReg(), RegTy, NarrowTy, LeftoverTy, SrcRegs,
5958 uint64_t OpStart =
MI.getOperand(3).getImm();
5960 for (
int I = 0, E = SrcRegs.
size();
I != E; ++
I) {
5961 unsigned DstStart =
I * NarrowSize;
5963 if (DstStart == OpStart && NarrowTy == MRI.
getType(OpReg)) {
5971 if (MRI.
getType(SrcRegs[
I]) == LeftoverTy) {
5977 if (DstStart + NarrowSize <= OpStart || DstStart >= OpStart + OpSize) {
5985 int64_t ExtractOffset, InsertOffset;
5987 if (OpStart < DstStart) {
5989 ExtractOffset = DstStart - OpStart;
5990 SegSize = std::min(NarrowSize, OpStart + OpSize - DstStart);
5992 InsertOffset = OpStart - DstStart;
5995 std::min(NarrowSize - InsertOffset, OpStart + OpSize - DstStart);
5999 if (ExtractOffset != 0 || SegSize != OpSize) {
6019 MI.eraseFromParent();
6029 assert(
MI.getNumOperands() == 3 && TypeIdx == 0);
6035 if (!
extractParts(
MI.getOperand(1).getReg(), DstTy, NarrowTy, LeftoverTy,
6036 Src0Regs, Src0LeftoverRegs,
MIRBuilder, MRI))
6040 if (!
extractParts(
MI.getOperand(2).getReg(), DstTy, NarrowTy, Unused,
6041 Src1Regs, Src1LeftoverRegs,
MIRBuilder, MRI))
6044 for (
unsigned I = 0, E = Src1Regs.
size();
I != E; ++
I) {
6046 {Src0Regs[I], Src1Regs[I]});
6050 for (
unsigned I = 0, E = Src1LeftoverRegs.
size();
I != E; ++
I) {
6053 {LeftoverTy}, {Src0LeftoverRegs[I], Src1LeftoverRegs[I]});
6054 DstLeftoverRegs.
push_back(Inst.getReg(0));
6057 insertParts(DstReg, DstTy, NarrowTy, DstRegs,
6058 LeftoverTy, DstLeftoverRegs);
6060 MI.eraseFromParent();
6070 auto [DstReg, SrcReg] =
MI.getFirst2Regs();
6077 LLT GCDTy = extractGCDType(Parts, DstTy, NarrowTy, SrcReg);
6078 LLT LCMTy = buildLCMMergePieces(DstTy, NarrowTy, GCDTy, Parts,
MI.getOpcode());
6079 buildWidenedRemergeToDst(DstReg, LCMTy, Parts);
6081 MI.eraseFromParent();
6091 Register CondReg =
MI.getOperand(1).getReg();
6103 if (!
extractParts(
MI.getOperand(2).getReg(), DstTy, NarrowTy, LeftoverTy,
6104 Src1Regs, Src1LeftoverRegs,
MIRBuilder, MRI))
6108 if (!
extractParts(
MI.getOperand(3).getReg(), DstTy, NarrowTy, Unused,
6109 Src2Regs, Src2LeftoverRegs,
MIRBuilder, MRI))
6112 for (
unsigned I = 0, E = Src1Regs.
size();
I != E; ++
I) {
6114 CondReg, Src1Regs[
I], Src2Regs[
I]);
6118 for (
unsigned I = 0, E = Src1LeftoverRegs.
size();
I != E; ++
I) {
6120 LeftoverTy, CondReg, Src1LeftoverRegs[
I], Src2LeftoverRegs[
I]);
6124 insertParts(DstReg, DstTy, NarrowTy, DstRegs,
6125 LeftoverTy, DstLeftoverRegs);
6127 MI.eraseFromParent();
6137 auto [DstReg, DstTy, SrcReg, SrcTy] =
MI.getFirst2RegLLTs();
6141 const bool IsUndef =
MI.getOpcode() == TargetOpcode::G_CTLZ_ZERO_UNDEF;
6144 auto UnmergeSrc =
B.buildUnmerge(NarrowTy, SrcReg);
6146 auto C_0 =
B.buildConstant(NarrowTy, 0);
6148 UnmergeSrc.getReg(1), C_0);
6149 auto LoCTLZ = IsUndef ?
6150 B.buildCTLZ_ZERO_UNDEF(DstTy, UnmergeSrc.getReg(0)) :
6151 B.buildCTLZ(DstTy, UnmergeSrc.getReg(0));
6152 auto C_NarrowSize =
B.buildConstant(DstTy, NarrowSize);
6153 auto HiIsZeroCTLZ =
B.buildAdd(DstTy, LoCTLZ, C_NarrowSize);
6154 auto HiCTLZ =
B.buildCTLZ_ZERO_UNDEF(DstTy, UnmergeSrc.getReg(1));
6155 B.buildSelect(DstReg, HiIsZero, HiIsZeroCTLZ, HiCTLZ);
6157 MI.eraseFromParent();
6170 auto [DstReg, DstTy, SrcReg, SrcTy] =
MI.getFirst2RegLLTs();
6174 const bool IsUndef =
MI.getOpcode() == TargetOpcode::G_CTTZ_ZERO_UNDEF;
6177 auto UnmergeSrc =
B.buildUnmerge(NarrowTy, SrcReg);
6179 auto C_0 =
B.buildConstant(NarrowTy, 0);
6181 UnmergeSrc.getReg(0), C_0);
6182 auto HiCTTZ = IsUndef ?
6183 B.buildCTTZ_ZERO_UNDEF(DstTy, UnmergeSrc.getReg(1)) :
6184 B.buildCTTZ(DstTy, UnmergeSrc.getReg(1));
6185 auto C_NarrowSize =
B.buildConstant(DstTy, NarrowSize);
6186 auto LoIsZeroCTTZ =
B.buildAdd(DstTy, HiCTTZ, C_NarrowSize);
6187 auto LoCTTZ =
B.buildCTTZ_ZERO_UNDEF(DstTy, UnmergeSrc.getReg(0));
6188 B.buildSelect(DstReg, LoIsZero, LoIsZeroCTTZ, LoCTTZ);
6190 MI.eraseFromParent();
6203 auto [DstReg, DstTy, SrcReg, SrcTy] =
MI.getFirst2RegLLTs();
6213 MI.eraseFromParent();
6233 auto MinExp =
B.buildConstant(ExpTy,
minIntN(ClampSize));
6234 auto ClampMin =
B.buildSMax(ExpTy, ExpReg, MinExp);
6235 auto MaxExp =
B.buildConstant(ExpTy,
maxIntN(ClampSize));
6236 auto Clamp =
B.buildSMin(ExpTy, ClampMin, MaxExp);
6238 auto Trunc =
B.buildTrunc(NarrowTy, Clamp);
6240 MI.getOperand(2).setReg(Trunc.getReg(0));
6247 unsigned Opc =
MI.getOpcode();
6256 case TargetOpcode::G_CTLZ_ZERO_UNDEF: {
6259 MI.setDesc(
TII.get(TargetOpcode::G_CTLZ));
6263 case TargetOpcode::G_CTLZ: {
6264 auto [DstReg, DstTy, SrcReg, SrcTy] =
MI.getFirst2RegLLTs();
6267 if (isSupported({TargetOpcode::G_CTLZ_ZERO_UNDEF, {DstTy, SrcTy}})) {
6275 MI.eraseFromParent();
6291 for (
unsigned i = 0; (1U << i) <= (NewLen / 2); ++i) {
6295 Op = MIBOp.getReg(0);
6300 MI.eraseFromParent();
6303 case TargetOpcode::G_CTTZ_ZERO_UNDEF: {
6306 MI.setDesc(
TII.get(TargetOpcode::G_CTTZ));
6310 case TargetOpcode::G_CTTZ: {
6311 auto [DstReg, DstTy, SrcReg, SrcTy] =
MI.getFirst2RegLLTs();
6314 if (isSupported({TargetOpcode::G_CTTZ_ZERO_UNDEF, {DstTy, SrcTy}})) {
6323 MI.eraseFromParent();
6334 if (!isSupported({TargetOpcode::G_CTPOP, {SrcTy, SrcTy}}) &&
6335 isSupported({TargetOpcode::G_CTLZ, {SrcTy, SrcTy}})) {
6339 MI.eraseFromParent();
6343 MI.setDesc(
TII.get(TargetOpcode::G_CTPOP));
6344 MI.getOperand(1).setReg(MIBTmp.getReg(0));
6348 case TargetOpcode::G_CTPOP: {
6359 auto C_1 =
B.buildConstant(Ty, 1);
6360 auto B2Set1LoTo1Hi =
B.buildLShr(Ty, SrcReg, C_1);
6362 auto C_B2Mask1HiTo0 =
B.buildConstant(Ty, B2Mask1HiTo0);
6363 auto B2Count1Hi =
B.buildAnd(Ty, B2Set1LoTo1Hi, C_B2Mask1HiTo0);
6364 auto B2Count =
B.buildSub(Ty, SrcReg, B2Count1Hi);
6368 auto C_2 =
B.buildConstant(Ty, 2);
6369 auto B4Set2LoTo2Hi =
B.buildLShr(Ty, B2Count, C_2);
6371 auto C_B4Mask2HiTo0 =
B.buildConstant(Ty, B4Mask2HiTo0);
6372 auto B4HiB2Count =
B.buildAnd(Ty, B4Set2LoTo2Hi, C_B4Mask2HiTo0);
6373 auto B4LoB2Count =
B.buildAnd(Ty, B2Count, C_B4Mask2HiTo0);
6374 auto B4Count =
B.buildAdd(Ty, B4HiB2Count, B4LoB2Count);
6381 auto C_4 =
B.buildConstant(Ty, 4);
6382 auto B8HiB4Count =
B.buildLShr(Ty, B4Count, C_4);
6383 auto B8CountDirty4Hi =
B.buildAdd(Ty, B8HiB4Count, B4Count);
6385 auto C_B8Mask4HiTo0 =
B.buildConstant(Ty, B8Mask4HiTo0);
6386 auto B8Count =
B.buildAnd(Ty, B8CountDirty4Hi, C_B8Mask4HiTo0);
6388 assert(
Size<=128 &&
"Scalar size is too large for CTPOP lower algorithm");
6392 auto ResTmp =
B.buildMul(Ty, B8Count, MulMask);
6395 auto C_SizeM8 =
B.buildConstant(Ty,
Size - 8);
6396 B.buildLShr(
MI.getOperand(0).getReg(), ResTmp, C_SizeM8);
6398 MI.eraseFromParent();
6411 const ConstantInt *CI = dyn_cast_or_null<ConstantInt>(
C);
6419 auto [Dst,
X,
Y, Z] =
MI.getFirst4Regs();
6428 const bool IsFSHL =
MI.getOpcode() == TargetOpcode::G_FSHL;
6429 unsigned RevOpcode = IsFSHL ? TargetOpcode::G_FSHR : TargetOpcode::G_FSHL;
6452 MI.eraseFromParent();
6458 auto [Dst,
X,
Y, Z] =
MI.getFirst4Regs();
6463 const bool IsFSHL =
MI.getOpcode() == TargetOpcode::G_FSHL;
6507 MI.eraseFromParent();
6521 bool IsFSHL =
MI.getOpcode() == TargetOpcode::G_FSHL;
6522 unsigned RevOpcode = IsFSHL ? TargetOpcode::G_FSHR : TargetOpcode::G_FSHL;
6526 return lowerFunnelShiftAsShifts(
MI);
6530 if (Result == UnableToLegalize)
6531 return lowerFunnelShiftAsShifts(
MI);
6536 auto [Dst, Src] =
MI.getFirst2Regs();
6550 if (SrcTyScalarSize * 2 < DstTyScalarSize) {
6564 {UnmergeSrc.getReg(0)});
6566 {UnmergeSrc.getReg(1)});
6571 MI.eraseFromParent();
6588 assert(
MI.getOpcode() == TargetOpcode::G_TRUNC);
6592 LLT DstTy =
MRI.getType(DstReg);
6593 LLT SrcTy =
MRI.getType(SrcReg);
6613 for (
unsigned I = 0;
I < SplitSrcs.
size(); ++
I) {
6627 MI.eraseFromParent();
6636 auto [Dst, DstTy, Src, SrcTy, Amt, AmtTy] =
MI.getFirst3RegLLTs();
6638 bool IsLeft =
MI.getOpcode() == TargetOpcode::G_ROTL;
6639 unsigned RevRot = IsLeft ? TargetOpcode::G_ROTR : TargetOpcode::G_ROTL;
6642 MI.eraseFromParent();
6647 auto [Dst, DstTy, Src, SrcTy, Amt, AmtTy] =
MI.getFirst3RegLLTs();
6649 unsigned EltSizeInBits = DstTy.getScalarSizeInBits();
6650 bool IsLeft =
MI.getOpcode() == TargetOpcode::G_ROTL;
6655 unsigned RevRot = IsLeft ? TargetOpcode::G_ROTR : TargetOpcode::G_ROTL;
6658 return lowerRotateWithReverseRotate(
MI);
6661 unsigned FShOpc = IsLeft ? TargetOpcode::G_FSHL : TargetOpcode::G_FSHR;
6662 unsigned RevFsh = !IsLeft ? TargetOpcode::G_FSHL : TargetOpcode::G_FSHR;
6663 bool IsFShLegal =
false;
6664 if ((IsFShLegal = LI.isLegalOrCustom({FShOpc, {DstTy, AmtTy}})) ||
6665 LI.isLegalOrCustom({RevFsh, {DstTy, AmtTy}})) {
6669 MI.eraseFromParent();
6674 return buildFunnelShift(FShOpc, Dst, Src, Amt);
6677 return buildFunnelShift(RevFsh, Dst, Src, Amt);
6682 unsigned ShOpc = IsLeft ? TargetOpcode::G_SHL : TargetOpcode::G_LSHR;
6683 unsigned RevShiftOpc = IsLeft ? TargetOpcode::G_LSHR : TargetOpcode::G_SHL;
6684 auto BitWidthMinusOneC = MIRBuilder.
buildConstant(AmtTy, EltSizeInBits - 1);
6690 auto NegAmt = MIRBuilder.
buildSub(AmtTy, Zero, Amt);
6691 auto ShAmt = MIRBuilder.
buildAnd(AmtTy, Amt, BitWidthMinusOneC);
6693 auto RevAmt = MIRBuilder.
buildAnd(AmtTy, NegAmt, BitWidthMinusOneC);
6699 auto BitWidthC = MIRBuilder.
buildConstant(AmtTy, EltSizeInBits);
6700 auto ShAmt = MIRBuilder.
buildURem(AmtTy, Amt, BitWidthC);
6702 auto RevAmt = MIRBuilder.
buildSub(AmtTy, BitWidthMinusOneC, ShAmt);
6704 auto Inner = MIRBuilder.
buildInstr(RevShiftOpc, {DstTy}, {Src, One});
6708 MIRBuilder.
buildOr(Dst, ShVal, RevShiftVal);
6709 MI.eraseFromParent();
6717 auto [Dst, Src] =
MI.getFirst2Regs();
6767 MI.eraseFromParent();
6772 auto [Dst, DstTy, Src, SrcTy] =
MI.getFirst2RegLLTs();
6778 MI.eraseFromParent();
6797 auto [Dst, DstTy, Src, SrcTy] =
MI.getFirst2RegLLTs();
6807 MI.eraseFromParent();
6832 MI.eraseFromParent();
6840 auto [Dst, DstTy, Src, SrcTy] =
MI.getFirst2RegLLTs();
6844 if (SrcTy !=
S64 && SrcTy !=
S32)
6846 if (DstTy !=
S32 && DstTy !=
S64)
6875 MI.eraseFromParent();
6880 auto [Dst, DstTy, Src, SrcTy] =
MI.getFirst2RegLLTs();
6939 MI.eraseFromParent();
6949 auto [Dst, Src] =
MI.getFirst2Regs();
6957 unsigned Flags =
MI.getFlags();
6960 MI.eraseFromParent();
6964 const unsigned ExpMask = 0x7ff;
6965 const unsigned ExpBiasf64 = 1023;
6966 const unsigned ExpBiasf16 = 15;
7055 MI.eraseFromParent();
7061 auto [DstTy, SrcTy] =
MI.getFirst2LLTs();
7074 auto [Dst, Src0, Src1] =
MI.getFirst3Regs();
7079 MI.eraseFromParent();
7085 case TargetOpcode::G_SMIN:
7087 case TargetOpcode::G_SMAX:
7089 case TargetOpcode::G_UMIN:
7091 case TargetOpcode::G_UMAX:
7099 auto [Dst, Src0, Src1] =
MI.getFirst3Regs();
7107 MI.eraseFromParent();
7113 auto [Dst, DstTy, Src0, Src0Ty, Src1, Src1Ty] =
MI.getFirst3RegLLTs();
7114 const int Src0Size = Src0Ty.getScalarSizeInBits();
7115 const int Src1Size = Src1Ty.getScalarSizeInBits();
7125 if (Src0Ty == Src1Ty) {
7127 }
else if (Src0Size > Src1Size) {
7142 unsigned Flags =
MI.getFlags();
7145 MI.eraseFromParent();
7151 unsigned NewOp =
MI.getOpcode() == TargetOpcode::G_FMINNUM ?
7152 TargetOpcode::G_FMINNUM_IEEE : TargetOpcode::G_FMAXNUM_IEEE;
7154 auto [Dst, Src0, Src1] =
MI.getFirst3Regs();
7174 MI.eraseFromParent();
7182 unsigned Flags =
MI.getFlags();
7187 MI.eraseFromParent();
7193 auto [DstReg,
X] =
MI.getFirst2Regs();
7194 const unsigned Flags =
MI.getFlags();
7221 MI.eraseFromParent();
7226 auto [DstReg, SrcReg] =
MI.getFirst2Regs();
7227 unsigned Flags =
MI.getFlags();
7239 SrcReg, Zero, Flags);
7241 SrcReg, Trunc, Flags);
7246 MI.eraseFromParent();
7252 const unsigned NumOps =
MI.getNumOperands();
7253 auto [DstReg, DstTy, Src0Reg, Src0Ty] =
MI.getFirst2RegLLTs();
7254 unsigned PartSize = Src0Ty.getSizeInBits();
7259 for (
unsigned I = 2;
I != NumOps; ++
I) {
7260 const unsigned Offset = (
I - 1) * PartSize;
7265 Register NextResult =
I + 1 == NumOps && WideTy == DstTy ? DstReg :
7271 ResultReg = NextResult;
7274 if (DstTy.isPointer()) {
7276 DstTy.getAddressSpace())) {
7284 MI.eraseFromParent();
7290 const unsigned NumDst =
MI.getNumOperands() - 1;
7291 Register SrcReg =
MI.getOperand(NumDst).getReg();
7292 Register Dst0Reg =
MI.getOperand(0).getReg();
7307 unsigned Offset = DstSize;
7308 for (
unsigned I = 1;
I != NumDst; ++
I,
Offset += DstSize) {
7314 MI.eraseFromParent();
7333 if (
MI.getOpcode() == TargetOpcode::G_INSERT_VECTOR_ELT)
7334 InsertVal =
MI.getOperand(2).getReg();
7348 SrcRegs[IdxVal] =
MI.getOperand(2).getReg();
7354 MI.eraseFromParent();
7359 LLVM_DEBUG(
dbgs() <<
"Can't handle non-byte element vectors yet\n");
7377 int64_t
Offset = IdxVal * EltBytes;
7396 MI.eraseFromParent();
7402 auto [DstReg, DstTy, Src0Reg, Src0Ty, Src1Reg, Src1Ty] =
7403 MI.getFirst3RegLLTs();
7411 for (
int Idx : Mask) {
7413 if (!Undef.isValid())
7419 if (Src0Ty.isScalar()) {
7422 int NumElts = Src0Ty.getNumElements();
7423 Register SrcVec =
Idx < NumElts ? Src0Reg : Src1Reg;
7424 int ExtractIdx =
Idx < NumElts ?
Idx :
Idx - NumElts;
7431 if (DstTy.isScalar())
7435 MI.eraseFromParent();
7452 if (Alignment >
Align(1)) {
7464 const auto &MF = *
MI.getMF();
7465 const auto &TFI = *MF.getSubtarget().getFrameLowering();
7470 Register AllocSize =
MI.getOperand(1).getReg();
7481 MI.eraseFromParent();
7492 MI.eraseFromParent();
7503 MI.eraseFromParent();
7509 auto [DstReg, DstTy, SrcReg, SrcTy] =
MI.getFirst2RegLLTs();
7510 unsigned Offset =
MI.getOperand(2).getImm();
7515 unsigned DstSize = DstTy.getSizeInBits();
7517 if ((
Offset % SrcEltSize == 0) && (DstSize % SrcEltSize == 0) &&
7524 for (
unsigned Idx =
Offset / SrcEltSize;
7528 if (SubVectorElts.
size() == 1)
7533 MI.eraseFromParent();
7538 if (DstTy.isScalar() &&
7541 LLT SrcIntTy = SrcTy;
7555 MI.eraseFromParent();
7563 auto [Dst, Src, InsertSrc] =
MI.getFirst3Regs();
7575 if ((
Offset % EltSize == 0) && (InsertSize % EltSize == 0) &&
7588 for (
unsigned i = 0;
Idx < (
Offset + InsertSize) / EltSize;
7590 DstElts.
push_back(UnmergeInsertSrc.getReg(i));
7603 MI.eraseFromParent();
7617 LLVM_DEBUG(
dbgs() <<
"Not casting non-integral address space integer\n");
7621 LLT IntDstTy = DstTy;
7647 MI.eraseFromParent();
7653 auto [Dst0, Dst0Ty, Dst1, Dst1Ty,
LHS, LHSTy,
RHS, RHSTy] =
7654 MI.getFirst4RegLLTs();
7655 const bool IsAdd =
MI.getOpcode() == TargetOpcode::G_SADDO;
7658 LLT BoolTy = Dst1Ty;
7675 auto ResultLowerThanLHS =
7681 MI.eraseFromParent();
7687 auto [Res,
LHS,
RHS] =
MI.getFirst3Regs();
7692 switch (
MI.getOpcode()) {
7695 case TargetOpcode::G_UADDSAT:
7698 BaseOp = TargetOpcode::G_ADD;
7700 case TargetOpcode::G_SADDSAT:
7703 BaseOp = TargetOpcode::G_ADD;
7705 case TargetOpcode::G_USUBSAT:
7708 BaseOp = TargetOpcode::G_SUB;
7710 case TargetOpcode::G_SSUBSAT:
7713 BaseOp = TargetOpcode::G_SUB;
7756 MI.eraseFromParent();
7762 auto [Res,
LHS,
RHS] =
MI.getFirst3Regs();
7767 unsigned OverflowOp;
7768 switch (
MI.getOpcode()) {
7771 case TargetOpcode::G_UADDSAT:
7774 OverflowOp = TargetOpcode::G_UADDO;
7776 case TargetOpcode::G_SADDSAT:
7779 OverflowOp = TargetOpcode::G_SADDO;
7781 case TargetOpcode::G_USUBSAT:
7784 OverflowOp = TargetOpcode::G_USUBO;
7786 case TargetOpcode::G_SSUBSAT:
7789 OverflowOp = TargetOpcode::G_SSUBO;
7795 Register Tmp = OverflowRes.getReg(0);
7796 Register Ov = OverflowRes.getReg(1);
7822 MI.eraseFromParent();
7828 assert((
MI.getOpcode() == TargetOpcode::G_SSHLSAT ||
7829 MI.getOpcode() == TargetOpcode::G_USHLSAT) &&
7830 "Expected shlsat opcode!");
7831 bool IsSigned =
MI.getOpcode() == TargetOpcode::G_SSHLSAT;
7832 auto [Res,
LHS,
RHS] =
MI.getFirst3Regs();
7854 MI.eraseFromParent();
7859 auto [Dst, Src] =
MI.getFirst2Regs();
7862 unsigned BaseShiftAmt = (SizeInBytes - 1) * 8;
7871 for (
unsigned i = 1; i < SizeInBytes / 2; ++i) {
7873 APInt APMask(SizeInBytes * 8, 0xFF << (i * 8));
7885 Res.getInstr()->getOperand(0).setReg(Dst);
7887 MI.eraseFromParent();
7894 const LLT Ty = Dst.getLLTTy(*
B.getMRI());
7897 auto LHS =
B.buildLShr(Ty,
B.buildAnd(Ty, Src, MaskLoNTo0), C_N);
7898 auto RHS =
B.buildAnd(Ty,
B.buildShl(Ty, Src, C_N), MaskLoNTo0);
7899 return B.buildOr(Dst,
LHS,
RHS);
7904 auto [Dst, Src] =
MI.getFirst2Regs();
7928 MI.eraseFromParent();
7936 bool IsRead =
MI.getOpcode() == TargetOpcode::G_READ_REGISTER;
7937 int NameOpIdx = IsRead ? 1 : 0;
7938 int ValRegIndex = IsRead ? 0 : 1;
7940 Register ValReg =
MI.getOperand(ValRegIndex).getReg();
7942 const MDString *RegStr = cast<MDString>(
7943 cast<MDNode>(
MI.getOperand(NameOpIdx).getMetadata())->getOperand(0));
7954 MI.eraseFromParent();
7960 bool IsSigned =
MI.getOpcode() == TargetOpcode::G_SMULH;
7961 unsigned ExtOp = IsSigned ? TargetOpcode::G_SEXT : TargetOpcode::G_ZEXT;
7970 unsigned ShiftOp = IsSigned ? TargetOpcode::G_ASHR : TargetOpcode::G_LSHR;
7976 MI.eraseFromParent();
7982 auto [DstReg, DstTy, SrcReg, SrcTy] =
MI.getFirst2RegLLTs();
7987 MI.eraseFromParent();
7992 MI.eraseFromParent();
8011 APInt ExpMask = Inf;
8029 LLT DstTyCopy = DstTy;
8044 Mask &= ~fcPosFinite;
8051 Mask &= ~fcNegFinite;
8062 Mask &= ~PartialCheck;
8071 else if (PartialCheck ==
fcZero)
8090 appendToRes(SubnormalRes);
8097 else if (PartialCheck ==
fcInf)
8110 if (PartialCheck ==
fcNan) {
8114 }
else if (PartialCheck ==
fcQNan) {
8124 Abs, InfWithQnanBitC);
8132 APInt ExpLSB = ExpMask & ~(ExpMask.
shl(1));
8135 APInt MaxExpMinusOne = ExpMask - ExpLSB;
8146 appendToRes(NormalRes);
8150 MI.eraseFromParent();
8156 auto [DstReg, DstTy, MaskReg, MaskTy, Op1Reg, Op1Ty, Op2Reg, Op2Ty] =
8157 MI.getFirst4RegLLTs();
8159 bool IsEltPtr = DstTy.isPointerOrPointerVector();
8168 if (MaskTy.isScalar()) {
8182 if (DstTy.isVector()) {
8185 MaskReg = ShufSplat.
getReg(0);
8190 }
else if (!DstTy.isVector()) {
8195 if (MaskTy.getSizeInBits() != DstTy.getSizeInBits()) {
8208 MI.eraseFromParent();
8214 unsigned Opcode =
MI.getOpcode();
8217 Opcode == TargetOpcode::G_SDIVREM ? TargetOpcode::G_SDIV
8218 : TargetOpcode::G_UDIV,
8219 {
MI.getOperand(0).
getReg()}, {
MI.getOperand(2),
MI.getOperand(3)});
8221 Opcode == TargetOpcode::G_SDIVREM ? TargetOpcode::G_SREM
8222 : TargetOpcode::G_UREM,
8223 {
MI.getOperand(1).
getReg()}, {
MI.getOperand(2),
MI.getOperand(3)});
8224 MI.eraseFromParent();
8241 MI.eraseFromParent();
8256 MI.eraseFromParent();
8263 Register DestReg =
MI.getOperand(0).getReg();
8269 MI.eraseFromParent();
8298 Register ListPtr =
MI.getOperand(1).getReg();
8308 const Align A(
MI.getOperand(2).getImm());
8315 VAList = AndDst.
getReg(0);
8333 Align EltAlignment =
DL.getABITypeAlign(Ty);
8338 MI.eraseFromParent();
8353 unsigned Limit,
const MemOp &
Op,
8354 unsigned DstAS,
unsigned SrcAS,
8357 if (
Op.isMemcpyWithFixedDstAlign() &&
Op.getSrcAlign() <
Op.getDstAlign())
8367 if (
Op.isFixedDstAlign())
8375 unsigned NumMemOps = 0;
8379 while (TySize >
Size) {
8388 assert(NewTySize > 0 &&
"Could not find appropriate type");
8395 if (NumMemOps &&
Op.allowOverlap() && NewTySize <
Size &&
8397 VT, DstAS,
Op.isFixedDstAlign() ?
Op.getDstAlign() :
Align(1),
8407 if (++NumMemOps > Limit)
8410 MemOps.push_back(Ty);
8429 if (!Ty.
isVector() && ValVRegAndVal) {
8430 APInt Scalar = ValVRegAndVal->Value.trunc(8);
8438 if (ValVRegAndVal && ValVRegAndVal->Value == 0) {
8461 auto &MF = *
MI.getParent()->getParent();
8462 const auto &TLI = *MF.getSubtarget().getTargetLowering();
8463 auto &
DL = MF.getDataLayout();
8466 assert(KnownLen != 0 &&
"Have a zero length memset length!");
8468 bool DstAlignCanChange =
false;
8474 DstAlignCanChange =
true;
8477 std::vector<LLT> MemOps;
8479 const auto &DstMMO = **
MI.memoperands_begin();
8483 bool IsZeroVal = ValVRegAndVal && ValVRegAndVal->Value == 0;
8491 MF.getFunction().getAttributes(), TLI))
8494 if (DstAlignCanChange) {
8497 Align NewAlign =
DL.getABITypeAlign(IRTy);
8498 if (NewAlign > Alignment) {
8499 Alignment = NewAlign;
8509 LLT LargestTy = MemOps[0];
8510 for (
unsigned i = 1; i < MemOps.size(); i++)
8512 LargestTy = MemOps[i];
8525 unsigned DstOff = 0;
8526 unsigned Size = KnownLen;
8527 for (
unsigned I = 0;
I < MemOps.size();
I++) {
8530 if (TySize >
Size) {
8533 assert(
I == MemOps.size() - 1 &&
I != 0);
8534 DstOff -= TySize -
Size;
8545 Value = MIB.buildTrunc(Ty, MemSetValue).getReg(0);
8552 auto *StoreMMO = MF.getMachineMemOperand(&DstMMO, DstOff, Ty);
8558 Ptr = MIB.buildPtrAdd(PtrTy, Dst,
Offset).getReg(0);
8561 MIB.buildStore(
Value,
Ptr, *StoreMMO);
8566 MI.eraseFromParent();
8572 assert(
MI.getOpcode() == TargetOpcode::G_MEMCPY_INLINE);
8574 auto [Dst, Src, Len] =
MI.getFirst3Regs();
8576 const auto *MMOIt =
MI.memoperands_begin();
8578 bool IsVolatile =
MemOp->isVolatile();
8584 "inline memcpy with dynamic size is not yet supported");
8585 uint64_t KnownLen = LenVRegAndVal->Value.getZExtValue();
8586 if (KnownLen == 0) {
8587 MI.eraseFromParent();
8591 const auto &DstMMO = **
MI.memoperands_begin();
8592 const auto &SrcMMO = **std::next(
MI.memoperands_begin());
8593 Align DstAlign = DstMMO.getBaseAlign();
8594 Align SrcAlign = SrcMMO.getBaseAlign();
8596 return lowerMemcpyInline(
MI, Dst, Src, KnownLen, DstAlign, SrcAlign,
8603 Align SrcAlign,
bool IsVolatile) {
8604 assert(
MI.getOpcode() == TargetOpcode::G_MEMCPY_INLINE);
8605 return lowerMemcpy(
MI, Dst, Src, KnownLen,
8606 std::numeric_limits<uint64_t>::max(), DstAlign, SrcAlign,
8613 Align SrcAlign,
bool IsVolatile) {
8614 auto &MF = *
MI.getParent()->getParent();
8615 const auto &TLI = *MF.getSubtarget().getTargetLowering();
8616 auto &
DL = MF.getDataLayout();
8619 assert(KnownLen != 0 &&
"Have a zero length memcpy length!");
8621 bool DstAlignCanChange =
false;
8623 Align Alignment = std::min(DstAlign, SrcAlign);
8627 DstAlignCanChange =
true;
8633 std::vector<LLT> MemOps;
8635 const auto &DstMMO = **
MI.memoperands_begin();
8636 const auto &SrcMMO = **std::next(
MI.memoperands_begin());
8642 MemOp::Copy(KnownLen, DstAlignCanChange, Alignment, SrcAlign,
8645 MF.getFunction().getAttributes(), TLI))
8648 if (DstAlignCanChange) {
8651 Align NewAlign =
DL.getABITypeAlign(IRTy);
8656 if (!
TRI->hasStackRealignment(MF))
8657 while (NewAlign > Alignment &&
DL.exceedsNaturalStackAlignment(NewAlign))
8660 if (NewAlign > Alignment) {
8661 Alignment = NewAlign;
8669 LLVM_DEBUG(
dbgs() <<
"Inlining memcpy: " <<
MI <<
" into loads & stores\n");
8677 unsigned CurrOffset = 0;
8678 unsigned Size = KnownLen;
8679 for (
auto CopyTy : MemOps) {
8682 if (CopyTy.getSizeInBytes() >
Size)
8683 CurrOffset -= CopyTy.getSizeInBytes() -
Size;
8687 MF.getMachineMemOperand(&SrcMMO, CurrOffset, CopyTy.getSizeInBytes());
8689 MF.getMachineMemOperand(&DstMMO, CurrOffset, CopyTy.getSizeInBytes());
8694 if (CurrOffset != 0) {
8698 LoadPtr = MIB.buildPtrAdd(SrcTy, Src,
Offset).getReg(0);
8700 auto LdVal = MIB.buildLoad(CopyTy, LoadPtr, *LoadMMO);
8704 if (CurrOffset != 0) {
8706 StorePtr = MIB.buildPtrAdd(DstTy, Dst,
Offset).getReg(0);
8708 MIB.buildStore(LdVal, StorePtr, *StoreMMO);
8709 CurrOffset += CopyTy.getSizeInBytes();
8710 Size -= CopyTy.getSizeInBytes();
8713 MI.eraseFromParent();
8721 auto &MF = *
MI.getParent()->getParent();
8722 const auto &TLI = *MF.getSubtarget().getTargetLowering();
8723 auto &
DL = MF.getDataLayout();
8726 assert(KnownLen != 0 &&
"Have a zero length memmove length!");
8728 bool DstAlignCanChange =
false;
8731 Align Alignment = std::min(DstAlign, SrcAlign);
8735 DstAlignCanChange =
true;
8738 std::vector<LLT> MemOps;
8740 const auto &DstMMO = **
MI.memoperands_begin();
8741 const auto &SrcMMO = **std::next(
MI.memoperands_begin());
8750 MemOp::Copy(KnownLen, DstAlignCanChange, Alignment, SrcAlign,
8753 MF.getFunction().getAttributes(), TLI))
8756 if (DstAlignCanChange) {
8759 Align NewAlign =
DL.getABITypeAlign(IRTy);
8764 if (!
TRI->hasStackRealignment(MF))
8765 while (NewAlign > Alignment &&
DL.exceedsNaturalStackAlignment(NewAlign))
8768 if (NewAlign > Alignment) {
8769 Alignment = NewAlign;
8777 LLVM_DEBUG(
dbgs() <<
"Inlining memmove: " <<
MI <<
" into loads & stores\n");
8783 unsigned CurrOffset = 0;
8785 for (
auto CopyTy : MemOps) {
8788 MF.getMachineMemOperand(&SrcMMO, CurrOffset, CopyTy.getSizeInBytes());
8792 if (CurrOffset != 0) {
8796 LoadPtr = MIB.buildPtrAdd(SrcTy, Src,
Offset).getReg(0);
8798 LoadVals.
push_back(MIB.buildLoad(CopyTy, LoadPtr, *LoadMMO).getReg(0));
8799 CurrOffset += CopyTy.getSizeInBytes();
8803 for (
unsigned I = 0;
I < MemOps.size(); ++
I) {
8804 LLT CopyTy = MemOps[
I];
8807 MF.getMachineMemOperand(&DstMMO, CurrOffset, CopyTy.
getSizeInBytes());
8810 if (CurrOffset != 0) {
8814 StorePtr = MIB.buildPtrAdd(DstTy, Dst,
Offset).getReg(0);
8816 MIB.buildStore(LoadVals[
I], StorePtr, *StoreMMO);
8819 MI.eraseFromParent();
8825 const unsigned Opc =
MI.getOpcode();
8828 assert((Opc == TargetOpcode::G_MEMCPY || Opc == TargetOpcode::G_MEMMOVE ||
8829 Opc == TargetOpcode::G_MEMSET) &&
8830 "Expected memcpy like instruction");
8832 auto MMOIt =
MI.memoperands_begin();
8837 auto [Dst, Src, Len] =
MI.getFirst3Regs();
8839 if (Opc != TargetOpcode::G_MEMSET) {
8840 assert(MMOIt !=
MI.memoperands_end() &&
"Expected a second MMO on MI");
8842 SrcAlign =
MemOp->getBaseAlign();
8849 uint64_t KnownLen = LenVRegAndVal->Value.getZExtValue();
8851 if (KnownLen == 0) {
8852 MI.eraseFromParent();
8856 bool IsVolatile =
MemOp->isVolatile();
8857 if (Opc == TargetOpcode::G_MEMCPY_INLINE)
8858 return lowerMemcpyInline(
MI, Dst, Src, KnownLen, DstAlign, SrcAlign,
8865 if (MaxLen && KnownLen > MaxLen)
8868 if (Opc == TargetOpcode::G_MEMCPY) {
8869 auto &MF = *
MI.getParent()->getParent();
8870 const auto &TLI = *MF.getSubtarget().getTargetLowering();
8873 return lowerMemcpy(
MI, Dst, Src, KnownLen, Limit, DstAlign, SrcAlign,
8876 if (Opc == TargetOpcode::G_MEMMOVE)
8877 return lowerMemmove(
MI, Dst, Src, KnownLen, DstAlign, SrcAlign, IsVolatile);
8878 if (Opc == TargetOpcode::G_MEMSET)
8879 return lowerMemset(
MI, Dst, Src, KnownLen, DstAlign, IsVolatile);
unsigned const MachineRegisterInfo * MRI
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
amdgpu AMDGPU Register Bank Select
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")
static GCRegistry::Add< StatepointGC > D("statepoint-example", "an example strategy for statepoint")
Analysis containing CSE Info
This file describes how to lower LLVM calls to machine code calls.
#define GISEL_VECREDUCE_CASES_NONSEQ
static std::optional< bool > isBigEndian(const SmallDenseMap< int64_t, int64_t, 8 > &MemOffset2Idx, int64_t LowestIdx)
Given a map from byte offsets in memory to indices in a load/store, determine if that map corresponds...
static Type * getTypeForLLT(LLT Ty, LLVMContext &C)
Returns the sub type a function will return at a given Idx Should correspond to the result type of an ExtractValue instruction executed with just that one unsigned Idx
static GCMetadataPrinterRegistry::Add< ErlangGCPrinter > X("erlang", "erlang-compatible garbage collector")
Rewrite Partial Register Uses
This contains common code to allow clients to notify changes to machine instr.
Provides analysis for querying information about KnownBits during GISel passes.
Declares convenience wrapper classes for interpreting MachineInstr instances as specific generic oper...
const HexagonInstrInfo * TII
#define RTLIBCASE_INT(LibcallPrefix)
static bool findGISelOptimalMemOpLowering(std::vector< LLT > &MemOps, unsigned Limit, const MemOp &Op, unsigned DstAS, unsigned SrcAS, const AttributeList &FuncAttributes, const TargetLowering &TLI)
static RTLIB::Libcall getOutlineAtomicLibcall(MachineInstr &MI)
static Register buildBitFieldInsert(MachineIRBuilder &B, Register TargetReg, Register InsertReg, Register OffsetBits)
Emit code to insert InsertReg into TargetRet at OffsetBits in TargetReg, while preserving other bits ...
static Register getMemsetValue(Register Val, LLT Ty, MachineIRBuilder &MIB)
static RTLIB::Libcall getRTLibDesc(unsigned Opcode, unsigned Size)
static void broadcastSrcOp(SmallVectorImpl< SrcOp > &Ops, unsigned N, MachineOperand &Op)
Operand Op is used on N sub-instructions.
static bool isLibCallInTailPosition(const CallLowering::ArgInfo &Result, MachineInstr &MI, const TargetInstrInfo &TII, MachineRegisterInfo &MRI)
True if an instruction is in tail position in its caller.
static LegalizerHelper::LegalizeResult simpleLibcall(MachineInstr &MI, MachineIRBuilder &MIRBuilder, unsigned Size, Type *OpType, LostDebugLocObserver &LocObserver)
static Register getBitcastWiderVectorElementOffset(MachineIRBuilder &B, Register Idx, unsigned NewEltSize, unsigned OldEltSize)
Figure out the bit offset into a register when coercing a vector index for the wide element type.
static void makeDstOps(SmallVectorImpl< DstOp > &DstOps, LLT Ty, unsigned NumElts)
Fill DstOps with DstOps that have same number of elements combined as the Ty.
static bool shouldLowerMemFuncForSize(const MachineFunction &MF)
static MachineInstrBuilder SwapN(unsigned N, DstOp Dst, MachineIRBuilder &B, MachineInstrBuilder Src, const APInt &Mask)
static void emitLoadFromConstantPool(Register DstReg, const Constant *ConstVal, MachineIRBuilder &MIRBuilder)
static void getUnmergePieces(SmallVectorImpl< Register > &Pieces, MachineIRBuilder &B, Register Src, LLT Ty)
static CmpInst::Predicate minMaxToCompare(unsigned Opc)
static LegalizerHelper::LegalizeResult createAtomicLibcall(MachineIRBuilder &MIRBuilder, MachineInstr &MI)
static RTLIB::Libcall getStateLibraryFunctionFor(MachineInstr &MI, const TargetLowering &TLI)
static std::pair< int, int > getNarrowTypeBreakDown(LLT OrigTy, LLT NarrowTy, LLT &LeftoverTy)
Try to break down OrigTy into NarrowTy sized pieces.
static bool hasSameNumEltsOnAllVectorOperands(GenericMachineInstr &MI, MachineRegisterInfo &MRI, std::initializer_list< unsigned > NonVecOpIndices)
Check that all vector operands have same number of elements.
static LegalizerHelper::LegalizeResult conversionLibcall(MachineInstr &MI, MachineIRBuilder &MIRBuilder, Type *ToType, Type *FromType, LostDebugLocObserver &LocObserver)
static Register clampVectorIndex(MachineIRBuilder &B, Register IdxReg, LLT VecTy)
static RTLIB::Libcall getConvRTLibDesc(unsigned Opcode, Type *ToType, Type *FromType)
static void getUnmergeResults(SmallVectorImpl< Register > &Regs, const MachineInstr &MI)
Append the result registers of G_UNMERGE_VALUES MI to Regs.
static bool isNonZeroModBitWidthOrUndef(const MachineRegisterInfo &MRI, Register Reg, unsigned BW)
#define RTLIBCASE(LibcallPrefix)
static Type * getFloatTypeForLLT(LLVMContext &Ctx, LLT Ty)
Interface for Targets to specify which operations they can successfully select and how the others sho...
Tracks DebugLocs between checkpoints and verifies that they are transferred.
Contains matchers for matching SSA Machine Instructions.
This file declares the MachineConstantPool class which is an abstract constant pool to keep track of ...
This file declares the MachineIRBuilder class.
unsigned const TargetRegisterInfo * TRI
static unsigned getReg(const MCDisassembler *D, unsigned RC, unsigned RegNo)
static GCMetadataPrinterRegistry::Add< OcamlGCMetadataPrinter > Y("ocaml", "ocaml 3.10-compatible collector")
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
This file describes how to lower LLVM code to machine code.
opStatus convertFromAPInt(const APInt &Input, bool IsSigned, roundingMode RM)
APInt bitcastToAPInt() const
static APFloat getLargest(const fltSemantics &Sem, bool Negative=false)
Returns the largest finite number in the given semantics.
static APFloat getInf(const fltSemantics &Sem, bool Negative=false)
Factory for Positive and Negative Infinity.
Class for arbitrary precision integers.
static APInt getAllOnes(unsigned numBits)
Return an APInt of a specified width with all bits set.
APInt zext(unsigned width) const
Zero extend to a new width.
static APInt getSignMask(unsigned BitWidth)
Get the SignMask for a specific bit width.
unsigned getActiveBits() const
Compute the number of active bits in the value.
APInt trunc(unsigned width) const
Truncate to new width.
static APInt getMaxValue(unsigned numBits)
Gets maximum unsigned value of APInt for specific bit width.
bool ugt(const APInt &RHS) const
Unsigned greater than comparison.
bool isZero() const
Determine if this value is zero, i.e. all bits are clear.
APInt urem(const APInt &RHS) const
Unsigned remainder operation.
static APInt getSignedMaxValue(unsigned numBits)
Gets maximum signed value of APInt for a specific bit width.
void negate()
Negate this APInt in place.
static APInt getSplat(unsigned NewLen, const APInt &V)
Return a value containing V broadcasted over NewLen bits.
static APInt getSignedMinValue(unsigned numBits)
Gets minimum signed value of APInt for a specific bit width.
APInt sext(unsigned width) const
Sign extend to a new width.
APInt shl(unsigned shiftAmt) const
Left-shift function.
static APInt getLowBitsSet(unsigned numBits, unsigned loBitsSet)
Constructs an APInt value that has the bottom loBitsSet bits set.
static APInt getZero(unsigned numBits)
Get the '0' value for the specified bit-width.
static APInt getOneBitSet(unsigned numBits, unsigned BitNo)
Return an APInt with exactly one bit set in the result.
APInt lshr(unsigned shiftAmt) const
Logical right-shift function.
static APInt getBitsSetWithWrap(unsigned numBits, unsigned loBit, unsigned hiBit)
Wrap version of getBitsSet.
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
size_t size() const
size - Get the array size.
bool empty() const
empty - Check if the array is empty.
bool hasAttributes() const
Return true if the builder has IR-level attributes.
AttrBuilder & removeAttribute(Attribute::AttrKind Val)
Remove an attribute from the builder.
AttributeSet getRetAttrs() const
The attributes for the ret value are returned.
bool hasRetAttr(Attribute::AttrKind Kind) const
Return true if the attribute exists for the return value.
Predicate
This enumeration lists the possible predicates for CmpInst subclasses.
@ ICMP_SLT
signed less than
@ FCMP_OLT
0 1 0 0 True if ordered and less than
@ FCMP_OGE
0 0 1 1 True if ordered and greater than or equal
@ ICMP_UGE
unsigned greater or equal
@ ICMP_UGT
unsigned greater than
@ ICMP_SGT
signed greater than
@ FCMP_ULT
1 1 0 0 True if unordered or less than
@ FCMP_ONE
0 1 1 0 True if ordered and operands are unequal
@ ICMP_ULT
unsigned less than
const APFloat & getValueAPF() const
This is the shared class of boolean and integer constants.
const APInt & getValue() const
Return the constant as an APInt value reference.
This is an important base class in LLVM.
This class represents an Operation in the Expression.
A parsed version of the target data layout string in and methods for querying it.
bool isNonIntegralAddressSpace(unsigned AddrSpace) const
static constexpr ElementCount getFixed(ScalarTy MinVal)
static constexpr ElementCount get(ScalarTy MinVal, bool Scalable)
static FixedVectorType * get(Type *ElementType, unsigned NumElts)
bool hasOptSize() const
Optimize this function for size (-Os) or minimum size (-Oz).
bool hasMinSize() const
Optimize this function for minimum size (-Oz).
LLVMContext & getContext() const
getContext - Return a reference to the LLVMContext associated with this function.
Type * getReturnType() const
Returns the type of the ret val.
Represents any generic load, including sign/zero extending variants.
Register getDstReg() const
Get the definition register of the loaded value.
Abstract class that contains various methods for clients to notify about changes.
virtual void changingInstr(MachineInstr &MI)=0
This instruction is about to be mutated in some way.
virtual void changedInstr(MachineInstr &MI)=0
This instruction was mutated in some way.
Represents any type of generic load or store.
Register getPointerReg() const
Get the source register of the pointer value.
MachineMemOperand & getMMO() const
Get the MachineMemOperand on this instruction.
LocationSize getMemSize() const
Returns the size in bytes of the memory access.
bool isAtomic() const
Returns true if the attached MachineMemOperand has the atomic flag set.
Register getValueReg() const
Get the stored value register.
A base class for all GenericMachineInstrs.
Register getReg(unsigned Idx) const
Access the Idx'th operand as a register and return it.
bool isTailCall(const MachineInstr &MI) const override
bool isEquality() const
Return true if this predicate is either EQ or NE.
Predicate getUnsignedPredicate() const
For example, EQ->EQ, SLE->ULE, UGT->UGT, etc.
static IntegerType * get(LLVMContext &C, unsigned NumBits)
This static method is the primary way of constructing an IntegerType.
constexpr unsigned getScalarSizeInBits() const
constexpr bool isScalar() const
constexpr LLT changeElementType(LLT NewEltTy) const
If this type is a vector, return a vector with the same number of elements but the new element type.
static constexpr LLT vector(ElementCount EC, unsigned ScalarSizeInBits)
Get a low-level vector of some number of elements and element width.
constexpr bool isPointerVector() const
static constexpr LLT scalar(unsigned SizeInBits)
Get a low-level scalar or aggregate "bag of bits".
constexpr bool isValid() const
constexpr uint16_t getNumElements() const
Returns the number of elements in a vector LLT.
constexpr bool isVector() const
static constexpr LLT pointer(unsigned AddressSpace, unsigned SizeInBits)
Get a low-level pointer in the given address space.
constexpr bool isScalable() const
Returns true if the LLT is a scalable vector.
constexpr bool isByteSized() const
constexpr TypeSize getSizeInBits() const
Returns the total size of the type. Must only be called on sized types.
constexpr bool isPointer() const
constexpr LLT getElementType() const
Returns the vector's element type. Only valid for vector types.
constexpr ElementCount getElementCount() const
constexpr LLT changeElementSize(unsigned NewEltSize) const
If this type is a vector, return a vector with the same number of elements but the new element size.
constexpr unsigned getAddressSpace() const
static constexpr LLT fixed_vector(unsigned NumElements, unsigned ScalarSizeInBits)
Get a low-level fixed-width vector of some number of elements and element width.
constexpr LLT changeElementCount(ElementCount EC) const
Return a vector or scalar with the same element type and the new element count.
constexpr LLT getScalarType() const
constexpr TypeSize getSizeInBytes() const
Returns the total size of the type in bytes, i.e.
static constexpr LLT scalarOrVector(ElementCount EC, LLT ScalarTy)
This is an important class for using LLVM in a threaded context.
LegalizeResult lowerShlSat(MachineInstr &MI)
LegalizeResult narrowScalarCTPOP(MachineInstr &MI, unsigned TypeIdx, LLT Ty)
LegalizeResult lowerFPTRUNC_F64_TO_F16(MachineInstr &MI)
LegalizeResult equalizeVectorShuffleLengths(MachineInstr &MI)
Equalize source and destination vector sizes of G_SHUFFLE_VECTOR.
LegalizeResult bitcastInsertVectorElt(MachineInstr &MI, unsigned TypeIdx, LLT CastTy)
Perform Bitcast legalize action on G_INSERT_VECTOR_ELT.
LegalizeResult lowerSITOFP(MachineInstr &MI)
LegalizeResult lowerDynStackAlloc(MachineInstr &MI)
LegalizeResult lowerBitCount(MachineInstr &MI)
LegalizeResult narrowScalarMul(MachineInstr &MI, LLT Ty)
LegalizeResult lowerFMinNumMaxNum(MachineInstr &MI)
LegalizeResult lowerIntrinsicRound(MachineInstr &MI)
void widenScalarSrc(MachineInstr &MI, LLT WideTy, unsigned OpIdx, unsigned ExtOpcode)
Legalize a single operand OpIdx of the machine instruction MI as a Use by extending the operand's typ...
LegalizeResult moreElementsVectorShuffle(MachineInstr &MI, unsigned TypeIdx, LLT MoreTy)
LegalizeResult lowerSMULH_UMULH(MachineInstr &MI)
LegalizeResult lowerLoad(GAnyLoad &MI)
LegalizeResult fewerElementsVectorShuffle(MachineInstr &MI, unsigned TypeIdx, LLT NarrowTy)
LegalizeResult lowerAbsToAddXor(MachineInstr &MI)
void moreElementsVectorDst(MachineInstr &MI, LLT MoreTy, unsigned OpIdx)
Legalize a single operand OpIdx of the machine instruction MI as a Def by performing it with addition...
LegalizeResult lowerFConstant(MachineInstr &MI)
LegalizeResult narrowScalarCTTZ(MachineInstr &MI, unsigned TypeIdx, LLT Ty)
LegalizeResult lowerBitreverse(MachineInstr &MI)
LegalizeResult narrowScalarShift(MachineInstr &MI, unsigned TypeIdx, LLT Ty)
LegalizeResult lowerExtractInsertVectorElt(MachineInstr &MI)
Lower a vector extract or insert by writing the vector to a stack temporary and reloading the element...
LegalizeResult moreElementsVector(MachineInstr &MI, unsigned TypeIdx, LLT MoreTy)
Legalize a vector instruction by increasing the number of vector elements involved and ignoring the a...
LegalizeResult lowerFunnelShiftWithInverse(MachineInstr &MI)
LegalizeResult lowerAbsToMaxNeg(MachineInstr &MI)
LegalizeResult lowerEXT(MachineInstr &MI)
LegalizeResult lowerStore(GStore &MI)
LegalizeResult lowerAbsToCNeg(MachineInstr &MI)
LegalizeResult lowerSADDO_SSUBO(MachineInstr &MI)
MachineInstrBuilder createStackTemporary(TypeSize Bytes, Align Alignment, MachinePointerInfo &PtrInfo)
Create a stack temporary based on the size in bytes and the alignment.
void narrowScalarSrc(MachineInstr &MI, LLT NarrowTy, unsigned OpIdx)
Legalize a single operand OpIdx of the machine instruction MI as a Use by truncating the operand's ty...
LegalizeResult fewerElementsVectorPhi(GenericMachineInstr &MI, unsigned NumElts)
LegalizeResult lowerFPTOUI(MachineInstr &MI)
LegalizeResult narrowScalar(MachineInstr &MI, unsigned TypeIdx, LLT NarrowTy)
Legalize an instruction by reducing the width of the underlying scalar type.
LegalizeResult narrowScalarFPTOI(MachineInstr &MI, unsigned TypeIdx, LLT Ty)
LegalizeResult lowerUnmergeValues(MachineInstr &MI)
LegalizeResult bitcast(MachineInstr &MI, unsigned TypeIdx, LLT Ty)
Legalize an instruction by replacing the value type.
LegalizeResult lowerBitcast(MachineInstr &MI)
LegalizeResult lowerMinMax(MachineInstr &MI)
LegalizeResult lowerFunnelShiftAsShifts(MachineInstr &MI)
LegalizeResult lowerInsert(MachineInstr &MI)
LegalizeResult lowerReadWriteRegister(MachineInstr &MI)
LegalizeResult lowerExtract(MachineInstr &MI)
LegalizeResult fewerElementsBitcast(MachineInstr &MI, unsigned TypeIdx, LLT NarrowTy)
LegalizeResult narrowScalarShiftByConstant(MachineInstr &MI, const APInt &Amt, LLT HalfTy, LLT ShiftAmtTy)
LegalizeResult lowerISFPCLASS(MachineInstr &MI)
LegalizeResult lowerAddSubSatToMinMax(MachineInstr &MI)
LegalizeResult lowerFPOWI(MachineInstr &MI)
LegalizeResult narrowScalarBasic(MachineInstr &MI, unsigned TypeIdx, LLT Ty)
LegalizeResult lowerVectorReduction(MachineInstr &MI)
LegalizeResult reduceLoadStoreWidth(GLoadStore &MI, unsigned TypeIdx, LLT NarrowTy)
LegalizeResult fewerElementsVectorMultiEltType(GenericMachineInstr &MI, unsigned NumElts, std::initializer_list< unsigned > NonVecOpIndices={})
Handles most opcodes.
LegalizeResult narrowScalarSelect(MachineInstr &MI, unsigned TypeIdx, LLT Ty)
LegalizeResult lowerVAArg(MachineInstr &MI)
@ Legalized
Instruction has been legalized and the MachineFunction changed.
@ AlreadyLegal
Instruction was already legal and no change was made to the MachineFunction.
@ UnableToLegalize
Some kind of error has occurred and we could not legalize this instruction.
LegalizeResult moreElementsVectorPhi(MachineInstr &MI, unsigned TypeIdx, LLT MoreTy)
LegalizeResult lowerU64ToF32BitOps(MachineInstr &MI)
LegalizeResult lowerFCopySign(MachineInstr &MI)
LegalizerHelper(MachineFunction &MF, GISelChangeObserver &Observer, MachineIRBuilder &B)
LegalizeResult lowerRotateWithReverseRotate(MachineInstr &MI)
LegalizeResult lower(MachineInstr &MI, unsigned TypeIdx, LLT Ty)
Legalize an instruction by splitting it into simpler parts, hopefully understood by the target.
LegalizeResult lowerFunnelShift(MachineInstr &MI)
LegalizeResult fewerElementsVector(MachineInstr &MI, unsigned TypeIdx, LLT NarrowTy)
Legalize a vector instruction by splitting into multiple components, each acting on the same scalar t...
GISelChangeObserver & Observer
To keep track of changes made by the LegalizerHelper.
void bitcastDst(MachineInstr &MI, LLT CastTy, unsigned OpIdx)
Legalize a single operand OpIdx of the machine instruction MI as a def by inserting a G_BITCAST from ...
LegalizeResult lowerFPTRUNC(MachineInstr &MI)
LegalizeResult lowerFMad(MachineInstr &MI)
LegalizeResult widenScalar(MachineInstr &MI, unsigned TypeIdx, LLT WideTy)
Legalize an instruction by performing the operation on a wider scalar type (for example a 16-bit addi...
LegalizeResult lowerAddSubSatToAddoSubo(MachineInstr &MI)
LegalizeResult narrowScalarExtract(MachineInstr &MI, unsigned TypeIdx, LLT Ty)
LegalizeResult lowerFFloor(MachineInstr &MI)
LegalizeResult narrowScalarExt(MachineInstr &MI, unsigned TypeIdx, LLT Ty)
LegalizeResult fewerElementsVectorSeqReductions(MachineInstr &MI, unsigned TypeIdx, LLT NarrowTy)
Register getDynStackAllocTargetPtr(Register SPReg, Register AllocSize, Align Alignment, LLT PtrTy)
LegalizeResult lowerFPTOSI(MachineInstr &MI)
LegalizeResult lowerUITOFP(MachineInstr &MI)
LegalizeResult lowerShuffleVector(MachineInstr &MI)
LegalizeResult fewerElementsVectorMerge(MachineInstr &MI, unsigned TypeIdx, LLT NarrowTy)
LegalizeResult lowerMergeValues(MachineInstr &MI)
LegalizeResult fewerElementsVectorUnmergeValues(MachineInstr &MI, unsigned TypeIdx, LLT NarrowTy)
void moreElementsVectorSrc(MachineInstr &MI, LLT MoreTy, unsigned OpIdx)
Legalize a single operand OpIdx of the machine instruction MI as a Use by producing a vector with und...
LegalizeResult bitcastExtractVectorElt(MachineInstr &MI, unsigned TypeIdx, LLT CastTy)
Perform Bitcast legalize action on G_EXTRACT_VECTOR_ELT.
LegalizeResult lowerRotate(MachineInstr &MI)
LegalizeResult lowerMemCpyFamily(MachineInstr &MI, unsigned MaxLen=0)
Register coerceToScalar(Register Val)
Cast the given value to an LLT::scalar with an equivalent size.
LegalizeResult lowerDIVREM(MachineInstr &MI)
LegalizeResult lowerSelect(MachineInstr &MI)
LegalizeResult narrowScalarInsert(MachineInstr &MI, unsigned TypeIdx, LLT Ty)
LegalizeResult narrowScalarFLDEXP(MachineInstr &MI, unsigned TypeIdx, LLT Ty)
void bitcastSrc(MachineInstr &MI, LLT CastTy, unsigned OpIdx)
Legalize a single operand OpIdx of the machine instruction MI as a use by inserting a G_BITCAST to Ca...
void narrowScalarDst(MachineInstr &MI, LLT NarrowTy, unsigned OpIdx, unsigned ExtOpcode)
LegalizeResult libcall(MachineInstr &MI, LostDebugLocObserver &LocObserver)
Legalize an instruction by emiting a runtime library call instead.
LegalizeResult lowerStackRestore(MachineInstr &MI)
LegalizeResult fewerElementsVectorReductions(MachineInstr &MI, unsigned TypeIdx, LLT NarrowTy)
LegalizeResult lowerStackSave(MachineInstr &MI)
LegalizeResult fewerElementsVectorExtractInsertVectorElt(MachineInstr &MI, unsigned TypeIdx, LLT NarrowTy)
LegalizeResult narrowScalarCTLZ(MachineInstr &MI, unsigned TypeIdx, LLT Ty)
MachineIRBuilder & MIRBuilder
Expose MIRBuilder so clients can set their own RecordInsertInstruction functions.
LegalizeResult lowerTRUNC(MachineInstr &MI)
LegalizeResult lowerBswap(MachineInstr &MI)
Register getVectorElementPointer(Register VecPtr, LLT VecTy, Register Index)
Get a pointer to vector element Index located in memory for a vector of type VecTy starting at a base...
LegalizeResult narrowScalarAddSub(MachineInstr &MI, unsigned TypeIdx, LLT NarrowTy)
Align getStackTemporaryAlignment(LLT Type, Align MinAlign=Align()) const
Return the alignment to use for a stack temporary object with the given type.
LegalizeResult lowerConstant(MachineInstr &MI)
void widenScalarDst(MachineInstr &MI, LLT WideTy, unsigned OpIdx=0, unsigned TruncOpcode=TargetOpcode::G_TRUNC)
Legalize a single operand OpIdx of the machine instruction MI as a Def by extending the operand's typ...
LegalizeResult legalizeInstrStep(MachineInstr &MI, LostDebugLocObserver &LocObserver)
Replace MI by a sequence of legal instructions that can implement the same operation.
virtual unsigned getExtOpcodeForWideningConstant(LLT SmallTy) const
Return the opcode (SEXT/ZEXT/ANYEXT) that should be performed while widening a constant of type Small...
bool isLegalOrCustom(const LegalityQuery &Query) const
virtual bool legalizeCustom(LegalizerHelper &Helper, MachineInstr &MI, LostDebugLocObserver &LocObserver) const
Called for instructions with the Custom LegalizationAction.
virtual bool legalizeIntrinsic(LegalizerHelper &Helper, MachineInstr &MI) const
LegalizeActionStep getAction(const LegalityQuery &Query) const
Determine what action should be taken to legalize the described instruction.
TypeSize getValue() const
void checkpoint(bool CheckDebugLocs=true)
Call this to indicate that it's a good point to assess whether locations have been lost.
const MCInstrDesc & get(unsigned Opcode) const
Return the machine instruction descriptor that corresponds to the specified instruction opcode.
StringRef getName(unsigned Opcode) const
Returns the name for the instructions with the given opcode.
StringRef getString() const
static MVT getVT(Type *Ty, bool HandleUnknown=false)
Return the value type corresponding to the specified type.
iterator getFirstTerminatorForward()
Finds the first terminator in a block by scanning forward.
iterator getFirstTerminator()
Returns an iterator to the first terminator instruction of this basic block.
iterator getFirstNonPHI()
Returns a pointer to the first instruction in this block that is not a PHINode instruction.
instr_iterator instr_end()
const MachineFunction * getParent() const
Return the MachineFunction containing this basic block.
unsigned getConstantPoolIndex(const Constant *C, Align Alignment)
getConstantPoolIndex - Create a new entry in the constant pool or return an existing one.
The MachineFrameInfo class represents an abstract stack frame until prolog/epilog code is inserted.
int CreateStackObject(uint64_t Size, Align Alignment, bool isSpillSlot, const AllocaInst *Alloca=nullptr, uint8_t ID=0)
Create a new statically sized stack object, returning a nonnegative identifier to represent it.
Align getObjectAlign(int ObjectIdx) const
Return the alignment of the specified stack object.
bool isFixedObjectIndex(int ObjectIdx) const
Returns true if the specified index corresponds to a fixed stack object.
void setObjectAlignment(int ObjectIdx, Align Alignment)
setObjectAlignment - Change the alignment of the specified stack object.
const TargetSubtargetInfo & getSubtarget() const
getSubtarget - Return the subtarget for which this machine code is being compiled.
MachineMemOperand * getMachineMemOperand(MachinePointerInfo PtrInfo, MachineMemOperand::Flags f, LLT MemTy, Align base_alignment, const AAMDNodes &AAInfo=AAMDNodes(), const MDNode *Ranges=nullptr, SyncScope::ID SSID=SyncScope::System, AtomicOrdering Ordering=AtomicOrdering::NotAtomic, AtomicOrdering FailureOrdering=AtomicOrdering::NotAtomic)
getMachineMemOperand - Allocate a new MachineMemOperand.
MachineFrameInfo & getFrameInfo()
getFrameInfo - Return the frame info object for the current function.
Function & getFunction()
Return the LLVM function that this machine code represents.
const LLVMTargetMachine & getTarget() const
getTarget - Return the target machine this machine code is compiled with
MachineConstantPool * getConstantPool()
getConstantPool - Return the constant pool object for the current function.
Helper class to build MachineInstr.
MachineInstrBuilder buildFSub(const DstOp &Dst, const SrcOp &Src0, const SrcOp &Src1, std::optional< unsigned > Flags=std::nullopt)
Build and insert Res = G_FSUB Op0, Op1.
MachineInstrBuilder buildFPTOSI(const DstOp &Dst, const SrcOp &Src0)
Build and insert Res = G_FPTOSI Src0.
MachineInstrBuilder buildFMul(const DstOp &Dst, const SrcOp &Src0, const SrcOp &Src1, std::optional< unsigned > Flags=std::nullopt)
MachineInstrBuilder buildFreeze(const DstOp &Dst, const SrcOp &Src)
Build and insert Dst = G_FREEZE Src.
void setInsertPt(MachineBasicBlock &MBB, MachineBasicBlock::iterator II)
Set the insertion point before the specified position.
std::optional< MachineInstrBuilder > materializePtrAdd(Register &Res, Register Op0, const LLT ValueTy, uint64_t Value)
Materialize and insert Res = G_PTR_ADD Op0, (G_CONSTANT Value)
MachineInstrBuilder buildAdd(const DstOp &Dst, const SrcOp &Src0, const SrcOp &Src1, std::optional< unsigned > Flags=std::nullopt)
Build and insert Res = G_ADD Op0, Op1.
MachineInstrBuilder buildUndef(const DstOp &Res)
Build and insert Res = IMPLICIT_DEF.
MachineInstrBuilder buildNot(const DstOp &Dst, const SrcOp &Src0)
Build and insert a bitwise not, NegOne = G_CONSTANT -1 Res = G_OR Op0, NegOne.
MachineInstrBuilder buildAShr(const DstOp &Dst, const SrcOp &Src0, const SrcOp &Src1, std::optional< unsigned > Flags=std::nullopt)
MachineInstrBuilder buildConstantPool(const DstOp &Res, unsigned Idx)
Build and insert Res = G_CONSTANT_POOL Idx.
MachineInstrBuilder buildUnmerge(ArrayRef< LLT > Res, const SrcOp &Op)
Build and insert Res0, ... = G_UNMERGE_VALUES Op.
MachineInstrBuilder buildFAbs(const DstOp &Dst, const SrcOp &Src0, std::optional< unsigned > Flags=std::nullopt)
Build and insert Res = G_FABS Op0.
MachineInstrBuilder buildSelect(const DstOp &Res, const SrcOp &Tst, const SrcOp &Op0, const SrcOp &Op1, std::optional< unsigned > Flags=std::nullopt)
Build and insert a Res = G_SELECT Tst, Op0, Op1.
MachineInstrBuilder buildZExtInReg(const DstOp &Res, const SrcOp &Op, int64_t ImmOp)
Build and inserts Res = G_AND Op, LowBitsSet(ImmOp) Since there is no G_ZEXT_INREG like G_SEXT_INREG,...
MachineInstrBuilder buildExtract(const DstOp &Res, const SrcOp &Src, uint64_t Index)
Build and insert Res0, ... = G_EXTRACT Src, Idx0.
MachineInstrBuilder buildMul(const DstOp &Dst, const SrcOp &Src0, const SrcOp &Src1, std::optional< unsigned > Flags=std::nullopt)
Build and insert Res = G_MUL Op0, Op1.
MachineInstrBuilder buildAnd(const DstOp &Dst, const SrcOp &Src0, const SrcOp &Src1)
Build and insert Res = G_AND Op0, Op1.
MachineInstrBuilder buildICmp(CmpInst::Predicate Pred, const DstOp &Res, const SrcOp &Op0, const SrcOp &Op1)
Build and insert a Res = G_ICMP Pred, Op0, Op1.
MachineInstrBuilder buildCast(const DstOp &Dst, const SrcOp &Src)
Build and insert an appropriate cast between two registers of equal size.
const TargetInstrInfo & getTII()
MachineInstrBuilder buildURem(const DstOp &Dst, const SrcOp &Src0, const SrcOp &Src1, std::optional< unsigned > Flags=std::nullopt)
Build and insert Res = G_UREM Op0, Op1.
MachineInstrBuilder buildLShr(const DstOp &Dst, const SrcOp &Src0, const SrcOp &Src1, std::optional< unsigned > Flags=std::nullopt)
MachineInstrBuilder buildFPow(const DstOp &Dst, const SrcOp &Src0, const SrcOp &Src1, std::optional< unsigned > Flags=std::nullopt)
Build and insert Dst = G_FPOW Src0, Src1.
MachineInstrBuilder buildAnyExtOrTrunc(const DstOp &Res, const SrcOp &Op)
Res = COPY Op depending on the differing sizes of Res and Op.
MachineInstrBuilder buildSExt(const DstOp &Res, const SrcOp &Op)
Build and insert Res = G_SEXT Op.
MachineInstrBuilder buildIntrinsicTrunc(const DstOp &Dst, const SrcOp &Src0, std::optional< unsigned > Flags=std::nullopt)
Build and insert Dst = G_INTRINSIC_TRUNC Src0.
MachineBasicBlock::iterator getInsertPt()
Current insertion point for new instructions.
MachineInstrBuilder buildSExtOrTrunc(const DstOp &Res, const SrcOp &Op)
Build and insert Res = G_SEXT Op, Res = G_TRUNC Op, or Res = COPY Op depending on the differing sizes...
MachineInstrBuilder buildShuffleSplat(const DstOp &Res, const SrcOp &Src)
Build and insert a vector splat of a scalar Src using a G_INSERT_VECTOR_ELT and G_SHUFFLE_VECTOR idio...
MachineInstrBuilder buildConcatVectors(const DstOp &Res, ArrayRef< Register > Ops)
Build and insert Res = G_CONCAT_VECTORS Op0, ...
MachineInstrBuilder buildSub(const DstOp &Dst, const SrcOp &Src0, const SrcOp &Src1, std::optional< unsigned > Flags=std::nullopt)
Build and insert Res = G_SUB Op0, Op1.
MachineInstrBuilder buildCTLZ_ZERO_UNDEF(const DstOp &Dst, const SrcOp &Src0)
Build and insert Res = G_CTLZ_ZERO_UNDEF Op0, Src0.
MachineInstrBuilder buildVScale(const DstOp &Res, unsigned MinElts)
Build and insert Res = G_VSCALE MinElts.
MachineInstrBuilder buildSplatBuildVector(const DstOp &Res, const SrcOp &Src)
Build and insert Res = G_BUILD_VECTOR with Src replicated to fill the number of elements.
MachineInstrBuilder buildIntToPtr(const DstOp &Dst, const SrcOp &Src)
Build and insert a G_INTTOPTR instruction.
unsigned getBoolExtOp(bool IsVec, bool IsFP) const
MachineInstrBuilder buildBuildVector(const DstOp &Res, ArrayRef< Register > Ops)
Build and insert Res = G_BUILD_VECTOR Op0, ...
MachineInstrBuilder buildNeg(const DstOp &Dst, const SrcOp &Src0)
Build and insert integer negation Zero = G_CONSTANT 0 Res = G_SUB Zero, Op0.
MachineInstrBuilder buildCTLZ(const DstOp &Dst, const SrcOp &Src0)
Build and insert Res = G_CTLZ Op0, Src0.
MachineInstrBuilder buildSMax(const DstOp &Dst, const SrcOp &Src0, const SrcOp &Src1)
Build and insert Res = G_SMAX Op0, Op1.
MachineInstrBuilder buildAssertZExt(const DstOp &Res, const SrcOp &Op, unsigned Size)
Build and insert Res = G_ASSERT_ZEXT Op, Size.
MachineInstrBuilder buildStrictFAdd(const DstOp &Dst, const SrcOp &Src0, const SrcOp &Src1, std::optional< unsigned > Flags=std::nullopt)
Build and insert Res = G_STRICT_FADD Op0, Op1.
MachineInstrBuilder buildMergeLikeInstr(const DstOp &Res, ArrayRef< Register > Ops)
Build and insert Res = G_MERGE_VALUES Op0, ... or Res = G_BUILD_VECTOR Op0, ... or Res = G_CONCAT_VEC...
MachineInstrBuilder buildExtractVectorElement(const DstOp &Res, const SrcOp &Val, const SrcOp &Idx)
Build and insert Res = G_EXTRACT_VECTOR_ELT Val, Idx.
MachineInstrBuilder buildLoad(const DstOp &Res, const SrcOp &Addr, MachineMemOperand &MMO)
Build and insert Res = G_LOAD Addr, MMO.
MachineInstrBuilder buildPtrAdd(const DstOp &Res, const SrcOp &Op0, const SrcOp &Op1, std::optional< unsigned > Flags=std::nullopt)
Build and insert Res = G_PTR_ADD Op0, Op1.
MachineInstrBuilder buildZExtOrTrunc(const DstOp &Res, const SrcOp &Op)
Build and insert Res = G_ZEXT Op, Res = G_TRUNC Op, or Res = COPY Op depending on the differing sizes...
MachineInstrBuilder buildExtractVectorElementConstant(const DstOp &Res, const SrcOp &Val, const int Idx)
Build and insert Res = G_EXTRACT_VECTOR_ELT Val, Idx.
MachineInstrBuilder buildCTTZ_ZERO_UNDEF(const DstOp &Dst, const SrcOp &Src0)
Build and insert Res = G_CTTZ_ZERO_UNDEF Op0, Src0.
virtual MachineInstrBuilder buildFConstant(const DstOp &Res, const ConstantFP &Val)
Build and insert Res = G_FCONSTANT Val.
MachineInstrBuilder buildShl(const DstOp &Dst, const SrcOp &Src0, const SrcOp &Src1, std::optional< unsigned > Flags=std::nullopt)
MachineInstrBuilder buildUITOFP(const DstOp &Dst, const SrcOp &Src0)
Build and insert Res = G_UITOFP Src0.
MachineInstrBuilder buildStore(const SrcOp &Val, const SrcOp &Addr, MachineMemOperand &MMO)
Build and insert G_STORE Val, Addr, MMO.
MachineInstrBuilder buildAtomicCmpXchg(Register OldValRes, Register Addr, Register CmpVal, Register NewVal, MachineMemOperand &MMO)
Build and insert OldValRes<def> = G_ATOMIC_CMPXCHG Addr, CmpVal, NewVal, MMO.
MachineInstrBuilder buildInstr(unsigned Opcode)
Build and insert <empty> = Opcode <empty>.
MachineInstrBuilder buildSITOFP(const DstOp &Dst, const SrcOp &Src0)
Build and insert Res = G_SITOFP Src0.
MachineInstrBuilder buildPadVectorWithUndefElements(const DstOp &Res, const SrcOp &Op0)
Build and insert a, b, ..., x = G_UNMERGE_VALUES Op0 Res = G_BUILD_VECTOR a, b, .....
MachineInstrBuilder buildFrameIndex(const DstOp &Res, int Idx)
Build and insert Res = G_FRAME_INDEX Idx.
MachineInstrBuilder buildCTPOP(const DstOp &Dst, const SrcOp &Src0)
Build and insert Res = G_CTPOP Op0, Src0.
MachineInstrBuilder buildZExt(const DstOp &Res, const SrcOp &Op)
Build and insert Res = G_ZEXT Op.
MachineFunction & getMF()
Getter for the function we currently build.
MachineInstrBuilder buildSMin(const DstOp &Dst, const SrcOp &Src0, const SrcOp &Src1)
Build and insert Res = G_SMIN Op0, Op1.
MachineInstrBuilder buildInsert(const DstOp &Res, const SrcOp &Src, const SrcOp &Op, unsigned Index)
void setInstrAndDebugLoc(MachineInstr &MI)
Set the insertion point to before MI, and set the debug loc to MI's loc.
MachineInstrBuilder buildFCopysign(const DstOp &Dst, const SrcOp &Src0, const SrcOp &Src1)
Build and insert Res = G_FCOPYSIGN Op0, Op1.
const MachineBasicBlock & getMBB() const
Getter for the basic block we currently build.
MachineInstrBuilder buildFNeg(const DstOp &Dst, const SrcOp &Src0, std::optional< unsigned > Flags=std::nullopt)
Build and insert Res = G_FNEG Op0.
MachineInstrBuilder buildInsertVectorElement(const DstOp &Res, const SrcOp &Val, const SrcOp &Elt, const SrcOp &Idx)
Build and insert Res = G_INSERT_VECTOR_ELT Val, Elt, Idx.
MachineInstrBuilder buildAnyExt(const DstOp &Res, const SrcOp &Op)
Build and insert Res = G_ANYEXT Op0.
MachineInstrBuilder buildBitcast(const DstOp &Dst, const SrcOp &Src)
Build and insert Dst = G_BITCAST Src.
MachineInstrBuilder buildTrunc(const DstOp &Res, const SrcOp &Op)
Build and insert Res = G_TRUNC Op.
MachineInstrBuilder buildDeleteTrailingVectorElements(const DstOp &Res, const SrcOp &Op0)
Build and insert a, b, ..., x, y, z = G_UNMERGE_VALUES Op0 Res = G_BUILD_VECTOR a,...
MachineRegisterInfo * getMRI()
Getter for MRI.
MachineInstrBuilder buildFPTrunc(const DstOp &Res, const SrcOp &Op, std::optional< unsigned > Flags=std::nullopt)
Build and insert Res = G_FPTRUNC Op.
MachineInstrBuilder buildOr(const DstOp &Dst, const SrcOp &Src0, const SrcOp &Src1, std::optional< unsigned > Flags=std::nullopt)
Build and insert Res = G_OR Op0, Op1.
MachineInstrBuilder buildShuffleVector(const DstOp &Res, const SrcOp &Src1, const SrcOp &Src2, ArrayRef< int > Mask)
Build and insert Res = G_SHUFFLE_VECTOR Src1, Src2, Mask.
MachineInstrBuilder buildCopy(const DstOp &Res, const SrcOp &Op)
Build and insert Res = COPY Op.
const DataLayout & getDataLayout() const
MachineInstrBuilder buildLoadInstr(unsigned Opcode, const DstOp &Res, const SrcOp &Addr, MachineMemOperand &MMO)
Build and insert Res = <opcode> Addr, MMO.
MachineInstrBuilder buildXor(const DstOp &Dst, const SrcOp &Src0, const SrcOp &Src1)
Build and insert Res = G_XOR Op0, Op1.
MachineInstrBuilder buildMaskLowPtrBits(const DstOp &Res, const SrcOp &Op0, uint32_t NumBits)
Build and insert Res = G_PTRMASK Op0, G_CONSTANT (1 << NumBits) - 1.
virtual MachineInstrBuilder buildConstant(const DstOp &Res, const ConstantInt &Val)
Build and insert Res = G_CONSTANT Val.
MachineInstrBuilder buildUMin(const DstOp &Dst, const SrcOp &Src0, const SrcOp &Src1)
Build and insert Res = G_UMIN Op0, Op1.
MachineInstrBuilder buildFCmp(CmpInst::Predicate Pred, const DstOp &Res, const SrcOp &Op0, const SrcOp &Op1, std::optional< unsigned > Flags=std::nullopt)
Build and insert a Res = G_FCMP PredOp0, Op1.
MachineInstrBuilder buildFAdd(const DstOp &Dst, const SrcOp &Src0, const SrcOp &Src1, std::optional< unsigned > Flags=std::nullopt)
Build and insert Res = G_FADD Op0, Op1.
MachineInstrBuilder buildPtrToInt(const DstOp &Dst, const SrcOp &Src)
Build and insert a G_PTRTOINT instruction.
MachineInstrBuilder buildFCanonicalize(const DstOp &Dst, const SrcOp &Src0, std::optional< unsigned > Flags=std::nullopt)
Build and insert Dst = G_FCANONICALIZE Src0.
MachineInstrBuilder buildSExtInReg(const DstOp &Res, const SrcOp &Op, int64_t ImmOp)
Build and insert Res = G_SEXT_INREG Op, ImmOp.
Register getReg(unsigned Idx) const
Get the register for the operand index.
const MachineInstrBuilder & add(const MachineOperand &MO) const
const MachineInstrBuilder & addUse(Register RegNo, unsigned Flags=0, unsigned SubReg=0) const
Add a virtual register use operand.
const MachineInstrBuilder & addDef(Register RegNo, unsigned Flags=0, unsigned SubReg=0) const
Add a virtual register definition operand.
Representation of each machine instruction.
unsigned getOpcode() const
Returns the opcode of this MachineInstr.
bool isReturn(QueryType Type=AnyInBundle) const
bool isDebugInstr() const
unsigned getNumOperands() const
Retuns the total number of operands.
mmo_iterator memoperands_begin() const
Access to memory operands of the instruction.
void eraseFromParent()
Unlink 'this' from the containing basic block and delete it.
const MachineOperand & getOperand(unsigned i) const
A description of a memory reference used in the backend.
void setType(LLT NewTy)
Reset the tracked memory type.
LLT getMemoryType() const
Return the memory type of the memory reference.
@ MOLoad
The memory access reads data.
@ MOStore
The memory access writes data.
const MachinePointerInfo & getPointerInfo() const
LocationSize getSizeInBits() const
Return the size in bits of the memory reference.
MachineOperand class - Representation of each machine instruction operand.
static MachineOperand CreateES(const char *SymName, unsigned TargetFlags=0)
const ConstantInt * getCImm() const
void setReg(Register Reg)
Change the register this operand corresponds to.
void setCImm(const ConstantInt *CI)
Register getReg() const
getReg - Returns the register number.
const ConstantFP * getFPImm() const
MachineRegisterInfo - Keep track of information for virtual and physical registers,...
LLT getType(Register Reg) const
Get the low-level type of Reg or LLT{} if Reg is not a generic (target independent) virtual register.
Register createGenericVirtualRegister(LLT Ty, StringRef Name="")
Create and return a new generic virtual register with low-level type Ty.
static PointerType * get(Type *ElementType, unsigned AddressSpace)
This constructs a pointer to an object of the specified type in a numbered address space.
Wrapper class representing virtual and physical registers.
constexpr bool isValid() const
constexpr bool isVirtual() const
Return true if the specified register number is in the virtual register namespace.
constexpr bool isPhysical() const
Return true if the specified register number is in the physical register namespace.
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
reference emplace_back(ArgTypes &&... Args)
void reserve(size_type N)
void append(ItTy in_start, ItTy in_end)
Add the specified range to the end of the SmallVector.
iterator insert(iterator I, T &&Elt)
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
constexpr const char * data() const
data - Get a pointer to the start of the string (which may not be null terminated).
static StructType * get(LLVMContext &Context, ArrayRef< Type * > Elements, bool isPacked=false)
This static method is the primary way to create a literal StructType.
TargetInstrInfo - Interface to description of machine instruction set.
virtual MVT getVectorIdxTy(const DataLayout &DL) const
Returns the type to be used for the index operand of: ISD::INSERT_VECTOR_ELT, ISD::EXTRACT_VECTOR_ELT...
unsigned getMaxStoresPerMemcpy(bool OptSize) const
Get maximum # of store operations permitted for llvm.memcpy.
Register getStackPointerRegisterToSaveRestore() const
If a physical register, this specifies the register that llvm.savestack/llvm.restorestack should save...
virtual bool allowsMisalignedMemoryAccesses(EVT, unsigned AddrSpace=0, Align Alignment=Align(1), MachineMemOperand::Flags Flags=MachineMemOperand::MONone, unsigned *=nullptr) const
Determine if the target supports unaligned memory accesses.
virtual LLT getOptimalMemOpLLT(const MemOp &Op, const AttributeList &) const
LLT returning variant.
virtual bool isTruncateFree(Type *FromTy, Type *ToTy) const
Return true if it's free to truncate a value of type FromTy to type ToTy.
virtual bool allowsMemoryAccess(LLVMContext &Context, const DataLayout &DL, EVT VT, unsigned AddrSpace=0, Align Alignment=Align(1), MachineMemOperand::Flags Flags=MachineMemOperand::MONone, unsigned *Fast=nullptr) const
Return true if the target supports a memory access of this type for the given address space and align...
unsigned getMaxStoresPerMemmove(bool OptSize) const
Get maximum # of store operations permitted for llvm.memmove.
Align getMinStackArgumentAlignment() const
Return the minimum stack alignment of an argument.
unsigned getMaxStoresPerMemset(bool OptSize) const
Get maximum # of store operations permitted for llvm.memset.
This class defines information used to lower LLVM code to legal SelectionDAG operators that the targe...
virtual Register getRegisterByName(const char *RegName, LLT Ty, const MachineFunction &MF) const
Return the register ID of the name passed in.
const Triple & getTargetTriple() const
unsigned UnsafeFPMath
UnsafeFPMath - This flag is enabled when the -enable-unsafe-fp-math flag is specified on the command ...
TargetRegisterInfo base class - We assume that the target defines a static array of TargetRegisterDes...
virtual const CallLowering * getCallLowering() const
virtual const TargetLowering * getTargetLowering() const
bool isOSDarwin() const
Is this a "Darwin" OS (macOS, iOS, tvOS, watchOS, XROS, or DriverKit).
static constexpr TypeSize getFixed(ScalarTy ExactSize)
The instances of the Type class are immutable: once they are created, they are never changed.
static Type * getHalfTy(LLVMContext &C)
static Type * getDoubleTy(LLVMContext &C)
static Type * getX86_FP80Ty(LLVMContext &C)
static Type * getVoidTy(LLVMContext &C)
static Type * getFP128Ty(LLVMContext &C)
static IntegerType * getInt32Ty(LLVMContext &C)
static IntegerType * getInt64Ty(LLVMContext &C)
static Type * getFloatTy(LLVMContext &C)
LLVM Value Representation.
Type * getType() const
All values are typed, get the type of this value.
constexpr LeafTy divideCoefficientBy(ScalarTy RHS) const
We do not provide the '/' operator here because division for polynomial types does not work in the sa...
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
@ Fast
Attempts to make calls as fast as possible (e.g.
@ C
The default llvm calling convention, compatible with C.
@ FewerElements
The (vector) operation should be implemented by splitting it into sub-vectors where the operation is ...
@ Libcall
The operation should be implemented as a call to some kind of runtime support library.
@ WidenScalar
The operation should be implemented in terms of a wider scalar base-type.
@ Bitcast
Perform the operation on a different, but equivalently sized type.
@ NarrowScalar
The operation should be synthesized from multiple instructions acting on a narrower scalar base-type.
@ Custom
The target wants to do something special with this combination of operand and type.
@ MoreElements
The (vector) operation should be implemented by widening the input vector and ignoring the lanes adde...
ConstantMatch< APInt > m_ICst(APInt &Cst)
bool mi_match(Reg R, const MachineRegisterInfo &MRI, Pattern &&P)
Libcall getSINTTOFP(EVT OpVT, EVT RetVT)
getSINTTOFP - Return the SINTTOFP_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
Libcall getUINTTOFP(EVT OpVT, EVT RetVT)
getUINTTOFP - Return the UINTTOFP_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
Libcall
RTLIB::Libcall enum - This enum defines all of the runtime library calls the backend can emit.
Libcall getFPTOUINT(EVT OpVT, EVT RetVT)
getFPTOUINT - Return the FPTOUINT_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
Libcall getFPTOSINT(EVT OpVT, EVT RetVT)
getFPTOSINT - Return the FPTOSINT_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
Libcall getFPEXT(EVT OpVT, EVT RetVT)
getFPEXT - Return the FPEXT_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
Libcall getFPROUND(EVT OpVT, EVT RetVT)
getFPROUND - Return the FPROUND_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
This is an optimization pass for GlobalISel generic memory operations.
IterT next_nodbg(IterT It, IterT End, bool SkipPseudoOp=true)
Increment It, then continue incrementing it while it points to a debug instruction.
auto drop_begin(T &&RangeOrContainer, size_t N=1)
Return a range covering RangeOrContainer with the first N elements excluded.
detail::zippy< detail::zip_shortest, T, U, Args... > zip(T &&t, U &&u, Args &&...args)
zip iterator for two or more iteratable types.
int64_t maxIntN(int64_t N)
Gets the maximum value for a N-bit signed integer.
MachineInstr * getOpcodeDef(unsigned Opcode, Register Reg, const MachineRegisterInfo &MRI)
See if Reg is defined by an single def instruction that is Opcode.
auto size(R &&Range, std::enable_if_t< std::is_base_of< std::random_access_iterator_tag, typename std::iterator_traits< decltype(Range.begin())>::iterator_category >::value, void > *=nullptr)
Get the size of a range.
const llvm::fltSemantics & getFltSemanticForLLT(LLT Ty)
Get the appropriate floating point arithmetic semantic based on the bit size of the given scalar LLT.
MVT getMVTForLLT(LLT Ty)
Get a rough equivalent of an MVT for a given LLT.
bool matchUnaryPredicate(const MachineRegisterInfo &MRI, Register Reg, std::function< bool(const Constant *ConstVal)> Match, bool AllowUndefs=false)
Attempt to match a unary predicate against a scalar/splat constant or every element of a constant G_B...
LegalizerHelper::LegalizeResult createMemLibcall(MachineIRBuilder &MIRBuilder, MachineRegisterInfo &MRI, MachineInstr &MI, LostDebugLocObserver &LocObserver)
Create a libcall to memcpy et al.
uint64_t PowerOf2Ceil(uint64_t A)
Returns the power of two which is greater than or equal to the given value.
LLVM_READNONE LLT getLCMType(LLT OrigTy, LLT TargetTy)
Return the least common multiple type of OrigTy and TargetTy, by changing the number of vector elemen...
unsigned Log2_32(uint32_t Value)
Return the floor log base 2 of the specified value, -1 if the value is zero.
constexpr bool isPowerOf2_32(uint32_t Value)
Return true if the argument is a power of two > 0.
FPClassTest
Floating-point class tests, supported by 'is_fpclass' intrinsic.
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
LegalizerHelper::LegalizeResult createLibcall(MachineIRBuilder &MIRBuilder, const char *Name, const CallLowering::ArgInfo &Result, ArrayRef< CallLowering::ArgInfo > Args, CallingConv::ID CC, LostDebugLocObserver &LocObserver, MachineInstr *MI=nullptr)
Helper function that creates a libcall to the given Name using the given calling convention CC.
void extractParts(Register Reg, LLT Ty, int NumParts, SmallVectorImpl< Register > &VRegs, MachineIRBuilder &MIRBuilder, MachineRegisterInfo &MRI)
Helper function to split a wide generic register into bitwise blocks with the given Type (which impli...
@ Mul
Product of integers.
@ Xor
Bitwise or logical XOR of integers.
uint64_t alignTo(uint64_t Size, Align A)
Returns a multiple of A needed to store Size bytes.
constexpr uint64_t MinAlign(uint64_t A, uint64_t B)
A and B are either alignments or offsets.
int64_t minIntN(int64_t N)
Gets the minimum value for a N-bit signed integer.
std::optional< ValueAndVReg > getIConstantVRegValWithLookThrough(Register VReg, const MachineRegisterInfo &MRI, bool LookThroughInstrs=true)
If VReg is defined by a statically evaluable chain of instructions rooted on a G_CONSTANT returns its...
bool isKnownNeverSNaN(Register Val, const MachineRegisterInfo &MRI)
Returns true if Val can be assumed to never be a signaling NaN.
bool is_contained(R &&Range, const E &Element)
Returns true if Element is found in Range.
Align commonAlignment(Align A, uint64_t Offset)
Returns the alignment that satisfies both alignments.
Align assumeAligned(uint64_t Value)
Treats the value 0 as a 1, so Align is always at least 1.
unsigned Log2(Align A)
Returns the log2 of the alignment.
LLVM_READNONE LLT getGCDType(LLT OrigTy, LLT TargetTy)
Return a type where the total size is the greatest common divisor of OrigTy and TargetTy.
T bit_floor(T Value)
Returns the largest integral power of two no greater than Value if Value is nonzero.
void extractVectorParts(Register Reg, unsigned NumElts, SmallVectorImpl< Register > &VRegs, MachineIRBuilder &MIRBuilder, MachineRegisterInfo &MRI)
Version which handles irregular sub-vector splits.
static const fltSemantics & IEEEsingle() LLVM_READNONE
static constexpr roundingMode rmNearestTiesToEven
static const fltSemantics & IEEEdouble() LLVM_READNONE
This struct is a compact representation of a valid (non-zero power of two) alignment.
uint64_t value() const
This is a hole in the type system and should not be abused.
The LegalityQuery object bundles together all the information that's needed to decide whether a given...
LegalizeAction Action
The action to take or the final answer.
This class contains a discriminated union of information about pointers in memory operands,...
unsigned getAddrSpace() const
Return the LLVM IR address space number that this pointer points into.
static MachinePointerInfo getConstantPool(MachineFunction &MF)
Return a MachinePointerInfo record that refers to the constant pool.
MachinePointerInfo getWithOffset(int64_t O) const
static MachinePointerInfo getFixedStack(MachineFunction &MF, int FI, int64_t Offset=0)
Return a MachinePointerInfo record that refers to the specified FrameIndex.
static MemOp Set(uint64_t Size, bool DstAlignCanChange, Align DstAlign, bool IsZeroMemset, bool IsVolatile)
static MemOp Copy(uint64_t Size, bool DstAlignCanChange, Align DstAlign, Align SrcAlign, bool IsVolatile, bool MemcpyStrSrc=false)