44#define DEBUG_TYPE "gi-combiner"
47using namespace MIPatternMatch;
53 cl::desc(
"Force all indexed operations to be "
54 "legal for the GlobalISel combiner"));
60 : Builder(
B),
MRI(Builder.getMF().getRegInfo()), Observer(Observer), KB(KB),
61 MDT(MDT), IsPreLegalize(IsPreLegalize), LI(LI),
62 RBI(Builder.getMF().getSubtarget().getRegBankInfo()),
63 TRI(Builder.getMF().getSubtarget().getRegisterInfo()) {
76 assert(
I < ByteWidth &&
"I must be in [0, ByteWidth)");
95 assert(
I < ByteWidth &&
"I must be in [0, ByteWidth)");
96 return ByteWidth -
I - 1;
116static std::optional<bool>
120 unsigned Width = MemOffset2Idx.
size();
123 bool BigEndian =
true, LittleEndian =
true;
124 for (
unsigned MemOffset = 0; MemOffset < Width; ++ MemOffset) {
125 auto MemOffsetAndIdx = MemOffset2Idx.
find(MemOffset);
126 if (MemOffsetAndIdx == MemOffset2Idx.
end())
128 const int64_t
Idx = MemOffsetAndIdx->second - LowestIdx;
129 assert(
Idx >= 0 &&
"Expected non-negative byte offset?");
132 if (!BigEndian && !LittleEndian)
136 assert((BigEndian != LittleEndian) &&
137 "Pattern cannot be both big and little endian!");
144 assert(
LI &&
"Must have LegalizerInfo to query isLegal!");
160 return isLegal({TargetOpcode::G_BUILD_VECTOR, {Ty, EltTy}}) &&
161 isLegal({TargetOpcode::G_CONSTANT, {EltTy}});
188 unsigned ToOpcode)
const {
213 if (
MI.getOpcode() != TargetOpcode::COPY)
222 MI.eraseFromParent();
228 assert(
MI.getOpcode() == TargetOpcode::G_CONCAT_VECTORS &&
229 "Invalid instruction");
239 assert(Def &&
"Operand not defined");
242 switch (Def->getOpcode()) {
243 case TargetOpcode::G_BUILD_VECTOR:
250 case TargetOpcode::G_IMPLICIT_DEF: {
259 "All undefs should have the same type");
263 EltIdx != EltEnd; ++EltIdx)
264 Ops.
push_back(Undef->getOperand(0).getReg());
275 {TargetOpcode::G_BUILD_VECTOR, {DstTy,
MRI.
getType(Ops[0])}})) {
302 MI.eraseFromParent();
317 assert(
MI.getOpcode() == TargetOpcode::G_SHUFFLE_VECTOR &&
318 "Invalid instruction kind");
343 if (DstNumElts < 2 * SrcNumElts && DstNumElts != 1)
348 if (DstNumElts % SrcNumElts != 0)
354 unsigned NumConcat = DstNumElts / SrcNumElts;
357 for (
unsigned i = 0; i != DstNumElts; ++i) {
364 if ((
Idx % SrcNumElts != (i % SrcNumElts)) ||
365 (ConcatSrcs[i / SrcNumElts] >= 0 &&
366 ConcatSrcs[i / SrcNumElts] != (
int)(
Idx / SrcNumElts)))
369 ConcatSrcs[i / SrcNumElts] =
Idx / SrcNumElts;
376 for (
auto Src : ConcatSrcs) {
402 MI.eraseFromParent();
407 assert(
MI.getOpcode() == TargetOpcode::G_SHUFFLE_VECTOR &&
408 "Invalid instruction kind");
411 return Mask.size() == 1;
418 int I =
MI.getOperand(3).getShuffleMask()[0];
423 if (
I >= Src1NumElts) {
424 SrcReg =
MI.getOperand(2).getReg();
436 MI.eraseFromParent();
445 const LLT TyForCandidate,
446 unsigned OpcodeForCandidate,
451 return {TyForCandidate, OpcodeForCandidate, MIForCandidate};
462 if (OpcodeForCandidate == TargetOpcode::G_ANYEXT &&
465 else if (CurrentUse.
ExtendOpcode == TargetOpcode::G_ANYEXT &&
466 OpcodeForCandidate != TargetOpcode::G_ANYEXT)
467 return {TyForCandidate, OpcodeForCandidate, MIForCandidate};
473 if (!isa<GZExtLoad>(LoadMI) && CurrentUse.
Ty == TyForCandidate) {
475 OpcodeForCandidate == TargetOpcode::G_ZEXT)
477 else if (CurrentUse.
ExtendOpcode == TargetOpcode::G_ZEXT &&
478 OpcodeForCandidate == TargetOpcode::G_SEXT)
479 return {TyForCandidate, OpcodeForCandidate, MIForCandidate};
488 return {TyForCandidate, OpcodeForCandidate, MIForCandidate};
499static void InsertInsnsWithoutSideEffectsBeforeUse(
511 InsertBB = PredBB->
getMBB();
516 if (InsertBB ==
DefMI.getParent()) {
518 Inserter(InsertBB, std::next(InsertPt), UseMO);
537 unsigned CandidateLoadOpc;
539 case TargetOpcode::G_ANYEXT:
540 CandidateLoadOpc = TargetOpcode::G_LOAD;
542 case TargetOpcode::G_SEXT:
543 CandidateLoadOpc = TargetOpcode::G_SEXTLOAD;
545 case TargetOpcode::G_ZEXT:
546 CandidateLoadOpc = TargetOpcode::G_ZEXTLOAD;
551 return CandidateLoadOpc;
582 if (!llvm::has_single_bit<uint32_t>(LoadValueTy.
getSizeInBits()))
590 unsigned PreferredOpcode =
592 ? TargetOpcode::G_ANYEXT
593 : isa<GSExtLoad>(&
MI) ? TargetOpcode::G_SEXT : TargetOpcode::G_ZEXT;
594 Preferred = {
LLT(), PreferredOpcode,
nullptr};
596 if (
UseMI.getOpcode() == TargetOpcode::G_SEXT ||
597 UseMI.getOpcode() == TargetOpcode::G_ZEXT ||
598 (
UseMI.getOpcode() == TargetOpcode::G_ANYEXT)) {
599 const auto &MMO = LoadMI->
getMMO();
601 if (MMO.isAtomic() &&
UseMI.getOpcode() != TargetOpcode::G_ANYEXT)
609 if (
LI->
getAction({CandidateLoadOpc, {UseTy, SrcTy}, {MMDesc}})
613 Preferred = ChoosePreferredUse(
MI, Preferred,
624 assert(Preferred.Ty != LoadValueTy &&
"Extending to same type?");
642 if (PreviouslyEmitted) {
652 EmittedInsns[InsertIntoBB] = NewMI;
661 auto &LoadValue =
MI.getOperand(0);
664 Uses.push_back(&UseMO);
666 for (
auto *UseMO :
Uses) {
676 if (UseDstReg != ChosenDstReg) {
677 if (Preferred.
Ty == UseDstTy) {
714 InsertInsnsWithoutSideEffectsBeforeUse(
Builder,
MI, *UseMO,
729 InsertInsnsWithoutSideEffectsBeforeUse(
Builder,
MI, *UseMO, InsertTruncAt);
732 MI.getOperand(0).setReg(ChosenDstReg);
738 assert(
MI.getOpcode() == TargetOpcode::G_AND);
757 APInt MaskVal = MaybeMask->Value;
778 if (MaskSizeBits > LoadSizeBits.
getValue())
798 else if (LoadSizeBits.
getValue() > MaskSizeBits ||
804 {TargetOpcode::G_ZEXTLOAD, {RegTy,
MRI.
getType(PtrReg)}, {MemDesc}}))
808 B.setInstrAndDebugLoc(*LoadMI);
809 auto &MF =
B.getMF();
811 auto *NewMMO = MF.getMachineMemOperand(&MMO, PtrInfo, MemDesc.
MemoryTy);
812 B.buildLoadInstr(TargetOpcode::G_ZEXTLOAD, Dst, PtrReg, *NewMMO);
821 "shouldn't consider debug uses");
829 if (DefOrUse ==
MBB.
end())
831 return &*DefOrUse == &
DefMI;
837 "shouldn't consider debug uses");
840 else if (
DefMI.getParent() !=
UseMI.getParent())
847 assert(
MI.getOpcode() == TargetOpcode::G_SEXT_INREG);
858 uint64_t SizeInBits =
MI.getOperand(2).getImm();
861 if (
auto *LoadMI = getOpcodeDef<GSExtLoad>(LoadUser,
MRI)) {
863 auto LoadSizeBits = LoadMI->getMemSizeInBits();
867 if (LoadSizeBits == SizeInBits)
874 assert(
MI.getOpcode() == TargetOpcode::G_SEXT_INREG);
877 MI.eraseFromParent();
882 assert(
MI.getOpcode() == TargetOpcode::G_SEXT_INREG);
892 auto *LoadDef = getOpcodeDef<GLoad>(SrcReg,
MRI);
896 uint64_t MemBits = LoadDef->getMemSizeInBits().getValue();
901 unsigned NewSizeBits = std::min((
uint64_t)
MI.getOperand(2).getImm(), MemBits);
916 if (LoadDef->isSimple())
918 else if (MemBits > NewSizeBits || MemBits == RegTy.
getSizeInBits())
928 MatchInfo = std::make_tuple(LoadDef->getDstReg(), NewSizeBits);
934 assert(
MI.getOpcode() == TargetOpcode::G_SEXT_INREG);
936 unsigned ScalarSizeBits;
937 std::tie(LoadReg, ScalarSizeBits) = MatchInfo;
946 auto &MMO = LoadDef->
getMMO();
949 auto PtrInfo = MMO.getPointerInfo();
953 MI.eraseFromParent();
968 auto *MF =
MI->getMF();
969 auto *
Addr = getOpcodeDef<GPtrAdd>(
MI->getPointerReg(),
MRI);
975 AM.
BaseOffs = CstOff->getSExtValue();
980 MF->getDataLayout(), AM,
982 MF->getFunction().getContext()),
983 MI->getMMO().getAddrSpace());
988 case TargetOpcode::G_LOAD:
989 return TargetOpcode::G_INDEXED_LOAD;
990 case TargetOpcode::G_STORE:
991 return TargetOpcode::G_INDEXED_STORE;
992 case TargetOpcode::G_ZEXTLOAD:
993 return TargetOpcode::G_INDEXED_ZEXTLOAD;
994 case TargetOpcode::G_SEXTLOAD:
995 return TargetOpcode::G_INDEXED_SEXTLOAD;
1001bool CombinerHelper::isIndexedLoadStoreLegal(
GLoadStore &LdSt)
const {
1010 if (IndexedOpc == TargetOpcode::G_INDEXED_STORE)
1011 OpTys = {PtrTy, Ty, Ty};
1013 OpTys = {Ty, PtrTy};
1021 cl::desc(
"Number of uses of a base pointer to check before it is no longer "
1022 "considered for post-indexing."));
1026 bool &RematOffset) {
1039 if (!isIndexedLoadStoreLegal(LdSt))
1048 unsigned NumUsesChecked = 0;
1053 auto *PtrAdd = dyn_cast<GPtrAdd>(&
Use);
1061 if (StoredValDef == &
Use)
1064 Offset = PtrAdd->getOffsetReg();
1066 !TLI.isIndexingLegal(LdSt, PtrAdd->getBaseReg(),
Offset,
1072 RematOffset =
false;
1076 if (OffsetDef->
getOpcode() != TargetOpcode::G_CONSTANT)
1082 if (&BasePtrUse == PtrDef)
1087 auto *BasePtrLdSt = dyn_cast<GLoadStore>(&BasePtrUse);
1088 if (BasePtrLdSt && BasePtrLdSt != &LdSt &&
1090 isIndexedLoadStoreLegal(*BasePtrLdSt))
1095 if (
auto *BasePtrUseDef = dyn_cast<GPtrAdd>(&BasePtrUse)) {
1096 Register PtrAddDefReg = BasePtrUseDef->getReg(0);
1100 if (BaseUseUse.getParent() != LdSt.
getParent())
1103 if (
auto *UseUseLdSt = dyn_cast<GLoadStore>(&BaseUseUse))
1112 Addr = PtrAdd->getReg(0);
1113 Base = PtrAdd->getBaseReg();
1134 if (!isIndexedLoadStoreLegal(LdSt))
1138 if (BaseDef->
getOpcode() == TargetOpcode::G_FRAME_INDEX)
1141 if (
auto *St = dyn_cast<GStore>(&LdSt)) {
1143 if (
Base == St->getValueReg())
1148 if (St->getValueReg() ==
Addr)
1154 if (AddrUse.getParent() != LdSt.
getParent())
1159 bool RealUse =
false;
1166 if (
auto *UseLdSt = dyn_cast<GLoadStore>(&AddrUse)) {
1178 assert(
MI.getOpcode() == TargetOpcode::G_EXTRACT_VECTOR_ELT);
1181 auto *LoadMI = getOpcodeDef<GLoad>(
MI.getOperand(1).getReg(),
MRI);
1195 if (!LoadMI->isSimple())
1207 const unsigned MaxIter = 20;
1209 for (
auto II = LoadMI->
getIterator(), IE =
MI.getIterator(); II != IE; ++II) {
1210 if (II->isLoadFoldBarrier())
1212 if (Iter++ == MaxIter)
1228 int Elt = CVal->getZExtValue();
1241 Register VecPtr = LoadMI->getPointerReg();
1249 LegalityQuery Q = {TargetOpcode::G_LOAD, {VecEltTy, PtrTy}, {MMDesc}};
1274 B.buildLoad(Result, finalPtr, PtrInfo, Alignment);
1284 auto &LdSt = cast<GLoadStore>(
MI);
1289 MatchInfo.
IsPre = findPreIndexCandidate(LdSt, MatchInfo.
Addr, MatchInfo.
Base,
1291 if (!MatchInfo.
IsPre &&
1292 !findPostIndexCandidate(LdSt, MatchInfo.
Addr, MatchInfo.
Base,
1303 unsigned Opcode =
MI.getOpcode();
1304 bool IsStore = Opcode == TargetOpcode::G_STORE;
1312 *OldCst->getOperand(1).getCImm());
1313 MatchInfo.
Offset = NewCst.getReg(0);
1319 MIB.
addUse(
MI.getOperand(0).getReg());
1321 MIB.
addDef(
MI.getOperand(0).getReg());
1329 MI.eraseFromParent();
1337 unsigned Opcode =
MI.getOpcode();
1338 bool IsDiv, IsSigned;
1343 case TargetOpcode::G_SDIV:
1344 case TargetOpcode::G_UDIV: {
1346 IsSigned = Opcode == TargetOpcode::G_SDIV;
1349 case TargetOpcode::G_SREM:
1350 case TargetOpcode::G_UREM: {
1352 IsSigned = Opcode == TargetOpcode::G_SREM;
1358 unsigned DivOpcode, RemOpcode, DivremOpcode;
1360 DivOpcode = TargetOpcode::G_SDIV;
1361 RemOpcode = TargetOpcode::G_SREM;
1362 DivremOpcode = TargetOpcode::G_SDIVREM;
1364 DivOpcode = TargetOpcode::G_UDIV;
1365 RemOpcode = TargetOpcode::G_UREM;
1366 DivremOpcode = TargetOpcode::G_UDIVREM;
1385 if (
MI.getParent() ==
UseMI.getParent() &&
1386 ((IsDiv &&
UseMI.getOpcode() == RemOpcode) ||
1387 (!IsDiv &&
UseMI.getOpcode() == DivOpcode)) &&
1400 unsigned Opcode =
MI.getOpcode();
1401 assert(OtherMI &&
"OtherMI shouldn't be empty.");
1404 if (Opcode == TargetOpcode::G_SDIV || Opcode == TargetOpcode::G_UDIV) {
1405 DestDivReg =
MI.getOperand(0).getReg();
1409 DestRemReg =
MI.getOperand(0).getReg();
1413 Opcode == TargetOpcode::G_SDIV || Opcode == TargetOpcode::G_SREM;
1425 FirstInst = OtherMI;
1429 : TargetOpcode::G_UDIVREM,
1430 {DestDivReg, DestRemReg},
1432 MI.eraseFromParent();
1438 assert(
MI.getOpcode() == TargetOpcode::G_BR);
1457 assert(std::next(BrIt) ==
MBB->
end() &&
"expected G_BR to be a terminator");
1459 BrCond = &*std::prev(BrIt);
1460 if (BrCond->
getOpcode() != TargetOpcode::G_BRCOND)
1466 return BrCondTarget !=
MI.getOperand(0).getMBB() &&
1484 MI.getOperand(0).setMBB(FallthroughBB);
1500 return Helper.lowerMemcpyInline(
MI) ==
1516 switch (
MI.getOpcode()) {
1519 case TargetOpcode::G_FNEG: {
1520 Result.changeSign();
1523 case TargetOpcode::G_FABS: {
1527 case TargetOpcode::G_FPTRUNC: {
1529 LLT DstTy =
MRI.getType(
MI.getOperand(0).getReg());
1534 case TargetOpcode::G_FSQRT: {
1538 Result =
APFloat(sqrt(Result.convertToDouble()));
1541 case TargetOpcode::G_FLOG2: {
1563 MI.eraseFromParent();
1574 if (
MI.getOpcode() != TargetOpcode::G_PTR_ADD)
1584 if (!Add2Def || Add2Def->
getOpcode() != TargetOpcode::G_PTR_ADD)
1597 Type *AccessTy =
nullptr;
1598 auto &MF = *
MI.getMF();
1600 if (
auto *LdSt = dyn_cast<GLoadStore>(&
UseMI)) {
1602 MF.getFunction().getContext());
1607 APInt CombinedImm = MaybeImmVal->Value + MaybeImm2Val->Value;
1612 AMOld.
BaseOffs = MaybeImmVal->Value.getSExtValue();
1615 const auto &TLI = *MF.getSubtarget().getTargetLowering();
1616 if (TLI.isLegalAddressingMode(MF.getDataLayout(), AMOld, AccessTy, AS) &&
1617 !TLI.isLegalAddressingMode(MF.getDataLayout(), AMNew, AccessTy, AS))
1630 assert(
MI.getOpcode() == TargetOpcode::G_PTR_ADD &&
"Expected G_PTR_ADD");
1636 MI.getOperand(1).setReg(MatchInfo.
Base);
1637 MI.getOperand(2).setReg(NewOffset.getReg(0));
1650 unsigned Opcode =
MI.getOpcode();
1651 assert((Opcode == TargetOpcode::G_SHL || Opcode == TargetOpcode::G_ASHR ||
1652 Opcode == TargetOpcode::G_LSHR || Opcode == TargetOpcode::G_SSHLSAT ||
1653 Opcode == TargetOpcode::G_USHLSAT) &&
1654 "Expected G_SHL, G_ASHR, G_LSHR, G_SSHLSAT or G_USHLSAT");
1674 (MaybeImmVal->Value.getZExtValue() + MaybeImm2Val->Value).getZExtValue();
1679 if (Opcode == TargetOpcode::G_USHLSAT &&
1688 unsigned Opcode =
MI.getOpcode();
1689 assert((Opcode == TargetOpcode::G_SHL || Opcode == TargetOpcode::G_ASHR ||
1690 Opcode == TargetOpcode::G_LSHR || Opcode == TargetOpcode::G_SSHLSAT ||
1691 Opcode == TargetOpcode::G_USHLSAT) &&
1692 "Expected G_SHL, G_ASHR, G_LSHR, G_SSHLSAT or G_USHLSAT");
1697 auto Imm = MatchInfo.
Imm;
1699 if (Imm >= ScalarSizeInBits) {
1701 if (Opcode == TargetOpcode::G_SHL || Opcode == TargetOpcode::G_LSHR) {
1703 MI.eraseFromParent();
1708 Imm = ScalarSizeInBits - 1;
1714 MI.getOperand(1).setReg(MatchInfo.
Reg);
1715 MI.getOperand(2).setReg(NewImm);
1731 unsigned ShiftOpcode =
MI.getOpcode();
1732 assert((ShiftOpcode == TargetOpcode::G_SHL ||
1733 ShiftOpcode == TargetOpcode::G_ASHR ||
1734 ShiftOpcode == TargetOpcode::G_LSHR ||
1735 ShiftOpcode == TargetOpcode::G_USHLSAT ||
1736 ShiftOpcode == TargetOpcode::G_SSHLSAT) &&
1737 "Expected G_SHL, G_ASHR, G_LSHR, G_USHLSAT and G_SSHLSAT");
1740 Register LogicDest =
MI.getOperand(1).getReg();
1745 unsigned LogicOpcode = LogicMI->
getOpcode();
1746 if (LogicOpcode != TargetOpcode::G_AND && LogicOpcode != TargetOpcode::G_OR &&
1747 LogicOpcode != TargetOpcode::G_XOR)
1751 const Register C1 =
MI.getOperand(2).getReg();
1753 if (!MaybeImmVal || MaybeImmVal->Value == 0)
1756 const uint64_t C1Val = MaybeImmVal->Value.getZExtValue();
1760 if (
MI->getOpcode() != ShiftOpcode ||
1770 ShiftVal = MaybeImmVal->Value.getSExtValue();
1781 if (matchFirstShift(LogicMIOp1, C0Val)) {
1783 MatchInfo.
Shift2 = LogicMIOp1;
1784 }
else if (matchFirstShift(LogicMIOp2, C0Val)) {
1786 MatchInfo.
Shift2 = LogicMIOp2;
1790 MatchInfo.
ValSum = C0Val + C1Val;
1796 MatchInfo.
Logic = LogicMI;
1802 unsigned Opcode =
MI.getOpcode();
1803 assert((Opcode == TargetOpcode::G_SHL || Opcode == TargetOpcode::G_ASHR ||
1804 Opcode == TargetOpcode::G_LSHR || Opcode == TargetOpcode::G_USHLSAT ||
1805 Opcode == TargetOpcode::G_SSHLSAT) &&
1806 "Expected G_SHL, G_ASHR, G_LSHR, G_USHLSAT and G_SSHLSAT");
1825 Register Shift2Const =
MI.getOperand(2).getReg();
1837 MI.eraseFromParent();
1841 assert(
MI.getOpcode() == TargetOpcode::G_SHL &&
"Expected G_SHL");
1844 auto &Shl = cast<GenericMachineInstr>(
MI);
1864 assert((SrcDef->getOpcode() == TargetOpcode::G_ADD ||
1865 SrcDef->getOpcode() == TargetOpcode::G_OR) &&
"Unexpected op");
1868 auto S1 =
B.buildShl(SrcTy,
X, ShiftReg);
1869 auto S2 =
B.buildShl(SrcTy, C1, ShiftReg);
1870 B.buildInstr(SrcDef->getOpcode(), {DstReg}, {
S1, S2});
1876 unsigned &ShiftVal) {
1877 assert(
MI.getOpcode() == TargetOpcode::G_MUL &&
"Expected a G_MUL");
1883 ShiftVal = MaybeImmVal->Value.exactLogBase2();
1884 return (
static_cast<int32_t
>(ShiftVal) != -1);
1888 unsigned &ShiftVal) {
1889 assert(
MI.getOpcode() == TargetOpcode::G_MUL &&
"Expected a G_MUL");
1894 MI.setDesc(MIB.
getTII().
get(TargetOpcode::G_SHL));
1895 MI.getOperand(2).setReg(ShiftCst.getReg(0));
1902 assert(
MI.getOpcode() == TargetOpcode::G_SHL &&
KB);
1917 if (!MaybeShiftAmtVal)
1931 int64_t ShiftAmt = MaybeShiftAmtVal->getSExtValue();
1932 MatchData.
Reg = ExtSrc;
1933 MatchData.
Imm = ShiftAmt;
1937 return MinLeadingZeros >= ShiftAmt && ShiftAmt < SrcTySize;
1943 int64_t ShiftAmtVal = MatchData.
Imm;
1951 MI.eraseFromParent();
1958 for (
unsigned I = 0;
I <
Merge.getNumSources(); ++
I)
1961 auto *Unmerge = getOpcodeDef<GUnmerge>(MergedValues[0],
MRI);
1962 if (!Unmerge || Unmerge->getNumDefs() !=
Merge.getNumSources())
1965 for (
unsigned I = 0;
I < MergedValues.
size(); ++
I)
1966 if (MergedValues[
I] != Unmerge->getReg(
I))
1969 MatchInfo = Unmerge->getSourceReg();
1983 assert(
MI.getOpcode() == TargetOpcode::G_UNMERGE_VALUES &&
1984 "Expected an unmerge");
1985 auto &Unmerge = cast<GUnmerge>(
MI);
1988 auto *SrcInstr = getOpcodeDef<GMergeLikeInstr>(SrcReg,
MRI);
1996 if (SrcMergeTy != Dst0Ty && !SameSize)
2000 for (
unsigned Idx = 0;
Idx < SrcInstr->getNumSources(); ++
Idx)
2007 assert(
MI.getOpcode() == TargetOpcode::G_UNMERGE_VALUES &&
2008 "Expected an unmerge");
2010 "Not enough operands to replace all defs");
2011 unsigned NumElems =
MI.getNumOperands() - 1;
2015 bool CanReuseInputDirectly = DstTy == SrcTy;
2017 for (
unsigned Idx = 0;
Idx < NumElems; ++
Idx) {
2029 if (CanReuseInputDirectly)
2034 MI.eraseFromParent();
2039 unsigned SrcIdx =
MI.getNumOperands() - 1;
2040 Register SrcReg =
MI.getOperand(SrcIdx).getReg();
2042 if (SrcInstr->
getOpcode() != TargetOpcode::G_CONSTANT &&
2043 SrcInstr->
getOpcode() != TargetOpcode::G_FCONSTANT)
2054 for (
unsigned Idx = 0;
Idx != SrcIdx; ++
Idx) {
2056 Val = Val.
lshr(ShiftAmt);
2064 assert(
MI.getOpcode() == TargetOpcode::G_UNMERGE_VALUES &&
2065 "Expected an unmerge");
2067 "Not enough operands to replace all defs");
2068 unsigned NumElems =
MI.getNumOperands() - 1;
2070 for (
unsigned Idx = 0;
Idx < NumElems; ++
Idx) {
2075 MI.eraseFromParent();
2080 unsigned SrcIdx =
MI.getNumOperands() - 1;
2081 Register SrcReg =
MI.getOperand(SrcIdx).getReg();
2083 unsigned NumElems =
MI.getNumOperands() - 1;
2084 for (
unsigned Idx = 0;
Idx < NumElems; ++
Idx) {
2086 B.buildUndef(DstReg);
2093 assert(
MI.getOpcode() == TargetOpcode::G_UNMERGE_VALUES &&
2094 "Expected an unmerge");
2099 for (
unsigned Idx = 1, EndIdx =
MI.getNumDefs();
Idx != EndIdx; ++
Idx) {
2108 Register SrcReg =
MI.getOperand(
MI.getNumDefs()).getReg();
2109 Register Dst0Reg =
MI.getOperand(0).getReg();
2111 MI.eraseFromParent();
2115 assert(
MI.getOpcode() == TargetOpcode::G_UNMERGE_VALUES &&
2116 "Expected an unmerge");
2117 Register Dst0Reg =
MI.getOperand(0).getReg();
2124 Register SrcReg =
MI.getOperand(
MI.getNumDefs()).getReg();
2141 assert(
MI.getOpcode() == TargetOpcode::G_UNMERGE_VALUES &&
2142 "Expected an unmerge");
2144 Register Dst0Reg =
MI.getOperand(0).getReg();
2149 "Expecting a G_ZEXT");
2161 "ZExt src doesn't fit in destination");
2166 for (
unsigned Idx = 1, EndIdx =
MI.getNumDefs();
Idx != EndIdx; ++
Idx) {
2171 MI.eraseFromParent();
2175 unsigned TargetShiftSize,
2176 unsigned &ShiftVal) {
2177 assert((
MI.getOpcode() == TargetOpcode::G_SHL ||
2178 MI.getOpcode() == TargetOpcode::G_LSHR ||
2179 MI.getOpcode() == TargetOpcode::G_ASHR) &&
"Expected a shift");
2187 if (
Size <= TargetShiftSize)
2195 ShiftVal = MaybeImmVal->Value.getSExtValue();
2196 return ShiftVal >=
Size / 2 && ShiftVal <
Size;
2200 const unsigned &ShiftVal) {
2205 unsigned HalfSize =
Size / 2;
2206 assert(ShiftVal >= HalfSize);
2212 unsigned NarrowShiftAmt = ShiftVal - HalfSize;
2214 if (
MI.getOpcode() == TargetOpcode::G_LSHR) {
2215 Register Narrowed = Unmerge.getReg(1);
2222 if (NarrowShiftAmt != 0) {
2229 }
else if (
MI.getOpcode() == TargetOpcode::G_SHL) {
2230 Register Narrowed = Unmerge.getReg(0);
2235 if (NarrowShiftAmt != 0) {
2243 assert(
MI.getOpcode() == TargetOpcode::G_ASHR);
2245 HalfTy, Unmerge.getReg(1),
2248 if (ShiftVal == HalfSize) {
2252 }
else if (ShiftVal ==
Size - 1) {
2260 HalfTy, Unmerge.getReg(1),
2269 MI.eraseFromParent();
2273 unsigned TargetShiftAmount) {
2284 assert(
MI.getOpcode() == TargetOpcode::G_INTTOPTR &&
"Expected a G_INTTOPTR");
2293 assert(
MI.getOpcode() == TargetOpcode::G_INTTOPTR &&
"Expected a G_INTTOPTR");
2297 MI.eraseFromParent();
2301 assert(
MI.getOpcode() == TargetOpcode::G_PTRTOINT &&
"Expected a G_PTRTOINT");
2305 MI.eraseFromParent();
2310 assert(
MI.getOpcode() == TargetOpcode::G_ADD);
2317 PtrReg.second =
false;
2327 PtrReg.second =
true;
2339 const bool DoCommute = PtrReg.second;
2349 MI.eraseFromParent();
2354 auto &PtrAdd = cast<GPtrAdd>(
MI);
2365 NewCst += RHSCst->
sextOrTrunc(DstTy.getSizeInBits());
2375 auto &PtrAdd = cast<GPtrAdd>(
MI);
2380 PtrAdd.eraseFromParent();
2384 assert(
MI.getOpcode() == TargetOpcode::G_ANYEXT &&
"Expected a G_ANYEXT");
2389 SrcReg = OriginalSrcReg;
2396 assert(
MI.getOpcode() == TargetOpcode::G_ZEXT &&
"Expected a G_ZEXT");
2411 assert((
MI.getOpcode() == TargetOpcode::G_ANYEXT ||
2412 MI.getOpcode() == TargetOpcode::G_SEXT ||
2413 MI.getOpcode() == TargetOpcode::G_ZEXT) &&
2414 "Expected a G_[ASZ]EXT");
2418 SrcReg = OriginalSrcReg;
2421 unsigned Opc =
MI.getOpcode();
2423 if (Opc == SrcOpc ||
2424 (Opc == TargetOpcode::G_ANYEXT &&
2425 (SrcOpc == TargetOpcode::G_SEXT || SrcOpc == TargetOpcode::G_ZEXT)) ||
2426 (Opc == TargetOpcode::G_SEXT && SrcOpc == TargetOpcode::G_ZEXT)) {
2435 assert((
MI.getOpcode() == TargetOpcode::G_ANYEXT ||
2436 MI.getOpcode() == TargetOpcode::G_SEXT ||
2437 MI.getOpcode() == TargetOpcode::G_ZEXT) &&
2438 "Expected a G_[ASZ]EXT");
2440 Register Reg = std::get<0>(MatchInfo);
2441 unsigned SrcExtOp = std::get<1>(MatchInfo);
2444 if (
MI.getOpcode() == SrcExtOp) {
2446 MI.getOperand(1).setReg(Reg);
2454 if (
MI.getOpcode() == TargetOpcode::G_ANYEXT ||
2455 (
MI.getOpcode() == TargetOpcode::G_SEXT &&
2456 SrcExtOp == TargetOpcode::G_ZEXT)) {
2460 MI.eraseFromParent();
2466 assert(
MI.getOpcode() == TargetOpcode::G_TRUNC &&
"Expected a G_TRUNC");
2470 if (SrcOpc == TargetOpcode::G_ANYEXT || SrcOpc == TargetOpcode::G_SEXT ||
2471 SrcOpc == TargetOpcode::G_ZEXT) {
2480 assert(
MI.getOpcode() == TargetOpcode::G_TRUNC &&
"Expected a G_TRUNC");
2482 unsigned SrcExtOp = MatchInfo.second;
2486 if (SrcTy == DstTy) {
2487 MI.eraseFromParent();
2496 MI.eraseFromParent();
2504 if (ShiftSize > 32 && TruncSize < 32)
2518 assert(
MI.getOpcode() == TargetOpcode::G_TRUNC &&
"Expected a G_TRUNC");
2535 case TargetOpcode::G_SHL: {
2544 case TargetOpcode::G_LSHR:
2545 case TargetOpcode::G_ASHR: {
2552 if (
User.getOpcode() == TargetOpcode::G_STORE)
2556 if (NewShiftTy == SrcTy)
2570 {NewShiftTy, TL.getPreferredShiftAmountTy(NewShiftTy)}}))
2573 MatchInfo = std::make_pair(SrcMI, NewShiftTy);
2582 LLT NewShiftTy = MatchInfo.second;
2596 if (NewShiftTy == DstTy)
2606 return MO.isReg() &&
2607 getOpcodeDef(TargetOpcode::G_IMPLICIT_DEF, MO.getReg(), MRI);
2613 return !MO.isReg() ||
2614 getOpcodeDef(TargetOpcode::G_IMPLICIT_DEF, MO.getReg(), MRI);
2619 assert(
MI.getOpcode() == TargetOpcode::G_SHUFFLE_VECTOR);
2621 return all_of(Mask, [](
int Elt) {
return Elt < 0; });
2625 assert(
MI.getOpcode() == TargetOpcode::G_STORE);
2626 return getOpcodeDef(TargetOpcode::G_IMPLICIT_DEF,
MI.getOperand(0).getReg(),
2631 assert(
MI.getOpcode() == TargetOpcode::G_SELECT);
2632 return getOpcodeDef(TargetOpcode::G_IMPLICIT_DEF,
MI.getOperand(1).getReg(),
2637 assert((
MI.getOpcode() == TargetOpcode::G_INSERT_VECTOR_ELT ||
2638 MI.getOpcode() == TargetOpcode::G_EXTRACT_VECTOR_ELT) &&
2639 "Expected an insert/extract element op");
2642 MI.getOpcode() == TargetOpcode::G_EXTRACT_VECTOR_ELT ? 2 : 3;
2655 OpIdx = Cst->isZero() ? 3 : 2;
2700 if (I1->mayLoadOrStore() && !I1->isDereferenceableInvariantLoad())
2727 return MO.isReg() && MO.getReg().isPhysical();
2737 return I1->isIdenticalTo(*I2);
2752 return I1->findRegisterDefOperandIdx(InstAndDef1->Reg) ==
2763 return MaybeCst && MaybeCst->getBitWidth() <= 64 &&
2764 MaybeCst->getSExtValue() ==
C;
2770 std::optional<FPValueAndVReg> MaybeCst;
2774 return MaybeCst->Value.isExactlyValue(
C);
2779 assert(
MI.getNumExplicitDefs() == 1 &&
"Expected one explicit def?");
2781 Register Replacement =
MI.getOperand(OpIdx).getReg();
2783 MI.eraseFromParent();
2789 assert(
MI.getNumExplicitDefs() == 1 &&
"Expected one explicit def?");
2792 MI.eraseFromParent();
2797 unsigned ConstIdx) {
2798 Register ConstReg =
MI.getOperand(ConstIdx).getReg();
2811 assert((
MI.getOpcode() == TargetOpcode::G_FSHL ||
2812 MI.getOpcode() == TargetOpcode::G_FSHR) &&
2813 "This is not a funnel shift operation");
2815 Register ConstReg =
MI.getOperand(3).getReg();
2820 assert((VRegAndVal) &&
"Value is not a constant");
2823 APInt NewConst = VRegAndVal->Value.
urem(
2829 MI.getOpcode(), {MI.getOperand(0)},
2830 {MI.getOperand(1), MI.getOperand(2), NewConstInstr.getReg(0)});
2832 MI.eraseFromParent();
2836 assert(
MI.getOpcode() == TargetOpcode::G_SELECT);
2857 return MO.
isReg() &&
2868 assert(
MI.getNumDefs() == 1 &&
"Expected only one def?");
2871 MI.eraseFromParent();
2875 assert(
MI.getNumDefs() == 1 &&
"Expected only one def?");
2878 MI.eraseFromParent();
2882 assert(
MI.getNumDefs() == 1 &&
"Expected only one def?");
2885 MI.eraseFromParent();
2889 assert(
MI.getNumDefs() == 1 &&
"Expected only one def?");
2892 MI.eraseFromParent();
2896 assert(
MI.getNumDefs() == 1 &&
"Expected only one def?");
2899 MI.eraseFromParent();
2906 Register &NewLHS = std::get<0>(MatchInfo);
2907 Register &NewRHS = std::get<1>(MatchInfo);
2915 NewLHS = MaybeNewLHS;
2924 assert(
MI.getOpcode() == TargetOpcode::G_INSERT_VECTOR_ELT &&
2933 TargetOpcode::G_INSERT_VECTOR_ELT)
2939 MatchInfo.
resize(NumElts);
2943 if (IntImm >= NumElts || IntImm < 0)
2945 if (!MatchInfo[IntImm])
2946 MatchInfo[IntImm] = TmpReg;
2950 if (CurrInst->
getOpcode() == TargetOpcode::G_INSERT_VECTOR_ELT)
2952 if (TmpInst->
getOpcode() == TargetOpcode::G_BUILD_VECTOR) {
2960 return TmpInst->
getOpcode() == TargetOpcode::G_IMPLICIT_DEF;
2967 auto GetUndef = [&]() {
2974 for (
unsigned I = 0;
I < MatchInfo.
size(); ++
I) {
2976 MatchInfo[
I] = GetUndef();
2979 MI.eraseFromParent();
2986 std::tie(SubLHS, SubRHS) = MatchInfo;
2988 MI.eraseFromParent();
2999 unsigned LogicOpcode =
MI.getOpcode();
3000 assert(LogicOpcode == TargetOpcode::G_AND ||
3001 LogicOpcode == TargetOpcode::G_OR ||
3002 LogicOpcode == TargetOpcode::G_XOR);
3015 if (!LeftHandInst || !RightHandInst)
3017 unsigned HandOpcode = LeftHandInst->
getOpcode();
3018 if (HandOpcode != RightHandInst->
getOpcode())
3030 if (!XTy.
isValid() || XTy != YTy)
3035 switch (HandOpcode) {
3038 case TargetOpcode::G_ANYEXT:
3039 case TargetOpcode::G_SEXT:
3040 case TargetOpcode::G_ZEXT: {
3044 case TargetOpcode::G_AND:
3045 case TargetOpcode::G_ASHR:
3046 case TargetOpcode::G_LSHR:
3047 case TargetOpcode::G_SHL: {
3052 ExtraHandOpSrcReg = ZOp.
getReg();
3074 if (ExtraHandOpSrcReg.
isValid())
3086 "Expected at least one instr to build?");
3089 assert(InstrToBuild.Opcode &&
"Expected a valid opcode?");
3090 assert(InstrToBuild.OperandFns.size() &&
"Expected at least one operand?");
3092 for (
auto &OperandFn : InstrToBuild.OperandFns)
3095 MI.eraseFromParent();
3100 assert(
MI.getOpcode() == TargetOpcode::G_ASHR);
3101 int64_t ShlCst, AshrCst;
3107 if (ShlCst != AshrCst)
3110 {TargetOpcode::G_SEXT_INREG, {
MRI.
getType(Src)}}))
3112 MatchInfo = std::make_tuple(Src, ShlCst);
3118 assert(
MI.getOpcode() == TargetOpcode::G_ASHR);
3121 std::tie(Src, ShiftAmt) = MatchInfo;
3125 MI.eraseFromParent();
3131 assert(
MI.getOpcode() == TargetOpcode::G_AND);
3146 B.buildAnd(Dst, R,
B.buildConstant(Ty, C1 & C2));
3149 auto Zero =
B.buildConstant(Ty, 0);
3172 assert(
MI.getOpcode() == TargetOpcode::G_AND);
3189 (LHSBits.
Zero | RHSBits.
One).isAllOnes()) {
3196 (LHSBits.
One | RHSBits.
Zero).isAllOnes()) {
3212 assert(
MI.getOpcode() == TargetOpcode::G_OR);
3229 (LHSBits.
One | RHSBits.
Zero).isAllOnes()) {
3236 (LHSBits.
Zero | RHSBits.
One).isAllOnes()) {
3247 unsigned ExtBits =
MI.getOperand(2).getImm();
3253 int64_t Cst,
bool IsVector,
bool IsFP) {
3255 return (ScalarSizeBits == 1 && Cst == -1) ||
3261 assert(
MI.getOpcode() == TargetOpcode::G_XOR);
3281 for (
unsigned I = 0;
I < RegsToNegate.
size(); ++
I) {
3286 switch (Def->getOpcode()) {
3291 case TargetOpcode::G_ICMP:
3297 case TargetOpcode::G_FCMP:
3303 case TargetOpcode::G_AND:
3304 case TargetOpcode::G_OR:
3310 RegsToNegate.
push_back(Def->getOperand(1).getReg());
3311 RegsToNegate.
push_back(Def->getOperand(2).getReg());
3338 for (
Register Reg : RegsToNegate) {
3343 switch (Def->getOpcode()) {
3346 case TargetOpcode::G_ICMP:
3347 case TargetOpcode::G_FCMP: {
3354 case TargetOpcode::G_AND:
3357 case TargetOpcode::G_OR:
3365 MI.eraseFromParent();
3371 assert(
MI.getOpcode() == TargetOpcode::G_XOR);
3375 Register SharedReg =
MI.getOperand(2).getReg();
3396 return Y == SharedReg;
3404 std::tie(
X,
Y) = MatchInfo;
3408 MI.getOperand(1).setReg(Not->getOperand(0).getReg());
3409 MI.getOperand(2).setReg(
Y);
3414 auto &PtrAdd = cast<GPtrAdd>(
MI);
3415 Register DstReg = PtrAdd.getReg(0);
3424 return ConstVal && *ConstVal == 0;
3433 auto &PtrAdd = cast<GPtrAdd>(
MI);
3436 PtrAdd.eraseFromParent();
3443 Register Pow2Src1 =
MI.getOperand(2).getReg();
3451 MI.eraseFromParent();
3455 unsigned &SelectOpNo) {
3465 if (
Select->getOpcode() != TargetOpcode::G_SELECT ||
3467 OtherOperandReg =
LHS;
3470 if (
Select->getOpcode() != TargetOpcode::G_SELECT ||
3487 unsigned BinOpcode =
MI.getOpcode();
3492 bool CanFoldNonConst =
3493 (BinOpcode == TargetOpcode::G_AND || BinOpcode == TargetOpcode::G_OR) &&
3498 if (CanFoldNonConst)
3509 const unsigned &SelectOperand) {
3522 unsigned BinOpcode =
MI.getOpcode();
3529 if (SelectOperand == 1) {
3543 MI.eraseFromParent();
3546std::optional<SmallVector<Register, 8>>
3547CombinerHelper::findCandidatesForLoadOrCombine(
const MachineInstr *Root)
const {
3548 assert(Root->
getOpcode() == TargetOpcode::G_OR &&
"Expected G_OR only!");
3577 const unsigned MaxIter =
3579 for (
unsigned Iter = 0; Iter < MaxIter; ++Iter) {
3588 return std::nullopt;
3604 if (RegsToVisit.
empty() || RegsToVisit.
size() % 2 != 0)
3605 return std::nullopt;
3617static std::optional<std::pair<GZExtLoad *, int64_t>>
3621 "Expected Reg to only have one non-debug use?");
3630 if (Shift % MemSizeInBits != 0)
3631 return std::nullopt;
3634 auto *Load = getOpcodeDef<GZExtLoad>(MaybeLoad,
MRI);
3636 return std::nullopt;
3638 if (!Load->isUnordered() || Load->getMemSizeInBits() != MemSizeInBits)
3639 return std::nullopt;
3641 return std::make_pair(Load, Shift / MemSizeInBits);
3644std::optional<std::tuple<GZExtLoad *, int64_t, GZExtLoad *>>
3645CombinerHelper::findLoadOffsetsForLoadOrCombine(
3679 for (
auto Reg : RegsToVisit) {
3684 return std::nullopt;
3687 std::tie(Load, DstPos) = *LoadAndPos;
3695 return std::nullopt;
3698 auto &LoadMMO =
Load->getMMO();
3702 return std::nullopt;
3709 LoadPtr =
Load->getOperand(1).getReg();
3715 return std::nullopt;
3722 if (BasePtr != LoadPtr)
3723 return std::nullopt;
3725 if (
Idx < LowestIdx) {
3727 LowestIdxLoad =
Load;
3735 return std::nullopt;
3743 if (!EarliestLoad ||
dominates(*Load, *EarliestLoad))
3744 EarliestLoad =
Load;
3745 if (!LatestLoad ||
dominates(*LatestLoad, *Load))
3752 "Expected to find a load for each register?");
3753 assert(EarliestLoad != LatestLoad && EarliestLoad &&
3754 LatestLoad &&
"Expected at least two loads?");
3763 const unsigned MaxIter = 20;
3769 if (
MI.isLoadFoldBarrier())
3770 return std::nullopt;
3771 if (Iter++ == MaxIter)
3772 return std::nullopt;
3775 return std::make_tuple(LowestIdxLoad, LowestIdx, LatestLoad);
3780 assert(
MI.getOpcode() == TargetOpcode::G_OR);
3800 if (WideMemSizeInBits < 16 || WideMemSizeInBits % 8 != 0)
3804 auto RegsToVisit = findCandidatesForLoadOrCombine(&
MI);
3811 const unsigned NarrowMemSizeInBits = WideMemSizeInBits / RegsToVisit->size();
3812 if (NarrowMemSizeInBits % 8 != 0)
3825 auto MaybeLoadInfo = findLoadOffsetsForLoadOrCombine(
3826 MemOffset2Idx, *RegsToVisit, NarrowMemSizeInBits);
3829 std::tie(LowestIdxLoad, LowestIdx, LatestLoad) = *MaybeLoadInfo;
3836 std::optional<bool> IsBigEndian =
isBigEndian(MemOffset2Idx, LowestIdx);
3839 bool NeedsBSwap = IsBigEndianTarget != *IsBigEndian;
3851 const unsigned NumLoadsInTy = WideMemSizeInBits / NarrowMemSizeInBits;
3852 const unsigned ZeroByteOffset =
3856 auto ZeroOffsetIdx = MemOffset2Idx.
find(ZeroByteOffset);
3857 if (ZeroOffsetIdx == MemOffset2Idx.
end() ||
3858 ZeroOffsetIdx->second != LowestIdx)
3882 MIB.setInstrAndDebugLoc(*LatestLoad);
3884 MIB.buildLoad(LoadDst,
Ptr, *NewMMO);
3886 MIB.buildBSwap(Dst, LoadDst);
3893 auto &
PHI = cast<GPhi>(
MI);
3906 case TargetOpcode::G_ANYEXT:
3908 case TargetOpcode::G_ZEXT:
3909 case TargetOpcode::G_SEXT:
3923 for (
unsigned I = 0;
I <
PHI.getNumIncomingValues(); ++
I) {
3926 case TargetOpcode::G_LOAD:
3927 case TargetOpcode::G_TRUNC:
3928 case TargetOpcode::G_SEXT:
3929 case TargetOpcode::G_ZEXT:
3930 case TargetOpcode::G_ANYEXT:
3931 case TargetOpcode::G_CONSTANT:
3935 if (InSrcs.
size() > 2)
3947 auto &
PHI = cast<GPhi>(
MI);
3956 for (
unsigned I = 0;
I <
PHI.getNumIncomingValues(); ++
I) {
3957 auto SrcReg =
PHI.getIncomingValue(
I);
3959 if (!SrcMIs.
insert(SrcMI))
3965 if (InsertPt !=
MBB->
end() && InsertPt->isPHI())
3971 OldToNewSrcMap[SrcMI] = NewExt;
3980 NewPhi.
addMBB(MO.getMBB());
3984 NewPhi.addUse(NewSrc->getOperand(0).getReg());
3992 assert(
MI.getOpcode() == TargetOpcode::G_EXTRACT_VECTOR_ELT);
4002 unsigned VecIdx = Cst->Value.getZExtValue();
4007 if (SrcVecMI->
getOpcode() == TargetOpcode::G_TRUNC) {
4011 if (SrcVecMI->
getOpcode() != TargetOpcode::G_BUILD_VECTOR &&
4012 SrcVecMI->
getOpcode() != TargetOpcode::G_BUILD_VECTOR_TRUNC)
4033 if (ScalarTy != DstTy) {
4036 MI.eraseFromParent();
4045 assert(
MI.getOpcode() == TargetOpcode::G_BUILD_VECTOR);
4068 if (II.getOpcode() != TargetOpcode::G_EXTRACT_VECTOR_ELT)
4073 unsigned Idx = Cst->getZExtValue();
4077 SrcDstPairs.emplace_back(
4078 std::make_pair(
MI.getOperand(
Idx + 1).getReg(), &II));
4081 return ExtractedElts.
all();
4087 assert(
MI.getOpcode() == TargetOpcode::G_BUILD_VECTOR);
4088 for (
auto &Pair : SrcDstPairs) {
4089 auto *ExtMI = Pair.second;
4091 ExtMI->eraseFromParent();
4093 MI.eraseFromParent();
4100 MI.eraseFromParent();
4111 assert(
MI.getOpcode() == TargetOpcode::G_OR);
4117 Register ShlSrc, ShlAmt, LShrSrc, LShrAmt, Amt;
4118 unsigned FshOpc = 0;
4129 int64_t CstShlAmt, CstLShrAmt;
4132 CstShlAmt + CstLShrAmt ==
BitWidth) {
4133 FshOpc = TargetOpcode::G_FSHR;
4140 FshOpc = TargetOpcode::G_FSHL;
4146 FshOpc = TargetOpcode::G_FSHR;
4157 B.buildInstr(FshOpc, {Dst}, {ShlSrc, LShrSrc, Amt});
4164 unsigned Opc =
MI.getOpcode();
4165 assert(Opc == TargetOpcode::G_FSHL || Opc == TargetOpcode::G_FSHR);
4170 unsigned RotateOpc =
4171 Opc == TargetOpcode::G_FSHL ? TargetOpcode::G_ROTL : TargetOpcode::G_ROTR;
4176 unsigned Opc =
MI.getOpcode();
4177 assert(Opc == TargetOpcode::G_FSHL || Opc == TargetOpcode::G_FSHR);
4178 bool IsFSHL = Opc == TargetOpcode::G_FSHL;
4181 : TargetOpcode::G_ROTR));
4182 MI.removeOperand(2);
4188 assert(
MI.getOpcode() == TargetOpcode::G_ROTL ||
4189 MI.getOpcode() == TargetOpcode::G_ROTR);
4193 bool OutOfRange =
false;
4194 auto MatchOutOfRange = [Bitsize, &OutOfRange](
const Constant *
C) {
4195 if (
auto *CI = dyn_cast<ConstantInt>(
C))
4196 OutOfRange |= CI->getValue().uge(Bitsize);
4203 assert(
MI.getOpcode() == TargetOpcode::G_ROTL ||
4204 MI.getOpcode() == TargetOpcode::G_ROTR);
4213 MI.getOperand(2).setReg(Amt);
4218 int64_t &MatchInfo) {
4219 assert(
MI.getOpcode() == TargetOpcode::G_ICMP);
4223 std::optional<bool> KnownVal;
4272 assert(
MI.getOpcode() == TargetOpcode::G_ICMP);
4297 if (KnownLHS.getMinValue() != 0 || KnownLHS.getMaxValue() != 1)
4303 unsigned Op = TargetOpcode::COPY;
4304 if (DstSize != LHSSize)
4305 Op = DstSize < LHSSize ? TargetOpcode::G_TRUNC : TargetOpcode::G_ZEXT;
4315 assert(
MI.getOpcode() == TargetOpcode::G_AND);
4325 int64_t AndMaskBits;
4333 if (AndMaskBits & OrMaskBits)
4339 if (
MI.getOperand(1).getReg() == AndMaskReg)
4340 MI.getOperand(2).setReg(AndMaskReg);
4341 MI.getOperand(1).setReg(Src);
4350 assert(
MI.getOpcode() == TargetOpcode::G_SEXT_INREG);
4357 int64_t Width =
MI.getOperand(2).getImm();
4369 auto Cst1 =
B.buildConstant(ExtractTy, ShiftImm);
4370 auto Cst2 =
B.buildConstant(ExtractTy, Width);
4371 B.buildSbfx(Dst, ShiftSrc, Cst1, Cst2);
4379 assert(
MI.getOpcode() == TargetOpcode::G_AND);
4386 int64_t AndImm, LSBImm;
4395 auto MaybeMask =
static_cast<uint64_t>(AndImm);
4396 if (MaybeMask & (MaybeMask + 1))
4405 auto WidthCst =
B.buildConstant(ExtractTy, Width);
4406 auto LSBCst =
B.buildConstant(ExtractTy, LSBImm);
4407 B.buildInstr(TargetOpcode::G_UBFX, {Dst}, {ShiftSrc, LSBCst, WidthCst});
4414 const unsigned Opcode =
MI.getOpcode();
4415 assert(Opcode == TargetOpcode::G_ASHR || Opcode == TargetOpcode::G_LSHR);
4417 const Register Dst =
MI.getOperand(0).getReg();
4419 const unsigned ExtrOpcode = Opcode == TargetOpcode::G_ASHR
4420 ? TargetOpcode::G_SBFX
4421 : TargetOpcode::G_UBFX;
4442 if (ShlAmt < 0 || ShlAmt > ShrAmt || ShrAmt >=
Size)
4446 if (Opcode == TargetOpcode::G_ASHR && ShlAmt == ShrAmt)
4450 const int64_t Pos = ShrAmt - ShlAmt;
4451 const int64_t Width =
Size - ShrAmt;
4454 auto WidthCst =
B.buildConstant(ExtractTy, Width);
4455 auto PosCst =
B.buildConstant(ExtractTy, Pos);
4456 B.buildInstr(ExtrOpcode, {Dst}, {ShlSrc, PosCst, WidthCst});
4463 const unsigned Opcode =
MI.getOpcode();
4464 assert(Opcode == TargetOpcode::G_LSHR || Opcode == TargetOpcode::G_ASHR);
4466 const Register Dst =
MI.getOperand(0).getReg();
4483 if (ShrAmt < 0 || ShrAmt >=
Size)
4487 if (0 == (SMask >> ShrAmt)) {
4489 B.buildConstant(Dst, 0);
4496 UMask |= maskTrailingOnes<uint64_t>(ShrAmt);
4497 UMask &= maskTrailingOnes<uint64_t>(
Size);
4502 const int64_t Pos = ShrAmt;
4507 if (Opcode == TargetOpcode::G_ASHR && Width + ShrAmt ==
Size)
4511 auto WidthCst =
B.buildConstant(ExtractTy, Width);
4512 auto PosCst =
B.buildConstant(ExtractTy, Pos);
4513 B.buildInstr(TargetOpcode::G_UBFX, {Dst}, {AndSrc, PosCst, WidthCst});
4518bool CombinerHelper::reassociationCanBreakAddressingModePattern(
4520 auto &PtrAdd = cast<GPtrAdd>(
MI);
4522 Register Src1Reg = PtrAdd.getBaseReg();
4523 auto *Src1Def = getOpcodeDef<GPtrAdd>(Src1Reg,
MRI);
4527 Register Src2Reg = PtrAdd.getOffsetReg();
4539 const APInt &C1APIntVal = *C1;
4540 const APInt &C2APIntVal = *C2;
4541 const int64_t CombinedValue = (C1APIntVal + C2APIntVal).getSExtValue();
4547 unsigned ConvUseOpc = ConvUseMI->
getOpcode();
4548 while (ConvUseOpc == TargetOpcode::G_INTTOPTR ||
4549 ConvUseOpc == TargetOpcode::G_PTRTOINT) {
4556 auto *LdStMI = dyn_cast<GLoadStore>(ConvUseMI);
4567 PtrAdd.getMF()->getFunction().getContext());
4568 const auto &TLI = *PtrAdd.getMF()->getSubtarget().getTargetLowering();
4569 if (!TLI.isLegalAddressingMode(PtrAdd.getMF()->getDataLayout(), AM,
4575 if (!TLI.isLegalAddressingMode(PtrAdd.getMF()->getDataLayout(), AM,
4587 Register Src1Reg =
MI.getOperand(1).getReg();
4588 if (
RHS->getOpcode() != TargetOpcode::G_ADD)
4600 MI.getOperand(1).setReg(NewBase.getReg(0));
4601 MI.getOperand(2).setReg(
RHS->getOperand(2).getReg());
4604 return !reassociationCanBreakAddressingModePattern(
MI);
4614 std::optional<ValueAndVReg> LHSCstOff;
4619 auto *LHSPtrAdd = cast<GPtrAdd>(
LHS);
4624 LHSPtrAdd->moveBefore(&
MI);
4627 auto NewCst =
B.buildConstant(
MRI.
getType(RHSReg), LHSCstOff->Value);
4629 MI.getOperand(2).setReg(NewCst.getReg(0));
4632 LHSPtrAdd->getOperand(2).setReg(RHSReg);
4635 return !reassociationCanBreakAddressingModePattern(
MI);
4643 auto *LHSPtrAdd = dyn_cast<GPtrAdd>(
LHS);
4647 Register Src2Reg =
MI.getOperand(2).getReg();
4648 Register LHSSrc1 = LHSPtrAdd->getBaseReg();
4649 Register LHSSrc2 = LHSPtrAdd->getOffsetReg();
4658 auto NewCst =
B.buildConstant(
MRI.
getType(Src2Reg), *C1 + *C2);
4660 MI.getOperand(1).setReg(LHSSrc1);
4661 MI.getOperand(2).setReg(NewCst.getReg(0));
4664 return !reassociationCanBreakAddressingModePattern(
MI);
4669 auto &PtrAdd = cast<GPtrAdd>(
MI);
4721 auto NewCst =
B.buildInstr(Opc, {OpRHSTy}, {OpLHSRHS, OpRHS});
4722 B.buildInstr(Opc, {DstReg}, {OpLHSLHS, NewCst});
4730 auto NewLHSLHS =
B.buildInstr(Opc, {OpRHSTy}, {OpLHSLHS, OpRHS});
4731 B.buildInstr(Opc, {DstReg}, {NewLHSLHS, OpLHSRHS});
4744 unsigned Opc =
MI.getOpcode();
4761 MatchInfo = *MaybeCst;
4774 MatchInfo = *MaybeCst;
4785 ConstantFP::get(
MI.getMF()->getFunction().getContext(), *MaybeCst);
4791 assert(
MI.getOpcode() == TargetOpcode::G_FMA ||
4792 MI.getOpcode() == TargetOpcode::G_FMAD);
4793 auto [
_, Op1, Op2, Op3] =
MI.getFirst4Regs();
4810 MatchInfo = ConstantFP::get(
MI.getMF()->getFunction().getContext(), Op1F);
4832 assert(
MI.getOpcode() == TargetOpcode::G_AND);
4856 case TargetOpcode::G_ADD:
4857 case TargetOpcode::G_SUB:
4858 case TargetOpcode::G_MUL:
4859 case TargetOpcode::G_AND:
4860 case TargetOpcode::G_OR:
4861 case TargetOpcode::G_XOR:
4869 auto Mask = Cst->Value;
4874 unsigned NarrowWidth = Mask.countr_one();
4880 auto &MF = *
MI.getMF();
4883 auto &
DL = MF.getDataLayout();
4884 if (!TLI.isTruncateFree(WideTy, NarrowTy,
DL, Ctx) ||
4885 !TLI.isZExtFree(NarrowTy, WideTy,
DL, Ctx))
4899 MI.getOperand(1).setReg(Ext.getReg(0));
4906 unsigned Opc =
MI.getOpcode();
4907 assert(Opc == TargetOpcode::G_UMULO || Opc == TargetOpcode::G_SMULO);
4914 unsigned NewOpc = Opc == TargetOpcode::G_UMULO ? TargetOpcode::G_UADDO
4915 : TargetOpcode::G_SADDO;
4917 MI.getOperand(3).setReg(
MI.getOperand(2).getReg());
4925 assert(
MI.getOpcode() == TargetOpcode::G_UMULO ||
4926 MI.getOpcode() == TargetOpcode::G_SMULO);
4935 B.buildConstant(Dst, 0);
4936 B.buildConstant(Carry, 0);
4944 assert(
MI.getOpcode() == TargetOpcode::G_UADDE ||
4945 MI.getOpcode() == TargetOpcode::G_SADDE ||
4946 MI.getOpcode() == TargetOpcode::G_USUBE ||
4947 MI.getOpcode() == TargetOpcode::G_SSUBE);
4952 switch (
MI.getOpcode()) {
4953 case TargetOpcode::G_UADDE:
4954 NewOpcode = TargetOpcode::G_UADDO;
4956 case TargetOpcode::G_SADDE:
4957 NewOpcode = TargetOpcode::G_SADDO;
4959 case TargetOpcode::G_USUBE:
4960 NewOpcode = TargetOpcode::G_USUBO;
4962 case TargetOpcode::G_SSUBE:
4963 NewOpcode = TargetOpcode::G_SSUBO;
4967 MI.setDesc(
B.getTII().get(NewOpcode));
4968 MI.removeOperand(4);
4976 assert(
MI.getOpcode() == TargetOpcode::G_SUB);
5010 B.buildSub(Dst, Zero, ReplaceReg);
5019 assert(
MI.getOpcode() == TargetOpcode::G_UDIV);
5020 auto &UDiv = cast<GenericMachineInstr>(
MI);
5032 bool UseNPQ =
false;
5035 auto BuildUDIVPattern = [&](
const Constant *
C) {
5036 auto *CI = cast<ConstantInt>(
C);
5037 const APInt &Divisor = CI->getValue();
5039 bool SelNPQ =
false;
5041 unsigned PreShift = 0, PostShift = 0;
5046 if (!Divisor.
isOne()) {
5050 Magic = std::move(magics.
Magic);
5053 "We shouldn't generate an undefined shift!");
5055 "We shouldn't generate an undefined shift!");
5059 SelNPQ = magics.
IsAdd;
5063 MIB.buildConstant(ScalarShiftAmtTy, PreShift).getReg(0));
5064 MagicFactors.
push_back(MIB.buildConstant(ScalarTy, Magic).getReg(0));
5066 MIB.buildConstant(ScalarTy,
5071 MIB.buildConstant(ScalarShiftAmtTy, PostShift).getReg(0));
5079 assert(Matched &&
"Expected unary predicate match to succeed");
5081 Register PreShift, PostShift, MagicFactor, NPQFactor;
5082 auto *RHSDef = getOpcodeDef<GBuildVector>(
RHS,
MRI);
5084 PreShift = MIB.buildBuildVector(ShiftAmtTy, PreShifts).getReg(0);
5085 MagicFactor = MIB.buildBuildVector(Ty, MagicFactors).getReg(0);
5086 NPQFactor = MIB.buildBuildVector(Ty, NPQFactors).getReg(0);
5087 PostShift = MIB.buildBuildVector(ShiftAmtTy, PostShifts).getReg(0);
5090 "Non-build_vector operation should have been a scalar");
5091 PreShift = PreShifts[0];
5092 MagicFactor = MagicFactors[0];
5093 PostShift = PostShifts[0];
5097 Q = MIB.buildLShr(Ty, Q, PreShift).getReg(0);
5100 Q = MIB.buildUMulH(Ty, Q, MagicFactor).getReg(0);
5103 Register NPQ = MIB.buildSub(Ty,
LHS, Q).getReg(0);
5108 NPQ = MIB.buildUMulH(Ty, NPQ, NPQFactor).getReg(0);
5110 NPQ = MIB.buildLShr(Ty, NPQ, MIB.buildConstant(ShiftAmtTy, 1)).getReg(0);
5112 Q = MIB.buildAdd(Ty, NPQ, Q).getReg(0);
5115 Q = MIB.buildLShr(Ty, Q, PostShift).getReg(0);
5116 auto One = MIB.buildConstant(Ty, 1);
5117 auto IsOne = MIB.buildICmp(
5120 return MIB.buildSelect(Ty, IsOne,
LHS, Q);
5124 assert(
MI.getOpcode() == TargetOpcode::G_UDIV);
5132 auto &MF = *
MI.getMF();
5136 auto &
DL = MF.getDataLayout();
5142 if (MF.getFunction().hasMinSize())
5152 {TargetOpcode::G_ICMP,
5158 auto CheckEltValue = [&](
const Constant *
C) {
5159 if (
auto *CI = dyn_cast_or_null<ConstantInt>(
C))
5160 return !CI->isZero();
5172 assert(
MI.getOpcode() == TargetOpcode::G_SDIV &&
"Expected SDIV");
5177 auto &MF = *
MI.getMF();
5181 auto &
DL = MF.getDataLayout();
5187 if (MF.getFunction().hasMinSize())
5206 assert(
MI.getOpcode() == TargetOpcode::G_SDIV &&
"Expected SDIV");
5207 auto &SDiv = cast<GenericMachineInstr>(
MI);
5218 bool UseSRA =
false;
5224 auto BuildSDIVPattern = [&](
const Constant *
C) {
5226 if (IsSplat && !Factors.
empty()) {
5232 auto *CI = cast<ConstantInt>(
C);
5233 APInt Divisor = CI->getValue();
5246 Shifts.
push_back(MIB.buildConstant(ScalarShiftAmtTy, Shift).getReg(0));
5247 Factors.
push_back(MIB.buildConstant(ScalarTy, Factor).getReg(0));
5254 assert(Matched &&
"Expected unary predicate match to succeed");
5258 Shift = MIB.buildBuildVector(ShiftAmtTy, Shifts).getReg(0);
5259 Factor = MIB.buildBuildVector(Ty, Factors).getReg(0);
5262 Factor = Factors[0];
5270 return MIB.buildMul(Ty, Res, Factor);
5274 assert(
MI.getOpcode() == TargetOpcode::G_UMULH);
5279 auto MatchPow2ExceptOne = [&](
const Constant *
C) {
5280 if (
auto *CI = dyn_cast<ConstantInt>(
C))
5281 return CI->getValue().isPowerOf2() && !CI->getValue().isOne();
5303 MI.eraseFromParent();
5308 unsigned Opc =
MI.getOpcode();
5309 assert(Opc == TargetOpcode::G_FADD || Opc == TargetOpcode::G_FSUB ||
5310 Opc == TargetOpcode::G_FMUL || Opc == TargetOpcode::G_FDIV ||
5311 Opc == TargetOpcode::G_FMAD || Opc == TargetOpcode::G_FMA);
5323 Opc = TargetOpcode::G_FSUB;
5328 Opc = TargetOpcode::G_FADD;
5334 else if ((Opc == TargetOpcode::G_FMUL || Opc == TargetOpcode::G_FDIV ||
5335 Opc == TargetOpcode::G_FMAD || Opc == TargetOpcode::G_FMA) &&
5344 MI.setDesc(
B.getTII().get(Opc));
5345 MI.getOperand(1).setReg(
X);
5346 MI.getOperand(2).setReg(
Y);
5353 assert(
MI.getOpcode() == TargetOpcode::G_FSUB);
5356 MatchInfo =
MI.getOperand(2).getReg();
5366 if (LHSCst->Value.isNegZero())
5370 if (LHSCst->Value.isPosZero())
5387 if (
MI.getOpcode() != TargetOpcode::G_FMUL)
5395 MRI.use_instr_nodbg_end()) >
5397 MRI.use_instr_nodbg_end());
5401 bool &AllowFusionGlobally,
5403 bool CanReassociate) {
5405 auto *MF =
MI.getMF();
5406 const auto &TLI = *MF->getSubtarget().getTargetLowering();
5410 if (CanReassociate &&
5417 bool HasFMA = TLI.isFMAFasterThanFMulAndFAdd(*MF, DstType) &&
5420 if (!HasFMAD && !HasFMA)
5424 Options.UnsafeFPMath || HasFMAD;
5429 Aggressive = TLI.enableAggressiveFMAFusion(DstType);
5435 assert(
MI.getOpcode() == TargetOpcode::G_FADD);
5437 bool AllowFusionGlobally, HasFMAD,
Aggressive;
5445 unsigned PreferredFusedOpcode =
5446 HasFMAD ? TargetOpcode::G_FMAD : TargetOpcode::G_FMA;
5460 B.buildInstr(PreferredFusedOpcode, {
MI.getOperand(0).getReg()},
5461 {
LHS.MI->getOperand(1).getReg(),
5462 LHS.MI->getOperand(2).getReg(),
RHS.Reg});
5471 B.buildInstr(PreferredFusedOpcode, {
MI.getOperand(0).getReg()},
5472 {
RHS.MI->getOperand(1).getReg(),
5473 RHS.MI->getOperand(2).getReg(),
LHS.Reg});
5483 assert(
MI.getOpcode() == TargetOpcode::G_FADD);
5485 bool AllowFusionGlobally, HasFMAD,
Aggressive;
5489 const auto &TLI = *
MI.getMF()->getSubtarget().getTargetLowering();
5496 unsigned PreferredFusedOpcode =
5497 HasFMAD ? TargetOpcode::G_FMAD : TargetOpcode::G_FMA;
5511 TLI.isFPExtFoldable(
MI, PreferredFusedOpcode, DstType,
5516 B.buildInstr(PreferredFusedOpcode, {
MI.getOperand(0).getReg()},
5517 {FpExtX.getReg(0), FpExtY.getReg(0),
RHS.Reg});
5526 TLI.isFPExtFoldable(
MI, PreferredFusedOpcode, DstType,
5531 B.buildInstr(PreferredFusedOpcode, {
MI.getOperand(0).getReg()},
5532 {FpExtX.getReg(0), FpExtY.getReg(0),
LHS.Reg});
5542 assert(
MI.getOpcode() == TargetOpcode::G_FADD);
5544 bool AllowFusionGlobally, HasFMAD,
Aggressive;
5554 unsigned PreferredFusedOpcode =
5555 HasFMAD ? TargetOpcode::G_FMAD : TargetOpcode::G_FMA;
5568 if (
LHS.MI->getOpcode() == PreferredFusedOpcode &&
5570 TargetOpcode::G_FMUL) &&
5577 else if (
RHS.MI->getOpcode() == PreferredFusedOpcode &&
5579 TargetOpcode::G_FMUL) &&
5588 Register X = FMA->getOperand(1).getReg();
5589 Register Y = FMA->getOperand(2).getReg();
5595 B.buildInstr(PreferredFusedOpcode, {InnerFMA}, {U, V, Z});
5596 B.buildInstr(PreferredFusedOpcode, {
MI.getOperand(0).getReg()},
5607 assert(
MI.getOpcode() == TargetOpcode::G_FADD);
5609 bool AllowFusionGlobally, HasFMAD,
Aggressive;
5616 const auto &TLI = *
MI.getMF()->getSubtarget().getTargetLowering();
5623 unsigned PreferredFusedOpcode =
5624 HasFMAD ? TargetOpcode::G_FMAD : TargetOpcode::G_FMA;
5637 Register FpExtU =
B.buildFPExt(DstType, U).getReg(0);
5638 Register FpExtV =
B.buildFPExt(DstType, V).getReg(0);
5640 B.buildInstr(PreferredFusedOpcode, {DstType}, {FpExtU, FpExtV, Z})
5642 B.buildInstr(PreferredFusedOpcode, {
MI.getOperand(0).getReg()},
5649 if (
LHS.MI->getOpcode() == PreferredFusedOpcode &&
5653 TLI.isFPExtFoldable(
MI, PreferredFusedOpcode, DstType,
5658 LHS.MI->getOperand(1).getReg(),
5659 LHS.MI->getOperand(2).getReg(),
B);
5670 FMAMI->
getOpcode() == PreferredFusedOpcode) {
5673 TLI.isFPExtFoldable(
MI, PreferredFusedOpcode, DstType,
5678 X =
B.buildFPExt(DstType,
X).getReg(0);
5679 Y =
B.buildFPExt(DstType,
Y).getReg(0);
5690 if (
RHS.MI->getOpcode() == PreferredFusedOpcode &&
5694 TLI.isFPExtFoldable(
MI, PreferredFusedOpcode, DstType,
5699 RHS.MI->getOperand(1).getReg(),
5700 RHS.MI->getOperand(2).getReg(),
B);
5711 FMAMI->
getOpcode() == PreferredFusedOpcode) {
5714 TLI.isFPExtFoldable(
MI, PreferredFusedOpcode, DstType,
5719 X =
B.buildFPExt(DstType,
X).getReg(0);
5720 Y =
B.buildFPExt(DstType,
Y).getReg(0);
5733 assert(
MI.getOpcode() == TargetOpcode::G_FSUB);
5735 bool AllowFusionGlobally, HasFMAD,
Aggressive;
5747 int FirstMulHasFewerUses =
true;
5751 FirstMulHasFewerUses =
false;
5753 unsigned PreferredFusedOpcode =
5754 HasFMAD ? TargetOpcode::G_FMAD : TargetOpcode::G_FMA;
5757 if (FirstMulHasFewerUses &&
5762 B.buildInstr(PreferredFusedOpcode, {
MI.getOperand(0).getReg()},
5763 {
LHS.MI->getOperand(1).getReg(),
5764 LHS.MI->getOperand(2).getReg(), NegZ});
5773 B.buildFNeg(DstTy,
RHS.MI->getOperand(1).getReg()).
getReg(0);
5774 B.buildInstr(PreferredFusedOpcode, {
MI.getOperand(0).getReg()},
5775 {NegY,
RHS.MI->getOperand(2).getReg(),
LHS.Reg});
5785 assert(
MI.getOpcode() == TargetOpcode::G_FSUB);
5787 bool AllowFusionGlobally, HasFMAD,
Aggressive;
5795 unsigned PreferredFusedOpcode =
5796 HasFMAD ? TargetOpcode::G_FMAD : TargetOpcode::G_FMA;
5807 Register NegZ =
B.buildFNeg(DstTy, RHSReg).getReg(0);
5808 B.buildInstr(PreferredFusedOpcode, {
MI.getOperand(0).getReg()},
5820 B.buildInstr(PreferredFusedOpcode, {
MI.getOperand(0).getReg()},
5832 assert(
MI.getOpcode() == TargetOpcode::G_FSUB);
5834 bool AllowFusionGlobally, HasFMAD,
Aggressive;
5842 unsigned PreferredFusedOpcode =
5843 HasFMAD ? TargetOpcode::G_FMAD : TargetOpcode::G_FMA;
5855 Register NegZ =
B.buildFNeg(DstTy, RHSReg).getReg(0);
5856 B.buildInstr(PreferredFusedOpcode, {
MI.getOperand(0).getReg()},
5857 {FpExtX, FpExtY, NegZ});
5869 Register NegY =
B.buildFNeg(DstTy, FpExtY).getReg(0);
5872 B.buildInstr(PreferredFusedOpcode, {
MI.getOperand(0).getReg()},
5873 {NegY, FpExtZ, LHSReg});
5883 assert(
MI.getOpcode() == TargetOpcode::G_FSUB);
5885 bool AllowFusionGlobally, HasFMAD,
Aggressive;
5889 const auto &TLI = *
MI.getMF()->getSubtarget().getTargetLowering();
5894 unsigned PreferredFusedOpcode =
5895 HasFMAD ? TargetOpcode::G_FMAD : TargetOpcode::G_FMA;
5899 Register FpExtX =
B.buildFPExt(DstTy,
X).getReg(0);
5900 Register FpExtY =
B.buildFPExt(DstTy,
Y).getReg(0);
5901 B.buildInstr(PreferredFusedOpcode, {Dst}, {FpExtX, FpExtY, Z});
5912 TLI.isFPExtFoldable(
MI, PreferredFusedOpcode, DstTy,
5918 B.buildFNeg(
MI.getOperand(0).getReg(), FMAReg);
5928 TLI.isFPExtFoldable(
MI, PreferredFusedOpcode, DstTy,
5941 unsigned &IdxToPropagate) {
5943 switch (
MI.getOpcode()) {
5946 case TargetOpcode::G_FMINNUM:
5947 case TargetOpcode::G_FMAXNUM:
5948 PropagateNaN =
false;
5950 case TargetOpcode::G_FMINIMUM:
5951 case TargetOpcode::G_FMAXIMUM:
5952 PropagateNaN =
true;
5956 auto MatchNaN = [&](
unsigned Idx) {
5961 IdxToPropagate = PropagateNaN ?
Idx : (
Idx == 1 ? 2 : 1);
5965 return MatchNaN(1) || MatchNaN(2);
5969 assert(
MI.getOpcode() == TargetOpcode::G_ADD &&
"Expected a G_ADD");
5979 Reg == MaybeSameReg;
6014 std::optional<ValueAndVReg> ShiftAmount;
6045 std::optional<ValueAndVReg> ShiftAmt;
6052 return ShiftAmt->Value.getZExtValue() == MatchTy.
getSizeInBits() &&
6056unsigned CombinerHelper::getFPMinMaxOpcForSelect(
6058 SelectPatternNaNBehaviour VsNaNRetVal)
const {
6059 assert(VsNaNRetVal != SelectPatternNaNBehaviour::NOT_APPLICABLE &&
6060 "Expected a NaN behaviour?");
6070 if (VsNaNRetVal == SelectPatternNaNBehaviour::RETURNS_OTHER)
6071 return TargetOpcode::G_FMAXNUM;
6072 if (VsNaNRetVal == SelectPatternNaNBehaviour::RETURNS_NAN)
6073 return TargetOpcode::G_FMAXIMUM;
6074 if (
isLegal({TargetOpcode::G_FMAXNUM, {DstTy}}))
6075 return TargetOpcode::G_FMAXNUM;
6076 if (
isLegal({TargetOpcode::G_FMAXIMUM, {DstTy}}))
6077 return TargetOpcode::G_FMAXIMUM;
6083 if (VsNaNRetVal == SelectPatternNaNBehaviour::RETURNS_OTHER)
6084 return TargetOpcode::G_FMINNUM;
6085 if (VsNaNRetVal == SelectPatternNaNBehaviour::RETURNS_NAN)
6086 return TargetOpcode::G_FMINIMUM;
6087 if (
isLegal({TargetOpcode::G_FMINNUM, {DstTy}}))
6088 return TargetOpcode::G_FMINNUM;
6089 if (!
isLegal({TargetOpcode::G_FMINIMUM, {DstTy}}))
6091 return TargetOpcode::G_FMINIMUM;
6095CombinerHelper::SelectPatternNaNBehaviour
6097 bool IsOrderedComparison)
const {
6101 if (!LHSSafe && !RHSSafe)
6102 return SelectPatternNaNBehaviour::NOT_APPLICABLE;
6103 if (LHSSafe && RHSSafe)
6104 return SelectPatternNaNBehaviour::RETURNS_ANY;
6107 if (IsOrderedComparison)
6108 return LHSSafe ? SelectPatternNaNBehaviour::RETURNS_NAN
6109 : SelectPatternNaNBehaviour::RETURNS_OTHER;
6112 return LHSSafe ? SelectPatternNaNBehaviour::RETURNS_OTHER
6113 : SelectPatternNaNBehaviour::RETURNS_NAN;
6135 SelectPatternNaNBehaviour ResWithKnownNaNInfo =
6137 if (ResWithKnownNaNInfo == SelectPatternNaNBehaviour::NOT_APPLICABLE)
6139 if (TrueVal == CmpRHS && FalseVal == CmpLHS) {
6142 if (ResWithKnownNaNInfo == SelectPatternNaNBehaviour::RETURNS_NAN)
6143 ResWithKnownNaNInfo = SelectPatternNaNBehaviour::RETURNS_OTHER;
6144 else if (ResWithKnownNaNInfo == SelectPatternNaNBehaviour::RETURNS_OTHER)
6145 ResWithKnownNaNInfo = SelectPatternNaNBehaviour::RETURNS_NAN;
6147 if (TrueVal != CmpLHS || FalseVal != CmpRHS)
6150 unsigned Opc = getFPMinMaxOpcForSelect(Pred, DstTy, ResWithKnownNaNInfo);
6151 if (!Opc || !
isLegal({Opc, {DstTy}}))
6155 if (Opc != TargetOpcode::G_FMAXIMUM && Opc != TargetOpcode::G_FMINIMUM) {
6160 if (!KnownNonZeroSide || !KnownNonZeroSide->Value.isNonZero()) {
6162 if (!KnownNonZeroSide || !KnownNonZeroSide->Value.isNonZero())
6167 B.buildInstr(Opc, {Dst}, {CmpLHS, CmpRHS});
6175 assert(
MI.getOpcode() == TargetOpcode::G_SELECT);
6182 Register TrueVal =
MI.getOperand(2).getReg();
6183 Register FalseVal =
MI.getOperand(3).getReg();
6184 return matchFPSelectToMinMax(Dst,
Cond, TrueVal, FalseVal, MatchInfo);
6189 assert(
MI.getOpcode() == TargetOpcode::G_ICMP);
6202 if (MatchedSub &&
X != OpLHS)
6210 Y =
X == OpLHS ? OpRHS :
X == OpRHS ? OpLHS :
Register();
6214 B.buildICmp(Pred, Dst,
Y, Zero);
6220 Register ShiftReg =
MI.getOperand(2).getReg();
6222 auto IsShiftTooBig = [&](
const Constant *
C) {
6223 auto *CI = dyn_cast<ConstantInt>(
C);
6238 if (LHSDef->getOpcode() != TargetOpcode::G_CONSTANT_FOLD_BARRIER)
6241 TargetOpcode::G_CONSTANT_FOLD_BARRIER &&
6248 std::optional<FPValueAndVReg> ValAndVReg;
6258 MI.getOperand(1).setReg(RHSReg);
6259 MI.getOperand(2).setReg(LHSReg);
6263bool CombinerHelper::isOneOrOneSplat(
Register Src,
bool AllowUndefs) {
6266 return isConstantSplatVector(Src, 1, AllowUndefs);
6268 if (AllowUndefs && getOpcodeDef<GImplicitDef>(Src,
MRI) !=
nullptr)
6271 return IConstant && IConstant->Value == 1;
6276bool CombinerHelper::isZeroOrZeroSplat(
Register Src,
bool AllowUndefs) {
6279 return isConstantSplatVector(Src, 0, AllowUndefs);
6281 if (AllowUndefs && getOpcodeDef<GImplicitDef>(Src,
MRI) !=
nullptr)
6284 return IConstant && IConstant->Value == 0;
6291bool CombinerHelper::isConstantSplatVector(
Register Src, int64_t SplatValue,
6298 for (
unsigned I = 0;
I < NumSources; ++
I) {
6301 if (ImplicitDef && AllowUndefs)
6303 if (ImplicitDef && !AllowUndefs)
6305 std::optional<ValueAndVReg> IConstant =
6307 if (IConstant && IConstant->Value == SplatValue)
6317CombinerHelper::getConstantOrConstantSplatVector(
Register Src) {
6320 return IConstant->Value;
6324 return std::nullopt;
6327 std::optional<APInt>
Value = std::nullopt;
6328 for (
unsigned I = 0;
I < NumSources; ++
I) {
6329 std::optional<ValueAndVReg> IConstant =
6332 return std::nullopt;
6336 return std::nullopt;
6342bool CombinerHelper::isConstantOrConstantVectorI(
Register Src)
const {
6352 for (
unsigned I = 0;
I < NumSources; ++
I) {
6353 std::optional<ValueAndVReg> IConstant =
6362bool CombinerHelper::tryFoldSelectOfConstants(
GSelect *
Select,
6380 std::optional<ValueAndVReg> TrueOpt =
6382 std::optional<ValueAndVReg> FalseOpt =
6385 if (!TrueOpt || !FalseOpt)
6388 APInt TrueValue = TrueOpt->Value;
6389 APInt FalseValue = FalseOpt->Value;
6394 B.setInstrAndDebugLoc(*
Select);
6395 B.buildZExtOrTrunc(Dest,
Cond);
6403 B.setInstrAndDebugLoc(*
Select);
6404 B.buildSExtOrTrunc(Dest,
Cond);
6412 B.setInstrAndDebugLoc(*
Select);
6414 B.buildNot(Inner,
Cond);
6415 B.buildZExtOrTrunc(Dest, Inner);
6423 B.setInstrAndDebugLoc(*
Select);
6425 B.buildNot(Inner,
Cond);
6426 B.buildSExtOrTrunc(Dest, Inner);
6432 if (TrueValue - 1 == FalseValue) {
6434 B.setInstrAndDebugLoc(*
Select);
6436 B.buildZExtOrTrunc(Inner,
Cond);
6437 B.buildAdd(Dest, Inner, False);
6443 if (TrueValue + 1 == FalseValue) {
6445 B.setInstrAndDebugLoc(*
Select);
6447 B.buildSExtOrTrunc(Inner,
Cond);
6448 B.buildAdd(Dest, Inner, False);
6456 B.setInstrAndDebugLoc(*
Select);
6458 B.buildZExtOrTrunc(Inner,
Cond);
6461 auto ShAmtC =
B.buildConstant(ShiftTy, TrueValue.
exactLogBase2());
6462 B.buildShl(Dest, Inner, ShAmtC, Flags);
6469 B.setInstrAndDebugLoc(*
Select);
6471 B.buildSExtOrTrunc(Inner,
Cond);
6472 B.buildOr(Dest, Inner, False, Flags);
6480 B.setInstrAndDebugLoc(*
Select);
6482 B.buildNot(Not,
Cond);
6484 B.buildSExtOrTrunc(Inner, Not);
6485 B.buildOr(Dest, Inner, True, Flags);
6494bool CombinerHelper::tryFoldBoolSelectToLogic(
GSelect *
Select,
6511 if (CondTy != TrueTy)
6516 if ((
Cond == True) || isOneOrOneSplat(True,
true)) {
6518 B.setInstrAndDebugLoc(*
Select);
6520 B.buildZExtOrTrunc(Ext,
Cond);
6521 auto FreezeFalse =
B.buildFreeze(TrueTy, False);
6522 B.buildOr(DstReg, Ext, FreezeFalse, Flags);
6529 if ((
Cond == False) || isZeroOrZeroSplat(False,
true)) {
6531 B.setInstrAndDebugLoc(*
Select);
6533 B.buildZExtOrTrunc(Ext,
Cond);
6534 auto FreezeTrue =
B.buildFreeze(TrueTy, True);
6535 B.buildAnd(DstReg, Ext, FreezeTrue);
6541 if (isOneOrOneSplat(False,
true)) {
6543 B.setInstrAndDebugLoc(*
Select);
6546 B.buildNot(Inner,
Cond);
6549 B.buildZExtOrTrunc(Ext, Inner);
6550 auto FreezeTrue =
B.buildFreeze(TrueTy, True);
6551 B.buildOr(DstReg, Ext, FreezeTrue, Flags);
6557 if (isZeroOrZeroSplat(True,
true)) {
6559 B.setInstrAndDebugLoc(*
Select);
6562 B.buildNot(Inner,
Cond);
6565 B.buildZExtOrTrunc(Ext, Inner);
6566 auto FreezeFalse =
B.buildFreeze(TrueTy, False);
6567 B.buildAnd(DstReg, Ext, FreezeFalse);
6575bool CombinerHelper::tryFoldSelectToIntMinMax(
GSelect *
Select,
6605 if (True == CmpRHS && False == CmpLHS) {
6613 if (True == CmpLHS && False == CmpRHS) {
6620 B.buildUMax(DstReg, True, False);
6629 B.buildSMax(DstReg, True, False);
6638 B.buildUMin(DstReg, True, False);
6647 B.buildSMin(DstReg, True, False);
6662 if (tryFoldSelectOfConstants(
Select, MatchInfo))
6665 if (tryFoldBoolSelectToLogic(
Select, MatchInfo))
6668 if (tryFoldSelectToIntMinMax(
Select, MatchInfo))
6678bool CombinerHelper::tryFoldAndOrOrICmpsUsingRanges(
GLogicalBinOp *Logic,
6680 assert(Logic->
getOpcode() != TargetOpcode::G_XOR &&
"unexpected xor");
6681 bool IsAnd = Logic->
getOpcode() == TargetOpcode::G_AND;
6685 unsigned Flags = Logic->
getFlags();
6688 GICmp *Cmp1 = getOpcodeDef<GICmp>(LHS,
MRI);
6693 GICmp *Cmp2 = getOpcodeDef<GICmp>(RHS,
MRI);
6704 std::optional<ValueAndVReg> MaybeC1 =
6708 C1 = MaybeC1->Value;
6710 std::optional<ValueAndVReg> MaybeC2 =
6714 C2 = MaybeC2->Value;
6735 std::optional<APInt> Offset1;
6736 std::optional<APInt> Offset2;
6738 if (
GAdd *
Add = getOpcodeDef<GAdd>(R1,
MRI)) {
6739 std::optional<ValueAndVReg> MaybeOffset1 =
6742 R1 =
Add->getLHSReg();
6743 Offset1 = MaybeOffset1->Value;
6747 std::optional<ValueAndVReg> MaybeOffset2 =
6750 R2 =
Add->getLHSReg();
6751 Offset2 = MaybeOffset2->Value;
6770 bool CreateMask =
false;
6783 if (!LowerDiff.
isPowerOf2() || LowerDiff != UpperDiff ||
6796 CR->getEquivalentICmp(NewPred, NewC,
Offset);
6806 if (CreateMask &&
Offset != 0) {
6807 auto TildeLowerDiff =
B.buildConstant(CmpOperandTy, ~LowerDiff);
6808 auto And =
B.buildAnd(CmpOperandTy, R1, TildeLowerDiff);
6809 auto OffsetC =
B.buildConstant(CmpOperandTy,
Offset);
6810 auto Add =
B.buildAdd(CmpOperandTy,
And, OffsetC, Flags);
6811 auto NewCon =
B.buildConstant(CmpOperandTy, NewC);
6812 auto ICmp =
B.buildICmp(NewPred, CmpTy,
Add, NewCon);
6813 B.buildZExtOrTrunc(DstReg, ICmp);
6814 }
else if (CreateMask &&
Offset == 0) {
6815 auto TildeLowerDiff =
B.buildConstant(CmpOperandTy, ~LowerDiff);
6816 auto And =
B.buildAnd(CmpOperandTy, R1, TildeLowerDiff);
6817 auto NewCon =
B.buildConstant(CmpOperandTy, NewC);
6818 auto ICmp =
B.buildICmp(NewPred, CmpTy,
And, NewCon);
6819 B.buildZExtOrTrunc(DstReg, ICmp);
6820 }
else if (!CreateMask &&
Offset != 0) {
6821 auto OffsetC =
B.buildConstant(CmpOperandTy,
Offset);
6822 auto Add =
B.buildAdd(CmpOperandTy, R1, OffsetC, Flags);
6823 auto NewCon =
B.buildConstant(CmpOperandTy, NewC);
6824 auto ICmp =
B.buildICmp(NewPred, CmpTy,
Add, NewCon);
6825 B.buildZExtOrTrunc(DstReg, ICmp);
6826 }
else if (!CreateMask &&
Offset == 0) {
6827 auto NewCon =
B.buildConstant(CmpOperandTy, NewC);
6828 auto ICmp =
B.buildICmp(NewPred, CmpTy, R1, NewCon);
6829 B.buildZExtOrTrunc(DstReg, ICmp);
6837bool CombinerHelper::tryFoldLogicOfFCmps(
GLogicalBinOp *Logic,
6843 bool IsAnd = Logic->
getOpcode() == TargetOpcode::G_AND;
6846 GFCmp *Cmp1 = getOpcodeDef<GFCmp>(LHS,
MRI);
6851 GFCmp *Cmp2 = getOpcodeDef<GFCmp>(RHS,
MRI);
6861 {TargetOpcode::G_FCMP, {CmpTy, CmpOperandTy}}) ||
6875 if (LHS0 == RHS1 && LHS1 == RHS0) {
6881 if (LHS0 == RHS0 && LHS1 == RHS1) {
6885 unsigned NewPred = IsAnd ? CmpCodeL & CmpCodeR : CmpCodeL | CmpCodeR;
6892 auto False =
B.buildConstant(CmpTy, 0);
6893 B.buildZExtOrTrunc(DestReg, False);
6900 B.buildZExtOrTrunc(DestReg, True);
6902 auto Cmp =
B.buildFCmp(Pred, CmpTy, LHS0, LHS1, Flags);
6903 B.buildZExtOrTrunc(DestReg, Cmp);
6915 if (tryFoldAndOrOrICmpsUsingRanges(
And, MatchInfo))
6918 if (tryFoldLogicOfFCmps(
And, MatchInfo))
6927 if (tryFoldAndOrOrICmpsUsingRanges(
Or, MatchInfo))
6930 if (tryFoldLogicOfFCmps(
Or, MatchInfo))
6944 bool IsSigned =
Add->isSigned();
6957 B.buildUndef(Carry);
6967 if (isConstantOrConstantVectorI(
LHS) && !isConstantOrConstantVectorI(
RHS)) {
6970 B.buildSAddo(Dst, Carry,
RHS,
LHS);
6976 B.buildUAddo(Dst, Carry,
RHS,
LHS);
6981 std::optional<APInt> MaybeLHS = getConstantOrConstantSplatVector(
LHS);
6982 std::optional<APInt> MaybeRHS = getConstantOrConstantSplatVector(
RHS);
6988 APInt Result = IsSigned ? MaybeLHS->sadd_ov(*MaybeRHS, Overflow)
6989 : MaybeLHS->uadd_ov(*MaybeRHS, Overflow);
6991 B.buildConstant(Dst, Result);
6992 B.buildConstant(Carry, Overflow);
7000 B.buildCopy(Dst,
LHS);
7001 B.buildConstant(Carry, 0);
7013 std::optional<APInt> MaybeAddRHS =
7014 getConstantOrConstantSplatVector(AddLHS->
getRHSReg());
7017 APInt NewC = IsSigned ? MaybeAddRHS->sadd_ov(*MaybeRHS, Overflow)
7018 : MaybeAddRHS->uadd_ov(*MaybeRHS, Overflow);
7022 auto ConstRHS =
B.buildConstant(DstTy, NewC);
7023 B.buildSAddo(Dst, Carry, AddLHS->
getLHSReg(), ConstRHS);
7029 auto ConstRHS =
B.buildConstant(DstTy, NewC);
7030 B.buildUAddo(Dst, Carry, AddLHS->
getLHSReg(), ConstRHS);
7055 B.buildConstant(Carry, 0);
7063 B.buildConstant(Carry, 1);
7078 B.buildConstant(Carry, 0);
7094 B.buildConstant(Carry, 0);
7102 B.buildConstant(Carry, 1);
unsigned const MachineRegisterInfo * MRI
MachineInstrBuilder & UseMI
MachineInstrBuilder MachineInstrBuilder & DefMI
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
amdgpu AMDGPU Register Bank Select
This file declares a class to represent arbitrary precision floating point values and provide a varie...
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
static bool hasMoreUses(const MachineInstr &MI0, const MachineInstr &MI1, const MachineRegisterInfo &MRI)
static bool isContractableFMul(MachineInstr &MI, bool AllowFusionGlobally)
Checks if MI is TargetOpcode::G_FMUL and contractable either due to global flags or MachineInstr flag...
static unsigned getIndexedOpc(unsigned LdStOpc)
static APFloat constantFoldFpUnary(const MachineInstr &MI, const MachineRegisterInfo &MRI, const APFloat &Val)
static std::optional< std::pair< GZExtLoad *, int64_t > > matchLoadAndBytePosition(Register Reg, unsigned MemSizeInBits, const MachineRegisterInfo &MRI)
Helper function for findLoadOffsetsForLoadOrCombine.
static Register peekThroughBitcast(Register Reg, const MachineRegisterInfo &MRI)
static unsigned bigEndianByteAt(const unsigned ByteWidth, const unsigned I)
static cl::opt< bool > ForceLegalIndexing("force-legal-indexing", cl::Hidden, cl::init(false), cl::desc("Force all indexed operations to be " "legal for the GlobalISel combiner"))
static cl::opt< unsigned > PostIndexUseThreshold("post-index-use-threshold", cl::Hidden, cl::init(32), cl::desc("Number of uses of a base pointer to check before it is no longer " "considered for post-indexing."))
static std::optional< bool > isBigEndian(const SmallDenseMap< int64_t, int64_t, 8 > &MemOffset2Idx, int64_t LowestIdx)
Given a map from byte offsets in memory to indices in a load/store, determine if that map corresponds...
static unsigned getExtLoadOpcForExtend(unsigned ExtOpc)
static Type * getTypeForLLT(LLT Ty, LLVMContext &C)
static bool isConstValidTrue(const TargetLowering &TLI, unsigned ScalarSizeBits, int64_t Cst, bool IsVector, bool IsFP)
static LLT getMidVTForTruncRightShiftCombine(LLT ShiftTy, LLT TruncTy)
static bool canFoldInAddressingMode(GLoadStore *MI, const TargetLowering &TLI, MachineRegisterInfo &MRI)
Return true if 'MI' is a load or a store that may be fold it's address operand into the load / store ...
static unsigned littleEndianByteAt(const unsigned ByteWidth, const unsigned I)
static Register buildLogBase2(Register V, MachineIRBuilder &MIB)
Determines the LogBase2 value for a non-null input value using the transform: LogBase2(V) = (EltBits ...
This contains common combine transformations that may be used in a combine pass,or by the target else...
Returns the sub type a function will return at a given Idx Should correspond to the result type of an ExtractValue instruction executed with just that one unsigned Idx
static GCMetadataPrinterRegistry::Add< ErlangGCPrinter > X("erlang", "erlang-compatible garbage collector")
Rewrite Partial Register Uses
This contains common code to allow clients to notify changes to machine instr.
Provides analysis for querying information about KnownBits during GISel passes.
Declares convenience wrapper classes for interpreting MachineInstr instances as specific generic oper...
Interface for Targets to specify which operations they can successfully select and how the others sho...
Implement a low-level type suitable for MachineInstr level instruction selection.
Contains matchers for matching SSA Machine Instructions.
mir Rename Register Operands
This file declares the MachineIRBuilder class.
unsigned const TargetRegisterInfo * TRI
static unsigned getReg(const MCDisassembler *D, unsigned RC, unsigned RegNo)
static GCMetadataPrinterRegistry::Add< OcamlGCMetadataPrinter > Y("ocaml", "ocaml 3.10-compatible collector")
const SmallVectorImpl< MachineOperand > & Cond
static bool isValid(const char C)
Returns true if C is a valid mangled character: <0-9a-zA-Z_>.
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
This file implements a set that has insertion order iteration characteristics.
This file implements the SmallBitVector class.
This file describes how to lower LLVM code to machine code.
const fltSemantics & getSemantics() const
opStatus fusedMultiplyAdd(const APFloat &Multiplicand, const APFloat &Addend, roundingMode RM)
APInt bitcastToAPInt() const
Class for arbitrary precision integers.
APInt zext(unsigned width) const
Zero extend to a new width.
uint64_t getZExtValue() const
Get zero extended value.
APInt multiplicativeInverse(const APInt &modulo) const
Computes the multiplicative inverse of this APInt for a given modulo.
APInt zextOrTrunc(unsigned width) const
Zero extend or truncate to width.
APInt trunc(unsigned width) const
Truncate to new width.
bool isAllOnes() const
Determine if all bits are set. This is true for zero-width values.
bool ugt(const APInt &RHS) const
Unsigned greater than comparison.
bool isZero() const
Determine if this value is zero, i.e. all bits are clear.
APInt urem(const APInt &RHS) const
Unsigned remainder operation.
unsigned getBitWidth() const
Return the number of bits in the APInt.
bool ult(const APInt &RHS) const
Unsigned less than comparison.
int32_t exactLogBase2() const
void ashrInPlace(unsigned ShiftAmt)
Arithmetic right-shift this APInt by ShiftAmt in place.
unsigned countr_zero() const
Count the number of trailing zero bits.
static APInt getSignedMinValue(unsigned numBits)
Gets minimum signed value of APInt for a specific bit width.
APInt sextOrTrunc(unsigned width) const
Sign extend or truncate to width.
unsigned countl_one() const
Count the number of leading one bits.
bool isMask(unsigned numBits) const
bool isPowerOf2() const
Check if this APInt's value is a power of two greater than zero.
static APInt getZero(unsigned numBits)
Get the '0' value for the specified bit-width.
bool isOne() const
Determine if this is a value of 1.
static APInt getOneBitSet(unsigned numBits, unsigned BitNo)
Return an APInt with exactly one bit set in the result.
int64_t getSExtValue() const
Get sign extended value.
APInt lshr(unsigned shiftAmt) const
Logical right-shift function.
unsigned countr_one() const
Count the number of trailing one bits.
bool uge(const APInt &RHS) const
Unsigned greater or equal comparison.
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
size_t size() const
size - Get the array size.
AttributeSet getAttributes(unsigned Index) const
The attributes for the specified index are returned.
bool isEquality() const
Determine if this is an equals/not equals predicate.
Predicate
This enumeration lists the possible predicates for CmpInst subclasses.
@ FCMP_TRUE
1 1 1 1 Always true (always folded)
@ ICMP_SLT
signed less than
@ ICMP_SLE
signed less or equal
@ FCMP_OLT
0 1 0 0 True if ordered and less than
@ FCMP_ULE
1 1 0 1 True if unordered, less than, or equal
@ FCMP_OGT
0 0 1 0 True if ordered and greater than
@ FCMP_OGE
0 0 1 1 True if ordered and greater than or equal
@ ICMP_UGE
unsigned greater or equal
@ ICMP_UGT
unsigned greater than
@ ICMP_SGT
signed greater than
@ FCMP_ULT
1 1 0 0 True if unordered or less than
@ ICMP_ULT
unsigned less than
@ FCMP_UGT
1 0 1 0 True if unordered or greater than
@ FCMP_OLE
0 1 0 1 True if ordered and less than or equal
@ ICMP_SGE
signed greater or equal
@ ICMP_ULE
unsigned less or equal
@ FCMP_UGE
1 0 1 1 True if unordered, greater than, or equal
@ FCMP_FALSE
0 0 0 0 Always false (always folded)
Predicate getSwappedPredicate() const
For example, EQ->EQ, SLE->SGE, ULT->UGT, OEQ->OEQ, ULE->UGE, OLT->OGT, etc.
Predicate getInversePredicate() const
For example, EQ -> NE, UGT -> ULE, SLT -> SGE, OEQ -> UNE, UGT -> OLE, OLT -> UGE,...
static bool isOrdered(Predicate predicate)
Determine if the predicate is an ordered operation.
void applyUDivByConst(MachineInstr &MI)
void applyCombineMulToShl(MachineInstr &MI, unsigned &ShiftVal)
bool matchCombineShuffleVector(MachineInstr &MI, SmallVectorImpl< Register > &Ops)
Check if the G_SHUFFLE_VECTOR MI can be replaced by a concat_vectors.
bool matchPtrAddZero(MachineInstr &MI)
}
bool matchAllExplicitUsesAreUndef(MachineInstr &MI)
Return true if all register explicit use operands on MI are defined by a G_IMPLICIT_DEF.
void replaceSingleDefInstWithOperand(MachineInstr &MI, unsigned OpIdx)
Delete MI and replace all of its uses with its OpIdx-th operand.
const RegisterBank * getRegBank(Register Reg) const
Get the register bank of Reg.
bool matchReassocPtrAdd(MachineInstr &MI, BuildFnTy &MatchInfo)
Reassociate pointer calculations with G_ADD involved, to allow better addressing mode usage.
bool matchUDivByConst(MachineInstr &MI)
Combine G_UDIV by constant into a multiply by magic constant.
void applyExtractVecEltBuildVec(MachineInstr &MI, Register &Reg)
bool matchInsertExtractVecEltOutOfBounds(MachineInstr &MI)
Return true if a G_{EXTRACT,INSERT}_VECTOR_ELT has an out of range index.
bool matchShiftsTooBig(MachineInstr &MI)
Match shifts greater or equal to the bitwidth of the operation.
bool tryCombineCopy(MachineInstr &MI)
If MI is COPY, try to combine it.
bool matchTruncLshrBuildVectorFold(MachineInstr &MI, Register &MatchInfo)
bool matchUndefStore(MachineInstr &MI)
Return true if a G_STORE instruction MI is storing an undef value.
bool matchRedundantBinOpInEquality(MachineInstr &MI, BuildFnTy &MatchInfo)
Transform: (X + Y) == X -> Y == 0 (X - Y) == X -> Y == 0 (X ^ Y) == X -> Y == 0 (X + Y) !...
bool matchRedundantSExtInReg(MachineInstr &MI)
bool matchCombineFAddFpExtFMulToFMadOrFMAAggressive(MachineInstr &MI, BuildFnTy &MatchInfo)
bool matchReassocConstantInnerRHS(GPtrAdd &MI, MachineInstr *RHS, BuildFnTy &MatchInfo)
bool matchSubAddSameReg(MachineInstr &MI, BuildFnTy &MatchInfo)
Transform: (x + y) - y -> x (x + y) - x -> y x - (y + x) -> 0 - y x - (x + z) -> 0 - z.
bool matchConstantFoldFPBinOp(MachineInstr &MI, ConstantFP *&MatchInfo)
Do constant FP folding when opportunities are exposed after MIR building.
void applyCombineShiftToUnmerge(MachineInstr &MI, const unsigned &ShiftVal)
void applyCombineUnmergeZExtToZExt(MachineInstr &MI)
void applyCommuteBinOpOperands(MachineInstr &MI)
bool matchBinOpSameVal(MachineInstr &MI)
Optimize (x op x) -> x.
void applyCombineUnmergeConstant(MachineInstr &MI, SmallVectorImpl< APInt > &Csts)
bool matchCombineFSubFNegFMulToFMadOrFMA(MachineInstr &MI, BuildFnTy &MatchInfo)
Transform (fsub (fneg (fmul, x, y)), z) -> (fma (fneg x), y, (fneg z)) (fsub (fneg (fmul,...
bool matchCombineCopy(MachineInstr &MI)
bool matchConstantSelectCmp(MachineInstr &MI, unsigned &OpIdx)
Return true if a G_SELECT instruction MI has a constant comparison.
void eraseInst(MachineInstr &MI)
Erase MI.
void replaceRegWith(MachineRegisterInfo &MRI, Register FromReg, Register ToReg) const
MachineRegisterInfo::replaceRegWith() and inform the observer of the changes.
void replaceRegOpWith(MachineRegisterInfo &MRI, MachineOperand &FromRegOp, Register ToReg) const
Replace a single register operand with a new register and inform the observer of the changes.
bool matchCombineFAddFMAFMulToFMadOrFMA(MachineInstr &MI, BuildFnTy &MatchInfo)
Transform (fadd (fma x, y, (fmul u, v)), z) -> (fma x, y, (fma u, v, z)) (fadd (fmad x,...
void applySimplifyAddToSub(MachineInstr &MI, std::tuple< Register, Register > &MatchInfo)
bool matchSimplifySelectToMinMax(MachineInstr &MI, BuildFnTy &MatchInfo)
bool matchCombineConcatVectors(MachineInstr &MI, SmallVector< Register > &Ops)
If MI is G_CONCAT_VECTORS, try to combine it.
bool matchAddSubSameReg(MachineInstr &MI, Register &Src)
Transform G_ADD(x, G_SUB(y, x)) to y.
void applyRotateOutOfRange(MachineInstr &MI)
bool matchMulOBy2(MachineInstr &MI, BuildFnTy &MatchInfo)
Match: (G_UMULO x, 2) -> (G_UADDO x, x) (G_SMULO x, 2) -> (G_SADDO x, x)
bool matchRotateOutOfRange(MachineInstr &MI)
void applyCombineConstPtrAddToI2P(MachineInstr &MI, APInt &NewCst)
void applyCombineTruncOfShift(MachineInstr &MI, std::pair< MachineInstr *, LLT > &MatchInfo)
void applyCombineShuffleVector(MachineInstr &MI, const ArrayRef< Register > Ops)
Replace MI with a concat_vectors with Ops.
const TargetLowering & getTargetLowering() const
void applyBuildFnNoErase(MachineInstr &MI, BuildFnTy &MatchInfo)
Use a function which takes in a MachineIRBuilder to perform a combine.
void applyPtrAddZero(MachineInstr &MI)
bool matchTruncBuildVectorFold(MachineInstr &MI, Register &MatchInfo)
void setRegBank(Register Reg, const RegisterBank *RegBank)
Set the register bank of Reg.
bool matchRedundantAnd(MachineInstr &MI, Register &Replacement)
void replaceInstWithConstant(MachineInstr &MI, int64_t C)
Replace an instruction with a G_CONSTANT with value C.
bool matchAshrShlToSextInreg(MachineInstr &MI, std::tuple< Register, int64_t > &MatchInfo)
Match ashr (shl x, C), C -> sext_inreg (C)
bool tryCombineExtendingLoads(MachineInstr &MI)
If MI is extend that consumes the result of a load, try to combine it.
bool tryCombineShiftToUnmerge(MachineInstr &MI, unsigned TargetShiftAmount)
bool matchCombineUnmergeUndef(MachineInstr &MI, std::function< void(MachineIRBuilder &)> &MatchInfo)
Transform G_UNMERGE G_IMPLICIT_DEF -> G_IMPLICIT_DEF, G_IMPLICIT_DEF, ...
void applySDivByConst(MachineInstr &MI)
bool matchUndefSelectCmp(MachineInstr &MI)
Return true if a G_SELECT instruction MI has an undef comparison.
void replaceInstWithUndef(MachineInstr &MI)
Replace an instruction with a G_IMPLICIT_DEF.
bool matchRedundantOr(MachineInstr &MI, Register &Replacement)
bool matchOperandIsUndef(MachineInstr &MI, unsigned OpIdx)
Check if operand OpIdx is undef.
void applyBuildFn(MachineInstr &MI, BuildFnTy &MatchInfo)
Use a function which takes in a MachineIRBuilder to perform a combine.
bool matchCombineConstPtrAddToI2P(MachineInstr &MI, APInt &NewCst)
void replaceInstWithFConstant(MachineInstr &MI, double C)
Replace an instruction with a G_FCONSTANT with value C.
bool matchBitfieldExtractFromSExtInReg(MachineInstr &MI, BuildFnTy &MatchInfo)
Form a G_SBFX from a G_SEXT_INREG fed by a right shift.
bool matchEqualDefs(const MachineOperand &MOP1, const MachineOperand &MOP2)
Return true if MOP1 and MOP2 are register operands are defined by equivalent instructions.
bool matchShiftImmedChain(MachineInstr &MI, RegisterImmPair &MatchInfo)
Fold (shift (shift base, x), y) -> (shift base (x+y))
bool matchPtrAddImmedChain(MachineInstr &MI, PtrAddChain &MatchInfo)
void applyShiftImmedChain(MachineInstr &MI, RegisterImmPair &MatchInfo)
void applyOptBrCondByInvertingCond(MachineInstr &MI, MachineInstr *&BrCond)
bool matchMulOBy0(MachineInstr &MI, BuildFnTy &MatchInfo)
Match: (G_*MULO x, 0) -> 0 + no carry out.
void replaceSingleDefInstWithReg(MachineInstr &MI, Register Replacement)
Delete MI and replace all of its uses with Replacement.
bool matchFunnelShiftToRotate(MachineInstr &MI)
Match an FSHL or FSHR that can be combined to a ROTR or ROTL rotate.
bool matchNotCmp(MachineInstr &MI, SmallVectorImpl< Register > &RegsToNegate)
Combine inverting a result of a compare into the opposite cond code.
void applyCombineExtOfExt(MachineInstr &MI, std::tuple< Register, unsigned > &MatchInfo)
void replaceOpcodeWith(MachineInstr &FromMI, unsigned ToOpcode) const
Replace the opcode in instruction with a new opcode and inform the observer of the changes.
bool matchOperandIsKnownToBeAPowerOfTwo(MachineInstr &MI, unsigned OpIdx)
Check if operand OpIdx is known to be a power of 2.
void applyCombineCopy(MachineInstr &MI)
void applyCombineTruncOfExt(MachineInstr &MI, std::pair< Register, unsigned > &MatchInfo)
bool matchAnyExplicitUseIsUndef(MachineInstr &MI)
Return true if any explicit use operand on MI is defined by a G_IMPLICIT_DEF.
bool matchFsubToFneg(MachineInstr &MI, Register &MatchInfo)
void applyCombineAddP2IToPtrAdd(MachineInstr &MI, std::pair< Register, bool > &PtrRegAndCommute)
bool matchNarrowBinopFeedingAnd(MachineInstr &MI, BuildFnTy &MatchInfo)
void applyCombineConcatVectors(MachineInstr &MI, SmallVector< Register > &Ops)
Replace MI with a flattened build_vector with Ops or an implicit_def if Ops is empty.
bool matchSextTruncSextLoad(MachineInstr &MI)
bool matchShiftOfShiftedLogic(MachineInstr &MI, ShiftOfShiftedLogic &MatchInfo)
If we have a shift-by-constant of a bitwise logic op that itself has a shift-by-constant operand with...
bool matchExtractAllEltsFromBuildVector(MachineInstr &MI, SmallVectorImpl< std::pair< Register, MachineInstr * > > &MatchInfo)
void applyCombineIndexedLoadStore(MachineInstr &MI, IndexedLoadStoreMatchInfo &MatchInfo)
MachineInstr * buildSDivUsingMul(MachineInstr &MI)
Given an G_SDIV MI expressing a signed divide by constant, return an expression that implements it by...
void applyFunnelShiftConstantModulo(MachineInstr &MI)
Replaces the shift amount in MI with ShiftAmt % BW.
bool matchConstantFoldBinOp(MachineInstr &MI, APInt &MatchInfo)
Do constant folding when opportunities are exposed after MIR building.
bool isPreLegalize() const
bool matchCombineLoadWithAndMask(MachineInstr &MI, BuildFnTy &MatchInfo)
Match (and (load x), mask) -> zextload x.
bool matchConstantOp(const MachineOperand &MOP, int64_t C)
Return true if MOP is defined by a G_CONSTANT or splat with a value equal to C.
bool matchCombineFSubFMulToFMadOrFMA(MachineInstr &MI, BuildFnTy &MatchInfo)
Transform (fsub (fmul x, y), z) -> (fma x, y, -z) (fsub (fmul x, y), z) -> (fmad x,...
bool matchAnd(MachineInstr &MI, BuildFnTy &MatchInfo)
Combine ands.
void applyCombineI2PToP2I(MachineInstr &MI, Register &Reg)
void applyNotCmp(MachineInstr &MI, SmallVectorImpl< Register > &RegsToNegate)
void applyCombineExtendingLoads(MachineInstr &MI, PreferredTuple &MatchInfo)
bool matchConstantFPOp(const MachineOperand &MOP, double C)
Return true if MOP is defined by a G_FCONSTANT or splat with a value exactly equal to C.
bool matchSimplifyAddToSub(MachineInstr &MI, std::tuple< Register, Register > &MatchInfo)
Return true if MI is a G_ADD which can be simplified to a G_SUB.
bool tryCombineMemCpyFamily(MachineInstr &MI, unsigned MaxLen=0)
Optimize memcpy intrinsics et al, e.g.
bool matchSelectSameVal(MachineInstr &MI)
Optimize (cond ? x : x) -> x.
void applyCombineConstantFoldFpUnary(MachineInstr &MI, const ConstantFP *Cst)
Transform fp_instr(cst) to constant result of the fp operation.
bool matchCombineExtendingLoads(MachineInstr &MI, PreferredTuple &MatchInfo)
bool tryReassocBinOp(unsigned Opc, Register DstReg, Register Op0, Register Op1, BuildFnTy &MatchInfo)
Try to reassociate to reassociate operands of a commutative binop.
bool isConstantLegalOrBeforeLegalizer(const LLT Ty) const
bool tryEmitMemcpyInline(MachineInstr &MI)
Emit loads and stores that perform the given memcpy.
void applyXorOfAndWithSameReg(MachineInstr &MI, std::pair< Register, Register > &MatchInfo)
bool matchXorOfAndWithSameReg(MachineInstr &MI, std::pair< Register, Register > &MatchInfo)
Fold (xor (and x, y), y) -> (and (not x), y) {.
bool matchCombineFSubFpExtFMulToFMadOrFMA(MachineInstr &MI, BuildFnTy &MatchInfo)
Transform (fsub (fpext (fmul x, y)), z) -> (fma (fpext x), (fpext y), (fneg z)) (fsub (fpext (fmul x,...
bool matchCombineFMinMaxNaN(MachineInstr &MI, unsigned &Info)
bool matchCombineShlOfExtend(MachineInstr &MI, RegisterImmPair &MatchData)
bool matchConstantFoldFMA(MachineInstr &MI, ConstantFP *&MatchInfo)
Constant fold G_FMA/G_FMAD.
bool matchBitfieldExtractFromAnd(MachineInstr &MI, BuildFnTy &MatchInfo)
Match: and (lshr x, cst), mask -> ubfx x, cst, width.
void applyShiftOfShiftedLogic(MachineInstr &MI, ShiftOfShiftedLogic &MatchInfo)
bool isLegal(const LegalityQuery &Query) const
bool matchSelect(MachineInstr &MI, BuildFnTy &MatchInfo)
Combine selects.
bool matchCombineUnmergeConstant(MachineInstr &MI, SmallVectorImpl< APInt > &Csts)
Transform G_UNMERGE Constant -> Constant1, Constant2, ...
bool matchICmpToTrueFalseKnownBits(MachineInstr &MI, int64_t &MatchInfo)
bool matchCombineAnyExtTrunc(MachineInstr &MI, Register &Reg)
Transform anyext(trunc(x)) to x.
void applySimplifyURemByPow2(MachineInstr &MI)
Combine G_UREM x, (known power of 2) to an add and bitmasking.
bool matchReassocFoldConstantsInSubTree(GPtrAdd &MI, MachineInstr *LHS, MachineInstr *RHS, BuildFnTy &MatchInfo)
MachineRegisterInfo & MRI
void applyUMulHToLShr(MachineInstr &MI)
bool matchLoadOrCombine(MachineInstr &MI, BuildFnTy &MatchInfo)
Match expression trees of the form.
bool matchShuffleToExtract(MachineInstr &MI)
bool matchUndefShuffleVectorMask(MachineInstr &MI)
Return true if a G_SHUFFLE_VECTOR instruction MI has an undef mask.
bool isLegalOrBeforeLegalizer(const LegalityQuery &Query) const
bool matchExtendThroughPhis(MachineInstr &MI, MachineInstr *&ExtMI)
bool matchAndOrDisjointMask(MachineInstr &MI, BuildFnTy &MatchInfo)
bool matchCombineExtractedVectorLoad(MachineInstr &MI, BuildFnTy &MatchInfo)
Combine a G_EXTRACT_VECTOR_ELT of a load into a narrowed load.
bool matchCombineMulToShl(MachineInstr &MI, unsigned &ShiftVal)
Transform a multiply by a power-of-2 value to a left shift.
bool matchBitfieldExtractFromShr(MachineInstr &MI, BuildFnTy &MatchInfo)
Match: shr (shl x, n), k -> sbfx/ubfx x, pos, width.
void applyFoldBinOpIntoSelect(MachineInstr &MI, const unsigned &SelectOpNo)
SelectOperand is the operand in binary operator MI that is the select to fold.
bool matchBuildVectorIdentityFold(MachineInstr &MI, Register &MatchInfo)
bool matchCombineFAddFMulToFMadOrFMA(MachineInstr &MI, BuildFnTy &MatchInfo)
Transform (fadd (fmul x, y), z) -> (fma x, y, z) (fadd (fmul x, y), z) -> (fmad x,...
bool matchRedundantNegOperands(MachineInstr &MI, BuildFnTy &MatchInfo)
Transform (fadd x, fneg(y)) -> (fsub x, y) (fadd fneg(x), y) -> (fsub y, x) (fsub x,...
bool matchCombineMergeUnmerge(MachineInstr &MI, Register &MatchInfo)
Fold away a merge of an unmerge of the corresponding values.
void applyCombineInsertVecElts(MachineInstr &MI, SmallVectorImpl< Register > &MatchInfo)
bool matchCombineUnmergeZExtToZExt(MachineInstr &MI)
Transform X, Y = G_UNMERGE(G_ZEXT(Z)) -> X = G_ZEXT(Z); Y = G_CONSTANT 0.
bool matchCombineUnmergeWithDeadLanesToTrunc(MachineInstr &MI)
Transform X, Y<dead> = G_UNMERGE Z -> X = G_TRUNC Z.
bool matchConstantLargerBitWidth(MachineInstr &MI, unsigned ConstIdx)
Checks if constant at ConstIdx is larger than MI 's bitwidth.
CombinerHelper(GISelChangeObserver &Observer, MachineIRBuilder &B, bool IsPreLegalize, GISelKnownBits *KB=nullptr, MachineDominatorTree *MDT=nullptr, const LegalizerInfo *LI=nullptr)
bool matchCombineDivRem(MachineInstr &MI, MachineInstr *&OtherMI)
Try to combine G_[SU]DIV and G_[SU]REM into a single G_[SU]DIVREM when their source operands are iden...
bool matchCombineTruncOfExt(MachineInstr &MI, std::pair< Register, unsigned > &MatchInfo)
Transform trunc ([asz]ext x) to x or ([asz]ext x) or (trunc x).
bool isPredecessor(const MachineInstr &DefMI, const MachineInstr &UseMI)
Returns true if DefMI precedes UseMI or they are the same instruction.
bool matchExtractVecEltBuildVec(MachineInstr &MI, Register &Reg)
bool matchUMulHToLShr(MachineInstr &MI)
bool dominates(const MachineInstr &DefMI, const MachineInstr &UseMI)
Returns true if DefMI dominates UseMI.
MachineInstr * buildUDivUsingMul(MachineInstr &MI)
Given an G_UDIV MI expressing a divide by constant, return an expression that implements it by multip...
bool matchCombineZextTrunc(MachineInstr &MI, Register &Reg)
Transform zext(trunc(x)) to x.
void applyCombineShlOfExtend(MachineInstr &MI, const RegisterImmPair &MatchData)
bool canCombineFMadOrFMA(MachineInstr &MI, bool &AllowFusionGlobally, bool &HasFMAD, bool &Aggressive, bool CanReassociate=false)
void applyCombineUnmergeWithDeadLanesToTrunc(MachineInstr &MI)
void applyShuffleToExtract(MachineInstr &MI)
MachineDominatorTree * MDT
bool matchSDivByConst(MachineInstr &MI)
void applySextInRegOfLoad(MachineInstr &MI, std::tuple< Register, unsigned > &MatchInfo)
bool matchCombineUnmergeMergeToPlainValues(MachineInstr &MI, SmallVectorImpl< Register > &Operands)
Transform <ty,...> G_UNMERGE(G_MERGE ty X, Y, Z) -> ty X, Y, Z.
void applyExtractAllEltsFromBuildVector(MachineInstr &MI, SmallVectorImpl< std::pair< Register, MachineInstr * > > &MatchInfo)
bool matchCombineTruncOfShift(MachineInstr &MI, std::pair< MachineInstr *, LLT > &MatchInfo)
Transform trunc (shl x, K) to shl (trunc x), K if K < VT.getScalarSizeInBits().
const RegisterBankInfo * RBI
bool matchCommuteShift(MachineInstr &MI, BuildFnTy &MatchInfo)
bool matchCombineFAddFpExtFMulToFMadOrFMA(MachineInstr &MI, BuildFnTy &MatchInfo)
Transform (fadd (fpext (fmul x, y)), z) -> (fma (fpext x), (fpext y), z) (fadd (fpext (fmul x,...
bool matchReassocConstantInnerLHS(GPtrAdd &MI, MachineInstr *LHS, MachineInstr *RHS, BuildFnTy &MatchInfo)
void applyExtendThroughPhis(MachineInstr &MI, MachineInstr *&ExtMI)
const TargetRegisterInfo * TRI
bool tryCombineShuffleVector(MachineInstr &MI)
Try to combine G_SHUFFLE_VECTOR into G_CONCAT_VECTORS.
bool matchCombineI2PToP2I(MachineInstr &MI, Register &Reg)
Transform IntToPtr(PtrToInt(x)) to x if cast is in the same address space.
bool matchICmpToLHSKnownBits(MachineInstr &MI, BuildFnTy &MatchInfo)
GISelChangeObserver & Observer
bool matchCombineExtOfExt(MachineInstr &MI, std::tuple< Register, unsigned > &MatchInfo)
Transform [asz]ext([asz]ext(x)) to [asz]ext x.
bool matchOverlappingAnd(MachineInstr &MI, BuildFnTy &MatchInfo)
Fold and(and(x, C1), C2) -> C1&C2 ? and(x, C1&C2) : 0.
bool matchSextInRegOfLoad(MachineInstr &MI, std::tuple< Register, unsigned > &MatchInfo)
Match sext_inreg(load p), imm -> sextload p.
bool matchCombineInsertVecElts(MachineInstr &MI, SmallVectorImpl< Register > &MatchInfo)
bool matchCombineAddP2IToPtrAdd(MachineInstr &MI, std::pair< Register, bool > &PtrRegAndCommute)
Transform G_ADD (G_PTRTOINT x), y -> G_PTRTOINT (G_PTR_ADD x, y) Transform G_ADD y,...
bool matchOr(MachineInstr &MI, BuildFnTy &MatchInfo)
Combine ors.
void applyFunnelShiftToRotate(MachineInstr &MI)
void applyCombineUnmergeMergeToPlainValues(MachineInstr &MI, SmallVectorImpl< Register > &Operands)
bool matchOptBrCondByInvertingCond(MachineInstr &MI, MachineInstr *&BrCond)
If a brcond's true block is not the fallthrough, make it so by inverting the condition and swapping o...
bool matchAddEToAddO(MachineInstr &MI, BuildFnTy &MatchInfo)
Match: (G_*ADDE x, y, 0) -> (G_*ADDO x, y) (G_*SUBE x, y, 0) -> (G_*SUBO x, y)
bool matchAddOverflow(MachineInstr &MI, BuildFnTy &MatchInfo)
Combine addos.
void applyCombineP2IToI2P(MachineInstr &MI, Register &Reg)
Transform PtrToInt(IntToPtr(x)) to x.
bool matchCombineShiftToUnmerge(MachineInstr &MI, unsigned TargetShiftSize, unsigned &ShiftVal)
Reduce a shift by a constant to an unmerge and a shift on a half sized type.
bool matchCommuteConstantToRHS(MachineInstr &MI)
Match constant LHS ops that should be commuted.
void applyPtrAddImmedChain(MachineInstr &MI, PtrAddChain &MatchInfo)
void applyCombineDivRem(MachineInstr &MI, MachineInstr *&OtherMI)
void applyFsubToFneg(MachineInstr &MI, Register &MatchInfo)
void applyBuildInstructionSteps(MachineInstr &MI, InstructionStepsMatchInfo &MatchInfo)
Replace MI with a series of instructions described in MatchInfo.
bool matchCombineFSubFpExtFNegFMulToFMadOrFMA(MachineInstr &MI, BuildFnTy &MatchInfo)
Transform (fsub (fpext (fneg (fmul x, y))), z) -> (fneg (fma (fpext x), (fpext y),...
MachineIRBuilder & Builder
bool matchBitfieldExtractFromShrAnd(MachineInstr &MI, BuildFnTy &MatchInfo)
Match: shr (and x, n), k -> ubfx x, pos, width.
bool matchReassocCommBinOp(MachineInstr &MI, BuildFnTy &MatchInfo)
Reassociate commutative binary operations like G_ADD.
bool matchFoldBinOpIntoSelect(MachineInstr &MI, unsigned &SelectOpNo)
Push a binary operator through a select on constants.
bool matchConstantFoldCastOp(MachineInstr &MI, APInt &MatchInfo)
Do constant folding when opportunities are exposed after MIR building.
bool matchOperandIsZero(MachineInstr &MI, unsigned OpIdx)
Check if operand OpIdx is zero.
bool matchOrShiftToFunnelShift(MachineInstr &MI, BuildFnTy &MatchInfo)
bool matchHoistLogicOpWithSameOpcodeHands(MachineInstr &MI, InstructionStepsMatchInfo &MatchInfo)
Match (logic_op (op x...), (op y...)) -> (op (logic_op x, y))
void applyAshShlToSextInreg(MachineInstr &MI, std::tuple< Register, int64_t > &MatchInfo)
void applySextTruncSextLoad(MachineInstr &MI)
bool matchCombineIndexedLoadStore(MachineInstr &MI, IndexedLoadStoreMatchInfo &MatchInfo)
bool matchCommuteFPConstantToRHS(MachineInstr &MI)
Match constant LHS FP ops that should be commuted.
ConstantFP - Floating Point Values [float, double].
const APFloat & getValue() const
const APFloat & getValueAPF() const
const APInt & getValue() const
Return the constant as an APInt value reference.
This class represents a range of values.
std::optional< ConstantRange > exactUnionWith(const ConstantRange &CR) const
Union the two ranges and return the result if it can be represented exactly, otherwise return std::nu...
ConstantRange subtract(const APInt &CI) const
Subtract the specified constant from the endpoints of this constant range.
static ConstantRange fromKnownBits(const KnownBits &Known, bool IsSigned)
Initialize a range based on a known bits constraint.
const APInt & getLower() const
Return the lower value for this range.
OverflowResult unsignedAddMayOverflow(const ConstantRange &Other) const
Return whether unsigned add of the two ranges always/never overflows.
bool isWrappedSet() const
Return true if this set wraps around the unsigned domain.
const APInt & getUpper() const
Return the upper value for this range.
static ConstantRange makeExactICmpRegion(CmpInst::Predicate Pred, const APInt &Other)
Produce the exact range such that all values in the returned range satisfy the given predicate with a...
OverflowResult signedAddMayOverflow(const ConstantRange &Other) const
Return whether signed add of the two ranges always/never overflows.
@ NeverOverflows
Never overflows.
@ AlwaysOverflowsHigh
Always overflows in the direction of signed/unsigned max value.
@ AlwaysOverflowsLow
Always overflows in the direction of signed/unsigned min value.
@ MayOverflow
May or may not overflow.
This is an important base class in LLVM.
This class represents an Operation in the Expression.
A parsed version of the target data layout string in and methods for querying it.
ValueT lookup(const_arg_type_t< KeyT > Val) const
lookup - Return the entry for the specified key, or a default constructed value if no such entry exis...
iterator find(const_arg_type_t< KeyT > Val)
std::pair< iterator, bool > try_emplace(KeyT &&Key, Ts &&... Args)
static FixedVectorType * get(Type *ElementType, unsigned NumElts)
LLVMContext & getContext() const
getContext - Return a reference to the LLVMContext associated with this function.
Represents overflowing add operations.
Represents an integer addition.
Represents a logical and.
CmpInst::Predicate getCond() const
Register getLHSReg() const
Register getRHSReg() const
Represents any generic load, including sign/zero extending variants.
Register getDstReg() const
Get the definition register of the loaded value.
Register getLHSReg() const
Register getRHSReg() const
Represents a G_BUILD_VECTOR.
Abstract class that contains various methods for clients to notify about changes.
virtual void changingInstr(MachineInstr &MI)=0
This instruction is about to be mutated in some way.
void finishedChangingAllUsesOfReg()
All instructions reported as changing by changingAllUsesOfReg() have finished being changed.
virtual void changedInstr(MachineInstr &MI)=0
This instruction was mutated in some way.
virtual void erasingInstr(MachineInstr &MI)=0
An instruction is about to be erased.
void changingAllUsesOfReg(const MachineRegisterInfo &MRI, Register Reg)
All the instructions using the given register are being changed.
unsigned computeNumSignBits(Register R, const APInt &DemandedElts, unsigned Depth=0)
KnownBits getKnownBits(Register R)
APInt getKnownZeroes(Register R)
Simple wrapper observer that takes several observers, and calls each one for each event.
Represents a G_IMPLICIT_DEF.
Represents any type of generic load or store.
Register getPointerReg() const
Get the source register of the pointer value.
Represents a logical binary operation.
MachineMemOperand & getMMO() const
Get the MachineMemOperand on this instruction.
bool isAtomic() const
Returns true if the attached MachineMemOperand has the atomic flag set.
LocationSize getMemSizeInBits() const
Returns the size in bits of the memory access.
bool isSimple() const
Returns true if the memory operation is neither atomic or volatile.
Register getSourceReg(unsigned I) const
Returns the I'th source register.
unsigned getNumSources() const
Returns the number of source registers.
Represents a G_MERGE_VALUES.
Register getCondReg() const
Register getReg(unsigned Idx) const
Access the Idx'th operand as a register and return it.
static IntegerType * get(LLVMContext &C, unsigned NumBits)
This static method is the primary way of constructing an IntegerType.
constexpr bool isScalableVector() const
Returns true if the LLT is a scalable vector.
constexpr unsigned getScalarSizeInBits() const
constexpr bool isScalar() const
static constexpr LLT scalar(unsigned SizeInBits)
Get a low-level scalar or aggregate "bag of bits".
constexpr bool isValid() const
constexpr uint16_t getNumElements() const
Returns the number of elements in a vector LLT.
constexpr bool isVector() const
constexpr bool isByteSized() const
constexpr TypeSize getSizeInBits() const
Returns the total size of the type. Must only be called on sized types.
constexpr bool isPointer() const
constexpr LLT getElementType() const
Returns the vector's element type. Only valid for vector types.
constexpr LLT changeElementSize(unsigned NewEltSize) const
If this type is a vector, return a vector with the same number of elements but the new element size.
constexpr unsigned getAddressSpace() const
constexpr bool isFixedVector() const
Returns true if the LLT is a fixed vector.
constexpr LLT getScalarType() const
constexpr TypeSize getSizeInBytes() const
Returns the total size of the type in bytes, i.e.
This is an important class for using LLVM in a threaded context.
@ Legalized
Instruction has been legalized and the MachineFunction changed.
LegalizeResult lowerMemCpyFamily(MachineInstr &MI, unsigned MaxLen=0)
Register getVectorElementPointer(Register VecPtr, LLT VecTy, Register Index)
Get a pointer to vector element Index located in memory for a vector of type VecTy starting at a base...
bool isLegalOrCustom(const LegalityQuery &Query) const
LegalizeActionStep getAction(const LegalityQuery &Query) const
Determine what action should be taken to legalize the described instruction.
TypeSize getValue() const
const MCInstrDesc & get(unsigned Opcode) const
Return the machine instruction descriptor that corresponds to the specified instruction opcode.
iterator getFirstNonPHI()
Returns a pointer to the first instruction in this block that is not a PHINode instruction.
bool isLayoutSuccessor(const MachineBasicBlock *MBB) const
Return true if the specified MBB will be emitted immediately after this block, such that if this bloc...
const MachineFunction * getParent() const
Return the MachineFunction containing this basic block.
DominatorTree Class - Concrete subclass of DominatorTreeBase that is used to compute a normal dominat...
bool dominates(const MachineDomTreeNode *A, const MachineDomTreeNode *B) const
const TargetSubtargetInfo & getSubtarget() const
getSubtarget - Return the subtarget for which this machine code is being compiled.
MachineMemOperand * getMachineMemOperand(MachinePointerInfo PtrInfo, MachineMemOperand::Flags f, LLT MemTy, Align base_alignment, const AAMDNodes &AAInfo=AAMDNodes(), const MDNode *Ranges=nullptr, SyncScope::ID SSID=SyncScope::System, AtomicOrdering Ordering=AtomicOrdering::NotAtomic, AtomicOrdering FailureOrdering=AtomicOrdering::NotAtomic)
getMachineMemOperand - Allocate a new MachineMemOperand.
MachineRegisterInfo & getRegInfo()
getRegInfo - Return information about the registers currently in use.
const DataLayout & getDataLayout() const
Return the DataLayout attached to the Module associated to this MF.
Function & getFunction()
Return the LLVM function that this machine code represents.
Helper class to build MachineInstr.
MachineInstrBuilder insertInstr(MachineInstrBuilder MIB)
Insert an existing instruction at the insertion point.
void setInsertPt(MachineBasicBlock &MBB, MachineBasicBlock::iterator II)
Set the insertion point before the specified position.
LLVMContext & getContext() const
MachineInstrBuilder buildAdd(const DstOp &Dst, const SrcOp &Src0, const SrcOp &Src1, std::optional< unsigned > Flags=std::nullopt)
Build and insert Res = G_ADD Op0, Op1.
MachineInstrBuilder buildUndef(const DstOp &Res)
Build and insert Res = IMPLICIT_DEF.
MachineInstrBuilder buildNot(const DstOp &Dst, const SrcOp &Src0)
Build and insert a bitwise not, NegOne = G_CONSTANT -1 Res = G_OR Op0, NegOne.
MachineInstrBuilder buildAShr(const DstOp &Dst, const SrcOp &Src0, const SrcOp &Src1, std::optional< unsigned > Flags=std::nullopt)
MachineInstrBuilder buildUnmerge(ArrayRef< LLT > Res, const SrcOp &Op)
Build and insert Res0, ... = G_UNMERGE_VALUES Op.
MachineInstrBuilder buildSelect(const DstOp &Res, const SrcOp &Tst, const SrcOp &Op0, const SrcOp &Op1, std::optional< unsigned > Flags=std::nullopt)
Build and insert a Res = G_SELECT Tst, Op0, Op1.
MachineInstrBuilder buildAnd(const DstOp &Dst, const SrcOp &Src0, const SrcOp &Src1)
Build and insert Res = G_AND Op0, Op1.
MachineInstrBuilder buildCast(const DstOp &Dst, const SrcOp &Src)
Build and insert an appropriate cast between two registers of equal size.
const TargetInstrInfo & getTII()
MachineInstrBuilder buildURem(const DstOp &Dst, const SrcOp &Src0, const SrcOp &Src1, std::optional< unsigned > Flags=std::nullopt)
Build and insert Res = G_UREM Op0, Op1.
MachineInstrBuilder buildLShr(const DstOp &Dst, const SrcOp &Src0, const SrcOp &Src1, std::optional< unsigned > Flags=std::nullopt)
MachineInstrBuilder buildSub(const DstOp &Dst, const SrcOp &Src0, const SrcOp &Src1, std::optional< unsigned > Flags=std::nullopt)
Build and insert Res = G_SUB Op0, Op1.
void setInstr(MachineInstr &MI)
Set the insertion point to before MI.
MachineInstrBuilder buildIntToPtr(const DstOp &Dst, const SrcOp &Src)
Build and insert a G_INTTOPTR instruction.
MachineInstrBuilder buildBuildVector(const DstOp &Res, ArrayRef< Register > Ops)
Build and insert Res = G_BUILD_VECTOR Op0, ...
MachineInstrBuilder buildCTLZ(const DstOp &Dst, const SrcOp &Src0)
Build and insert Res = G_CTLZ Op0, Src0.
MachineInstrBuilder buildMergeLikeInstr(const DstOp &Res, ArrayRef< Register > Ops)
Build and insert Res = G_MERGE_VALUES Op0, ... or Res = G_BUILD_VECTOR Op0, ... or Res = G_CONCAT_VEC...
MachineInstrBuilder buildPtrAdd(const DstOp &Res, const SrcOp &Op0, const SrcOp &Op1, std::optional< unsigned > Flags=std::nullopt)
Build and insert Res = G_PTR_ADD Op0, Op1.
MachineInstrBuilder buildZExtOrTrunc(const DstOp &Res, const SrcOp &Op)
Build and insert Res = G_ZEXT Op, Res = G_TRUNC Op, or Res = COPY Op depending on the differing sizes...
MachineInstrBuilder buildExtractVectorElementConstant(const DstOp &Res, const SrcOp &Val, const int Idx)
Build and insert Res = G_EXTRACT_VECTOR_ELT Val, Idx.
virtual MachineInstrBuilder buildFConstant(const DstOp &Res, const ConstantFP &Val)
Build and insert Res = G_FCONSTANT Val.
MachineInstrBuilder buildShl(const DstOp &Dst, const SrcOp &Src0, const SrcOp &Src1, std::optional< unsigned > Flags=std::nullopt)
MachineInstrBuilder buildInstr(unsigned Opcode)
Build and insert <empty> = Opcode <empty>.
MachineInstrBuilder buildZExt(const DstOp &Res, const SrcOp &Op)
Build and insert Res = G_ZEXT Op.
MachineFunction & getMF()
Getter for the function we currently build.
void setInstrAndDebugLoc(MachineInstr &MI)
Set the insertion point to before MI, and set the debug loc to MI's loc.
MachineInstrBuilder buildExtOrTrunc(unsigned ExtOpc, const DstOp &Res, const SrcOp &Op)
Build and insert Res = ExtOpc, Res = G_TRUNC Op, or Res = COPY Op depending on the differing sizes of...
void setDebugLoc(const DebugLoc &DL)
Set the debug location to DL for all the next build instructions.
MachineInstrBuilder buildFNeg(const DstOp &Dst, const SrcOp &Src0, std::optional< unsigned > Flags=std::nullopt)
Build and insert Res = G_FNEG Op0.
MachineInstrBuilder buildTrunc(const DstOp &Res, const SrcOp &Op)
Build and insert Res = G_TRUNC Op.
MachineRegisterInfo * getMRI()
Getter for MRI.
MachineInstrBuilder buildInstrNoInsert(unsigned Opcode)
Build but don't insert <empty> = Opcode <empty>.
MachineInstrBuilder buildCopy(const DstOp &Res, const SrcOp &Op)
Build and insert Res = COPY Op.
MachineInstrBuilder buildLoadInstr(unsigned Opcode, const DstOp &Res, const SrcOp &Addr, MachineMemOperand &MMO)
Build and insert Res = <opcode> Addr, MMO.
MachineInstrBuilder buildXor(const DstOp &Dst, const SrcOp &Src0, const SrcOp &Src1)
Build and insert Res = G_XOR Op0, Op1.
virtual MachineInstrBuilder buildConstant(const DstOp &Res, const ConstantInt &Val)
Build and insert Res = G_CONSTANT Val.
MachineInstrBuilder buildPtrToInt(const DstOp &Dst, const SrcOp &Src)
Build and insert a G_PTRTOINT instruction.
MachineInstrBuilder buildFCanonicalize(const DstOp &Dst, const SrcOp &Src0, std::optional< unsigned > Flags=std::nullopt)
Build and insert Dst = G_FCANONICALIZE Src0.
MachineInstrBuilder buildSExtInReg(const DstOp &Res, const SrcOp &Op, int64_t ImmOp)
Build and insert Res = G_SEXT_INREG Op, ImmOp.
Register getReg(unsigned Idx) const
Get the register for the operand index.
const MachineInstrBuilder & addImm(int64_t Val) const
Add a new immediate operand.
const MachineInstrBuilder & addMBB(MachineBasicBlock *MBB, unsigned TargetFlags=0) const
const MachineInstrBuilder & addUse(Register RegNo, unsigned Flags=0, unsigned SubReg=0) const
Add a virtual register use operand.
const MachineInstrBuilder & addDef(Register RegNo, unsigned Flags=0, unsigned SubReg=0) const
Add a virtual register definition operand.
Representation of each machine instruction.
unsigned getOpcode() const
Returns the opcode of this MachineInstr.
bool mayLoadOrStore(QueryType Type=AnyInBundle) const
Return true if this instruction could possibly read or modify memory.
const MachineBasicBlock * getParent() const
int findRegisterDefOperandIdx(Register Reg, bool isDead=false, bool Overlap=false, const TargetRegisterInfo *TRI=nullptr) const
Returns the operand index that is a def of the specified register or -1 if it is not found.
bool isDereferenceableInvariantLoad() const
Return true if this load instruction never traps and points to a memory location whose value doesn't ...
bool getFlag(MIFlag Flag) const
Return whether an MI flag is set.
void cloneMemRefs(MachineFunction &MF, const MachineInstr &MI)
Clone another MachineInstr's memory reference descriptor list and replace ours with it.
unsigned getNumOperands() const
Retuns the total number of operands.
void setDesc(const MCInstrDesc &TID)
Replace the instruction descriptor (thus opcode) of the current instruction with a new one.
void eraseFromParent()
Unlink 'this' from the containing basic block and delete it.
const MachineOperand & getOperand(unsigned i) const
uint32_t getFlags() const
Return the MI flags bitvector.
A description of a memory reference used in the backend.
LLT getMemoryType() const
Return the memory type of the memory reference.
unsigned getAddrSpace() const
const MachinePointerInfo & getPointerInfo() const
Align getAlign() const
Return the minimum known alignment in bytes of the actual memory reference.
MachineOperand class - Representation of each machine instruction operand.
const ConstantInt * getCImm() const
bool isReg() const
isReg - Tests if this is a MO_Register operand.
MachineBasicBlock * getMBB() const
void setReg(Register Reg)
Change the register this operand corresponds to.
MachineInstr * getParent()
getParent - Return the instruction that this operand belongs to.
void setMBB(MachineBasicBlock *MBB)
void setPredicate(unsigned Predicate)
Register getReg() const
getReg - Returns the register number.
const ConstantFP * getFPImm() const
unsigned getPredicate() const
MachineRegisterInfo - Keep track of information for virtual and physical registers,...
bool hasOneNonDBGUse(Register RegNo) const
hasOneNonDBGUse - Return true if there is exactly one non-Debug use of the specified register.
MachineInstr * getVRegDef(Register Reg) const
getVRegDef - Return the machine instr that defines the specified virtual register or null if none is ...
use_instr_iterator use_instr_begin(Register RegNo) const
bool use_nodbg_empty(Register RegNo) const
use_nodbg_empty - Return true if there are no non-Debug instructions using the specified register.
const RegClassOrRegBank & getRegClassOrRegBank(Register Reg) const
Return the register bank or register class of Reg.
void setRegClassOrRegBank(Register Reg, const RegClassOrRegBank &RCOrRB)
LLT getType(Register Reg) const
Get the low-level type of Reg or LLT{} if Reg is not a generic (target independent) virtual register.
bool hasOneUse(Register RegNo) const
hasOneUse - Return true if there is exactly one instruction using the specified register.
use_instr_nodbg_iterator use_instr_nodbg_begin(Register RegNo) const
void setRegBank(Register Reg, const RegisterBank &RegBank)
Set the register bank to RegBank for Reg.
iterator_range< use_instr_nodbg_iterator > use_nodbg_instructions(Register Reg) const
Register createGenericVirtualRegister(LLT Ty, StringRef Name="")
Create and return a new generic virtual register with low-level type Ty.
iterator_range< use_instr_iterator > use_instructions(Register Reg) const
Register cloneVirtualRegister(Register VReg, StringRef Name="")
Create and return a new virtual register in the function with the same attributes as the given regist...
bool constrainRegAttrs(Register Reg, Register ConstrainingReg, unsigned MinNumRegs=0)
Constrain the register class or the register bank of the virtual register Reg (and low-level type) to...
iterator_range< use_iterator > use_operands(Register Reg) const
void replaceRegWith(Register FromReg, Register ToReg)
replaceRegWith - Replace all instances of FromReg with ToReg in the machine function.
MachineInstr * getUniqueVRegDef(Register Reg) const
getUniqueVRegDef - Return the unique machine instr that defines the specified virtual register or nul...
const RegisterBank & getRegBank(unsigned ID)
Get the register bank identified by ID.
This class implements the register bank concept.
Wrapper class representing virtual and physical registers.
constexpr bool isValid() const
size_type size() const
Determine the number of elements in the SetVector.
size_type count(const key_type &key) const
Count the number of elements of a given key in the SetVector.
bool insert(const value_type &X)
Insert a new element into the SetVector.
This is a 'bitvector' (really, a variable-sized bit array), optimized for the case when the array is ...
bool all() const
Returns true if all bits are set.
std::pair< iterator, bool > insert(PtrType Ptr)
Inserts Ptr if and only if there is no element in the container equal to Ptr.
SmallPtrSet - This class implements a set which is optimized for holding SmallSize or less elements.
A SetVector that performs no allocations if smaller than a certain size.
SmallSet - This maintains a set of unique values, optimizing for the case when the set is small (less...
std::pair< const_iterator, bool > insert(const T &V)
insert - Insert an element into the set if it isn't already there.
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
reference emplace_back(ArgTypes &&... Args)
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
virtual bool isExtendLikelyToBeFolded(MachineInstr &ExtMI, MachineRegisterInfo &MRI) const
Given the generic extension instruction ExtMI, returns true if this extension is a likely candidate f...
virtual bool produceSameValue(const MachineInstr &MI0, const MachineInstr &MI1, const MachineRegisterInfo *MRI=nullptr) const
Return true if two machine instructions would produce identical values.
virtual LLVM_READONLY LLT getPreferredShiftAmountTy(LLT ShiftValueTy) const
Return the preferred type to use for a shift opcode, given the shifted amount type is ShiftValueTy.
virtual bool isLegalAddressingMode(const DataLayout &DL, const AddrMode &AM, Type *Ty, unsigned AddrSpace, Instruction *I=nullptr) const
Return true if the addressing mode represented by AM is legal for this target, for a load/store of th...
This class defines information used to lower LLVM code to legal SelectionDAG operators that the targe...
virtual bool isReassocProfitable(SelectionDAG &DAG, SDValue N0, SDValue N1) const
virtual const TargetLowering * getTargetLowering() const
The instances of the Type class are immutable: once they are created, they are never changed.
A Use represents the edge between a Value definition and its users.
LLVM Value Representation.
Value(Type *Ty, unsigned scid)
self_iterator getIterator()
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
@ Fast
Attempts to make calls as fast as possible (e.g.
@ C
The default llvm calling convention, compatible with C.
@ Legal
The operation is expected to be selectable directly by the target, and no transformation is necessary...
operand_type_match m_Reg()
SpecificConstantOrSplatMatch m_SpecificICstOrSplat(int64_t RequestedValue)
Matches a RequestedValue constant or a constant splat of RequestedValue.
BinaryOp_match< LHS, RHS, TargetOpcode::G_BUILD_VECTOR, false > m_GBuildVector(const LHS &L, const RHS &R)
GCstAndRegMatch m_GCst(std::optional< ValueAndVReg > &ValReg)
SpecificConstantMatch m_SpecificICst(int64_t RequestedValue)
Matches a constant equal to RequestedValue.
operand_type_match m_Pred()
UnaryOp_match< SrcTy, TargetOpcode::G_ZEXT > m_GZExt(const SrcTy &Src)
BinaryOp_match< LHS, RHS, TargetOpcode::G_XOR, true > m_GXor(const LHS &L, const RHS &R)
UnaryOp_match< SrcTy, TargetOpcode::G_SEXT > m_GSExt(const SrcTy &Src)
UnaryOp_match< SrcTy, TargetOpcode::G_FPEXT > m_GFPExt(const SrcTy &Src)
ConstantMatch< APInt > m_ICst(APInt &Cst)
UnaryOp_match< SrcTy, TargetOpcode::G_INTTOPTR > m_GIntToPtr(const SrcTy &Src)
BinaryOp_match< LHS, RHS, TargetOpcode::G_ADD, true > m_GAdd(const LHS &L, const RHS &R)
BinaryOp_match< LHS, RHS, TargetOpcode::G_OR, true > m_GOr(const LHS &L, const RHS &R)
ICstOrSplatMatch< APInt > m_ICstOrSplat(APInt &Cst)
ImplicitDefMatch m_GImplicitDef()
OneNonDBGUse_match< SubPat > m_OneNonDBGUse(const SubPat &SP)
CheckType m_SpecificType(LLT Ty)
BinaryOp_match< LHS, RHS, TargetOpcode::G_FADD, true > m_GFAdd(const LHS &L, const RHS &R)
UnaryOp_match< SrcTy, TargetOpcode::G_PTRTOINT > m_GPtrToInt(const SrcTy &Src)
BinaryOp_match< LHS, RHS, TargetOpcode::G_FSUB, false > m_GFSub(const LHS &L, const RHS &R)
BinaryOp_match< LHS, RHS, TargetOpcode::G_SUB > m_GSub(const LHS &L, const RHS &R)
BinaryOp_match< LHS, RHS, TargetOpcode::G_ASHR, false > m_GAShr(const LHS &L, const RHS &R)
bool mi_match(Reg R, const MachineRegisterInfo &MRI, Pattern &&P)
BinaryOp_match< LHS, RHS, TargetOpcode::G_PTR_ADD, false > m_GPtrAdd(const LHS &L, const RHS &R)
BinaryOp_match< LHS, RHS, TargetOpcode::G_SHL, false > m_GShl(const LHS &L, const RHS &R)
Or< Preds... > m_any_of(Preds &&... preds)
BinaryOp_match< LHS, RHS, TargetOpcode::G_AND, true > m_GAnd(const LHS &L, const RHS &R)
UnaryOp_match< SrcTy, TargetOpcode::G_BITCAST > m_GBitcast(const SrcTy &Src)
BinaryOp_match< LHS, RHS, TargetOpcode::G_BUILD_VECTOR_TRUNC, false > m_GBuildVectorTrunc(const LHS &L, const RHS &R)
bind_ty< MachineInstr * > m_MInstr(MachineInstr *&MI)
UnaryOp_match< SrcTy, TargetOpcode::G_FNEG > m_GFNeg(const SrcTy &Src)
CompareOp_match< Pred, LHS, RHS, TargetOpcode::G_ICMP, true > m_c_GICmp(const Pred &P, const LHS &L, const RHS &R)
G_ICMP matcher that also matches commuted compares.
TernaryOp_match< Src0Ty, Src1Ty, Src2Ty, TargetOpcode::G_INSERT_VECTOR_ELT > m_GInsertVecElt(const Src0Ty &Src0, const Src1Ty &Src1, const Src2Ty &Src2)
GFCstOrSplatGFCstMatch m_GFCstOrSplat(std::optional< FPValueAndVReg > &FPValReg)
And< Preds... > m_all_of(Preds &&... preds)
BinaryOp_match< LHS, RHS, TargetOpcode::G_LSHR, false > m_GLShr(const LHS &L, const RHS &R)
UnaryOp_match< SrcTy, TargetOpcode::G_ANYEXT > m_GAnyExt(const SrcTy &Src)
UnaryOp_match< SrcTy, TargetOpcode::G_TRUNC > m_GTrunc(const SrcTy &Src)
CompareOp_match< Pred, LHS, RHS, TargetOpcode::G_FCMP > m_GFCmp(const Pred &P, const LHS &L, const RHS &R)
class_match< BinaryOperator > m_BinOp()
Match an arbitrary binary operation and ignore it.
BinaryOp_match< cst_pred_ty< is_zero_int >, ValTy, Instruction::Sub > m_Neg(const ValTy &V)
Matches a 'Neg' as 'sub 0, V'.
initializer< Ty > init(const Ty &Val)
This is an optimization pass for GlobalISel generic memory operations.
auto drop_begin(T &&RangeOrContainer, size_t N=1)
Return a range covering RangeOrContainer with the first N elements excluded.
bool isBuildVectorAllZeros(const MachineInstr &MI, const MachineRegisterInfo &MRI, bool AllowUndef=false)
Return true if the specified instruction is a G_BUILD_VECTOR or G_BUILD_VECTOR_TRUNC where all of the...
bool all_of(R &&range, UnaryPredicate P)
Provide wrappers to std::all_of which take ranges instead of having to pass begin/end explicitly.
MachineInstr * getOpcodeDef(unsigned Opcode, Register Reg, const MachineRegisterInfo &MRI)
See if Reg is defined by an single def instruction that is Opcode.
static double log2(double V)
const ConstantFP * getConstantFPVRegVal(Register VReg, const MachineRegisterInfo &MRI)
EVT getApproximateEVTForLLT(LLT Ty, const DataLayout &DL, LLVMContext &Ctx)
std::optional< APInt > getIConstantVRegVal(Register VReg, const MachineRegisterInfo &MRI)
If VReg is defined by a G_CONSTANT, return the corresponding value.
std::optional< APInt > getIConstantSplatVal(const Register Reg, const MachineRegisterInfo &MRI)
bool isAllOnesOrAllOnesSplat(const MachineInstr &MI, const MachineRegisterInfo &MRI, bool AllowUndefs=false)
Return true if the value is a constant -1 integer or a splatted vector of a constant -1 integer (with...
int countr_one(T Value)
Count the number of ones from the least significant bit to the first zero bit.
const llvm::fltSemantics & getFltSemanticForLLT(LLT Ty)
Get the appropriate floating point arithmetic semantic based on the bit size of the given scalar LLT.
std::optional< APFloat > ConstantFoldFPBinOp(unsigned Opcode, const Register Op1, const Register Op2, const MachineRegisterInfo &MRI)
MVT getMVTForLLT(LLT Ty)
Get a rough equivalent of an MVT for a given LLT.
bool isKnownToBeAPowerOfTwo(const Value *V, const DataLayout &DL, bool OrZero=false, unsigned Depth=0, AssumptionCache *AC=nullptr, const Instruction *CxtI=nullptr, const DominatorTree *DT=nullptr, bool UseInstrInfo=true)
Return true if the given value is known to have exactly one bit set when defined.
std::optional< APInt > isConstantOrConstantSplatVector(MachineInstr &MI, const MachineRegisterInfo &MRI)
Determines if MI defines a constant integer or a splat vector of constant integers.
bool isNullOrNullSplat(const MachineInstr &MI, const MachineRegisterInfo &MRI, bool AllowUndefs=false)
Return true if the value is a constant 0 integer or a splatted vector of a constant 0 integer (with n...
MachineInstr * getDefIgnoringCopies(Register Reg, const MachineRegisterInfo &MRI)
Find the def instruction for Reg, folding away any trivial copies.
bool matchUnaryPredicate(const MachineRegisterInfo &MRI, Register Reg, std::function< bool(const Constant *ConstVal)> Match, bool AllowUndefs=false)
Attempt to match a unary predicate against a scalar/splat constant or every element of a constant G_B...
bool isConstTrueVal(const TargetLowering &TLI, int64_t Val, bool IsVector, bool IsFP)
Returns true if given the TargetLowering's boolean contents information, the value Val contains a tru...
std::function< void(MachineIRBuilder &)> BuildFnTy
std::optional< APInt > ConstantFoldBinOp(unsigned Opcode, const Register Op1, const Register Op2, const MachineRegisterInfo &MRI)
bool any_of(R &&range, UnaryPredicate P)
Provide wrappers to std::any_of which take ranges instead of having to pass begin/end explicitly.
bool isConstantOrConstantVector(const MachineInstr &MI, const MachineRegisterInfo &MRI, bool AllowFP=true, bool AllowOpaqueConstants=true)
Return true if the specified instruction is known to be a constant, or a vector of constants.
constexpr bool isPowerOf2_32(uint32_t Value)
Return true if the argument is a power of two > 0.
bool canReplaceReg(Register DstReg, Register SrcReg, MachineRegisterInfo &MRI)
Check if DstReg can be replaced with SrcReg depending on the register constraints.
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
constexpr bool isMask_64(uint64_t Value)
Return true if the argument is a non-empty sequence of ones starting at the least significant bit wit...
auto instructionsWithoutDebug(IterT It, IterT End, bool SkipPseudoOp=true)
Construct a range iterator which begins at It and moves forwards until End is reached,...
std::optional< FPValueAndVReg > getFConstantSplat(Register VReg, const MachineRegisterInfo &MRI, bool AllowUndef=true)
Returns a floating point scalar constant of a build vector splat if it exists.
std::optional< APInt > ConstantFoldCastOp(unsigned Opcode, LLT DstTy, const Register Op0, const MachineRegisterInfo &MRI)
@ Xor
Bitwise or logical XOR of integers.
DWARFExpression::Operation Op
std::optional< FPValueAndVReg > getFConstantVRegValWithLookThrough(Register VReg, const MachineRegisterInfo &MRI, bool LookThroughInstrs=true)
If VReg is defined by a statically evaluable chain of instructions rooted on a G_FCONSTANT returns it...
constexpr unsigned BitWidth
int64_t getICmpTrueVal(const TargetLowering &TLI, bool IsVector, bool IsFP)
Returns an integer representing true, as defined by the TargetBooleanContents.
std::optional< ValueAndVReg > getIConstantVRegValWithLookThrough(Register VReg, const MachineRegisterInfo &MRI, bool LookThroughInstrs=true)
If VReg is defined by a statically evaluable chain of instructions rooted on a G_CONSTANT returns its...
auto find_if(R &&Range, UnaryPredicate P)
Provide wrappers to std::find_if which take ranges instead of having to pass begin/end explicitly.
std::optional< DefinitionAndSourceRegister > getDefSrcRegIgnoringCopies(Register Reg, const MachineRegisterInfo &MRI)
Find the def instruction for Reg, and underlying value Register folding away any copies.
Align commonAlignment(Align A, uint64_t Offset)
Returns the alignment that satisfies both alignments.
bool isKnownNeverNaN(const Value *V, unsigned Depth, const SimplifyQuery &SQ)
Return true if the floating-point scalar value is not a NaN or if the floating-point vector value has...
Register getSrcRegIgnoringCopies(Register Reg, const MachineRegisterInfo &MRI)
Find the source register for Reg, folding away any trivial copies.
unsigned getFCmpCode(CmpInst::Predicate CC)
Similar to getICmpCode but for FCmpInst.
std::optional< int64_t > getIConstantSplatSExtVal(const Register Reg, const MachineRegisterInfo &MRI)
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
static constexpr roundingMode rmNearestTiesToEven
static const fltSemantics & IEEEdouble() LLVM_READNONE
This struct is a compact representation of a valid (non-zero power of two) alignment.
Simple struct used to hold a Register value and the instruction which defines it.
SmallVector< InstructionBuildSteps, 2 > InstrsToBuild
Describes instructions to be built during a combine.
static std::optional< bool > eq(const KnownBits &LHS, const KnownBits &RHS)
Determine if these known bits always give the same ICMP_EQ result.
static std::optional< bool > ne(const KnownBits &LHS, const KnownBits &RHS)
Determine if these known bits always give the same ICMP_NE result.
static std::optional< bool > sge(const KnownBits &LHS, const KnownBits &RHS)
Determine if these known bits always give the same ICMP_SGE result.
unsigned countMinLeadingZeros() const
Returns the minimum number of leading zero bits.
APInt getMaxValue() const
Return the maximal unsigned value possible given these KnownBits.
static std::optional< bool > ugt(const KnownBits &LHS, const KnownBits &RHS)
Determine if these known bits always give the same ICMP_UGT result.
static std::optional< bool > slt(const KnownBits &LHS, const KnownBits &RHS)
Determine if these known bits always give the same ICMP_SLT result.
static std::optional< bool > ult(const KnownBits &LHS, const KnownBits &RHS)
Determine if these known bits always give the same ICMP_ULT result.
static std::optional< bool > ule(const KnownBits &LHS, const KnownBits &RHS)
Determine if these known bits always give the same ICMP_ULE result.
static std::optional< bool > sle(const KnownBits &LHS, const KnownBits &RHS)
Determine if these known bits always give the same ICMP_SLE result.
static std::optional< bool > sgt(const KnownBits &LHS, const KnownBits &RHS)
Determine if these known bits always give the same ICMP_SGT result.
static std::optional< bool > uge(const KnownBits &LHS, const KnownBits &RHS)
Determine if these known bits always give the same ICMP_UGE result.
The LegalityQuery object bundles together all the information that's needed to decide whether a given...
LegalizeAction Action
The action to take or the final answer.
This class contains a discriminated union of information about pointers in memory operands,...
unsigned getAddrSpace() const
Return the LLVM IR address space number that this pointer points into.
MachinePointerInfo getWithOffset(int64_t O) const
const RegisterBank * Bank
Register LogicNonShiftReg
This represents an addressing mode of: BaseGV + BaseOffs + BaseReg + Scale*ScaleReg If BaseGV is null...
Magic data for optimising unsigned division by a constant.
unsigned PreShift
pre-shift amount
static UnsignedDivisionByConstantInfo get(const APInt &D, unsigned LeadingZeros=0, bool AllowEvenDivisorOptimization=true)
Calculate the magic numbers required to implement an unsigned integer division by a constant as a seq...
unsigned PostShift
post-shift amount