50#define DEBUG_TYPE "x86-instr-info"
52#define GET_INSTRINFO_CTOR_DTOR
53#include "X86GenInstrInfo.inc"
57 cl::desc(
"Disable fusing of spill code into instructions"),
61 cl::desc(
"Print instructions that the allocator wants to"
62 " fuse, but the X86 backend currently can't"),
66 cl::desc(
"Re-materialize load from stub in PIC mode"),
70 cl::desc(
"Clearance between two register writes "
71 "for inserting XOR to avoid partial "
76 cl::desc(
"How many idle instructions we would like before "
77 "certain undef register reads"),
82void X86InstrInfo::anchor() {}
86 : X86::ADJCALLSTACKDOWN32),
87 (STI.isTarget64BitLP64() ? X86::ADJCALLSTACKUP64
88 : X86::ADJCALLSTACKUP32),
90 (STI.
is64Bit() ? X86::RET64 : X86::RET32)),
91 Subtarget(STI), RI(STI.getTargetTriple()) {
97 unsigned &SubIdx)
const {
98 switch (
MI.getOpcode()) {
100 case X86::MOVSX16rr8:
101 case X86::MOVZX16rr8:
102 case X86::MOVSX32rr8:
103 case X86::MOVZX32rr8:
104 case X86::MOVSX64rr8:
105 if (!Subtarget.is64Bit())
110 case X86::MOVSX32rr16:
111 case X86::MOVZX32rr16:
112 case X86::MOVSX64rr16:
113 case X86::MOVSX64rr32: {
114 if (
MI.getOperand(0).getSubReg() ||
MI.getOperand(1).getSubReg())
117 SrcReg =
MI.getOperand(1).getReg();
118 DstReg =
MI.getOperand(0).getReg();
119 switch (
MI.getOpcode()) {
121 case X86::MOVSX16rr8:
122 case X86::MOVZX16rr8:
123 case X86::MOVSX32rr8:
124 case X86::MOVZX32rr8:
125 case X86::MOVSX64rr8:
126 SubIdx = X86::sub_8bit;
128 case X86::MOVSX32rr16:
129 case X86::MOVZX32rr16:
130 case X86::MOVSX64rr16:
131 SubIdx = X86::sub_16bit;
133 case X86::MOVSX64rr32:
134 SubIdx = X86::sub_32bit;
144 if (
MI.mayLoad() ||
MI.mayStore())
149 if (
MI.isCopyLike() ||
MI.isInsertSubreg())
152 unsigned Opcode =
MI.getOpcode();
163 if (isBSF(Opcode) || isBSR(Opcode) || isLZCNT(Opcode) || isPOPCNT(Opcode) ||
169 if (isBLCFILL(Opcode) || isBLCI(Opcode) || isBLCIC(Opcode) ||
170 isBLCMSK(Opcode) || isBLCS(Opcode) || isBLSFILL(Opcode) ||
171 isBLSI(Opcode) || isBLSIC(Opcode) || isBLSMSK(Opcode) || isBLSR(Opcode) ||
176 if (isBEXTR(Opcode) || isBZHI(Opcode))
179 if (isROL(Opcode) || isROR(Opcode) || isSAR(Opcode) || isSHL(Opcode) ||
180 isSHR(Opcode) || isSHLD(Opcode) || isSHRD(Opcode))
183 if (isADC(Opcode) || isADD(Opcode) || isAND(Opcode) || isOR(Opcode) ||
184 isSBB(Opcode) || isSUB(Opcode) || isXOR(Opcode))
187 if (isADCX(Opcode) || isADOX(Opcode) || isANDN(Opcode))
190 if (isDEC(Opcode) || isINC(Opcode) || isNEG(Opcode))
198 if (isMOVSX(Opcode) || isMOVZX(Opcode) || isMOVSXD(Opcode) || isMOV(Opcode))
201 if (isRORX(Opcode) || isSARX(Opcode) || isSHLX(Opcode) || isSHRX(Opcode))
211 switch (
MI.getOpcode()) {
220 case X86::IMUL16rmi8:
223 case X86::IMUL32rmi8:
226 case X86::IMUL64rmi32:
227 case X86::IMUL64rmi8:
242 case X86::POPCNT16rm:
243 case X86::POPCNT32rm:
244 case X86::POPCNT64rm:
252 case X86::BLCFILL32rm:
253 case X86::BLCFILL64rm:
258 case X86::BLCMSK32rm:
259 case X86::BLCMSK64rm:
262 case X86::BLSFILL32rm:
263 case X86::BLSFILL64rm:
268 case X86::BLSMSK32rm:
269 case X86::BLSMSK64rm:
279 case X86::BEXTRI32mi:
280 case X86::BEXTRI64mi:
337 case X86::CVTTSD2SI64rm:
338 case X86::VCVTTSD2SI64rm:
339 case X86::VCVTTSD2SI64Zrm:
340 case X86::CVTTSD2SIrm:
341 case X86::VCVTTSD2SIrm:
342 case X86::VCVTTSD2SIZrm:
343 case X86::CVTTSS2SI64rm:
344 case X86::VCVTTSS2SI64rm:
345 case X86::VCVTTSS2SI64Zrm:
346 case X86::CVTTSS2SIrm:
347 case X86::VCVTTSS2SIrm:
348 case X86::VCVTTSS2SIZrm:
349 case X86::CVTSI2SDrm:
350 case X86::VCVTSI2SDrm:
351 case X86::VCVTSI2SDZrm:
352 case X86::CVTSI2SSrm:
353 case X86::VCVTSI2SSrm:
354 case X86::VCVTSI2SSZrm:
355 case X86::CVTSI642SDrm:
356 case X86::VCVTSI642SDrm:
357 case X86::VCVTSI642SDZrm:
358 case X86::CVTSI642SSrm:
359 case X86::VCVTSI642SSrm:
360 case X86::VCVTSI642SSZrm:
361 case X86::CVTSS2SDrm:
362 case X86::VCVTSS2SDrm:
363 case X86::VCVTSS2SDZrm:
364 case X86::CVTSD2SSrm:
365 case X86::VCVTSD2SSrm:
366 case X86::VCVTSD2SSZrm:
368 case X86::VCVTTSD2USI64Zrm:
369 case X86::VCVTTSD2USIZrm:
370 case X86::VCVTTSS2USI64Zrm:
371 case X86::VCVTTSS2USIZrm:
372 case X86::VCVTUSI2SDZrm:
373 case X86::VCVTUSI642SDZrm:
374 case X86::VCVTUSI2SSZrm:
375 case X86::VCVTUSI642SSZrm:
379 case X86::MOV8rm_NOREX:
383 case X86::MOVSX16rm8:
384 case X86::MOVSX32rm16:
385 case X86::MOVSX32rm8:
386 case X86::MOVSX32rm8_NOREX:
387 case X86::MOVSX64rm16:
388 case X86::MOVSX64rm32:
389 case X86::MOVSX64rm8:
390 case X86::MOVZX16rm8:
391 case X86::MOVZX32rm16:
392 case X86::MOVZX32rm8:
393 case X86::MOVZX32rm8_NOREX:
394 case X86::MOVZX64rm16:
395 case X86::MOVZX64rm8:
404 if (isFrameInstr(
MI)) {
407 if (!isFrameSetup(
MI))
419 if (
I->getOpcode() == getCallFrameDestroyOpcode() ||
426 if (
I->getOpcode() != getCallFrameDestroyOpcode())
429 return -(
I->getOperand(1).getImm());
434 switch (
MI.getOpcode()) {
454bool X86InstrInfo::isFrameOperand(
const MachineInstr &
MI,
unsigned int Op,
455 int &FrameIndex)
const {
480 case X86::VMOVSHZrm_alt:
485 case X86::MOVSSrm_alt:
487 case X86::VMOVSSrm_alt:
489 case X86::VMOVSSZrm_alt:
496 case X86::MOVSDrm_alt:
498 case X86::VMOVSDrm_alt:
500 case X86::VMOVSDZrm_alt:
501 case X86::MMX_MOVD64rm:
502 case X86::MMX_MOVQ64rm:
518 case X86::VMOVAPSZ128rm:
519 case X86::VMOVUPSZ128rm:
520 case X86::VMOVAPSZ128rm_NOVLX:
521 case X86::VMOVUPSZ128rm_NOVLX:
522 case X86::VMOVAPDZ128rm:
523 case X86::VMOVUPDZ128rm:
524 case X86::VMOVDQU8Z128rm:
525 case X86::VMOVDQU16Z128rm:
526 case X86::VMOVDQA32Z128rm:
527 case X86::VMOVDQU32Z128rm:
528 case X86::VMOVDQA64Z128rm:
529 case X86::VMOVDQU64Z128rm:
532 case X86::VMOVAPSYrm:
533 case X86::VMOVUPSYrm:
534 case X86::VMOVAPDYrm:
535 case X86::VMOVUPDYrm:
536 case X86::VMOVDQAYrm:
537 case X86::VMOVDQUYrm:
538 case X86::VMOVAPSZ256rm:
539 case X86::VMOVUPSZ256rm:
540 case X86::VMOVAPSZ256rm_NOVLX:
541 case X86::VMOVUPSZ256rm_NOVLX:
542 case X86::VMOVAPDZ256rm:
543 case X86::VMOVUPDZ256rm:
544 case X86::VMOVDQU8Z256rm:
545 case X86::VMOVDQU16Z256rm:
546 case X86::VMOVDQA32Z256rm:
547 case X86::VMOVDQU32Z256rm:
548 case X86::VMOVDQA64Z256rm:
549 case X86::VMOVDQU64Z256rm:
552 case X86::VMOVAPSZrm:
553 case X86::VMOVUPSZrm:
554 case X86::VMOVAPDZrm:
555 case X86::VMOVUPDZrm:
556 case X86::VMOVDQU8Zrm:
557 case X86::VMOVDQU16Zrm:
558 case X86::VMOVDQA32Zrm:
559 case X86::VMOVDQU32Zrm:
560 case X86::VMOVDQA64Zrm:
561 case X86::VMOVDQU64Zrm:
592 case X86::MMX_MOVD64mr:
593 case X86::MMX_MOVQ64mr:
594 case X86::MMX_MOVNTQmr:
610 case X86::VMOVUPSZ128mr:
611 case X86::VMOVAPSZ128mr:
612 case X86::VMOVUPSZ128mr_NOVLX:
613 case X86::VMOVAPSZ128mr_NOVLX:
614 case X86::VMOVUPDZ128mr:
615 case X86::VMOVAPDZ128mr:
616 case X86::VMOVDQA32Z128mr:
617 case X86::VMOVDQU32Z128mr:
618 case X86::VMOVDQA64Z128mr:
619 case X86::VMOVDQU64Z128mr:
620 case X86::VMOVDQU8Z128mr:
621 case X86::VMOVDQU16Z128mr:
624 case X86::VMOVUPSYmr:
625 case X86::VMOVAPSYmr:
626 case X86::VMOVUPDYmr:
627 case X86::VMOVAPDYmr:
628 case X86::VMOVDQUYmr:
629 case X86::VMOVDQAYmr:
630 case X86::VMOVUPSZ256mr:
631 case X86::VMOVAPSZ256mr:
632 case X86::VMOVUPSZ256mr_NOVLX:
633 case X86::VMOVAPSZ256mr_NOVLX:
634 case X86::VMOVUPDZ256mr:
635 case X86::VMOVAPDZ256mr:
636 case X86::VMOVDQU8Z256mr:
637 case X86::VMOVDQU16Z256mr:
638 case X86::VMOVDQA32Z256mr:
639 case X86::VMOVDQU32Z256mr:
640 case X86::VMOVDQA64Z256mr:
641 case X86::VMOVDQU64Z256mr:
644 case X86::VMOVUPSZmr:
645 case X86::VMOVAPSZmr:
646 case X86::VMOVUPDZmr:
647 case X86::VMOVAPDZmr:
648 case X86::VMOVDQU8Zmr:
649 case X86::VMOVDQU16Zmr:
650 case X86::VMOVDQA32Zmr:
651 case X86::VMOVDQU32Zmr:
652 case X86::VMOVDQA64Zmr:
653 case X86::VMOVDQU64Zmr:
661 int &FrameIndex)
const {
668 unsigned &MemBytes)
const {
670 if (
MI.getOperand(0).getSubReg() == 0 && isFrameOperand(
MI, 1, FrameIndex))
671 return MI.getOperand(0).getReg();
676 int &FrameIndex)
const {
684 if (hasLoadFromStackSlot(
MI, Accesses)) {
686 cast<FixedStackPseudoSourceValue>(Accesses.
front()->getPseudoValue())
688 return MI.getOperand(0).getReg();
695 int &FrameIndex)
const {
702 unsigned &MemBytes)
const {
705 isFrameOperand(
MI, 0, FrameIndex))
711 int &FrameIndex)
const {
719 if (hasStoreToStackSlot(
MI, Accesses)) {
721 cast<FixedStackPseudoSourceValue>(Accesses.
front()->getPseudoValue())
734 bool isPICBase =
false;
736 E =
MRI.def_instr_end();
I !=
E; ++
I) {
740 assert(!isPICBase &&
"More than one PIC base?");
748 switch (
MI.getOpcode()) {
755 case X86::LOAD_STACK_GUARD:
756 case X86::AVX1_SETALLONES:
757 case X86::AVX2_SETALLONES:
758 case X86::AVX512_128_SET0:
759 case X86::AVX512_256_SET0:
760 case X86::AVX512_512_SET0:
761 case X86::AVX512_512_SETALLONES:
762 case X86::AVX512_FsFLD0SD:
763 case X86::AVX512_FsFLD0SH:
764 case X86::AVX512_FsFLD0SS:
765 case X86::AVX512_FsFLD0F128:
770 case X86::FsFLD0F128:
778 case X86::MOV32ImmSExti8:
783 case X86::MOV64ImmSExti8:
785 case X86::V_SETALLONES:
791 case X86::PTILEZEROV:
795 case X86::MOV8rm_NOREX:
800 case X86::MOVSSrm_alt:
802 case X86::MOVSDrm_alt:
810 case X86::VMOVSSrm_alt:
812 case X86::VMOVSDrm_alt:
819 case X86::VMOVAPSYrm:
820 case X86::VMOVUPSYrm:
821 case X86::VMOVAPDYrm:
822 case X86::VMOVUPDYrm:
823 case X86::VMOVDQAYrm:
824 case X86::VMOVDQUYrm:
825 case X86::MMX_MOVD64rm:
826 case X86::MMX_MOVQ64rm:
829 case X86::VMOVSSZrm_alt:
831 case X86::VMOVSDZrm_alt:
833 case X86::VMOVSHZrm_alt:
834 case X86::VMOVAPDZ128rm:
835 case X86::VMOVAPDZ256rm:
836 case X86::VMOVAPDZrm:
837 case X86::VMOVAPSZ128rm:
838 case X86::VMOVAPSZ256rm:
839 case X86::VMOVAPSZ128rm_NOVLX:
840 case X86::VMOVAPSZ256rm_NOVLX:
841 case X86::VMOVAPSZrm:
842 case X86::VMOVDQA32Z128rm:
843 case X86::VMOVDQA32Z256rm:
844 case X86::VMOVDQA32Zrm:
845 case X86::VMOVDQA64Z128rm:
846 case X86::VMOVDQA64Z256rm:
847 case X86::VMOVDQA64Zrm:
848 case X86::VMOVDQU16Z128rm:
849 case X86::VMOVDQU16Z256rm:
850 case X86::VMOVDQU16Zrm:
851 case X86::VMOVDQU32Z128rm:
852 case X86::VMOVDQU32Z256rm:
853 case X86::VMOVDQU32Zrm:
854 case X86::VMOVDQU64Z128rm:
855 case X86::VMOVDQU64Z256rm:
856 case X86::VMOVDQU64Zrm:
857 case X86::VMOVDQU8Z128rm:
858 case X86::VMOVDQU8Z256rm:
859 case X86::VMOVDQU8Zrm:
860 case X86::VMOVUPDZ128rm:
861 case X86::VMOVUPDZ256rm:
862 case X86::VMOVUPDZrm:
863 case X86::VMOVUPSZ128rm:
864 case X86::VMOVUPSZ256rm:
865 case X86::VMOVUPSZ128rm_NOVLX:
866 case X86::VMOVUPSZ256rm_NOVLX:
867 case X86::VMOVUPSZrm: {
873 MI.isDereferenceableInvariantLoad()) {
875 if (BaseReg == 0 || BaseReg == X86::RIP)
921 case X86::MOV32r0:
Value = 0;
break;
922 case X86::MOV32r1:
Value = 1;
break;
923 case X86::MOV32r_1:
Value = -1;
break;
944 if (MO.isReg() && MO.isDef() &&
945 MO.getReg() == X86::EFLAGS && !MO.isDead()) {
954 unsigned ShiftAmtOperandIdx) {
956 unsigned ShiftCountMask = (
MI.getDesc().TSFlags &
X86II::REX_W) ? 63 : 31;
957 unsigned Imm =
MI.getOperand(ShiftAmtOperandIdx).getImm();
958 return Imm & ShiftCountMask;
969 return ShAmt < 4 && ShAmt > 0;
977 bool &NoSignFlag,
bool &ClearsOverflowFlag) {
978 if (CmpValDefInstr.
getOpcode() != X86::SUBREG_TO_REG)
981 if (CmpInstr.
getOpcode() != X86::TEST64rr)
988 "CmpInstr is an analyzable TEST64rr, and `X86InstrInfo::analyzeCompare` "
989 "requires two reg operands are the same.");
997 "Caller guarantees that TEST64rr is a user of SUBREG_TO_REG.");
1012 assert(VregDefInstr &&
"Must have a definition (SSA)");
1022 if (X86::isAND(VregDefInstr->
getOpcode())) {
1037 if (Instr.modifiesRegister(X86::EFLAGS,
TRI))
1041 *AndInstr = VregDefInstr;
1062 ClearsOverflowFlag =
true;
1069 unsigned Opc,
bool AllowSP,
Register &NewSrc,
1075 RC = Opc != X86::LEA32r ? &X86::GR64RegClass : &X86::GR32RegClass;
1077 RC = Opc != X86::LEA32r ?
1078 &X86::GR64_NOSPRegClass : &X86::GR32_NOSPRegClass;
1081 isKill =
MI.killsRegister(SrcReg);
1085 if (Opc != X86::LEA64_32r) {
1087 assert(!Src.isUndef() &&
"Undef op doesn't need optimization");
1103 assert(!Src.isUndef() &&
"Undef op doesn't need optimization");
1133MachineInstr *X86InstrInfo::convertToThreeAddressWithLEA(
unsigned MIOpc,
1137 bool Is8BitOp)
const {
1143 "Unexpected type for LEA transform");
1152 if (!Subtarget.is64Bit())
1155 unsigned Opcode = X86::LEA64_32r;
1171 bool IsDead =
MI.getOperand(0).isDead();
1172 bool IsKill =
MI.getOperand(1).isKill();
1173 unsigned SubReg = Is8BitOp ? X86::sub_8bit : X86::sub_16bit;
1174 assert(!
MI.getOperand(1).isUndef() &&
"Undef op doesn't need optimization");
1189 case X86::SHL16ri: {
1190 unsigned ShAmt =
MI.getOperand(2).getImm();
1207 case X86::ADD8ri_DB:
1210 case X86::ADD16ri_DB:
1211 case X86::ADD16ri8_DB:
1215 case X86::ADD8rr_DB:
1217 case X86::ADD16rr_DB: {
1218 Src2 =
MI.getOperand(2).getReg();
1219 bool IsKill2 =
MI.getOperand(2).isKill();
1220 assert(!
MI.getOperand(2).isUndef() &&
"Undef op doesn't need optimization");
1224 addRegReg(MIB, InRegLEA,
true, InRegLEA,
false);
1226 if (Subtarget.is64Bit())
1227 InRegLEA2 =
RegInfo.createVirtualRegister(&X86::GR64_NOSPRegClass);
1229 InRegLEA2 =
RegInfo.createVirtualRegister(&X86::GR32_NOSPRegClass);
1232 ImpDef2 =
BuildMI(
MBB, &*MIB,
MI.getDebugLoc(),
get(X86::IMPLICIT_DEF),
1234 InsMI2 =
BuildMI(
MBB, &*MIB,
MI.getDebugLoc(),
get(TargetOpcode::COPY))
1237 addRegReg(MIB, InRegLEA,
true, InRegLEA2,
true);
1239 if (LV && IsKill2 && InsMI2)
1335 if (
MI.getNumOperands() > 2)
1336 if (
MI.getOperand(2).isReg() &&
MI.getOperand(2).isUndef())
1341 bool Is64Bit = Subtarget.is64Bit();
1343 bool Is8BitOp =
false;
1344 unsigned NumRegOperands = 2;
1345 unsigned MIOpc =
MI.getOpcode();
1348 case X86::SHL64ri: {
1349 assert(
MI.getNumOperands() >= 3 &&
"Unknown shift instruction!");
1355 Src.getReg(), &X86::GR64_NOSPRegClass))
1358 NewMI =
BuildMI(MF,
MI.getDebugLoc(),
get(X86::LEA64r))
1367 case X86::SHL32ri: {
1368 assert(
MI.getNumOperands() >= 3 &&
"Unknown shift instruction!");
1372 unsigned Opc = Is64Bit ? X86::LEA64_32r : X86::LEA32r;
1378 ImplicitOp, LV, LIS))
1389 if (ImplicitOp.
getReg() != 0)
1390 MIB.
add(ImplicitOp);
1394 if (LV && SrcReg != Src.getReg())
1401 case X86::SHL16ri: {
1402 assert(
MI.getNumOperands() >= 3 &&
"Unknown shift instruction!");
1406 return convertToThreeAddressWithLEA(MIOpc,
MI, LV, LIS, Is8BitOp);
1410 assert(
MI.getNumOperands() >= 2 &&
"Unknown inc instruction!");
1411 unsigned Opc = MIOpc == X86::INC64r ? X86::LEA64r :
1412 (Is64Bit ? X86::LEA64_32r : X86::LEA32r);
1416 ImplicitOp, LV, LIS))
1423 if (ImplicitOp.
getReg() != 0)
1424 MIB.
add(ImplicitOp);
1429 if (LV && SrcReg != Src.getReg())
1435 assert(
MI.getNumOperands() >= 2 &&
"Unknown dec instruction!");
1436 unsigned Opc = MIOpc == X86::DEC64r ? X86::LEA64r
1437 : (Is64Bit ? X86::LEA64_32r : X86::LEA32r);
1442 ImplicitOp, LV, LIS))
1448 if (ImplicitOp.
getReg() != 0)
1449 MIB.
add(ImplicitOp);
1454 if (LV && SrcReg != Src.getReg())
1464 return convertToThreeAddressWithLEA(MIOpc,
MI, LV, LIS, Is8BitOp);
1466 case X86::ADD64rr_DB:
1468 case X86::ADD32rr_DB: {
1469 assert(
MI.getNumOperands() >= 3 &&
"Unknown add instruction!");
1471 if (MIOpc == X86::ADD64rr || MIOpc == X86::ADD64rr_DB)
1474 Opc = Is64Bit ? X86::LEA64_32r : X86::LEA32r;
1480 ImplicitOp2, LV, LIS))
1485 if (Src.getReg() == Src2.
getReg()) {
1492 ImplicitOp, LV, LIS))
1497 if (ImplicitOp.
getReg() != 0)
1498 MIB.
add(ImplicitOp);
1499 if (ImplicitOp2.
getReg() != 0)
1500 MIB.
add(ImplicitOp2);
1502 NewMI =
addRegReg(MIB, SrcReg, isKill, SrcReg2, isKill2);
1506 if (SrcReg2 != Src2.
getReg())
1508 if (SrcReg != SrcReg2 && SrcReg != Src.getReg())
1515 case X86::ADD8rr_DB:
1519 case X86::ADD16rr_DB:
1520 return convertToThreeAddressWithLEA(MIOpc,
MI, LV, LIS, Is8BitOp);
1521 case X86::ADD64ri32:
1523 case X86::ADD64ri32_DB:
1524 case X86::ADD64ri8_DB:
1525 assert(
MI.getNumOperands() >= 3 &&
"Unknown add instruction!");
1527 BuildMI(MF,
MI.getDebugLoc(),
get(X86::LEA64r)).add(Dest).add(Src),
1532 case X86::ADD32ri_DB:
1533 case X86::ADD32ri8_DB: {
1534 assert(
MI.getNumOperands() >= 3 &&
"Unknown add instruction!");
1535 unsigned Opc = Is64Bit ? X86::LEA64_32r : X86::LEA32r;
1540 ImplicitOp, LV, LIS))
1546 if (ImplicitOp.
getReg() != 0)
1547 MIB.
add(ImplicitOp);
1552 if (LV && SrcReg != Src.getReg())
1557 case X86::ADD8ri_DB:
1562 case X86::ADD16ri_DB:
1563 case X86::ADD16ri8_DB:
1564 return convertToThreeAddressWithLEA(MIOpc,
MI, LV, LIS, Is8BitOp);
1571 case X86::SUB32ri: {
1572 if (!
MI.getOperand(2).isImm())
1574 int64_t Imm =
MI.getOperand(2).getImm();
1575 if (!isInt<32>(-Imm))
1578 assert(
MI.getNumOperands() >= 3 &&
"Unknown add instruction!");
1579 unsigned Opc = Is64Bit ? X86::LEA64_32r : X86::LEA32r;
1584 ImplicitOp, LV, LIS))
1590 if (ImplicitOp.
getReg() != 0)
1591 MIB.
add(ImplicitOp);
1596 if (LV && SrcReg != Src.getReg())
1602 case X86::SUB64ri32: {
1603 if (!
MI.getOperand(2).isImm())
1605 int64_t Imm =
MI.getOperand(2).getImm();
1606 if (!isInt<32>(-Imm))
1609 assert(
MI.getNumOperands() >= 3 &&
"Unknown sub instruction!");
1617 case X86::VMOVDQU8Z128rmk:
1618 case X86::VMOVDQU8Z256rmk:
1619 case X86::VMOVDQU8Zrmk:
1620 case X86::VMOVDQU16Z128rmk:
1621 case X86::VMOVDQU16Z256rmk:
1622 case X86::VMOVDQU16Zrmk:
1623 case X86::VMOVDQU32Z128rmk:
case X86::VMOVDQA32Z128rmk:
1624 case X86::VMOVDQU32Z256rmk:
case X86::VMOVDQA32Z256rmk:
1625 case X86::VMOVDQU32Zrmk:
case X86::VMOVDQA32Zrmk:
1626 case X86::VMOVDQU64Z128rmk:
case X86::VMOVDQA64Z128rmk:
1627 case X86::VMOVDQU64Z256rmk:
case X86::VMOVDQA64Z256rmk:
1628 case X86::VMOVDQU64Zrmk:
case X86::VMOVDQA64Zrmk:
1629 case X86::VMOVUPDZ128rmk:
case X86::VMOVAPDZ128rmk:
1630 case X86::VMOVUPDZ256rmk:
case X86::VMOVAPDZ256rmk:
1631 case X86::VMOVUPDZrmk:
case X86::VMOVAPDZrmk:
1632 case X86::VMOVUPSZ128rmk:
case X86::VMOVAPSZ128rmk:
1633 case X86::VMOVUPSZ256rmk:
case X86::VMOVAPSZ256rmk:
1634 case X86::VMOVUPSZrmk:
case X86::VMOVAPSZrmk:
1635 case X86::VBROADCASTSDZ256rmk:
1636 case X86::VBROADCASTSDZrmk:
1637 case X86::VBROADCASTSSZ128rmk:
1638 case X86::VBROADCASTSSZ256rmk:
1639 case X86::VBROADCASTSSZrmk:
1640 case X86::VPBROADCASTDZ128rmk:
1641 case X86::VPBROADCASTDZ256rmk:
1642 case X86::VPBROADCASTDZrmk:
1643 case X86::VPBROADCASTQZ128rmk:
1644 case X86::VPBROADCASTQZ256rmk:
1645 case X86::VPBROADCASTQZrmk: {
1649 case X86::VMOVDQU8Z128rmk: Opc = X86::VPBLENDMBZ128rmk;
break;
1650 case X86::VMOVDQU8Z256rmk: Opc = X86::VPBLENDMBZ256rmk;
break;
1651 case X86::VMOVDQU8Zrmk: Opc = X86::VPBLENDMBZrmk;
break;
1652 case X86::VMOVDQU16Z128rmk: Opc = X86::VPBLENDMWZ128rmk;
break;
1653 case X86::VMOVDQU16Z256rmk: Opc = X86::VPBLENDMWZ256rmk;
break;
1654 case X86::VMOVDQU16Zrmk: Opc = X86::VPBLENDMWZrmk;
break;
1655 case X86::VMOVDQU32Z128rmk: Opc = X86::VPBLENDMDZ128rmk;
break;
1656 case X86::VMOVDQU32Z256rmk: Opc = X86::VPBLENDMDZ256rmk;
break;
1657 case X86::VMOVDQU32Zrmk: Opc = X86::VPBLENDMDZrmk;
break;
1658 case X86::VMOVDQU64Z128rmk: Opc = X86::VPBLENDMQZ128rmk;
break;
1659 case X86::VMOVDQU64Z256rmk: Opc = X86::VPBLENDMQZ256rmk;
break;
1660 case X86::VMOVDQU64Zrmk: Opc = X86::VPBLENDMQZrmk;
break;
1661 case X86::VMOVUPDZ128rmk: Opc = X86::VBLENDMPDZ128rmk;
break;
1662 case X86::VMOVUPDZ256rmk: Opc = X86::VBLENDMPDZ256rmk;
break;
1663 case X86::VMOVUPDZrmk: Opc = X86::VBLENDMPDZrmk;
break;
1664 case X86::VMOVUPSZ128rmk: Opc = X86::VBLENDMPSZ128rmk;
break;
1665 case X86::VMOVUPSZ256rmk: Opc = X86::VBLENDMPSZ256rmk;
break;
1666 case X86::VMOVUPSZrmk: Opc = X86::VBLENDMPSZrmk;
break;
1667 case X86::VMOVDQA32Z128rmk: Opc = X86::VPBLENDMDZ128rmk;
break;
1668 case X86::VMOVDQA32Z256rmk: Opc = X86::VPBLENDMDZ256rmk;
break;
1669 case X86::VMOVDQA32Zrmk: Opc = X86::VPBLENDMDZrmk;
break;
1670 case X86::VMOVDQA64Z128rmk: Opc = X86::VPBLENDMQZ128rmk;
break;
1671 case X86::VMOVDQA64Z256rmk: Opc = X86::VPBLENDMQZ256rmk;
break;
1672 case X86::VMOVDQA64Zrmk: Opc = X86::VPBLENDMQZrmk;
break;
1673 case X86::VMOVAPDZ128rmk: Opc = X86::VBLENDMPDZ128rmk;
break;
1674 case X86::VMOVAPDZ256rmk: Opc = X86::VBLENDMPDZ256rmk;
break;
1675 case X86::VMOVAPDZrmk: Opc = X86::VBLENDMPDZrmk;
break;
1676 case X86::VMOVAPSZ128rmk: Opc = X86::VBLENDMPSZ128rmk;
break;
1677 case X86::VMOVAPSZ256rmk: Opc = X86::VBLENDMPSZ256rmk;
break;
1678 case X86::VMOVAPSZrmk: Opc = X86::VBLENDMPSZrmk;
break;
1679 case X86::VBROADCASTSDZ256rmk: Opc = X86::VBLENDMPDZ256rmbk;
break;
1680 case X86::VBROADCASTSDZrmk: Opc = X86::VBLENDMPDZrmbk;
break;
1681 case X86::VBROADCASTSSZ128rmk: Opc = X86::VBLENDMPSZ128rmbk;
break;
1682 case X86::VBROADCASTSSZ256rmk: Opc = X86::VBLENDMPSZ256rmbk;
break;
1683 case X86::VBROADCASTSSZrmk: Opc = X86::VBLENDMPSZrmbk;
break;
1684 case X86::VPBROADCASTDZ128rmk: Opc = X86::VPBLENDMDZ128rmbk;
break;
1685 case X86::VPBROADCASTDZ256rmk: Opc = X86::VPBLENDMDZ256rmbk;
break;
1686 case X86::VPBROADCASTDZrmk: Opc = X86::VPBLENDMDZrmbk;
break;
1687 case X86::VPBROADCASTQZ128rmk: Opc = X86::VPBLENDMQZ128rmbk;
break;
1688 case X86::VPBROADCASTQZ256rmk: Opc = X86::VPBLENDMQZ256rmbk;
break;
1689 case X86::VPBROADCASTQZrmk: Opc = X86::VPBLENDMQZrmbk;
break;
1694 .
add(
MI.getOperand(2))
1696 .
add(
MI.getOperand(3))
1697 .
add(
MI.getOperand(4))
1698 .
add(
MI.getOperand(5))
1699 .
add(
MI.getOperand(6))
1700 .
add(
MI.getOperand(7));
1705 case X86::VMOVDQU8Z128rrk:
1706 case X86::VMOVDQU8Z256rrk:
1707 case X86::VMOVDQU8Zrrk:
1708 case X86::VMOVDQU16Z128rrk:
1709 case X86::VMOVDQU16Z256rrk:
1710 case X86::VMOVDQU16Zrrk:
1711 case X86::VMOVDQU32Z128rrk:
case X86::VMOVDQA32Z128rrk:
1712 case X86::VMOVDQU32Z256rrk:
case X86::VMOVDQA32Z256rrk:
1713 case X86::VMOVDQU32Zrrk:
case X86::VMOVDQA32Zrrk:
1714 case X86::VMOVDQU64Z128rrk:
case X86::VMOVDQA64Z128rrk:
1715 case X86::VMOVDQU64Z256rrk:
case X86::VMOVDQA64Z256rrk:
1716 case X86::VMOVDQU64Zrrk:
case X86::VMOVDQA64Zrrk:
1717 case X86::VMOVUPDZ128rrk:
case X86::VMOVAPDZ128rrk:
1718 case X86::VMOVUPDZ256rrk:
case X86::VMOVAPDZ256rrk:
1719 case X86::VMOVUPDZrrk:
case X86::VMOVAPDZrrk:
1720 case X86::VMOVUPSZ128rrk:
case X86::VMOVAPSZ128rrk:
1721 case X86::VMOVUPSZ256rrk:
case X86::VMOVAPSZ256rrk:
1722 case X86::VMOVUPSZrrk:
case X86::VMOVAPSZrrk: {
1726 case X86::VMOVDQU8Z128rrk: Opc = X86::VPBLENDMBZ128rrk;
break;
1727 case X86::VMOVDQU8Z256rrk: Opc = X86::VPBLENDMBZ256rrk;
break;
1728 case X86::VMOVDQU8Zrrk: Opc = X86::VPBLENDMBZrrk;
break;
1729 case X86::VMOVDQU16Z128rrk: Opc = X86::VPBLENDMWZ128rrk;
break;
1730 case X86::VMOVDQU16Z256rrk: Opc = X86::VPBLENDMWZ256rrk;
break;
1731 case X86::VMOVDQU16Zrrk: Opc = X86::VPBLENDMWZrrk;
break;
1732 case X86::VMOVDQU32Z128rrk: Opc = X86::VPBLENDMDZ128rrk;
break;
1733 case X86::VMOVDQU32Z256rrk: Opc = X86::VPBLENDMDZ256rrk;
break;
1734 case X86::VMOVDQU32Zrrk: Opc = X86::VPBLENDMDZrrk;
break;
1735 case X86::VMOVDQU64Z128rrk: Opc = X86::VPBLENDMQZ128rrk;
break;
1736 case X86::VMOVDQU64Z256rrk: Opc = X86::VPBLENDMQZ256rrk;
break;
1737 case X86::VMOVDQU64Zrrk: Opc = X86::VPBLENDMQZrrk;
break;
1738 case X86::VMOVUPDZ128rrk: Opc = X86::VBLENDMPDZ128rrk;
break;
1739 case X86::VMOVUPDZ256rrk: Opc = X86::VBLENDMPDZ256rrk;
break;
1740 case X86::VMOVUPDZrrk: Opc = X86::VBLENDMPDZrrk;
break;
1741 case X86::VMOVUPSZ128rrk: Opc = X86::VBLENDMPSZ128rrk;
break;
1742 case X86::VMOVUPSZ256rrk: Opc = X86::VBLENDMPSZ256rrk;
break;
1743 case X86::VMOVUPSZrrk: Opc = X86::VBLENDMPSZrrk;
break;
1744 case X86::VMOVDQA32Z128rrk: Opc = X86::VPBLENDMDZ128rrk;
break;
1745 case X86::VMOVDQA32Z256rrk: Opc = X86::VPBLENDMDZ256rrk;
break;
1746 case X86::VMOVDQA32Zrrk: Opc = X86::VPBLENDMDZrrk;
break;
1747 case X86::VMOVDQA64Z128rrk: Opc = X86::VPBLENDMQZ128rrk;
break;
1748 case X86::VMOVDQA64Z256rrk: Opc = X86::VPBLENDMQZ256rrk;
break;
1749 case X86::VMOVDQA64Zrrk: Opc = X86::VPBLENDMQZrrk;
break;
1750 case X86::VMOVAPDZ128rrk: Opc = X86::VBLENDMPDZ128rrk;
break;
1751 case X86::VMOVAPDZ256rrk: Opc = X86::VBLENDMPDZ256rrk;
break;
1752 case X86::VMOVAPDZrrk: Opc = X86::VBLENDMPDZrrk;
break;
1753 case X86::VMOVAPSZ128rrk: Opc = X86::VBLENDMPSZ128rrk;
break;
1754 case X86::VMOVAPSZ256rrk: Opc = X86::VBLENDMPSZ256rrk;
break;
1755 case X86::VMOVAPSZrrk: Opc = X86::VBLENDMPSZrrk;
break;
1760 .
add(
MI.getOperand(2))
1762 .
add(
MI.getOperand(3));
1768 if (!NewMI)
return nullptr;
1771 for (
unsigned I = 0;
I < NumRegOperands; ++
I) {
1773 if (Op.isReg() && (Op.isDead() || Op.isKill()))
1800 unsigned SrcOpIdx2) {
1802 if (SrcOpIdx1 > SrcOpIdx2)
1805 unsigned Op1 = 1, Op2 = 2, Op3 = 3;
1811 if (SrcOpIdx1 == Op1 && SrcOpIdx2 == Op2)
1813 if (SrcOpIdx1 == Op1 && SrcOpIdx2 == Op3)
1815 if (SrcOpIdx1 == Op2 && SrcOpIdx2 == Op3)
1824 unsigned Opc =
MI.getOpcode();
1833 "Intrinsic instructions can't commute operand 1");
1838 assert(Case < 3 &&
"Unexpected case number!");
1843 const unsigned Form132Index = 0;
1844 const unsigned Form213Index = 1;
1845 const unsigned Form231Index = 2;
1846 static const unsigned FormMapping[][3] = {
1851 { Form231Index, Form213Index, Form132Index },
1856 { Form132Index, Form231Index, Form213Index },
1861 { Form213Index, Form132Index, Form231Index }
1864 unsigned FMAForms[3];
1870 for (
unsigned FormIndex = 0; FormIndex < 3; FormIndex++)
1871 if (Opc == FMAForms[FormIndex])
1872 return FMAForms[FormMapping[Case][FormIndex]];
1878 unsigned SrcOpIdx2) {
1882 assert(Case < 3 &&
"Unexpected case value!");
1885 static const uint8_t SwapMasks[3][4] = {
1886 { 0x04, 0x10, 0x08, 0x20 },
1887 { 0x02, 0x10, 0x08, 0x40 },
1888 { 0x02, 0x04, 0x20, 0x40 },
1891 uint8_t Imm =
MI.getOperand(
MI.getNumOperands()-1).getImm();
1893 uint8_t NewImm = Imm & ~(SwapMasks[Case][0] | SwapMasks[Case][1] |
1894 SwapMasks[Case][2] | SwapMasks[Case][3]);
1896 if (Imm & SwapMasks[Case][0]) NewImm |= SwapMasks[Case][1];
1897 if (Imm & SwapMasks[Case][1]) NewImm |= SwapMasks[Case][0];
1898 if (Imm & SwapMasks[Case][2]) NewImm |= SwapMasks[Case][3];
1899 if (Imm & SwapMasks[Case][3]) NewImm |= SwapMasks[Case][2];
1900 MI.getOperand(
MI.getNumOperands()-1).setImm(NewImm);
1906#define VPERM_CASES(Suffix) \
1907 case X86::VPERMI2##Suffix##128rr: case X86::VPERMT2##Suffix##128rr: \
1908 case X86::VPERMI2##Suffix##256rr: case X86::VPERMT2##Suffix##256rr: \
1909 case X86::VPERMI2##Suffix##rr: case X86::VPERMT2##Suffix##rr: \
1910 case X86::VPERMI2##Suffix##128rm: case X86::VPERMT2##Suffix##128rm: \
1911 case X86::VPERMI2##Suffix##256rm: case X86::VPERMT2##Suffix##256rm: \
1912 case X86::VPERMI2##Suffix##rm: case X86::VPERMT2##Suffix##rm: \
1913 case X86::VPERMI2##Suffix##128rrkz: case X86::VPERMT2##Suffix##128rrkz: \
1914 case X86::VPERMI2##Suffix##256rrkz: case X86::VPERMT2##Suffix##256rrkz: \
1915 case X86::VPERMI2##Suffix##rrkz: case X86::VPERMT2##Suffix##rrkz: \
1916 case X86::VPERMI2##Suffix##128rmkz: case X86::VPERMT2##Suffix##128rmkz: \
1917 case X86::VPERMI2##Suffix##256rmkz: case X86::VPERMT2##Suffix##256rmkz: \
1918 case X86::VPERMI2##Suffix##rmkz: case X86::VPERMT2##Suffix##rmkz:
1920#define VPERM_CASES_BROADCAST(Suffix) \
1921 VPERM_CASES(Suffix) \
1922 case X86::VPERMI2##Suffix##128rmb: case X86::VPERMT2##Suffix##128rmb: \
1923 case X86::VPERMI2##Suffix##256rmb: case X86::VPERMT2##Suffix##256rmb: \
1924 case X86::VPERMI2##Suffix##rmb: case X86::VPERMT2##Suffix##rmb: \
1925 case X86::VPERMI2##Suffix##128rmbkz: case X86::VPERMT2##Suffix##128rmbkz: \
1926 case X86::VPERMI2##Suffix##256rmbkz: case X86::VPERMT2##Suffix##256rmbkz: \
1927 case X86::VPERMI2##Suffix##rmbkz: case X86::VPERMT2##Suffix##rmbkz:
1930 default:
return false;
1939#undef VPERM_CASES_BROADCAST
1946#define VPERM_CASES(Orig, New) \
1947 case X86::Orig##128rr: return X86::New##128rr; \
1948 case X86::Orig##128rrkz: return X86::New##128rrkz; \
1949 case X86::Orig##128rm: return X86::New##128rm; \
1950 case X86::Orig##128rmkz: return X86::New##128rmkz; \
1951 case X86::Orig##256rr: return X86::New##256rr; \
1952 case X86::Orig##256rrkz: return X86::New##256rrkz; \
1953 case X86::Orig##256rm: return X86::New##256rm; \
1954 case X86::Orig##256rmkz: return X86::New##256rmkz; \
1955 case X86::Orig##rr: return X86::New##rr; \
1956 case X86::Orig##rrkz: return X86::New##rrkz; \
1957 case X86::Orig##rm: return X86::New##rm; \
1958 case X86::Orig##rmkz: return X86::New##rmkz;
1960#define VPERM_CASES_BROADCAST(Orig, New) \
1961 VPERM_CASES(Orig, New) \
1962 case X86::Orig##128rmb: return X86::New##128rmb; \
1963 case X86::Orig##128rmbkz: return X86::New##128rmbkz; \
1964 case X86::Orig##256rmb: return X86::New##256rmb; \
1965 case X86::Orig##256rmbkz: return X86::New##256rmbkz; \
1966 case X86::Orig##rmb: return X86::New##rmb; \
1967 case X86::Orig##rmbkz: return X86::New##rmbkz;
1985#undef VPERM_CASES_BROADCAST
1991 unsigned OpIdx2)
const {
1994 return *
MI.getParent()->getParent()->CloneMachineInstr(&
MI);
1998 switch (
MI.getOpcode()) {
1999 case X86::SHRD16rri8:
2000 case X86::SHLD16rri8:
2001 case X86::SHRD32rri8:
2002 case X86::SHLD32rri8:
2003 case X86::SHRD64rri8:
2004 case X86::SHLD64rri8:{
2007 switch (
MI.getOpcode()) {
2009 case X86::SHRD16rri8:
Size = 16; Opc = X86::SHLD16rri8;
break;
2010 case X86::SHLD16rri8:
Size = 16; Opc = X86::SHRD16rri8;
break;
2011 case X86::SHRD32rri8:
Size = 32; Opc = X86::SHLD32rri8;
break;
2012 case X86::SHLD32rri8:
Size = 32; Opc = X86::SHRD32rri8;
break;
2013 case X86::SHRD64rri8:
Size = 64; Opc = X86::SHLD64rri8;
break;
2014 case X86::SHLD64rri8:
Size = 64; Opc = X86::SHRD64rri8;
break;
2016 unsigned Amt =
MI.getOperand(3).getImm();
2017 auto &WorkingMI = cloneIfNew(
MI);
2018 WorkingMI.setDesc(
get(Opc));
2019 WorkingMI.getOperand(3).setImm(
Size - Amt);
2024 case X86::PFSUBRrr: {
2028 (X86::PFSUBRrr ==
MI.getOpcode() ? X86::PFSUBrr : X86::PFSUBRrr);
2029 auto &WorkingMI = cloneIfNew(
MI);
2030 WorkingMI.setDesc(
get(Opc));
2034 case X86::BLENDPDrri:
2035 case X86::BLENDPSrri:
2036 case X86::VBLENDPDrri:
2037 case X86::VBLENDPSrri:
2039 if (
MI.getParent()->getParent()->getFunction().hasOptSize()) {
2041 switch (
MI.getOpcode()) {
2043 case X86::BLENDPDrri: Opc = X86::MOVSDrr; Mask = 0x03;
break;
2044 case X86::BLENDPSrri: Opc = X86::MOVSSrr; Mask = 0x0F;
break;
2045 case X86::VBLENDPDrri: Opc = X86::VMOVSDrr; Mask = 0x03;
break;
2046 case X86::VBLENDPSrri: Opc = X86::VMOVSSrr; Mask = 0x0F;
break;
2048 if ((
MI.getOperand(3).getImm() ^ Mask) == 1) {
2049 auto &WorkingMI = cloneIfNew(
MI);
2050 WorkingMI.setDesc(
get(Opc));
2051 WorkingMI.removeOperand(3);
2058 case X86::PBLENDWrri:
2059 case X86::VBLENDPDYrri:
2060 case X86::VBLENDPSYrri:
2061 case X86::VPBLENDDrri:
2062 case X86::VPBLENDWrri:
2063 case X86::VPBLENDDYrri:
2064 case X86::VPBLENDWYrri:{
2066 switch (
MI.getOpcode()) {
2068 case X86::BLENDPDrri: Mask = (int8_t)0x03;
break;
2069 case X86::BLENDPSrri: Mask = (int8_t)0x0F;
break;
2070 case X86::PBLENDWrri: Mask = (int8_t)0xFF;
break;
2071 case X86::VBLENDPDrri: Mask = (int8_t)0x03;
break;
2072 case X86::VBLENDPSrri: Mask = (int8_t)0x0F;
break;
2073 case X86::VBLENDPDYrri: Mask = (int8_t)0x0F;
break;
2074 case X86::VBLENDPSYrri: Mask = (int8_t)0xFF;
break;
2075 case X86::VPBLENDDrri: Mask = (int8_t)0x0F;
break;
2076 case X86::VPBLENDWrri: Mask = (int8_t)0xFF;
break;
2077 case X86::VPBLENDDYrri: Mask = (int8_t)0xFF;
break;
2078 case X86::VPBLENDWYrri: Mask = (int8_t)0xFF;
break;
2083 int8_t Imm =
MI.getOperand(3).getImm() & Mask;
2084 auto &WorkingMI = cloneIfNew(
MI);
2085 WorkingMI.getOperand(3).setImm(Mask ^ Imm);
2089 case X86::INSERTPSrr:
2090 case X86::VINSERTPSrr:
2091 case X86::VINSERTPSZrr: {
2092 unsigned Imm =
MI.getOperand(
MI.getNumOperands() - 1).getImm();
2093 unsigned ZMask = Imm & 15;
2094 unsigned DstIdx = (Imm >> 4) & 3;
2095 unsigned SrcIdx = (Imm >> 6) & 3;
2099 if (DstIdx == SrcIdx && (ZMask & (1 << DstIdx)) == 0 &&
2102 assert(AltIdx < 4 &&
"Illegal insertion index");
2103 unsigned AltImm = (AltIdx << 6) | (AltIdx << 4) | ZMask;
2104 auto &WorkingMI = cloneIfNew(
MI);
2105 WorkingMI.getOperand(
MI.getNumOperands() - 1).setImm(AltImm);
2114 case X86::VMOVSSrr:{
2118 switch (
MI.getOpcode()) {
2120 case X86::MOVSDrr: Opc = X86::BLENDPDrri; Mask = 0x02;
break;
2121 case X86::MOVSSrr: Opc = X86::BLENDPSrri; Mask = 0x0E;
break;
2122 case X86::VMOVSDrr: Opc = X86::VBLENDPDrri; Mask = 0x02;
break;
2123 case X86::VMOVSSrr: Opc = X86::VBLENDPSrri; Mask = 0x0E;
break;
2126 auto &WorkingMI = cloneIfNew(
MI);
2127 WorkingMI.setDesc(
get(Opc));
2134 assert(
MI.getOpcode() == X86::MOVSDrr &&
2135 "Can only commute MOVSDrr without SSE4.1");
2137 auto &WorkingMI = cloneIfNew(
MI);
2138 WorkingMI.setDesc(
get(X86::SHUFPDrri));
2143 case X86::SHUFPDrri: {
2145 assert(
MI.getOperand(3).getImm() == 0x02 &&
"Unexpected immediate!");
2146 auto &WorkingMI = cloneIfNew(
MI);
2147 WorkingMI.setDesc(
get(X86::MOVSDrr));
2148 WorkingMI.removeOperand(3);
2152 case X86::PCLMULQDQrr:
2153 case X86::VPCLMULQDQrr:
2154 case X86::VPCLMULQDQYrr:
2155 case X86::VPCLMULQDQZrr:
2156 case X86::VPCLMULQDQZ128rr:
2157 case X86::VPCLMULQDQZ256rr: {
2160 unsigned Imm =
MI.getOperand(3).getImm();
2161 unsigned Src1Hi = Imm & 0x01;
2162 unsigned Src2Hi = Imm & 0x10;
2163 auto &WorkingMI = cloneIfNew(
MI);
2164 WorkingMI.getOperand(3).setImm((Src1Hi << 4) | (Src2Hi >> 4));
2168 case X86::VPCMPBZ128rri:
case X86::VPCMPUBZ128rri:
2169 case X86::VPCMPBZ256rri:
case X86::VPCMPUBZ256rri:
2170 case X86::VPCMPBZrri:
case X86::VPCMPUBZrri:
2171 case X86::VPCMPDZ128rri:
case X86::VPCMPUDZ128rri:
2172 case X86::VPCMPDZ256rri:
case X86::VPCMPUDZ256rri:
2173 case X86::VPCMPDZrri:
case X86::VPCMPUDZrri:
2174 case X86::VPCMPQZ128rri:
case X86::VPCMPUQZ128rri:
2175 case X86::VPCMPQZ256rri:
case X86::VPCMPUQZ256rri:
2176 case X86::VPCMPQZrri:
case X86::VPCMPUQZrri:
2177 case X86::VPCMPWZ128rri:
case X86::VPCMPUWZ128rri:
2178 case X86::VPCMPWZ256rri:
case X86::VPCMPUWZ256rri:
2179 case X86::VPCMPWZrri:
case X86::VPCMPUWZrri:
2180 case X86::VPCMPBZ128rrik:
case X86::VPCMPUBZ128rrik:
2181 case X86::VPCMPBZ256rrik:
case X86::VPCMPUBZ256rrik:
2182 case X86::VPCMPBZrrik:
case X86::VPCMPUBZrrik:
2183 case X86::VPCMPDZ128rrik:
case X86::VPCMPUDZ128rrik:
2184 case X86::VPCMPDZ256rrik:
case X86::VPCMPUDZ256rrik:
2185 case X86::VPCMPDZrrik:
case X86::VPCMPUDZrrik:
2186 case X86::VPCMPQZ128rrik:
case X86::VPCMPUQZ128rrik:
2187 case X86::VPCMPQZ256rrik:
case X86::VPCMPUQZ256rrik:
2188 case X86::VPCMPQZrrik:
case X86::VPCMPUQZrrik:
2189 case X86::VPCMPWZ128rrik:
case X86::VPCMPUWZ128rrik:
2190 case X86::VPCMPWZ256rrik:
case X86::VPCMPUWZ256rrik:
2191 case X86::VPCMPWZrrik:
case X86::VPCMPUWZrrik: {
2193 unsigned Imm =
MI.getOperand(
MI.getNumOperands() - 1).getImm() & 0x7;
2195 auto &WorkingMI = cloneIfNew(
MI);
2196 WorkingMI.getOperand(
MI.getNumOperands() - 1).setImm(Imm);
2200 case X86::VPCOMBri:
case X86::VPCOMUBri:
2201 case X86::VPCOMDri:
case X86::VPCOMUDri:
2202 case X86::VPCOMQri:
case X86::VPCOMUQri:
2203 case X86::VPCOMWri:
case X86::VPCOMUWri: {
2205 unsigned Imm =
MI.getOperand(3).getImm() & 0x7;
2207 auto &WorkingMI = cloneIfNew(
MI);
2208 WorkingMI.getOperand(3).setImm(Imm);
2212 case X86::VCMPSDZrr:
2213 case X86::VCMPSSZrr:
2214 case X86::VCMPPDZrri:
2215 case X86::VCMPPSZrri:
2216 case X86::VCMPSHZrr:
2217 case X86::VCMPPHZrri:
2218 case X86::VCMPPHZ128rri:
2219 case X86::VCMPPHZ256rri:
2220 case X86::VCMPPDZ128rri:
2221 case X86::VCMPPSZ128rri:
2222 case X86::VCMPPDZ256rri:
2223 case X86::VCMPPSZ256rri:
2224 case X86::VCMPPDZrrik:
2225 case X86::VCMPPSZrrik:
2226 case X86::VCMPPDZ128rrik:
2227 case X86::VCMPPSZ128rrik:
2228 case X86::VCMPPDZ256rrik:
2229 case X86::VCMPPSZ256rrik: {
2231 MI.getOperand(
MI.getNumExplicitOperands() - 1).getImm() & 0x1f;
2233 auto &WorkingMI = cloneIfNew(
MI);
2234 WorkingMI.getOperand(
MI.getNumExplicitOperands() - 1).setImm(Imm);
2238 case X86::VPERM2F128rr:
2239 case X86::VPERM2I128rr: {
2243 int8_t Imm =
MI.getOperand(3).getImm() & 0xFF;
2244 auto &WorkingMI = cloneIfNew(
MI);
2245 WorkingMI.getOperand(3).setImm(Imm ^ 0x22);
2249 case X86::MOVHLPSrr:
2250 case X86::UNPCKHPDrr:
2251 case X86::VMOVHLPSrr:
2252 case X86::VUNPCKHPDrr:
2253 case X86::VMOVHLPSZrr:
2254 case X86::VUNPCKHPDZ128rr: {
2255 assert(Subtarget.
hasSSE2() &&
"Commuting MOVHLP/UNPCKHPD requires SSE2!");
2257 unsigned Opc =
MI.getOpcode();
2260 case X86::MOVHLPSrr: Opc = X86::UNPCKHPDrr;
break;
2261 case X86::UNPCKHPDrr: Opc = X86::MOVHLPSrr;
break;
2262 case X86::VMOVHLPSrr: Opc = X86::VUNPCKHPDrr;
break;
2263 case X86::VUNPCKHPDrr: Opc = X86::VMOVHLPSrr;
break;
2264 case X86::VMOVHLPSZrr: Opc = X86::VUNPCKHPDZ128rr;
break;
2265 case X86::VUNPCKHPDZ128rr: Opc = X86::VMOVHLPSZrr;
break;
2267 auto &WorkingMI = cloneIfNew(
MI);
2268 WorkingMI.setDesc(
get(Opc));
2272 case X86::CMOV16rr:
case X86::CMOV32rr:
case X86::CMOV64rr: {
2273 auto &WorkingMI = cloneIfNew(
MI);
2274 unsigned OpNo =
MI.getDesc().getNumOperands() - 1;
2280 case X86::VPTERNLOGDZrri:
case X86::VPTERNLOGDZrmi:
2281 case X86::VPTERNLOGDZ128rri:
case X86::VPTERNLOGDZ128rmi:
2282 case X86::VPTERNLOGDZ256rri:
case X86::VPTERNLOGDZ256rmi:
2283 case X86::VPTERNLOGQZrri:
case X86::VPTERNLOGQZrmi:
2284 case X86::VPTERNLOGQZ128rri:
case X86::VPTERNLOGQZ128rmi:
2285 case X86::VPTERNLOGQZ256rri:
case X86::VPTERNLOGQZ256rmi:
2286 case X86::VPTERNLOGDZrrik:
2287 case X86::VPTERNLOGDZ128rrik:
2288 case X86::VPTERNLOGDZ256rrik:
2289 case X86::VPTERNLOGQZrrik:
2290 case X86::VPTERNLOGQZ128rrik:
2291 case X86::VPTERNLOGQZ256rrik:
2292 case X86::VPTERNLOGDZrrikz:
case X86::VPTERNLOGDZrmikz:
2293 case X86::VPTERNLOGDZ128rrikz:
case X86::VPTERNLOGDZ128rmikz:
2294 case X86::VPTERNLOGDZ256rrikz:
case X86::VPTERNLOGDZ256rmikz:
2295 case X86::VPTERNLOGQZrrikz:
case X86::VPTERNLOGQZrmikz:
2296 case X86::VPTERNLOGQZ128rrikz:
case X86::VPTERNLOGQZ128rmikz:
2297 case X86::VPTERNLOGQZ256rrikz:
case X86::VPTERNLOGQZ256rmikz:
2298 case X86::VPTERNLOGDZ128rmbi:
2299 case X86::VPTERNLOGDZ256rmbi:
2300 case X86::VPTERNLOGDZrmbi:
2301 case X86::VPTERNLOGQZ128rmbi:
2302 case X86::VPTERNLOGQZ256rmbi:
2303 case X86::VPTERNLOGQZrmbi:
2304 case X86::VPTERNLOGDZ128rmbikz:
2305 case X86::VPTERNLOGDZ256rmbikz:
2306 case X86::VPTERNLOGDZrmbikz:
2307 case X86::VPTERNLOGQZ128rmbikz:
2308 case X86::VPTERNLOGQZ256rmbikz:
2309 case X86::VPTERNLOGQZrmbikz: {
2310 auto &WorkingMI = cloneIfNew(
MI);
2318 auto &WorkingMI = cloneIfNew(
MI);
2319 WorkingMI.setDesc(
get(Opc));
2325 MI.getDesc().TSFlags);
2329 auto &WorkingMI = cloneIfNew(
MI);
2330 WorkingMI.setDesc(
get(Opc));
2341X86InstrInfo::findThreeSrcCommutedOpIndices(
const MachineInstr &
MI,
2342 unsigned &SrcOpIdx1,
2343 unsigned &SrcOpIdx2,
2344 bool IsIntrinsic)
const {
2347 unsigned FirstCommutableVecOp = 1;
2348 unsigned LastCommutableVecOp = 3;
2349 unsigned KMaskOp = -1U;
2372 FirstCommutableVecOp = 3;
2374 LastCommutableVecOp++;
2375 }
else if (IsIntrinsic) {
2378 FirstCommutableVecOp = 2;
2381 if (
isMem(
MI, LastCommutableVecOp))
2382 LastCommutableVecOp--;
2387 if (SrcOpIdx1 != CommuteAnyOperandIndex &&
2388 (SrcOpIdx1 < FirstCommutableVecOp || SrcOpIdx1 > LastCommutableVecOp ||
2389 SrcOpIdx1 == KMaskOp))
2391 if (SrcOpIdx2 != CommuteAnyOperandIndex &&
2392 (SrcOpIdx2 < FirstCommutableVecOp || SrcOpIdx2 > LastCommutableVecOp ||
2393 SrcOpIdx2 == KMaskOp))
2398 if (SrcOpIdx1 == CommuteAnyOperandIndex ||
2399 SrcOpIdx2 == CommuteAnyOperandIndex) {
2400 unsigned CommutableOpIdx2 = SrcOpIdx2;
2404 if (SrcOpIdx1 == SrcOpIdx2)
2407 CommutableOpIdx2 = LastCommutableVecOp;
2408 else if (SrcOpIdx2 == CommuteAnyOperandIndex)
2410 CommutableOpIdx2 = SrcOpIdx1;
2414 Register Op2Reg =
MI.getOperand(CommutableOpIdx2).getReg();
2416 unsigned CommutableOpIdx1;
2417 for (CommutableOpIdx1 = LastCommutableVecOp;
2418 CommutableOpIdx1 >= FirstCommutableVecOp; CommutableOpIdx1--) {
2420 if (CommutableOpIdx1 == KMaskOp)
2426 if (Op2Reg !=
MI.getOperand(CommutableOpIdx1).getReg())
2431 if (CommutableOpIdx1 < FirstCommutableVecOp)
2436 if (!fixCommutedOpIndices(SrcOpIdx1, SrcOpIdx2,
2437 CommutableOpIdx1, CommutableOpIdx2))
2445 unsigned &SrcOpIdx1,
2446 unsigned &SrcOpIdx2)
const {
2451 switch (
MI.getOpcode()) {
2458 case X86::VCMPPDrri:
2459 case X86::VCMPPSrri:
2460 case X86::VCMPPDYrri:
2461 case X86::VCMPPSYrri:
2462 case X86::VCMPSDZrr:
2463 case X86::VCMPSSZrr:
2464 case X86::VCMPPDZrri:
2465 case X86::VCMPPSZrri:
2466 case X86::VCMPSHZrr:
2467 case X86::VCMPPHZrri:
2468 case X86::VCMPPHZ128rri:
2469 case X86::VCMPPHZ256rri:
2470 case X86::VCMPPDZ128rri:
2471 case X86::VCMPPSZ128rri:
2472 case X86::VCMPPDZ256rri:
2473 case X86::VCMPPSZ256rri:
2474 case X86::VCMPPDZrrik:
2475 case X86::VCMPPSZrrik:
2476 case X86::VCMPPDZ128rrik:
2477 case X86::VCMPPSZ128rrik:
2478 case X86::VCMPPDZ256rrik:
2479 case X86::VCMPPSZ256rrik: {
2484 unsigned Imm =
MI.getOperand(3 + OpOffset).getImm() & 0x7;
2501 return fixCommutedOpIndices(SrcOpIdx1, SrcOpIdx2, 1 + OpOffset,
2511 case X86::SHUFPDrri:
2513 if (
MI.getOperand(3).getImm() == 0x02)
2516 case X86::MOVHLPSrr:
2517 case X86::UNPCKHPDrr:
2518 case X86::VMOVHLPSrr:
2519 case X86::VUNPCKHPDrr:
2520 case X86::VMOVHLPSZrr:
2521 case X86::VUNPCKHPDZ128rr:
2525 case X86::VPTERNLOGDZrri:
case X86::VPTERNLOGDZrmi:
2526 case X86::VPTERNLOGDZ128rri:
case X86::VPTERNLOGDZ128rmi:
2527 case X86::VPTERNLOGDZ256rri:
case X86::VPTERNLOGDZ256rmi:
2528 case X86::VPTERNLOGQZrri:
case X86::VPTERNLOGQZrmi:
2529 case X86::VPTERNLOGQZ128rri:
case X86::VPTERNLOGQZ128rmi:
2530 case X86::VPTERNLOGQZ256rri:
case X86::VPTERNLOGQZ256rmi:
2531 case X86::VPTERNLOGDZrrik:
2532 case X86::VPTERNLOGDZ128rrik:
2533 case X86::VPTERNLOGDZ256rrik:
2534 case X86::VPTERNLOGQZrrik:
2535 case X86::VPTERNLOGQZ128rrik:
2536 case X86::VPTERNLOGQZ256rrik:
2537 case X86::VPTERNLOGDZrrikz:
case X86::VPTERNLOGDZrmikz:
2538 case X86::VPTERNLOGDZ128rrikz:
case X86::VPTERNLOGDZ128rmikz:
2539 case X86::VPTERNLOGDZ256rrikz:
case X86::VPTERNLOGDZ256rmikz:
2540 case X86::VPTERNLOGQZrrikz:
case X86::VPTERNLOGQZrmikz:
2541 case X86::VPTERNLOGQZ128rrikz:
case X86::VPTERNLOGQZ128rmikz:
2542 case X86::VPTERNLOGQZ256rrikz:
case X86::VPTERNLOGQZ256rmikz:
2543 case X86::VPTERNLOGDZ128rmbi:
2544 case X86::VPTERNLOGDZ256rmbi:
2545 case X86::VPTERNLOGDZrmbi:
2546 case X86::VPTERNLOGQZ128rmbi:
2547 case X86::VPTERNLOGQZ256rmbi:
2548 case X86::VPTERNLOGQZrmbi:
2549 case X86::VPTERNLOGDZ128rmbikz:
2550 case X86::VPTERNLOGDZ256rmbikz:
2551 case X86::VPTERNLOGDZrmbikz:
2552 case X86::VPTERNLOGQZ128rmbikz:
2553 case X86::VPTERNLOGQZ256rmbikz:
2554 case X86::VPTERNLOGQZrmbikz:
2555 return findThreeSrcCommutedOpIndices(
MI, SrcOpIdx1, SrcOpIdx2);
2556 case X86::VPDPWSSDYrr:
2557 case X86::VPDPWSSDrr:
2558 case X86::VPDPWSSDSYrr:
2559 case X86::VPDPWSSDSrr:
2560 case X86::VPDPBSSDSrr:
2561 case X86::VPDPBSSDSYrr:
2562 case X86::VPDPBSSDrr:
2563 case X86::VPDPBSSDYrr:
2564 case X86::VPDPBUUDSrr:
2565 case X86::VPDPBUUDSYrr:
2566 case X86::VPDPBUUDrr:
2567 case X86::VPDPBUUDYrr:
2568 case X86::VPDPWSSDZ128r:
2569 case X86::VPDPWSSDZ128rk:
2570 case X86::VPDPWSSDZ128rkz:
2571 case X86::VPDPWSSDZ256r:
2572 case X86::VPDPWSSDZ256rk:
2573 case X86::VPDPWSSDZ256rkz:
2574 case X86::VPDPWSSDZr:
2575 case X86::VPDPWSSDZrk:
2576 case X86::VPDPWSSDZrkz:
2577 case X86::VPDPWSSDSZ128r:
2578 case X86::VPDPWSSDSZ128rk:
2579 case X86::VPDPWSSDSZ128rkz:
2580 case X86::VPDPWSSDSZ256r:
2581 case X86::VPDPWSSDSZ256rk:
2582 case X86::VPDPWSSDSZ256rkz:
2583 case X86::VPDPWSSDSZr:
2584 case X86::VPDPWSSDSZrk:
2585 case X86::VPDPWSSDSZrkz:
2586 case X86::VPMADD52HUQrr:
2587 case X86::VPMADD52HUQYrr:
2588 case X86::VPMADD52HUQZ128r:
2589 case X86::VPMADD52HUQZ128rk:
2590 case X86::VPMADD52HUQZ128rkz:
2591 case X86::VPMADD52HUQZ256r:
2592 case X86::VPMADD52HUQZ256rk:
2593 case X86::VPMADD52HUQZ256rkz:
2594 case X86::VPMADD52HUQZr:
2595 case X86::VPMADD52HUQZrk:
2596 case X86::VPMADD52HUQZrkz:
2597 case X86::VPMADD52LUQrr:
2598 case X86::VPMADD52LUQYrr:
2599 case X86::VPMADD52LUQZ128r:
2600 case X86::VPMADD52LUQZ128rk:
2601 case X86::VPMADD52LUQZ128rkz:
2602 case X86::VPMADD52LUQZ256r:
2603 case X86::VPMADD52LUQZ256rk:
2604 case X86::VPMADD52LUQZ256rkz:
2605 case X86::VPMADD52LUQZr:
2606 case X86::VPMADD52LUQZrk:
2607 case X86::VPMADD52LUQZrkz:
2608 case X86::VFMADDCPHZr:
2609 case X86::VFMADDCPHZrk:
2610 case X86::VFMADDCPHZrkz:
2611 case X86::VFMADDCPHZ128r:
2612 case X86::VFMADDCPHZ128rk:
2613 case X86::VFMADDCPHZ128rkz:
2614 case X86::VFMADDCPHZ256r:
2615 case X86::VFMADDCPHZ256rk:
2616 case X86::VFMADDCPHZ256rkz:
2617 case X86::VFMADDCSHZr:
2618 case X86::VFMADDCSHZrk:
2619 case X86::VFMADDCSHZrkz: {
2620 unsigned CommutableOpIdx1 = 2;
2621 unsigned CommutableOpIdx2 = 3;
2627 if (!fixCommutedOpIndices(SrcOpIdx1, SrcOpIdx2,
2628 CommutableOpIdx1, CommutableOpIdx2))
2630 if (!
MI.getOperand(SrcOpIdx1).isReg() ||
2631 !
MI.getOperand(SrcOpIdx2).isReg())
2639 MI.getDesc().TSFlags);
2641 return findThreeSrcCommutedOpIndices(
MI, SrcOpIdx1, SrcOpIdx2,
2648 unsigned CommutableOpIdx1 = Desc.
getNumDefs() + 1;
2649 unsigned CommutableOpIdx2 = Desc.
getNumDefs() + 2;
2652 if ((
MI.getDesc().getOperandConstraint(Desc.
getNumDefs(),
2667 if (!fixCommutedOpIndices(SrcOpIdx1, SrcOpIdx2,
2668 CommutableOpIdx1, CommutableOpIdx2))
2671 if (!
MI.getOperand(SrcOpIdx1).isReg() ||
2672 !
MI.getOperand(SrcOpIdx2).isReg())
2684 unsigned Opcode =
MI->getOpcode();
2685 if (Opcode != X86::LEA32r && Opcode != X86::LEA64r &&
2686 Opcode != X86::LEA64_32r)
2708 unsigned Opcode =
MI.getOpcode();
2709 if (Opcode != X86::ADD32rr && Opcode != X86::ADD64rr)
2737 if (!(X86::isJCC(Opcode) || X86::isSETCC(Opcode) || X86::isCMOVCC(Opcode)))
2812std::pair<X86::CondCode, bool>
2815 bool NeedSwap =
false;
2816 switch (Predicate) {
2847 return std::make_pair(
CC, NeedSwap);
2854 case 2:
return HasMemoryOperand ? X86::CMOV16rm : X86::CMOV16rr;
2855 case 4:
return HasMemoryOperand ? X86::CMOV32rm : X86::CMOV32rr;
2856 case 8:
return HasMemoryOperand ? X86::CMOV64rm : X86::CMOV64rr;
2881 case 0x01: Imm = 0x06;
break;
2882 case 0x02: Imm = 0x05;
break;
2883 case 0x05: Imm = 0x02;
break;
2884 case 0x06: Imm = 0x01;
break;
2899 case 0x00: Imm = 0x02;
break;
2900 case 0x01: Imm = 0x03;
break;
2901 case 0x02: Imm = 0x00;
break;
2902 case 0x03: Imm = 0x01;
break;
2916 switch (Imm & 0x3) {
2918 case 0x00:
case 0x03:
2921 case 0x01:
case 0x02:
2932 return (Reg == X86::FPCW || Reg == X86::FPSW ||
2933 (Reg >= X86::ST0 && Reg <= X86::ST7));
2948 switch (
MI.getOpcode()) {
2949 case X86::TCRETURNdi:
2950 case X86::TCRETURNri:
2951 case X86::TCRETURNmi:
2952 case X86::TCRETURNdi64:
2953 case X86::TCRETURNri64:
2954 case X86::TCRETURNmi64:
2973 if (Symbol.equals(
"__x86_indirect_thunk_r11"))
2978 if (TailCall.getOpcode() != X86::TCRETURNdi &&
2979 TailCall.getOpcode() != X86::TCRETURNdi64) {
2997 TailCall.getOperand(1).getImm() != 0) {
3013 if (
I->isDebugInstr())
3016 assert(0 &&
"Can't find the branch to replace!");
3020 if (
CC != BranchCond[0].getImm())
3026 unsigned Opc = TailCall.getOpcode() == X86::TCRETURNdi ? X86::TCRETURNdicc
3027 : X86::TCRETURNdi64cc;
3041 for (
const auto &
C : Clobbers) {
3046 I->eraseFromParent();
3060 if (Succ->isEHPad() || (Succ ==
TBB && FallthroughBB))
3063 if (FallthroughBB && FallthroughBB !=
TBB)
3065 FallthroughBB = Succ;
3067 return FallthroughBB;
3070bool X86InstrInfo::AnalyzeBranchImpl(
3081 if (
I->isDebugInstr())
3086 if (!isUnpredicatedTerminator(*
I))
3095 if (
I->getOpcode() == X86::JMP_1) {
3099 TBB =
I->getOperand(0).getMBB();
3112 I->eraseFromParent();
3114 UnCondBrIter =
MBB.
end();
3119 TBB =
I->getOperand(0).getMBB();
3130 if (
I->findRegisterUseOperand(X86::EFLAGS)->isUndef())
3136 TBB =
I->getOperand(0).getMBB();
3150 auto NewTBB =
I->getOperand(0).getMBB();
3151 if (OldBranchCode == BranchCode &&
TBB == NewTBB)
3157 if (
TBB == NewTBB &&
3190 Cond[0].setImm(BranchCode);
3201 bool AllowModify)
const {
3203 return AnalyzeBranchImpl(
MBB,
TBB, FBB,
Cond, CondBranches, AllowModify);
3207 MachineBranchPredicate &MBP,
3208 bool AllowModify)
const {
3209 using namespace std::placeholders;
3213 if (AnalyzeBranchImpl(
MBB, MBP.TrueDest, MBP.FalseDest,
Cond, CondBranches,
3217 if (
Cond.size() != 1)
3220 assert(MBP.TrueDest &&
"expected!");
3228 bool SingleUseCondition =
true;
3231 if (
MI.modifiesRegister(X86::EFLAGS,
TRI)) {
3236 if (
MI.readsRegister(X86::EFLAGS,
TRI))
3237 SingleUseCondition =
false;
3243 if (SingleUseCondition) {
3245 if (Succ->isLiveIn(X86::EFLAGS))
3246 SingleUseCondition =
false;
3249 MBP.ConditionDef = ConditionDef;
3250 MBP.SingleUseCondition = SingleUseCondition;
3257 const unsigned TestOpcode =
3258 Subtarget.is64Bit() ? X86::TEST64rr : X86::TEST32rr;
3260 if (ConditionDef->
getOpcode() == TestOpcode &&
3267 ? MachineBranchPredicate::PRED_NE
3268 : MachineBranchPredicate::PRED_EQ;
3276 int *BytesRemoved)
const {
3277 assert(!BytesRemoved &&
"code size not handled");
3284 if (
I->isDebugInstr())
3286 if (
I->getOpcode() != X86::JMP_1 &&
3290 I->eraseFromParent();
3303 int *BytesAdded)
const {
3305 assert(
TBB &&
"insertBranch must not be told to insert a fallthrough");
3307 "X86 branch conditions have one component!");
3308 assert(!BytesAdded &&
"code size not handled");
3312 assert(!FBB &&
"Unconditional branch with multiple successors!");
3318 bool FallThru = FBB ==
nullptr;
3333 if (FBB ==
nullptr) {
3335 assert(FBB &&
"MBB cannot be the last block in function when the false "
3336 "body is a fall-through.");
3360 Register FalseReg,
int &CondCycles,
3361 int &TrueCycles,
int &FalseCycles)
const {
3365 if (
Cond.size() != 1)
3374 RI.getCommonSubClass(
MRI.getRegClass(TrueReg),
MRI.getRegClass(FalseReg));
3379 if (X86::GR16RegClass.hasSubClassEq(RC) ||
3380 X86::GR32RegClass.hasSubClassEq(RC) ||
3381 X86::GR64RegClass.hasSubClassEq(RC)) {
3402 assert(
Cond.size() == 1 &&
"Invalid Cond array");
3413 return X86::GR8_ABCD_HRegClass.contains(Reg);
3419 bool HasAVX = Subtarget.
hasAVX();
3426 if (X86::VK16RegClass.
contains(SrcReg)) {
3427 if (X86::GR64RegClass.
contains(DestReg)) {
3428 assert(Subtarget.hasBWI());
3429 return X86::KMOVQrk;
3431 if (X86::GR32RegClass.
contains(DestReg))
3432 return Subtarget.hasBWI() ? X86::KMOVDrk : X86::KMOVWrk;
3439 if (X86::VK16RegClass.
contains(DestReg)) {
3440 if (X86::GR64RegClass.
contains(SrcReg)) {
3441 assert(Subtarget.hasBWI());
3442 return X86::KMOVQkr;
3444 if (X86::GR32RegClass.
contains(SrcReg))
3445 return Subtarget.hasBWI() ? X86::KMOVDkr : X86::KMOVWkr;
3454 if (X86::GR64RegClass.
contains(DestReg)) {
3455 if (X86::VR128XRegClass.
contains(SrcReg))
3457 return HasAVX512 ? X86::VMOVPQIto64Zrr :
3458 HasAVX ? X86::VMOVPQIto64rr :
3460 if (X86::VR64RegClass.
contains(SrcReg))
3462 return X86::MMX_MOVD64from64rr;
3463 }
else if (X86::GR64RegClass.
contains(SrcReg)) {
3465 if (X86::VR128XRegClass.
contains(DestReg))
3466 return HasAVX512 ? X86::VMOV64toPQIZrr :
3467 HasAVX ? X86::VMOV64toPQIrr :
3470 if (X86::VR64RegClass.
contains(DestReg))
3471 return X86::MMX_MOVD64to64rr;
3477 if (X86::GR32RegClass.
contains(DestReg) &&
3478 X86::VR128XRegClass.
contains(SrcReg))
3480 return HasAVX512 ? X86::VMOVPDI2DIZrr :
3481 HasAVX ? X86::VMOVPDI2DIrr :
3484 if (X86::VR128XRegClass.
contains(DestReg) &&
3485 X86::GR32RegClass.
contains(SrcReg))
3487 return HasAVX512 ? X86::VMOVDI2PDIZrr :
3488 HasAVX ? X86::VMOVDI2PDIrr :
3498 bool HasAVX = Subtarget.
hasAVX();
3499 bool HasVLX = Subtarget.hasVLX();
3501 if (X86::GR64RegClass.
contains(DestReg, SrcReg))
3503 else if (X86::GR32RegClass.
contains(DestReg, SrcReg))
3505 else if (X86::GR16RegClass.
contains(DestReg, SrcReg))
3507 else if (X86::GR8RegClass.
contains(DestReg, SrcReg)) {
3511 Subtarget.is64Bit()) {
3512 Opc = X86::MOV8rr_NOREX;
3515 "8-bit H register can not be copied outside GR8_NOREX");
3519 else if (X86::VR64RegClass.
contains(DestReg, SrcReg))
3520 Opc = X86::MMX_MOVQ64rr;
3521 else if (X86::VR128XRegClass.
contains(DestReg, SrcReg)) {
3523 Opc = X86::VMOVAPSZ128rr;
3524 else if (X86::VR128RegClass.
contains(DestReg, SrcReg))
3525 Opc = HasAVX ? X86::VMOVAPSrr : X86::MOVAPSrr;
3529 Opc = X86::VMOVAPSZrr;
3531 DestReg =
TRI->getMatchingSuperReg(DestReg, X86::sub_xmm,
3532 &X86::VR512RegClass);
3533 SrcReg =
TRI->getMatchingSuperReg(SrcReg, X86::sub_xmm,
3534 &X86::VR512RegClass);
3536 }
else if (X86::VR256XRegClass.
contains(DestReg, SrcReg)) {
3538 Opc = X86::VMOVAPSZ256rr;
3539 else if (X86::VR256RegClass.
contains(DestReg, SrcReg))
3540 Opc = X86::VMOVAPSYrr;
3544 Opc = X86::VMOVAPSZrr;
3546 DestReg =
TRI->getMatchingSuperReg(DestReg, X86::sub_ymm,
3547 &X86::VR512RegClass);
3548 SrcReg =
TRI->getMatchingSuperReg(SrcReg, X86::sub_ymm,
3549 &X86::VR512RegClass);
3551 }
else if (X86::VR512RegClass.
contains(DestReg, SrcReg))
3552 Opc = X86::VMOVAPSZrr;
3554 else if (X86::VK16RegClass.
contains(DestReg, SrcReg))
3555 Opc = Subtarget.hasBWI() ? X86::KMOVQkk : X86::KMOVWkk;
3565 if (SrcReg == X86::EFLAGS || DestReg == X86::EFLAGS) {
3573 LLVM_DEBUG(
dbgs() <<
"Cannot copy " << RI.getName(SrcReg) <<
" to "
3574 << RI.getName(DestReg) <<
'\n');
3578std::optional<DestSourcePair>
3582 return std::nullopt;
3587 return Load ? X86::VMOVSHZrm_alt : X86::VMOVSHZmr;
3590 : STI.
hasAVX() ? X86::VMOVSSrm
3594 : STI.
hasAVX() ? X86::VMOVSSmr
3600 bool IsStackAligned,
3602 bool HasAVX = STI.
hasAVX();
3604 bool HasVLX = STI.hasVLX();
3610 assert(X86::GR8RegClass.hasSubClassEq(RC) &&
"Unknown 1-byte regclass");
3614 if (
isHReg(Reg) || X86::GR8_ABCD_HRegClass.hasSubClassEq(RC))
3615 return Load ? X86::MOV8rm_NOREX : X86::MOV8mr_NOREX;
3616 return Load ? X86::MOV8rm : X86::MOV8mr;
3618 if (X86::VK16RegClass.hasSubClassEq(RC))
3619 return Load ? X86::KMOVWkm : X86::KMOVWmk;
3620 assert(X86::GR16RegClass.hasSubClassEq(RC) &&
"Unknown 2-byte regclass");
3621 return Load ? X86::MOV16rm : X86::MOV16mr;
3623 if (X86::GR32RegClass.hasSubClassEq(RC))
3624 return Load ? X86::MOV32rm : X86::MOV32mr;
3625 if (X86::FR32XRegClass.hasSubClassEq(RC))
3627 (HasAVX512 ? X86::VMOVSSZrm_alt :
3628 HasAVX ? X86::VMOVSSrm_alt :
3630 (HasAVX512 ? X86::VMOVSSZmr :
3631 HasAVX ? X86::VMOVSSmr :
3633 if (X86::RFP32RegClass.hasSubClassEq(RC))
3634 return Load ? X86::LD_Fp32m : X86::ST_Fp32m;
3635 if (X86::VK32RegClass.hasSubClassEq(RC)) {
3636 assert(STI.hasBWI() &&
"KMOVD requires BWI");
3637 return Load ? X86::KMOVDkm : X86::KMOVDmk;
3641 if (X86::VK1PAIRRegClass.hasSubClassEq(RC) ||
3642 X86::VK2PAIRRegClass.hasSubClassEq(RC) ||
3643 X86::VK4PAIRRegClass.hasSubClassEq(RC) ||
3644 X86::VK8PAIRRegClass.hasSubClassEq(RC) ||
3645 X86::VK16PAIRRegClass.hasSubClassEq(RC))
3646 return Load ? X86::MASKPAIR16LOAD : X86::MASKPAIR16STORE;
3647 if (X86::FR16RegClass.hasSubClassEq(RC) ||
3648 X86::FR16XRegClass.hasSubClassEq(RC))
3652 if (X86::GR64RegClass.hasSubClassEq(RC))
3653 return Load ? X86::MOV64rm : X86::MOV64mr;
3654 if (X86::FR64XRegClass.hasSubClassEq(RC))
3656 (HasAVX512 ? X86::VMOVSDZrm_alt :
3657 HasAVX ? X86::VMOVSDrm_alt :
3659 (HasAVX512 ? X86::VMOVSDZmr :
3660 HasAVX ? X86::VMOVSDmr :
3662 if (X86::VR64RegClass.hasSubClassEq(RC))
3663 return Load ? X86::MMX_MOVQ64rm : X86::MMX_MOVQ64mr;
3664 if (X86::RFP64RegClass.hasSubClassEq(RC))
3665 return Load ? X86::LD_Fp64m : X86::ST_Fp64m;
3666 if (X86::VK64RegClass.hasSubClassEq(RC)) {
3667 assert(STI.hasBWI() &&
"KMOVQ requires BWI");
3668 return Load ? X86::KMOVQkm : X86::KMOVQmk;
3672 assert(X86::RFP80RegClass.hasSubClassEq(RC) &&
"Unknown 10-byte regclass");
3673 return Load ? X86::LD_Fp80m : X86::ST_FpP80m;
3675 if (X86::VR128XRegClass.hasSubClassEq(RC)) {
3679 (HasVLX ? X86::VMOVAPSZ128rm :
3680 HasAVX512 ? X86::VMOVAPSZ128rm_NOVLX :
3681 HasAVX ? X86::VMOVAPSrm :
3683 (HasVLX ? X86::VMOVAPSZ128mr :
3684 HasAVX512 ? X86::VMOVAPSZ128mr_NOVLX :
3685 HasAVX ? X86::VMOVAPSmr :
3689 (HasVLX ? X86::VMOVUPSZ128rm :
3690 HasAVX512 ? X86::VMOVUPSZ128rm_NOVLX :
3691 HasAVX ? X86::VMOVUPSrm :
3693 (HasVLX ? X86::VMOVUPSZ128mr :
3694 HasAVX512 ? X86::VMOVUPSZ128mr_NOVLX :
3695 HasAVX ? X86::VMOVUPSmr :
3701 assert(X86::VR256XRegClass.hasSubClassEq(RC) &&
"Unknown 32-byte regclass");
3705 (HasVLX ? X86::VMOVAPSZ256rm :
3706 HasAVX512 ? X86::VMOVAPSZ256rm_NOVLX :
3708 (HasVLX ? X86::VMOVAPSZ256mr :
3709 HasAVX512 ? X86::VMOVAPSZ256mr_NOVLX :
3713 (HasVLX ? X86::VMOVUPSZ256rm :
3714 HasAVX512 ? X86::VMOVUPSZ256rm_NOVLX :
3716 (HasVLX ? X86::VMOVUPSZ256mr :
3717 HasAVX512 ? X86::VMOVUPSZ256mr_NOVLX :
3720 assert(X86::VR512RegClass.hasSubClassEq(RC) &&
"Unknown 64-byte regclass");
3723 return Load ? X86::VMOVAPSZrm : X86::VMOVAPSZmr;
3725 return Load ? X86::VMOVUPSZrm : X86::VMOVUPSZmr;
3727 assert(X86::TILERegClass.hasSubClassEq(RC) &&
"Unknown 1024-byte regclass");
3728 assert(STI.hasAMXTILE() &&
"Using 8*1024-bit register requires AMX-TILE");
3729 return Load ? X86::TILELOADD : X86::TILESTORED;
3733std::optional<ExtAddrMode>
3738 if (MemRefBegin < 0)
3739 return std::nullopt;
3744 if (!BaseOp.isReg())
3745 return std::nullopt;
3749 if (!DispMO.
isImm())
3750 return std::nullopt;
3776 ErrInfo =
"Scale factor in address must be 1, 2, 4 or 8";
3781 ErrInfo =
"Displacement in address must fit into 32-bit signed "
3791 int64_t &ImmVal)
const {
3792 if (
MI.getOpcode() != X86::MOV32ri &&
MI.getOpcode() != X86::MOV64ri)
3795 if (!
MI.getOperand(1).isImm() ||
MI.getOperand(0).getReg() != Reg)
3797 ImmVal =
MI.getOperand(1).getImm();
3804 if (!
MI->modifiesRegister(NullValueReg,
TRI))
3806 switch (
MI->getOpcode()) {
3813 assert(
MI->getOperand(0).isDef() &&
MI->getOperand(1).isUse() &&
3814 "expected for shift opcode!");
3815 return MI->getOperand(0).getReg() == NullValueReg &&
3816 MI->getOperand(1).getReg() == NullValueReg;
3821 return TRI->isSubRegisterEq(NullValueReg, MO.getReg());
3831 int64_t &
Offset,
bool &OffsetIsScalable,
unsigned &Width,
3835 if (MemRefBegin < 0)
3842 if (!BaseOp->
isReg())
3855 if (!DispMO.
isImm())
3860 if (!BaseOp->
isReg())
3863 OffsetIsScalable =
false;
3868 !
MemOp.memoperands_empty() ?
MemOp.memoperands().front()->getSize() : 0;
3875 bool IsStackAligned,
3890 case X86::TILELOADD:
3891 case X86::TILESTORED:
3898 unsigned Opc,
Register Reg,
int FrameIdx,
3899 bool isKill)
const {
3903 case X86::TILESTORED: {
3916 case X86::TILELOADD: {
3938 "Stack slot too small for store");
3940 unsigned Alignment = std::max<uint32_t>(
TRI->getSpillSize(*RC), 16);
3962 "Load size exceeds stack slot");
3963 unsigned Alignment = std::max<uint32_t>(
TRI->getSpillSize(*RC), 16);
3977 Register &SrcReg2, int64_t &CmpMask,
3978 int64_t &CmpValue)
const {
3979 switch (
MI.getOpcode()) {
3981 case X86::CMP64ri32:
3988 SrcReg =
MI.getOperand(0).getReg();
3990 if (
MI.getOperand(1).isImm()) {
3992 CmpValue =
MI.getOperand(1).getImm();
3994 CmpMask = CmpValue = 0;
4002 SrcReg =
MI.getOperand(1).getReg();
4011 SrcReg =
MI.getOperand(1).getReg();
4012 SrcReg2 =
MI.getOperand(2).getReg();
4016 case X86::SUB64ri32:
4023 SrcReg =
MI.getOperand(1).getReg();
4025 if (
MI.getOperand(2).isImm()) {
4027 CmpValue =
MI.getOperand(2).getImm();
4029 CmpMask = CmpValue = 0;
4036 SrcReg =
MI.getOperand(0).getReg();
4037 SrcReg2 =
MI.getOperand(1).getReg();
4045 SrcReg =
MI.getOperand(0).getReg();
4046 if (
MI.getOperand(1).getReg() != SrcReg)
4057bool X86InstrInfo::isRedundantFlagInstr(
const MachineInstr &FlagI,
4059 int64_t ImmMask, int64_t ImmValue,
4061 int64_t *ImmDelta)
const {
4076 OIMask != ImmMask || OIValue != ImmValue)
4078 if (SrcReg == OISrcReg && SrcReg2 == OISrcReg2) {
4082 if (SrcReg == OISrcReg2 && SrcReg2 == OISrcReg) {
4088 case X86::CMP64ri32:
4095 case X86::SUB64ri32:
4105 case X86::TEST8rr: {
4112 SrcReg == OISrcReg && ImmMask == OIMask) {
4113 if (OIValue == ImmValue) {
4116 }
else if (
static_cast<uint64_t>(ImmValue) ==
4117 static_cast<uint64_t>(OIValue) - 1) {
4120 }
else if (
static_cast<uint64_t>(ImmValue) ==
4121 static_cast<uint64_t>(OIValue) + 1) {
4139 bool &ClearsOverflowFlag) {
4141 ClearsOverflowFlag =
false;
4147 if (
MI.getOpcode() == X86::ADD64rm ||
MI.getOpcode() == X86::ADD32rm) {
4148 unsigned Flags =
MI.getOperand(5).getTargetFlags();
4154 switch (
MI.getOpcode()) {
4155 default:
return false;
4159 case X86::SAR8ri:
case X86::SAR16ri:
case X86::SAR32ri:
case X86::SAR64ri:
4160 case X86::SHR8ri:
case X86::SHR16ri:
case X86::SHR32ri:
case X86::SHR64ri:
4165 case X86::SHL8ri:
case X86::SHL16ri:
case X86::SHL32ri:
case X86::SHL64ri:{
4171 case X86::SHRD16rri8:
case X86::SHRD32rri8:
case X86::SHRD64rri8:
4172 case X86::SHLD16rri8:
case X86::SHLD32rri8:
case X86::SHLD64rri8:
4175 case X86::SUB64ri32:
case X86::SUB64ri8:
case X86::SUB32ri:
4176 case X86::SUB32ri8:
case X86::SUB16ri:
case X86::SUB16ri8:
4177 case X86::SUB8ri:
case X86::SUB64rr:
case X86::SUB32rr:
4178 case X86::SUB16rr:
case X86::SUB8rr:
case X86::SUB64rm:
4179 case X86::SUB32rm:
case X86::SUB16rm:
case X86::SUB8rm:
4180 case X86::DEC64r:
case X86::DEC32r:
case X86::DEC16r:
case X86::DEC8r:
4181 case X86::ADD64ri32:
case X86::ADD64ri8:
case X86::ADD32ri:
4182 case X86::ADD32ri8:
case X86::ADD16ri:
case X86::ADD16ri8:
4183 case X86::ADD8ri:
case X86::ADD64rr:
case X86::ADD32rr:
4184 case X86::ADD16rr:
case X86::ADD8rr:
case X86::ADD64rm:
4185 case X86::ADD32rm:
case X86::ADD16rm:
case X86::ADD8rm:
4186 case X86::INC64r:
case X86::INC32r:
case X86::INC16r:
case X86::INC8r:
4187 case X86::ADC64ri32:
case X86::ADC64ri8:
case X86::ADC32ri:
4188 case X86::ADC32ri8:
case X86::ADC16ri:
case X86::ADC16ri8:
4189 case X86::ADC8ri:
case X86::ADC64rr:
case X86::ADC32rr:
4190 case X86::ADC16rr:
case X86::ADC8rr:
case X86::ADC64rm:
4191 case X86::ADC32rm:
case X86::ADC16rm:
case X86::ADC8rm:
4192 case X86::SBB64ri32:
case X86::SBB64ri8:
case X86::SBB32ri:
4193 case X86::SBB32ri8:
case X86::SBB16ri:
case X86::SBB16ri8:
4194 case X86::SBB8ri:
case X86::SBB64rr:
case X86::SBB32rr:
4195 case X86::SBB16rr:
case X86::SBB8rr:
case X86::SBB64rm:
4196 case X86::SBB32rm:
case X86::SBB16rm:
case X86::SBB8rm:
4197 case X86::NEG8r:
case X86::NEG16r:
case X86::NEG32r:
case X86::NEG64r:
4198 case X86::SAR8r1:
case X86::SAR16r1:
case X86::SAR32r1:
case X86::SAR64r1:
4199 case X86::SHR8r1:
case X86::SHR16r1:
case X86::SHR32r1:
case X86::SHR64r1:
4200 case X86::SHL8r1:
case X86::SHL16r1:
case X86::SHL32r1:
case X86::SHL64r1:
4201 case X86::LZCNT16rr:
case X86::LZCNT16rm:
4202 case X86::LZCNT32rr:
case X86::LZCNT32rm:
4203 case X86::LZCNT64rr:
case X86::LZCNT64rm:
4204 case X86::POPCNT16rr:
case X86::POPCNT16rm:
4205 case X86::POPCNT32rr:
case X86::POPCNT32rm:
4206 case X86::POPCNT64rr:
case X86::POPCNT64rm:
4207 case X86::TZCNT16rr:
case X86::TZCNT16rm:
4208 case X86::TZCNT32rr:
case X86::TZCNT32rm:
4209 case X86::TZCNT64rr:
case X86::TZCNT64rm:
4211 case X86::AND64ri32:
case X86::AND64ri8:
case X86::AND32ri:
4212 case X86::AND32ri8:
case X86::AND16ri:
case X86::AND16ri8:
4213 case X86::AND8ri:
case X86::AND64rr:
case X86::AND32rr:
4214 case X86::AND16rr:
case X86::AND8rr:
case X86::AND64rm:
4215 case X86::AND32rm:
case X86::AND16rm:
case X86::AND8rm:
4216 case X86::XOR64ri32:
case X86::XOR64ri8:
case X86::XOR32ri:
4217 case X86::XOR32ri8:
case X86::XOR16ri:
case X86::XOR16ri8:
4218 case X86::XOR8ri:
case X86::XOR64rr:
case X86::XOR32rr:
4219 case X86::XOR16rr:
case X86::XOR8rr:
case X86::XOR64rm:
4220 case X86::XOR32rm:
case X86::XOR16rm:
case X86::XOR8rm:
4221 case X86::OR64ri32:
case X86::OR64ri8:
case X86::OR32ri:
4222 case X86::OR32ri8:
case X86::OR16ri:
case X86::OR16ri8:
4223 case X86::OR8ri:
case X86::OR64rr:
case X86::OR32rr:
4224 case X86::OR16rr:
case X86::OR8rr:
case X86::OR64rm:
4225 case X86::OR32rm:
case X86::OR16rm:
case X86::OR8rm:
4226 case X86::ANDN32rr:
case X86::ANDN32rm:
4227 case X86::ANDN64rr:
case X86::ANDN64rm:
4228 case X86::BLSI32rr:
case X86::BLSI32rm:
4229 case X86::BLSI64rr:
case X86::BLSI64rm:
4230 case X86::BLSMSK32rr:
case X86::BLSMSK32rm:
4231 case X86::BLSMSK64rr:
case X86::BLSMSK64rm:
4232 case X86::BLSR32rr:
case X86::BLSR32rm:
4233 case X86::BLSR64rr:
case X86::BLSR64rm:
4234 case X86::BLCFILL32rr:
case X86::BLCFILL32rm:
4235 case X86::BLCFILL64rr:
case X86::BLCFILL64rm:
4236 case X86::BLCI32rr:
case X86::BLCI32rm:
4237 case X86::BLCI64rr:
case X86::BLCI64rm:
4238 case X86::BLCIC32rr:
case X86::BLCIC32rm:
4239 case X86::BLCIC64rr:
case X86::BLCIC64rm:
4240 case X86::BLCMSK32rr:
case X86::BLCMSK32rm:
4241 case X86::BLCMSK64rr:
case X86::BLCMSK64rm:
4242 case X86::BLCS32rr:
case X86::BLCS32rm:
4243 case X86::BLCS64rr:
case X86::BLCS64rm:
4244 case X86::BLSFILL32rr:
case X86::BLSFILL32rm:
4245 case X86::BLSFILL64rr:
case X86::BLSFILL64rm:
4246 case X86::BLSIC32rr:
case X86::BLSIC32rm:
4247 case X86::BLSIC64rr:
case X86::BLSIC64rm:
4248 case X86::BZHI32rr:
case X86::BZHI32rm:
4249 case X86::BZHI64rr:
case X86::BZHI64rm:
4250 case X86::T1MSKC32rr:
case X86::T1MSKC32rm:
4251 case X86::T1MSKC64rr:
case X86::T1MSKC64rm:
4252 case X86::TZMSK32rr:
case X86::TZMSK32rm:
4253 case X86::TZMSK64rr:
case X86::TZMSK64rm:
4257 ClearsOverflowFlag =
true;
4259 case X86::BEXTR32rr:
case X86::BEXTR64rr:
4260 case X86::BEXTR32rm:
case X86::BEXTR64rm:
4261 case X86::BEXTRI32ri:
case X86::BEXTRI32mi:
4262 case X86::BEXTRI64ri:
case X86::BEXTRI64mi:
4272 switch (
MI.getOpcode()) {
4279 case X86::LZCNT16rr:
4280 case X86::LZCNT32rr:
4281 case X86::LZCNT64rr:
4283 case X86::POPCNT16rr:
4284 case X86::POPCNT32rr:
4285 case X86::POPCNT64rr:
4287 case X86::TZCNT16rr:
4288 case X86::TZCNT32rr:
4289 case X86::TZCNT64rr:
4303 case X86::BLSMSK32rr:
4304 case X86::BLSMSK64rr:
4320 case X86::SUB64ri32:
4338 unsigned NewOpcode = 0;
4341 case X86::SUB64rm: NewOpcode = X86::CMP64rm;
break;
4342 case X86::SUB32rm: NewOpcode = X86::CMP32rm;
break;
4343 case X86::SUB16rm: NewOpcode = X86::CMP16rm;
break;
4344 case X86::SUB8rm: NewOpcode = X86::CMP8rm;
break;
4345 case X86::SUB64rr: NewOpcode = X86::CMP64rr;
break;
4346 case X86::SUB32rr: NewOpcode = X86::CMP32rr;
break;
4347 case X86::SUB16rr: NewOpcode = X86::CMP16rr;
break;
4348 case X86::SUB8rr: NewOpcode = X86::CMP8rr;
break;
4349 case X86::SUB64ri32: NewOpcode = X86::CMP64ri32;
break;
4350 case X86::SUB64ri8: NewOpcode = X86::CMP64ri8;
break;
4351 case X86::SUB32ri: NewOpcode = X86::CMP32ri;
break;
4352 case X86::SUB32ri8: NewOpcode = X86::CMP32ri8;
break;
4353 case X86::SUB16ri: NewOpcode = X86::CMP16ri;
break;
4354 case X86::SUB16ri8: NewOpcode = X86::CMP16ri8;
break;
4355 case X86::SUB8ri: NewOpcode = X86::CMP8ri;
break;
4362 if (NewOpcode == X86::CMP64rm || NewOpcode == X86::CMP32rm ||
4363 NewOpcode == X86::CMP16rm || NewOpcode == X86::CMP8rm)
4371 bool IsCmpZero = (CmpMask != 0 && CmpValue == 0);
4377 assert(SrcRegDef &&
"Must have a definition (SSA)");
4382 bool NoSignFlag =
false;
4383 bool ClearsOverflowFlag =
false;
4384 bool ShouldUpdateCC =
false;
4385 bool IsSwapped =
false;
4387 int64_t ImmDelta = 0;
4400 if (&Inst == SrcRegDef) {
4418 NoSignFlag, ClearsOverflowFlag)) {
4427 if (Inst.modifiesRegister(X86::EFLAGS,
TRI)) {
4438 Inst.getOperand(1).getReg() == SrcReg) {
4439 ShouldUpdateCC =
true;
4450 if (isRedundantFlagInstr(CmpInstr, SrcReg, SrcReg2, CmpMask, CmpValue,
4451 Inst, &IsSwapped, &ImmDelta)) {
4459 if (!Movr0Inst && Inst.
getOpcode() == X86::MOV32r0 &&
4460 Inst.registerDefIsDead(X86::EFLAGS,
TRI)) {
4485 bool FlagsMayLiveOut =
true;
4490 bool ModifyEFLAGS = Instr.modifiesRegister(X86::EFLAGS,
TRI);
4491 bool UseEFLAGS = Instr.readsRegister(X86::EFLAGS,
TRI);
4493 if (!UseEFLAGS && ModifyEFLAGS) {
4495 FlagsMayLiveOut =
false;
4498 if (!UseEFLAGS && !ModifyEFLAGS)
4523 if (!ClearsOverflowFlag)
4541 ReplacementCC = NewCC;
4547 }
else if (IsSwapped) {
4554 ShouldUpdateCC =
true;
4555 }
else if (ImmDelta != 0) {
4556 unsigned BitWidth =
TRI->getRegSizeInBits(*
MRI->getRegClass(SrcReg));
4566 if (ImmDelta != 1 || CmpValue == 0)
4576 if (ImmDelta != 1 || CmpValue == 0)
4603 ShouldUpdateCC =
true;
4606 if (ShouldUpdateCC && ReplacementCC != OldCC) {
4610 OpsToUpdate.
push_back(std::make_pair(&Instr, ReplacementCC));
4612 if (ModifyEFLAGS || Instr.killsRegister(X86::EFLAGS,
TRI)) {
4614 FlagsMayLiveOut =
false;
4621 if ((
MI !=
nullptr || ShouldUpdateCC) && FlagsMayLiveOut) {
4628 assert((
MI ==
nullptr || Sub ==
nullptr) &&
"Should not have Sub and MI set");
4629 Sub =
MI !=
nullptr ?
MI : Sub;
4635 if (&CmpMBB != SubBB)
4640 for (; InsertI != InsertE; ++InsertI) {
4650 if (InsertI == InsertE)
4656 assert(FlagDef &&
"Unable to locate a def EFLAGS operand");
4662 for (
auto &Op : OpsToUpdate) {
4663 Op.first->getOperand(Op.first->getDesc().getNumOperands() - 1)
4685 DefMI =
MRI->getVRegDef(FoldAsLoadDefReg);
4687 bool SawStore =
false;
4693 for (
unsigned i = 0, e =
MI.getNumOperands(); i != e; ++i) {
4698 if (Reg != FoldAsLoadDefReg)
4705 if (SrcOperandIds.
empty())
4710 FoldAsLoadDefReg = 0;
4735 MIB.
getReg(2) == Reg &&
"Misplaced operand");
4765 MIB->
setDesc(
TII.get(MinusOne ? X86::DEC32r : X86::INC32r));
4777 assert(Imm != 0 &&
"Using push/pop for 0 is not efficient.");
4780 int StackAdjustment;
4782 if (Subtarget.is64Bit()) {
4784 MIB->
getOpcode() == X86::MOV32ImmSExti8);
4791 X86::MOV32ImmSExti8 ? X86::MOV32ri : X86::MOV64ri));
4797 StackAdjustment = 8;
4804 StackAdjustment = 4;
4816 bool EmitCFI = !TFL->
hasFP(MF) && NeedsDwarfCFI;
4857 MIB->
getOpcode() == X86::XOR64_FP ? X86::XOR64rr : X86::XOR32rr;
4873 if (
TRI->getEncodingValue(DestReg) < 16) {
4880 DestReg =
TRI->getMatchingSuperReg(DestReg, SubIdx, &X86::VR512RegClass);
4896 if (
TRI->getEncodingValue(SrcReg) < 16) {
4903 SrcReg =
TRI->getMatchingSuperReg(SrcReg, SubIdx, &X86::VR512RegClass);
4925 bool HasAVX = Subtarget.
hasAVX();
4927 switch (
MI.getOpcode()) {
4934 case X86::MOV32ImmSExti8:
4935 case X86::MOV64ImmSExti8:
4937 case X86::SETB_C32r:
4939 case X86::SETB_C64r:
4947 case X86::FsFLD0F128:
4949 case X86::AVX_SET0: {
4950 assert(HasAVX &&
"AVX not supported");
4953 Register XReg =
TRI->getSubReg(SrcReg, X86::sub_xmm);
4959 case X86::AVX512_128_SET0:
4960 case X86::AVX512_FsFLD0SH:
4961 case X86::AVX512_FsFLD0SS:
4962 case X86::AVX512_FsFLD0SD:
4963 case X86::AVX512_FsFLD0F128: {
4964 bool HasVLX = Subtarget.hasVLX();
4967 if (HasVLX ||
TRI->getEncodingValue(SrcReg) < 16)
4969 get(HasVLX ? X86::VPXORDZ128rr : X86::VXORPSrr));
4972 TRI->getMatchingSuperReg(SrcReg, X86::sub_xmm, &X86::VR512RegClass);
4976 case X86::AVX512_256_SET0:
4977 case X86::AVX512_512_SET0: {
4978 bool HasVLX = Subtarget.hasVLX();
4981 if (HasVLX ||
TRI->getEncodingValue(SrcReg) < 16) {
4982 Register XReg =
TRI->getSubReg(SrcReg, X86::sub_xmm);
4985 get(HasVLX ? X86::VPXORDZ128rr : X86::VXORPSrr));
4989 if (
MI.getOpcode() == X86::AVX512_256_SET0) {
4992 TRI->getMatchingSuperReg(SrcReg, X86::sub_ymm, &X86::VR512RegClass);
4997 case X86::V_SETALLONES:
4999 case X86::AVX2_SETALLONES:
5001 case X86::AVX1_SETALLONES: {