49 #define DEBUG_TYPE "x86-instr-info"
51 #define GET_INSTRINFO_CTOR_DTOR
52 #include "X86GenInstrInfo.inc"
56 cl::desc(
"Disable fusing of spill code into instructions"),
60 cl::desc(
"Print instructions that the allocator wants to"
61 " fuse, but the X86 backend currently can't"),
65 cl::desc(
"Re-materialize load from stub in PIC mode"),
69 cl::desc(
"Clearance between two register writes "
70 "for inserting XOR to avoid partial "
75 cl::desc(
"How many idle instructions we would like before "
76 "certain undef register reads"),
81 void X86InstrInfo::anchor() {}
85 :
X86::ADJCALLSTACKDOWN32),
86 (STI.isTarget64BitLP64() ?
X86::ADJCALLSTACKUP64
87 :
X86::ADJCALLSTACKUP32),
90 Subtarget(STI), RI(STI.getTargetTriple()) {
96 unsigned &SubIdx)
const {
97 switch (
MI.getOpcode()) {
100 case X86::MOVZX16rr8:
101 case X86::MOVSX32rr8:
102 case X86::MOVZX32rr8:
103 case X86::MOVSX64rr8:
104 if (!Subtarget.is64Bit())
109 case X86::MOVSX32rr16:
110 case X86::MOVZX32rr16:
111 case X86::MOVSX64rr16:
112 case X86::MOVSX64rr32: {
113 if (
MI.getOperand(0).getSubReg() ||
MI.getOperand(1).getSubReg())
116 SrcReg =
MI.getOperand(1).getReg();
117 DstReg =
MI.getOperand(0).getReg();
118 switch (
MI.getOpcode()) {
120 case X86::MOVSX16rr8:
121 case X86::MOVZX16rr8:
122 case X86::MOVSX32rr8:
123 case X86::MOVZX32rr8:
124 case X86::MOVSX64rr8:
125 SubIdx = X86::sub_8bit;
127 case X86::MOVSX32rr16:
128 case X86::MOVZX32rr16:
129 case X86::MOVSX64rr16:
130 SubIdx = X86::sub_16bit;
132 case X86::MOVSX64rr32:
133 SubIdx = X86::sub_32bit;
143 if (
MI.mayLoad() ||
MI.mayStore())
148 if (
MI.isCopyLike() ||
MI.isInsertSubreg())
151 unsigned Opcode =
MI.getOpcode();
162 if (isBSF(Opcode) || isBSR(Opcode) || isLZCNT(Opcode) || isPOPCNT(Opcode) ||
168 if (isBLCFILL(Opcode) || isBLCI(Opcode) || isBLCIC(Opcode) ||
169 isBLCMSK(Opcode) || isBLCS(Opcode) || isBLSFILL(Opcode) ||
170 isBLSI(Opcode) || isBLSIC(Opcode) || isBLSMSK(Opcode) || isBLSR(Opcode) ||
175 if (isBEXTR(Opcode) || isBZHI(Opcode))
178 if (isROL(Opcode) || isROR(Opcode) || isSAR(Opcode) || isSHL(Opcode) ||
179 isSHR(Opcode) || isSHLD(Opcode) || isSHRD(Opcode))
182 if (isADC(Opcode) || isADD(Opcode) || isAND(Opcode) || isOR(Opcode) ||
183 isSBB(Opcode) || isSUB(Opcode) || isXOR(Opcode))
186 if (isADCX(Opcode) || isADOX(Opcode) || isANDN(Opcode))
189 if (isDEC(Opcode) || isINC(Opcode) || isNEG(Opcode))
197 if (isMOVSX(Opcode) || isMOVZX(Opcode) || isMOVSXD(Opcode) || isMOV(Opcode))
200 if (isRORX(Opcode) || isSARX(Opcode) || isSHLX(Opcode) || isSHRX(Opcode))
210 switch (
MI.getOpcode()) {
219 case X86::IMUL16rmi8:
222 case X86::IMUL32rmi8:
225 case X86::IMUL64rmi32:
226 case X86::IMUL64rmi8:
241 case X86::POPCNT16rm:
242 case X86::POPCNT32rm:
243 case X86::POPCNT64rm:
251 case X86::BLCFILL32rm:
252 case X86::BLCFILL64rm:
257 case X86::BLCMSK32rm:
258 case X86::BLCMSK64rm:
261 case X86::BLSFILL32rm:
262 case X86::BLSFILL64rm:
267 case X86::BLSMSK32rm:
268 case X86::BLSMSK64rm:
278 case X86::BEXTRI32mi:
279 case X86::BEXTRI64mi:
336 case X86::CVTTSD2SI64rm:
337 case X86::VCVTTSD2SI64rm:
338 case X86::VCVTTSD2SI64Zrm:
339 case X86::CVTTSD2SIrm:
340 case X86::VCVTTSD2SIrm:
341 case X86::VCVTTSD2SIZrm:
342 case X86::CVTTSS2SI64rm:
343 case X86::VCVTTSS2SI64rm:
344 case X86::VCVTTSS2SI64Zrm:
345 case X86::CVTTSS2SIrm:
346 case X86::VCVTTSS2SIrm:
347 case X86::VCVTTSS2SIZrm:
348 case X86::CVTSI2SDrm:
349 case X86::VCVTSI2SDrm:
350 case X86::VCVTSI2SDZrm:
351 case X86::CVTSI2SSrm:
352 case X86::VCVTSI2SSrm:
353 case X86::VCVTSI2SSZrm:
354 case X86::CVTSI642SDrm:
355 case X86::VCVTSI642SDrm:
356 case X86::VCVTSI642SDZrm:
357 case X86::CVTSI642SSrm:
358 case X86::VCVTSI642SSrm:
359 case X86::VCVTSI642SSZrm:
360 case X86::CVTSS2SDrm:
361 case X86::VCVTSS2SDrm:
362 case X86::VCVTSS2SDZrm:
363 case X86::CVTSD2SSrm:
364 case X86::VCVTSD2SSrm:
365 case X86::VCVTSD2SSZrm:
367 case X86::VCVTTSD2USI64Zrm:
368 case X86::VCVTTSD2USIZrm:
369 case X86::VCVTTSS2USI64Zrm:
370 case X86::VCVTTSS2USIZrm:
371 case X86::VCVTUSI2SDZrm:
372 case X86::VCVTUSI642SDZrm:
373 case X86::VCVTUSI2SSZrm:
374 case X86::VCVTUSI642SSZrm:
378 case X86::MOV8rm_NOREX:
382 case X86::MOVSX16rm8:
383 case X86::MOVSX32rm16:
384 case X86::MOVSX32rm8:
385 case X86::MOVSX32rm8_NOREX:
386 case X86::MOVSX64rm16:
387 case X86::MOVSX64rm32:
388 case X86::MOVSX64rm8:
389 case X86::MOVZX16rm8:
390 case X86::MOVZX32rm16:
391 case X86::MOVZX32rm8:
392 case X86::MOVZX32rm8_NOREX:
393 case X86::MOVZX64rm16:
394 case X86::MOVZX64rm8:
403 if (isFrameInstr(
MI)) {
406 if (!isFrameSetup(
MI))
418 if (
I->getOpcode() == getCallFrameDestroyOpcode() ||
425 if (
I->getOpcode() != getCallFrameDestroyOpcode())
428 return -(
I->getOperand(1).getImm());
433 switch (
MI.getOpcode()) {
479 case X86::VMOVSHZrm_alt:
484 case X86::MOVSSrm_alt:
486 case X86::VMOVSSrm_alt:
488 case X86::VMOVSSZrm_alt:
495 case X86::MOVSDrm_alt:
497 case X86::VMOVSDrm_alt:
499 case X86::VMOVSDZrm_alt:
500 case X86::MMX_MOVD64rm:
501 case X86::MMX_MOVQ64rm:
517 case X86::VMOVAPSZ128rm:
518 case X86::VMOVUPSZ128rm:
519 case X86::VMOVAPSZ128rm_NOVLX:
520 case X86::VMOVUPSZ128rm_NOVLX:
521 case X86::VMOVAPDZ128rm:
522 case X86::VMOVUPDZ128rm:
523 case X86::VMOVDQU8Z128rm:
524 case X86::VMOVDQU16Z128rm:
525 case X86::VMOVDQA32Z128rm:
526 case X86::VMOVDQU32Z128rm:
527 case X86::VMOVDQA64Z128rm:
528 case X86::VMOVDQU64Z128rm:
531 case X86::VMOVAPSYrm:
532 case X86::VMOVUPSYrm:
533 case X86::VMOVAPDYrm:
534 case X86::VMOVUPDYrm:
535 case X86::VMOVDQAYrm:
536 case X86::VMOVDQUYrm:
537 case X86::VMOVAPSZ256rm:
538 case X86::VMOVUPSZ256rm:
539 case X86::VMOVAPSZ256rm_NOVLX:
540 case X86::VMOVUPSZ256rm_NOVLX:
541 case X86::VMOVAPDZ256rm:
542 case X86::VMOVUPDZ256rm:
543 case X86::VMOVDQU8Z256rm:
544 case X86::VMOVDQU16Z256rm:
545 case X86::VMOVDQA32Z256rm:
546 case X86::VMOVDQU32Z256rm:
547 case X86::VMOVDQA64Z256rm:
548 case X86::VMOVDQU64Z256rm:
551 case X86::VMOVAPSZrm:
552 case X86::VMOVUPSZrm:
553 case X86::VMOVAPDZrm:
554 case X86::VMOVUPDZrm:
555 case X86::VMOVDQU8Zrm:
556 case X86::VMOVDQU16Zrm:
557 case X86::VMOVDQA32Zrm:
558 case X86::VMOVDQU32Zrm:
559 case X86::VMOVDQA64Zrm:
560 case X86::VMOVDQU64Zrm:
591 case X86::MMX_MOVD64mr:
592 case X86::MMX_MOVQ64mr:
593 case X86::MMX_MOVNTQmr:
609 case X86::VMOVUPSZ128mr:
610 case X86::VMOVAPSZ128mr:
611 case X86::VMOVUPSZ128mr_NOVLX:
612 case X86::VMOVAPSZ128mr_NOVLX:
613 case X86::VMOVUPDZ128mr:
614 case X86::VMOVAPDZ128mr:
615 case X86::VMOVDQA32Z128mr:
616 case X86::VMOVDQU32Z128mr:
617 case X86::VMOVDQA64Z128mr:
618 case X86::VMOVDQU64Z128mr:
619 case X86::VMOVDQU8Z128mr:
620 case X86::VMOVDQU16Z128mr:
623 case X86::VMOVUPSYmr:
624 case X86::VMOVAPSYmr:
625 case X86::VMOVUPDYmr:
626 case X86::VMOVAPDYmr:
627 case X86::VMOVDQUYmr:
628 case X86::VMOVDQAYmr:
629 case X86::VMOVUPSZ256mr:
630 case X86::VMOVAPSZ256mr:
631 case X86::VMOVUPSZ256mr_NOVLX:
632 case X86::VMOVAPSZ256mr_NOVLX:
633 case X86::VMOVUPDZ256mr:
634 case X86::VMOVAPDZ256mr:
635 case X86::VMOVDQU8Z256mr:
636 case X86::VMOVDQU16Z256mr:
637 case X86::VMOVDQA32Z256mr:
638 case X86::VMOVDQU32Z256mr:
639 case X86::VMOVDQA64Z256mr:
640 case X86::VMOVDQU64Z256mr:
643 case X86::VMOVUPSZmr:
644 case X86::VMOVAPSZmr:
645 case X86::VMOVUPDZmr:
646 case X86::VMOVAPDZmr:
647 case X86::VMOVDQU8Zmr:
648 case X86::VMOVDQU16Zmr:
649 case X86::VMOVDQA32Zmr:
650 case X86::VMOVDQU32Zmr:
651 case X86::VMOVDQA64Zmr:
652 case X86::VMOVDQU64Zmr:
667 unsigned &MemBytes)
const {
669 if (
MI.getOperand(0).getSubReg() == 0 && isFrameOperand(
MI, 1,
FrameIndex))
670 return MI.getOperand(0).getReg();
683 if (hasLoadFromStackSlot(
MI, Accesses)) {
685 cast<FixedStackPseudoSourceValue>(Accesses.front()->getPseudoValue())
687 return MI.getOperand(0).getReg();
701 unsigned &MemBytes)
const {
718 if (hasStoreToStackSlot(
MI, Accesses)) {
720 cast<FixedStackPseudoSourceValue>(Accesses.front()->getPseudoValue())
733 bool isPICBase =
false;
739 assert(!isPICBase &&
"More than one PIC base?");
747 switch (
MI.getOpcode()) {
754 case X86::LOAD_STACK_GUARD:
755 case X86::AVX1_SETALLONES:
756 case X86::AVX2_SETALLONES:
757 case X86::AVX512_128_SET0:
758 case X86::AVX512_256_SET0:
759 case X86::AVX512_512_SET0:
760 case X86::AVX512_512_SETALLONES:
761 case X86::AVX512_FsFLD0SD:
762 case X86::AVX512_FsFLD0SH:
763 case X86::AVX512_FsFLD0SS:
764 case X86::AVX512_FsFLD0F128:
768 case X86::FsFLD0F128:
776 case X86::MOV32ImmSExti8:
781 case X86::MOV64ImmSExti8:
783 case X86::V_SETALLONES:
789 case X86::PTILEZEROV:
793 case X86::MOV8rm_NOREX:
798 case X86::MOVSSrm_alt:
800 case X86::MOVSDrm_alt:
808 case X86::VMOVSSrm_alt:
810 case X86::VMOVSDrm_alt:
817 case X86::VMOVAPSYrm:
818 case X86::VMOVUPSYrm:
819 case X86::VMOVAPDYrm:
820 case X86::VMOVUPDYrm:
821 case X86::VMOVDQAYrm:
822 case X86::VMOVDQUYrm:
823 case X86::MMX_MOVD64rm:
824 case X86::MMX_MOVQ64rm:
827 case X86::VMOVSSZrm_alt:
829 case X86::VMOVSDZrm_alt:
831 case X86::VMOVSHZrm_alt:
832 case X86::VMOVAPDZ128rm:
833 case X86::VMOVAPDZ256rm:
834 case X86::VMOVAPDZrm:
835 case X86::VMOVAPSZ128rm:
836 case X86::VMOVAPSZ256rm:
837 case X86::VMOVAPSZ128rm_NOVLX:
838 case X86::VMOVAPSZ256rm_NOVLX:
839 case X86::VMOVAPSZrm:
840 case X86::VMOVDQA32Z128rm:
841 case X86::VMOVDQA32Z256rm:
842 case X86::VMOVDQA32Zrm:
843 case X86::VMOVDQA64Z128rm:
844 case X86::VMOVDQA64Z256rm:
845 case X86::VMOVDQA64Zrm:
846 case X86::VMOVDQU16Z128rm:
847 case X86::VMOVDQU16Z256rm:
848 case X86::VMOVDQU16Zrm:
849 case X86::VMOVDQU32Z128rm:
850 case X86::VMOVDQU32Z256rm:
851 case X86::VMOVDQU32Zrm:
852 case X86::VMOVDQU64Z128rm:
853 case X86::VMOVDQU64Z256rm:
854 case X86::VMOVDQU64Zrm:
855 case X86::VMOVDQU8Z128rm:
856 case X86::VMOVDQU8Z256rm:
857 case X86::VMOVDQU8Zrm:
858 case X86::VMOVUPDZ128rm:
859 case X86::VMOVUPDZ256rm:
860 case X86::VMOVUPDZrm:
861 case X86::VMOVUPSZ128rm:
862 case X86::VMOVUPSZ256rm:
863 case X86::VMOVUPSZ128rm_NOVLX:
864 case X86::VMOVUPSZ256rm_NOVLX:
865 case X86::VMOVUPSZrm: {
871 MI.isDereferenceableInvariantLoad(
AA)) {
873 if (BaseReg == 0 || BaseReg == X86::RIP)
919 case X86::MOV32r0:
Value = 0;
break;
920 case X86::MOV32r1:
Value = 1;
break;
921 case X86::MOV32r_1:
Value = -1;
break;
942 if (MO.isReg() && MO.isDef() &&
943 MO.getReg() == X86::EFLAGS && !MO.isDead()) {
952 unsigned ShiftAmtOperandIdx) {
954 unsigned ShiftCountMask = (
MI.getDesc().TSFlags &
X86II::REX_W) ? 63 : 31;
955 unsigned Imm =
MI.getOperand(ShiftAmtOperandIdx).getImm();
956 return Imm & ShiftCountMask;
967 return ShAmt < 4 && ShAmt > 0;
975 bool &NoSignFlag,
bool &ClearsOverflowFlag) {
976 if (CmpValDefInstr.
getOpcode() != X86::SUBREG_TO_REG)
979 if (CmpInstr.
getOpcode() != X86::TEST64rr)
986 "CmpInstr is an analyzable TEST64rr, and `X86InstrInfo::analyzeCompare` "
987 "requires two reg operands are the same.");
995 "Caller guarantees that TEST64rr is a user of SUBREG_TO_REG.");
1010 assert(VregDefInstr &&
"Must have a definition (SSA)");
1020 if (X86::isAND(VregDefInstr->
getOpcode())) {
1035 if (Instr.modifiesRegister(X86::EFLAGS,
TRI))
1039 *AndInstr = VregDefInstr;
1060 ClearsOverflowFlag =
true;
1067 unsigned Opc,
bool AllowSP,
Register &NewSrc,
1073 RC = Opc != X86::LEA32r ? &X86::GR64RegClass : &X86::GR32RegClass;
1075 RC = Opc != X86::LEA32r ?
1076 &X86::GR64_NOSPRegClass : &X86::GR32_NOSPRegClass;
1079 isKill =
MI.killsRegister(SrcReg);
1083 if (Opc != X86::LEA64_32r) {
1085 assert(!Src.isUndef() &&
"Undef op doesn't need optimization");
1100 assert(!Src.isUndef() &&
"Undef op doesn't need optimization");
1121 if (
S->end.getBaseIndex() == Idx)
1130 MachineInstr *X86InstrInfo::convertToThreeAddressWithLEA(
unsigned MIOpc,
1134 bool Is8BitOp)
const {
1140 "Unexpected type for LEA transform");
1149 if (!Subtarget.is64Bit())
1152 unsigned Opcode = X86::LEA64_32r;
1168 bool IsDead =
MI.getOperand(0).isDead();
1169 bool IsKill =
MI.getOperand(1).isKill();
1170 unsigned SubReg = Is8BitOp ? X86::sub_8bit : X86::sub_16bit;
1171 assert(!
MI.getOperand(1).isUndef() &&
"Undef op doesn't need optimization");
1186 case X86::SHL16ri: {
1187 unsigned ShAmt =
MI.getOperand(2).getImm();
1201 case X86::ADD8ri_DB:
1204 case X86::ADD16ri_DB:
1205 case X86::ADD16ri8_DB:
1209 case X86::ADD8rr_DB:
1211 case X86::ADD16rr_DB: {
1212 Src2 =
MI.getOperand(2).getReg();
1213 bool IsKill2 =
MI.getOperand(2).isKill();
1214 assert(!
MI.getOperand(2).isUndef() &&
"Undef op doesn't need optimization");
1218 addRegReg(MIB, InRegLEA,
true, InRegLEA,
false);
1220 if (Subtarget.is64Bit())
1226 ImpDef2 =
BuildMI(
MBB, &*MIB,
MI.getDebugLoc(),
get(X86::IMPLICIT_DEF),
1228 InsMI2 =
BuildMI(
MBB, &*MIB,
MI.getDebugLoc(),
get(TargetOpcode::COPY))
1231 addRegReg(MIB, InRegLEA,
true, InRegLEA2,
true);
1233 if (LV && IsKill2 && InsMI2)
1327 if (
MI.getNumOperands() > 2)
1328 if (
MI.getOperand(2).isReg() &&
MI.getOperand(2).isUndef())
1333 bool Is64Bit = Subtarget.is64Bit();
1335 bool Is8BitOp =
false;
1336 unsigned MIOpc =
MI.getOpcode();
1339 case X86::SHL64ri: {
1340 assert(
MI.getNumOperands() >= 3 &&
"Unknown shift instruction!");
1346 Src.getReg(), &X86::GR64_NOSPRegClass))
1349 NewMI =
BuildMI(MF,
MI.getDebugLoc(),
get(X86::LEA64r))
1358 case X86::SHL32ri: {
1359 assert(
MI.getNumOperands() >= 3 &&
"Unknown shift instruction!");
1363 unsigned Opc = Is64Bit ? X86::LEA64_32r : X86::LEA32r;
1369 ImplicitOp, LV, LIS))
1380 if (ImplicitOp.
getReg() != 0)
1381 MIB.
add(ImplicitOp);
1389 case X86::SHL16ri: {
1390 assert(
MI.getNumOperands() >= 3 &&
"Unknown shift instruction!");
1394 return convertToThreeAddressWithLEA(MIOpc,
MI, LV, LIS, Is8BitOp);
1398 assert(
MI.getNumOperands() >= 2 &&
"Unknown inc instruction!");
1399 unsigned Opc = MIOpc == X86::INC64r ? X86::LEA64r :
1400 (Is64Bit ? X86::LEA64_32r : X86::LEA32r);
1404 ImplicitOp, LV, LIS))
1411 if (ImplicitOp.
getReg() != 0)
1412 MIB.
add(ImplicitOp);
1419 assert(
MI.getNumOperands() >= 2 &&
"Unknown dec instruction!");
1420 unsigned Opc = MIOpc == X86::DEC64r ? X86::LEA64r
1421 : (Is64Bit ? X86::LEA64_32r : X86::LEA32r);
1426 ImplicitOp, LV, LIS))
1432 if (ImplicitOp.
getReg() != 0)
1433 MIB.
add(ImplicitOp);
1445 return convertToThreeAddressWithLEA(MIOpc,
MI, LV, LIS, Is8BitOp);
1447 case X86::ADD64rr_DB:
1449 case X86::ADD32rr_DB: {
1450 assert(
MI.getNumOperands() >= 3 &&
"Unknown add instruction!");
1452 if (MIOpc == X86::ADD64rr || MIOpc == X86::ADD64rr_DB)
1455 Opc = Is64Bit ? X86::LEA64_32r : X86::LEA32r;
1461 ImplicitOp2, LV, LIS))
1466 if (Src.getReg() == Src2.
getReg()) {
1473 ImplicitOp, LV, LIS))
1478 if (ImplicitOp.
getReg() != 0)
1479 MIB.
add(ImplicitOp);
1480 if (ImplicitOp2.
getReg() != 0)
1481 MIB.
add(ImplicitOp2);
1483 NewMI =
addRegReg(MIB, SrcReg, isKill, SrcReg2, isKill2);
1489 case X86::ADD8rr_DB:
1493 case X86::ADD16rr_DB:
1494 return convertToThreeAddressWithLEA(MIOpc,
MI, LV, LIS, Is8BitOp);
1495 case X86::ADD64ri32:
1497 case X86::ADD64ri32_DB:
1498 case X86::ADD64ri8_DB:
1499 assert(
MI.getNumOperands() >= 3 &&
"Unknown add instruction!");
1506 case X86::ADD32ri_DB:
1507 case X86::ADD32ri8_DB: {
1508 assert(
MI.getNumOperands() >= 3 &&
"Unknown add instruction!");
1509 unsigned Opc = Is64Bit ? X86::LEA64_32r : X86::LEA32r;
1514 ImplicitOp, LV, LIS))
1520 if (ImplicitOp.
getReg() != 0)
1521 MIB.
add(ImplicitOp);
1527 case X86::ADD8ri_DB:
1532 case X86::ADD16ri_DB:
1533 case X86::ADD16ri8_DB:
1534 return convertToThreeAddressWithLEA(MIOpc,
MI, LV, LIS, Is8BitOp);
1541 case X86::SUB32ri: {
1542 if (!
MI.getOperand(2).isImm())
1544 int64_t
Imm =
MI.getOperand(2).getImm();
1548 assert(
MI.getNumOperands() >= 3 &&
"Unknown add instruction!");
1549 unsigned Opc = Is64Bit ? X86::LEA64_32r : X86::LEA32r;
1554 ImplicitOp, LV, LIS))
1560 if (ImplicitOp.
getReg() != 0)
1561 MIB.
add(ImplicitOp);
1568 case X86::SUB64ri32: {
1569 if (!
MI.getOperand(2).isImm())
1571 int64_t
Imm =
MI.getOperand(2).getImm();
1575 assert(
MI.getNumOperands() >= 3 &&
"Unknown sub instruction!");
1583 case X86::VMOVDQU8Z128rmk:
1584 case X86::VMOVDQU8Z256rmk:
1585 case X86::VMOVDQU8Zrmk:
1586 case X86::VMOVDQU16Z128rmk:
1587 case X86::VMOVDQU16Z256rmk:
1588 case X86::VMOVDQU16Zrmk:
1589 case X86::VMOVDQU32Z128rmk:
case X86::VMOVDQA32Z128rmk:
1590 case X86::VMOVDQU32Z256rmk:
case X86::VMOVDQA32Z256rmk:
1591 case X86::VMOVDQU32Zrmk:
case X86::VMOVDQA32Zrmk:
1592 case X86::VMOVDQU64Z128rmk:
case X86::VMOVDQA64Z128rmk:
1593 case X86::VMOVDQU64Z256rmk:
case X86::VMOVDQA64Z256rmk:
1594 case X86::VMOVDQU64Zrmk:
case X86::VMOVDQA64Zrmk:
1595 case X86::VMOVUPDZ128rmk:
case X86::VMOVAPDZ128rmk:
1596 case X86::VMOVUPDZ256rmk:
case X86::VMOVAPDZ256rmk:
1597 case X86::VMOVUPDZrmk:
case X86::VMOVAPDZrmk:
1598 case X86::VMOVUPSZ128rmk:
case X86::VMOVAPSZ128rmk:
1599 case X86::VMOVUPSZ256rmk:
case X86::VMOVAPSZ256rmk:
1600 case X86::VMOVUPSZrmk:
case X86::VMOVAPSZrmk:
1601 case X86::VBROADCASTSDZ256rmk:
1602 case X86::VBROADCASTSDZrmk:
1603 case X86::VBROADCASTSSZ128rmk:
1604 case X86::VBROADCASTSSZ256rmk:
1605 case X86::VBROADCASTSSZrmk:
1606 case X86::VPBROADCASTDZ128rmk:
1607 case X86::VPBROADCASTDZ256rmk:
1608 case X86::VPBROADCASTDZrmk:
1609 case X86::VPBROADCASTQZ128rmk:
1610 case X86::VPBROADCASTQZ256rmk:
1611 case X86::VPBROADCASTQZrmk: {
1615 case X86::VMOVDQU8Z128rmk: Opc = X86::VPBLENDMBZ128rmk;
break;
1616 case X86::VMOVDQU8Z256rmk: Opc = X86::VPBLENDMBZ256rmk;
break;
1617 case X86::VMOVDQU8Zrmk: Opc = X86::VPBLENDMBZrmk;
break;
1618 case X86::VMOVDQU16Z128rmk: Opc = X86::VPBLENDMWZ128rmk;
break;
1619 case X86::VMOVDQU16Z256rmk: Opc = X86::VPBLENDMWZ256rmk;
break;
1620 case X86::VMOVDQU16Zrmk: Opc = X86::VPBLENDMWZrmk;
break;
1621 case X86::VMOVDQU32Z128rmk: Opc = X86::VPBLENDMDZ128rmk;
break;
1622 case X86::VMOVDQU32Z256rmk: Opc = X86::VPBLENDMDZ256rmk;
break;
1623 case X86::VMOVDQU32Zrmk: Opc = X86::VPBLENDMDZrmk;
break;
1624 case X86::VMOVDQU64Z128rmk: Opc = X86::VPBLENDMQZ128rmk;
break;
1625 case X86::VMOVDQU64Z256rmk: Opc = X86::VPBLENDMQZ256rmk;
break;
1626 case X86::VMOVDQU64Zrmk: Opc = X86::VPBLENDMQZrmk;
break;
1627 case X86::VMOVUPDZ128rmk: Opc = X86::VBLENDMPDZ128rmk;
break;
1628 case X86::VMOVUPDZ256rmk: Opc = X86::VBLENDMPDZ256rmk;
break;
1629 case X86::VMOVUPDZrmk: Opc = X86::VBLENDMPDZrmk;
break;
1630 case X86::VMOVUPSZ128rmk: Opc = X86::VBLENDMPSZ128rmk;
break;
1631 case X86::VMOVUPSZ256rmk: Opc = X86::VBLENDMPSZ256rmk;
break;
1632 case X86::VMOVUPSZrmk: Opc = X86::VBLENDMPSZrmk;
break;
1633 case X86::VMOVDQA32Z128rmk: Opc = X86::VPBLENDMDZ128rmk;
break;
1634 case X86::VMOVDQA32Z256rmk: Opc = X86::VPBLENDMDZ256rmk;
break;
1635 case X86::VMOVDQA32Zrmk: Opc = X86::VPBLENDMDZrmk;
break;
1636 case X86::VMOVDQA64Z128rmk: Opc = X86::VPBLENDMQZ128rmk;
break;
1637 case X86::VMOVDQA64Z256rmk: Opc = X86::VPBLENDMQZ256rmk;
break;
1638 case X86::VMOVDQA64Zrmk: Opc = X86::VPBLENDMQZrmk;
break;
1639 case X86::VMOVAPDZ128rmk: Opc = X86::VBLENDMPDZ128rmk;
break;
1640 case X86::VMOVAPDZ256rmk: Opc = X86::VBLENDMPDZ256rmk;
break;
1641 case X86::VMOVAPDZrmk: Opc = X86::VBLENDMPDZrmk;
break;
1642 case X86::VMOVAPSZ128rmk: Opc = X86::VBLENDMPSZ128rmk;
break;
1643 case X86::VMOVAPSZ256rmk: Opc = X86::VBLENDMPSZ256rmk;
break;
1644 case X86::VMOVAPSZrmk: Opc = X86::VBLENDMPSZrmk;
break;
1645 case X86::VBROADCASTSDZ256rmk: Opc = X86::VBLENDMPDZ256rmbk;
break;
1646 case X86::VBROADCASTSDZrmk: Opc = X86::VBLENDMPDZrmbk;
break;
1647 case X86::VBROADCASTSSZ128rmk: Opc = X86::VBLENDMPSZ128rmbk;
break;
1648 case X86::VBROADCASTSSZ256rmk: Opc = X86::VBLENDMPSZ256rmbk;
break;
1649 case X86::VBROADCASTSSZrmk: Opc = X86::VBLENDMPSZrmbk;
break;
1650 case X86::VPBROADCASTDZ128rmk: Opc = X86::VPBLENDMDZ128rmbk;
break;
1651 case X86::VPBROADCASTDZ256rmk: Opc = X86::VPBLENDMDZ256rmbk;
break;
1652 case X86::VPBROADCASTDZrmk: Opc = X86::VPBLENDMDZrmbk;
break;
1653 case X86::VPBROADCASTQZ128rmk: Opc = X86::VPBLENDMQZ128rmbk;
break;
1654 case X86::VPBROADCASTQZ256rmk: Opc = X86::VPBLENDMQZ256rmbk;
break;
1655 case X86::VPBROADCASTQZrmk: Opc = X86::VPBLENDMQZrmbk;
break;
1660 .
add(
MI.getOperand(2))
1662 .
add(
MI.getOperand(3))
1663 .
add(
MI.getOperand(4))
1664 .
add(
MI.getOperand(5))
1665 .
add(
MI.getOperand(6))
1666 .
add(
MI.getOperand(7));
1670 case X86::VMOVDQU8Z128rrk:
1671 case X86::VMOVDQU8Z256rrk:
1672 case X86::VMOVDQU8Zrrk:
1673 case X86::VMOVDQU16Z128rrk:
1674 case X86::VMOVDQU16Z256rrk:
1675 case X86::VMOVDQU16Zrrk:
1676 case X86::VMOVDQU32Z128rrk:
case X86::VMOVDQA32Z128rrk:
1677 case X86::VMOVDQU32Z256rrk:
case X86::VMOVDQA32Z256rrk:
1678 case X86::VMOVDQU32Zrrk:
case X86::VMOVDQA32Zrrk:
1679 case X86::VMOVDQU64Z128rrk:
case X86::VMOVDQA64Z128rrk:
1680 case X86::VMOVDQU64Z256rrk:
case X86::VMOVDQA64Z256rrk:
1681 case X86::VMOVDQU64Zrrk:
case X86::VMOVDQA64Zrrk:
1682 case X86::VMOVUPDZ128rrk:
case X86::VMOVAPDZ128rrk:
1683 case X86::VMOVUPDZ256rrk:
case X86::VMOVAPDZ256rrk:
1684 case X86::VMOVUPDZrrk:
case X86::VMOVAPDZrrk:
1685 case X86::VMOVUPSZ128rrk:
case X86::VMOVAPSZ128rrk:
1686 case X86::VMOVUPSZ256rrk:
case X86::VMOVAPSZ256rrk:
1687 case X86::VMOVUPSZrrk:
case X86::VMOVAPSZrrk: {
1691 case X86::VMOVDQU8Z128rrk: Opc = X86::VPBLENDMBZ128rrk;
break;
1692 case X86::VMOVDQU8Z256rrk: Opc = X86::VPBLENDMBZ256rrk;
break;
1693 case X86::VMOVDQU8Zrrk: Opc = X86::VPBLENDMBZrrk;
break;
1694 case X86::VMOVDQU16Z128rrk: Opc = X86::VPBLENDMWZ128rrk;
break;
1695 case X86::VMOVDQU16Z256rrk: Opc = X86::VPBLENDMWZ256rrk;
break;
1696 case X86::VMOVDQU16Zrrk: Opc = X86::VPBLENDMWZrrk;
break;
1697 case X86::VMOVDQU32Z128rrk: Opc = X86::VPBLENDMDZ128rrk;
break;
1698 case X86::VMOVDQU32Z256rrk: Opc = X86::VPBLENDMDZ256rrk;
break;
1699 case X86::VMOVDQU32Zrrk: Opc = X86::VPBLENDMDZrrk;
break;
1700 case X86::VMOVDQU64Z128rrk: Opc = X86::VPBLENDMQZ128rrk;
break;
1701 case X86::VMOVDQU64Z256rrk: Opc = X86::VPBLENDMQZ256rrk;
break;
1702 case X86::VMOVDQU64Zrrk: Opc = X86::VPBLENDMQZrrk;
break;
1703 case X86::VMOVUPDZ128rrk: Opc = X86::VBLENDMPDZ128rrk;
break;
1704 case X86::VMOVUPDZ256rrk: Opc = X86::VBLENDMPDZ256rrk;
break;
1705 case X86::VMOVUPDZrrk: Opc = X86::VBLENDMPDZrrk;
break;
1706 case X86::VMOVUPSZ128rrk: Opc = X86::VBLENDMPSZ128rrk;
break;
1707 case X86::VMOVUPSZ256rrk: Opc = X86::VBLENDMPSZ256rrk;
break;
1708 case X86::VMOVUPSZrrk: Opc = X86::VBLENDMPSZrrk;
break;
1709 case X86::VMOVDQA32Z128rrk: Opc = X86::VPBLENDMDZ128rrk;
break;
1710 case X86::VMOVDQA32Z256rrk: Opc = X86::VPBLENDMDZ256rrk;
break;
1711 case X86::VMOVDQA32Zrrk: Opc = X86::VPBLENDMDZrrk;
break;
1712 case X86::VMOVDQA64Z128rrk: Opc = X86::VPBLENDMQZ128rrk;
break;
1713 case X86::VMOVDQA64Z256rrk: Opc = X86::VPBLENDMQZ256rrk;
break;
1714 case X86::VMOVDQA64Zrrk: Opc = X86::VPBLENDMQZrrk;
break;
1715 case X86::VMOVAPDZ128rrk: Opc = X86::VBLENDMPDZ128rrk;
break;
1716 case X86::VMOVAPDZ256rrk: Opc = X86::VBLENDMPDZ256rrk;
break;
1717 case X86::VMOVAPDZrrk: Opc = X86::VBLENDMPDZrrk;
break;
1718 case X86::VMOVAPSZ128rrk: Opc = X86::VBLENDMPSZ128rrk;
break;
1719 case X86::VMOVAPSZ256rrk: Opc = X86::VBLENDMPSZ256rrk;
break;
1720 case X86::VMOVAPSZrrk: Opc = X86::VBLENDMPSZrrk;
break;
1725 .
add(
MI.getOperand(2))
1727 .
add(
MI.getOperand(3));
1732 if (!NewMI)
return nullptr;
1763 unsigned SrcOpIdx2) {
1765 if (SrcOpIdx1 > SrcOpIdx2)
1768 unsigned Op1 = 1, Op2 = 2, Op3 = 3;
1774 if (SrcOpIdx1 == Op1 && SrcOpIdx2 == Op2)
1776 if (SrcOpIdx1 == Op1 && SrcOpIdx2 == Op3)
1778 if (SrcOpIdx1 == Op2 && SrcOpIdx2 == Op3)
1787 unsigned Opc =
MI.getOpcode();
1796 "Intrinsic instructions can't commute operand 1");
1801 assert(Case < 3 &&
"Unexpected case number!");
1806 const unsigned Form132Index = 0;
1807 const unsigned Form213Index = 1;
1808 const unsigned Form231Index = 2;
1809 static const unsigned FormMapping[][3] = {
1814 { Form231Index, Form213Index, Form132Index },
1819 { Form132Index, Form231Index, Form213Index },
1824 { Form213Index, Form132Index, Form231Index }
1827 unsigned FMAForms[3];
1833 for (
unsigned FormIndex = 0; FormIndex < 3; FormIndex++)
1834 if (Opc == FMAForms[FormIndex])
1835 return FMAForms[FormMapping[Case][FormIndex]];
1841 unsigned SrcOpIdx2) {
1845 assert(Case < 3 &&
"Unexpected case value!");
1848 static const uint8_t SwapMasks[3][4] = {
1849 { 0x04, 0x10, 0x08, 0x20 },
1850 { 0x02, 0x10, 0x08, 0x40 },
1851 { 0x02, 0x04, 0x20, 0x40 },
1854 uint8_t
Imm =
MI.getOperand(
MI.getNumOperands()-1).getImm();
1856 uint8_t NewImm =
Imm & ~(SwapMasks[Case][0] | SwapMasks[Case][1] |
1857 SwapMasks[Case][2] | SwapMasks[Case][3]);
1859 if (
Imm & SwapMasks[Case][0]) NewImm |= SwapMasks[Case][1];
1860 if (
Imm & SwapMasks[Case][1]) NewImm |= SwapMasks[Case][0];
1861 if (
Imm & SwapMasks[Case][2]) NewImm |= SwapMasks[Case][3];
1862 if (
Imm & SwapMasks[Case][3]) NewImm |= SwapMasks[Case][2];
1863 MI.getOperand(
MI.getNumOperands()-1).setImm(NewImm);
1869 #define VPERM_CASES(Suffix) \
1870 case X86::VPERMI2##Suffix##128rr: case X86::VPERMT2##Suffix##128rr: \
1871 case X86::VPERMI2##Suffix##256rr: case X86::VPERMT2##Suffix##256rr: \
1872 case X86::VPERMI2##Suffix##rr: case X86::VPERMT2##Suffix##rr: \
1873 case X86::VPERMI2##Suffix##128rm: case X86::VPERMT2##Suffix##128rm: \
1874 case X86::VPERMI2##Suffix##256rm: case X86::VPERMT2##Suffix##256rm: \
1875 case X86::VPERMI2##Suffix##rm: case X86::VPERMT2##Suffix##rm: \
1876 case X86::VPERMI2##Suffix##128rrkz: case X86::VPERMT2##Suffix##128rrkz: \
1877 case X86::VPERMI2##Suffix##256rrkz: case X86::VPERMT2##Suffix##256rrkz: \
1878 case X86::VPERMI2##Suffix##rrkz: case X86::VPERMT2##Suffix##rrkz: \
1879 case X86::VPERMI2##Suffix##128rmkz: case X86::VPERMT2##Suffix##128rmkz: \
1880 case X86::VPERMI2##Suffix##256rmkz: case X86::VPERMT2##Suffix##256rmkz: \
1881 case X86::VPERMI2##Suffix##rmkz: case X86::VPERMT2##Suffix##rmkz:
1883 #define VPERM_CASES_BROADCAST(Suffix) \
1884 VPERM_CASES(Suffix) \
1885 case X86::VPERMI2##Suffix##128rmb: case X86::VPERMT2##Suffix##128rmb: \
1886 case X86::VPERMI2##Suffix##256rmb: case X86::VPERMT2##Suffix##256rmb: \
1887 case X86::VPERMI2##Suffix##rmb: case X86::VPERMT2##Suffix##rmb: \
1888 case X86::VPERMI2##Suffix##128rmbkz: case X86::VPERMT2##Suffix##128rmbkz: \
1889 case X86::VPERMI2##Suffix##256rmbkz: case X86::VPERMT2##Suffix##256rmbkz: \
1890 case X86::VPERMI2##Suffix##rmbkz: case X86::VPERMT2##Suffix##rmbkz:
1893 default:
return false;
1902 #undef VPERM_CASES_BROADCAST
1909 #define VPERM_CASES(Orig, New) \
1910 case X86::Orig##128rr: return X86::New##128rr; \
1911 case X86::Orig##128rrkz: return X86::New##128rrkz; \
1912 case X86::Orig##128rm: return X86::New##128rm; \
1913 case X86::Orig##128rmkz: return X86::New##128rmkz; \
1914 case X86::Orig##256rr: return X86::New##256rr; \
1915 case X86::Orig##256rrkz: return X86::New##256rrkz; \
1916 case X86::Orig##256rm: return X86::New##256rm; \
1917 case X86::Orig##256rmkz: return X86::New##256rmkz; \
1918 case X86::Orig##rr: return X86::New##rr; \
1919 case X86::Orig##rrkz: return X86::New##rrkz; \
1920 case X86::Orig##rm: return X86::New##rm; \
1921 case X86::Orig##rmkz: return X86::New##rmkz;
1923 #define VPERM_CASES_BROADCAST(Orig, New) \
1924 VPERM_CASES(Orig, New) \
1925 case X86::Orig##128rmb: return X86::New##128rmb; \
1926 case X86::Orig##128rmbkz: return X86::New##128rmbkz; \
1927 case X86::Orig##256rmb: return X86::New##256rmb; \
1928 case X86::Orig##256rmbkz: return X86::New##256rmbkz; \
1929 case X86::Orig##rmb: return X86::New##rmb; \
1930 case X86::Orig##rmbkz: return X86::New##rmbkz;
1948 #undef VPERM_CASES_BROADCAST
1954 unsigned OpIdx2)
const {
1957 return *
MI.getParent()->getParent()->CloneMachineInstr(&
MI);
1961 switch (
MI.getOpcode()) {
1962 case X86::SHRD16rri8:
1963 case X86::SHLD16rri8:
1964 case X86::SHRD32rri8:
1965 case X86::SHLD32rri8:
1966 case X86::SHRD64rri8:
1967 case X86::SHLD64rri8:{
1970 switch (
MI.getOpcode()) {
1972 case X86::SHRD16rri8: Size = 16; Opc = X86::SHLD16rri8;
break;
1973 case X86::SHLD16rri8: Size = 16; Opc = X86::SHRD16rri8;
break;
1974 case X86::SHRD32rri8: Size = 32; Opc = X86::SHLD32rri8;
break;
1975 case X86::SHLD32rri8: Size = 32; Opc = X86::SHRD32rri8;
break;
1976 case X86::SHRD64rri8: Size = 64; Opc = X86::SHLD64rri8;
break;
1977 case X86::SHLD64rri8: Size = 64; Opc = X86::SHRD64rri8;
break;
1979 unsigned Amt =
MI.getOperand(3).getImm();
1980 auto &WorkingMI = cloneIfNew(
MI);
1981 WorkingMI.setDesc(
get(Opc));
1982 WorkingMI.getOperand(3).setImm(Size - Amt);
1987 case X86::PFSUBRrr: {
1991 (X86::PFSUBRrr ==
MI.getOpcode() ? X86::PFSUBrr : X86::PFSUBRrr);
1992 auto &WorkingMI = cloneIfNew(
MI);
1993 WorkingMI.setDesc(
get(Opc));
1997 case X86::BLENDPDrri:
1998 case X86::BLENDPSrri:
1999 case X86::VBLENDPDrri:
2000 case X86::VBLENDPSrri:
2002 if (
MI.getParent()->getParent()->getFunction().hasOptSize()) {
2004 switch (
MI.getOpcode()) {
2006 case X86::BLENDPDrri: Opc = X86::MOVSDrr;
Mask = 0x03;
break;
2007 case X86::BLENDPSrri: Opc = X86::MOVSSrr;
Mask = 0x0F;
break;
2008 case X86::VBLENDPDrri: Opc = X86::VMOVSDrr;
Mask = 0x03;
break;
2009 case X86::VBLENDPSrri: Opc = X86::VMOVSSrr;
Mask = 0x0F;
break;
2011 if ((
MI.getOperand(3).getImm() ^
Mask) == 1) {
2012 auto &WorkingMI = cloneIfNew(
MI);
2013 WorkingMI.setDesc(
get(Opc));
2014 WorkingMI.removeOperand(3);
2021 case X86::PBLENDWrri:
2022 case X86::VBLENDPDYrri:
2023 case X86::VBLENDPSYrri:
2024 case X86::VPBLENDDrri:
2025 case X86::VPBLENDWrri:
2026 case X86::VPBLENDDYrri:
2027 case X86::VPBLENDWYrri:{
2029 switch (
MI.getOpcode()) {
2031 case X86::BLENDPDrri:
Mask = (int8_t)0x03;
break;
2032 case X86::BLENDPSrri:
Mask = (int8_t)0x0F;
break;
2033 case X86::PBLENDWrri:
Mask = (int8_t)0xFF;
break;
2034 case X86::VBLENDPDrri:
Mask = (int8_t)0x03;
break;
2035 case X86::VBLENDPSrri:
Mask = (int8_t)0x0F;
break;
2036 case X86::VBLENDPDYrri:
Mask = (int8_t)0x0F;
break;
2037 case X86::VBLENDPSYrri:
Mask = (int8_t)0xFF;
break;
2038 case X86::VPBLENDDrri:
Mask = (int8_t)0x0F;
break;
2039 case X86::VPBLENDWrri:
Mask = (int8_t)0xFF;
break;
2040 case X86::VPBLENDDYrri:
Mask = (int8_t)0xFF;
break;
2041 case X86::VPBLENDWYrri:
Mask = (int8_t)0xFF;
break;
2046 int8_t
Imm =
MI.getOperand(3).getImm() &
Mask;
2047 auto &WorkingMI = cloneIfNew(
MI);
2048 WorkingMI.getOperand(3).setImm(
Mask ^
Imm);
2052 case X86::INSERTPSrr:
2053 case X86::VINSERTPSrr:
2054 case X86::VINSERTPSZrr: {
2055 unsigned Imm =
MI.getOperand(
MI.getNumOperands() - 1).getImm();
2056 unsigned ZMask =
Imm & 15;
2057 unsigned DstIdx = (
Imm >> 4) & 3;
2058 unsigned SrcIdx = (
Imm >> 6) & 3;
2062 if (DstIdx == SrcIdx && (ZMask & (1 << DstIdx)) == 0 &&
2064 unsigned AltIdx =
findFirstSet((ZMask | (1 << DstIdx)) ^ 15);
2065 assert(AltIdx < 4 &&
"Illegal insertion index");
2066 unsigned AltImm = (AltIdx << 6) | (AltIdx << 4) | ZMask;
2067 auto &WorkingMI = cloneIfNew(
MI);
2068 WorkingMI.getOperand(
MI.getNumOperands() - 1).setImm(AltImm);
2077 case X86::VMOVSSrr:{
2081 switch (
MI.getOpcode()) {
2083 case X86::MOVSDrr: Opc = X86::BLENDPDrri;
Mask = 0x02;
break;
2084 case X86::MOVSSrr: Opc = X86::BLENDPSrri;
Mask = 0x0E;
break;
2085 case X86::VMOVSDrr: Opc = X86::VBLENDPDrri;
Mask = 0x02;
break;
2086 case X86::VMOVSSrr: Opc = X86::VBLENDPSrri;
Mask = 0x0E;
break;
2089 auto &WorkingMI = cloneIfNew(
MI);
2090 WorkingMI.setDesc(
get(Opc));
2097 assert(
MI.getOpcode() == X86::MOVSDrr &&
2098 "Can only commute MOVSDrr without SSE4.1");
2100 auto &WorkingMI = cloneIfNew(
MI);
2101 WorkingMI.setDesc(
get(X86::SHUFPDrri));
2106 case X86::SHUFPDrri: {
2108 assert(
MI.getOperand(3).getImm() == 0x02 &&
"Unexpected immediate!");
2109 auto &WorkingMI = cloneIfNew(
MI);
2110 WorkingMI.setDesc(
get(X86::MOVSDrr));
2111 WorkingMI.removeOperand(3);
2115 case X86::PCLMULQDQrr:
2116 case X86::VPCLMULQDQrr:
2117 case X86::VPCLMULQDQYrr:
2118 case X86::VPCLMULQDQZrr:
2119 case X86::VPCLMULQDQZ128rr:
2120 case X86::VPCLMULQDQZ256rr: {
2123 unsigned Imm =
MI.getOperand(3).getImm();
2124 unsigned Src1Hi =
Imm & 0x01;
2125 unsigned Src2Hi =
Imm & 0x10;
2126 auto &WorkingMI = cloneIfNew(
MI);
2127 WorkingMI.getOperand(3).setImm((Src1Hi << 4) | (Src2Hi >> 4));
2131 case X86::VPCMPBZ128rri:
case X86::VPCMPUBZ128rri:
2132 case X86::VPCMPBZ256rri:
case X86::VPCMPUBZ256rri:
2133 case X86::VPCMPBZrri:
case X86::VPCMPUBZrri:
2134 case X86::VPCMPDZ128rri:
case X86::VPCMPUDZ128rri:
2135 case X86::VPCMPDZ256rri:
case X86::VPCMPUDZ256rri:
2136 case X86::VPCMPDZrri:
case X86::VPCMPUDZrri:
2137 case X86::VPCMPQZ128rri:
case X86::VPCMPUQZ128rri:
2138 case X86::VPCMPQZ256rri:
case X86::VPCMPUQZ256rri:
2139 case X86::VPCMPQZrri:
case X86::VPCMPUQZrri:
2140 case X86::VPCMPWZ128rri:
case X86::VPCMPUWZ128rri:
2141 case X86::VPCMPWZ256rri:
case X86::VPCMPUWZ256rri:
2142 case X86::VPCMPWZrri:
case X86::VPCMPUWZrri:
2143 case X86::VPCMPBZ128rrik:
case X86::VPCMPUBZ128rrik:
2144 case X86::VPCMPBZ256rrik:
case X86::VPCMPUBZ256rrik:
2145 case X86::VPCMPBZrrik:
case X86::VPCMPUBZrrik:
2146 case X86::VPCMPDZ128rrik:
case X86::VPCMPUDZ128rrik:
2147 case X86::VPCMPDZ256rrik:
case X86::VPCMPUDZ256rrik:
2148 case X86::VPCMPDZrrik:
case X86::VPCMPUDZrrik:
2149 case X86::VPCMPQZ128rrik:
case X86::VPCMPUQZ128rrik:
2150 case X86::VPCMPQZ256rrik:
case X86::VPCMPUQZ256rrik:
2151 case X86::VPCMPQZrrik:
case X86::VPCMPUQZrrik:
2152 case X86::VPCMPWZ128rrik:
case X86::VPCMPUWZ128rrik:
2153 case X86::VPCMPWZ256rrik:
case X86::VPCMPUWZ256rrik:
2154 case X86::VPCMPWZrrik:
case X86::VPCMPUWZrrik: {
2156 unsigned Imm =
MI.getOperand(
MI.getNumOperands() - 1).getImm() & 0x7;
2158 auto &WorkingMI = cloneIfNew(
MI);
2159 WorkingMI.getOperand(
MI.getNumOperands() - 1).setImm(
Imm);
2163 case X86::VPCOMBri:
case X86::VPCOMUBri:
2164 case X86::VPCOMDri:
case X86::VPCOMUDri:
2165 case X86::VPCOMQri:
case X86::VPCOMUQri:
2166 case X86::VPCOMWri:
case X86::VPCOMUWri: {
2168 unsigned Imm =
MI.getOperand(3).getImm() & 0x7;
2170 auto &WorkingMI = cloneIfNew(
MI);
2171 WorkingMI.getOperand(3).setImm(
Imm);
2175 case X86::VCMPSDZrr:
2176 case X86::VCMPSSZrr:
2177 case X86::VCMPPDZrri:
2178 case X86::VCMPPSZrri:
2179 case X86::VCMPSHZrr:
2180 case X86::VCMPPHZrri:
2181 case X86::VCMPPHZ128rri:
2182 case X86::VCMPPHZ256rri:
2183 case X86::VCMPPDZ128rri:
2184 case X86::VCMPPSZ128rri:
2185 case X86::VCMPPDZ256rri:
2186 case X86::VCMPPSZ256rri:
2187 case X86::VCMPPDZrrik:
2188 case X86::VCMPPSZrrik:
2189 case X86::VCMPPDZ128rrik:
2190 case X86::VCMPPSZ128rrik:
2191 case X86::VCMPPDZ256rrik:
2192 case X86::VCMPPSZ256rrik: {
2194 MI.getOperand(
MI.getNumExplicitOperands() - 1).getImm() & 0x1f;
2196 auto &WorkingMI = cloneIfNew(
MI);
2197 WorkingMI.getOperand(
MI.getNumExplicitOperands() - 1).setImm(
Imm);
2201 case X86::VPERM2F128rr:
2202 case X86::VPERM2I128rr: {
2206 int8_t
Imm =
MI.getOperand(3).getImm() & 0xFF;
2207 auto &WorkingMI = cloneIfNew(
MI);
2208 WorkingMI.getOperand(3).setImm(
Imm ^ 0x22);
2212 case X86::MOVHLPSrr:
2213 case X86::UNPCKHPDrr:
2214 case X86::VMOVHLPSrr:
2215 case X86::VUNPCKHPDrr:
2216 case X86::VMOVHLPSZrr:
2217 case X86::VUNPCKHPDZ128rr: {
2218 assert(Subtarget.
hasSSE2() &&
"Commuting MOVHLP/UNPCKHPD requires SSE2!");
2220 unsigned Opc =
MI.getOpcode();
2223 case X86::MOVHLPSrr: Opc = X86::UNPCKHPDrr;
break;
2224 case X86::UNPCKHPDrr: Opc = X86::MOVHLPSrr;
break;
2225 case X86::VMOVHLPSrr: Opc = X86::VUNPCKHPDrr;
break;
2226 case X86::VUNPCKHPDrr: Opc = X86::VMOVHLPSrr;
break;
2227 case X86::VMOVHLPSZrr: Opc = X86::VUNPCKHPDZ128rr;
break;
2228 case X86::VUNPCKHPDZ128rr: Opc = X86::VMOVHLPSZrr;
break;
2230 auto &WorkingMI = cloneIfNew(
MI);
2231 WorkingMI.setDesc(
get(Opc));
2235 case X86::CMOV16rr:
case X86::CMOV32rr:
case X86::CMOV64rr: {
2236 auto &WorkingMI = cloneIfNew(
MI);
2237 unsigned OpNo =
MI.getDesc().getNumOperands() - 1;
2243 case X86::VPTERNLOGDZrri:
case X86::VPTERNLOGDZrmi:
2244 case X86::VPTERNLOGDZ128rri:
case X86::VPTERNLOGDZ128rmi:
2245 case X86::VPTERNLOGDZ256rri:
case X86::VPTERNLOGDZ256rmi:
2246 case X86::VPTERNLOGQZrri:
case X86::VPTERNLOGQZrmi:
2247 case X86::VPTERNLOGQZ128rri:
case X86::VPTERNLOGQZ128rmi:
2248 case X86::VPTERNLOGQZ256rri:
case X86::VPTERNLOGQZ256rmi:
2249 case X86::VPTERNLOGDZrrik:
2250 case X86::VPTERNLOGDZ128rrik:
2251 case X86::VPTERNLOGDZ256rrik:
2252 case X86::VPTERNLOGQZrrik:
2253 case X86::VPTERNLOGQZ128rrik:
2254 case X86::VPTERNLOGQZ256rrik:
2255 case X86::VPTERNLOGDZrrikz:
case X86::VPTERNLOGDZrmikz:
2256 case X86::VPTERNLOGDZ128rrikz:
case X86::VPTERNLOGDZ128rmikz:
2257 case X86::VPTERNLOGDZ256rrikz:
case X86::VPTERNLOGDZ256rmikz:
2258 case X86::VPTERNLOGQZrrikz:
case X86::VPTERNLOGQZrmikz:
2259 case X86::VPTERNLOGQZ128rrikz:
case X86::VPTERNLOGQZ128rmikz:
2260 case X86::VPTERNLOGQZ256rrikz:
case X86::VPTERNLOGQZ256rmikz:
2261 case X86::VPTERNLOGDZ128rmbi:
2262 case X86::VPTERNLOGDZ256rmbi:
2263 case X86::VPTERNLOGDZrmbi:
2264 case X86::VPTERNLOGQZ128rmbi:
2265 case X86::VPTERNLOGQZ256rmbi:
2266 case X86::VPTERNLOGQZrmbi:
2267 case X86::VPTERNLOGDZ128rmbikz:
2268 case X86::VPTERNLOGDZ256rmbikz:
2269 case X86::VPTERNLOGDZrmbikz:
2270 case X86::VPTERNLOGQZ128rmbikz:
2271 case X86::VPTERNLOGQZ256rmbikz:
2272 case X86::VPTERNLOGQZrmbikz: {
2273 auto &WorkingMI = cloneIfNew(
MI);
2281 auto &WorkingMI = cloneIfNew(
MI);
2282 WorkingMI.setDesc(
get(Opc));
2288 MI.getDesc().TSFlags);
2292 auto &WorkingMI = cloneIfNew(
MI);
2293 WorkingMI.setDesc(
get(Opc));
2304 X86InstrInfo::findThreeSrcCommutedOpIndices(
const MachineInstr &
MI,
2305 unsigned &SrcOpIdx1,
2306 unsigned &SrcOpIdx2,
2307 bool IsIntrinsic)
const {
2310 unsigned FirstCommutableVecOp = 1;
2311 unsigned LastCommutableVecOp = 3;
2312 unsigned KMaskOp = -1U;
2335 FirstCommutableVecOp = 3;
2337 LastCommutableVecOp++;
2338 }
else if (IsIntrinsic) {
2341 FirstCommutableVecOp = 2;
2344 if (
isMem(
MI, LastCommutableVecOp))
2345 LastCommutableVecOp--;
2350 if (SrcOpIdx1 != CommuteAnyOperandIndex &&
2351 (SrcOpIdx1 < FirstCommutableVecOp || SrcOpIdx1 > LastCommutableVecOp ||
2352 SrcOpIdx1 == KMaskOp))
2354 if (SrcOpIdx2 != CommuteAnyOperandIndex &&
2355 (SrcOpIdx2 < FirstCommutableVecOp || SrcOpIdx2 > LastCommutableVecOp ||
2356 SrcOpIdx2 == KMaskOp))
2361 if (SrcOpIdx1 == CommuteAnyOperandIndex ||
2362 SrcOpIdx2 == CommuteAnyOperandIndex) {
2363 unsigned CommutableOpIdx2 = SrcOpIdx2;
2367 if (SrcOpIdx1 == SrcOpIdx2)
2370 CommutableOpIdx2 = LastCommutableVecOp;
2371 else if (SrcOpIdx2 == CommuteAnyOperandIndex)
2373 CommutableOpIdx2 = SrcOpIdx1;
2377 Register Op2Reg =
MI.getOperand(CommutableOpIdx2).getReg();
2379 unsigned CommutableOpIdx1;
2380 for (CommutableOpIdx1 = LastCommutableVecOp;
2381 CommutableOpIdx1 >= FirstCommutableVecOp; CommutableOpIdx1--) {
2383 if (CommutableOpIdx1 == KMaskOp)
2389 if (Op2Reg !=
MI.getOperand(CommutableOpIdx1).getReg())
2394 if (CommutableOpIdx1 < FirstCommutableVecOp)
2399 if (!fixCommutedOpIndices(SrcOpIdx1, SrcOpIdx2,
2400 CommutableOpIdx1, CommutableOpIdx2))
2408 unsigned &SrcOpIdx1,
2409 unsigned &SrcOpIdx2)
const {
2414 switch (
MI.getOpcode()) {
2421 case X86::VCMPPDrri:
2422 case X86::VCMPPSrri:
2423 case X86::VCMPPDYrri:
2424 case X86::VCMPPSYrri:
2425 case X86::VCMPSDZrr:
2426 case X86::VCMPSSZrr:
2427 case X86::VCMPPDZrri:
2428 case X86::VCMPPSZrri:
2429 case X86::VCMPSHZrr:
2430 case X86::VCMPPHZrri:
2431 case X86::VCMPPHZ128rri:
2432 case X86::VCMPPHZ256rri:
2433 case X86::VCMPPDZ128rri:
2434 case X86::VCMPPSZ128rri:
2435 case X86::VCMPPDZ256rri:
2436 case X86::VCMPPSZ256rri:
2437 case X86::VCMPPDZrrik:
2438 case X86::VCMPPSZrrik:
2439 case X86::VCMPPDZ128rrik:
2440 case X86::VCMPPSZ128rrik:
2441 case X86::VCMPPDZ256rrik:
2442 case X86::VCMPPSZ256rrik: {
2447 unsigned Imm =
MI.getOperand(3 + OpOffset).getImm() & 0x7;
2464 return fixCommutedOpIndices(SrcOpIdx1, SrcOpIdx2, 1 + OpOffset,
2474 case X86::SHUFPDrri:
2476 if (
MI.getOperand(3).getImm() == 0x02)
2479 case X86::MOVHLPSrr:
2480 case X86::UNPCKHPDrr:
2481 case X86::VMOVHLPSrr:
2482 case X86::VUNPCKHPDrr:
2483 case X86::VMOVHLPSZrr:
2484 case X86::VUNPCKHPDZ128rr:
2488 case X86::VPTERNLOGDZrri:
case X86::VPTERNLOGDZrmi:
2489 case X86::VPTERNLOGDZ128rri:
case X86::VPTERNLOGDZ128rmi:
2490 case X86::VPTERNLOGDZ256rri:
case X86::VPTERNLOGDZ256rmi:
2491 case X86::VPTERNLOGQZrri:
case X86::VPTERNLOGQZrmi:
2492 case X86::VPTERNLOGQZ128rri:
case X86::VPTERNLOGQZ128rmi:
2493 case X86::VPTERNLOGQZ256rri:
case X86::VPTERNLOGQZ256rmi:
2494 case X86::VPTERNLOGDZrrik:
2495 case X86::VPTERNLOGDZ128rrik:
2496 case X86::VPTERNLOGDZ256rrik:
2497 case X86::VPTERNLOGQZrrik:
2498 case X86::VPTERNLOGQZ128rrik:
2499 case X86::VPTERNLOGQZ256rrik:
2500 case X86::VPTERNLOGDZrrikz:
case X86::VPTERNLOGDZrmikz:
2501 case X86::VPTERNLOGDZ128rrikz:
case X86::VPTERNLOGDZ128rmikz:
2502 case X86::VPTERNLOGDZ256rrikz:
case X86::VPTERNLOGDZ256rmikz:
2503 case X86::VPTERNLOGQZrrikz:
case X86::VPTERNLOGQZrmikz:
2504 case X86::VPTERNLOGQZ128rrikz:
case X86::VPTERNLOGQZ128rmikz:
2505 case X86::VPTERNLOGQZ256rrikz:
case X86::VPTERNLOGQZ256rmikz:
2506 case X86::VPTERNLOGDZ128rmbi:
2507 case X86::VPTERNLOGDZ256rmbi:
2508 case X86::VPTERNLOGDZrmbi:
2509 case X86::VPTERNLOGQZ128rmbi:
2510 case X86::VPTERNLOGQZ256rmbi:
2511 case X86::VPTERNLOGQZrmbi:
2512 case X86::VPTERNLOGDZ128rmbikz:
2513 case X86::VPTERNLOGDZ256rmbikz:
2514 case X86::VPTERNLOGDZrmbikz:
2515 case X86::VPTERNLOGQZ128rmbikz:
2516 case X86::VPTERNLOGQZ256rmbikz:
2517 case X86::VPTERNLOGQZrmbikz:
2518 return findThreeSrcCommutedOpIndices(
MI, SrcOpIdx1, SrcOpIdx2);
2519 case X86::VPDPWSSDYrr:
2520 case X86::VPDPWSSDrr:
2521 case X86::VPDPWSSDSYrr:
2522 case X86::VPDPWSSDSrr:
2523 case X86::VPDPWSSDZ128r:
2524 case X86::VPDPWSSDZ128rk:
2525 case X86::VPDPWSSDZ128rkz:
2526 case X86::VPDPWSSDZ256r:
2527 case X86::VPDPWSSDZ256rk:
2528 case X86::VPDPWSSDZ256rkz:
2529 case X86::VPDPWSSDZr:
2530 case X86::VPDPWSSDZrk:
2531 case X86::VPDPWSSDZrkz:
2532 case X86::VPDPWSSDSZ128r:
2533 case X86::VPDPWSSDSZ128rk:
2534 case X86::VPDPWSSDSZ128rkz:
2535 case X86::VPDPWSSDSZ256r:
2536 case X86::VPDPWSSDSZ256rk:
2537 case X86::VPDPWSSDSZ256rkz:
2538 case X86::VPDPWSSDSZr:
2539 case X86::VPDPWSSDSZrk:
2540 case X86::VPDPWSSDSZrkz:
2541 case X86::VPMADD52HUQZ128r:
2542 case X86::VPMADD52HUQZ128rk:
2543 case X86::VPMADD52HUQZ128rkz:
2544 case X86::VPMADD52HUQZ256r:
2545 case X86::VPMADD52HUQZ256rk:
2546 case X86::VPMADD52HUQZ256rkz:
2547 case X86::VPMADD52HUQZr:
2548 case X86::VPMADD52HUQZrk:
2549 case X86::VPMADD52HUQZrkz:
2550 case X86::VPMADD52LUQZ128r:
2551 case X86::VPMADD52LUQZ128rk:
2552 case X86::VPMADD52LUQZ128rkz:
2553 case X86::VPMADD52LUQZ256r:
2554 case X86::VPMADD52LUQZ256rk:
2555 case X86::VPMADD52LUQZ256rkz:
2556 case X86::VPMADD52LUQZr:
2557 case X86::VPMADD52LUQZrk:
2558 case X86::VPMADD52LUQZrkz:
2559 case X86::VFMADDCPHZr:
2560 case X86::VFMADDCPHZrk:
2561 case X86::VFMADDCPHZrkz:
2562 case X86::VFMADDCPHZ128r:
2563 case X86::VFMADDCPHZ128rk:
2564 case X86::VFMADDCPHZ128rkz:
2565 case X86::VFMADDCPHZ256r:
2566 case X86::VFMADDCPHZ256rk:
2567 case X86::VFMADDCPHZ256rkz:
2568 case X86::VFMADDCSHZr:
2569 case X86::VFMADDCSHZrk:
2570 case X86::VFMADDCSHZrkz: {
2571 unsigned CommutableOpIdx1 = 2;
2572 unsigned CommutableOpIdx2 = 3;
2578 if (!fixCommutedOpIndices(SrcOpIdx1, SrcOpIdx2,
2579 CommutableOpIdx1, CommutableOpIdx2))
2581 if (!
MI.getOperand(SrcOpIdx1).isReg() ||
2582 !
MI.getOperand(SrcOpIdx2).isReg())
2590 MI.getDesc().TSFlags);
2592 return findThreeSrcCommutedOpIndices(
MI, SrcOpIdx1, SrcOpIdx2,
2599 unsigned CommutableOpIdx1 = Desc.
getNumDefs() + 1;
2600 unsigned CommutableOpIdx2 = Desc.
getNumDefs() + 2;
2603 if ((
MI.getDesc().getOperandConstraint(Desc.
getNumDefs(),
2618 if (!fixCommutedOpIndices(SrcOpIdx1, SrcOpIdx2,
2619 CommutableOpIdx1, CommutableOpIdx2))
2622 if (!
MI.getOperand(SrcOpIdx1).isReg() ||
2623 !
MI.getOperand(SrcOpIdx2).isReg())
2635 unsigned Opcode =
MI->getOpcode();
2636 if (Opcode != X86::LEA32r && Opcode != X86::LEA64r &&
2637 Opcode != X86::LEA64_32r)
2659 unsigned Opcode =
MI.getOpcode();
2660 if (Opcode != X86::ADD32rr && Opcode != X86::ADD64rr)
2688 if (!(X86::isJCC(Opcode) || X86::isSETCC(Opcode) || X86::isCMOVCC(Opcode)))
2763 std::pair<X86::CondCode, bool>
2766 bool NeedSwap =
false;
2798 return std::make_pair(CC, NeedSwap);
2805 case 2:
return HasMemoryOperand ? X86::CMOV16rm : X86::CMOV16rr;
2806 case 4:
return HasMemoryOperand ? X86::CMOV32rm : X86::CMOV32rr;
2807 case 8:
return HasMemoryOperand ? X86::CMOV64rm : X86::CMOV64rr;
2832 case 0x01:
Imm = 0x06;
break;
2833 case 0x02:
Imm = 0x05;
break;
2834 case 0x05:
Imm = 0x02;
break;
2835 case 0x06:
Imm = 0x01;
break;
2850 case 0x00:
Imm = 0x02;
break;
2851 case 0x01:
Imm = 0x03;
break;
2852 case 0x02:
Imm = 0x00;
break;
2853 case 0x03:
Imm = 0x01;
break;
2867 switch (
Imm & 0
x3) {
2869 case 0x00:
case 0x03:
2872 case 0x01:
case 0x02:
2883 return (
Reg == X86::FPCW ||
Reg == X86::FPSW ||
2884 (
Reg >= X86::ST0 &&
Reg <= X86::ST7));
2899 switch (
MI.getOpcode()) {
2900 case X86::TCRETURNdi:
2901 case X86::TCRETURNri:
2902 case X86::TCRETURNmi:
2903 case X86::TCRETURNdi64:
2904 case X86::TCRETURNri64:
2905 case X86::TCRETURNmi64:
2915 if (
TailCall.getOpcode() != X86::TCRETURNdi &&
2916 TailCall.getOpcode() != X86::TCRETURNdi64) {
2927 assert(BranchCond.size() == 1);
2935 TailCall.getOperand(1).getImm() != 0) {
2951 if (
I->isDebugInstr())
2954 assert(0 &&
"Can't find the branch to replace!");
2957 assert(BranchCond.size() == 1);
2958 if (CC != BranchCond[0].getImm())
2964 unsigned Opc =
TailCall.getOpcode() == X86::TCRETURNdi ? X86::TCRETURNdicc
2965 : X86::TCRETURNdi64cc;
2979 for (
const auto &
C : Clobbers) {
2984 I->eraseFromParent();
2998 if (Succ->isEHPad() || (Succ == TBB && FallthroughBB))
3001 if (FallthroughBB && FallthroughBB != TBB)
3003 FallthroughBB = Succ;
3005 return FallthroughBB;
3008 bool X86InstrInfo::AnalyzeBranchImpl(
3019 if (
I->isDebugInstr())
3024 if (!isUnpredicatedTerminator(*
I))
3033 if (
I->getOpcode() == X86::JMP_1) {
3037 TBB =
I->getOperand(0).getMBB();
3042 while (std::next(
I) !=
MBB.
end())
3043 std::next(
I)->eraseFromParent();
3051 I->eraseFromParent();
3053 UnCondBrIter =
MBB.
end();
3058 TBB =
I->getOperand(0).getMBB();
3069 if (
I->findRegisterUseOperand(X86::EFLAGS)->isUndef())
3075 if (AllowModify && UnCondBrIter !=
MBB.
end() &&
3098 .
addMBB(UnCondBrIter->getOperand(0).getMBB())
3103 OldInst->eraseFromParent();
3104 UnCondBrIter->eraseFromParent();
3107 UnCondBrIter =
MBB.
end();
3113 TBB =
I->getOperand(0).getMBB();
3115 CondBranches.push_back(&*
I);
3127 auto NewTBB =
I->getOperand(0).getMBB();
3128 if (OldBranchCode == BranchCode && TBB == NewTBB)
3134 if (TBB == NewTBB &&
3167 Cond[0].setImm(BranchCode);
3168 CondBranches.push_back(&*
I);
3178 bool AllowModify)
const {
3180 return AnalyzeBranchImpl(
MBB, TBB, FBB,
Cond, CondBranches, AllowModify);
3184 MachineBranchPredicate &MBP,
3185 bool AllowModify)
const {
3186 using namespace std::placeholders;
3190 if (AnalyzeBranchImpl(
MBB, MBP.TrueDest, MBP.FalseDest,
Cond, CondBranches,
3194 if (
Cond.size() != 1)
3197 assert(MBP.TrueDest &&
"expected!");
3205 bool SingleUseCondition =
true;
3208 if (
MI.modifiesRegister(X86::EFLAGS,
TRI)) {
3213 if (
MI.readsRegister(X86::EFLAGS,
TRI))
3214 SingleUseCondition =
false;
3220 if (SingleUseCondition) {
3222 if (Succ->isLiveIn(X86::EFLAGS))
3223 SingleUseCondition =
false;
3226 MBP.ConditionDef = ConditionDef;
3227 MBP.SingleUseCondition = SingleUseCondition;
3234 const unsigned TestOpcode =
3235 Subtarget.is64Bit() ? X86::TEST64rr : X86::TEST32rr;
3237 if (ConditionDef->
getOpcode() == TestOpcode &&
3253 int *BytesRemoved)
const {
3254 assert(!BytesRemoved &&
"code size not handled");
3261 if (
I->isDebugInstr())
3263 if (
I->getOpcode() != X86::JMP_1 &&
3267 I->eraseFromParent();
3280 int *BytesAdded)
const {
3282 assert(TBB &&
"insertBranch must not be told to insert a fallthrough");
3284 "X86 branch conditions have one component!");
3285 assert(!BytesAdded &&
"code size not handled");
3289 assert(!FBB &&
"Unconditional branch with multiple successors!");
3295 bool FallThru = FBB ==
nullptr;
3310 if (FBB ==
nullptr) {
3312 assert(FBB &&
"MBB cannot be the last block in function when the false "
3313 "body is a fall-through.");
3337 Register FalseReg,
int &CondCycles,
3338 int &TrueCycles,
int &FalseCycles)
const {
3342 if (
Cond.size() != 1)
3356 if (X86::GR16RegClass.hasSubClassEq(RC) ||
3357 X86::GR32RegClass.hasSubClassEq(RC) ||
3358 X86::GR64RegClass.hasSubClassEq(RC)) {
3379 assert(
Cond.size() == 1 &&
"Invalid Cond array");
3390 return X86::GR8_ABCD_HRegClass.contains(
Reg);
3396 bool HasAVX = Subtarget.
hasAVX();
3403 if (X86::VK16RegClass.
contains(SrcReg)) {
3404 if (X86::GR64RegClass.
contains(DestReg)) {
3405 assert(Subtarget.hasBWI());
3406 return X86::KMOVQrk;
3408 if (X86::GR32RegClass.
contains(DestReg))
3409 return Subtarget.hasBWI() ? X86::KMOVDrk : X86::KMOVWrk;
3416 if (X86::VK16RegClass.
contains(DestReg)) {
3417 if (X86::GR64RegClass.
contains(SrcReg)) {
3418 assert(Subtarget.hasBWI());
3419 return X86::KMOVQkr;
3421 if (X86::GR32RegClass.
contains(SrcReg))
3422 return Subtarget.hasBWI() ? X86::KMOVDkr : X86::KMOVWkr;
3431 if (X86::GR64RegClass.
contains(DestReg)) {
3432 if (X86::VR128XRegClass.
contains(SrcReg))
3434 return HasAVX512 ? X86::VMOVPQIto64Zrr :
3435 HasAVX ? X86::VMOVPQIto64rr :
3437 if (X86::VR64RegClass.
contains(SrcReg))
3439 return X86::MMX_MOVD64from64rr;
3440 }
else if (X86::GR64RegClass.
contains(SrcReg)) {
3442 if (X86::VR128XRegClass.
contains(DestReg))
3443 return HasAVX512 ? X86::VMOV64toPQIZrr :
3444 HasAVX ? X86::VMOV64toPQIrr :
3447 if (X86::VR64RegClass.
contains(DestReg))
3448 return X86::MMX_MOVD64to64rr;
3454 if (X86::GR32RegClass.
contains(DestReg) &&
3455 X86::VR128XRegClass.
contains(SrcReg))
3457 return HasAVX512 ? X86::VMOVPDI2DIZrr :
3458 HasAVX ? X86::VMOVPDI2DIrr :
3461 if (X86::VR128XRegClass.
contains(DestReg) &&
3462 X86::GR32RegClass.
contains(SrcReg))
3464 return HasAVX512 ? X86::VMOVDI2PDIZrr :
3465 HasAVX ? X86::VMOVDI2PDIrr :
3475 bool HasAVX = Subtarget.
hasAVX();
3476 bool HasVLX = Subtarget.hasVLX();
3478 if (X86::GR64RegClass.
contains(DestReg, SrcReg))
3480 else if (X86::GR32RegClass.
contains(DestReg, SrcReg))
3482 else if (X86::GR16RegClass.
contains(DestReg, SrcReg))
3484 else if (X86::GR8RegClass.
contains(DestReg, SrcReg)) {
3488 Subtarget.is64Bit()) {
3489 Opc = X86::MOV8rr_NOREX;
3492 "8-bit H register can not be copied outside GR8_NOREX");
3496 else if (X86::VR64RegClass.
contains(DestReg, SrcReg))
3497 Opc = X86::MMX_MOVQ64rr;
3498 else if (X86::VR128XRegClass.
contains(DestReg, SrcReg)) {
3500 Opc = X86::VMOVAPSZ128rr;
3501 else if (X86::VR128RegClass.
contains(DestReg, SrcReg))
3502 Opc = HasAVX ? X86::VMOVAPSrr : X86::MOVAPSrr;
3506 Opc = X86::VMOVAPSZrr;
3509 &X86::VR512RegClass);
3511 &X86::VR512RegClass);
3513 }
else if (X86::VR256XRegClass.
contains(DestReg, SrcReg)) {
3515 Opc = X86::VMOVAPSZ256rr;
3516 else if (X86::VR256RegClass.
contains(DestReg, SrcReg))
3517 Opc = X86::VMOVAPSYrr;
3521 Opc = X86::VMOVAPSZrr;
3524 &X86::VR512RegClass);
3526 &X86::VR512RegClass);
3528 }
else if (X86::VR512RegClass.
contains(DestReg, SrcReg))
3529 Opc = X86::VMOVAPSZrr;
3531 else if (X86::VK16RegClass.
contains(DestReg, SrcReg))
3532 Opc = Subtarget.hasBWI() ? X86::KMOVQkk : X86::KMOVWkk;
3542 if (SrcReg == X86::EFLAGS || DestReg == X86::EFLAGS) {
3550 LLVM_DEBUG(
dbgs() <<
"Cannot copy " << RI.getName(SrcReg) <<
" to "
3551 << RI.getName(DestReg) <<
'\n');
3564 bool IsStackAligned,
3566 bool HasAVX = STI.
hasAVX();
3568 bool HasVLX = STI.hasVLX();
3574 assert(X86::GR8RegClass.hasSubClassEq(RC) &&
"Unknown 1-byte regclass");
3578 if (
isHReg(
Reg) || X86::GR8_ABCD_HRegClass.hasSubClassEq(RC))
3579 return load ? X86::MOV8rm_NOREX : X86::MOV8mr_NOREX;
3580 return load ? X86::MOV8rm : X86::MOV8mr;
3582 if (X86::VK16RegClass.hasSubClassEq(RC))
3583 return load ? X86::KMOVWkm : X86::KMOVWmk;
3584 if (X86::FR16XRegClass.hasSubClassEq(RC)) {
3586 return load ? X86::VMOVSHZrm_alt : X86::VMOVSHZmr;
3588 assert(X86::GR16RegClass.hasSubClassEq(RC) &&
"Unknown 2-byte regclass");
3589 return load ? X86::MOV16rm : X86::MOV16mr;
3591 if (X86::GR32RegClass.hasSubClassEq(RC))
3592 return load ? X86::MOV32rm : X86::MOV32mr;
3593 if (X86::FR32XRegClass.hasSubClassEq(RC))
3595 (HasAVX512 ? X86::VMOVSSZrm_alt :
3596 HasAVX ? X86::VMOVSSrm_alt :
3598 (HasAVX512 ? X86::VMOVSSZmr :
3599 HasAVX ? X86::VMOVSSmr :
3601 if (X86::RFP32RegClass.hasSubClassEq(RC))
3602 return load ? X86::LD_Fp32m : X86::ST_Fp32m;
3603 if (X86::VK32RegClass.hasSubClassEq(RC)) {
3604 assert(STI.hasBWI() &&
"KMOVD requires BWI");
3605 return load ? X86::KMOVDkm : X86::KMOVDmk;
3609 if (X86::VK1PAIRRegClass.hasSubClassEq(RC) ||
3610 X86::VK2PAIRRegClass.hasSubClassEq(RC) ||
3611 X86::VK4PAIRRegClass.hasSubClassEq(RC) ||
3612 X86::VK8PAIRRegClass.hasSubClassEq(RC) ||
3613 X86::VK16PAIRRegClass.hasSubClassEq(RC))
3614 return load ? X86::MASKPAIR16LOAD : X86::MASKPAIR16STORE;
3617 if (X86::GR64RegClass.hasSubClassEq(RC))
3618 return load ? X86::MOV64rm : X86::MOV64mr;
3619 if (X86::FR64XRegClass.hasSubClassEq(RC))
3621 (HasAVX512 ? X86::VMOVSDZrm_alt :
3622 HasAVX ? X86::VMOVSDrm_alt :
3624 (HasAVX512 ? X86::VMOVSDZmr :
3625 HasAVX ? X86::VMOVSDmr :
3627 if (X86::VR64RegClass.hasSubClassEq(RC))
3628 return load ? X86::MMX_MOVQ64rm : X86::MMX_MOVQ64mr;
3629 if (X86::RFP64RegClass.hasSubClassEq(RC))
3630 return load ? X86::LD_Fp64m : X86::ST_Fp64m;
3631 if (X86::VK64RegClass.hasSubClassEq(RC)) {
3632 assert(STI.hasBWI() &&
"KMOVQ requires BWI");
3633 return load ? X86::KMOVQkm : X86::KMOVQmk;
3637 assert(X86::RFP80RegClass.hasSubClassEq(RC) &&
"Unknown 10-byte regclass");
3638 return load ? X86::LD_Fp80m : X86::ST_FpP80m;
3640 if (X86::VR128XRegClass.hasSubClassEq(RC)) {
3644 (HasVLX ? X86::VMOVAPSZ128rm :
3645 HasAVX512 ? X86::VMOVAPSZ128rm_NOVLX :
3646 HasAVX ? X86::VMOVAPSrm :
3648 (HasVLX ? X86::VMOVAPSZ128mr :
3649 HasAVX512 ? X86::VMOVAPSZ128mr_NOVLX :
3650 HasAVX ? X86::VMOVAPSmr :
3654 (HasVLX ? X86::VMOVUPSZ128rm :
3655 HasAVX512 ? X86::VMOVUPSZ128rm_NOVLX :
3656 HasAVX ? X86::VMOVUPSrm :
3658 (HasVLX ? X86::VMOVUPSZ128mr :
3659 HasAVX512 ? X86::VMOVUPSZ128mr_NOVLX :
3660 HasAVX ? X86::VMOVUPSmr :
3666 assert(X86::VR256XRegClass.hasSubClassEq(RC) &&
"Unknown 32-byte regclass");
3670 (HasVLX ? X86::VMOVAPSZ256rm :
3671 HasAVX512 ? X86::VMOVAPSZ256rm_NOVLX :
3673 (HasVLX ? X86::VMOVAPSZ256mr :
3674 HasAVX512 ? X86::VMOVAPSZ256mr_NOVLX :
3678 (HasVLX ? X86::VMOVUPSZ256rm :
3679 HasAVX512 ? X86::VMOVUPSZ256rm_NOVLX :
3681 (HasVLX ? X86::VMOVUPSZ256mr :
3682 HasAVX512 ? X86::VMOVUPSZ256mr_NOVLX :
3685 assert(X86::VR512RegClass.hasSubClassEq(RC) &&
"Unknown 64-byte regclass");
3688 return load ? X86::VMOVAPSZrm : X86::VMOVAPSZmr;
3690 return load ? X86::VMOVUPSZrm : X86::VMOVUPSZmr;
3699 if (MemRefBegin < 0)
3705 if (!BaseOp.isReg())
3710 if (!DispMO.
isImm())
3737 ErrInfo =
"Scale factor in address must be 1, 2, 4 or 8";
3742 ErrInfo =
"Displacement in address must fit into 32-bit signed "
3752 int64_t &ImmVal)
const {
3753 if (
MI.getOpcode() != X86::MOV32ri &&
MI.getOpcode() != X86::MOV64ri)
3756 if (!
MI.getOperand(1).isImm() ||
MI.getOperand(0).getReg() !=
Reg)
3758 ImmVal =
MI.getOperand(1).getImm();
3765 if (!
MI->modifiesRegister(NullValueReg,
TRI))
3767 switch (
MI->getOpcode()) {
3774 assert(
MI->getOperand(0).isDef() &&
MI->getOperand(1).isUse() &&
3775 "expected for shift opcode!");
3776 return MI->getOperand(0).getReg() == NullValueReg &&
3777 MI->getOperand(1).getReg() == NullValueReg;
3782 return TRI->isSubRegisterEq(NullValueReg, MO.getReg());
3792 int64_t &Offset,
bool &OffsetIsScalable,
unsigned &
Width,
3796 if (MemRefBegin < 0)
3803 if (!BaseOp->
isReg())
3816 if (!DispMO.
isImm())
3819 Offset = DispMO.
getImm();
3821 if (!BaseOp->
isReg())
3824 OffsetIsScalable =
false;
3829 !
MemOp.memoperands_empty() ?
MemOp.memoperands().front()->getSize() : 0;
3830 BaseOps.push_back(BaseOp);
3836 bool IsStackAligned,
3849 Register SrcReg,
bool isKill,
int FrameIdx,
3855 "Stack slot too small for store");
3856 if (RC->
getID() == X86::TILERegClassID) {
3857 unsigned Opc = X86::TILESTORED;
3884 if (RC->
getID() == X86::TILERegClassID) {
3885 unsigned Opc = X86::TILELOADD;
3910 Register &SrcReg2, int64_t &CmpMask,
3911 int64_t &CmpValue)
const {
3912 switch (
MI.getOpcode()) {
3914 case X86::CMP64ri32:
3921 SrcReg =
MI.getOperand(0).getReg();
3923 if (
MI.getOperand(1).isImm()) {
3925 CmpValue =
MI.getOperand(1).getImm();
3927 CmpMask = CmpValue = 0;
3935 SrcReg =
MI.getOperand(1).getReg();
3944 SrcReg =
MI.getOperand(1).getReg();
3945 SrcReg2 =
MI.getOperand(2).getReg();
3949 case X86::SUB64ri32:
3956 SrcReg =
MI.getOperand(1).getReg();
3958 if (
MI.getOperand(2).isImm()) {
3960 CmpValue =
MI.getOperand(2).getImm();
3962 CmpMask = CmpValue = 0;
3969 SrcReg =
MI.getOperand(0).getReg();
3970 SrcReg2 =
MI.getOperand(1).getReg();
3978 SrcReg =
MI.getOperand(0).getReg();
3979 if (
MI.getOperand(1).getReg() != SrcReg)
3990 bool X86InstrInfo::isRedundantFlagInstr(
const MachineInstr &FlagI,
3992 int64_t
ImmMask, int64_t ImmValue,
3994 int64_t *ImmDelta)
const {
4009 OIMask !=
ImmMask || OIValue != ImmValue)
4011 if (SrcReg == OISrcReg && SrcReg2 == OISrcReg2) {
4015 if (SrcReg == OISrcReg2 && SrcReg2 == OISrcReg) {
4021 case X86::CMP64ri32:
4028 case X86::SUB64ri32:
4038 case X86::TEST8rr: {
4045 SrcReg == OISrcReg &&
ImmMask == OIMask) {
4046 if (OIValue == ImmValue) {
4049 }
else if (
static_cast<uint64_t>(ImmValue) ==
4050 static_cast<uint64_t>(OIValue) - 1) {
4053 }
else if (
static_cast<uint64_t>(ImmValue) ==
4054 static_cast<uint64_t>(OIValue) + 1) {
4072 bool &ClearsOverflowFlag) {
4074 ClearsOverflowFlag =
false;
4076 switch (
MI.getOpcode()) {
4077 default:
return false;
4081 case X86::SAR8ri:
case X86::SAR16ri:
case X86::SAR32ri:
case X86::SAR64ri:
4082 case X86::SHR8ri:
case X86::SHR16ri:
case X86::SHR32ri:
case X86::SHR64ri:
4087 case X86::SHL8ri:
case X86::SHL16ri:
case X86::SHL32ri:
case X86::SHL64ri:{
4093 case X86::SHRD16rri8:
case X86::SHRD32rri8:
case X86::SHRD64rri8:
4094 case X86::SHLD16rri8:
case X86::SHLD32rri8:
case X86::SHLD64rri8:
4097 case X86::SUB64ri32:
case X86::SUB64ri8:
case X86::SUB32ri:
4098 case X86::SUB32ri8:
case X86::SUB16ri:
case X86::SUB16ri8:
4099 case X86::SUB8ri:
case X86::SUB64rr:
case X86::SUB32rr:
4100 case X86::SUB16rr:
case X86::SUB8rr:
case X86::SUB64rm:
4101 case X86::SUB32rm:
case X86::SUB16rm:
case X86::SUB8rm:
4102 case X86::DEC64r:
case X86::DEC32r:
case X86::DEC16r:
case X86::DEC8r:
4103 case X86::ADD64ri32:
case X86::ADD64ri8:
case X86::ADD32ri:
4104 case X86::ADD32ri8:
case X86::ADD16ri:
case X86::ADD16ri8:
4105 case X86::ADD8ri:
case X86::ADD64rr:
case X86::ADD32rr:
4106 case X86::ADD16rr:
case X86::ADD8rr:
case X86::ADD64rm:
4107 case X86::ADD32rm:
case X86::ADD16rm:
case X86::ADD8rm:
4108 case X86::INC64r:
case X86::INC32r:
case X86::INC16r:
case X86::INC8r:
4109 case X86::ADC64ri32:
case X86::ADC64ri8:
case X86::ADC32ri:
4110 case X86::ADC32ri8:
case X86::ADC16ri:
case X86::ADC16ri8:
4111 case X86::ADC8ri:
case X86::ADC64rr:
case X86::ADC32rr:
4112 case X86::ADC16rr:
case X86::ADC8rr:
case X86::ADC64rm:
4113 case X86::ADC32rm:
case X86::ADC16rm:
case X86::ADC8rm:
4114 case X86::SBB64ri32:
case X86::SBB64ri8:
case X86::SBB32ri:
4115 case X86::SBB32ri8:
case X86::SBB16ri:
case X86::SBB16ri8:
4116 case X86::SBB8ri:
case X86::SBB64rr:
case X86::SBB32rr:
4117 case X86::SBB16rr:
case X86::SBB8rr:
case X86::SBB64rm:
4118 case X86::SBB32rm:
case X86::SBB16rm:
case X86::SBB8rm:
4119 case X86::NEG8r:
case X86::NEG16r:
case X86::NEG32r:
case X86::NEG64r:
4120 case X86::SAR8r1:
case X86::SAR16r1:
case X86::SAR32r1:
case X86::SAR64r1:
4121 case X86::SHR8r1:
case X86::SHR16r1:
case X86::SHR32r1:
case X86::SHR64r1:
4122 case X86::SHL8r1:
case X86::SHL16r1:
case X86::SHL32r1:
case X86::SHL64r1:
4123 case X86::LZCNT16rr:
case X86::LZCNT16rm:
4124 case X86::LZCNT32rr:
case X86::LZCNT32rm:
4125 case X86::LZCNT64rr:
case X86::LZCNT64rm:
4126 case X86::POPCNT16rr:
case X86::POPCNT16rm:
4127 case X86::POPCNT32rr:
case X86::POPCNT32rm:
4128 case X86::POPCNT64rr:
case X86::POPCNT64rm:
4129 case X86::TZCNT16rr:
case X86::TZCNT16rm:
4130 case X86::TZCNT32rr:
case X86::TZCNT32rm:
4131 case X86::TZCNT64rr:
case X86::TZCNT64rm:
4133 case X86::AND64ri32:
case X86::AND64ri8:
case X86::AND32ri:
4134 case X86::AND32ri8:
case X86::AND16ri:
case X86::AND16ri8:
4135 case X86::AND8ri:
case X86::AND64rr:
case X86::AND32rr:
4136 case X86::AND16rr:
case X86::AND8rr:
case X86::AND64rm:
4137 case X86::AND32rm:
case X86::AND16rm:
case X86::AND8rm:
4138 case X86::XOR64ri32:
case X86::XOR64ri8:
case X86::XOR32ri:
4139 case X86::XOR32ri8:
case X86::XOR16ri:
case X86::XOR16ri8:
4140 case X86::XOR8ri:
case X86::XOR64rr:
case X86::XOR32rr:
4141 case X86::XOR16rr:
case X86::XOR8rr:
case X86::XOR64rm:
4142 case X86::XOR32rm:
case X86::XOR16rm:
case X86::XOR8rm:
4143 case X86::OR64ri32:
case X86::OR64ri8:
case X86::OR32ri:
4144 case X86::OR32ri8:
case X86::OR16ri:
case X86::OR16ri8:
4145 case X86::OR8ri:
case X86::OR64rr:
case X86::OR32rr:
4146 case X86::OR16rr:
case X86::OR8rr:
case X86::OR64rm:
4147 case X86::OR32rm:
case X86::OR16rm:
case X86::OR8rm:
4148 case X86::ANDN32rr:
case X86::ANDN32rm:
4149 case X86::ANDN64rr:
case X86::ANDN64rm:
4150 case X86::BLSI32rr:
case X86::BLSI32rm:
4151 case X86::BLSI64rr:
case X86::BLSI64rm:
4152 case X86::BLSMSK32rr:
case X86::BLSMSK32rm:
4153 case X86::BLSMSK64rr:
case X86::BLSMSK64rm:
4154 case X86::BLSR32rr:
case X86::BLSR32rm:
4155 case X86::BLSR64rr:
case X86::BLSR64rm:
4156 case X86::BLCFILL32rr:
case X86::BLCFILL32rm:
4157 case X86::BLCFILL64rr:
case X86::BLCFILL64rm:
4158 case X86::BLCI32rr:
case X86::BLCI32rm:
4159 case X86::BLCI64rr:
case X86::BLCI64rm:
4160 case X86::BLCIC32rr:
case X86::BLCIC32rm:
4161 case X86::BLCIC64rr:
case X86::BLCIC64rm:
4162 case X86::BLCMSK32rr:
case X86::BLCMSK32rm:
4163 case X86::BLCMSK64rr:
case X86::BLCMSK64rm:
4164 case X86::BLCS32rr:
case X86::BLCS32rm:
4165 case X86::BLCS64rr:
case X86::BLCS64rm:
4166 case X86::BLSFILL32rr:
case X86::BLSFILL32rm:
4167 case X86::BLSFILL64rr:
case X86::BLSFILL64rm:
4168 case X86::BLSIC32rr:
case X86::BLSIC32rm:
4169 case X86::BLSIC64rr:
case X86::BLSIC64rm:
4170 case X86::BZHI32rr:
case X86::BZHI32rm:
4171 case X86::BZHI64rr:
case X86::BZHI64rm:
4172 case X86::T1MSKC32rr:
case X86::T1MSKC32rm:
4173 case X86::T1MSKC64rr:
case X86::T1MSKC64rm:
4174 case X86::TZMSK32rr:
case X86::TZMSK32rm:
4175 case X86::TZMSK64rr:
case X86::TZMSK64rm:
4179 ClearsOverflowFlag =
true;
4181 case X86::BEXTR32rr:
case X86::BEXTR64rr:
4182 case X86::BEXTR32rm:
case X86::BEXTR64rm:
4183 case X86::BEXTRI32ri:
case X86::BEXTRI32mi:
4184 case X86::BEXTRI64ri:
case X86::BEXTRI64mi:
4194 switch (
MI.getOpcode()) {
4201 case X86::LZCNT16rr:
4202 case X86::LZCNT32rr:
4203 case X86::LZCNT64rr:
4205 case X86::POPCNT16rr:
4206 case X86::POPCNT32rr:
4207 case X86::POPCNT64rr:
4209 case X86::TZCNT16rr:
4210 case X86::TZCNT32rr:
4211 case X86::TZCNT64rr:
4225 case X86::BLSMSK32rr:
4226 case X86::BLSMSK64rr:
4242 case X86::SUB64ri32:
4260 unsigned NewOpcode = 0;
4263 case X86::SUB64rm: NewOpcode = X86::CMP64rm;
break;
4264 case X86::SUB32rm: NewOpcode = X86::CMP32rm;
break;
4265 case X86::SUB16rm: NewOpcode = X86::CMP16rm;
break;
4266 case X86::SUB8rm: NewOpcode = X86::CMP8rm;
break;
4267 case X86::SUB64rr: NewOpcode = X86::CMP64rr;
break;
4268 case X86::SUB32rr: NewOpcode = X86::CMP32rr;
break;
4269 case X86::SUB16rr: NewOpcode = X86::CMP16rr;
break;
4270 case X86::SUB8rr: NewOpcode = X86::CMP8rr;
break;
4271 case X86::SUB64ri32: NewOpcode = X86::CMP64ri32;
break;
4272 case X86::SUB64ri8: NewOpcode = X86::CMP64ri8;
break;
4273 case X86::SUB32ri: NewOpcode = X86::CMP32ri;
break;
4274 case X86::SUB32ri8: NewOpcode = X86::CMP32ri8;
break;
4275 case X86::SUB16ri: NewOpcode = X86::CMP16ri;
break;
4276 case X86::SUB16ri8: NewOpcode = X86::CMP16ri8;
break;
4277 case X86::SUB8ri: NewOpcode = X86::CMP8ri;
break;
4284 if (NewOpcode == X86::CMP64rm || NewOpcode == X86::CMP32rm ||
4285 NewOpcode == X86::CMP16rm || NewOpcode == X86::CMP8rm)
4293 bool IsCmpZero = (CmpMask != 0 && CmpValue == 0);
4299 assert(SrcRegDef &&
"Must have a definition (SSA)");
4304 bool NoSignFlag =
false;
4305 bool ClearsOverflowFlag =
false;
4306 bool ShouldUpdateCC =
false;
4307 bool IsSwapped =
false;
4309 int64_t ImmDelta = 0;
4322 if (&Inst == SrcRegDef) {
4340 NoSignFlag, ClearsOverflowFlag)) {
4349 if (Inst.modifiesRegister(X86::EFLAGS,
TRI)) {
4360 Inst.getOperand(1).getReg() == SrcReg) {
4361 ShouldUpdateCC =
true;
4372 if (isRedundantFlagInstr(CmpInstr, SrcReg, SrcReg2, CmpMask, CmpValue,
4373 Inst, &IsSwapped, &ImmDelta)) {
4381 if (!Movr0Inst && Inst.
getOpcode() == X86::MOV32r0 &&
4382 Inst.registerDefIsDead(X86::EFLAGS,
TRI)) {
4407 bool FlagsMayLiveOut =
true;
4412 bool ModifyEFLAGS = Instr.modifiesRegister(X86::EFLAGS,
TRI);
4413 bool UseEFLAGS = Instr.readsRegister(X86::EFLAGS,
TRI);
4415 if (!UseEFLAGS && ModifyEFLAGS) {
4417 FlagsMayLiveOut =
false;
4420 if (!UseEFLAGS && !ModifyEFLAGS)
4425 if (
MI || IsSwapped || ImmDelta != 0) {
4427 if (Instr.isBranch())
4448 if (!ClearsOverflowFlag)
4466 ReplacementCC = NewCC;
4472 }
else if (IsSwapped) {
4479 ShouldUpdateCC =
true;
4480 }
else if (ImmDelta != 0) {
4491 if (ImmDelta != 1 || CmpValue == 0)
4501 if (ImmDelta != 1 || CmpValue == 0)
4528 ShouldUpdateCC =
true;
4531 if (ShouldUpdateCC && ReplacementCC != OldCC) {
4535 OpsToUpdate.push_back(std::make_pair(&Instr, ReplacementCC));
4537 if (ModifyEFLAGS || Instr.killsRegister(X86::EFLAGS,
TRI)) {
4539 FlagsMayLiveOut =
false;
4546 if ((
MI !=
nullptr || ShouldUpdateCC) && FlagsMayLiveOut) {
4553 assert((
MI ==
nullptr || Sub ==
nullptr) &&
"Should not have Sub and MI set");
4554 Sub =
MI !=
nullptr ?
MI : Sub;
4560 if (&CmpMBB != SubBB)
4565 for (; InsertI != InsertE; ++InsertI) {
4575 if (InsertI == InsertE)
4581 assert(FlagDef &&
"Unable to locate a def EFLAGS operand");
4587 for (
auto &
Op : OpsToUpdate) {
4588 Op.first->getOperand(
Op.first->getDesc().getNumOperands() - 1)
4612 bool SawStore =
false;
4618 for (
unsigned i = 0,
e =
MI.getNumOperands();
i !=
e; ++
i) {
4623 if (
Reg != FoldAsLoadDefReg)
4628 SrcOperandIds.push_back(
i);
4630 if (SrcOperandIds.empty())
4635 FoldAsLoadDefReg = 0;
4660 MIB.
getReg(2) ==
Reg &&
"Misplaced operand");
4690 MIB->
setDesc(
TII.get(MinusOne ? X86::DEC32r : X86::INC32r));
4702 assert(
Imm != 0 &&
"Using push/pop for 0 is not efficient.");
4707 if (Subtarget.is64Bit()) {
4709 MIB->
getOpcode() == X86::MOV32ImmSExti8);
4716 X86::MOV32ImmSExti8 ? X86::MOV32ri : X86::MOV64ri));
4741 bool EmitCFI = !TFL->
hasFP(MF) && NeedsDwarfCFI;
4782 MIB->
getOpcode() == X86::XOR64_FP ? X86::XOR64rr : X86::XOR32rr;
4850 bool HasAVX = Subtarget.
hasAVX();
4852 switch (
MI.getOpcode()) {
4859 case X86::MOV32ImmSExti8:
4860 case X86::MOV64ImmSExti8:
4862 case X86::SETB_C32r:
4864 case X86::SETB_C64r:
4871 case X86::FsFLD0F128:
4873 case X86::AVX_SET0: {
4874 assert(HasAVX &&
"AVX not supported");
4883 case X86::AVX512_128_SET0:
4884 case X86::AVX512_FsFLD0SH:
4885 case X86::AVX512_FsFLD0SS:
4886 case X86::AVX512_FsFLD0SD:
4887 case X86::AVX512_FsFLD0F128: {
4888 bool HasVLX = Subtarget.hasVLX();
4893 get(HasVLX ? X86::VPXORDZ128rr : X86::VXORPSrr));
4900 case X86::AVX512_256_SET0:
4901 case X86::AVX512_512_SET0: {
4902 bool HasVLX = Subtarget.hasVLX();
4909 get(HasVLX ? X86::VPXORDZ128rr : X86::VXORPSrr));
4913 if (
MI.getOpcode() == X86::AVX512_256_SET0) {
4921 case X86::V_SETALLONES:
4923 case X86::AVX2_SETALLONES:
4925 case X86::AVX1_SETALLONES: {
4932 case X86::AVX512_512_SETALLONES: {
4941 case X86::AVX512_512_SEXT_MASK_32:
4942 case X86::AVX512_512_SEXT_MASK_64: {
4946 unsigned Opc = (
MI.getOpcode() == X86::AVX512_512_SEXT_MASK_64) ?
4947 X86::VPTERNLOGQZrrikz : X86::VPTERNLOGDZrrikz;
4948 MI.removeOperand(1);
4956 case X86::VMOVAPSZ128rm_NOVLX:
4958 get(X86::VBROADCASTF32X4rm), X86::sub_xmm);
4959 case X86::VMOVUPSZ128rm_NOVLX:
4961 get(X86::VBROADCASTF32X4rm), X86::sub_xmm);
4962 case X86::VMOVAPSZ256rm_NOVLX:
4964 get(X86::VBROADCASTF64X4rm), X86::sub_ymm);
4965 case X86::VMOVUPSZ256rm_NOVLX:
4967 get(X86::VBROADCASTF64X4rm), X86::sub_ymm);
4968 case X86::VMOVAPSZ128mr_NOVLX:
4970 get(X86::VEXTRACTF32x4Zmr), X86::sub_xmm);
4971 case X86::VMOVUPSZ128mr_NOVLX:
4973 get(X86::VEXTRACTF32x4Zmr), X86::sub_xmm);
4974 case X86::VMOVAPSZ256mr_NOVLX:
4976 get(X86::VEXTRACTF64x4Zmr), X86::sub_ymm);
4977 case X86::VMOVUPSZ256mr_NOVLX:
4979 get(X86::VEXTRACTF64x4Zmr), X86::sub_ymm);
4980 case X86::MOV32ri64: {
4982 Register Reg32 = RI.getSubReg(
Reg, X86::sub_32bit);
4983 MI.setDesc(
get(X86::MOV32ri));
5002 case TargetOpcode::LOAD_STACK_GUARD:
5012 case X86::ADD8rr_DB: MIB->
setDesc(
get(X86::OR8rr));
break;
5013 case X86::ADD16rr_DB: MIB->
setDesc(
get(X86::OR16rr));
break;
5014 case X86::ADD32rr_DB: MIB->
setDesc(
get(X86::OR32rr));
break;
5015 case X86::ADD64rr_DB: MIB->
setDesc(
get(X86::OR64rr));
break;
5016 case X86::ADD8ri_DB: MIB->
setDesc(
get(X86::OR8ri));
break;
5017 case X86::ADD16ri_DB: MIB->
setDesc(
get(X86::OR16ri));
break;
5018 case X86::ADD32ri_DB: MIB->
setDesc(
get(X86::OR32ri));
break;
5019 case X86::ADD64ri32_DB: MIB->
setDesc(
get(X86::OR64ri32));
break;
5020 case X86::ADD16ri8_DB: MIB->
setDesc(
get(X86::OR16ri8));
break;
5021 case X86::ADD32ri8_DB: MIB->
setDesc(
get(X86::OR32ri8));
break;
5022 case X86::ADD64ri8_DB: MIB->
setDesc(
get(X86::OR64ri8));
break;
5045 bool ForLoadFold =
false) {
5047 case X86::CVTSI2SSrr:
5048 case X86::CVTSI2SSrm:
5049 case X86::CVTSI642SSrr:
5050 case X86::CVTSI642SSrm:
5051 case X86::CVTSI2SDrr:
5052 case X86::CVTSI2SDrm:
5053 case X86::CVTSI642SDrr:
5054 case X86::CVTSI642SDrm:
5057 return !ForLoadFold;
5058 case X86::CVTSD2SSrr:
5059 case X86::CVTSD2SSrm:
5060 case X86::CVTSS2SDrr:
5061 case X86::CVTSS2SDrm:
5068 case X86::RCPSSr_Int:
5069 case X86::RCPSSm_Int:
5076 case X86::RSQRTSSr_Int:
5077 case X86::RSQRTSSm_Int:
5080 case X86::SQRTSSr_Int:
5081 case X86::SQRTSSm_Int:
5084 case X86::SQRTSDr_Int:
5085 case X86::SQRTSDm_Int:
5087 case X86::VFCMULCPHZ128rm:
5088 case X86::VFCMULCPHZ128rmb:
5089 case X86::VFCMULCPHZ128rmbkz:
5090 case X86::VFCMULCPHZ128rmkz:
5091 case X86::VFCMULCPHZ128rr:
5092 case X86::VFCMULCPHZ128rrkz:
5093 case X86::VFCMULCPHZ256rm:
5094 case X86::VFCMULCPHZ256rmb:
5095 case X86::VFCMULCPHZ256rmbkz:
5096 case X86::VFCMULCPHZ256rmkz:
5097 case X86::VFCMULCPHZ256rr:
5098 case X86::VFCMULCPHZ256rrkz:
5099 case X86::VFCMULCPHZrm:
5100 case X86::VFCMULCPHZrmb:
5101 case X86::VFCMULCPHZrmbkz:
5102 case X86::VFCMULCPHZrmkz:
5103 case X86::VFCMULCPHZrr:
5104 case X86::VFCMULCPHZrrb:
5105 case X86::VFCMULCPHZrrbkz:
5106 case X86::VFCMULCPHZrrkz:
5107 case X86::VFMULCPHZ128rm:
5108 case X86::VFMULCPHZ128rmb:
5109 case X86::VFMULCPHZ128rmbkz:
5110 case X86::VFMULCPHZ128rmkz:
5111 case X86::VFMULCPHZ128rr:
5112 case X86::VFMULCPHZ128rrkz:
5113 case X86::VFMULCPHZ256rm:
5114 case X86::VFMULCPHZ256rmb:
5115 case X86::VFMULCPHZ256rmbkz:
5116 case X86::VFMULCPHZ256rmkz:
5117 case X86::VFMULCPHZ256rr:
5118 case X86::VFMULCPHZ256rrkz:
5119 case X86::VFMULCPHZrm:
5120 case X86::VFMULCPHZrmb:
5121 case X86::VFMULCPHZrmbkz:
5122 case X86::VFMULCPHZrmkz:
5123 case X86::VFMULCPHZrr:
5124 case X86::VFMULCPHZrrb:
5125 case X86::VFMULCPHZrrbkz:
5126 case X86::VFMULCPHZrrkz:
5127 case X86::VFCMULCSHZrm:
5128 case X86::VFCMULCSHZrmkz:
5129 case X86::VFCMULCSHZrr:
5130 case X86::VFCMULCSHZrrb:
5131 case X86::VFCMULCSHZrrbkz:
5132 case X86::VFCMULCSHZrrkz:
5133 case X86::VFMULCSHZrm:
5134 case X86::VFMULCSHZrmkz:
5135 case X86::VFMULCSHZrr:
5136 case X86::VFMULCSHZrrb:
5137 case X86::VFMULCSHZrrbkz:
5138 case X86::VFMULCSHZrrkz:
5139 return Subtarget.hasMULCFalseDeps();
5140 case X86::VPERMDYrm:
5141 case X86::VPERMDYrr:
5142 case X86::VPERMQYmi:
5143 case X86::VPERMQYri:
5144 case X86::VPERMPSYrm:
5145 case X86::VPERMPSYrr:
5146 case X86::VPERMPDYmi:
5147 case X86::VPERMPDYri:
5148 case X86::VPERMDZ256rm:
5149 case X86::VPERMDZ256rmb:
5150 case X86::VPERMDZ256rmbkz:
5151 case X86::VPERMDZ256rmkz:
5152 case X86::VPERMDZ256rr:
5153 case X86::VPERMDZ256rrkz:
5154 case X86::VPERMDZrm:
5155 case X86::VPERMDZrmb:
5156 case X86::VPERMDZrmbkz:
5157 case X86::VPERMDZrmkz:
5158 case X86::VPERMDZrr:
5159 case X86::VPERMDZrrkz:
5160 case X86::VPERMQZ256mbi:
5161 case X86::VPERMQZ256mbikz:
5162 case X86::VPERMQZ256mi:
5163 case X86::VPERMQZ256mikz:
5164 case X86::VPERMQZ256ri:
5165 case X86::VPERMQZ256rikz:
5166 case X86::VPERMQZ256rm:
5167 case X86::VPERMQZ256rmb:
5168 case X86::VPERMQZ256rmbkz:
5169 case X86::VPERMQZ256rmkz:
5170 case X86::VPERMQZ256rr:
5171 case X86::VPERMQZ256rrkz:
5172 case X86::VPERMQZmbi:
5173 case X86::VPERMQZmbikz:
5174 case X86::VPERMQZmi:
5175 case X86::VPERMQZmikz:
5176 case X86::VPERMQZri:
5177 case X86::VPERMQZrikz:
5178 case X86::VPERMQZrm:
5179 case X86::VPERMQZrmb:
5180 case X86::VPERMQZrmbkz: