LLVM 19.0.0git
X86InstrInfo.cpp
Go to the documentation of this file.
1//===-- X86InstrInfo.cpp - X86 Instruction Information --------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file contains the X86 implementation of the TargetInstrInfo class.
10//
11//===----------------------------------------------------------------------===//
12
13#include "X86InstrInfo.h"
14#include "X86.h"
15#include "X86InstrBuilder.h"
16#include "X86InstrFoldTables.h"
18#include "X86Subtarget.h"
19#include "X86TargetMachine.h"
20#include "llvm/ADT/STLExtras.h"
21#include "llvm/ADT/Sequence.h"
37#include "llvm/IR/Function.h"
38#include "llvm/IR/InstrTypes.h"
39#include "llvm/MC/MCAsmInfo.h"
40#include "llvm/MC/MCExpr.h"
41#include "llvm/MC/MCInst.h"
43#include "llvm/Support/Debug.h"
47#include <optional>
48
49using namespace llvm;
50
51#define DEBUG_TYPE "x86-instr-info"
52
53#define GET_INSTRINFO_CTOR_DTOR
54#include "X86GenInstrInfo.inc"
55
56static cl::opt<bool>
57 NoFusing("disable-spill-fusing",
58 cl::desc("Disable fusing of spill code into instructions"),
60static cl::opt<bool>
61 PrintFailedFusing("print-failed-fuse-candidates",
62 cl::desc("Print instructions that the allocator wants to"
63 " fuse, but the X86 backend currently can't"),
65static cl::opt<bool>
66 ReMatPICStubLoad("remat-pic-stub-load",
67 cl::desc("Re-materialize load from stub in PIC mode"),
68 cl::init(false), cl::Hidden);
70 PartialRegUpdateClearance("partial-reg-update-clearance",
71 cl::desc("Clearance between two register writes "
72 "for inserting XOR to avoid partial "
73 "register update"),
74 cl::init(64), cl::Hidden);
76 "undef-reg-clearance",
77 cl::desc("How many idle instructions we would like before "
78 "certain undef register reads"),
79 cl::init(128), cl::Hidden);
80
81// Pin the vtable to this file.
82void X86InstrInfo::anchor() {}
83
85 : X86GenInstrInfo((STI.isTarget64BitLP64() ? X86::ADJCALLSTACKDOWN64
86 : X86::ADJCALLSTACKDOWN32),
87 (STI.isTarget64BitLP64() ? X86::ADJCALLSTACKUP64
88 : X86::ADJCALLSTACKUP32),
89 X86::CATCHRET, (STI.is64Bit() ? X86::RET64 : X86::RET32)),
90 Subtarget(STI), RI(STI.getTargetTriple()) {}
91
93X86InstrInfo::getRegClass(const MCInstrDesc &MCID, unsigned OpNum,
95 const MachineFunction &MF) const {
96 auto *RC = TargetInstrInfo::getRegClass(MCID, OpNum, TRI, MF);
97 // If the target does not have egpr, then r16-r31 will be resereved for all
98 // instructions.
99 if (!RC || !Subtarget.hasEGPR())
100 return RC;
101
103 return RC;
104
105 switch (RC->getID()) {
106 default:
107 return RC;
108 case X86::GR8RegClassID:
109 return &X86::GR8_NOREX2RegClass;
110 case X86::GR16RegClassID:
111 return &X86::GR16_NOREX2RegClass;
112 case X86::GR32RegClassID:
113 return &X86::GR32_NOREX2RegClass;
114 case X86::GR64RegClassID:
115 return &X86::GR64_NOREX2RegClass;
116 case X86::GR32_NOSPRegClassID:
117 return &X86::GR32_NOREX2_NOSPRegClass;
118 case X86::GR64_NOSPRegClassID:
119 return &X86::GR64_NOREX2_NOSPRegClass;
120 }
121}
122
124 Register &SrcReg, Register &DstReg,
125 unsigned &SubIdx) const {
126 switch (MI.getOpcode()) {
127 default:
128 break;
129 case X86::MOVSX16rr8:
130 case X86::MOVZX16rr8:
131 case X86::MOVSX32rr8:
132 case X86::MOVZX32rr8:
133 case X86::MOVSX64rr8:
134 if (!Subtarget.is64Bit())
135 // It's not always legal to reference the low 8-bit of the larger
136 // register in 32-bit mode.
137 return false;
138 [[fallthrough]];
139 case X86::MOVSX32rr16:
140 case X86::MOVZX32rr16:
141 case X86::MOVSX64rr16:
142 case X86::MOVSX64rr32: {
143 if (MI.getOperand(0).getSubReg() || MI.getOperand(1).getSubReg())
144 // Be conservative.
145 return false;
146 SrcReg = MI.getOperand(1).getReg();
147 DstReg = MI.getOperand(0).getReg();
148 switch (MI.getOpcode()) {
149 default:
150 llvm_unreachable("Unreachable!");
151 case X86::MOVSX16rr8:
152 case X86::MOVZX16rr8:
153 case X86::MOVSX32rr8:
154 case X86::MOVZX32rr8:
155 case X86::MOVSX64rr8:
156 SubIdx = X86::sub_8bit;
157 break;
158 case X86::MOVSX32rr16:
159 case X86::MOVZX32rr16:
160 case X86::MOVSX64rr16:
161 SubIdx = X86::sub_16bit;
162 break;
163 case X86::MOVSX64rr32:
164 SubIdx = X86::sub_32bit;
165 break;
166 }
167 return true;
168 }
169 }
170 return false;
171}
172
174 if (MI.mayLoad() || MI.mayStore())
175 return false;
176
177 // Some target-independent operations that trivially lower to data-invariant
178 // instructions.
179 if (MI.isCopyLike() || MI.isInsertSubreg())
180 return true;
181
182 unsigned Opcode = MI.getOpcode();
183 using namespace X86;
184 // On x86 it is believed that imul is constant time w.r.t. the loaded data.
185 // However, they set flags and are perhaps the most surprisingly constant
186 // time operations so we call them out here separately.
187 if (isIMUL(Opcode))
188 return true;
189 // Bit scanning and counting instructions that are somewhat surprisingly
190 // constant time as they scan across bits and do other fairly complex
191 // operations like popcnt, but are believed to be constant time on x86.
192 // However, these set flags.
193 if (isBSF(Opcode) || isBSR(Opcode) || isLZCNT(Opcode) || isPOPCNT(Opcode) ||
194 isTZCNT(Opcode))
195 return true;
196 // Bit manipulation instructions are effectively combinations of basic
197 // arithmetic ops, and should still execute in constant time. These also
198 // set flags.
199 if (isBLCFILL(Opcode) || isBLCI(Opcode) || isBLCIC(Opcode) ||
200 isBLCMSK(Opcode) || isBLCS(Opcode) || isBLSFILL(Opcode) ||
201 isBLSI(Opcode) || isBLSIC(Opcode) || isBLSMSK(Opcode) || isBLSR(Opcode) ||
202 isTZMSK(Opcode))
203 return true;
204 // Bit extracting and clearing instructions should execute in constant time,
205 // and set flags.
206 if (isBEXTR(Opcode) || isBZHI(Opcode))
207 return true;
208 // Shift and rotate.
209 if (isROL(Opcode) || isROR(Opcode) || isSAR(Opcode) || isSHL(Opcode) ||
210 isSHR(Opcode) || isSHLD(Opcode) || isSHRD(Opcode))
211 return true;
212 // Basic arithmetic is constant time on the input but does set flags.
213 if (isADC(Opcode) || isADD(Opcode) || isAND(Opcode) || isOR(Opcode) ||
214 isSBB(Opcode) || isSUB(Opcode) || isXOR(Opcode))
215 return true;
216 // Arithmetic with just 32-bit and 64-bit variants and no immediates.
217 if (isANDN(Opcode))
218 return true;
219 // Unary arithmetic operations.
220 if (isDEC(Opcode) || isINC(Opcode) || isNEG(Opcode))
221 return true;
222 // Unlike other arithmetic, NOT doesn't set EFLAGS.
223 if (isNOT(Opcode))
224 return true;
225 // Various move instructions used to zero or sign extend things. Note that we
226 // intentionally don't support the _NOREX variants as we can't handle that
227 // register constraint anyways.
228 if (isMOVSX(Opcode) || isMOVZX(Opcode) || isMOVSXD(Opcode) || isMOV(Opcode))
229 return true;
230 // Arithmetic instructions that are both constant time and don't set flags.
231 if (isRORX(Opcode) || isSARX(Opcode) || isSHLX(Opcode) || isSHRX(Opcode))
232 return true;
233 // LEA doesn't actually access memory, and its arithmetic is constant time.
234 if (isLEA(Opcode))
235 return true;
236 // By default, assume that the instruction is not data invariant.
237 return false;
238}
239
241 switch (MI.getOpcode()) {
242 default:
243 // By default, assume that the load will immediately leak.
244 return false;
245
246 // On x86 it is believed that imul is constant time w.r.t. the loaded data.
247 // However, they set flags and are perhaps the most surprisingly constant
248 // time operations so we call them out here separately.
249 case X86::IMUL16rm:
250 case X86::IMUL16rmi:
251 case X86::IMUL32rm:
252 case X86::IMUL32rmi:
253 case X86::IMUL64rm:
254 case X86::IMUL64rmi32:
255
256 // Bit scanning and counting instructions that are somewhat surprisingly
257 // constant time as they scan across bits and do other fairly complex
258 // operations like popcnt, but are believed to be constant time on x86.
259 // However, these set flags.
260 case X86::BSF16rm:
261 case X86::BSF32rm:
262 case X86::BSF64rm:
263 case X86::BSR16rm:
264 case X86::BSR32rm:
265 case X86::BSR64rm:
266 case X86::LZCNT16rm:
267 case X86::LZCNT32rm:
268 case X86::LZCNT64rm:
269 case X86::POPCNT16rm:
270 case X86::POPCNT32rm:
271 case X86::POPCNT64rm:
272 case X86::TZCNT16rm:
273 case X86::TZCNT32rm:
274 case X86::TZCNT64rm:
275
276 // Bit manipulation instructions are effectively combinations of basic
277 // arithmetic ops, and should still execute in constant time. These also
278 // set flags.
279 case X86::BLCFILL32rm:
280 case X86::BLCFILL64rm:
281 case X86::BLCI32rm:
282 case X86::BLCI64rm:
283 case X86::BLCIC32rm:
284 case X86::BLCIC64rm:
285 case X86::BLCMSK32rm:
286 case X86::BLCMSK64rm:
287 case X86::BLCS32rm:
288 case X86::BLCS64rm:
289 case X86::BLSFILL32rm:
290 case X86::BLSFILL64rm:
291 case X86::BLSI32rm:
292 case X86::BLSI64rm:
293 case X86::BLSIC32rm:
294 case X86::BLSIC64rm:
295 case X86::BLSMSK32rm:
296 case X86::BLSMSK64rm:
297 case X86::BLSR32rm:
298 case X86::BLSR64rm:
299 case X86::TZMSK32rm:
300 case X86::TZMSK64rm:
301
302 // Bit extracting and clearing instructions should execute in constant time,
303 // and set flags.
304 case X86::BEXTR32rm:
305 case X86::BEXTR64rm:
306 case X86::BEXTRI32mi:
307 case X86::BEXTRI64mi:
308 case X86::BZHI32rm:
309 case X86::BZHI64rm:
310
311 // Basic arithmetic is constant time on the input but does set flags.
312 case X86::ADC8rm:
313 case X86::ADC16rm:
314 case X86::ADC32rm:
315 case X86::ADC64rm:
316 case X86::ADD8rm:
317 case X86::ADD16rm:
318 case X86::ADD32rm:
319 case X86::ADD64rm:
320 case X86::AND8rm:
321 case X86::AND16rm:
322 case X86::AND32rm:
323 case X86::AND64rm:
324 case X86::ANDN32rm:
325 case X86::ANDN64rm:
326 case X86::OR8rm:
327 case X86::OR16rm:
328 case X86::OR32rm:
329 case X86::OR64rm:
330 case X86::SBB8rm:
331 case X86::SBB16rm:
332 case X86::SBB32rm:
333 case X86::SBB64rm:
334 case X86::SUB8rm:
335 case X86::SUB16rm:
336 case X86::SUB32rm:
337 case X86::SUB64rm:
338 case X86::XOR8rm:
339 case X86::XOR16rm:
340 case X86::XOR32rm:
341 case X86::XOR64rm:
342
343 // Integer multiply w/o affecting flags is still believed to be constant
344 // time on x86. Called out separately as this is among the most surprising
345 // instructions to exhibit that behavior.
346 case X86::MULX32rm:
347 case X86::MULX64rm:
348
349 // Arithmetic instructions that are both constant time and don't set flags.
350 case X86::RORX32mi:
351 case X86::RORX64mi:
352 case X86::SARX32rm:
353 case X86::SARX64rm:
354 case X86::SHLX32rm:
355 case X86::SHLX64rm:
356 case X86::SHRX32rm:
357 case X86::SHRX64rm:
358
359 // Conversions are believed to be constant time and don't set flags.
360 case X86::CVTTSD2SI64rm:
361 case X86::VCVTTSD2SI64rm:
362 case X86::VCVTTSD2SI64Zrm:
363 case X86::CVTTSD2SIrm:
364 case X86::VCVTTSD2SIrm:
365 case X86::VCVTTSD2SIZrm:
366 case X86::CVTTSS2SI64rm:
367 case X86::VCVTTSS2SI64rm:
368 case X86::VCVTTSS2SI64Zrm:
369 case X86::CVTTSS2SIrm:
370 case X86::VCVTTSS2SIrm:
371 case X86::VCVTTSS2SIZrm:
372 case X86::CVTSI2SDrm:
373 case X86::VCVTSI2SDrm:
374 case X86::VCVTSI2SDZrm:
375 case X86::CVTSI2SSrm:
376 case X86::VCVTSI2SSrm:
377 case X86::VCVTSI2SSZrm:
378 case X86::CVTSI642SDrm:
379 case X86::VCVTSI642SDrm:
380 case X86::VCVTSI642SDZrm:
381 case X86::CVTSI642SSrm:
382 case X86::VCVTSI642SSrm:
383 case X86::VCVTSI642SSZrm:
384 case X86::CVTSS2SDrm:
385 case X86::VCVTSS2SDrm:
386 case X86::VCVTSS2SDZrm:
387 case X86::CVTSD2SSrm:
388 case X86::VCVTSD2SSrm:
389 case X86::VCVTSD2SSZrm:
390 // AVX512 added unsigned integer conversions.
391 case X86::VCVTTSD2USI64Zrm:
392 case X86::VCVTTSD2USIZrm:
393 case X86::VCVTTSS2USI64Zrm:
394 case X86::VCVTTSS2USIZrm:
395 case X86::VCVTUSI2SDZrm:
396 case X86::VCVTUSI642SDZrm:
397 case X86::VCVTUSI2SSZrm:
398 case X86::VCVTUSI642SSZrm:
399
400 // Loads to register don't set flags.
401 case X86::MOV8rm:
402 case X86::MOV8rm_NOREX:
403 case X86::MOV16rm:
404 case X86::MOV32rm:
405 case X86::MOV64rm:
406 case X86::MOVSX16rm8:
407 case X86::MOVSX32rm16:
408 case X86::MOVSX32rm8:
409 case X86::MOVSX32rm8_NOREX:
410 case X86::MOVSX64rm16:
411 case X86::MOVSX64rm32:
412 case X86::MOVSX64rm8:
413 case X86::MOVZX16rm8:
414 case X86::MOVZX32rm16:
415 case X86::MOVZX32rm8:
416 case X86::MOVZX32rm8_NOREX:
417 case X86::MOVZX64rm16:
418 case X86::MOVZX64rm8:
419 return true;
420 }
421}
422
424 const MachineFunction *MF = MI.getParent()->getParent();
426
427 if (isFrameInstr(MI)) {
428 int SPAdj = alignTo(getFrameSize(MI), TFI->getStackAlign());
429 SPAdj -= getFrameAdjustment(MI);
430 if (!isFrameSetup(MI))
431 SPAdj = -SPAdj;
432 return SPAdj;
433 }
434
435 // To know whether a call adjusts the stack, we need information
436 // that is bound to the following ADJCALLSTACKUP pseudo.
437 // Look for the next ADJCALLSTACKUP that follows the call.
438 if (MI.isCall()) {
439 const MachineBasicBlock *MBB = MI.getParent();
441 for (auto E = MBB->end(); I != E; ++I) {
442 if (I->getOpcode() == getCallFrameDestroyOpcode() || I->isCall())
443 break;
444 }
445
446 // If we could not find a frame destroy opcode, then it has already
447 // been simplified, so we don't care.
448 if (I->getOpcode() != getCallFrameDestroyOpcode())
449 return 0;
450
451 return -(I->getOperand(1).getImm());
452 }
453
454 // Currently handle only PUSHes we can reasonably expect to see
455 // in call sequences
456 switch (MI.getOpcode()) {
457 default:
458 return 0;
459 case X86::PUSH32r:
460 case X86::PUSH32rmm:
461 case X86::PUSH32rmr:
462 case X86::PUSH32i:
463 return 4;
464 case X86::PUSH64r:
465 case X86::PUSH64rmm:
466 case X86::PUSH64rmr:
467 case X86::PUSH64i32:
468 return 8;
469 }
470}
471
472/// Return true and the FrameIndex if the specified
473/// operand and follow operands form a reference to the stack frame.
474bool X86InstrInfo::isFrameOperand(const MachineInstr &MI, unsigned int Op,
475 int &FrameIndex) const {
476 if (MI.getOperand(Op + X86::AddrBaseReg).isFI() &&
477 MI.getOperand(Op + X86::AddrScaleAmt).isImm() &&
478 MI.getOperand(Op + X86::AddrIndexReg).isReg() &&
479 MI.getOperand(Op + X86::AddrDisp).isImm() &&
480 MI.getOperand(Op + X86::AddrScaleAmt).getImm() == 1 &&
481 MI.getOperand(Op + X86::AddrIndexReg).getReg() == 0 &&
482 MI.getOperand(Op + X86::AddrDisp).getImm() == 0) {
483 FrameIndex = MI.getOperand(Op + X86::AddrBaseReg).getIndex();
484 return true;
485 }
486 return false;
487}
488
489static bool isFrameLoadOpcode(int Opcode, unsigned &MemBytes) {
490 switch (Opcode) {
491 default:
492 return false;
493 case X86::MOV8rm:
494 case X86::KMOVBkm:
495 case X86::KMOVBkm_EVEX:
496 MemBytes = 1;
497 return true;
498 case X86::MOV16rm:
499 case X86::KMOVWkm:
500 case X86::KMOVWkm_EVEX:
501 case X86::VMOVSHZrm:
502 case X86::VMOVSHZrm_alt:
503 MemBytes = 2;
504 return true;
505 case X86::MOV32rm:
506 case X86::MOVSSrm:
507 case X86::MOVSSrm_alt:
508 case X86::VMOVSSrm:
509 case X86::VMOVSSrm_alt:
510 case X86::VMOVSSZrm:
511 case X86::VMOVSSZrm_alt:
512 case X86::KMOVDkm:
513 case X86::KMOVDkm_EVEX:
514 MemBytes = 4;
515 return true;
516 case X86::MOV64rm:
517 case X86::LD_Fp64m:
518 case X86::MOVSDrm:
519 case X86::MOVSDrm_alt:
520 case X86::VMOVSDrm:
521 case X86::VMOVSDrm_alt:
522 case X86::VMOVSDZrm:
523 case X86::VMOVSDZrm_alt:
524 case X86::MMX_MOVD64rm:
525 case X86::MMX_MOVQ64rm:
526 case X86::KMOVQkm:
527 case X86::KMOVQkm_EVEX:
528 MemBytes = 8;
529 return true;
530 case X86::MOVAPSrm:
531 case X86::MOVUPSrm:
532 case X86::MOVAPDrm:
533 case X86::MOVUPDrm:
534 case X86::MOVDQArm:
535 case X86::MOVDQUrm:
536 case X86::VMOVAPSrm:
537 case X86::VMOVUPSrm:
538 case X86::VMOVAPDrm:
539 case X86::VMOVUPDrm:
540 case X86::VMOVDQArm:
541 case X86::VMOVDQUrm:
542 case X86::VMOVAPSZ128rm:
543 case X86::VMOVUPSZ128rm:
544 case X86::VMOVAPSZ128rm_NOVLX:
545 case X86::VMOVUPSZ128rm_NOVLX:
546 case X86::VMOVAPDZ128rm:
547 case X86::VMOVUPDZ128rm:
548 case X86::VMOVDQU8Z128rm:
549 case X86::VMOVDQU16Z128rm:
550 case X86::VMOVDQA32Z128rm:
551 case X86::VMOVDQU32Z128rm:
552 case X86::VMOVDQA64Z128rm:
553 case X86::VMOVDQU64Z128rm:
554 MemBytes = 16;
555 return true;
556 case X86::VMOVAPSYrm:
557 case X86::VMOVUPSYrm:
558 case X86::VMOVAPDYrm:
559 case X86::VMOVUPDYrm:
560 case X86::VMOVDQAYrm:
561 case X86::VMOVDQUYrm:
562 case X86::VMOVAPSZ256rm:
563 case X86::VMOVUPSZ256rm:
564 case X86::VMOVAPSZ256rm_NOVLX:
565 case X86::VMOVUPSZ256rm_NOVLX:
566 case X86::VMOVAPDZ256rm:
567 case X86::VMOVUPDZ256rm:
568 case X86::VMOVDQU8Z256rm:
569 case X86::VMOVDQU16Z256rm:
570 case X86::VMOVDQA32Z256rm:
571 case X86::VMOVDQU32Z256rm:
572 case X86::VMOVDQA64Z256rm:
573 case X86::VMOVDQU64Z256rm:
574 MemBytes = 32;
575 return true;
576 case X86::VMOVAPSZrm:
577 case X86::VMOVUPSZrm:
578 case X86::VMOVAPDZrm:
579 case X86::VMOVUPDZrm:
580 case X86::VMOVDQU8Zrm:
581 case X86::VMOVDQU16Zrm:
582 case X86::VMOVDQA32Zrm:
583 case X86::VMOVDQU32Zrm:
584 case X86::VMOVDQA64Zrm:
585 case X86::VMOVDQU64Zrm:
586 MemBytes = 64;
587 return true;
588 }
589}
590
591static bool isFrameStoreOpcode(int Opcode, unsigned &MemBytes) {
592 switch (Opcode) {
593 default:
594 return false;
595 case X86::MOV8mr:
596 case X86::KMOVBmk:
597 case X86::KMOVBmk_EVEX:
598 MemBytes = 1;
599 return true;
600 case X86::MOV16mr:
601 case X86::KMOVWmk:
602 case X86::KMOVWmk_EVEX:
603 case X86::VMOVSHZmr:
604 MemBytes = 2;
605 return true;
606 case X86::MOV32mr:
607 case X86::MOVSSmr:
608 case X86::VMOVSSmr:
609 case X86::VMOVSSZmr:
610 case X86::KMOVDmk:
611 case X86::KMOVDmk_EVEX:
612 MemBytes = 4;
613 return true;
614 case X86::MOV64mr:
615 case X86::ST_FpP64m:
616 case X86::MOVSDmr:
617 case X86::VMOVSDmr:
618 case X86::VMOVSDZmr:
619 case X86::MMX_MOVD64mr:
620 case X86::MMX_MOVQ64mr:
621 case X86::MMX_MOVNTQmr:
622 case X86::KMOVQmk:
623 case X86::KMOVQmk_EVEX:
624 MemBytes = 8;
625 return true;
626 case X86::MOVAPSmr:
627 case X86::MOVUPSmr:
628 case X86::MOVAPDmr:
629 case X86::MOVUPDmr:
630 case X86::MOVDQAmr:
631 case X86::MOVDQUmr:
632 case X86::VMOVAPSmr:
633 case X86::VMOVUPSmr:
634 case X86::VMOVAPDmr:
635 case X86::VMOVUPDmr:
636 case X86::VMOVDQAmr:
637 case X86::VMOVDQUmr:
638 case X86::VMOVUPSZ128mr:
639 case X86::VMOVAPSZ128mr:
640 case X86::VMOVUPSZ128mr_NOVLX:
641 case X86::VMOVAPSZ128mr_NOVLX:
642 case X86::VMOVUPDZ128mr:
643 case X86::VMOVAPDZ128mr:
644 case X86::VMOVDQA32Z128mr:
645 case X86::VMOVDQU32Z128mr:
646 case X86::VMOVDQA64Z128mr:
647 case X86::VMOVDQU64Z128mr:
648 case X86::VMOVDQU8Z128mr:
649 case X86::VMOVDQU16Z128mr:
650 MemBytes = 16;
651 return true;
652 case X86::VMOVUPSYmr:
653 case X86::VMOVAPSYmr:
654 case X86::VMOVUPDYmr:
655 case X86::VMOVAPDYmr:
656 case X86::VMOVDQUYmr:
657 case X86::VMOVDQAYmr:
658 case X86::VMOVUPSZ256mr:
659 case X86::VMOVAPSZ256mr:
660 case X86::VMOVUPSZ256mr_NOVLX:
661 case X86::VMOVAPSZ256mr_NOVLX:
662 case X86::VMOVUPDZ256mr:
663 case X86::VMOVAPDZ256mr:
664 case X86::VMOVDQU8Z256mr:
665 case X86::VMOVDQU16Z256mr:
666 case X86::VMOVDQA32Z256mr:
667 case X86::VMOVDQU32Z256mr:
668 case X86::VMOVDQA64Z256mr:
669 case X86::VMOVDQU64Z256mr:
670 MemBytes = 32;
671 return true;
672 case X86::VMOVUPSZmr:
673 case X86::VMOVAPSZmr:
674 case X86::VMOVUPDZmr:
675 case X86::VMOVAPDZmr:
676 case X86::VMOVDQU8Zmr:
677 case X86::VMOVDQU16Zmr:
678 case X86::VMOVDQA32Zmr:
679 case X86::VMOVDQU32Zmr:
680 case X86::VMOVDQA64Zmr:
681 case X86::VMOVDQU64Zmr:
682 MemBytes = 64;
683 return true;
684 }
685 return false;
686}
687
689 int &FrameIndex) const {
690 unsigned Dummy;
691 return X86InstrInfo::isLoadFromStackSlot(MI, FrameIndex, Dummy);
692}
693
695 int &FrameIndex,
696 unsigned &MemBytes) const {
697 if (isFrameLoadOpcode(MI.getOpcode(), MemBytes))
698 if (MI.getOperand(0).getSubReg() == 0 && isFrameOperand(MI, 1, FrameIndex))
699 return MI.getOperand(0).getReg();
700 return 0;
701}
702
704 int &FrameIndex) const {
705 unsigned Dummy;
706 if (isFrameLoadOpcode(MI.getOpcode(), Dummy)) {
707 unsigned Reg;
708 if ((Reg = isLoadFromStackSlot(MI, FrameIndex)))
709 return Reg;
710 // Check for post-frame index elimination operations
712 if (hasLoadFromStackSlot(MI, Accesses)) {
713 FrameIndex =
714 cast<FixedStackPseudoSourceValue>(Accesses.front()->getPseudoValue())
715 ->getFrameIndex();
716 return MI.getOperand(0).getReg();
717 }
718 }
719 return 0;
720}
721
723 int &FrameIndex) const {
724 unsigned Dummy;
725 return X86InstrInfo::isStoreToStackSlot(MI, FrameIndex, Dummy);
726}
727
729 int &FrameIndex,
730 unsigned &MemBytes) const {
731 if (isFrameStoreOpcode(MI.getOpcode(), MemBytes))
732 if (MI.getOperand(X86::AddrNumOperands).getSubReg() == 0 &&
733 isFrameOperand(MI, 0, FrameIndex))
734 return MI.getOperand(X86::AddrNumOperands).getReg();
735 return 0;
736}
737
739 int &FrameIndex) const {
740 unsigned Dummy;
741 if (isFrameStoreOpcode(MI.getOpcode(), Dummy)) {
742 unsigned Reg;
743 if ((Reg = isStoreToStackSlot(MI, FrameIndex)))
744 return Reg;
745 // Check for post-frame index elimination operations
747 if (hasStoreToStackSlot(MI, Accesses)) {
748 FrameIndex =
749 cast<FixedStackPseudoSourceValue>(Accesses.front()->getPseudoValue())
750 ->getFrameIndex();
751 return MI.getOperand(X86::AddrNumOperands).getReg();
752 }
753 }
754 return 0;
755}
756
757/// Return true if register is PIC base; i.e.g defined by X86::MOVPC32r.
758static bool regIsPICBase(Register BaseReg, const MachineRegisterInfo &MRI) {
759 // Don't waste compile time scanning use-def chains of physregs.
760 if (!BaseReg.isVirtual())
761 return false;
762 bool isPICBase = false;
763 for (const MachineInstr &DefMI : MRI.def_instructions(BaseReg)) {
764 if (DefMI.getOpcode() != X86::MOVPC32r)
765 return false;
766 assert(!isPICBase && "More than one PIC base?");
767 isPICBase = true;
768 }
769 return isPICBase;
770}
771
773 const MachineInstr &MI) const {
774 switch (MI.getOpcode()) {
775 default:
776 // This function should only be called for opcodes with the ReMaterializable
777 // flag set.
778 llvm_unreachable("Unknown rematerializable operation!");
779 break;
780 case X86::IMPLICIT_DEF:
781 // Defer to generic logic.
782 break;
783 case X86::LOAD_STACK_GUARD:
784 case X86::LD_Fp032:
785 case X86::LD_Fp064:
786 case X86::LD_Fp080:
787 case X86::LD_Fp132:
788 case X86::LD_Fp164:
789 case X86::LD_Fp180:
790 case X86::AVX1_SETALLONES:
791 case X86::AVX2_SETALLONES:
792 case X86::AVX512_128_SET0:
793 case X86::AVX512_256_SET0:
794 case X86::AVX512_512_SET0:
795 case X86::AVX512_512_SETALLONES:
796 case X86::AVX512_FsFLD0SD:
797 case X86::AVX512_FsFLD0SH:
798 case X86::AVX512_FsFLD0SS:
799 case X86::AVX512_FsFLD0F128:
800 case X86::AVX_SET0:
801 case X86::FsFLD0SD:
802 case X86::FsFLD0SS:
803 case X86::FsFLD0SH:
804 case X86::FsFLD0F128:
805 case X86::KSET0D:
806 case X86::KSET0Q:
807 case X86::KSET0W:
808 case X86::KSET1D:
809 case X86::KSET1Q:
810 case X86::KSET1W:
811 case X86::MMX_SET0:
812 case X86::MOV32ImmSExti8:
813 case X86::MOV32r0:
814 case X86::MOV32r1:
815 case X86::MOV32r_1:
816 case X86::MOV32ri64:
817 case X86::MOV64ImmSExti8:
818 case X86::V_SET0:
819 case X86::V_SETALLONES:
820 case X86::MOV16ri:
821 case X86::MOV32ri:
822 case X86::MOV64ri:
823 case X86::MOV64ri32:
824 case X86::MOV8ri:
825 case X86::PTILEZEROV:
826 return true;
827
828 case X86::MOV8rm:
829 case X86::MOV8rm_NOREX:
830 case X86::MOV16rm:
831 case X86::MOV32rm:
832 case X86::MOV64rm:
833 case X86::MOVSSrm:
834 case X86::MOVSSrm_alt:
835 case X86::MOVSDrm:
836 case X86::MOVSDrm_alt:
837 case X86::MOVAPSrm:
838 case X86::MOVUPSrm:
839 case X86::MOVAPDrm:
840 case X86::MOVUPDrm:
841 case X86::MOVDQArm:
842 case X86::MOVDQUrm:
843 case X86::VMOVSSrm:
844 case X86::VMOVSSrm_alt:
845 case X86::VMOVSDrm:
846 case X86::VMOVSDrm_alt:
847 case X86::VMOVAPSrm:
848 case X86::VMOVUPSrm:
849 case X86::VMOVAPDrm:
850 case X86::VMOVUPDrm:
851 case X86::VMOVDQArm:
852 case X86::VMOVDQUrm:
853 case X86::VMOVAPSYrm:
854 case X86::VMOVUPSYrm:
855 case X86::VMOVAPDYrm:
856 case X86::VMOVUPDYrm:
857 case X86::VMOVDQAYrm:
858 case X86::VMOVDQUYrm:
859 case X86::MMX_MOVD64rm:
860 case X86::MMX_MOVQ64rm:
861 case X86::VBROADCASTSSrm:
862 case X86::VBROADCASTSSYrm:
863 case X86::VBROADCASTSDYrm:
864 // AVX-512
865 case X86::VPBROADCASTBZ128rm:
866 case X86::VPBROADCASTBZ256rm:
867 case X86::VPBROADCASTBZrm:
868 case X86::VBROADCASTF32X2Z256rm:
869 case X86::VBROADCASTF32X2Zrm:
870 case X86::VBROADCASTI32X2Z128rm:
871 case X86::VBROADCASTI32X2Z256rm:
872 case X86::VBROADCASTI32X2Zrm:
873 case X86::VPBROADCASTWZ128rm:
874 case X86::VPBROADCASTWZ256rm:
875 case X86::VPBROADCASTWZrm:
876 case X86::VPBROADCASTDZ128rm:
877 case X86::VPBROADCASTDZ256rm:
878 case X86::VPBROADCASTDZrm:
879 case X86::VBROADCASTSSZ128rm:
880 case X86::VBROADCASTSSZ256rm:
881 case X86::VBROADCASTSSZrm:
882 case X86::VPBROADCASTQZ128rm:
883 case X86::VPBROADCASTQZ256rm:
884 case X86::VPBROADCASTQZrm:
885 case X86::VBROADCASTSDZ256rm:
886 case X86::VBROADCASTSDZrm:
887 case X86::VMOVSSZrm:
888 case X86::VMOVSSZrm_alt:
889 case X86::VMOVSDZrm:
890 case X86::VMOVSDZrm_alt:
891 case X86::VMOVSHZrm:
892 case X86::VMOVSHZrm_alt:
893 case X86::VMOVAPDZ128rm:
894 case X86::VMOVAPDZ256rm:
895 case X86::VMOVAPDZrm:
896 case X86::VMOVAPSZ128rm:
897 case X86::VMOVAPSZ256rm:
898 case X86::VMOVAPSZ128rm_NOVLX:
899 case X86::VMOVAPSZ256rm_NOVLX:
900 case X86::VMOVAPSZrm:
901 case X86::VMOVDQA32Z128rm:
902 case X86::VMOVDQA32Z256rm:
903 case X86::VMOVDQA32Zrm:
904 case X86::VMOVDQA64Z128rm:
905 case X86::VMOVDQA64Z256rm:
906 case X86::VMOVDQA64Zrm:
907 case X86::VMOVDQU16Z128rm:
908 case X86::VMOVDQU16Z256rm:
909 case X86::VMOVDQU16Zrm:
910 case X86::VMOVDQU32Z128rm:
911 case X86::VMOVDQU32Z256rm:
912 case X86::VMOVDQU32Zrm:
913 case X86::VMOVDQU64Z128rm:
914 case X86::VMOVDQU64Z256rm:
915 case X86::VMOVDQU64Zrm:
916 case X86::VMOVDQU8Z128rm:
917 case X86::VMOVDQU8Z256rm:
918 case X86::VMOVDQU8Zrm:
919 case X86::VMOVUPDZ128rm:
920 case X86::VMOVUPDZ256rm:
921 case X86::VMOVUPDZrm:
922 case X86::VMOVUPSZ128rm:
923 case X86::VMOVUPSZ256rm:
924 case X86::VMOVUPSZ128rm_NOVLX:
925 case X86::VMOVUPSZ256rm_NOVLX:
926 case X86::VMOVUPSZrm: {
927 // Loads from constant pools are trivially rematerializable.
928 if (MI.getOperand(1 + X86::AddrBaseReg).isReg() &&
929 MI.getOperand(1 + X86::AddrScaleAmt).isImm() &&
930 MI.getOperand(1 + X86::AddrIndexReg).isReg() &&
931 MI.getOperand(1 + X86::AddrIndexReg).getReg() == 0 &&
932 MI.isDereferenceableInvariantLoad()) {
933 Register BaseReg = MI.getOperand(1 + X86::AddrBaseReg).getReg();
934 if (BaseReg == 0 || BaseReg == X86::RIP)
935 return true;
936 // Allow re-materialization of PIC load.
937 if (!(!ReMatPICStubLoad && MI.getOperand(1 + X86::AddrDisp).isGlobal())) {
938 const MachineFunction &MF = *MI.getParent()->getParent();
939 const MachineRegisterInfo &MRI = MF.getRegInfo();
940 if (regIsPICBase(BaseReg, MRI))
941 return true;
942 }
943 }
944 break;
945 }
946
947 case X86::LEA32r:
948 case X86::LEA64r: {
949 if (MI.getOperand(1 + X86::AddrScaleAmt).isImm() &&
950 MI.getOperand(1 + X86::AddrIndexReg).isReg() &&
951 MI.getOperand(1 + X86::AddrIndexReg).getReg() == 0 &&
952 !MI.getOperand(1 + X86::AddrDisp).isReg()) {
953 // lea fi#, lea GV, etc. are all rematerializable.
954 if (!MI.getOperand(1 + X86::AddrBaseReg).isReg())
955 return true;
956 Register BaseReg = MI.getOperand(1 + X86::AddrBaseReg).getReg();
957 if (BaseReg == 0)
958 return true;
959 // Allow re-materialization of lea PICBase + x.
960 const MachineFunction &MF = *MI.getParent()->getParent();
961 const MachineRegisterInfo &MRI = MF.getRegInfo();
962 if (regIsPICBase(BaseReg, MRI))
963 return true;
964 }
965 break;
966 }
967 }
969}
970
973 Register DestReg, unsigned SubIdx,
974 const MachineInstr &Orig,
975 const TargetRegisterInfo &TRI) const {
976 bool ClobbersEFLAGS = Orig.modifiesRegister(X86::EFLAGS, &TRI);
977 if (ClobbersEFLAGS && MBB.computeRegisterLiveness(&TRI, X86::EFLAGS, I) !=
979 // The instruction clobbers EFLAGS. Re-materialize as MOV32ri to avoid side
980 // effects.
981 int Value;
982 switch (Orig.getOpcode()) {
983 case X86::MOV32r0:
984 Value = 0;
985 break;
986 case X86::MOV32r1:
987 Value = 1;
988 break;
989 case X86::MOV32r_1:
990 Value = -1;
991 break;
992 default:
993 llvm_unreachable("Unexpected instruction!");
994 }
995
996 const DebugLoc &DL = Orig.getDebugLoc();
997 BuildMI(MBB, I, DL, get(X86::MOV32ri))
998 .add(Orig.getOperand(0))
999 .addImm(Value);
1000 } else {
1002 MBB.insert(I, MI);
1003 }
1004
1005 MachineInstr &NewMI = *std::prev(I);
1006 NewMI.substituteRegister(Orig.getOperand(0).getReg(), DestReg, SubIdx, TRI);
1007}
1008
1009/// True if MI has a condition code def, e.g. EFLAGS, that is not marked dead.
1011 for (const MachineOperand &MO : MI.operands()) {
1012 if (MO.isReg() && MO.isDef() && MO.getReg() == X86::EFLAGS &&
1013 !MO.isDead()) {
1014 return true;
1015 }
1016 }
1017 return false;
1018}
1019
1020/// Check whether the shift count for a machine operand is non-zero.
1021inline static unsigned getTruncatedShiftCount(const MachineInstr &MI,
1022 unsigned ShiftAmtOperandIdx) {
1023 // The shift count is six bits with the REX.W prefix and five bits without.
1024 unsigned ShiftCountMask = (MI.getDesc().TSFlags & X86II::REX_W) ? 63 : 31;
1025 unsigned Imm = MI.getOperand(ShiftAmtOperandIdx).getImm();
1026 return Imm & ShiftCountMask;
1027}
1028
1029/// Check whether the given shift count is appropriate
1030/// can be represented by a LEA instruction.
1031inline static bool isTruncatedShiftCountForLEA(unsigned ShAmt) {
1032 // Left shift instructions can be transformed into load-effective-address
1033 // instructions if we can encode them appropriately.
1034 // A LEA instruction utilizes a SIB byte to encode its scale factor.
1035 // The SIB.scale field is two bits wide which means that we can encode any
1036 // shift amount less than 4.
1037 return ShAmt < 4 && ShAmt > 0;
1038}
1039
1041 MachineInstr &CmpValDefInstr,
1042 const MachineRegisterInfo *MRI,
1043 MachineInstr **AndInstr,
1044 const TargetRegisterInfo *TRI,
1045 bool &NoSignFlag, bool &ClearsOverflowFlag) {
1046 if (!(CmpValDefInstr.getOpcode() == X86::SUBREG_TO_REG &&
1047 CmpInstr.getOpcode() == X86::TEST64rr) &&
1048 !(CmpValDefInstr.getOpcode() == X86::COPY &&
1049 CmpInstr.getOpcode() == X86::TEST16rr))
1050 return false;
1051
1052 // CmpInstr is a TEST16rr/TEST64rr instruction, and
1053 // `X86InstrInfo::analyzeCompare` guarantees that it's analyzable only if two
1054 // registers are identical.
1055 assert((CmpInstr.getOperand(0).getReg() == CmpInstr.getOperand(1).getReg()) &&
1056 "CmpInstr is an analyzable TEST16rr/TEST64rr, and "
1057 "`X86InstrInfo::analyzeCompare` requires two reg operands are the"
1058 "same.");
1059
1060 // Caller (`X86InstrInfo::optimizeCompareInstr`) guarantees that
1061 // `CmpValDefInstr` defines the value that's used by `CmpInstr`; in this case
1062 // if `CmpValDefInstr` sets the EFLAGS, it is likely that `CmpInstr` is
1063 // redundant.
1064 assert(
1065 (MRI->getVRegDef(CmpInstr.getOperand(0).getReg()) == &CmpValDefInstr) &&
1066 "Caller guarantees that TEST64rr is a user of SUBREG_TO_REG or TEST16rr "
1067 "is a user of COPY sub16bit.");
1068 MachineInstr *VregDefInstr = nullptr;
1069 if (CmpInstr.getOpcode() == X86::TEST16rr) {
1070 if (!CmpValDefInstr.getOperand(1).getReg().isVirtual())
1071 return false;
1072 VregDefInstr = MRI->getVRegDef(CmpValDefInstr.getOperand(1).getReg());
1073 if (!VregDefInstr)
1074 return false;
1075 // We can only remove test when AND32ri or AND64ri32 whose imm can fit 16bit
1076 // size, others 32/64 bit ops would test higher bits which test16rr don't
1077 // want to.
1078 if (!((VregDefInstr->getOpcode() == X86::AND32ri ||
1079 VregDefInstr->getOpcode() == X86::AND64ri32) &&
1080 isUInt<16>(VregDefInstr->getOperand(2).getImm())))
1081 return false;
1082 }
1083
1084 if (CmpInstr.getOpcode() == X86::TEST64rr) {
1085 // As seen in X86 td files, CmpValDefInstr.getOperand(1).getImm() is
1086 // typically 0.
1087 if (CmpValDefInstr.getOperand(1).getImm() != 0)
1088 return false;
1089
1090 // As seen in X86 td files, CmpValDefInstr.getOperand(3) is typically
1091 // sub_32bit or sub_xmm.
1092 if (CmpValDefInstr.getOperand(3).getImm() != X86::sub_32bit)
1093 return false;
1094
1095 VregDefInstr = MRI->getVRegDef(CmpValDefInstr.getOperand(2).getReg());
1096 }
1097
1098 assert(VregDefInstr && "Must have a definition (SSA)");
1099
1100 // Requires `CmpValDefInstr` and `VregDefInstr` are from the same MBB
1101 // to simplify the subsequent analysis.
1102 //
1103 // FIXME: If `VregDefInstr->getParent()` is the only predecessor of
1104 // `CmpValDefInstr.getParent()`, this could be handled.
1105 if (VregDefInstr->getParent() != CmpValDefInstr.getParent())
1106 return false;
1107
1108 if (X86::isAND(VregDefInstr->getOpcode())) {
1109 // Get a sequence of instructions like
1110 // %reg = and* ... // Set EFLAGS
1111 // ... // EFLAGS not changed
1112 // %extended_reg = subreg_to_reg 0, %reg, %subreg.sub_32bit
1113 // test64rr %extended_reg, %extended_reg, implicit-def $eflags
1114 // or
1115 // %reg = and32* ...
1116 // ... // EFLAGS not changed.
1117 // %src_reg = copy %reg.sub_16bit:gr32
1118 // test16rr %src_reg, %src_reg, implicit-def $eflags
1119 //
1120 // If subsequent readers use a subset of bits that don't change
1121 // after `and*` instructions, it's likely that the test64rr could
1122 // be optimized away.
1123 for (const MachineInstr &Instr :
1124 make_range(std::next(MachineBasicBlock::iterator(VregDefInstr)),
1125 MachineBasicBlock::iterator(CmpValDefInstr))) {
1126 // There are instructions between 'VregDefInstr' and
1127 // 'CmpValDefInstr' that modifies EFLAGS.
1128 if (Instr.modifiesRegister(X86::EFLAGS, TRI))
1129 return false;
1130 }
1131
1132 *AndInstr = VregDefInstr;
1133
1134 // AND instruction will essentially update SF and clear OF, so
1135 // NoSignFlag should be false in the sense that SF is modified by `AND`.
1136 //
1137 // However, the implementation artifically sets `NoSignFlag` to true
1138 // to poison the SF bit; that is to say, if SF is looked at later, the
1139 // optimization (to erase TEST64rr) will be disabled.
1140 //
1141 // The reason to poison SF bit is that SF bit value could be different
1142 // in the `AND` and `TEST` operation; signed bit is not known for `AND`,
1143 // and is known to be 0 as a result of `TEST64rr`.
1144 //
1145 // FIXME: As opposed to poisoning the SF bit directly, consider peeking into
1146 // the AND instruction and using the static information to guide peephole
1147 // optimization if possible. For example, it's possible to fold a
1148 // conditional move into a copy if the relevant EFLAG bits could be deduced
1149 // from an immediate operand of and operation.
1150 //
1151 NoSignFlag = true;
1152 // ClearsOverflowFlag is true for AND operation (no surprise).
1153 ClearsOverflowFlag = true;
1154 return true;
1155 }
1156 return false;
1157}
1158
1160 unsigned Opc, bool AllowSP, Register &NewSrc,
1161 bool &isKill, MachineOperand &ImplicitOp,
1162 LiveVariables *LV, LiveIntervals *LIS) const {
1163 MachineFunction &MF = *MI.getParent()->getParent();
1164 const TargetRegisterClass *RC;
1165 if (AllowSP) {
1166 RC = Opc != X86::LEA32r ? &X86::GR64RegClass : &X86::GR32RegClass;
1167 } else {
1168 RC = Opc != X86::LEA32r ? &X86::GR64_NOSPRegClass : &X86::GR32_NOSPRegClass;
1169 }
1170 Register SrcReg = Src.getReg();
1171 isKill = MI.killsRegister(SrcReg);
1172
1173 // For both LEA64 and LEA32 the register already has essentially the right
1174 // type (32-bit or 64-bit) we may just need to forbid SP.
1175 if (Opc != X86::LEA64_32r) {
1176 NewSrc = SrcReg;
1177 assert(!Src.isUndef() && "Undef op doesn't need optimization");
1178
1179 if (NewSrc.isVirtual() && !MF.getRegInfo().constrainRegClass(NewSrc, RC))
1180 return false;
1181
1182 return true;
1183 }
1184
1185 // This is for an LEA64_32r and incoming registers are 32-bit. One way or
1186 // another we need to add 64-bit registers to the final MI.
1187 if (SrcReg.isPhysical()) {
1188 ImplicitOp = Src;
1189 ImplicitOp.setImplicit();
1190
1191 NewSrc = getX86SubSuperRegister(SrcReg, 64);
1192 assert(NewSrc.isValid() && "Invalid Operand");
1193 assert(!Src.isUndef() && "Undef op doesn't need optimization");
1194 } else {
1195 // Virtual register of the wrong class, we have to create a temporary 64-bit
1196 // vreg to feed into the LEA.
1197 NewSrc = MF.getRegInfo().createVirtualRegister(RC);
1198 MachineInstr *Copy =
1199 BuildMI(*MI.getParent(), MI, MI.getDebugLoc(), get(TargetOpcode::COPY))
1200 .addReg(NewSrc, RegState::Define | RegState::Undef, X86::sub_32bit)
1201 .addReg(SrcReg, getKillRegState(isKill));
1202
1203 // Which is obviously going to be dead after we're done with it.
1204 isKill = true;
1205
1206 if (LV)
1207 LV->replaceKillInstruction(SrcReg, MI, *Copy);
1208
1209 if (LIS) {
1210 SlotIndex CopyIdx = LIS->InsertMachineInstrInMaps(*Copy);
1212 LiveInterval &LI = LIS->getInterval(SrcReg);
1214 if (S->end.getBaseIndex() == Idx)
1215 S->end = CopyIdx.getRegSlot();
1216 }
1217 }
1218
1219 // We've set all the parameters without issue.
1220 return true;
1221}
1222
1223MachineInstr *X86InstrInfo::convertToThreeAddressWithLEA(unsigned MIOpc,
1225 LiveVariables *LV,
1226 LiveIntervals *LIS,
1227 bool Is8BitOp) const {
1228 // We handle 8-bit adds and various 16-bit opcodes in the switch below.
1229 MachineBasicBlock &MBB = *MI.getParent();
1231 assert((Is8BitOp ||
1233 *RegInfo.getRegClass(MI.getOperand(0).getReg())) == 16) &&
1234 "Unexpected type for LEA transform");
1235
1236 // TODO: For a 32-bit target, we need to adjust the LEA variables with
1237 // something like this:
1238 // Opcode = X86::LEA32r;
1239 // InRegLEA = RegInfo.createVirtualRegister(&X86::GR32_NOSPRegClass);
1240 // OutRegLEA =
1241 // Is8BitOp ? RegInfo.createVirtualRegister(&X86::GR32ABCD_RegClass)
1242 // : RegInfo.createVirtualRegister(&X86::GR32RegClass);
1243 if (!Subtarget.is64Bit())
1244 return nullptr;
1245
1246 unsigned Opcode = X86::LEA64_32r;
1247 Register InRegLEA = RegInfo.createVirtualRegister(&X86::GR64_NOSPRegClass);
1248 Register OutRegLEA = RegInfo.createVirtualRegister(&X86::GR32RegClass);
1249 Register InRegLEA2;
1250
1251 // Build and insert into an implicit UNDEF value. This is OK because
1252 // we will be shifting and then extracting the lower 8/16-bits.
1253 // This has the potential to cause partial register stall. e.g.
1254 // movw (%rbp,%rcx,2), %dx
1255 // leal -65(%rdx), %esi
1256 // But testing has shown this *does* help performance in 64-bit mode (at
1257 // least on modern x86 machines).
1258 MachineBasicBlock::iterator MBBI = MI.getIterator();
1259 Register Dest = MI.getOperand(0).getReg();
1260 Register Src = MI.getOperand(1).getReg();
1261 Register Src2;
1262 bool IsDead = MI.getOperand(0).isDead();
1263 bool IsKill = MI.getOperand(1).isKill();
1264 unsigned SubReg = Is8BitOp ? X86::sub_8bit : X86::sub_16bit;
1265 assert(!MI.getOperand(1).isUndef() && "Undef op doesn't need optimization");
1266 MachineInstr *ImpDef =
1267 BuildMI(MBB, MBBI, MI.getDebugLoc(), get(X86::IMPLICIT_DEF), InRegLEA);
1268 MachineInstr *InsMI =
1269 BuildMI(MBB, MBBI, MI.getDebugLoc(), get(TargetOpcode::COPY))
1270 .addReg(InRegLEA, RegState::Define, SubReg)
1271 .addReg(Src, getKillRegState(IsKill));
1272 MachineInstr *ImpDef2 = nullptr;
1273 MachineInstr *InsMI2 = nullptr;
1274
1276 BuildMI(MBB, MBBI, MI.getDebugLoc(), get(Opcode), OutRegLEA);
1277 switch (MIOpc) {
1278 default:
1279 llvm_unreachable("Unreachable!");
1280 case X86::SHL8ri:
1281 case X86::SHL16ri: {
1282 unsigned ShAmt = MI.getOperand(2).getImm();
1283 MIB.addReg(0)
1284 .addImm(1LL << ShAmt)
1285 .addReg(InRegLEA, RegState::Kill)
1286 .addImm(0)
1287 .addReg(0);
1288 break;
1289 }
1290 case X86::INC8r:
1291 case X86::INC16r:
1292 addRegOffset(MIB, InRegLEA, true, 1);
1293 break;
1294 case X86::DEC8r:
1295 case X86::DEC16r:
1296 addRegOffset(MIB, InRegLEA, true, -1);
1297 break;
1298 case X86::ADD8ri:
1299 case X86::ADD8ri_DB:
1300 case X86::ADD16ri:
1301 case X86::ADD16ri_DB:
1302 addRegOffset(MIB, InRegLEA, true, MI.getOperand(2).getImm());
1303 break;
1304 case X86::ADD8rr:
1305 case X86::ADD8rr_DB:
1306 case X86::ADD16rr:
1307 case X86::ADD16rr_DB: {
1308 Src2 = MI.getOperand(2).getReg();
1309 bool IsKill2 = MI.getOperand(2).isKill();
1310 assert(!MI.getOperand(2).isUndef() && "Undef op doesn't need optimization");
1311 if (Src == Src2) {
1312 // ADD8rr/ADD16rr killed %reg1028, %reg1028
1313 // just a single insert_subreg.
1314 addRegReg(MIB, InRegLEA, true, InRegLEA, false);
1315 } else {
1316 if (Subtarget.is64Bit())
1317 InRegLEA2 = RegInfo.createVirtualRegister(&X86::GR64_NOSPRegClass);
1318 else
1319 InRegLEA2 = RegInfo.createVirtualRegister(&X86::GR32_NOSPRegClass);
1320 // Build and insert into an implicit UNDEF value. This is OK because
1321 // we will be shifting and then extracting the lower 8/16-bits.
1322 ImpDef2 = BuildMI(MBB, &*MIB, MI.getDebugLoc(), get(X86::IMPLICIT_DEF),
1323 InRegLEA2);
1324 InsMI2 = BuildMI(MBB, &*MIB, MI.getDebugLoc(), get(TargetOpcode::COPY))
1325 .addReg(InRegLEA2, RegState::Define, SubReg)
1326 .addReg(Src2, getKillRegState(IsKill2));
1327 addRegReg(MIB, InRegLEA, true, InRegLEA2, true);
1328 }
1329 if (LV && IsKill2 && InsMI2)
1330 LV->replaceKillInstruction(Src2, MI, *InsMI2);
1331 break;
1332 }
1333 }
1334
1335 MachineInstr *NewMI = MIB;
1336 MachineInstr *ExtMI =
1337 BuildMI(MBB, MBBI, MI.getDebugLoc(), get(TargetOpcode::COPY))
1339 .addReg(OutRegLEA, RegState::Kill, SubReg);
1340
1341 if (LV) {
1342 // Update live variables.
1343 LV->getVarInfo(InRegLEA).Kills.push_back(NewMI);
1344 if (InRegLEA2)
1345 LV->getVarInfo(InRegLEA2).Kills.push_back(NewMI);
1346 LV->getVarInfo(OutRegLEA).Kills.push_back(ExtMI);
1347 if (IsKill)
1348 LV->replaceKillInstruction(Src, MI, *InsMI);
1349 if (IsDead)
1350 LV->replaceKillInstruction(Dest, MI, *ExtMI);
1351 }
1352
1353 if (LIS) {
1354 LIS->InsertMachineInstrInMaps(*ImpDef);
1355 SlotIndex InsIdx = LIS->InsertMachineInstrInMaps(*InsMI);
1356 if (ImpDef2)
1357 LIS->InsertMachineInstrInMaps(*ImpDef2);
1358 SlotIndex Ins2Idx;
1359 if (InsMI2)
1360 Ins2Idx = LIS->InsertMachineInstrInMaps(*InsMI2);
1361 SlotIndex NewIdx = LIS->ReplaceMachineInstrInMaps(MI, *NewMI);
1362 SlotIndex ExtIdx = LIS->InsertMachineInstrInMaps(*ExtMI);
1363 LIS->getInterval(InRegLEA);
1364 LIS->getInterval(OutRegLEA);
1365 if (InRegLEA2)
1366 LIS->getInterval(InRegLEA2);
1367
1368 // Move the use of Src up to InsMI.
1369 LiveInterval &SrcLI = LIS->getInterval(Src);
1370 LiveRange::Segment *SrcSeg = SrcLI.getSegmentContaining(NewIdx);
1371 if (SrcSeg->end == NewIdx.getRegSlot())
1372 SrcSeg->end = InsIdx.getRegSlot();
1373
1374 if (InsMI2) {
1375 // Move the use of Src2 up to InsMI2.
1376 LiveInterval &Src2LI = LIS->getInterval(Src2);
1377 LiveRange::Segment *Src2Seg = Src2LI.getSegmentContaining(NewIdx);
1378 if (Src2Seg->end == NewIdx.getRegSlot())
1379 Src2Seg->end = Ins2Idx.getRegSlot();
1380 }
1381
1382 // Move the definition of Dest down to ExtMI.
1383 LiveInterval &DestLI = LIS->getInterval(Dest);
1384 LiveRange::Segment *DestSeg =
1385 DestLI.getSegmentContaining(NewIdx.getRegSlot());
1386 assert(DestSeg->start == NewIdx.getRegSlot() &&
1387 DestSeg->valno->def == NewIdx.getRegSlot());
1388 DestSeg->start = ExtIdx.getRegSlot();
1389 DestSeg->valno->def = ExtIdx.getRegSlot();
1390 }
1391
1392 return ExtMI;
1393}
1394
1395/// This method must be implemented by targets that
1396/// set the M_CONVERTIBLE_TO_3_ADDR flag. When this flag is set, the target
1397/// may be able to convert a two-address instruction into a true
1398/// three-address instruction on demand. This allows the X86 target (for
1399/// example) to convert ADD and SHL instructions into LEA instructions if they
1400/// would require register copies due to two-addressness.
1401///
1402/// This method returns a null pointer if the transformation cannot be
1403/// performed, otherwise it returns the new instruction.
1404///
1406 LiveVariables *LV,
1407 LiveIntervals *LIS) const {
1408 // The following opcodes also sets the condition code register(s). Only
1409 // convert them to equivalent lea if the condition code register def's
1410 // are dead!
1412 return nullptr;
1413
1414 MachineFunction &MF = *MI.getParent()->getParent();
1415 // All instructions input are two-addr instructions. Get the known operands.
1416 const MachineOperand &Dest = MI.getOperand(0);
1417 const MachineOperand &Src = MI.getOperand(1);
1418
1419 // Ideally, operations with undef should be folded before we get here, but we
1420 // can't guarantee it. Bail out because optimizing undefs is a waste of time.
1421 // Without this, we have to forward undef state to new register operands to
1422 // avoid machine verifier errors.
1423 if (Src.isUndef())
1424 return nullptr;
1425 if (MI.getNumOperands() > 2)
1426 if (MI.getOperand(2).isReg() && MI.getOperand(2).isUndef())
1427 return nullptr;
1428
1429 MachineInstr *NewMI = nullptr;
1430 Register SrcReg, SrcReg2;
1431 bool Is64Bit = Subtarget.is64Bit();
1432
1433 bool Is8BitOp = false;
1434 unsigned NumRegOperands = 2;
1435 unsigned MIOpc = MI.getOpcode();
1436 switch (MIOpc) {
1437 default:
1438 llvm_unreachable("Unreachable!");
1439 case X86::SHL64ri: {
1440 assert(MI.getNumOperands() >= 3 && "Unknown shift instruction!");
1441 unsigned ShAmt = getTruncatedShiftCount(MI, 2);
1442 if (!isTruncatedShiftCountForLEA(ShAmt))
1443 return nullptr;
1444
1445 // LEA can't handle RSP.
1446 if (Src.getReg().isVirtual() && !MF.getRegInfo().constrainRegClass(
1447 Src.getReg(), &X86::GR64_NOSPRegClass))
1448 return nullptr;
1449
1450 NewMI = BuildMI(MF, MI.getDebugLoc(), get(X86::LEA64r))
1451 .add(Dest)
1452 .addReg(0)
1453 .addImm(1LL << ShAmt)
1454 .add(Src)
1455 .addImm(0)
1456 .addReg(0);
1457 break;
1458 }
1459 case X86::SHL32ri: {
1460 assert(MI.getNumOperands() >= 3 && "Unknown shift instruction!");
1461 unsigned ShAmt = getTruncatedShiftCount(MI, 2);
1462 if (!isTruncatedShiftCountForLEA(ShAmt))
1463 return nullptr;
1464
1465 unsigned Opc = Is64Bit ? X86::LEA64_32r : X86::LEA32r;
1466
1467 // LEA can't handle ESP.
1468 bool isKill;
1469 MachineOperand ImplicitOp = MachineOperand::CreateReg(0, false);
1470 if (!classifyLEAReg(MI, Src, Opc, /*AllowSP=*/false, SrcReg, isKill,
1471 ImplicitOp, LV, LIS))
1472 return nullptr;
1473
1474 MachineInstrBuilder MIB = BuildMI(MF, MI.getDebugLoc(), get(Opc))
1475 .add(Dest)
1476 .addReg(0)
1477 .addImm(1LL << ShAmt)
1478 .addReg(SrcReg, getKillRegState(isKill))
1479 .addImm(0)
1480 .addReg(0);
1481 if (ImplicitOp.getReg() != 0)
1482 MIB.add(ImplicitOp);
1483 NewMI = MIB;
1484
1485 // Add kills if classifyLEAReg created a new register.
1486 if (LV && SrcReg != Src.getReg())
1487 LV->getVarInfo(SrcReg).Kills.push_back(NewMI);
1488 break;
1489 }
1490 case X86::SHL8ri:
1491 Is8BitOp = true;
1492 [[fallthrough]];
1493 case X86::SHL16ri: {
1494 assert(MI.getNumOperands() >= 3 && "Unknown shift instruction!");
1495 unsigned ShAmt = getTruncatedShiftCount(MI, 2);
1496 if (!isTruncatedShiftCountForLEA(ShAmt))
1497 return nullptr;
1498 return convertToThreeAddressWithLEA(MIOpc, MI, LV, LIS, Is8BitOp);
1499 }
1500 case X86::INC64r:
1501 case X86::INC32r: {
1502 assert(MI.getNumOperands() >= 2 && "Unknown inc instruction!");
1503 unsigned Opc = MIOpc == X86::INC64r
1504 ? X86::LEA64r
1505 : (Is64Bit ? X86::LEA64_32r : X86::LEA32r);
1506 bool isKill;
1507 MachineOperand ImplicitOp = MachineOperand::CreateReg(0, false);
1508 if (!classifyLEAReg(MI, Src, Opc, /*AllowSP=*/false, SrcReg, isKill,
1509 ImplicitOp, LV, LIS))
1510 return nullptr;
1511
1512 MachineInstrBuilder MIB = BuildMI(MF, MI.getDebugLoc(), get(Opc))
1513 .add(Dest)
1514 .addReg(SrcReg, getKillRegState(isKill));
1515 if (ImplicitOp.getReg() != 0)
1516 MIB.add(ImplicitOp);
1517
1518 NewMI = addOffset(MIB, 1);
1519
1520 // Add kills if classifyLEAReg created a new register.
1521 if (LV && SrcReg != Src.getReg())
1522 LV->getVarInfo(SrcReg).Kills.push_back(NewMI);
1523 break;
1524 }
1525 case X86::DEC64r:
1526 case X86::DEC32r: {
1527 assert(MI.getNumOperands() >= 2 && "Unknown dec instruction!");
1528 unsigned Opc = MIOpc == X86::DEC64r
1529 ? X86::LEA64r
1530 : (Is64Bit ? X86::LEA64_32r : X86::LEA32r);
1531
1532 bool isKill;
1533 MachineOperand ImplicitOp = MachineOperand::CreateReg(0, false);
1534 if (!classifyLEAReg(MI, Src, Opc, /*AllowSP=*/false, SrcReg, isKill,
1535 ImplicitOp, LV, LIS))
1536 return nullptr;
1537
1538 MachineInstrBuilder MIB = BuildMI(MF, MI.getDebugLoc(), get(Opc))
1539 .add(Dest)
1540 .addReg(SrcReg, getKillRegState(isKill));
1541 if (ImplicitOp.getReg() != 0)
1542 MIB.add(ImplicitOp);
1543
1544 NewMI = addOffset(MIB, -1);
1545
1546 // Add kills if classifyLEAReg created a new register.
1547 if (LV && SrcReg != Src.getReg())
1548 LV->getVarInfo(SrcReg).Kills.push_back(NewMI);
1549 break;
1550 }
1551 case X86::DEC8r:
1552 case X86::INC8r:
1553 Is8BitOp = true;
1554 [[fallthrough]];
1555 case X86::DEC16r:
1556 case X86::INC16r:
1557 return convertToThreeAddressWithLEA(MIOpc, MI, LV, LIS, Is8BitOp);
1558 case X86::ADD64rr:
1559 case X86::ADD64rr_DB:
1560 case X86::ADD32rr:
1561 case X86::ADD32rr_DB: {
1562 assert(MI.getNumOperands() >= 3 && "Unknown add instruction!");
1563 unsigned Opc;
1564 if (MIOpc == X86::ADD64rr || MIOpc == X86::ADD64rr_DB)
1565 Opc = X86::LEA64r;
1566 else
1567 Opc = Is64Bit ? X86::LEA64_32r : X86::LEA32r;
1568
1569 const MachineOperand &Src2 = MI.getOperand(2);
1570 bool isKill2;
1571 MachineOperand ImplicitOp2 = MachineOperand::CreateReg(0, false);
1572 if (!classifyLEAReg(MI, Src2, Opc, /*AllowSP=*/false, SrcReg2, isKill2,
1573 ImplicitOp2, LV, LIS))
1574 return nullptr;
1575
1576 bool isKill;
1577 MachineOperand ImplicitOp = MachineOperand::CreateReg(0, false);
1578 if (Src.getReg() == Src2.getReg()) {
1579 // Don't call classify LEAReg a second time on the same register, in case
1580 // the first call inserted a COPY from Src2 and marked it as killed.
1581 isKill = isKill2;
1582 SrcReg = SrcReg2;
1583 } else {
1584 if (!classifyLEAReg(MI, Src, Opc, /*AllowSP=*/true, SrcReg, isKill,
1585 ImplicitOp, LV, LIS))
1586 return nullptr;
1587 }
1588
1589 MachineInstrBuilder MIB = BuildMI(MF, MI.getDebugLoc(), get(Opc)).add(Dest);
1590 if (ImplicitOp.getReg() != 0)
1591 MIB.add(ImplicitOp);
1592 if (ImplicitOp2.getReg() != 0)
1593 MIB.add(ImplicitOp2);
1594
1595 NewMI = addRegReg(MIB, SrcReg, isKill, SrcReg2, isKill2);
1596
1597 // Add kills if classifyLEAReg created a new register.
1598 if (LV) {
1599 if (SrcReg2 != Src2.getReg())
1600 LV->getVarInfo(SrcReg2).Kills.push_back(NewMI);
1601 if (SrcReg != SrcReg2 && SrcReg != Src.getReg())
1602 LV->getVarInfo(SrcReg).Kills.push_back(NewMI);
1603 }
1604 NumRegOperands = 3;
1605 break;
1606 }
1607 case X86::ADD8rr:
1608 case X86::ADD8rr_DB:
1609 Is8BitOp = true;
1610 [[fallthrough]];
1611 case X86::ADD16rr:
1612 case X86::ADD16rr_DB:
1613 return convertToThreeAddressWithLEA(MIOpc, MI, LV, LIS, Is8BitOp);
1614 case X86::ADD64ri32:
1615 case X86::ADD64ri32_DB:
1616 assert(MI.getNumOperands() >= 3 && "Unknown add instruction!");
1617 NewMI = addOffset(
1618 BuildMI(MF, MI.getDebugLoc(), get(X86::LEA64r)).add(Dest).add(Src),
1619 MI.getOperand(2));
1620 break;
1621 case X86::ADD32ri:
1622 case X86::ADD32ri_DB: {
1623 assert(MI.getNumOperands() >= 3 && "Unknown add instruction!");
1624 unsigned Opc = Is64Bit ? X86::LEA64_32r : X86::LEA32r;
1625
1626 bool isKill;
1627 MachineOperand ImplicitOp = MachineOperand::CreateReg(0, false);
1628 if (!classifyLEAReg(MI, Src, Opc, /*AllowSP=*/true, SrcReg, isKill,
1629 ImplicitOp, LV, LIS))
1630 return nullptr;
1631
1632 MachineInstrBuilder MIB = BuildMI(MF, MI.getDebugLoc(), get(Opc))
1633 .add(Dest)
1634 .addReg(SrcReg, getKillRegState(isKill));
1635 if (ImplicitOp.getReg() != 0)
1636 MIB.add(ImplicitOp);
1637
1638 NewMI = addOffset(MIB, MI.getOperand(2));
1639
1640 // Add kills if classifyLEAReg created a new register.
1641 if (LV && SrcReg != Src.getReg())
1642 LV->getVarInfo(SrcReg).Kills.push_back(NewMI);
1643 break;
1644 }
1645 case X86::ADD8ri:
1646 case X86::ADD8ri_DB:
1647 Is8BitOp = true;
1648 [[fallthrough]];
1649 case X86::ADD16ri:
1650 case X86::ADD16ri_DB:
1651 return convertToThreeAddressWithLEA(MIOpc, MI, LV, LIS, Is8BitOp);
1652 case X86::SUB8ri:
1653 case X86::SUB16ri:
1654 /// FIXME: Support these similar to ADD8ri/ADD16ri*.
1655 return nullptr;
1656 case X86::SUB32ri: {
1657 if (!MI.getOperand(2).isImm())
1658 return nullptr;
1659 int64_t Imm = MI.getOperand(2).getImm();
1660 if (!isInt<32>(-Imm))
1661 return nullptr;
1662
1663 assert(MI.getNumOperands() >= 3 && "Unknown add instruction!");
1664 unsigned Opc = Is64Bit ? X86::LEA64_32r : X86::LEA32r;
1665
1666 bool isKill;
1667 MachineOperand ImplicitOp = MachineOperand::CreateReg(0, false);
1668 if (!classifyLEAReg(MI, Src, Opc, /*AllowSP=*/true, SrcReg, isKill,
1669 ImplicitOp, LV, LIS))
1670 return nullptr;
1671
1672 MachineInstrBuilder MIB = BuildMI(MF, MI.getDebugLoc(), get(Opc))
1673 .add(Dest)
1674 .addReg(SrcReg, getKillRegState(isKill));
1675 if (ImplicitOp.getReg() != 0)
1676 MIB.add(ImplicitOp);
1677
1678 NewMI = addOffset(MIB, -Imm);
1679
1680 // Add kills if classifyLEAReg created a new register.
1681 if (LV && SrcReg != Src.getReg())
1682 LV->getVarInfo(SrcReg).Kills.push_back(NewMI);
1683 break;
1684 }
1685
1686 case X86::SUB64ri32: {
1687 if (!MI.getOperand(2).isImm())
1688 return nullptr;
1689 int64_t Imm = MI.getOperand(2).getImm();
1690 if (!isInt<32>(-Imm))
1691 return nullptr;
1692
1693 assert(MI.getNumOperands() >= 3 && "Unknown sub instruction!");
1694
1696 BuildMI(MF, MI.getDebugLoc(), get(X86::LEA64r)).add(Dest).add(Src);
1697 NewMI = addOffset(MIB, -Imm);
1698 break;
1699 }
1700
1701 case X86::VMOVDQU8Z128rmk:
1702 case X86::VMOVDQU8Z256rmk:
1703 case X86::VMOVDQU8Zrmk:
1704 case X86::VMOVDQU16Z128rmk:
1705 case X86::VMOVDQU16Z256rmk:
1706 case X86::VMOVDQU16Zrmk:
1707 case X86::VMOVDQU32Z128rmk:
1708 case X86::VMOVDQA32Z128rmk:
1709 case X86::VMOVDQU32Z256rmk:
1710 case X86::VMOVDQA32Z256rmk:
1711 case X86::VMOVDQU32Zrmk:
1712 case X86::VMOVDQA32Zrmk:
1713 case X86::VMOVDQU64Z128rmk:
1714 case X86::VMOVDQA64Z128rmk:
1715 case X86::VMOVDQU64Z256rmk:
1716 case X86::VMOVDQA64Z256rmk:
1717 case X86::VMOVDQU64Zrmk:
1718 case X86::VMOVDQA64Zrmk:
1719 case X86::VMOVUPDZ128rmk:
1720 case X86::VMOVAPDZ128rmk:
1721 case X86::VMOVUPDZ256rmk:
1722 case X86::VMOVAPDZ256rmk:
1723 case X86::VMOVUPDZrmk:
1724 case X86::VMOVAPDZrmk:
1725 case X86::VMOVUPSZ128rmk:
1726 case X86::VMOVAPSZ128rmk:
1727 case X86::VMOVUPSZ256rmk:
1728 case X86::VMOVAPSZ256rmk:
1729 case X86::VMOVUPSZrmk:
1730 case X86::VMOVAPSZrmk:
1731 case X86::VBROADCASTSDZ256rmk:
1732 case X86::VBROADCASTSDZrmk:
1733 case X86::VBROADCASTSSZ128rmk:
1734 case X86::VBROADCASTSSZ256rmk:
1735 case X86::VBROADCASTSSZrmk:
1736 case X86::VPBROADCASTDZ128rmk:
1737 case X86::VPBROADCASTDZ256rmk:
1738 case X86::VPBROADCASTDZrmk:
1739 case X86::VPBROADCASTQZ128rmk:
1740 case X86::VPBROADCASTQZ256rmk:
1741 case X86::VPBROADCASTQZrmk: {
1742 unsigned Opc;
1743 switch (MIOpc) {
1744 default:
1745 llvm_unreachable("Unreachable!");
1746 case X86::VMOVDQU8Z128rmk:
1747 Opc = X86::VPBLENDMBZ128rmk;
1748 break;
1749 case X86::VMOVDQU8Z256rmk:
1750 Opc = X86::VPBLENDMBZ256rmk;
1751 break;
1752 case X86::VMOVDQU8Zrmk:
1753 Opc = X86::VPBLENDMBZrmk;
1754 break;
1755 case X86::VMOVDQU16Z128rmk:
1756 Opc = X86::VPBLENDMWZ128rmk;
1757 break;
1758 case X86::VMOVDQU16Z256rmk:
1759 Opc = X86::VPBLENDMWZ256rmk;
1760 break;
1761 case X86::VMOVDQU16Zrmk:
1762 Opc = X86::VPBLENDMWZrmk;
1763 break;
1764 case X86::VMOVDQU32Z128rmk:
1765 Opc = X86::VPBLENDMDZ128rmk;
1766 break;
1767 case X86::VMOVDQU32Z256rmk:
1768 Opc = X86::VPBLENDMDZ256rmk;
1769 break;
1770 case X86::VMOVDQU32Zrmk:
1771 Opc = X86::VPBLENDMDZrmk;
1772 break;
1773 case X86::VMOVDQU64Z128rmk:
1774 Opc = X86::VPBLENDMQZ128rmk;
1775 break;
1776 case X86::VMOVDQU64Z256rmk:
1777 Opc = X86::VPBLENDMQZ256rmk;
1778 break;
1779 case X86::VMOVDQU64Zrmk:
1780 Opc = X86::VPBLENDMQZrmk;
1781 break;
1782 case X86::VMOVUPDZ128rmk:
1783 Opc = X86::VBLENDMPDZ128rmk;
1784 break;
1785 case X86::VMOVUPDZ256rmk:
1786 Opc = X86::VBLENDMPDZ256rmk;
1787 break;
1788 case X86::VMOVUPDZrmk:
1789 Opc = X86::VBLENDMPDZrmk;
1790 break;
1791 case X86::VMOVUPSZ128rmk:
1792 Opc = X86::VBLENDMPSZ128rmk;
1793 break;
1794 case X86::VMOVUPSZ256rmk:
1795 Opc = X86::VBLENDMPSZ256rmk;
1796 break;
1797 case X86::VMOVUPSZrmk:
1798 Opc = X86::VBLENDMPSZrmk;
1799 break;
1800 case X86::VMOVDQA32Z128rmk:
1801 Opc = X86::VPBLENDMDZ128rmk;
1802 break;
1803 case X86::VMOVDQA32Z256rmk:
1804 Opc = X86::VPBLENDMDZ256rmk;
1805 break;
1806 case X86::VMOVDQA32Zrmk:
1807 Opc = X86::VPBLENDMDZrmk;
1808 break;
1809 case X86::VMOVDQA64Z128rmk:
1810 Opc = X86::VPBLENDMQZ128rmk;
1811 break;
1812 case X86::VMOVDQA64Z256rmk:
1813 Opc = X86::VPBLENDMQZ256rmk;
1814 break;
1815 case X86::VMOVDQA64Zrmk:
1816 Opc = X86::VPBLENDMQZrmk;
1817 break;
1818 case X86::VMOVAPDZ128rmk:
1819 Opc = X86::VBLENDMPDZ128rmk;
1820 break;
1821 case X86::VMOVAPDZ256rmk:
1822 Opc = X86::VBLENDMPDZ256rmk;
1823 break;
1824 case X86::VMOVAPDZrmk:
1825 Opc = X86::VBLENDMPDZrmk;
1826 break;
1827 case X86::VMOVAPSZ128rmk:
1828 Opc = X86::VBLENDMPSZ128rmk;
1829 break;
1830 case X86::VMOVAPSZ256rmk:
1831 Opc = X86::VBLENDMPSZ256rmk;
1832 break;
1833 case X86::VMOVAPSZrmk:
1834 Opc = X86::VBLENDMPSZrmk;
1835 break;
1836 case X86::VBROADCASTSDZ256rmk:
1837 Opc = X86::VBLENDMPDZ256rmbk;
1838 break;
1839 case X86::VBROADCASTSDZrmk:
1840 Opc = X86::VBLENDMPDZrmbk;
1841 break;
1842 case X86::VBROADCASTSSZ128rmk:
1843 Opc = X86::VBLENDMPSZ128rmbk;
1844 break;
1845 case X86::VBROADCASTSSZ256rmk:
1846 Opc = X86::VBLENDMPSZ256rmbk;
1847 break;
1848 case X86::VBROADCASTSSZrmk:
1849 Opc = X86::VBLENDMPSZrmbk;
1850 break;
1851 case X86::VPBROADCASTDZ128rmk:
1852 Opc = X86::VPBLENDMDZ128rmbk;
1853 break;
1854 case X86::VPBROADCASTDZ256rmk:
1855 Opc = X86::VPBLENDMDZ256rmbk;
1856 break;
1857 case X86::VPBROADCASTDZrmk:
1858 Opc = X86::VPBLENDMDZrmbk;
1859 break;
1860 case X86::VPBROADCASTQZ128rmk:
1861 Opc = X86::VPBLENDMQZ128rmbk;
1862 break;
1863 case X86::VPBROADCASTQZ256rmk:
1864 Opc = X86::VPBLENDMQZ256rmbk;
1865 break;
1866 case X86::VPBROADCASTQZrmk:
1867 Opc = X86::VPBLENDMQZrmbk;
1868 break;
1869 }
1870
1871 NewMI = BuildMI(MF, MI.getDebugLoc(), get(Opc))
1872 .add(Dest)
1873 .add(MI.getOperand(2))
1874 .add(Src)
1875 .add(MI.getOperand(3))
1876 .add(MI.getOperand(4))
1877 .add(MI.getOperand(5))
1878 .add(MI.getOperand(6))
1879 .add(MI.getOperand(7));
1880 NumRegOperands = 4;
1881 break;
1882 }
1883
1884 case X86::VMOVDQU8Z128rrk:
1885 case X86::VMOVDQU8Z256rrk:
1886 case X86::VMOVDQU8Zrrk:
1887 case X86::VMOVDQU16Z128rrk:
1888 case X86::VMOVDQU16Z256rrk:
1889 case X86::VMOVDQU16Zrrk:
1890 case X86::VMOVDQU32Z128rrk:
1891 case X86::VMOVDQA32Z128rrk:
1892 case X86::VMOVDQU32Z256rrk:
1893 case X86::VMOVDQA32Z256rrk:
1894 case X86::VMOVDQU32Zrrk:
1895 case X86::VMOVDQA32Zrrk:
1896 case X86::VMOVDQU64Z128rrk:
1897 case X86::VMOVDQA64Z128rrk:
1898 case X86::VMOVDQU64Z256rrk:
1899 case X86::VMOVDQA64Z256rrk:
1900 case X86::VMOVDQU64Zrrk:
1901 case X86::VMOVDQA64Zrrk:
1902 case X86::VMOVUPDZ128rrk:
1903 case X86::VMOVAPDZ128rrk:
1904 case X86::VMOVUPDZ256rrk:
1905 case X86::VMOVAPDZ256rrk:
1906 case X86::VMOVUPDZrrk:
1907 case X86::VMOVAPDZrrk:
1908 case X86::VMOVUPSZ128rrk:
1909 case X86::VMOVAPSZ128rrk:
1910 case X86::VMOVUPSZ256rrk:
1911 case X86::VMOVAPSZ256rrk:
1912 case X86::VMOVUPSZrrk:
1913 case X86::VMOVAPSZrrk: {
1914 unsigned Opc;
1915 switch (MIOpc) {
1916 default:
1917 llvm_unreachable("Unreachable!");
1918 case X86::VMOVDQU8Z128rrk:
1919 Opc = X86::VPBLENDMBZ128rrk;
1920 break;
1921 case X86::VMOVDQU8Z256rrk:
1922 Opc = X86::VPBLENDMBZ256rrk;
1923 break;
1924 case X86::VMOVDQU8Zrrk:
1925 Opc = X86::VPBLENDMBZrrk;
1926 break;
1927 case X86::VMOVDQU16Z128rrk:
1928 Opc = X86::VPBLENDMWZ128rrk;
1929 break;
1930 case X86::VMOVDQU16Z256rrk:
1931 Opc = X86::VPBLENDMWZ256rrk;
1932 break;
1933 case X86::VMOVDQU16Zrrk:
1934 Opc = X86::VPBLENDMWZrrk;
1935 break;
1936 case X86::VMOVDQU32Z128rrk:
1937 Opc = X86::VPBLENDMDZ128rrk;
1938 break;
1939 case X86::VMOVDQU32Z256rrk:
1940 Opc = X86::VPBLENDMDZ256rrk;
1941 break;
1942 case X86::VMOVDQU32Zrrk:
1943 Opc = X86::VPBLENDMDZrrk;
1944 break;
1945 case X86::VMOVDQU64Z128rrk:
1946 Opc = X86::VPBLENDMQZ128rrk;
1947 break;
1948 case X86::VMOVDQU64Z256rrk:
1949 Opc = X86::VPBLENDMQZ256rrk;
1950 break;
1951 case X86::VMOVDQU64Zrrk:
1952 Opc = X86::VPBLENDMQZrrk;
1953 break;
1954 case X86::VMOVUPDZ128rrk:
1955 Opc = X86::VBLENDMPDZ128rrk;
1956 break;
1957 case X86::VMOVUPDZ256rrk:
1958 Opc = X86::VBLENDMPDZ256rrk;
1959 break;
1960 case X86::VMOVUPDZrrk:
1961 Opc = X86::VBLENDMPDZrrk;
1962 break;
1963 case X86::VMOVUPSZ128rrk:
1964 Opc = X86::VBLENDMPSZ128rrk;
1965 break;
1966 case X86::VMOVUPSZ256rrk:
1967 Opc = X86::VBLENDMPSZ256rrk;
1968 break;
1969 case X86::VMOVUPSZrrk:
1970 Opc = X86::VBLENDMPSZrrk;
1971 break;
1972 case X86::VMOVDQA32Z128rrk:
1973 Opc = X86::VPBLENDMDZ128rrk;
1974 break;
1975 case X86::VMOVDQA32Z256rrk:
1976 Opc = X86::VPBLENDMDZ256rrk;
1977 break;
1978 case X86::VMOVDQA32Zrrk:
1979 Opc = X86::VPBLENDMDZrrk;
1980 break;
1981 case X86::VMOVDQA64Z128rrk:
1982 Opc = X86::VPBLENDMQZ128rrk;
1983 break;
1984 case X86::VMOVDQA64Z256rrk:
1985 Opc = X86::VPBLENDMQZ256rrk;
1986 break;
1987 case X86::VMOVDQA64Zrrk:
1988 Opc = X86::VPBLENDMQZrrk;
1989 break;
1990 case X86::VMOVAPDZ128rrk:
1991 Opc = X86::VBLENDMPDZ128rrk;
1992 break;
1993 case X86::VMOVAPDZ256rrk:
1994 Opc = X86::VBLENDMPDZ256rrk;
1995 break;
1996 case X86::VMOVAPDZrrk:
1997 Opc = X86::VBLENDMPDZrrk;
1998 break;
1999 case X86::VMOVAPSZ128rrk:
2000 Opc = X86::VBLENDMPSZ128rrk;
2001 break;
2002 case X86::VMOVAPSZ256rrk:
2003 Opc = X86::VBLENDMPSZ256rrk;
2004 break;
2005 case X86::VMOVAPSZrrk:
2006 Opc = X86::VBLENDMPSZrrk;
2007 break;
2008 }
2009
2010 NewMI = BuildMI(MF, MI.getDebugLoc(), get(Opc))
2011 .add(Dest)
2012 .add(MI.getOperand(2))
2013 .add(Src)
2014 .add(MI.getOperand(3));
2015 NumRegOperands = 4;
2016 break;
2017 }
2018 }
2019
2020 if (!NewMI)
2021 return nullptr;
2022
2023 if (LV) { // Update live variables
2024 for (unsigned I = 0; I < NumRegOperands; ++I) {
2025 MachineOperand &Op = MI.getOperand(I);
2026 if (Op.isReg() && (Op.isDead() || Op.isKill()))
2027 LV->replaceKillInstruction(Op.getReg(), MI, *NewMI);
2028 }
2029 }
2030
2031 MachineBasicBlock &MBB = *MI.getParent();
2032 MBB.insert(MI.getIterator(), NewMI); // Insert the new inst
2033
2034 if (LIS) {
2035 LIS->ReplaceMachineInstrInMaps(MI, *NewMI);
2036 if (SrcReg)
2037 LIS->getInterval(SrcReg);
2038 if (SrcReg2)
2039 LIS->getInterval(SrcReg2);
2040 }
2041
2042 return NewMI;
2043}
2044
2045/// This determines which of three possible cases of a three source commute
2046/// the source indexes correspond to taking into account any mask operands.
2047/// All prevents commuting a passthru operand. Returns -1 if the commute isn't
2048/// possible.
2049/// Case 0 - Possible to commute the first and second operands.
2050/// Case 1 - Possible to commute the first and third operands.
2051/// Case 2 - Possible to commute the second and third operands.
2052static unsigned getThreeSrcCommuteCase(uint64_t TSFlags, unsigned SrcOpIdx1,
2053 unsigned SrcOpIdx2) {
2054 // Put the lowest index to SrcOpIdx1 to simplify the checks below.
2055 if (SrcOpIdx1 > SrcOpIdx2)
2056 std::swap(SrcOpIdx1, SrcOpIdx2);
2057
2058 unsigned Op1 = 1, Op2 = 2, Op3 = 3;
2059 if (X86II::isKMasked(TSFlags)) {
2060 Op2++;
2061 Op3++;
2062 }
2063
2064 if (SrcOpIdx1 == Op1 && SrcOpIdx2 == Op2)
2065 return 0;
2066 if (SrcOpIdx1 == Op1 && SrcOpIdx2 == Op3)
2067 return 1;
2068 if (SrcOpIdx1 == Op2 && SrcOpIdx2 == Op3)
2069 return 2;
2070 llvm_unreachable("Unknown three src commute case.");
2071}
2072
2074 const MachineInstr &MI, unsigned SrcOpIdx1, unsigned SrcOpIdx2,
2075 const X86InstrFMA3Group &FMA3Group) const {
2076
2077 unsigned Opc = MI.getOpcode();
2078
2079 // TODO: Commuting the 1st operand of FMA*_Int requires some additional
2080 // analysis. The commute optimization is legal only if all users of FMA*_Int
2081 // use only the lowest element of the FMA*_Int instruction. Such analysis are
2082 // not implemented yet. So, just return 0 in that case.
2083 // When such analysis are available this place will be the right place for
2084 // calling it.
2085 assert(!(FMA3Group.isIntrinsic() && (SrcOpIdx1 == 1 || SrcOpIdx2 == 1)) &&
2086 "Intrinsic instructions can't commute operand 1");
2087
2088 // Determine which case this commute is or if it can't be done.
2089 unsigned Case =
2090 getThreeSrcCommuteCase(MI.getDesc().TSFlags, SrcOpIdx1, SrcOpIdx2);
2091 assert(Case < 3 && "Unexpected case number!");
2092
2093 // Define the FMA forms mapping array that helps to map input FMA form
2094 // to output FMA form to preserve the operation semantics after
2095 // commuting the operands.
2096 const unsigned Form132Index = 0;
2097 const unsigned Form213Index = 1;
2098 const unsigned Form231Index = 2;
2099 static const unsigned FormMapping[][3] = {
2100 // 0: SrcOpIdx1 == 1 && SrcOpIdx2 == 2;
2101 // FMA132 A, C, b; ==> FMA231 C, A, b;
2102 // FMA213 B, A, c; ==> FMA213 A, B, c;
2103 // FMA231 C, A, b; ==> FMA132 A, C, b;
2104 {Form231Index, Form213Index, Form132Index},
2105 // 1: SrcOpIdx1 == 1 && SrcOpIdx2 == 3;
2106 // FMA132 A, c, B; ==> FMA132 B, c, A;
2107 // FMA213 B, a, C; ==> FMA231 C, a, B;
2108 // FMA231 C, a, B; ==> FMA213 B, a, C;
2109 {Form132Index, Form231Index, Form213Index},
2110 // 2: SrcOpIdx1 == 2 && SrcOpIdx2 == 3;
2111 // FMA132 a, C, B; ==> FMA213 a, B, C;
2112 // FMA213 b, A, C; ==> FMA132 b, C, A;
2113 // FMA231 c, A, B; ==> FMA231 c, B, A;
2114 {Form213Index, Form132Index, Form231Index}};
2115
2116 unsigned FMAForms[3];
2117 FMAForms[0] = FMA3Group.get132Opcode();
2118 FMAForms[1] = FMA3Group.get213Opcode();
2119 FMAForms[2] = FMA3Group.get231Opcode();
2120
2121 // Everything is ready, just adjust the FMA opcode and return it.
2122 for (unsigned FormIndex = 0; FormIndex < 3; FormIndex++)
2123 if (Opc == FMAForms[FormIndex])
2124 return FMAForms[FormMapping[Case][FormIndex]];
2125
2126 llvm_unreachable("Illegal FMA3 format");
2127}
2128
2129static void commuteVPTERNLOG(MachineInstr &MI, unsigned SrcOpIdx1,
2130 unsigned SrcOpIdx2) {
2131 // Determine which case this commute is or if it can't be done.
2132 unsigned Case =
2133 getThreeSrcCommuteCase(MI.getDesc().TSFlags, SrcOpIdx1, SrcOpIdx2);
2134 assert(Case < 3 && "Unexpected case value!");
2135
2136 // For each case we need to swap two pairs of bits in the final immediate.
2137 static const uint8_t SwapMasks[3][4] = {
2138 {0x04, 0x10, 0x08, 0x20}, // Swap bits 2/4 and 3/5.
2139 {0x02, 0x10, 0x08, 0x40}, // Swap bits 1/4 and 3/6.
2140 {0x02, 0x04, 0x20, 0x40}, // Swap bits 1/2 and 5/6.
2141 };
2142
2143 uint8_t Imm = MI.getOperand(MI.getNumOperands() - 1).getImm();
2144 // Clear out the bits we are swapping.
2145 uint8_t NewImm = Imm & ~(SwapMasks[Case][0] | SwapMasks[Case][1] |
2146 SwapMasks[Case][2] | SwapMasks[Case][3]);
2147 // If the immediate had a bit of the pair set, then set the opposite bit.
2148 if (Imm & SwapMasks[Case][0])
2149 NewImm |= SwapMasks[Case][1];
2150 if (Imm & SwapMasks[Case][1])
2151 NewImm |= SwapMasks[Case][0];
2152 if (Imm & SwapMasks[Case][2])
2153 NewImm |= SwapMasks[Case][3];
2154 if (Imm & SwapMasks[Case][3])
2155 NewImm |= SwapMasks[Case][2];
2156 MI.getOperand(MI.getNumOperands() - 1).setImm(NewImm);
2157}
2158
2159// Returns true if this is a VPERMI2 or VPERMT2 instruction that can be
2160// commuted.
2161static bool isCommutableVPERMV3Instruction(unsigned Opcode) {
2162#define VPERM_CASES(Suffix) \
2163 case X86::VPERMI2##Suffix##Z128rr: \
2164 case X86::VPERMT2##Suffix##Z128rr: \
2165 case X86::VPERMI2##Suffix##Z256rr: \
2166 case X86::VPERMT2##Suffix##Z256rr: \
2167 case X86::VPERMI2##Suffix##Zrr: \
2168 case X86::VPERMT2##Suffix##Zrr: \
2169 case X86::VPERMI2##Suffix##Z128rm: \
2170 case X86::VPERMT2##Suffix##Z128rm: \
2171 case X86::VPERMI2##Suffix##Z256rm: \
2172 case X86::VPERMT2##Suffix##Z256rm: \
2173 case X86::VPERMI2##Suffix##Zrm: \
2174 case X86::VPERMT2##Suffix##Zrm: \
2175 case X86::VPERMI2##Suffix##Z128rrkz: \
2176 case X86::VPERMT2##Suffix##Z128rrkz: \
2177 case X86::VPERMI2##Suffix##Z256rrkz: \
2178 case X86::VPERMT2##Suffix##Z256rrkz: \
2179 case X86::VPERMI2##Suffix##Zrrkz: \
2180 case X86::VPERMT2##Suffix##Zrrkz: \
2181 case X86::VPERMI2##Suffix##Z128rmkz: \
2182 case X86::VPERMT2##Suffix##Z128rmkz: \
2183 case X86::VPERMI2##Suffix##Z256rmkz: \
2184 case X86::VPERMT2##Suffix##Z256rmkz: \
2185 case X86::VPERMI2##Suffix##Zrmkz: \
2186 case X86::VPERMT2##Suffix##Zrmkz:
2187
2188#define VPERM_CASES_BROADCAST(Suffix) \
2189 VPERM_CASES(Suffix) \
2190 case X86::VPERMI2##Suffix##Z128rmb: \
2191 case X86::VPERMT2##Suffix##Z128rmb: \
2192 case X86::VPERMI2##Suffix##Z256rmb: \
2193 case X86::VPERMT2##Suffix##Z256rmb: \
2194 case X86::VPERMI2##Suffix##Zrmb: \
2195 case X86::VPERMT2##Suffix##Zrmb: \
2196 case X86::VPERMI2##Suffix##Z128rmbkz: \
2197 case X86::VPERMT2##Suffix##Z128rmbkz: \
2198 case X86::VPERMI2##Suffix##Z256rmbkz: \
2199 case X86::VPERMT2##Suffix##Z256rmbkz: \
2200 case X86::VPERMI2##Suffix##Zrmbkz: \
2201 case X86::VPERMT2##Suffix##Zrmbkz:
2202
2203 switch (Opcode) {
2204 default:
2205 return false;
2206 VPERM_CASES(B)
2211 VPERM_CASES(W)
2212 return true;
2213 }
2214#undef VPERM_CASES_BROADCAST
2215#undef VPERM_CASES
2216}
2217
2218// Returns commuted opcode for VPERMI2 and VPERMT2 instructions by switching
2219// from the I opcode to the T opcode and vice versa.
2220static unsigned getCommutedVPERMV3Opcode(unsigned Opcode) {
2221#define VPERM_CASES(Orig, New) \
2222 case X86::Orig##Z128rr: \
2223 return X86::New##Z128rr; \
2224 case X86::Orig##Z128rrkz: \
2225 return X86::New##Z128rrkz; \
2226 case X86::Orig##Z128rm: \
2227 return X86::New##Z128rm; \
2228 case X86::Orig##Z128rmkz: \
2229 return X86::New##Z128rmkz; \
2230 case X86::Orig##Z256rr: \
2231 return X86::New##Z256rr; \
2232 case X86::Orig##Z256rrkz: \
2233 return X86::New##Z256rrkz; \
2234 case X86::Orig##Z256rm: \
2235 return X86::New##Z256rm; \
2236 case X86::Orig##Z256rmkz: \
2237 return X86::New##Z256rmkz; \
2238 case X86::Orig##Zrr: \
2239 return X86::New##Zrr; \
2240 case X86::Orig##Zrrkz: \
2241 return X86::New##Zrrkz; \
2242 case X86::Orig##Zrm: \
2243 return X86::New##Zrm; \
2244 case X86::Orig##Zrmkz: \
2245 return X86::New##Zrmkz;
2246
2247#define VPERM_CASES_BROADCAST(Orig, New) \
2248 VPERM_CASES(Orig, New) \
2249 case X86::Orig##Z128rmb: \
2250 return X86::New##Z128rmb; \
2251 case X86::Orig##Z128rmbkz: \
2252 return X86::New##Z128rmbkz; \
2253 case X86::Orig##Z256rmb: \
2254 return X86::New##Z256rmb; \
2255 case X86::Orig##Z256rmbkz: \
2256 return X86::New##Z256rmbkz; \
2257 case X86::Orig##Zrmb: \
2258 return X86::New##Zrmb; \
2259 case X86::Orig##Zrmbkz: \
2260 return X86::New##Zrmbkz;
2261
2262 switch (Opcode) {
2263 VPERM_CASES(VPERMI2B, VPERMT2B)
2264 VPERM_CASES_BROADCAST(VPERMI2D, VPERMT2D)
2265 VPERM_CASES_BROADCAST(VPERMI2PD, VPERMT2PD)
2266 VPERM_CASES_BROADCAST(VPERMI2PS, VPERMT2PS)
2267 VPERM_CASES_BROADCAST(VPERMI2Q, VPERMT2Q)
2268 VPERM_CASES(VPERMI2W, VPERMT2W)
2269 VPERM_CASES(VPERMT2B, VPERMI2B)
2270 VPERM_CASES_BROADCAST(VPERMT2D, VPERMI2D)
2271 VPERM_CASES_BROADCAST(VPERMT2PD, VPERMI2PD)
2272 VPERM_CASES_BROADCAST(VPERMT2PS, VPERMI2PS)
2273 VPERM_CASES_BROADCAST(VPERMT2Q, VPERMI2Q)
2274 VPERM_CASES(VPERMT2W, VPERMI2W)
2275 }
2276
2277 llvm_unreachable("Unreachable!");
2278#undef VPERM_CASES_BROADCAST
2279#undef VPERM_CASES
2280}
2281
2283 unsigned OpIdx1,
2284 unsigned OpIdx2) const {
2285 auto CloneIfNew = [&](MachineInstr &MI) {
2286 return std::exchange(NewMI, false)
2287 ? MI.getParent()->getParent()->CloneMachineInstr(&MI)
2288 : &MI;
2289 };
2290 MachineInstr *WorkingMI = nullptr;
2291 unsigned Opc = MI.getOpcode();
2292
2293#define CASE_ND(OP) \
2294 case X86::OP: \
2295 case X86::OP##_ND:
2296
2297 switch (Opc) {
2298 // SHLD B, C, I <-> SHRD C, B, (BitWidth - I)
2299 CASE_ND(SHRD16rri8)
2300 CASE_ND(SHLD16rri8)
2301 CASE_ND(SHRD32rri8)
2302 CASE_ND(SHLD32rri8)
2303 CASE_ND(SHRD64rri8)
2304 CASE_ND(SHLD64rri8) {
2305 unsigned Size;
2306 switch (Opc) {
2307 default:
2308 llvm_unreachable("Unreachable!");
2309#define FROM_TO_SIZE(A, B, S) \
2310 case X86::A: \
2311 Opc = X86::B; \
2312 Size = S; \
2313 break; \
2314 case X86::A##_ND: \
2315 Opc = X86::B##_ND; \
2316 Size = S; \
2317 break; \
2318 case X86::B: \
2319 Opc = X86::A; \
2320 Size = S; \
2321 break; \
2322 case X86::B##_ND: \
2323 Opc = X86::A##_ND; \
2324 Size = S; \
2325 break;
2326
2327 FROM_TO_SIZE(SHRD16rri8, SHLD16rri8, 16)
2328 FROM_TO_SIZE(SHRD32rri8, SHLD32rri8, 32)
2329 FROM_TO_SIZE(SHRD64rri8, SHLD64rri8, 64)
2330#undef FROM_TO_SIZE
2331 }
2332 WorkingMI = CloneIfNew(MI);
2333 WorkingMI->setDesc(get(Opc));
2334 WorkingMI->getOperand(3).setImm(Size - MI.getOperand(3).getImm());
2335 break;
2336 }
2337 case X86::PFSUBrr:
2338 case X86::PFSUBRrr:
2339 // PFSUB x, y: x = x - y
2340 // PFSUBR x, y: x = y - x
2341 WorkingMI = CloneIfNew(MI);
2342 WorkingMI->setDesc(
2343 get(X86::PFSUBRrr == Opc ? X86::PFSUBrr : X86::PFSUBRrr));
2344 break;
2345 case X86::BLENDPDrri:
2346 case X86::BLENDPSrri:
2347 case X86::VBLENDPDrri:
2348 case X86::VBLENDPSrri:
2349 // If we're optimizing for size, try to use MOVSD/MOVSS.
2350 if (MI.getParent()->getParent()->getFunction().hasOptSize()) {
2351 unsigned Mask = (Opc == X86::BLENDPDrri || Opc == X86::VBLENDPDrri) ? 0x03: 0x0F;
2352 if ((MI.getOperand(3).getImm() ^ Mask) == 1) {
2353#define FROM_TO(FROM, TO) \
2354 case X86::FROM: \
2355 Opc = X86::TO; \
2356 break;
2357 switch (Opc) {
2358 default:
2359 llvm_unreachable("Unreachable!");
2360 FROM_TO(BLENDPDrri, MOVSDrr)
2361 FROM_TO(BLENDPSrri, MOVSSrr)
2362 FROM_TO(VBLENDPDrri, VMOVSDrr)
2363 FROM_TO(VBLENDPSrri, VMOVSSrr)
2364 }
2365 WorkingMI = CloneIfNew(MI);
2366 WorkingMI->setDesc(get(Opc));
2367 WorkingMI->removeOperand(3);
2368 break;
2369 }
2370#undef FROM_TO
2371 }
2372 [[fallthrough]];
2373 case X86::PBLENDWrri:
2374 case X86::VBLENDPDYrri:
2375 case X86::VBLENDPSYrri:
2376 case X86::VPBLENDDrri:
2377 case X86::VPBLENDWrri:
2378 case X86::VPBLENDDYrri:
2379 case X86::VPBLENDWYrri: {
2380 int8_t Mask;
2381 switch (Opc) {
2382 default:
2383 llvm_unreachable("Unreachable!");
2384 case X86::BLENDPDrri:
2385 Mask = (int8_t)0x03;
2386 break;
2387 case X86::BLENDPSrri:
2388 Mask = (int8_t)0x0F;
2389 break;
2390 case X86::PBLENDWrri:
2391 Mask = (int8_t)0xFF;
2392 break;
2393 case X86::VBLENDPDrri:
2394 Mask = (int8_t)0x03;
2395 break;
2396 case X86::VBLENDPSrri:
2397 Mask = (int8_t)0x0F;
2398 break;
2399 case X86::VBLENDPDYrri:
2400 Mask = (int8_t)0x0F;
2401 break;
2402 case X86::VBLENDPSYrri:
2403 Mask = (int8_t)0xFF;
2404 break;
2405 case X86::VPBLENDDrri:
2406 Mask = (int8_t)0x0F;
2407 break;
2408 case X86::VPBLENDWrri:
2409 Mask = (int8_t)0xFF;
2410 break;
2411 case X86::VPBLENDDYrri:
2412 Mask = (int8_t)0xFF;
2413 break;
2414 case X86::VPBLENDWYrri:
2415 Mask = (int8_t)0xFF;
2416 break;
2417 }
2418 // Only the least significant bits of Imm are used.
2419 // Using int8_t to ensure it will be sign extended to the int64_t that
2420 // setImm takes in order to match isel behavior.
2421 int8_t Imm = MI.getOperand(3).getImm() & Mask;
2422 WorkingMI = CloneIfNew(MI);
2423 WorkingMI->getOperand(3).setImm(Mask ^ Imm);
2424 break;
2425 }
2426 case X86::INSERTPSrr:
2427 case X86::VINSERTPSrr:
2428 case X86::VINSERTPSZrr: {
2429 unsigned Imm = MI.getOperand(MI.getNumOperands() - 1).getImm();
2430 unsigned ZMask = Imm & 15;
2431 unsigned DstIdx = (Imm >> 4) & 3;
2432 unsigned SrcIdx = (Imm >> 6) & 3;
2433
2434 // We can commute insertps if we zero 2 of the elements, the insertion is
2435 // "inline" and we don't override the insertion with a zero.
2436 if (DstIdx == SrcIdx && (ZMask & (1 << DstIdx)) == 0 &&
2437 llvm::popcount(ZMask) == 2) {
2438 unsigned AltIdx = llvm::countr_zero((ZMask | (1 << DstIdx)) ^ 15);
2439 assert(AltIdx < 4 && "Illegal insertion index");
2440 unsigned AltImm = (AltIdx << 6) | (AltIdx << 4) | ZMask;
2441 WorkingMI = CloneIfNew(MI);
2442 WorkingMI->getOperand(MI.getNumOperands() - 1).setImm(AltImm);
2443 break;
2444 }
2445 return nullptr;
2446 }
2447 case X86::MOVSDrr:
2448 case X86::MOVSSrr:
2449 case X86::VMOVSDrr:
2450 case X86::VMOVSSrr: {
2451 // On SSE41 or later we can commute a MOVSS/MOVSD to a BLENDPS/BLENDPD.
2452 if (Subtarget.hasSSE41()) {
2453 unsigned Mask;
2454 switch (Opc) {
2455 default:
2456 llvm_unreachable("Unreachable!");
2457 case X86::MOVSDrr:
2458 Opc = X86::BLENDPDrri;
2459 Mask = 0x02;
2460 break;
2461 case X86::MOVSSrr:
2462 Opc = X86::BLENDPSrri;
2463 Mask = 0x0E;
2464 break;
2465 case X86::VMOVSDrr:
2466 Opc = X86::VBLENDPDrri;
2467 Mask = 0x02;
2468 break;
2469 case X86::VMOVSSrr:
2470 Opc = X86::VBLENDPSrri;
2471 Mask = 0x0E;
2472 break;
2473 }
2474
2475 WorkingMI = CloneIfNew(MI);
2476 WorkingMI->setDesc(get(Opc));
2477 WorkingMI->addOperand(MachineOperand::CreateImm(Mask));
2478 break;
2479 }
2480
2481 WorkingMI = CloneIfNew(MI);
2482 WorkingMI->setDesc(get(X86::SHUFPDrri));
2483 WorkingMI->addOperand(MachineOperand::CreateImm(0x02));
2484 break;
2485 }
2486 case X86::SHUFPDrri: {
2487 // Commute to MOVSD.
2488 assert(MI.getOperand(3).getImm() == 0x02 && "Unexpected immediate!");
2489 WorkingMI = CloneIfNew(MI);
2490 WorkingMI->setDesc(get(X86::MOVSDrr));
2491 WorkingMI->removeOperand(3);
2492 break;
2493 }
2494 case X86::PCLMULQDQrri:
2495 case X86::VPCLMULQDQrri:
2496 case X86::VPCLMULQDQYrri:
2497 case X86::VPCLMULQDQZrri:
2498 case X86::VPCLMULQDQZ128rri:
2499 case X86::VPCLMULQDQZ256rri: {
2500 // SRC1 64bits = Imm[0] ? SRC1[127:64] : SRC1[63:0]
2501 // SRC2 64bits = Imm[4] ? SRC2[127:64] : SRC2[63:0]
2502 unsigned Imm = MI.getOperand(3).getImm();
2503 unsigned Src1Hi = Imm & 0x01;
2504 unsigned Src2Hi = Imm & 0x10;
2505 WorkingMI = CloneIfNew(MI);
2506 WorkingMI->getOperand(3).setImm((Src1Hi << 4) | (Src2Hi >> 4));
2507 break;
2508 }
2509 case X86::VPCMPBZ128rri:
2510 case X86::VPCMPUBZ128rri:
2511 case X86::VPCMPBZ256rri:
2512 case X86::VPCMPUBZ256rri:
2513 case X86::VPCMPBZrri:
2514 case X86::VPCMPUBZrri:
2515 case X86::VPCMPDZ128rri:
2516 case X86::VPCMPUDZ128rri:
2517 case X86::VPCMPDZ256rri:
2518 case X86::VPCMPUDZ256rri:
2519 case X86::VPCMPDZrri:
2520 case X86::VPCMPUDZrri:
2521 case X86::VPCMPQZ128rri:
2522 case X86::VPCMPUQZ128rri:
2523 case X86::VPCMPQZ256rri:
2524 case X86::VPCMPUQZ256rri:
2525 case X86::VPCMPQZrri:
2526 case X86::VPCMPUQZrri:
2527 case X86::VPCMPWZ128rri:
2528 case X86::VPCMPUWZ128rri:
2529 case X86::VPCMPWZ256rri:
2530 case X86::VPCMPUWZ256rri:
2531 case X86::VPCMPWZrri:
2532 case X86::VPCMPUWZrri:
2533 case X86::VPCMPBZ128rrik:
2534 case X86::VPCMPUBZ128rrik:
2535 case X86::VPCMPBZ256rrik:
2536 case X86::VPCMPUBZ256rrik:
2537 case X86::VPCMPBZrrik:
2538 case X86::VPCMPUBZrrik:
2539 case X86::VPCMPDZ128rrik:
2540 case X86::VPCMPUDZ128rrik:
2541 case X86::VPCMPDZ256rrik:
2542 case X86::VPCMPUDZ256rrik:
2543 case X86::VPCMPDZrrik:
2544 case X86::VPCMPUDZrrik:
2545 case X86::VPCMPQZ128rrik:
2546 case X86::VPCMPUQZ128rrik:
2547 case X86::VPCMPQZ256rrik:
2548 case X86::VPCMPUQZ256rrik:
2549 case X86::VPCMPQZrrik:
2550 case X86::VPCMPUQZrrik:
2551 case X86::VPCMPWZ128rrik:
2552 case X86::VPCMPUWZ128rrik:
2553 case X86::VPCMPWZ256rrik:
2554 case X86::VPCMPUWZ256rrik:
2555 case X86::VPCMPWZrrik:
2556 case X86::VPCMPUWZrrik:
2557 WorkingMI = CloneIfNew(MI);
2558 // Flip comparison mode immediate (if necessary).
2559 WorkingMI->getOperand(MI.getNumOperands() - 1)
2561 MI.getOperand(MI.getNumOperands() - 1).getImm() & 0x7));
2562 break;
2563 case X86::VPCOMBri:
2564 case X86::VPCOMUBri:
2565 case X86::VPCOMDri:
2566 case X86::VPCOMUDri:
2567 case X86::VPCOMQri:
2568 case X86::VPCOMUQri:
2569 case X86::VPCOMWri:
2570 case X86::VPCOMUWri:
2571 WorkingMI = CloneIfNew(MI);
2572 // Flip comparison mode immediate (if necessary).
2573 WorkingMI->getOperand(3).setImm(
2574 X86::getSwappedVPCOMImm(MI.getOperand(3).getImm() & 0x7));
2575 break;
2576 case X86::VCMPSDZrri:
2577 case X86::VCMPSSZrri:
2578 case X86::VCMPPDZrri:
2579 case X86::VCMPPSZrri:
2580 case X86::VCMPSHZrri:
2581 case X86::VCMPPHZrri:
2582 case X86::VCMPPHZ128rri:
2583 case X86::VCMPPHZ256rri:
2584 case X86::VCMPPDZ128rri:
2585 case X86::VCMPPSZ128rri:
2586 case X86::VCMPPDZ256rri:
2587 case X86::VCMPPSZ256rri:
2588 case X86::VCMPPDZrrik:
2589 case X86::VCMPPSZrrik:
2590 case X86::VCMPPDZ128rrik:
2591 case X86::VCMPPSZ128rrik:
2592 case X86::VCMPPDZ256rrik:
2593 case X86::VCMPPSZ256rrik:
2594 WorkingMI = CloneIfNew(MI);
2595 WorkingMI->getOperand(MI.getNumExplicitOperands() - 1)
2597 MI.getOperand(MI.getNumExplicitOperands() - 1).getImm() & 0x1f));
2598 break;
2599 case X86::VPERM2F128rr:
2600 case X86::VPERM2I128rr:
2601 // Flip permute source immediate.
2602 // Imm & 0x02: lo = if set, select Op1.lo/hi else Op0.lo/hi.
2603 // Imm & 0x20: hi = if set, select Op1.lo/hi else Op0.lo/hi.
2604 WorkingMI = CloneIfNew(MI);
2605 WorkingMI->getOperand(3).setImm((MI.getOperand(3).getImm() & 0xFF) ^ 0x22);
2606 break;
2607 case X86::MOVHLPSrr:
2608 case X86::UNPCKHPDrr:
2609 case X86::VMOVHLPSrr:
2610 case X86::VUNPCKHPDrr:
2611 case X86::VMOVHLPSZrr:
2612 case X86::VUNPCKHPDZ128rr:
2613 assert(Subtarget.hasSSE2() && "Commuting MOVHLP/UNPCKHPD requires SSE2!");
2614
2615 switch (Opc) {
2616 default:
2617 llvm_unreachable("Unreachable!");
2618 case X86::MOVHLPSrr:
2619 Opc = X86::UNPCKHPDrr;
2620 break;
2621 case X86::UNPCKHPDrr:
2622 Opc = X86::MOVHLPSrr;
2623 break;
2624 case X86::VMOVHLPSrr:
2625 Opc = X86::VUNPCKHPDrr;
2626 break;
2627 case X86::VUNPCKHPDrr:
2628 Opc = X86::VMOVHLPSrr;
2629 break;
2630 case X86::VMOVHLPSZrr:
2631 Opc = X86::VUNPCKHPDZ128rr;
2632 break;
2633 case X86::VUNPCKHPDZ128rr:
2634 Opc = X86::VMOVHLPSZrr;
2635 break;
2636 }
2637 WorkingMI = CloneIfNew(MI);
2638 WorkingMI->setDesc(get(Opc));
2639 break;
2640 CASE_ND(CMOV16rr)
2641 CASE_ND(CMOV32rr)
2642 CASE_ND(CMOV64rr) {
2643 WorkingMI = CloneIfNew(MI);
2644 unsigned OpNo = MI.getDesc().getNumOperands() - 1;
2645 X86::CondCode CC = static_cast<X86::CondCode>(MI.getOperand(OpNo).getImm());
2647 break;
2648 }
2649 case X86::VPTERNLOGDZrri:
2650 case X86::VPTERNLOGDZrmi:
2651 case X86::VPTERNLOGDZ128rri:
2652 case X86::VPTERNLOGDZ128rmi:
2653 case X86::VPTERNLOGDZ256rri:
2654 case X86::VPTERNLOGDZ256rmi:
2655 case X86::VPTERNLOGQZrri:
2656 case X86::VPTERNLOGQZrmi:
2657 case X86::VPTERNLOGQZ128rri:
2658 case X86::VPTERNLOGQZ128rmi:
2659 case X86::VPTERNLOGQZ256rri:
2660 case X86::VPTERNLOGQZ256rmi:
2661 case X86::VPTERNLOGDZrrik:
2662 case X86::VPTERNLOGDZ128rrik:
2663 case X86::VPTERNLOGDZ256rrik:
2664 case X86::VPTERNLOGQZrrik:
2665 case X86::VPTERNLOGQZ128rrik:
2666 case X86::VPTERNLOGQZ256rrik:
2667 case X86::VPTERNLOGDZrrikz:
2668 case X86::VPTERNLOGDZrmikz:
2669 case X86::VPTERNLOGDZ128rrikz:
2670 case X86::VPTERNLOGDZ128rmikz:
2671 case X86::VPTERNLOGDZ256rrikz:
2672 case X86::VPTERNLOGDZ256rmikz:
2673 case X86::VPTERNLOGQZrrikz:
2674 case X86::VPTERNLOGQZrmikz:
2675 case X86::VPTERNLOGQZ128rrikz:
2676 case X86::VPTERNLOGQZ128rmikz:
2677 case X86::VPTERNLOGQZ256rrikz:
2678 case X86::VPTERNLOGQZ256rmikz:
2679 case X86::VPTERNLOGDZ128rmbi:
2680 case X86::VPTERNLOGDZ256rmbi:
2681 case X86::VPTERNLOGDZrmbi:
2682 case X86::VPTERNLOGQZ128rmbi:
2683 case X86::VPTERNLOGQZ256rmbi:
2684 case X86::VPTERNLOGQZrmbi:
2685 case X86::VPTERNLOGDZ128rmbikz:
2686 case X86::VPTERNLOGDZ256rmbikz:
2687 case X86::VPTERNLOGDZrmbikz:
2688 case X86::VPTERNLOGQZ128rmbikz:
2689 case X86::VPTERNLOGQZ256rmbikz:
2690 case X86::VPTERNLOGQZrmbikz: {
2691 WorkingMI = CloneIfNew(MI);
2692 commuteVPTERNLOG(*WorkingMI, OpIdx1, OpIdx2);
2693 break;
2694 }
2695 default:
2697 WorkingMI = CloneIfNew(MI);
2698 WorkingMI->setDesc(get(getCommutedVPERMV3Opcode(Opc)));
2699 break;
2700 }
2701
2702 if (auto *FMA3Group = getFMA3Group(Opc, MI.getDesc().TSFlags)) {
2703 WorkingMI = CloneIfNew(MI);
2704 WorkingMI->setDesc(
2705 get(getFMA3OpcodeToCommuteOperands(MI, OpIdx1, OpIdx2, *FMA3Group)));
2706 break;
2707 }
2708 }
2709 return TargetInstrInfo::commuteInstructionImpl(MI, NewMI, OpIdx1, OpIdx2);
2710}
2711
2712bool X86InstrInfo::findThreeSrcCommutedOpIndices(const MachineInstr &MI,
2713 unsigned &SrcOpIdx1,
2714 unsigned &SrcOpIdx2,
2715 bool IsIntrinsic) const {
2716 uint64_t TSFlags = MI.getDesc().TSFlags;
2717
2718 unsigned FirstCommutableVecOp = 1;
2719 unsigned LastCommutableVecOp = 3;
2720 unsigned KMaskOp = -1U;
2721 if (X86II::isKMasked(TSFlags)) {
2722 // For k-zero-masked operations it is Ok to commute the first vector
2723 // operand. Unless this is an intrinsic instruction.
2724 // For regular k-masked operations a conservative choice is done as the
2725 // elements of the first vector operand, for which the corresponding bit
2726 // in the k-mask operand is set to 0, are copied to the result of the
2727 // instruction.
2728 // TODO/FIXME: The commute still may be legal if it is known that the
2729 // k-mask operand is set to either all ones or all zeroes.
2730 // It is also Ok to commute the 1st operand if all users of MI use only
2731 // the elements enabled by the k-mask operand. For example,
2732 // v4 = VFMADD213PSZrk v1, k, v2, v3; // v1[i] = k[i] ? v2[i]*v1[i]+v3[i]
2733 // : v1[i];
2734 // VMOVAPSZmrk <mem_addr>, k, v4; // this is the ONLY user of v4 ->
2735 // // Ok, to commute v1 in FMADD213PSZrk.
2736
2737 // The k-mask operand has index = 2 for masked and zero-masked operations.
2738 KMaskOp = 2;
2739
2740 // The operand with index = 1 is used as a source for those elements for
2741 // which the corresponding bit in the k-mask is set to 0.
2742 if (X86II::isKMergeMasked(TSFlags) || IsIntrinsic)
2743 FirstCommutableVecOp = 3;
2744
2745 LastCommutableVecOp++;
2746 } else if (IsIntrinsic) {
2747 // Commuting the first operand of an intrinsic instruction isn't possible
2748 // unless we can prove that only the lowest element of the result is used.
2749 FirstCommutableVecOp = 2;
2750 }
2751
2752 if (isMem(MI, LastCommutableVecOp))
2753 LastCommutableVecOp--;
2754
2755 // Only the first RegOpsNum operands are commutable.
2756 // Also, the value 'CommuteAnyOperandIndex' is valid here as it means
2757 // that the operand is not specified/fixed.
2758 if (SrcOpIdx1 != CommuteAnyOperandIndex &&
2759 (SrcOpIdx1 < FirstCommutableVecOp || SrcOpIdx1 > LastCommutableVecOp ||
2760 SrcOpIdx1 == KMaskOp))
2761 return false;
2762 if (SrcOpIdx2 != CommuteAnyOperandIndex &&
2763 (SrcOpIdx2 < FirstCommutableVecOp || SrcOpIdx2 > LastCommutableVecOp ||
2764 SrcOpIdx2 == KMaskOp))
2765 return false;
2766
2767 // Look for two different register operands assumed to be commutable
2768 // regardless of the FMA opcode. The FMA opcode is adjusted later.
2769 if (SrcOpIdx1 == CommuteAnyOperandIndex ||
2770 SrcOpIdx2 == CommuteAnyOperandIndex) {
2771 unsigned CommutableOpIdx2 = SrcOpIdx2;
2772
2773 // At least one of operands to be commuted is not specified and
2774 // this method is free to choose appropriate commutable operands.
2775 if (SrcOpIdx1 == SrcOpIdx2)
2776 // Both of operands are not fixed. By default set one of commutable
2777 // operands to the last register operand of the instruction.
2778 CommutableOpIdx2 = LastCommutableVecOp;
2779 else if (SrcOpIdx2 == CommuteAnyOperandIndex)
2780 // Only one of operands is not fixed.
2781 CommutableOpIdx2 = SrcOpIdx1;
2782
2783 // CommutableOpIdx2 is well defined now. Let's choose another commutable
2784 // operand and assign its index to CommutableOpIdx1.
2785 Register Op2Reg = MI.getOperand(CommutableOpIdx2).getReg();
2786
2787 unsigned CommutableOpIdx1;
2788 for (CommutableOpIdx1 = LastCommutableVecOp;
2789 CommutableOpIdx1 >= FirstCommutableVecOp; CommutableOpIdx1--) {
2790 // Just ignore and skip the k-mask operand.
2791 if (CommutableOpIdx1 == KMaskOp)
2792 continue;
2793
2794 // The commuted operands must have different registers.
2795 // Otherwise, the commute transformation does not change anything and
2796 // is useless then.
2797 if (Op2Reg != MI.getOperand(CommutableOpIdx1).getReg())
2798 break;
2799 }
2800
2801 // No appropriate commutable operands were found.
2802 if (CommutableOpIdx1 < FirstCommutableVecOp)
2803 return false;
2804
2805 // Assign the found pair of commutable indices to SrcOpIdx1 and SrcOpidx2
2806 // to return those values.
2807 if (!fixCommutedOpIndices(SrcOpIdx1, SrcOpIdx2, CommutableOpIdx1,
2808 CommutableOpIdx2))
2809 return false;
2810 }
2811
2812 return true;
2813}
2814
2816 unsigned &SrcOpIdx1,
2817 unsigned &SrcOpIdx2) const {
2818 const MCInstrDesc &Desc = MI.getDesc();
2819 if (!Desc.isCommutable())
2820 return false;
2821
2822 switch (MI.getOpcode()) {
2823 case X86::CMPSDrri:
2824 case X86::CMPSSrri:
2825 case X86::CMPPDrri:
2826 case X86::CMPPSrri:
2827 case X86::VCMPSDrri:
2828 case X86::VCMPSSrri:
2829 case X86::VCMPPDrri:
2830 case X86::VCMPPSrri:
2831 case X86::VCMPPDYrri:
2832 case X86::VCMPPSYrri:
2833 case X86::VCMPSDZrri:
2834 case X86::VCMPSSZrri:
2835 case X86::VCMPPDZrri:
2836 case X86::VCMPPSZrri:
2837 case X86::VCMPSHZrri:
2838 case X86::VCMPPHZrri:
2839 case X86::VCMPPHZ128rri:
2840 case X86::VCMPPHZ256rri:
2841 case X86::VCMPPDZ128rri:
2842 case X86::VCMPPSZ128rri:
2843 case X86::VCMPPDZ256rri:
2844 case X86::VCMPPSZ256rri:
2845 case X86::VCMPPDZrrik:
2846 case X86::VCMPPSZrrik:
2847 case X86::VCMPPDZ128rrik:
2848 case X86::VCMPPSZ128rrik:
2849 case X86::VCMPPDZ256rrik:
2850 case X86::VCMPPSZ256rrik: {
2851 unsigned OpOffset = X86II::isKMasked(Desc.TSFlags) ? 1 : 0;
2852
2853 // Float comparison can be safely commuted for
2854 // Ordered/Unordered/Equal/NotEqual tests
2855 unsigned Imm = MI.getOperand(3 + OpOffset).getImm() & 0x7;
2856 switch (Imm) {
2857 default:
2858 // EVEX versions can be commuted.
2859 if ((Desc.TSFlags & X86II::EncodingMask) == X86II::EVEX)
2860 break;
2861 return false;
2862 case 0x00: // EQUAL
2863 case 0x03: // UNORDERED
2864 case 0x04: // NOT EQUAL
2865 case 0x07: // ORDERED
2866 break;
2867 }
2868
2869 // The indices of the commutable operands are 1 and 2 (or 2 and 3
2870 // when masked).
2871 // Assign them to the returned operand indices here.
2872 return fixCommutedOpIndices(SrcOpIdx1, SrcOpIdx2, 1 + OpOffset,
2873 2 + OpOffset);
2874 }
2875 case X86::MOVSSrr:
2876 // X86::MOVSDrr is always commutable. MOVSS is only commutable if we can
2877 // form sse4.1 blend. We assume VMOVSSrr/VMOVSDrr is always commutable since
2878 // AVX implies sse4.1.
2879 if (Subtarget.hasSSE41())
2880 return TargetInstrInfo::findCommutedOpIndices(MI, SrcOpIdx1, SrcOpIdx2);
2881 return false;
2882 case X86::SHUFPDrri:
2883 // We can commute this to MOVSD.
2884 if (MI.getOperand(3).getImm() == 0x02)
2885 return TargetInstrInfo::findCommutedOpIndices(MI, SrcOpIdx1, SrcOpIdx2);
2886 return false;
2887 case X86::MOVHLPSrr:
2888 case X86::UNPCKHPDrr:
2889 case X86::VMOVHLPSrr:
2890 case X86::VUNPCKHPDrr:
2891 case X86::VMOVHLPSZrr:
2892 case X86::VUNPCKHPDZ128rr:
2893 if (Subtarget.hasSSE2())
2894 return TargetInstrInfo::findCommutedOpIndices(MI, SrcOpIdx1, SrcOpIdx2);
2895 return false;
2896 case X86::VPTERNLOGDZrri:
2897 case X86::VPTERNLOGDZrmi:
2898 case X86::VPTERNLOGDZ128rri:
2899 case X86::VPTERNLOGDZ128rmi:
2900 case X86::VPTERNLOGDZ256rri:
2901 case X86::VPTERNLOGDZ256rmi:
2902 case X86::VPTERNLOGQZrri:
2903 case X86::VPTERNLOGQZrmi:
2904 case X86::VPTERNLOGQZ128rri:
2905 case X86::VPTERNLOGQZ128rmi:
2906 case X86::VPTERNLOGQZ256rri:
2907 case X86::VPTERNLOGQZ256rmi:
2908 case X86::VPTERNLOGDZrrik:
2909 case X86::VPTERNLOGDZ128rrik:
2910 case X86::VPTERNLOGDZ256rrik:
2911 case X86::VPTERNLOGQZrrik:
2912 case X86::VPTERNLOGQZ128rrik:
2913 case X86::VPTERNLOGQZ256rrik:
2914 case X86::VPTERNLOGDZrrikz:
2915 case X86::VPTERNLOGDZrmikz:
2916 case X86::VPTERNLOGDZ128rrikz:
2917 case X86::VPTERNLOGDZ128rmikz:
2918 case X86::VPTERNLOGDZ256rrikz:
2919 case X86::VPTERNLOGDZ256rmikz:
2920 case X86::VPTERNLOGQZrrikz:
2921 case X86::VPTERNLOGQZrmikz:
2922 case X86::VPTERNLOGQZ128rrikz:
2923 case X86::VPTERNLOGQZ128rmikz:
2924 case X86::VPTERNLOGQZ256rrikz:
2925 case X86::VPTERNLOGQZ256rmikz:
2926 case X86::VPTERNLOGDZ128rmbi:
2927 case X86::VPTERNLOGDZ256rmbi:
2928 case X86::VPTERNLOGDZrmbi:
2929 case X86::VPTERNLOGQZ128rmbi:
2930 case X86::VPTERNLOGQZ256rmbi:
2931 case X86::VPTERNLOGQZrmbi:
2932 case X86::VPTERNLOGDZ128rmbikz:
2933 case X86::VPTERNLOGDZ256rmbikz:
2934 case X86::VPTERNLOGDZrmbikz:
2935 case X86::VPTERNLOGQZ128rmbikz:
2936 case X86::VPTERNLOGQZ256rmbikz:
2937 case X86::VPTERNLOGQZrmbikz:
2938 return findThreeSrcCommutedOpIndices(MI, SrcOpIdx1, SrcOpIdx2);
2939 case X86::VPDPWSSDYrr:
2940 case X86::VPDPWSSDrr:
2941 case X86::VPDPWSSDSYrr:
2942 case X86::VPDPWSSDSrr:
2943 case X86::VPDPWUUDrr:
2944 case X86::VPDPWUUDYrr:
2945 case X86::VPDPWUUDSrr:
2946 case X86::VPDPWUUDSYrr:
2947 case X86::VPDPBSSDSrr:
2948 case X86::VPDPBSSDSYrr:
2949 case X86::VPDPBSSDrr:
2950 case X86::VPDPBSSDYrr:
2951 case X86::VPDPBUUDSrr:
2952 case X86::VPDPBUUDSYrr:
2953 case X86::VPDPBUUDrr:
2954 case X86::VPDPBUUDYrr:
2955 case X86::VPDPWSSDZ128r:
2956 case X86::VPDPWSSDZ128rk:
2957 case X86::VPDPWSSDZ128rkz:
2958 case X86::VPDPWSSDZ256r:
2959 case X86::VPDPWSSDZ256rk:
2960 case X86::VPDPWSSDZ256rkz:
2961 case X86::VPDPWSSDZr:
2962 case X86::VPDPWSSDZrk:
2963 case X86::VPDPWSSDZrkz:
2964 case X86::VPDPWSSDSZ128r:
2965 case X86::VPDPWSSDSZ128rk:
2966 case X86::VPDPWSSDSZ128rkz:
2967 case X86::VPDPWSSDSZ256r:
2968 case X86::VPDPWSSDSZ256rk:
2969 case X86::VPDPWSSDSZ256rkz:
2970 case X86::VPDPWSSDSZr:
2971 case X86::VPDPWSSDSZrk:
2972 case X86::VPDPWSSDSZrkz:
2973 case X86::VPMADD52HUQrr:
2974 case X86::VPMADD52HUQYrr:
2975 case X86::VPMADD52HUQZ128r:
2976 case X86::VPMADD52HUQZ128rk:
2977 case X86::VPMADD52HUQZ128rkz:
2978 case X86::VPMADD52HUQZ256r:
2979 case X86::VPMADD52HUQZ256rk:
2980 case X86::VPMADD52HUQZ256rkz:
2981 case X86::VPMADD52HUQZr:
2982 case X86::VPMADD52HUQZrk:
2983 case X86::VPMADD52HUQZrkz:
2984 case X86::VPMADD52LUQrr:
2985 case X86::VPMADD52LUQYrr:
2986 case X86::VPMADD52LUQZ128r:
2987 case X86::VPMADD52LUQZ128rk:
2988 case X86::VPMADD52LUQZ128rkz:
2989 case X86::VPMADD52LUQZ256r:
2990 case X86::VPMADD52LUQZ256rk:
2991 case X86::VPMADD52LUQZ256rkz:
2992 case X86::VPMADD52LUQZr:
2993 case X86::VPMADD52LUQZrk:
2994 case X86::VPMADD52LUQZrkz:
2995 case X86::VFMADDCPHZr:
2996 case X86::VFMADDCPHZrk:
2997 case X86::VFMADDCPHZrkz:
2998 case X86::VFMADDCPHZ128r:
2999 case X86::VFMADDCPHZ128rk:
3000 case X86::VFMADDCPHZ128rkz:
3001 case X86::VFMADDCPHZ256r:
3002 case X86::VFMADDCPHZ256rk:
3003 case X86::VFMADDCPHZ256rkz:
3004 case X86::VFMADDCSHZr:
3005 case X86::VFMADDCSHZrk:
3006 case X86::VFMADDCSHZrkz: {
3007 unsigned CommutableOpIdx1 = 2;
3008 unsigned CommutableOpIdx2 = 3;
3009 if (X86II::isKMasked(Desc.TSFlags)) {
3010 // Skip the mask register.
3011 ++CommutableOpIdx1;
3012 ++CommutableOpIdx2;
3013 }
3014 if (!fixCommutedOpIndices(SrcOpIdx1, SrcOpIdx2, CommutableOpIdx1,
3015 CommutableOpIdx2))
3016 return false;
3017 if (!MI.getOperand(SrcOpIdx1).isReg() || !MI.getOperand(SrcOpIdx2).isReg())
3018 // No idea.
3019 return false;
3020 return true;
3021 }
3022
3023 default:
3024 const X86InstrFMA3Group *FMA3Group =
3025 getFMA3Group(MI.getOpcode(), MI.getDesc().TSFlags);
3026 if (FMA3Group)
3027 return findThreeSrcCommutedOpIndices(MI, SrcOpIdx1, SrcOpIdx2,
3028 FMA3Group->isIntrinsic());
3029
3030 // Handled masked instructions since we need to skip over the mask input
3031 // and the preserved input.
3032 if (X86II::isKMasked(Desc.TSFlags)) {
3033 // First assume that the first input is the mask operand and skip past it.
3034 unsigned CommutableOpIdx1 = Desc.getNumDefs() + 1;
3035 unsigned CommutableOpIdx2 = Desc.getNumDefs() + 2;
3036 // Check if the first input is tied. If there isn't one then we only
3037 // need to skip the mask operand which we did above.
3038 if ((MI.getDesc().getOperandConstraint(Desc.getNumDefs(),
3039 MCOI::TIED_TO) != -1)) {
3040 // If this is zero masking instruction with a tied operand, we need to
3041 // move the first index back to the first input since this must
3042 // be a 3 input instruction and we want the first two non-mask inputs.
3043 // Otherwise this is a 2 input instruction with a preserved input and
3044 // mask, so we need to move the indices to skip one more input.
3045 if (X86II::isKMergeMasked(Desc.TSFlags)) {
3046 ++CommutableOpIdx1;
3047 ++CommutableOpIdx2;
3048 } else {
3049 --CommutableOpIdx1;
3050 }
3051 }
3052
3053 if (!fixCommutedOpIndices(SrcOpIdx1, SrcOpIdx2, CommutableOpIdx1,
3054 CommutableOpIdx2))
3055 return false;
3056
3057 if (!MI.getOperand(SrcOpIdx1).isReg() ||
3058 !MI.getOperand(SrcOpIdx2).isReg())
3059 // No idea.
3060 return false;
3061 return true;
3062 }
3063
3064 return TargetInstrInfo::findCommutedOpIndices(MI, SrcOpIdx1, SrcOpIdx2);
3065 }
3066 return false;
3067}
3068
3070 unsigned Opcode = MI->getOpcode();
3071 if (Opcode != X86::LEA32r && Opcode != X86::LEA64r &&
3072 Opcode != X86::LEA64_32r)
3073 return false;
3074
3075 const MachineOperand &Scale = MI->getOperand(1 + X86::AddrScaleAmt);
3076 const MachineOperand &Disp = MI->getOperand(1 + X86::AddrDisp);
3077 const MachineOperand &Segment = MI->getOperand(1 + X86::AddrSegmentReg);
3078
3079 if (Segment.getReg() != 0 || !Disp.isImm() || Disp.getImm() != 0 ||
3080 Scale.getImm() > 1)
3081 return false;
3082
3083 return true;
3084}
3085
3087 // Currently we're interested in following sequence only.
3088 // r3 = lea r1, r2
3089 // r5 = add r3, r4
3090 // Both r3 and r4 are killed in add, we hope the add instruction has the
3091 // operand order
3092 // r5 = add r4, r3
3093 // So later in X86FixupLEAs the lea instruction can be rewritten as add.
3094 unsigned Opcode = MI.getOpcode();
3095 if (Opcode != X86::ADD32rr && Opcode != X86::ADD64rr)
3096 return false;
3097
3098 const MachineRegisterInfo &MRI = MI.getParent()->getParent()->getRegInfo();
3099 Register Reg1 = MI.getOperand(1).getReg();
3100 Register Reg2 = MI.getOperand(2).getReg();
3101
3102 // Check if Reg1 comes from LEA in the same MBB.
3103 if (MachineInstr *Inst = MRI.getUniqueVRegDef(Reg1)) {
3104 if (isConvertibleLEA(Inst) && Inst->getParent() == MI.getParent()) {
3105 Commute = true;
3106 return true;
3107 }
3108 }
3109
3110 // Check if Reg2 comes from LEA in the same MBB.
3111 if (MachineInstr *Inst = MRI.getUniqueVRegDef(Reg2)) {
3112 if (isConvertibleLEA(Inst) && Inst->getParent() == MI.getParent()) {
3113 Commute = false;
3114 return true;
3115 }
3116 }
3117
3118 return false;
3119}
3120
3122 unsigned Opcode = MCID.getOpcode();
3123 if (!(X86::isJCC(Opcode) || X86::isSETCC(Opcode) || X86::isCMOVCC(Opcode) ||
3124 X86::isCFCMOVCC(Opcode)))
3125 return -1;
3126 // Assume that condition code is always the last use operand.
3127 unsigned NumUses = MCID.getNumOperands() - MCID.getNumDefs();
3128 return NumUses - 1;
3129}
3130
3132 const MCInstrDesc &MCID = MI.getDesc();
3133 int CondNo = getCondSrcNoFromDesc(MCID);
3134 if (CondNo < 0)
3135 return X86::COND_INVALID;
3136 CondNo += MCID.getNumDefs();
3137 return static_cast<X86::CondCode>(MI.getOperand(CondNo).getImm());
3138}
3139
3141 return X86::isJCC(MI.getOpcode()) ? X86::getCondFromMI(MI)
3143}
3144
3146 return X86::isSETCC(MI.getOpcode()) ? X86::getCondFromMI(MI)
3148}
3149
3151 return X86::isCMOVCC(MI.getOpcode()) ? X86::getCondFromMI(MI)
3153}
3154
3156 return X86::isCFCMOVCC(MI.getOpcode()) ? X86::getCondFromMI(MI)
3158}
3159
3160/// Return the inverse of the specified condition,
3161/// e.g. turning COND_E to COND_NE.
3163 switch (CC) {
3164 default:
3165 llvm_unreachable("Illegal condition code!");
3166 case X86::COND_E:
3167 return X86::COND_NE;
3168 case X86::COND_NE:
3169 return X86::COND_E;
3170 case X86::COND_L:
3171 return X86::COND_GE;
3172 case X86::COND_LE:
3173 return X86::COND_G;
3174 case X86::COND_G:
3175 return X86::COND_LE;
3176 case X86::COND_GE:
3177 return X86::COND_L;
3178 case X86::COND_B:
3179 return X86::COND_AE;
3180 case X86::COND_BE:
3181 return X86::COND_A;
3182 case X86::COND_A:
3183 return X86::COND_BE;
3184 case X86::COND_AE:
3185 return X86::COND_B;
3186 case X86::COND_S:
3187 return X86::COND_NS;
3188 case X86::COND_NS:
3189 return X86::COND_S;
3190 case X86::COND_P:
3191 return X86::COND_NP;
3192 case X86::COND_NP:
3193 return X86::COND_P;
3194 case X86::COND_O:
3195 return X86::COND_NO;
3196 case X86::COND_NO:
3197 return X86::COND_O;
3198 case X86::COND_NE_OR_P:
3199 return X86::COND_E_AND_NP;
3200 case X86::COND_E_AND_NP:
3201 return X86::COND_NE_OR_P;
3202 }
3203}
3204
3205/// Assuming the flags are set by MI(a,b), return the condition code if we
3206/// modify the instructions such that flags are set by MI(b,a).
3208 switch (CC) {
3209 default:
3210 return X86::COND_INVALID;
3211 case X86::COND_E:
3212 return X86::COND_E;
3213 case X86::COND_NE:
3214 return X86::COND_NE;
3215 case X86::COND_L:
3216 return X86::COND_G;
3217 case X86::COND_LE:
3218 return X86::COND_GE;
3219 case X86::COND_G:
3220 return X86::COND_L;
3221 case X86::COND_GE:
3222 return X86::COND_LE;
3223 case X86::COND_B:
3224 return X86::COND_A;
3225 case X86::COND_BE:
3226 return X86::COND_AE;
3227 case X86::COND_A:
3228 return X86::COND_B;
3229 case X86::COND_AE:
3230 return X86::COND_BE;
3231 }
3232}
3233
3234std::pair<X86::CondCode, bool>
3237 bool NeedSwap = false;
3238 switch (Predicate) {
3239 default:
3240 break;
3241 // Floating-point Predicates
3242 case CmpInst::FCMP_UEQ:
3243 CC = X86::COND_E;
3244 break;
3245 case CmpInst::FCMP_OLT:
3246 NeedSwap = true;
3247 [[fallthrough]];
3248 case CmpInst::FCMP_OGT:
3249 CC = X86::COND_A;
3250 break;
3251 case CmpInst::FCMP_OLE:
3252 NeedSwap = true;
3253 [[fallthrough]];
3254 case CmpInst::FCMP_OGE:
3255 CC = X86::COND_AE;
3256 break;
3257 case CmpInst::FCMP_UGT:
3258 NeedSwap = true;
3259 [[fallthrough]];
3260 case CmpInst::FCMP_ULT:
3261 CC = X86::COND_B;
3262 break;
3263 case CmpInst::FCMP_UGE:
3264 NeedSwap = true;
3265 [[fallthrough]];
3266 case CmpInst::FCMP_ULE:
3267 CC = X86::COND_BE;
3268 break;
3269 case CmpInst::FCMP_ONE:
3270 CC = X86::COND_NE;
3271 break;
3272 case CmpInst::FCMP_UNO:
3273 CC = X86::COND_P;
3274 break;
3275 case CmpInst::FCMP_ORD:
3276 CC = X86::COND_NP;
3277 break;
3278 case CmpInst::FCMP_OEQ:
3279 [[fallthrough]];
3280 case CmpInst::FCMP_UNE:
3282 break;
3283
3284 // Integer Predicates
3285 case CmpInst::ICMP_EQ:
3286 CC = X86::COND_E;
3287 break;
3288 case CmpInst::ICMP_NE:
3289 CC = X86::COND_NE;
3290 break;
3291 case CmpInst::ICMP_UGT:
3292 CC = X86::COND_A;
3293 break;
3294 case CmpInst::ICMP_UGE:
3295 CC = X86::COND_AE;
3296 break;
3297 case CmpInst::ICMP_ULT:
3298 CC = X86::COND_B;
3299 break;
3300 case CmpInst::ICMP_ULE:
3301 CC = X86::COND_BE;
3302 break;
3303 case CmpInst::ICMP_SGT:
3304 CC = X86::COND_G;
3305 break;
3306 case CmpInst::ICMP_SGE:
3307 CC = X86::COND_GE;
3308 break;
3309 case CmpInst::ICMP_SLT:
3310 CC = X86::COND_L;
3311 break;
3312 case CmpInst::ICMP_SLE:
3313 CC = X86::COND_LE;
3314 break;
3315 }
3316
3317 return std::make_pair(CC, NeedSwap);
3318}
3319
3320/// Return a cmov opcode for the given register size in bytes, and operand type.
3321unsigned X86::getCMovOpcode(unsigned RegBytes, bool HasMemoryOperand,
3322 bool HasNDD) {
3323 switch (RegBytes) {
3324 default:
3325 llvm_unreachable("Illegal register size!");
3326#define GET_ND_IF_ENABLED(OPC) (HasNDD ? OPC##_ND : OPC)
3327 case 2:
3328 return HasMemoryOperand ? GET_ND_IF_ENABLED(X86::CMOV16rm)
3329 : GET_ND_IF_ENABLED(X86::CMOV16rr);
3330 case 4:
3331 return HasMemoryOperand ? GET_ND_IF_ENABLED(X86::CMOV32rm)
3332 : GET_ND_IF_ENABLED(X86::CMOV32rr);
3333 case 8:
3334 return HasMemoryOperand ? GET_ND_IF_ENABLED(X86::CMOV64rm)
3335 : GET_ND_IF_ENABLED(X86::CMOV64rr);
3336 }
3337}
3338
3339/// Get the VPCMP immediate for the given condition.
3341 switch (CC) {
3342 default:
3343 llvm_unreachable("Unexpected SETCC condition");
3344 case ISD::SETNE:
3345 return 4;
3346 case ISD::SETEQ:
3347 return 0;
3348 case ISD::SETULT:
3349 case ISD::SETLT:
3350 return 1;
3351 case ISD::SETUGT:
3352 case ISD::SETGT:
3353 return 6;
3354 case ISD::SETUGE:
3355 case ISD::SETGE:
3356 return 5;
3357 case ISD::SETULE:
3358 case ISD::SETLE:
3359 return 2;
3360 }
3361}
3362
3363/// Get the VPCMP immediate if the operands are swapped.
3364unsigned X86::getSwappedVPCMPImm(unsigned Imm) {
3365 switch (Imm) {
3366 default:
3367 llvm_unreachable("Unreachable!");
3368 case 0x01:
3369 Imm = 0x06;
3370 break; // LT -> NLE
3371 case 0x02:
3372 Imm = 0x05;
3373 break; // LE -> NLT
3374 case 0x05:
3375 Imm = 0x02;
3376 break; // NLT -> LE
3377 case 0x06:
3378 Imm = 0x01;
3379 break; // NLE -> LT
3380 case 0x00: // EQ
3381 case 0x03: // FALSE
3382 case 0x04: // NE
3383 case 0x07: // TRUE
3384 break;
3385 }
3386
3387 return Imm;
3388}
3389
3390/// Get the VPCOM immediate if the operands are swapped.
3391unsigned X86::getSwappedVPCOMImm(unsigned Imm) {
3392 switch (Imm) {
3393 default:
3394 llvm_unreachable("Unreachable!");
3395 case 0x00:
3396 Imm = 0x02;
3397 break; // LT -> GT
3398 case 0x01:
3399 Imm = 0x03;
3400 break; // LE -> GE
3401 case 0x02:
3402 Imm = 0x00;
3403 break; // GT -> LT
3404 case 0x03:
3405 Imm = 0x01;
3406 break; // GE -> LE
3407 case 0x04: // EQ
3408 case 0x05: // NE
3409 case 0x06: // FALSE
3410 case 0x07: // TRUE
3411 break;
3412 }
3413
3414 return Imm;
3415}
3416
3417/// Get the VCMP immediate if the operands are swapped.
3418unsigned X86::getSwappedVCMPImm(unsigned Imm) {
3419 // Only need the lower 2 bits to distinquish.
3420 switch (Imm & 0x3) {
3421 default:
3422 llvm_unreachable("Unreachable!");
3423 case 0x00:
3424 case 0x03:
3425 // EQ/NE/TRUE/FALSE/ORD/UNORD don't change immediate when commuted.
3426 break;
3427 case 0x01:
3428 case 0x02:
3429 // Need to toggle bits 3:0. Bit 4 stays the same.
3430 Imm ^= 0xf;
3431 break;
3432 }
3433
3434 return Imm;
3435}
3436
3438 if (Info.RegClass == X86::VR128RegClassID ||
3439 Info.RegClass == X86::VR128XRegClassID)
3440 return 128;
3441 if (Info.RegClass == X86::VR256RegClassID ||
3442 Info.RegClass == X86::VR256XRegClassID)
3443 return 256;
3444 if (Info.RegClass == X86::VR512RegClassID)
3445 return 512;
3446 llvm_unreachable("Unknown register class!");
3447}
3448
3449/// Return true if the Reg is X87 register.
3450static bool isX87Reg(unsigned Reg) {
3451 return (Reg == X86::FPCW || Reg == X86::FPSW ||
3452 (Reg >= X86::ST0 && Reg <= X86::ST7));
3453}
3454
3455/// check if the instruction is X87 instruction
3457 // Call defs X87 register, so we special case it here because
3458 // otherwise calls are incorrectly flagged as x87 instructions
3459 // as a result.
3460 if (MI.isCall())
3461 return false;
3462 for (const MachineOperand &MO : MI.operands()) {
3463 if (!MO.isReg())
3464 continue;
3465 if (isX87Reg(MO.getReg()))
3466 return true;
3467 }
3468 return false;
3469}
3470
3472 auto IsMemOp = [](const MCOperandInfo &OpInfo) {
3473 return OpInfo.OperandType == MCOI::OPERAND_MEMORY;
3474 };
3475
3476 const MCInstrDesc &Desc = MI.getDesc();
3477
3478 // Directly invoke the MC-layer routine for real (i.e., non-pseudo)
3479 // instructions (fast case).
3480 if (!X86II::isPseudo(Desc.TSFlags)) {
3481 int MemRefIdx = X86II::getMemoryOperandNo(Desc.TSFlags);
3482 if (MemRefIdx >= 0)
3483 return MemRefIdx + X86II::getOperandBias(Desc);
3484#ifdef EXPENSIVE_CHECKS
3485 assert(none_of(Desc.operands(), IsMemOp) &&
3486 "Got false negative from X86II::getMemoryOperandNo()!");
3487#endif
3488 return -1;
3489 }
3490
3491 // Otherwise, handle pseudo instructions by examining the type of their
3492 // operands (slow case). An instruction cannot have a memory reference if it
3493 // has fewer than AddrNumOperands (= 5) explicit operands.
3494 unsigned NumOps = Desc.getNumOperands();
3495 if (NumOps < X86::AddrNumOperands) {
3496#ifdef EXPENSIVE_CHECKS
3497 assert(none_of(Desc.operands(), IsMemOp) &&
3498 "Expected no operands to have OPERAND_MEMORY type!");
3499#endif
3500 return -1;
3501 }
3502
3503 // The first operand with type OPERAND_MEMORY indicates the start of a memory
3504 // reference. We expect the following AddrNumOperand-1 operands to also have
3505 // OPERAND_MEMORY type.
3506 for (unsigned I = 0, E = NumOps - X86::AddrNumOperands; I != E; ++I) {
3507 if (IsMemOp(Desc.operands()[I])) {
3508#ifdef EXPENSIVE_CHECKS
3509 assert(std::all_of(Desc.operands().begin() + I,
3510 Desc.operands().begin() + I + X86::AddrNumOperands,
3511 IsMemOp) &&
3512 "Expected all five operands in the memory reference to have "
3513 "OPERAND_MEMORY type!");
3514#endif
3515 return I;
3516 }
3517 }
3518
3519 return -1;
3520}
3521
3523 unsigned OpNo) {
3524 assert(MI.getNumOperands() >= (OpNo + X86::AddrNumOperands) &&
3525 "Unexpected number of operands!");
3526
3527 const MachineOperand &Index = MI.getOperand(OpNo + X86::AddrIndexReg);
3528 if (!Index.isReg() || Index.getReg() != X86::NoRegister)
3529 return nullptr;
3530
3531 const MachineOperand &Disp = MI.getOperand(OpNo + X86::AddrDisp);
3532 if (!Disp.isCPI() || Disp.getOffset() != 0)
3533 return nullptr;
3534
3536 MI.getParent()->getParent()->getConstantPool()->getConstants();
3537 const MachineConstantPoolEntry &ConstantEntry = Constants[Disp.getIndex()];
3538
3539 // Bail if this is a machine constant pool entry, we won't be able to dig out
3540 // anything useful.
3541 if (ConstantEntry.isMachineConstantPoolEntry())
3542 return nullptr;
3543
3544 return ConstantEntry.Val.ConstVal;
3545}
3546
3548 switch (MI.getOpcode()) {
3549 case X86::TCRETURNdi:
3550 case X86::TCRETURNri:
3551 case X86::TCRETURNmi:
3552 case X86::TCRETURNdi64:
3553 case X86::TCRETURNri64:
3554 case X86::TCRETURNmi64:
3555 return true;
3556 default:
3557 return false;
3558 }
3559}
3560
3563 const MachineInstr &TailCall) const {
3564
3565 const MachineFunction *MF = TailCall.getMF();
3566
3567 if (MF->getTarget().getCodeModel() == CodeModel::Kernel) {
3568 // Kernel patches thunk calls in runtime, these should never be conditional.
3569 const MachineOperand &Target = TailCall.getOperand(0);
3570 if (Target.isSymbol()) {
3571 StringRef Symbol(Target.getSymbolName());
3572 // this is currently only relevant to r11/kernel indirect thunk.
3573 if (Symbol.equals("__x86_indirect_thunk_r11"))
3574 return false;
3575 }
3576 }
3577
3578 if (TailCall.getOpcode() != X86::TCRETURNdi &&
3579 TailCall.getOpcode() != X86::TCRETURNdi64) {
3580 // Only direct calls can be done with a conditional branch.
3581 return false;
3582 }
3583
3584 if (Subtarget.isTargetWin64() && MF->hasWinCFI()) {
3585 // Conditional tail calls confuse the Win64 unwinder.
3586 return false;
3587 }
3588
3589 assert(BranchCond.size() == 1);
3590 if (BranchCond[0].getImm() > X86::LAST_VALID_COND) {
3591 // Can't make a conditional tail call with this condition.
3592 return false;
3593 }
3594
3596 if (X86FI->getTCReturnAddrDelta() != 0 ||
3597 TailCall.getOperand(1).getImm() != 0) {
3598 // A conditional tail call cannot do any stack adjustment.
3599 return false;
3600 }
3601
3602 return true;
3603}
3604
3607 const MachineInstr &TailCall) const {
3608 assert(canMakeTailCallConditional(BranchCond, TailCall));
3609
3611 while (I != MBB.begin()) {
3612 --I;
3613 if (I->isDebugInstr())
3614 continue;
3615 if (!I->isBranch())
3616 assert(0 && "Can't find the branch to replace!");
3617
3619 assert(BranchCond.size() == 1);
3620 if (CC != BranchCond[0].getImm())
3621 continue;
3622
3623 break;
3624 }
3625
3626 unsigned Opc = TailCall.getOpcode() == X86::TCRETURNdi ? X86::TCRETURNdicc
3627 : X86::TCRETURNdi64cc;
3628
3629 auto MIB = BuildMI(MBB, I, MBB.findDebugLoc(I), get(Opc));
3630 MIB->addOperand(TailCall.getOperand(0)); // Destination.
3631 MIB.addImm(0); // Stack offset (not used).
3632 MIB->addOperand(BranchCond[0]); // Condition.
3633 MIB.copyImplicitOps(TailCall); // Regmask and (imp-used) parameters.
3634
3635 // Add implicit uses and defs of all live regs potentially clobbered by the
3636 // call. This way they still appear live across the call.
3637 LivePhysRegs LiveRegs(getRegisterInfo());
3638 LiveRegs.addLiveOuts(MBB);
3640 LiveRegs.stepForward(*MIB, Clobbers);
3641 for (const auto &C : Clobbers) {
3642 MIB.addReg(C.first, RegState::Implicit);
3644 }
3645
3646 I->eraseFromParent();
3647}
3648
3649// Given a MBB and its TBB, find the FBB which was a fallthrough MBB (it may
3650// not be a fallthrough MBB now due to layout changes). Return nullptr if the
3651// fallthrough MBB cannot be identified.
3654 // Look for non-EHPad successors other than TBB. If we find exactly one, it
3655 // is the fallthrough MBB. If we find zero, then TBB is both the target MBB
3656 // and fallthrough MBB. If we find more than one, we cannot identify the
3657 // fallthrough MBB and should return nullptr.
3658 MachineBasicBlock *FallthroughBB = nullptr;
3659 for (MachineBasicBlock *Succ : MBB->successors()) {
3660 if (Succ->isEHPad() || (Succ == TBB && FallthroughBB))
3661 continue;
3662 // Return a nullptr if we found more than one fallthrough successor.
3663 if (FallthroughBB && FallthroughBB != TBB)
3664 return nullptr;
3665 FallthroughBB = Succ;
3666 }
3667 return FallthroughBB;
3668}
3669
3670bool X86InstrInfo::analyzeBranchImpl(
3673 SmallVectorImpl<MachineInstr *> &CondBranches, bool AllowModify) const {
3674
3675 // Start from the bottom of the block and work up, examining the
3676 // terminator instructions.
3678 MachineBasicBlock::iterator UnCondBrIter = MBB.end();
3679 while (I != MBB.begin()) {
3680 --I;
3681 if (I->isDebugInstr())
3682 continue;
3683
3684 // Working from the bottom, when we see a non-terminator instruction, we're
3685 // done.
3686 if (!isUnpredicatedTerminator(*I))
3687 break;
3688
3689 // A terminator that isn't a branch can't easily be handled by this
3690 // analysis.
3691 if (!I->isBranch())
3692 return true;
3693
3694 // Handle unconditional branches.
3695 if (I->getOpcode() == X86::JMP_1) {
3696 UnCondBrIter = I;
3697
3698 if (!AllowModify) {
3699 TBB = I->getOperand(0).getMBB();
3700 continue;
3701 }
3702
3703 // If the block has any instructions after a JMP, delete them.
3704 MBB.erase(std::next(I), MBB.end());
3705
3706 Cond.clear();
3707 FBB = nullptr;
3708
3709 // Delete the JMP if it's equivalent to a fall-through.
3710 if (MBB.isLayoutSuccessor(I->getOperand(0).getMBB())) {
3711 TBB = nullptr;
3712 I->eraseFromParent();
3713 I = MBB.end();
3714 UnCondBrIter = MBB.end();
3715 continue;
3716 }
3717
3718 // TBB is used to indicate the unconditional destination.
3719 TBB = I->getOperand(0).getMBB();
3720 continue;
3721 }
3722
3723 // Handle conditional branches.
3724 X86::CondCode BranchCode = X86::getCondFromBranch(*I);
3725 if (BranchCode == X86::COND_INVALID)
3726 return true; // Can't handle indirect branch.
3727
3728 // In practice we should never have an undef eflags operand, if we do
3729 // abort here as we are not prepared to preserve the flag.
3730 if (I->findRegisterUseOperand(X86::EFLAGS)->isUndef())
3731 return true;
3732
3733 // Working from the bottom, handle the first conditional branch.
3734 if (Cond.empty()) {
3735 FBB = TBB;
3736 TBB = I->getOperand(0).getMBB();
3737 Cond.push_back(MachineOperand::CreateImm(BranchCode));
3738 CondBranches.push_back(&*I);
3739 continue;
3740 }
3741
3742 // Handle subsequent conditional branches. Only handle the case where all
3743 // conditional branches branch to the same destination and their condition
3744 // opcodes fit one of the special multi-branch idioms.
3745 assert(Cond.size() == 1);
3746 assert(TBB);
3747
3748 // If the conditions are the same, we can leave them alone.
3749 X86::CondCode OldBranchCode = (X86::CondCode)Cond[0].getImm();
3750 auto NewTBB = I->getOperand(0).getMBB();
3751 if (OldBranchCode == BranchCode && TBB == NewTBB)
3752 continue;
3753
3754 // If they differ, see if they fit one of the known patterns. Theoretically,
3755 // we could handle more patterns here, but we shouldn't expect to see them
3756 // if instruction selection has done a reasonable job.
3757 if (TBB == NewTBB &&
3758 ((OldBranchCode == X86::COND_P && BranchCode == X86::COND_NE) ||
3759 (OldBranchCode == X86::COND_NE && BranchCode == X86::COND_P))) {
3760 BranchCode = X86::COND_NE_OR_P;
3761 } else if ((OldBranchCode == X86::COND_NP && BranchCode == X86::COND_NE) ||
3762 (OldBranchCode == X86::COND_E && BranchCode == X86::COND_P)) {
3763 if (NewTBB != (FBB ? FBB : getFallThroughMBB(&MBB, TBB)))
3764 return true;
3765
3766 // X86::COND_E_AND_NP usually has two different branch destinations.
3767 //
3768 // JP B1
3769 // JE B2
3770 // JMP B1
3771 // B1:
3772 // B2:
3773 //
3774 // Here this condition branches to B2 only if NP && E. It has another
3775 // equivalent form:
3776 //
3777 // JNE B1
3778 // JNP B2
3779 // JMP B1
3780 // B1:
3781 // B2:
3782 //
3783 // Similarly it branches to B2 only if E && NP. That is why this condition
3784 // is named with COND_E_AND_NP.
3785 BranchCode = X86::COND_E_AND_NP;
3786 } else
3787 return true;
3788
3789 // Update the MachineOperand.
3790 Cond[0].setImm(BranchCode);
3791 CondBranches.push_back(&*I);
3792 }
3793
3794 return false;
3795}
3796
3799 MachineBasicBlock *&FBB,
3801 bool AllowModify) const {
3802 SmallVector<MachineInstr *, 4> CondBranches;
3803 return analyzeBranchImpl(MBB, TBB, FBB, Cond, CondBranches, AllowModify);
3804}
3805
3807 const MCInstrDesc &Desc = MI.getDesc();
3808 int MemRefBegin = X86II::getMemoryOperandNo(Desc.TSFlags);
3809 assert(MemRefBegin >= 0 && "instr should have memory operand");
3810 MemRefBegin += X86II::getOperandBias(Desc);
3811
3812 const MachineOperand &MO = MI.getOperand(MemRefBegin + X86::AddrDisp);
3813 if (!MO.isJTI())
3814 return -1;
3815
3816 return MO.getIndex();
3817}
3818
3820 Register Reg) {
3821 if (!Reg.isVirtual())
3822 return -1;
3823 MachineInstr *MI = MRI.getUniqueVRegDef(Reg);
3824 if (MI == nullptr)
3825 return -1;
3826 unsigned Opcode = MI->getOpcode();
3827 if (Opcode != X86::LEA64r && Opcode != X86::LEA32r)
3828 return -1;
3830}
3831
3833 unsigned Opcode = MI.getOpcode();
3834 // Switch-jump pattern for non-PIC code looks like:
3835 // JMP64m $noreg, 8, %X, %jump-table.X, $noreg
3836 if (Opcode == X86::JMP64m || Opcode == X86::JMP32m) {
3838 }
3839 // The pattern for PIC code looks like:
3840 // %0 = LEA64r $rip, 1, $noreg, %jump-table.X
3841 // %1 = MOVSX64rm32 %0, 4, XX, 0, $noreg
3842 // %2 = ADD64rr %1, %0
3843 // JMP64r %2
3844 if (Opcode == X86::JMP64r || Opcode == X86::JMP32r) {
3845 Register Reg = MI.getOperand(0).getReg();
3846 if (!Reg.isVirtual())
3847 return -1;
3848 const MachineFunction &MF = *MI.getParent()->getParent();
3849 const MachineRegisterInfo &MRI = MF.getRegInfo();
3850 MachineInstr *Add = MRI.getUniqueVRegDef(Reg);
3851 if (Add == nullptr)
3852 return -1;
3853 if (Add->getOpcode() != X86::ADD64rr && Add->getOpcode() != X86::ADD32rr)
3854 return -1;
3855 int JTI1 = getJumpTableIndexFromReg(MRI, Add->getOperand(1).getReg());
3856 if (JTI1 >= 0)
3857 return JTI1;
3858 int JTI2 = getJumpTableIndexFromReg(MRI, Add->getOperand(2).getReg());
3859 if (JTI2 >= 0)
3860 return JTI2;
3861 }
3862 return -1;
3863}
3864
3866 MachineBranchPredicate &MBP,
3867 bool AllowModify) const {
3868 using namespace std::placeholders;
3869
3871 SmallVector<MachineInstr *, 4> CondBranches;
3872 if (analyzeBranchImpl(MBB, MBP.TrueDest, MBP.FalseDest, Cond, CondBranches,
3873 AllowModify))
3874 return true;
3875
3876 if (Cond.size() != 1)
3877 return true;
3878
3879 assert(MBP.TrueDest && "expected!");
3880
3881 if (!MBP.FalseDest)
3882 MBP.FalseDest = MBB.getNextNode();
3883
3885
3886 MachineInstr *ConditionDef = nullptr;
3887 bool SingleUseCondition = true;
3888
3890 if (MI.modifiesRegister(X86::EFLAGS, TRI)) {
3891 ConditionDef = &MI;
3892 break;
3893 }
3894
3895 if (MI.readsRegister(X86::EFLAGS, TRI))
3896 SingleUseCondition = false;
3897 }
3898
3899 if (!ConditionDef)
3900 return true;
3901
3902 if (SingleUseCondition) {
3903 for (auto *Succ : MBB.successors())
3904 if (Succ->isLiveIn(X86::EFLAGS))
3905 SingleUseCondition = false;
3906 }
3907
3908 MBP.ConditionDef = ConditionDef;
3909 MBP.SingleUseCondition = SingleUseCondition;
3910
3911 // Currently we only recognize the simple pattern:
3912 //
3913 // test %reg, %reg
3914 // je %label
3915 //
3916 const unsigned TestOpcode =
3917 Subtarget.is64Bit() ? X86::TEST64rr : X86::TEST32rr;
3918
3919 if (ConditionDef->getOpcode() == TestOpcode &&
3920 ConditionDef->getNumOperands() == 3 &&
3921 ConditionDef->getOperand(0).isIdenticalTo(ConditionDef->getOperand(1)) &&
3922 (Cond[0].getImm() == X86::COND_NE || Cond[0].getImm() == X86::COND_E)) {
3923 MBP.LHS = ConditionDef->getOperand(0);
3924 MBP.RHS = MachineOperand::CreateImm(0);
3925 MBP.Predicate = Cond[0].getImm() == X86::COND_NE
3926 ? MachineBranchPredicate::PRED_NE
3927 : MachineBranchPredicate::PRED_EQ;
3928 return false;
3929 }
3930
3931 return true;
3932}
3933
3935 int *BytesRemoved) const {
3936 assert(!BytesRemoved && "code size not handled");
3937
3939 unsigned Count = 0;
3940
3941 while (I != MBB.begin()) {
3942 --I;
3943 if (I->isDebugInstr())
3944 continue;
3945 if (I->getOpcode() != X86::JMP_1 &&
3947 break;
3948 // Remove the branch.
3949 I->eraseFromParent();
3950 I = MBB.end();
3951 ++Count;
3952 }
3953
3954 return Count;
3955}
3956
3959 MachineBasicBlock *FBB,
3961 const DebugLoc &DL, int *BytesAdded) const {
3962 // Shouldn't be a fall through.
3963 assert(TBB && "insertBranch must not be told to insert a fallthrough");
3964 assert((Cond.size() == 1 || Cond.size() == 0) &&
3965 "X86 branch conditions have one component!");
3966 assert(!BytesAdded && "code size not handled");
3967
3968 if (Cond.empty()) {
3969 // Unconditional branch?
3970 assert(!FBB && "Unconditional branch with multiple successors!");
3971 BuildMI(&MBB, DL, get(X86::JMP_1)).addMBB(TBB);
3972 return 1;
3973 }
3974
3975 // If FBB is null, it is implied to be a fall-through block.
3976 bool FallThru = FBB == nullptr;
3977
3978 // Conditional branch.
3979 unsigned Count = 0;
3980 X86::CondCode CC = (X86::CondCode)Cond[0].getImm();
3981 switch (CC) {
3982 case X86::COND_NE_OR_P:
3983 // Synthesize NE_OR_P with two branches.
3984 BuildMI(&MBB, DL, get(X86::JCC_1)).addMBB(TBB).addImm(X86::COND_NE);
3985 ++Count;
3986 BuildMI(&MBB, DL, get(X86::JCC_1)).addMBB(TBB).addImm(X86::COND_P);
3987 ++Count;
3988 break;
3989 case X86::COND_E_AND_NP:
3990 // Use the next block of MBB as FBB if it is null.
3991 if (FBB == nullptr) {
3992 FBB = getFallThroughMBB(&MBB, TBB);
3993 assert(FBB && "MBB cannot be the last block in function when the false "
3994 "body is a fall-through.");
3995 }
3996 // Synthesize COND_E_AND_NP with two branches.
3997 BuildMI(&MBB, DL, get(X86::JCC_1)).addMBB(FBB).addImm(X86::COND_NE);
3998 ++Count;
3999 BuildMI(&MBB, DL, get(X86::JCC_1)).addMBB(TBB).addImm(X86::COND_NP);
4000 ++Count;
4001 break;
4002 default: {
4003 BuildMI(&MBB, DL, get(X86::JCC_1)).addMBB(TBB).addImm(CC);
4004 ++Count;
4005 }
4006 }
4007 if (!FallThru) {
4008 // Two-way Conditional branch. Insert the second branch.
4009 BuildMI(&MBB, DL, get(X86::JMP_1)).addMBB(FBB);
4010 ++Count;
4011 }
4012 return Count;
4013}
4014
4017 Register DstReg, Register TrueReg,
4018 Register FalseReg, int &CondCycles,
4019 int &TrueCycles, int &FalseCycles) const {
4020 // Not all subtargets have cmov instructions.
4021 if (!Subtarget.canUseCMOV())
4022 return false;
4023 if (Cond.size() != 1)
4024 return false;
4025 // We cannot do the composite conditions, at least not in SSA form.
4026 if ((X86::CondCode)Cond[0].getImm() > X86::LAST_VALID_COND)
4027 return false;
4028
4029 // Check register classes.
4031 const TargetRegisterClass *RC =
4032 RI.getCommonSubClass(MRI.getRegClass(TrueReg), MRI.getRegClass(FalseReg));
4033 if (!RC)
4034 return false;
4035
4036 // We have cmov instructions for 16, 32, and 64 bit general purpose registers.
4037 if (X86::GR16RegClass.hasSubClassEq(RC) ||
4038 X86::GR32RegClass.hasSubClassEq(RC) ||
4039 X86::GR64RegClass.hasSubClassEq(RC)) {
4040 // This latency applies to Pentium M, Merom, Wolfdale, Nehalem, and Sandy
4041 // Bridge. Probably Ivy Bridge as well.
4042 CondCycles = 2;
4043 TrueCycles = 2;
4044 FalseCycles = 2;
4045 return true;
4046 }
4047
4048 // Can't do vectors.
4049 return false;
4050}
4051
4054 const DebugLoc &DL, Register DstReg,
4056 Register FalseReg) const {
4058 const TargetRegisterInfo &TRI = *MRI.getTargetRegisterInfo();
4059 const TargetRegisterClass &RC = *MRI.getRegClass(DstReg);
4060 assert(Cond.size() == 1 && "Invalid Cond array");
4061 unsigned Opc =
4062 X86::getCMovOpcode(TRI.getRegSizeInBits(RC) / 8,
4063 false /*HasMemoryOperand*/, Subtarget.hasNDD());
4064 BuildMI(MBB, I, DL, get(Opc), DstReg)
4065 .addReg(FalseReg)
4066 .addReg(TrueReg)
4067 .addImm(Cond[0].getImm());
4068}
4069
4070/// Test if the given register is a physical h register.
4071static bool isHReg(unsigned Reg) {
4072 return X86::GR8_ABCD_HRegClass.contains(Reg);
4073}
4074
4075// Try and copy between VR128/VR64 and GR64 registers.
4076static unsigned CopyToFromAsymmetricReg(unsigned DestReg, unsigned SrcReg,
4077 const X86Subtarget &Subtarget) {
4078 bool HasAVX = Subtarget.hasAVX();
4079 bool HasAVX512 = Subtarget.hasAVX512();
4080 bool HasEGPR = Subtarget.hasEGPR();
4081
4082 // SrcReg(MaskReg) -> DestReg(GR64)
4083 // SrcReg(MaskReg) -> DestReg(GR32)
4084
4085 // All KMASK RegClasses hold the same k registers, can be tested against
4086 // anyone.
4087 if (X86::VK16RegClass.contains(SrcReg)) {
4088 if (X86::GR64RegClass.contains(DestReg)) {
4089 assert(Subtarget.hasBWI());
4090 return HasEGPR ? X86::KMOVQrk_EVEX : X86::KMOVQrk;
4091 }
4092 if (X86::GR32RegClass.contains(DestReg))
4093 return Subtarget.hasBWI() ? (HasEGPR ? X86::KMOVDrk_EVEX : X86::KMOVDrk)
4094 : (HasEGPR ? X86::KMOVWrk_EVEX : X86::KMOVWrk);
4095 }
4096
4097 // SrcReg(GR64) -> DestReg(MaskReg)
4098 // SrcReg(GR32) -> DestReg(MaskReg)
4099
4100 // All KMASK RegClasses hold the same k registers, can be tested against
4101 // anyone.
4102 if (X86::VK16RegClass.contains(DestReg)) {
4103 if (X86::GR64RegClass.contains(SrcReg)) {
4104 assert(Subtarget.hasBWI());
4105 return HasEGPR ? X86::KMOVQkr_EVEX : X86::KMOVQkr;
4106 }
4107 if (X86::GR32RegClass.contains(SrcReg))
4108 return Subtarget.hasBWI() ? (HasEGPR ? X86::KMOVDkr_EVEX : X86::KMOVDkr)
4109 : (HasEGPR ? X86::KMOVWkr_EVEX : X86::KMOVWkr);
4110 }
4111
4112 // SrcReg(VR128) -> DestReg(GR64)
4113 // SrcReg(VR64) -> DestReg(GR64)
4114 // SrcReg(GR64) -> DestReg(VR128)
4115 // SrcReg(GR64) -> DestReg(VR64)
4116
4117 if (X86::GR64RegClass.contains(DestReg)) {
4118 if (X86::VR128XRegClass.contains(SrcReg))
4119 // Copy from a VR128 register to a GR64 register.
4120 return HasAVX512 ? X86::VMOVPQIto64Zrr
4121 : HasAVX ? X86::VMOVPQIto64rr
4122 : X86::MOVPQIto64rr;
4123 if (X86::VR64RegClass.contains(SrcReg))
4124 // Copy from a VR64 register to a GR64 register.
4125 return X86::MMX_MOVD64from64rr;
4126 } else if (X86::GR64RegClass.contains(SrcReg)) {
4127 // Copy from a GR64 register to a VR128 register.
4128 if (X86::VR128XRegClass.contains(DestReg))
4129 return HasAVX512 ? X86::VMOV64toPQIZrr
4130 : HasAVX ? X86::VMOV64toPQIrr
4131 : X86::MOV64toPQIrr;
4132 // Copy from a GR64 register to a VR64 register.
4133 if (X86::VR64RegClass.contains(DestReg))
4134 return X86::MMX_MOVD64to64rr;
4135 }
4136
4137 // SrcReg(VR128) -> DestReg(GR32)
4138 // SrcReg(GR32) -> DestReg(VR128)
4139
4140 if (X86::GR32RegClass.contains(DestReg) &&
4141 X86::VR128XRegClass.contains(SrcReg))
4142 // Copy from a VR128 register to a GR32 register.
4143 return HasAVX512 ? X86::VMOVPDI2DIZrr
4144 : HasAVX ? X86::VMOVPDI2DIrr
4145 : X86::MOVPDI2DIrr;
4146
4147 if (X86::VR128XRegClass.contains(DestReg) &&
4148 X86::GR32RegClass.contains(SrcReg))
4149 // Copy from a VR128 register to a VR128 register.
4150 return HasAVX512 ? X86::VMOVDI2PDIZrr
4151 : HasAVX ? X86::VMOVDI2PDIrr
4152 : X86::MOVDI2PDIrr;
4153 return 0;
4154}
4155
4158 const DebugLoc &DL, MCRegister DestReg,
4159 MCRegister SrcReg, bool KillSrc) const {
4160 // First deal with the normal symmetric copies.
4161 bool HasAVX = Subtarget.hasAVX();
4162 bool HasVLX = Subtarget.hasVLX();
4163 bool HasEGPR = Subtarget.hasEGPR();
4164 unsigned Opc = 0;
4165 if (X86::GR64RegClass.contains(DestReg, SrcReg))
4166 Opc = X86::MOV64rr;
4167 else if (X86::GR32RegClass.contains(DestReg, SrcReg))
4168 Opc = X86::MOV32rr;
4169 else if (X86::GR16RegClass.contains(DestReg, SrcReg))
4170 Opc = X86::MOV16rr;
4171 else if (X86::GR8RegClass.contains(DestReg, SrcReg)) {
4172 // Copying to or from a physical H register on x86-64 requires a NOREX
4173 // move. Otherwise use a normal move.
4174 if ((isHReg(DestReg) || isHReg(SrcReg)) && Subtarget.is64Bit()) {
4175 Opc = X86::MOV8rr_NOREX;
4176 // Both operands must be encodable without an REX prefix.
4177 assert(X86::GR8_NOREXRegClass.contains(SrcReg, DestReg) &&
4178 "8-bit H register can not be copied outside GR8_NOREX");
4179 } else
4180 Opc = X86::MOV8rr;
4181 } else if (X86::VR64RegClass.contains(DestReg, SrcReg))
4182 Opc = X86::MMX_MOVQ64rr;
4183 else if (X86::VR128XRegClass.contains(DestReg, SrcReg)) {
4184 if (HasVLX)
4185 Opc = X86::VMOVAPSZ128rr;
4186 else if (X86::VR128RegClass.contains(DestReg, SrcReg))
4187 Opc = HasAVX ? X86::VMOVAPSrr : X86::MOVAPSrr;
4188 else {
4189 // If this an extended register and we don't have VLX we need to use a
4190 // 512-bit move.
4191 Opc = X86::VMOVAPSZrr;
4193 DestReg =
4194 TRI->getMatchingSuperReg(DestReg, X86::sub_xmm, &X86::VR512RegClass);
4195 SrcReg =
4196 TRI->getMatchingSuperReg(SrcReg, X86::sub_xmm, &X86::VR512RegClass);
4197 }
4198 } else if (X86::VR256XRegClass.contains(DestReg, SrcReg)) {
4199 if (HasVLX)
4200 Opc = X86::VMOVAPSZ256rr;
4201 else if (X86::VR256RegClass.contains(DestReg, SrcReg))
4202 Opc = X86::VMOVAPSYrr;
4203 else {
4204 // If this an extended register and we don't have VLX we need to use a
4205 // 512-bit move.
4206 Opc = X86::VMOVAPSZrr;
4208 DestReg =
4209 TRI->getMatchingSuperReg(DestReg, X86::sub_ymm, &X86::VR512RegClass);
4210 SrcReg =
4211 TRI->getMatchingSuperReg(SrcReg, X86::sub_ymm, &X86::VR512RegClass);
4212 }
4213 } else if (X86::VR512RegClass.contains(DestReg, SrcReg))
4214 Opc = X86::VMOVAPSZrr;
4215 // All KMASK RegClasses hold the same k registers, can be tested against
4216 // anyone.
4217 else if (X86::VK16RegClass.contains(DestReg, SrcReg))
4218 Opc = Subtarget.hasBWI() ? (HasEGPR ? X86::KMOVQkk_EVEX : X86::KMOVQkk)
4219 : (HasEGPR ? X86::KMOVQkk_EVEX : X86::KMOVWkk);
4220 if (!Opc)
4221 Opc = CopyToFromAsymmetricReg(DestReg, SrcReg, Subtarget);
4222
4223 if (Opc) {
4224 BuildMI(MBB, MI, DL, get(Opc), DestReg)
4225 .addReg(SrcReg, getKillRegState(KillSrc));
4226 return;
4227 }
4228
4229 if (SrcReg == X86::EFLAGS || DestReg == X86::EFLAGS) {
4230 // FIXME: We use a fatal error here because historically LLVM has tried
4231 // lower some of these physreg copies and we want to ensure we get
4232 // reasonable bug reports if someone encounters a case no other testing
4233 // found. This path should be removed after the LLVM 7 release.
4234 report_fatal_error("Unable to copy EFLAGS physical register!");
4235 }
4236
4237 LLVM_DEBUG(dbgs() << "Cannot copy " << RI.getName(SrcReg) << " to "
4238 << RI.getName(DestReg) << '\n');
4239 report_fatal_error("Cannot emit physreg copy instruction");
4240}
4241
4242std::optional<DestSourcePair>
4244 if (MI.isMoveReg()) {
4245 // FIXME: Dirty hack for apparent invariant that doesn't hold when
4246 // subreg_to_reg is coalesced with ordinary copies, such that the bits that
4247 // were asserted as 0 are now undef.
4248 if (MI.getOperand(0).isUndef() && MI.getOperand(0).getSubReg())
4249 return std::nullopt;
4250
4251 return DestSourcePair{MI.getOperand(0), MI.getOperand(1)};
4252 }
4253 return std::nullopt;
4254}
4255
4256static unsigned getLoadStoreOpcodeForFP16(bool Load, const X86Subtarget &STI) {
4257 if (STI.hasFP16())
4258 return Load ? X86::VMOVSHZrm_alt : X86::VMOVSHZmr;
4259 if (Load)
4260 return STI.hasAVX512() ? X86::VMOVSSZrm
4261 : STI.hasAVX() ? X86::VMOVSSrm
4262 : X86::MOVSSrm;
4263 else
4264 return STI.hasAVX512() ? X86::VMOVSSZmr
4265 : STI.hasAVX() ? X86::VMOVSSmr
4266 : X86::MOVSSmr;
4267}
4268
4270 const TargetRegisterClass *RC,
4271 bool IsStackAligned,
4272 const X86Subtarget &STI, bool Load) {
4273 bool HasAVX = STI.hasAVX();
4274 bool HasAVX512 = STI.hasAVX512();
4275 bool HasVLX = STI.hasVLX();
4276 bool HasEGPR = STI.hasEGPR();
4277
4278 assert(RC != nullptr && "Invalid target register class");
4279 switch (STI.getRegisterInfo()->getSpillSize(*RC)) {
4280 default:
4281 llvm_unreachable("Unknown spill size");
4282 case 1:
4283 assert(X86::GR8RegClass.hasSubClassEq(RC) && "Unknown 1-byte regclass");
4284 if (STI.is64Bit())
4285 // Copying to or from a physical H register on x86-64 requires a NOREX
4286 // move. Otherwise use a normal move.
4287 if (isHReg(Reg) || X86::GR8_ABCD_HRegClass.hasSubClassEq(RC))
4288 return Load ? X86::MOV8rm_NOREX : X86::MOV8mr_NOREX;
4289 return Load ? X86::MOV8rm : X86::MOV8mr;
4290 case 2:
4291 if (X86::VK16RegClass.hasSubClassEq(RC))
4292 return Load ? (HasEGPR ? X86::KMOVWkm_EVEX : X86::KMOVWkm)
4293 : (HasEGPR ? X86::KMOVWmk_EVEX : X86::KMOVWmk);
4294 assert(X86::GR16RegClass.hasSubClassEq(RC) && "Unknown 2-byte regclass");
4295 return Load ? X86::MOV16rm : X86::MOV16mr;
4296 case 4:
4297 if (X86::GR32RegClass.hasSubClassEq(RC))
4298 return Load ? X86::MOV32rm : X86::MOV32mr;
4299 if (X86::FR32XRegClass.hasSubClassEq(RC))
4300 return Load ? (HasAVX512 ? X86::VMOVSSZrm_alt
4301 : HasAVX ? X86::VMOVSSrm_alt
4302 : X86::MOVSSrm_alt)
4303 : (HasAVX512 ? X86::VMOVSSZmr
4304 : HasAVX ? X86::VMOVSSmr
4305 : X86::MOVSSmr);
4306 if (X86::RFP32RegClass.hasSubClassEq(RC))
4307 return Load ? X86::LD_Fp32m : X86::ST_Fp32m;
4308 if (X86::VK32RegClass.hasSubClassEq(RC)) {
4309 assert(STI.hasBWI() && "KMOVD requires BWI");
4310 return Load ? (HasEGPR ? X86::KMOVDkm_EVEX : X86::KMOVDkm)
4311 : (HasEGPR ? X86::KMOVDmk_EVEX : X86::KMOVDmk);
4312 }
4313 // All of these mask pair classes have the same spill size, the same kind
4314 // of kmov instructions can be used with all of them.
4315 if (X86::VK1PAIRRegClass.hasSubClassEq(RC) ||
4316 X86::VK2PAIRRegClass.hasSubClassEq(RC) ||
4317 X86::VK4PAIRRegClass.hasSubClassEq(RC) ||
4318 X86::VK8PAIRRegClass.hasSubClassEq(RC) ||
4319 X86::VK16PAIRRegClass.hasSubClassEq(RC))
4320 return Load ? X86::MASKPAIR16LOAD : X86::MASKPAIR16STORE;
4321 if (X86::FR16RegClass.hasSubClassEq(RC) ||
4322 X86::FR16XRegClass.hasSubClassEq(RC))
4323 return getLoadStoreOpcodeForFP16(Load, STI);
4324 llvm_unreachable("Unknown 4-byte regclass");
4325 case 8:
4326 if (X86::GR64RegClass.hasSubClassEq(RC))
4327 return Load ? X86::MOV64rm : X86::MOV64mr;
4328 if (X86::FR64XRegClass.hasSubClassEq(RC))
4329 return Load ? (HasAVX512 ? X86::VMOVSDZrm_alt
4330 : HasAVX ? X86::VMOVSDrm_alt
4331 : X86::MOVSDrm_alt)
4332 : (HasAVX512 ? X86::VMOVSDZmr
4333 : HasAVX ? X86::VMOVSDmr
4334 : X86::MOVSDmr);
4335 if (X86::VR64RegClass.hasSubClassEq(RC))
4336 return Load ? X86::MMX_MOVQ64rm : X86::MMX_MOVQ64mr;
4337 if (X86::RFP64RegClass.hasSubClassEq(RC))
4338 return Load ? X86::LD_Fp64m : X86::ST_Fp64m;
4339 if (X86::VK64RegClass.hasSubClassEq(RC)) {
4340 assert(STI.hasBWI() && "KMOVQ requires BWI");
4341 return Load ? (HasEGPR ? X86::KMOVQkm_EVEX : X86::KMOVQkm)
4342 : (HasEGPR ? X86::KMOVQmk_EVEX : X86::KMOVQmk);
4343 }
4344 llvm_unreachable("Unknown 8-byte regclass");
4345 case 10:
4346 assert(X86::RFP80RegClass.hasSubClassEq(RC) && "Unknown 10-byte regclass");
4347 return Load ? X86::LD_Fp80m : X86::ST_FpP80m;
4348 case 16: {
4349 if (X86::VR128XRegClass.hasSubClassEq(RC)) {
4350 // If stack is realigned we can use aligned stores.
4351 if (IsStackAligned)
4352 return Load ? (HasVLX ? X86::VMOVAPSZ128rm
4353 : HasAVX512 ? X86::VMOVAPSZ128rm_NOVLX
4354 : HasAVX ? X86::VMOVAPSrm
4355 : X86::MOVAPSrm)
4356 : (HasVLX ? X86::VMOVAPSZ128mr
4357 : HasAVX512 ? X86::VMOVAPSZ128mr_NOVLX
4358 : HasAVX ? X86::VMOVAPSmr
4359 : X86::MOVAPSmr);
4360 else
4361 return Load ? (HasVLX ? X86::VMOVUPSZ128rm
4362 : HasAVX512 ? X86::VMOVUPSZ128rm_NOVLX
4363 : HasAVX ? X86::VMOVUPSrm
4364 : X86::MOVUPSrm)
4365 : (HasVLX ? X86::VMOVUPSZ128mr
4366 : HasAVX512 ? X86::VMOVUPSZ128mr_NOVLX
4367 : HasAVX ? X86::VMOVUPSmr
4368 : X86::MOVUPSmr);
4369 }
4370 llvm_unreachable("Unknown 16-byte regclass");
4371 }
4372 case 32:
4373 assert(X86::VR256XRegClass.hasSubClassEq(RC) && "Unknown 32-byte regclass");
4374 // If stack is realigned we can use aligned stores.
4375 if (IsStackAligned)
4376 return Load ? (HasVLX ? X86::VMOVAPSZ256rm
4377 : HasAVX512 ? X86::VMOVAPSZ256rm_NOVLX
4378 : X86::VMOVAPSYrm)
4379 : (HasVLX ? X86::VMOVAPSZ256mr
4380 : HasAVX512 ? X86::VMOVAPSZ256mr_NOVLX
4381 : X86::VMOVAPSYmr);
4382 else
4383 return Load ? (HasVLX ? X86::VMOVUPSZ256rm
4384 : HasAVX512 ? X86::VMOVUPSZ256rm_NOVLX
4385 : X86::VMOVUPSYrm)
4386 : (HasVLX ? X86::VMOVUPSZ256mr
4387 : HasAVX512 ? X86::VMOVUPSZ256mr_NOVLX
4388 : X86::VMOVUPSYmr);
4389 case 64:
4390 assert(X86::VR512RegClass.hasSubClassEq(RC) && "Unknown 64-byte regclass");
4391 assert(STI.hasAVX512() && "Using 512-bit register requires AVX512");
4392 if (IsStackAligned)
4393 return Load ? X86::VMOVAPSZrm : X86::VMOVAPSZmr;
4394 else
4395 return Load ? X86::VMOVUPSZrm : X86::VMOVUPSZmr;
4396 case 1024:
4397 assert(X86::TILERegClass.hasSubClassEq(RC) && "Unknown 1024-byte regclass");
4398 assert(STI.hasAMXTILE() && "Using 8*1024-bit register requires AMX-TILE");
4399#define GET_EGPR_IF_ENABLED(OPC) (STI.hasEGPR() ? OPC##_EVEX : OPC)
4400 return Load ? GET_EGPR_IF_ENABLED(X86::TILELOADD)
4401 : GET_EGPR_IF_ENABLED(X86::TILESTORED);
4402#undef GET_EGPR_IF_ENABLED
4403 }
4404}
4405
4406std::optional<ExtAddrMode>
4408 const TargetRegisterInfo *TRI) const {
4409 const MCInstrDesc &Desc = MemI.getDesc();
4410 int MemRefBegin = X86II::getMemoryOperandNo(Desc.TSFlags);
4411 if (MemRefBegin < 0)
4412 return std::nullopt;
4413
4414 MemRefBegin += X86II::getOperandBias(Desc);
4415
4416 auto &BaseOp = MemI.getOperand(MemRefBegin + X86::AddrBaseReg);
4417 if (!BaseOp.isReg()) // Can be an MO_FrameIndex
4418 return std::nullopt;
4419
4420 const MachineOperand &DispMO = MemI.getOperand(MemRefBegin + X86::AddrDisp);
4421 // Displacement can be symbolic
4422 if (!DispMO.isImm())
4423 return std::nullopt;
4424
4425 ExtAddrMode AM;
4426 AM.BaseReg = BaseOp.getReg();
4427 AM.ScaledReg = MemI.getOperand(MemRefBegin + X86::AddrIndexReg).getReg();
4428 AM.Scale = MemI.getOperand(MemRefBegin + X86::AddrScaleAmt).getImm();
4429 AM.Displacement = DispMO.getImm();
4430 return AM;
4431}
4432
4434 StringRef &ErrInfo) const {
4435 std::optional<ExtAddrMode> AMOrNone = getAddrModeFromMemoryOp(MI, nullptr);
4436 if (!AMOrNone)
4437 return true;
4438
4439 ExtAddrMode AM = *AMOrNone;
4441 if (AM.ScaledReg != X86::NoRegister) {
4442 switch (AM.Scale) {
4443 case 1:
4444 case 2:
4445 case 4:
4446 case 8:
4447 break;
4448 default:
4449 ErrInfo = "Scale factor in address must be 1, 2, 4 or 8";
4450 return false;
4451 }
4452 }
4453 if (!isInt<32>(AM.Displacement)) {
4454 ErrInfo = "Displacement in address must fit into 32-bit signed "
4455 "integer";
4456 return false;
4457 }
4458
4459 return true;
4460}
4461
4463 const Register Reg,
4464 int64_t &ImmVal) const {
4465 Register MovReg = Reg;
4466 const MachineInstr *MovMI = &MI;
4467
4468 // Follow use-def for SUBREG_TO_REG to find the real move immediate
4469 // instruction. It is quite common for x86-64.
4470 if (MI.isSubregToReg()) {
4471 // We use following pattern to setup 64b immediate.
4472 // %8:gr32 = MOV32r0 implicit-def dead $eflags
4473 // %6:gr64 = SUBREG_TO_REG 0, killed %8:gr32, %subreg.sub_32bit
4474 if (!MI.getOperand(1).isImm())
4475 return false;
4476 unsigned FillBits = MI.getOperand(1).getImm();
4477 unsigned SubIdx = MI.getOperand(3).getImm();
4478 MovReg = MI.getOperand(2).getReg();
4479 if (SubIdx != X86::sub_32bit || FillBits != 0)
4480 return false;
4481 const MachineRegisterInfo &MRI = MI.getParent()->getParent()->getRegInfo();
4482 MovMI = MRI.getUniqueVRegDef(MovReg);
4483 if (!MovMI)
4484 return false;
4485 }
4486
4487 if (MovMI->getOpcode() == X86::MOV32r0 &&
4488 MovMI->getOperand(0).getReg() == MovReg) {
4489 ImmVal = 0;
4490 return true;
4491 }
4492
4493 if (MovMI->getOpcode() != X86::MOV32ri &&
4494 MovMI->getOpcode() != X86::MOV64ri &&
4495 MovMI->getOpcode() != X86::MOV32ri64 && MovMI->getOpcode() != X86::MOV8ri)
4496 return false;
4497 // Mov Src can be a global address.
4498 if (!MovMI->getOperand(1).isImm() || MovMI->getOperand(0).getReg() != MovReg)
4499 return false;
4500 ImmVal = MovMI->getOperand(1).getImm();
4501 return true;
4502}
4503
4505 const MachineInstr *MI, const Register NullValueReg,
4506 const TargetRegisterInfo *TRI) const {
4507 if (!MI->modifiesRegister(NullValueReg, TRI))
4508 return true;
4509 switch (MI->getOpcode()) {
4510 // Shift right/left of a null unto itself is still a null, i.e. rax = shl rax
4511 // X.
4512 case X86::SHR64ri:
4513 case X86::SHR32ri:
4514 case X86::SHL64ri:
4515 case X86::SHL32ri:
4516 assert(MI->getOperand(0).isDef() && MI->getOperand(1).isUse() &&
4517 "expected for shift opcode!");
4518 return MI->getOperand(0).getReg() == NullValueReg &&
4519 MI->getOperand(1).getReg() == NullValueReg;
4520 // Zero extend of a sub-reg of NullValueReg into itself does not change the
4521 // null value.
4522 case X86::MOV32rr:
4523 return llvm::all_of(MI->operands(), [&](const MachineOperand &MO) {
4524 return TRI->isSubRegisterEq(NullValueReg, MO.getReg());
4525 });
4526 default:
4527 return false;
4528 }
4529 llvm_unreachable("Should be handled above!");
4530}
4531
4534 int64_t &Offset, bool &OffsetIsScalable, LocationSize &Width,
4535 const TargetRegisterInfo *TRI) const {
4536 const MCInstrDesc &Desc = MemOp.getDesc();
4537 int MemRefBegin = X86II::getMemoryOperandNo(Desc.TSFlags);
4538 if (MemRefBegin < 0)
4539 return false;
4540
4541 MemRefBegin += X86II::getOperandBias(Desc);
4542
4543 const MachineOperand *BaseOp =
4544 &MemOp.getOperand(MemRefBegin + X86::AddrBaseReg);
4545 if (!BaseOp->isReg()) // Can be an MO_FrameIndex
4546 return false;
4547
4548 if (MemOp.getOperand(MemRefBegin + X86::AddrScaleAmt).getImm() != 1)
4549 return false;
4550
4551 if (MemOp.getOperand(MemRefBegin + X86::AddrIndexReg).getReg() !=
4552 X86::NoRegister)
4553 return false;
4554
4555 const MachineOperand &DispMO = MemOp.getOperand(MemRefBegin + X86::AddrDisp);
4556
4557 // Displacement can be symbolic
4558 if (!DispMO.isImm())
4559 return false;
4560
4561 Offset = DispMO.getImm();
4562
4563 if (!BaseOp->isReg())
4564 return false;
4565
4566 OffsetIsScalable = false;
4567 // FIXME: Relying on memoperands() may not be right thing to do here. Check
4568 // with X86 maintainers, and fix it accordingly. For now, it is ok, since
4569 // there is no use of `Width` for X86 back-end at the moment.
4570 Width =
4571 !MemOp.memoperands_empty() ? MemOp.memoperands().front()->getSize() : 0;
4572 BaseOps.push_back(BaseOp);
4573 return true;
4574}
4575
4576static unsigned getStoreRegOpcode(Register SrcReg,
4577 const TargetRegisterClass *RC,
4578 bool IsStackAligned,
4579 const X86Subtarget &STI) {
4580 return getLoadStoreRegOpcode(SrcReg, RC, IsStackAligned, STI, false);
4581}
4582
4583static unsigned getLoadRegOpcode(Register DestReg,
4584 const TargetRegisterClass *RC,
4585 bool IsStackAligned, const X86Subtarget &STI) {
4586 return getLoadStoreRegOpcode(DestReg, RC, IsStackAligned, STI, true);
4587}
4588
4589static bool isAMXOpcode(unsigned Opc) {
4590 switch (Opc) {
4591 default:
4592 return false;
4593 case X86::TILELOADD:
4594 case X86::TILESTORED:
4595 case X86::TILELOADD_EVEX:
4596 case X86::TILESTORED_EVEX:
4597 return true;
4598 }
4599}
4600
4603 unsigned Opc, Register Reg, int FrameIdx,
4604 bool isKill) const {
4605 switch (Opc) {
4606 default:
4607 llvm_unreachable("Unexpected special opcode!");
4608 case X86::TILESTORED:
4609 case X86::TILESTORED_EVEX: {
4610 // tilestored %tmm, (%sp, %idx)
4612 Register VirtReg = RegInfo.createVirtualRegister(&X86::GR64_NOSPRegClass);
4613 BuildMI(MBB, MI, DebugLoc(), get(X86::MOV64ri), VirtReg).addImm(64);
4614 MachineInstr *NewMI =
4615 addFrameReference(BuildMI(MBB, MI, DebugLoc(), get(Opc)), FrameIdx)
4616 .addReg(Reg, getKillRegState(isKill));
4618 MO.setReg(VirtReg);
4619 MO.setIsKill(true);
4620 break;
4621 }
4622 case X86::TILELOADD:
4623 case X86::TILELOADD_EVEX: {
4624 // tileloadd (%sp, %idx), %tmm
4626 Register VirtReg = RegInfo.createVirtualRegister(&X86::GR64_NOSPRegClass);
4627 BuildMI(MBB, MI, DebugLoc(), get(X86::MOV64ri), VirtReg).addImm(64);
4629 BuildMI(MBB, MI, DebugLoc(), get(Opc), Reg), FrameIdx);
4631 MO.setReg(VirtReg);
4632 MO.setIsKill(true);
4633 break;
4634 }
4635 }
4636}
4637
4640 bool isKill, int FrameIdx, const TargetRegisterClass *RC,
4641 const TargetRegisterInfo *TRI, Register VReg) const {
4642 const MachineFunction &MF = *MBB.getParent();
4643 const MachineFrameInfo &MFI = MF.getFrameInfo();
4644 assert(MFI.getObjectSize(FrameIdx) >= TRI->getSpillSize(*RC) &&
4645 "Stack slot too small for store");
4646
4647 unsigned Alignment = std::max<uint32_t>(TRI->getSpillSize(*RC), 16);
4648 bool isAligned =
4649 (Subtarget.getFrameLowering()->getStackAlign() >= Alignment) ||
4650 (RI.canRealignStack(MF) && !MFI.isFixedObjectIndex(FrameIdx));
4651
4652 unsigned Opc = getStoreRegOpcode(SrcReg, RC, isAligned, Subtarget);
4653 if (isAMXOpcode(Opc))
4654 loadStoreTileReg(MBB, MI, Opc, SrcReg, FrameIdx, isKill);
4655 else
4656 addFrameReference(BuildMI(MBB, MI, DebugLoc(), get(Opc)), FrameIdx)
4657 .addReg(SrcReg, getKillRegState(isKill));
4658}
4659
4662 Register DestReg, int FrameIdx,
4663 const TargetRegisterClass *RC,
4664 const TargetRegisterInfo *TRI,
4665 Register VReg) const {
4666 const MachineFunction &MF = *MBB.getParent();
4667 const MachineFrameInfo &MFI = MF.getFrameInfo();
4668 assert(MFI.getObjectSize(FrameIdx) >= TRI->getSpillSize(*RC) &&
4669 "Load size exceeds stack slot");
4670 unsigned Alignment = std::max<uint32_t>(TRI->getSpillSize(*RC), 16);
4671 bool isAligned =
4672 (Subtarget.getFrameLowering()->getStackAlign() >= Alignment) ||
4673 (RI.canRealignStack(MF) && !MFI.isFixedObjectIndex(FrameIdx));
4674
4675 unsigned Opc = getLoadRegOpcode(DestReg, RC, isAligned, Subtarget);
4676 if (isAMXOpcode(Opc))
4677 loadStoreTileReg(MBB, MI, Opc, DestReg, FrameIdx);
4678 else
4679 addFrameReference(BuildMI(MBB, MI, DebugLoc(), get(Opc), DestReg),
4680 FrameIdx);
4681}
4682
4684 Register &SrcReg2, int64_t &CmpMask,
4685 int64_t &CmpValue) const {
4686 switch (MI.getOpcode()) {
4687 default:
4688 break;
4689 case X86::CMP64ri32:
4690 case X86::CMP32ri:
4691 case X86::CMP16ri:
4692 case X86::CMP8ri:
4693 SrcReg = MI.getOperand(0).getReg();
4694 SrcReg2 = 0;
4695 if (MI.getOperand(1).isImm()) {
4696 CmpMask = ~0;
4697 CmpValue = MI.getOperand(1).getImm();
4698 } else {
4699 CmpMask = CmpValue = 0;
4700 }
4701 return true;
4702 // A SUB can be used to perform comparison.
4703 CASE_ND(SUB64rm)
4704 CASE_ND(SUB32rm)
4705 CASE_ND(SUB16rm)
4706 CASE_ND(SUB8rm)
4707 SrcReg = MI.getOperand(1).getReg();
4708 SrcReg2 = 0;
4709 CmpMask = 0;
4710 CmpValue = 0;
4711 return true;
4712 CASE_ND(SUB64rr)
4713 CASE_ND(SUB32rr)
4714 CASE_ND(SUB16rr)
4715 CASE_ND(SUB8rr)
4716 SrcReg = MI.getOperand(1).getReg();
4717 SrcReg2 = MI.getOperand(2).getReg();
4718 CmpMask = 0;
4719 CmpValue = 0;
4720 return true;
4721 CASE_ND(SUB64ri32)
4722 CASE_ND(SUB32ri)
4723 CASE_ND(SUB16ri)
4724 CASE_ND(SUB8ri)
4725 SrcReg = MI.getOperand(1).getReg();
4726 SrcReg2 = 0;
4727 if (MI.getOperand(2).isImm()) {
4728 CmpMask = ~0;
4729 CmpValue = MI.getOperand(2).getImm();
4730 } else {
4731 CmpMask = CmpValue = 0;
4732 }
4733 return true;
4734 case X86::CMP64rr:
4735 case X86::CMP32rr:
4736 case X86::CMP16rr:
4737 case X86::CMP8rr:
4738 SrcReg = MI.getOperand(0).getReg();
4739 SrcReg2 = MI.getOperand(1).getReg();
4740 CmpMask = 0;
4741 CmpValue = 0;
4742 return true;
4743 case X86::TEST8rr:
4744 case X86::TEST16rr:
4745 case X86::TEST32rr:
4746 case X86::TEST64rr:
4747 SrcReg = MI.getOperand(0).getReg();
4748 if (MI.getOperand(1).getReg() != SrcReg)
4749 return false;
4750 // Compare against zero.
4751 SrcReg2 = 0;
4752 CmpMask = ~0;
4753 CmpValue = 0;
4754 return true;
4755 }
4756 return false;
4757}
4758
4759bool X86InstrInfo::isRedundantFlagInstr(const MachineInstr &FlagI,
4760 Register SrcReg, Register SrcReg2,
4761 int64_t ImmMask, int64_t ImmValue,
4762 const MachineInstr &OI, bool *IsSwapped,
4763 int64_t *ImmDelta) const {
4764 switch (OI.getOpcode()) {
4765 case X86::CMP64rr:
4766 case X86::CMP32rr:
4767 case X86::CMP16rr:
4768 case X86::CMP8rr:
4769 CASE_ND(SUB64rr)
4770 CASE_ND(SUB32rr)
4771 CASE_ND(SUB16rr)
4772 CASE_ND(SUB8rr) {
4773 Register OISrcReg;
4774 Register OISrcReg2;
4775 int64_t OIMask;
4776 int64_t OIValue;
4777 if (!analyzeCompare(OI, OISrcReg, OISrcReg2, OIMask, OIValue) ||
4778 OIMask != ImmMask || OIValue != ImmValue)
4779 return false;
4780 if (SrcReg == OISrcReg && SrcReg2 == OISrcReg2) {
4781 *IsSwapped = false;
4782 return true;
4783 }
4784 if (SrcReg == OISrcReg2 && SrcReg2 == OISrcReg) {
4785 *IsSwapped = true;
4786 return true;
4787 }
4788 return false;
4789 }
4790 case X86::CMP64ri32:
4791 case X86::CMP32ri:
4792 case X86::CMP16ri:
4793 case X86::CMP8ri:
4794 CASE_ND(SUB64ri32)
4795 CASE_ND(SUB32ri)
4796 CASE_ND(SUB16ri)
4797 CASE_ND(SUB8ri)
4798 case X86::TEST64rr:
4799 case X86::TEST32rr:
4800 case X86::TEST16rr:
4801 case X86::TEST8rr: {
4802 if (ImmMask != 0) {
4803 Register OISrcReg;
4804 Register OISrcReg2;
4805 int64_t OIMask;
4806 int64_t OIValue;
4807 if (analyzeCompare(OI, OISrcReg, OISrcReg2, OIMask, OIValue) &&
4808 SrcReg == OISrcReg && ImmMask == OIMask) {
4809 if (OIValue == ImmValue) {
4810 *ImmDelta = 0;
4811 return true;
4812 } else if (static_cast<uint64_t>(ImmValue) ==
4813 static_cast<uint64_t>(OIValue) - 1) {
4814 *ImmDelta = -1;
4815 return true;
4816 } else if (static_cast<uint64_t>(ImmValue) ==
4817 static_cast<uint64_t>(OIValue) + 1) {
4818 *ImmDelta = 1;
4819 return true;
4820 } else {
4821 return false;
4822 }
4823 }
4824 }
4825 return FlagI.isIdenticalTo(OI);
4826 }
4827 default:
4828 return false;
4829 }
4830}
4831
4832/// Check whether the definition can be converted
4833/// to remove a comparison against zero.
4834inline static bool isDefConvertible(const MachineInstr &MI, bool &NoSignFlag,
4835 bool &ClearsOverflowFlag) {
4836 NoSignFlag = false;
4837 ClearsOverflowFlag = false;
4838
4839 // "ELF Handling for Thread-Local Storage" specifies that x86-64 GOTTPOFF, and
4840 // i386 GOTNTPOFF/INDNTPOFF relocations can convert an ADD to a LEA during
4841 // Initial Exec to Local Exec relaxation. In these cases, we must not depend
4842 // on the EFLAGS modification of ADD actually happening in the final binary.
4843 if (MI.getOpcode() == X86::ADD64rm || MI.getOpcode() == X86::ADD32rm) {
4844 unsigned Flags = MI.getOperand(5).getTargetFlags();
4845 if (Flags == X86II::MO_GOTTPOFF || Flags == X86II::MO_INDNTPOFF ||
4846 Flags == X86II::MO_GOTNTPOFF)
4847 return false;
4848 }
4849
4850 switch (MI.getOpcode()) {
4851 default:
4852 return false;
4853
4854 // The shift instructions only modify ZF if their shift count is non-zero.
4855 // N.B.: The processor truncates the shift count depending on the encoding.
4856 CASE_ND(SAR8ri)
4857 CASE_ND(SAR16ri)
4858 CASE_ND(SAR32ri)
4859 CASE_ND(SAR64ri)
4860 CASE_ND(SHR8ri)
4861 CASE_ND(SHR16ri)
4862 CASE_ND(SHR32ri)
4863 CASE_ND(SHR64ri)
4864 return getTruncatedShiftCount(MI, 2) != 0;
4865
4866 // Some left shift instructions can be turned into LEA instructions but only
4867 // if their flags aren't used. Avoid transforming such instructions.
4868 CASE_ND(SHL8ri)
4869 CASE_ND(SHL16ri)
4870 CASE_ND(SHL32ri)
4871 CASE_ND(SHL64ri) {
4872 unsigned ShAmt = getTruncatedShiftCount(MI, 2);
4873 if (isTruncatedShiftCountForLEA(ShAmt))
4874 return false;
4875 return ShAmt != 0;
4876 }
4877
4878 CASE_ND(SHRD16rri8)
4879 CASE_ND(SHRD32rri8)
4880 CASE_ND(SHRD64rri8)
4881 CASE_ND(SHLD16rri8)
4882 CASE_ND(SHLD32rri8)
4883 CASE_ND(SHLD64rri8)
4884 return getTruncatedShiftCount(MI, 3) != 0;
4885
4886 CASE_ND(SUB64ri32)
4887 CASE_ND(SUB32ri)
4888 CASE_ND(SUB16ri)
4889 CASE_ND(SUB8ri)
4890 CASE_ND(SUB64rr)
4891 CASE_ND(SUB32rr)
4892 CASE_ND(SUB16rr)
4893 CASE_ND(SUB8rr)
4894 CASE_ND(SUB64rm)
4895 CASE_ND(SUB32rm)
4896 CASE_ND(SUB16rm)
4897 CASE_ND(SUB8rm)
4898 CASE_ND(DEC64r)
4899 CASE_ND(DEC32r)
4900 CASE_ND(DEC16r)
4901 CASE_ND(DEC8r)
4902 CASE_ND(ADD64ri32)
4903 CASE_ND(ADD32ri)
4904 CASE_ND(ADD16ri)
4905 CASE_ND(ADD8ri)
4906 CASE_ND(ADD64rr)
4907 CASE_ND(ADD32rr)
4908 CASE_ND(ADD16rr)
4909 CASE_ND(ADD8rr)
4910 CASE_ND(ADD64rm)
4911 CASE_ND(ADD32rm)
4912 CASE_ND(ADD16rm)
4913 CASE_ND(ADD8rm)
4914 CASE_ND(INC64r)
4915 CASE_ND(INC32r)
4916 CASE_ND(INC16r)
4917 CASE_ND(INC8r)
4918 CASE_ND(ADC64ri32)
4919 CASE_ND(ADC32ri)
4920 CASE_ND(ADC16ri)
4921 CASE_ND(ADC8ri)
4922 CASE_ND(ADC64rr)
4923 CASE_ND(ADC32rr)
4924 CASE_ND(ADC16rr)
4925 CASE_ND(ADC8rr)
4926 CASE_ND(ADC64rm)
4927 CASE_ND(ADC32rm)
4928 CASE_ND(ADC16rm)
4929 CASE_ND(ADC8rm)
4930 CASE_ND(SBB64ri32)
4931 CASE_ND(SBB32ri)
4932 CASE_ND(SBB16ri)
4933 CASE_ND(SBB8ri)
4934 CASE_ND(SBB64rr)
4935 CASE_ND(SBB32rr)
4936 CASE_ND(SBB16rr)
4937 CASE_ND(SBB8rr)
4938 CASE_ND(SBB64rm)
4939 CASE_ND(SBB32rm)
4940 CASE_ND(SBB16rm)
4941 CASE_ND(SBB8rm)
4942 CASE_ND(NEG8r)
4943 CASE_ND(NEG16r)
4944 CASE_ND(NEG32r)
4945 CASE_ND(NEG64r)
4946 case X86::LZCNT16rr:
4947 case X86::LZCNT16rm:
4948 case X86::LZCNT32rr:
4949 case X86::LZCNT32rm:
4950 case X86::LZCNT64rr:
4951 case X86::LZCNT64rm:
4952 case X86::POPCNT16rr:
4953 case X86::POPCNT16rm:
4954 case X86::POPCNT32rr:
4955 case X86::POPCNT32rm:
4956 case X86::POPCNT64rr:
4957 case X86::POPCNT64rm:
4958 case X86::TZCNT16rr:
4959 case X86::TZCNT16rm:
4960 case X86::TZCNT32rr:
4961 case X86::TZCNT32rm:
4962 case X86::TZCNT64rr:
4963 case X86::TZCNT64rm:
4964 return true;
4965 CASE_ND(AND64ri32)
4966 CASE_ND(AND32ri)
4967 CASE_ND(AND16ri)
4968 CASE_ND(AND8ri)
4969 CASE_ND(AND64rr)
4970 CASE_ND(AND32rr)
4971 CASE_ND(AND16rr)
4972 CASE_ND(AND8rr)
4973 CASE_ND(AND64rm)
4974 CASE_ND(AND32rm)
4975 CASE_ND(AND16rm)
4976 CASE_ND(AND8rm)
4977 CASE_ND(XOR64ri32)
4978 CASE_ND(XOR32ri)
4979 CASE_ND(XOR16ri)
4980 CASE_ND(XOR8ri)
4981 CASE_ND(XOR64rr)
4982 CASE_ND(XOR32rr)
4983 CASE_ND(XOR16rr)
4984 CASE_ND(XOR8rr)
4985 CASE_ND(XOR64rm)
4986 CASE_ND(XOR32rm)
4987 CASE_ND(XOR16rm)
4988 CASE_ND(XOR8rm)
4989 CASE_ND(OR64ri32)
4990 CASE_ND(OR32ri)
4991 CASE_ND(OR16ri)
4992 CASE_ND(OR8ri)
4993 CASE_ND(OR64rr)
4994 CASE_ND(OR32rr)
4995 CASE_ND(OR16rr)
4996 CASE_ND(OR8rr)
4997 CASE_ND(OR64rm)
4998 CASE_ND(OR32rm)
4999 CASE_ND(OR16rm)
5000 CASE_ND(OR8rm)
5001 case X86::ANDN32rr:
5002 case X86::ANDN32rm:
5003 case X86::ANDN64rr:
5004 case X86::ANDN64rm:
5005 case X86::BLSI32rr:
5006 case X86::BLSI32rm:
5007 case X86::BLSI64rr:
5008 case X86::BLSI64rm:
5009 case X86::BLSMSK32rr:
5010 case X86::BLSMSK32rm:
5011 case X86::BLSMSK64rr:
5012 case X86::BLSMSK64rm:
5013 case X86::BLSR32rr:
5014 case X86::BLSR32rm:
5015 case X86::BLSR64rr:
5016 case X86::BLSR64rm:
5017 case X86::BLCFILL32rr:
5018 case X86::BLCFILL32rm:
5019 case X86::BLCFILL64rr:
5020 case X86::BLCFILL64rm:
5021 case X86::BLCI32rr:
5022 case X86::BLCI32rm:
5023 case X86::BLCI64rr:
5024 case X86::BLCI64rm:
5025 case X86::BLCIC32rr:
5026 case X86::BLCIC32rm:
5027 case X86::BLCIC64rr:
5028 case X86::BLCIC64rm:
5029 case X86::BLCMSK32rr:
5030 case X86::BLCMSK32rm:
5031 case X86::BLCMSK64rr:
5032 case X86::BLCMSK64rm:
5033 case X86::BLCS32rr:
5034 case X86::BLCS32rm:
5035 case X86::BLCS64rr:
5036 case X86::BLCS64rm:
5037 case X86::BLSFILL32rr:
5038 case X86::BLSFILL32rm:
5039 case X86::BLSFILL64rr:
5040 case X86::BLSFILL64rm:
5041 case X86::BLSIC32rr:
5042 case X86::BLSIC32rm:
5043 case X86::BLSIC64rr:
5044 case X86::BLSIC64rm:
5045 case X86::BZHI32rr:
5046 case X86::BZHI32rm:
5047 case X86::BZHI64rr:
5048 case X86::BZHI64rm:
5049 case X86::T1MSKC32rr:
5050 case X86::T1MSKC32rm:
5051 case X86::T1MSKC64rr:
5052 case X86::T1MSKC64rm:
5053 case X86::TZMSK32rr:
5054 case X86::TZMSK32rm:
5055 case X86::TZMSK64rr:
5056 case X86::TZMSK64rm:
5057 // These instructions clear the overflow flag just like TEST.
5058 // FIXME: These are not the only instructions in this switch that clear the
5059 // overflow flag.
5060 ClearsOverflowFlag = true;
5061 return true;
5062 case X86::BEXTR32rr:
5063 case X86::BEXTR64rr:
5064 case X86::BEXTR32rm:
5065 case X86::BEXTR64rm:
5066 case X86::BEXTRI32ri:
5067 case X86::BEXTRI32mi:
5068 case X86::BEXTRI64ri:
5069 case X86::BEXTRI64mi:
5070 // BEXTR doesn't update the sign flag so we can't use it. It does clear
5071 // the overflow flag, but that's not useful without the sign flag.
5072 NoSignFlag = true;
5073 return true;
5074 }
5075}
5076
5077/// Check whether the use can be converted to remove a comparison against zero.
5079 switch (MI.getOpcode()) {
5080 default:
5081 return X86::COND_INVALID;
5082 CASE_ND(NEG8r)
5083 CASE_ND(NEG16r)
5084 CASE_ND(NEG32r)
5085 CASE_ND(NEG64r)
5086 return X86::COND_AE;
5087 case X86::LZCNT16rr:
5088 case X86::LZCNT32rr:
5089 case X86::LZCNT64rr:
5090 return X86::COND_B;
5091 case X86::POPCNT16rr:
5092 case X86::POPCNT32rr:
5093 case X86::POPCNT64rr:
5094 return X86::COND_E;
5095 case X86::TZCNT16rr:
5096 case X86::TZCNT32rr:
5097 case X86::TZCNT64rr:
5098 return X86::COND_B;
5099 case X86::BSF16rr:
5100 case X86::BSF32rr:
5101 case X86::BSF64rr:
5102 case X86::BSR16rr:
5103 case X86::BSR32rr:
5104 case X86::BSR64rr:
5105 return X86::COND_E;
5106 case X86::BLSI32rr:
5107 case X86::BLSI64rr:
5108 return X86::COND_AE;
5109 case X86::BLSR32rr:
5110 case X86::BLSR64rr:
5111 case X86::BLSMSK32rr:
5112 case X86::BLSMSK64rr:
5113 return X86::COND_B;
5114 // TODO: TBM instructions.
5115 }
5116}
5117
5118/// Check if there exists an earlier instruction that
5119/// operates on the same source operands and sets flags in the same way as
5120/// Compare; remove Compare if possible.
5122 Register SrcReg2, int64_t CmpMask,
5123 int64_t CmpValue,
5124 const MachineRegisterInfo *MRI) const {
5125 // Check whether we can replace SUB with CMP.
5126 switch (CmpInstr.getOpcode()) {
5127 default:
5128 break;
5129 CASE_ND(SUB64ri32)
5130 CASE_ND(SUB32ri)
5131 CASE_ND(SUB16ri)
5132 CASE_ND(SUB8ri)
5133 CASE_ND(SUB64rm)
5134 CASE_ND(SUB32rm)
5135 CASE_ND(SUB16rm)
5136 CASE_ND(SUB8rm)
5137 CASE_ND(SUB64rr)
5138 CASE_ND(SUB32rr)
5139 CASE_ND(SUB16rr)
5140 CASE_ND(SUB8rr) {
5141 if (!MRI->use_nodbg_empty(CmpInstr.getOperand(0).getReg()))
5142 return false;
5143 // There is no use of the destination register, we can replace SUB with CMP.
5144 unsigned NewOpcode = 0;
5145#define FROM_TO(A, B) \
5146 CASE_ND(A) NewOpcode = X86::B; \
5147 break;
5148 switch (CmpInstr.getOpcode()) {
5149 default:
5150 llvm_unreachable("Unreachable!");
5151 FROM_TO(SUB64rm, CMP64rm)
5152 FROM_TO(SUB32rm, CMP32rm)
5153 FROM_TO(SUB16rm, CMP16rm)
5154 FROM_TO(SUB8rm, CMP8rm)
5155 FROM_TO(SUB64rr, CMP64rr)
5156 FROM_TO(SUB32rr, CMP32rr)
5157 FROM_TO(SUB16rr, CMP16rr)
5158 FROM_TO(SUB8rr, CMP8rr)
5159 FROM_TO(SUB64ri32, CMP64ri32)
5160 FROM_TO(SUB32ri, CMP32ri)
5161 FROM_TO(SUB16ri, CMP16ri)
5162 FROM_TO(SUB8ri, CMP8ri)
5163 }
5164#undef FROM_TO
5165 CmpInstr.setDesc(get(NewOpcode));
5166 CmpInstr.removeOperand(0);
5167 // Mutating this instruction invalidates any debug data associated with it.
5168 CmpInstr.dropDebugNumber();
5169 // Fall through to optimize Cmp if Cmp is CMPrr or CMPri.
5170 if (NewOpcode == X86::CMP64rm || NewOpcode == X86::CMP32rm ||
5171 NewOpcode == X86::CMP16rm || NewOpcode == X86::CMP8rm)
5172 return false;
5173 }
5174 }
5175
5176 // The following code tries to remove the comparison by re-using EFLAGS
5177 // from earlier instructions.
5178
5179 bool IsCmpZero = (CmpMask != 0 && CmpValue == 0);
5180
5181 // Transformation currently requires SSA values.
5182 if (SrcReg2.isPhysical())
5183 return false;
5184 MachineInstr *SrcRegDef = MRI->getVRegDef(SrcReg);
5185 assert(SrcRegDef && "Must have a definition (SSA)");
5186
5187 MachineInstr *MI = nullptr;
5188 MachineInstr *Sub = nullptr;
5189 MachineInstr *Movr0Inst = nullptr;
5190 bool NoSignFlag = false;
5191 bool ClearsOverflowFlag = false;
5192 bool ShouldUpdateCC = false;
5193 bool IsSwapped = false;
5195 int64_t ImmDelta = 0;
5196
5197 // Search backward from CmpInstr for the next instruction defining EFLAGS.
5199 MachineBasicBlock &CmpMBB = *CmpInstr.getParent();
5201 std::next(MachineBasicBlock::reverse_iterator(CmpInstr));
5202 for (MachineBasicBlock *MBB = &CmpMBB;;) {
5203 for (MachineInstr &Inst : make_range(From, MBB->rend())) {
5204 // Try to use EFLAGS from the instruction defining %SrcReg. Example:
5205 // %eax = addl ...
5206 // ... // EFLAGS not changed
5207 // testl %eax, %eax // <-- can be removed
5208 if (&Inst == SrcRegDef) {
5209 if (IsCmpZero &&
5210 isDefConvertible(Inst, NoSignFlag, ClearsOverflowFlag)) {
5211 MI = &Inst;
5212 break;
5213 }
5214
5215 // Look back for the following pattern, in which case the
5216 // test16rr/test64rr instruction could be erased.
5217 //
5218 // Example for test16rr:
5219 // %reg = and32ri %in_reg, 5
5220 // ... // EFLAGS not changed.
5221 // %src_reg = copy %reg.sub_16bit:gr32
5222 // test16rr %src_reg, %src_reg, implicit-def $eflags
5223 // Example for test64rr:
5224 // %reg = and32ri %in_reg, 5
5225 // ... // EFLAGS not changed.
5226 // %src_reg = subreg_to_reg 0, %reg, %subreg.sub_index
5227 // test64rr %src_reg, %src_reg, implicit-def $eflags
5228 MachineInstr *AndInstr = nullptr;
5229 if (IsCmpZero &&
5230 findRedundantFlagInstr(CmpInstr, Inst, MRI, &AndInstr, TRI,
5231 NoSignFlag, ClearsOverflowFlag)) {
5232 assert(AndInstr != nullptr && X86::isAND(AndInstr->getOpcode()));
5233 MI = AndInstr;
5234 break;
5235 }
5236 // Cannot find other candidates before definition of SrcReg.
5237 return false;
5238 }
5239
5240 if (Inst.modifiesRegister(X86::EFLAGS, TRI)) {
5241 // Try to use EFLAGS produced by an instruction reading %SrcReg.
5242 // Example:
5243 // %eax = ...
5244 // ...
5245 // popcntl %eax
5246 // ... // EFLAGS not changed
5247 // testl %eax, %eax // <-- can be removed
5248 if (IsCmpZero) {
5249 NewCC = isUseDefConvertible(Inst);
5250 if (NewCC != X86::COND_INVALID && Inst.getOperand(1).isReg() &&
5251 Inst.getOperand(1).getReg() == SrcReg) {
5252 ShouldUpdateCC = true;
5253 MI = &Inst;
5254 break;
5255 }
5256 }
5257
5258 // Try to use EFLAGS from an instruction with similar flag results.
5259 // Example:
5260 // sub x, y or cmp x, y
5261 // ... // EFLAGS not changed
5262 // cmp x, y // <-- can be removed
5263 if (isRedundantFlagInstr(CmpInstr, SrcReg, SrcReg2, CmpMask, CmpValue,
5264 Inst, &IsSwapped, &ImmDelta)) {
5265 Sub = &Inst;
5266 break;
5267 }
5268
5269 // MOV32r0 is implemented with xor which clobbers condition code. It is
5270 // safe to move up, if the definition to EFLAGS is dead and earlier
5271 // instructions do not read or write EFLAGS.
5272 if (!Movr0Inst && Inst.getOpcode() == X86::MOV32r0 &&
5273 Inst.registerDefIsDead(X86::EFLAGS, TRI)) {
5274 Movr0Inst = &Inst;
5275 continue;
5276 }
5277
5278 // Cannot do anything for any other EFLAG changes.
5279 return false;
5280 }
5281 }
5282
5283 if (MI || Sub)
5284 break;
5285
5286 // Reached begin of basic block. Continue in predecessor if there is
5287 // exactly one.
5288 if (MBB->pred_size() != 1)
5289 return false;
5290 MBB = *MBB->pred_begin();
5291 From = MBB->rbegin();
5292 }
5293
5294 // Scan forward from the instruction after CmpInstr for uses of EFLAGS.
5295 // It is safe to remove CmpInstr if EFLAGS is redefined or killed.
5296 // If we are done with the basic block, we need to check whether EFLAGS is
5297 // live-out.
5298 bool FlagsMayLiveOut = true;
5300 MachineBasicBlock::iterator AfterCmpInstr =
5301 std::next(MachineBasicBlock::iterator(CmpInstr));
5302 for (MachineInstr &Instr : make_range(AfterCmpInstr, CmpMBB.end())) {
5303 bool ModifyEFLAGS = Instr.modifiesRegister(X86::EFLAGS, TRI);
5304 bool UseEFLAGS = Instr.readsRegister(X86::EFLAGS, TRI);
5305 // We should check the usage if this instruction uses and updates EFLAGS.
5306 if (!UseEFLAGS && ModifyEFLAGS) {
5307 // It is safe to remove CmpInstr if EFLAGS is updated again.
5308 FlagsMayLiveOut = false;
5309 break;
5310 }
5311 if (!UseEFLAGS && !ModifyEFLAGS)
5312 continue;
5313
5314 // EFLAGS is used by this instruction.
5315 X86::CondCode OldCC = X86::getCondFromMI(Instr);
5316