LLVM 20.0.0git
X86InstrInfo.cpp
Go to the documentation of this file.
1//===-- X86InstrInfo.cpp - X86 Instruction Information --------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file contains the X86 implementation of the TargetInstrInfo class.
10//
11//===----------------------------------------------------------------------===//
12
13#include "X86InstrInfo.h"
14#include "X86.h"
15#include "X86InstrBuilder.h"
16#include "X86InstrFoldTables.h"
18#include "X86Subtarget.h"
19#include "X86TargetMachine.h"
20#include "llvm/ADT/STLExtras.h"
21#include "llvm/ADT/Sequence.h"
36#include "llvm/IR/Function.h"
37#include "llvm/IR/InstrTypes.h"
38#include "llvm/IR/Module.h"
39#include "llvm/MC/MCAsmInfo.h"
40#include "llvm/MC/MCExpr.h"
41#include "llvm/MC/MCInst.h"
43#include "llvm/Support/Debug.h"
47#include <optional>
48
49using namespace llvm;
50
51#define DEBUG_TYPE "x86-instr-info"
52
53#define GET_INSTRINFO_CTOR_DTOR
54#include "X86GenInstrInfo.inc"
55
56static cl::opt<bool>
57 NoFusing("disable-spill-fusing",
58 cl::desc("Disable fusing of spill code into instructions"),
60static cl::opt<bool>
61 PrintFailedFusing("print-failed-fuse-candidates",
62 cl::desc("Print instructions that the allocator wants to"
63 " fuse, but the X86 backend currently can't"),
65static cl::opt<bool>
66 ReMatPICStubLoad("remat-pic-stub-load",
67 cl::desc("Re-materialize load from stub in PIC mode"),
68 cl::init(false), cl::Hidden);
70 PartialRegUpdateClearance("partial-reg-update-clearance",
71 cl::desc("Clearance between two register writes "
72 "for inserting XOR to avoid partial "
73 "register update"),
74 cl::init(64), cl::Hidden);
76 "undef-reg-clearance",
77 cl::desc("How many idle instructions we would like before "
78 "certain undef register reads"),
79 cl::init(128), cl::Hidden);
80
81// Pin the vtable to this file.
82void X86InstrInfo::anchor() {}
83
85 : X86GenInstrInfo((STI.isTarget64BitLP64() ? X86::ADJCALLSTACKDOWN64
86 : X86::ADJCALLSTACKDOWN32),
87 (STI.isTarget64BitLP64() ? X86::ADJCALLSTACKUP64
88 : X86::ADJCALLSTACKUP32),
89 X86::CATCHRET, (STI.is64Bit() ? X86::RET64 : X86::RET32)),
90 Subtarget(STI), RI(STI.getTargetTriple()) {}
91
93X86InstrInfo::getRegClass(const MCInstrDesc &MCID, unsigned OpNum,
95 const MachineFunction &MF) const {
96 auto *RC = TargetInstrInfo::getRegClass(MCID, OpNum, TRI, MF);
97 // If the target does not have egpr, then r16-r31 will be resereved for all
98 // instructions.
99 if (!RC || !Subtarget.hasEGPR())
100 return RC;
101
103 return RC;
104
105 switch (RC->getID()) {
106 default:
107 return RC;
108 case X86::GR8RegClassID:
109 return &X86::GR8_NOREX2RegClass;
110 case X86::GR16RegClassID:
111 return &X86::GR16_NOREX2RegClass;
112 case X86::GR32RegClassID:
113 return &X86::GR32_NOREX2RegClass;
114 case X86::GR64RegClassID:
115 return &X86::GR64_NOREX2RegClass;
116 case X86::GR32_NOSPRegClassID:
117 return &X86::GR32_NOREX2_NOSPRegClass;
118 case X86::GR64_NOSPRegClassID:
119 return &X86::GR64_NOREX2_NOSPRegClass;
120 }
121}
122
124 Register &SrcReg, Register &DstReg,
125 unsigned &SubIdx) const {
126 switch (MI.getOpcode()) {
127 default:
128 break;
129 case X86::MOVSX16rr8:
130 case X86::MOVZX16rr8:
131 case X86::MOVSX32rr8:
132 case X86::MOVZX32rr8:
133 case X86::MOVSX64rr8:
134 if (!Subtarget.is64Bit())
135 // It's not always legal to reference the low 8-bit of the larger
136 // register in 32-bit mode.
137 return false;
138 [[fallthrough]];
139 case X86::MOVSX32rr16:
140 case X86::MOVZX32rr16:
141 case X86::MOVSX64rr16:
142 case X86::MOVSX64rr32: {
143 if (MI.getOperand(0).getSubReg() || MI.getOperand(1).getSubReg())
144 // Be conservative.
145 return false;
146 SrcReg = MI.getOperand(1).getReg();
147 DstReg = MI.getOperand(0).getReg();
148 switch (MI.getOpcode()) {
149 default:
150 llvm_unreachable("Unreachable!");
151 case X86::MOVSX16rr8:
152 case X86::MOVZX16rr8:
153 case X86::MOVSX32rr8:
154 case X86::MOVZX32rr8:
155 case X86::MOVSX64rr8:
156 SubIdx = X86::sub_8bit;
157 break;
158 case X86::MOVSX32rr16:
159 case X86::MOVZX32rr16:
160 case X86::MOVSX64rr16:
161 SubIdx = X86::sub_16bit;
162 break;
163 case X86::MOVSX64rr32:
164 SubIdx = X86::sub_32bit;
165 break;
166 }
167 return true;
168 }
169 }
170 return false;
171}
172
174 if (MI.mayLoad() || MI.mayStore())
175 return false;
176
177 // Some target-independent operations that trivially lower to data-invariant
178 // instructions.
179 if (MI.isCopyLike() || MI.isInsertSubreg())
180 return true;
181
182 unsigned Opcode = MI.getOpcode();
183 using namespace X86;
184 // On x86 it is believed that imul is constant time w.r.t. the loaded data.
185 // However, they set flags and are perhaps the most surprisingly constant
186 // time operations so we call them out here separately.
187 if (isIMUL(Opcode))
188 return true;
189 // Bit scanning and counting instructions that are somewhat surprisingly
190 // constant time as they scan across bits and do other fairly complex
191 // operations like popcnt, but are believed to be constant time on x86.
192 // However, these set flags.
193 if (isBSF(Opcode) || isBSR(Opcode) || isLZCNT(Opcode) || isPOPCNT(Opcode) ||
194 isTZCNT(Opcode))
195 return true;
196 // Bit manipulation instructions are effectively combinations of basic
197 // arithmetic ops, and should still execute in constant time. These also
198 // set flags.
199 if (isBLCFILL(Opcode) || isBLCI(Opcode) || isBLCIC(Opcode) ||
200 isBLCMSK(Opcode) || isBLCS(Opcode) || isBLSFILL(Opcode) ||
201 isBLSI(Opcode) || isBLSIC(Opcode) || isBLSMSK(Opcode) || isBLSR(Opcode) ||
202 isTZMSK(Opcode))
203 return true;
204 // Bit extracting and clearing instructions should execute in constant time,
205 // and set flags.
206 if (isBEXTR(Opcode) || isBZHI(Opcode))
207 return true;
208 // Shift and rotate.
209 if (isROL(Opcode) || isROR(Opcode) || isSAR(Opcode) || isSHL(Opcode) ||
210 isSHR(Opcode) || isSHLD(Opcode) || isSHRD(Opcode))
211 return true;
212 // Basic arithmetic is constant time on the input but does set flags.
213 if (isADC(Opcode) || isADD(Opcode) || isAND(Opcode) || isOR(Opcode) ||
214 isSBB(Opcode) || isSUB(Opcode) || isXOR(Opcode))
215 return true;
216 // Arithmetic with just 32-bit and 64-bit variants and no immediates.
217 if (isANDN(Opcode))
218 return true;
219 // Unary arithmetic operations.
220 if (isDEC(Opcode) || isINC(Opcode) || isNEG(Opcode))
221 return true;
222 // Unlike other arithmetic, NOT doesn't set EFLAGS.
223 if (isNOT(Opcode))
224 return true;
225 // Various move instructions used to zero or sign extend things. Note that we
226 // intentionally don't support the _NOREX variants as we can't handle that
227 // register constraint anyways.
228 if (isMOVSX(Opcode) || isMOVZX(Opcode) || isMOVSXD(Opcode) || isMOV(Opcode))
229 return true;
230 // Arithmetic instructions that are both constant time and don't set flags.
231 if (isRORX(Opcode) || isSARX(Opcode) || isSHLX(Opcode) || isSHRX(Opcode))
232 return true;
233 // LEA doesn't actually access memory, and its arithmetic is constant time.
234 if (isLEA(Opcode))
235 return true;
236 // By default, assume that the instruction is not data invariant.
237 return false;
238}
239
241 switch (MI.getOpcode()) {
242 default:
243 // By default, assume that the load will immediately leak.
244 return false;
245
246 // On x86 it is believed that imul is constant time w.r.t. the loaded data.
247 // However, they set flags and are perhaps the most surprisingly constant
248 // time operations so we call them out here separately.
249 case X86::IMUL16rm:
250 case X86::IMUL16rmi:
251 case X86::IMUL32rm:
252 case X86::IMUL32rmi:
253 case X86::IMUL64rm:
254 case X86::IMUL64rmi32:
255
256 // Bit scanning and counting instructions that are somewhat surprisingly
257 // constant time as they scan across bits and do other fairly complex
258 // operations like popcnt, but are believed to be constant time on x86.
259 // However, these set flags.
260 case X86::BSF16rm:
261 case X86::BSF32rm:
262 case X86::BSF64rm:
263 case X86::BSR16rm:
264 case X86::BSR32rm:
265 case X86::BSR64rm:
266 case X86::LZCNT16rm:
267 case X86::LZCNT32rm:
268 case X86::LZCNT64rm:
269 case X86::POPCNT16rm:
270 case X86::POPCNT32rm:
271 case X86::POPCNT64rm:
272 case X86::TZCNT16rm:
273 case X86::TZCNT32rm:
274 case X86::TZCNT64rm:
275
276 // Bit manipulation instructions are effectively combinations of basic
277 // arithmetic ops, and should still execute in constant time. These also
278 // set flags.
279 case X86::BLCFILL32rm:
280 case X86::BLCFILL64rm:
281 case X86::BLCI32rm:
282 case X86::BLCI64rm:
283 case X86::BLCIC32rm:
284 case X86::BLCIC64rm:
285 case X86::BLCMSK32rm:
286 case X86::BLCMSK64rm:
287 case X86::BLCS32rm:
288 case X86::BLCS64rm:
289 case X86::BLSFILL32rm:
290 case X86::BLSFILL64rm:
291 case X86::BLSI32rm:
292 case X86::BLSI64rm:
293 case X86::BLSIC32rm:
294 case X86::BLSIC64rm:
295 case X86::BLSMSK32rm:
296 case X86::BLSMSK64rm:
297 case X86::BLSR32rm:
298 case X86::BLSR64rm:
299 case X86::TZMSK32rm:
300 case X86::TZMSK64rm:
301
302 // Bit extracting and clearing instructions should execute in constant time,
303 // and set flags.
304 case X86::BEXTR32rm:
305 case X86::BEXTR64rm:
306 case X86::BEXTRI32mi:
307 case X86::BEXTRI64mi:
308 case X86::BZHI32rm:
309 case X86::BZHI64rm:
310
311 // Basic arithmetic is constant time on the input but does set flags.
312 case X86::ADC8rm:
313 case X86::ADC16rm:
314 case X86::ADC32rm:
315 case X86::ADC64rm:
316 case X86::ADD8rm:
317 case X86::ADD16rm:
318 case X86::ADD32rm:
319 case X86::ADD64rm:
320 case X86::AND8rm:
321 case X86::AND16rm:
322 case X86::AND32rm:
323 case X86::AND64rm:
324 case X86::ANDN32rm:
325 case X86::ANDN64rm:
326 case X86::OR8rm:
327 case X86::OR16rm:
328 case X86::OR32rm:
329 case X86::OR64rm:
330 case X86::SBB8rm:
331 case X86::SBB16rm:
332 case X86::SBB32rm:
333 case X86::SBB64rm:
334 case X86::SUB8rm:
335 case X86::SUB16rm:
336 case X86::SUB32rm:
337 case X86::SUB64rm:
338 case X86::XOR8rm:
339 case X86::XOR16rm:
340 case X86::XOR32rm:
341 case X86::XOR64rm:
342
343 // Integer multiply w/o affecting flags is still believed to be constant
344 // time on x86. Called out separately as this is among the most surprising
345 // instructions to exhibit that behavior.
346 case X86::MULX32rm:
347 case X86::MULX64rm:
348
349 // Arithmetic instructions that are both constant time and don't set flags.
350 case X86::RORX32mi:
351 case X86::RORX64mi:
352 case X86::SARX32rm:
353 case X86::SARX64rm:
354 case X86::SHLX32rm:
355 case X86::SHLX64rm:
356 case X86::SHRX32rm:
357 case X86::SHRX64rm:
358
359 // Conversions are believed to be constant time and don't set flags.
360 case X86::CVTTSD2SI64rm:
361 case X86::VCVTTSD2SI64rm:
362 case X86::VCVTTSD2SI64Zrm:
363 case X86::CVTTSD2SIrm:
364 case X86::VCVTTSD2SIrm:
365 case X86::VCVTTSD2SIZrm:
366 case X86::CVTTSS2SI64rm:
367 case X86::VCVTTSS2SI64rm:
368 case X86::VCVTTSS2SI64Zrm:
369 case X86::CVTTSS2SIrm:
370 case X86::VCVTTSS2SIrm:
371 case X86::VCVTTSS2SIZrm:
372 case X86::CVTSI2SDrm:
373 case X86::VCVTSI2SDrm:
374 case X86::VCVTSI2SDZrm:
375 case X86::CVTSI2SSrm:
376 case X86::VCVTSI2SSrm:
377 case X86::VCVTSI2SSZrm:
378 case X86::CVTSI642SDrm:
379 case X86::VCVTSI642SDrm:
380 case X86::VCVTSI642SDZrm:
381 case X86::CVTSI642SSrm:
382 case X86::VCVTSI642SSrm:
383 case X86::VCVTSI642SSZrm:
384 case X86::CVTSS2SDrm:
385 case X86::VCVTSS2SDrm:
386 case X86::VCVTSS2SDZrm:
387 case X86::CVTSD2SSrm:
388 case X86::VCVTSD2SSrm:
389 case X86::VCVTSD2SSZrm:
390 // AVX512 added unsigned integer conversions.
391 case X86::VCVTTSD2USI64Zrm:
392 case X86::VCVTTSD2USIZrm:
393 case X86::VCVTTSS2USI64Zrm:
394 case X86::VCVTTSS2USIZrm:
395 case X86::VCVTUSI2SDZrm:
396 case X86::VCVTUSI642SDZrm:
397 case X86::VCVTUSI2SSZrm:
398 case X86::VCVTUSI642SSZrm:
399
400 // Loads to register don't set flags.
401 case X86::MOV8rm:
402 case X86::MOV8rm_NOREX:
403 case X86::MOV16rm:
404 case X86::MOV32rm:
405 case X86::MOV64rm:
406 case X86::MOVSX16rm8:
407 case X86::MOVSX32rm16:
408 case X86::MOVSX32rm8:
409 case X86::MOVSX32rm8_NOREX:
410 case X86::MOVSX64rm16:
411 case X86::MOVSX64rm32:
412 case X86::MOVSX64rm8:
413 case X86::MOVZX16rm8:
414 case X86::MOVZX32rm16:
415 case X86::MOVZX32rm8:
416 case X86::MOVZX32rm8_NOREX:
417 case X86::MOVZX64rm16:
418 case X86::MOVZX64rm8:
419 return true;
420 }
421}
422
424 const MachineFunction *MF = MI.getParent()->getParent();
426
427 if (isFrameInstr(MI)) {
428 int SPAdj = alignTo(getFrameSize(MI), TFI->getStackAlign());
429 SPAdj -= getFrameAdjustment(MI);
430 if (!isFrameSetup(MI))
431 SPAdj = -SPAdj;
432 return SPAdj;
433 }
434
435 // To know whether a call adjusts the stack, we need information
436 // that is bound to the following ADJCALLSTACKUP pseudo.
437 // Look for the next ADJCALLSTACKUP that follows the call.
438 if (MI.isCall()) {
439 const MachineBasicBlock *MBB = MI.getParent();
441 for (auto E = MBB->end(); I != E; ++I) {
442 if (I->getOpcode() == getCallFrameDestroyOpcode() || I->isCall())
443 break;
444 }
445
446 // If we could not find a frame destroy opcode, then it has already
447 // been simplified, so we don't care.
448 if (I->getOpcode() != getCallFrameDestroyOpcode())
449 return 0;
450
451 return -(I->getOperand(1).getImm());
452 }
453
454 // Currently handle only PUSHes we can reasonably expect to see
455 // in call sequences
456 switch (MI.getOpcode()) {
457 default:
458 return 0;
459 case X86::PUSH32r:
460 case X86::PUSH32rmm:
461 case X86::PUSH32rmr:
462 case X86::PUSH32i:
463 return 4;
464 case X86::PUSH64r:
465 case X86::PUSH64rmm:
466 case X86::PUSH64rmr:
467 case X86::PUSH64i32:
468 return 8;
469 }
470}
471
472/// Return true and the FrameIndex if the specified
473/// operand and follow operands form a reference to the stack frame.
474bool X86InstrInfo::isFrameOperand(const MachineInstr &MI, unsigned int Op,
475 int &FrameIndex) const {
476 if (MI.getOperand(Op + X86::AddrBaseReg).isFI() &&
477 MI.getOperand(Op + X86::AddrScaleAmt).isImm() &&
478 MI.getOperand(Op + X86::AddrIndexReg).isReg() &&
479 MI.getOperand(Op + X86::AddrDisp).isImm() &&
480 MI.getOperand(Op + X86::AddrScaleAmt).getImm() == 1 &&
481 MI.getOperand(Op + X86::AddrIndexReg).getReg() == 0 &&
482 MI.getOperand(Op + X86::AddrDisp).getImm() == 0) {
483 FrameIndex = MI.getOperand(Op + X86::AddrBaseReg).getIndex();
484 return true;
485 }
486 return false;
487}
488
489static bool isFrameLoadOpcode(int Opcode, unsigned &MemBytes) {
490 switch (Opcode) {
491 default:
492 return false;
493 case X86::MOV8rm:
494 case X86::KMOVBkm:
495 case X86::KMOVBkm_EVEX:
496 MemBytes = 1;
497 return true;
498 case X86::MOV16rm:
499 case X86::KMOVWkm:
500 case X86::KMOVWkm_EVEX:
501 case X86::VMOVSHZrm:
502 case X86::VMOVSHZrm_alt:
503 MemBytes = 2;
504 return true;
505 case X86::MOV32rm:
506 case X86::MOVSSrm:
507 case X86::MOVSSrm_alt:
508 case X86::VMOVSSrm:
509 case X86::VMOVSSrm_alt:
510 case X86::VMOVSSZrm:
511 case X86::VMOVSSZrm_alt:
512 case X86::KMOVDkm:
513 case X86::KMOVDkm_EVEX:
514 MemBytes = 4;
515 return true;
516 case X86::MOV64rm:
517 case X86::LD_Fp64m:
518 case X86::MOVSDrm:
519 case X86::MOVSDrm_alt:
520 case X86::VMOVSDrm:
521 case X86::VMOVSDrm_alt:
522 case X86::VMOVSDZrm:
523 case X86::VMOVSDZrm_alt:
524 case X86::MMX_MOVD64rm:
525 case X86::MMX_MOVQ64rm:
526 case X86::KMOVQkm:
527 case X86::KMOVQkm_EVEX:
528 MemBytes = 8;
529 return true;
530 case X86::MOVAPSrm:
531 case X86::MOVUPSrm:
532 case X86::MOVAPDrm:
533 case X86::MOVUPDrm:
534 case X86::MOVDQArm:
535 case X86::MOVDQUrm:
536 case X86::VMOVAPSrm:
537 case X86::VMOVUPSrm:
538 case X86::VMOVAPDrm:
539 case X86::VMOVUPDrm:
540 case X86::VMOVDQArm:
541 case X86::VMOVDQUrm:
542 case X86::VMOVAPSZ128rm:
543 case X86::VMOVUPSZ128rm:
544 case X86::VMOVAPSZ128rm_NOVLX:
545 case X86::VMOVUPSZ128rm_NOVLX:
546 case X86::VMOVAPDZ128rm:
547 case X86::VMOVUPDZ128rm:
548 case X86::VMOVDQU8Z128rm:
549 case X86::VMOVDQU16Z128rm:
550 case X86::VMOVDQA32Z128rm:
551 case X86::VMOVDQU32Z128rm:
552 case X86::VMOVDQA64Z128rm:
553 case X86::VMOVDQU64Z128rm:
554 MemBytes = 16;
555 return true;
556 case X86::VMOVAPSYrm:
557 case X86::VMOVUPSYrm:
558 case X86::VMOVAPDYrm:
559 case X86::VMOVUPDYrm:
560 case X86::VMOVDQAYrm:
561 case X86::VMOVDQUYrm:
562 case X86::VMOVAPSZ256rm:
563 case X86::VMOVUPSZ256rm:
564 case X86::VMOVAPSZ256rm_NOVLX:
565 case X86::VMOVUPSZ256rm_NOVLX:
566 case X86::VMOVAPDZ256rm:
567 case X86::VMOVUPDZ256rm:
568 case X86::VMOVDQU8Z256rm:
569 case X86::VMOVDQU16Z256rm:
570 case X86::VMOVDQA32Z256rm:
571 case X86::VMOVDQU32Z256rm:
572 case X86::VMOVDQA64Z256rm:
573 case X86::VMOVDQU64Z256rm:
574 MemBytes = 32;
575 return true;
576 case X86::VMOVAPSZrm:
577 case X86::VMOVUPSZrm:
578 case X86::VMOVAPDZrm:
579 case X86::VMOVUPDZrm:
580 case X86::VMOVDQU8Zrm:
581 case X86::VMOVDQU16Zrm:
582 case X86::VMOVDQA32Zrm:
583 case X86::VMOVDQU32Zrm:
584 case X86::VMOVDQA64Zrm:
585 case X86::VMOVDQU64Zrm:
586 MemBytes = 64;
587 return true;
588 }
589}
590
591static bool isFrameStoreOpcode(int Opcode, unsigned &MemBytes) {
592 switch (Opcode) {
593 default:
594 return false;
595 case X86::MOV8mr:
596 case X86::KMOVBmk:
597 case X86::KMOVBmk_EVEX:
598 MemBytes = 1;
599 return true;
600 case X86::MOV16mr:
601 case X86::KMOVWmk:
602 case X86::KMOVWmk_EVEX:
603 case X86::VMOVSHZmr:
604 MemBytes = 2;
605 return true;
606 case X86::MOV32mr:
607 case X86::MOVSSmr:
608 case X86::VMOVSSmr:
609 case X86::VMOVSSZmr:
610 case X86::KMOVDmk:
611 case X86::KMOVDmk_EVEX:
612 MemBytes = 4;
613 return true;
614 case X86::MOV64mr:
615 case X86::ST_FpP64m:
616 case X86::MOVSDmr:
617 case X86::VMOVSDmr:
618 case X86::VMOVSDZmr:
619 case X86::MMX_MOVD64mr:
620 case X86::MMX_MOVQ64mr:
621 case X86::MMX_MOVNTQmr:
622 case X86::KMOVQmk:
623 case X86::KMOVQmk_EVEX:
624 MemBytes = 8;
625 return true;
626 case X86::MOVAPSmr:
627 case X86::MOVUPSmr:
628 case X86::MOVAPDmr:
629 case X86::MOVUPDmr:
630 case X86::MOVDQAmr:
631 case X86::MOVDQUmr:
632 case X86::VMOVAPSmr:
633 case X86::VMOVUPSmr:
634 case X86::VMOVAPDmr:
635 case X86::VMOVUPDmr:
636 case X86::VMOVDQAmr:
637 case X86::VMOVDQUmr:
638 case X86::VMOVUPSZ128mr:
639 case X86::VMOVAPSZ128mr:
640 case X86::VMOVUPSZ128mr_NOVLX:
641 case X86::VMOVAPSZ128mr_NOVLX:
642 case X86::VMOVUPDZ128mr:
643 case X86::VMOVAPDZ128mr:
644 case X86::VMOVDQA32Z128mr:
645 case X86::VMOVDQU32Z128mr:
646 case X86::VMOVDQA64Z128mr:
647 case X86::VMOVDQU64Z128mr:
648 case X86::VMOVDQU8Z128mr:
649 case X86::VMOVDQU16Z128mr:
650 MemBytes = 16;
651 return true;
652 case X86::VMOVUPSYmr:
653 case X86::VMOVAPSYmr:
654 case X86::VMOVUPDYmr:
655 case X86::VMOVAPDYmr:
656 case X86::VMOVDQUYmr:
657 case X86::VMOVDQAYmr:
658 case X86::VMOVUPSZ256mr:
659 case X86::VMOVAPSZ256mr:
660 case X86::VMOVUPSZ256mr_NOVLX:
661 case X86::VMOVAPSZ256mr_NOVLX:
662 case X86::VMOVUPDZ256mr:
663 case X86::VMOVAPDZ256mr:
664 case X86::VMOVDQU8Z256mr:
665 case X86::VMOVDQU16Z256mr:
666 case X86::VMOVDQA32Z256mr:
667 case X86::VMOVDQU32Z256mr:
668 case X86::VMOVDQA64Z256mr:
669 case X86::VMOVDQU64Z256mr:
670 MemBytes = 32;
671 return true;
672 case X86::VMOVUPSZmr:
673 case X86::VMOVAPSZmr:
674 case X86::VMOVUPDZmr:
675 case X86::VMOVAPDZmr:
676 case X86::VMOVDQU8Zmr:
677 case X86::VMOVDQU16Zmr:
678 case X86::VMOVDQA32Zmr:
679 case X86::VMOVDQU32Zmr:
680 case X86::VMOVDQA64Zmr:
681 case X86::VMOVDQU64Zmr:
682 MemBytes = 64;
683 return true;
684 }
685 return false;
686}
687
689 int &FrameIndex) const {
690 unsigned Dummy;
691 return X86InstrInfo::isLoadFromStackSlot(MI, FrameIndex, Dummy);
692}
693
695 int &FrameIndex,
696 unsigned &MemBytes) const {
697 if (isFrameLoadOpcode(MI.getOpcode(), MemBytes))
698 if (MI.getOperand(0).getSubReg() == 0 && isFrameOperand(MI, 1, FrameIndex))
699 return MI.getOperand(0).getReg();
700 return 0;
701}
702
704 int &FrameIndex) const {
705 unsigned Dummy;
706 if (isFrameLoadOpcode(MI.getOpcode(), Dummy)) {
707 unsigned Reg;
708 if ((Reg = isLoadFromStackSlot(MI, FrameIndex)))
709 return Reg;
710 // Check for post-frame index elimination operations
712 if (hasLoadFromStackSlot(MI, Accesses)) {
713 FrameIndex =
714 cast<FixedStackPseudoSourceValue>(Accesses.front()->getPseudoValue())
715 ->getFrameIndex();
716 return MI.getOperand(0).getReg();
717 }
718 }
719 return 0;
720}
721
723 int &FrameIndex) const {
724 unsigned Dummy;
725 return X86InstrInfo::isStoreToStackSlot(MI, FrameIndex, Dummy);
726}
727
729 int &FrameIndex,
730 unsigned &MemBytes) const {
731 if (isFrameStoreOpcode(MI.getOpcode(), MemBytes))
732 if (MI.getOperand(X86::AddrNumOperands).getSubReg() == 0 &&
733 isFrameOperand(MI, 0, FrameIndex))
734 return MI.getOperand(X86::AddrNumOperands).getReg();
735 return 0;
736}
737
739 int &FrameIndex) const {
740 unsigned Dummy;
741 if (isFrameStoreOpcode(MI.getOpcode(), Dummy)) {
742 unsigned Reg;
743 if ((Reg = isStoreToStackSlot(MI, FrameIndex)))
744 return Reg;
745 // Check for post-frame index elimination operations
747 if (hasStoreToStackSlot(MI, Accesses)) {
748 FrameIndex =
749 cast<FixedStackPseudoSourceValue>(Accesses.front()->getPseudoValue())
750 ->getFrameIndex();
751 return MI.getOperand(X86::AddrNumOperands).getReg();
752 }
753 }
754 return 0;
755}
756
757/// Return true if register is PIC base; i.e.g defined by X86::MOVPC32r.
758static bool regIsPICBase(Register BaseReg, const MachineRegisterInfo &MRI) {
759 // Don't waste compile time scanning use-def chains of physregs.
760 if (!BaseReg.isVirtual())
761 return false;
762 bool isPICBase = false;
763 for (const MachineInstr &DefMI : MRI.def_instructions(BaseReg)) {
764 if (DefMI.getOpcode() != X86::MOVPC32r)
765 return false;
766 assert(!isPICBase && "More than one PIC base?");
767 isPICBase = true;
768 }
769 return isPICBase;
770}
771
773 const MachineInstr &MI) const {
774 switch (MI.getOpcode()) {
775 default:
776 // This function should only be called for opcodes with the ReMaterializable
777 // flag set.
778 llvm_unreachable("Unknown rematerializable operation!");
779 break;
780 case X86::IMPLICIT_DEF:
781 // Defer to generic logic.
782 break;
783 case X86::LOAD_STACK_GUARD:
784 case X86::LD_Fp032:
785 case X86::LD_Fp064:
786 case X86::LD_Fp080:
787 case X86::LD_Fp132:
788 case X86::LD_Fp164:
789 case X86::LD_Fp180:
790 case X86::AVX1_SETALLONES:
791 case X86::AVX2_SETALLONES:
792 case X86::AVX512_128_SET0:
793 case X86::AVX512_256_SET0:
794 case X86::AVX512_512_SET0:
795 case X86::AVX512_512_SETALLONES:
796 case X86::AVX512_FsFLD0SD:
797 case X86::AVX512_FsFLD0SH:
798 case X86::AVX512_FsFLD0SS:
799 case X86::AVX512_FsFLD0F128:
800 case X86::AVX_SET0:
801 case X86::FsFLD0SD:
802 case X86::FsFLD0SS:
803 case X86::FsFLD0SH:
804 case X86::FsFLD0F128:
805 case X86::KSET0D:
806 case X86::KSET0Q:
807 case X86::KSET0W:
808 case X86::KSET1D:
809 case X86::KSET1Q:
810 case X86::KSET1W:
811 case X86::MMX_SET0:
812 case X86::MOV32ImmSExti8:
813 case X86::MOV32r0:
814 case X86::MOV32r1:
815 case X86::MOV32r_1:
816 case X86::MOV32ri64:
817 case X86::MOV64ImmSExti8:
818 case X86::V_SET0:
819 case X86::V_SETALLONES:
820 case X86::MOV16ri:
821 case X86::MOV32ri:
822 case X86::MOV64ri:
823 case X86::MOV64ri32:
824 case X86::MOV8ri:
825 case X86::PTILEZEROV:
826 return true;
827
828 case X86::MOV8rm:
829 case X86::MOV8rm_NOREX:
830 case X86::MOV16rm:
831 case X86::MOV32rm:
832 case X86::MOV64rm:
833 case X86::MOVSSrm:
834 case X86::MOVSSrm_alt:
835 case X86::MOVSDrm:
836 case X86::MOVSDrm_alt:
837 case X86::MOVAPSrm:
838 case X86::MOVUPSrm:
839 case X86::MOVAPDrm:
840 case X86::MOVUPDrm:
841 case X86::MOVDQArm:
842 case X86::MOVDQUrm:
843 case X86::VMOVSSrm:
844 case X86::VMOVSSrm_alt:
845 case X86::VMOVSDrm:
846 case X86::VMOVSDrm_alt:
847 case X86::VMOVAPSrm:
848 case X86::VMOVUPSrm:
849 case X86::VMOVAPDrm:
850 case X86::VMOVUPDrm:
851 case X86::VMOVDQArm:
852 case X86::VMOVDQUrm:
853 case X86::VMOVAPSYrm:
854 case X86::VMOVUPSYrm:
855 case X86::VMOVAPDYrm:
856 case X86::VMOVUPDYrm:
857 case X86::VMOVDQAYrm:
858 case X86::VMOVDQUYrm:
859 case X86::MMX_MOVD64rm:
860 case X86::MMX_MOVQ64rm:
861 case X86::VBROADCASTSSrm:
862 case X86::VBROADCASTSSYrm:
863 case X86::VBROADCASTSDYrm:
864 // AVX-512
865 case X86::VPBROADCASTBZ128rm:
866 case X86::VPBROADCASTBZ256rm:
867 case X86::VPBROADCASTBZrm:
868 case X86::VBROADCASTF32X2Z256rm:
869 case X86::VBROADCASTF32X2Zrm:
870 case X86::VBROADCASTI32X2Z128rm:
871 case X86::VBROADCASTI32X2Z256rm:
872 case X86::VBROADCASTI32X2Zrm:
873 case X86::VPBROADCASTWZ128rm:
874 case X86::VPBROADCASTWZ256rm:
875 case X86::VPBROADCASTWZrm:
876 case X86::VPBROADCASTDZ128rm:
877 case X86::VPBROADCASTDZ256rm:
878 case X86::VPBROADCASTDZrm:
879 case X86::VBROADCASTSSZ128rm:
880 case X86::VBROADCASTSSZ256rm:
881 case X86::VBROADCASTSSZrm:
882 case X86::VPBROADCASTQZ128rm:
883 case X86::VPBROADCASTQZ256rm:
884 case X86::VPBROADCASTQZrm:
885 case X86::VBROADCASTSDZ256rm:
886 case X86::VBROADCASTSDZrm:
887 case X86::VMOVSSZrm:
888 case X86::VMOVSSZrm_alt:
889 case X86::VMOVSDZrm:
890 case X86::VMOVSDZrm_alt:
891 case X86::VMOVSHZrm:
892 case X86::VMOVSHZrm_alt:
893 case X86::VMOVAPDZ128rm:
894 case X86::VMOVAPDZ256rm:
895 case X86::VMOVAPDZrm:
896 case X86::VMOVAPSZ128rm:
897 case X86::VMOVAPSZ256rm:
898 case X86::VMOVAPSZ128rm_NOVLX:
899 case X86::VMOVAPSZ256rm_NOVLX:
900 case X86::VMOVAPSZrm:
901 case X86::VMOVDQA32Z128rm:
902 case X86::VMOVDQA32Z256rm:
903 case X86::VMOVDQA32Zrm:
904 case X86::VMOVDQA64Z128rm:
905 case X86::VMOVDQA64Z256rm:
906 case X86::VMOVDQA64Zrm:
907 case X86::VMOVDQU16Z128rm:
908 case X86::VMOVDQU16Z256rm:
909 case X86::VMOVDQU16Zrm:
910 case X86::VMOVDQU32Z128rm:
911 case X86::VMOVDQU32Z256rm:
912 case X86::VMOVDQU32Zrm:
913 case X86::VMOVDQU64Z128rm:
914 case X86::VMOVDQU64Z256rm:
915 case X86::VMOVDQU64Zrm:
916 case X86::VMOVDQU8Z128rm:
917 case X86::VMOVDQU8Z256rm:
918 case X86::VMOVDQU8Zrm:
919 case X86::VMOVUPDZ128rm:
920 case X86::VMOVUPDZ256rm:
921 case X86::VMOVUPDZrm:
922 case X86::VMOVUPSZ128rm:
923 case X86::VMOVUPSZ256rm:
924 case X86::VMOVUPSZ128rm_NOVLX:
925 case X86::VMOVUPSZ256rm_NOVLX:
926 case X86::VMOVUPSZrm: {
927 // Loads from constant pools are trivially rematerializable.
928 if (MI.getOperand(1 + X86::AddrBaseReg).isReg() &&
929 MI.getOperand(1 + X86::AddrScaleAmt).isImm() &&
930 MI.getOperand(1 + X86::AddrIndexReg).isReg() &&
931 MI.getOperand(1 + X86::AddrIndexReg).getReg() == 0 &&
932 MI.isDereferenceableInvariantLoad()) {
933 Register BaseReg = MI.getOperand(1 + X86::AddrBaseReg).getReg();
934 if (BaseReg == 0 || BaseReg == X86::RIP)
935 return true;
936 // Allow re-materialization of PIC load.
937 if (!(!ReMatPICStubLoad && MI.getOperand(1 + X86::AddrDisp).isGlobal())) {
938 const MachineFunction &MF = *MI.getParent()->getParent();
939 const MachineRegisterInfo &MRI = MF.getRegInfo();
940 if (regIsPICBase(BaseReg, MRI))
941 return true;
942 }
943 }
944 break;
945 }
946
947 case X86::LEA32r:
948 case X86::LEA64r: {
949 if (MI.getOperand(1 + X86::AddrScaleAmt).isImm() &&
950 MI.getOperand(1 + X86::AddrIndexReg).isReg() &&
951 MI.getOperand(1 + X86::AddrIndexReg).getReg() == 0 &&
952 !MI.getOperand(1 + X86::AddrDisp).isReg()) {
953 // lea fi#, lea GV, etc. are all rematerializable.
954 if (!MI.getOperand(1 + X86::AddrBaseReg).isReg())
955 return true;
956 Register BaseReg = MI.getOperand(1 + X86::AddrBaseReg).getReg();
957 if (BaseReg == 0)
958 return true;
959 // Allow re-materialization of lea PICBase + x.
960 const MachineFunction &MF = *MI.getParent()->getParent();
961 const MachineRegisterInfo &MRI = MF.getRegInfo();
962 if (regIsPICBase(BaseReg, MRI))
963 return true;
964 }
965 break;
966 }
967 }
969}
970
973 Register DestReg, unsigned SubIdx,
974 const MachineInstr &Orig,
975 const TargetRegisterInfo &TRI) const {
976 bool ClobbersEFLAGS = Orig.modifiesRegister(X86::EFLAGS, &TRI);
977 if (ClobbersEFLAGS && MBB.computeRegisterLiveness(&TRI, X86::EFLAGS, I) !=
979 // The instruction clobbers EFLAGS. Re-materialize as MOV32ri to avoid side
980 // effects.
981 int Value;
982 switch (Orig.getOpcode()) {
983 case X86::MOV32r0:
984 Value = 0;
985 break;
986 case X86::MOV32r1:
987 Value = 1;
988 break;
989 case X86::MOV32r_1:
990 Value = -1;
991 break;
992 default:
993 llvm_unreachable("Unexpected instruction!");
994 }
995
996 const DebugLoc &DL = Orig.getDebugLoc();
997 BuildMI(MBB, I, DL, get(X86::MOV32ri))
998 .add(Orig.getOperand(0))
999 .addImm(Value);
1000 } else {
1002 MBB.insert(I, MI);
1003 }
1004
1005 MachineInstr &NewMI = *std::prev(I);
1006 NewMI.substituteRegister(Orig.getOperand(0).getReg(), DestReg, SubIdx, TRI);
1007}
1008
1009/// True if MI has a condition code def, e.g. EFLAGS, that is not marked dead.
1011 for (const MachineOperand &MO : MI.operands()) {
1012 if (MO.isReg() && MO.isDef() && MO.getReg() == X86::EFLAGS &&
1013 !MO.isDead()) {
1014 return true;
1015 }
1016 }
1017 return false;
1018}
1019
1020/// Check whether the shift count for a machine operand is non-zero.
1021inline static unsigned getTruncatedShiftCount(const MachineInstr &MI,
1022 unsigned ShiftAmtOperandIdx) {
1023 // The shift count is six bits with the REX.W prefix and five bits without.
1024 unsigned ShiftCountMask = (MI.getDesc().TSFlags & X86II::REX_W) ? 63 : 31;
1025 unsigned Imm = MI.getOperand(ShiftAmtOperandIdx).getImm();
1026 return Imm & ShiftCountMask;
1027}
1028
1029/// Check whether the given shift count is appropriate
1030/// can be represented by a LEA instruction.
1031inline static bool isTruncatedShiftCountForLEA(unsigned ShAmt) {
1032 // Left shift instructions can be transformed into load-effective-address
1033 // instructions if we can encode them appropriately.
1034 // A LEA instruction utilizes a SIB byte to encode its scale factor.
1035 // The SIB.scale field is two bits wide which means that we can encode any
1036 // shift amount less than 4.
1037 return ShAmt < 4 && ShAmt > 0;
1038}
1039
1041 MachineInstr &CmpValDefInstr,
1042 const MachineRegisterInfo *MRI,
1043 MachineInstr **AndInstr,
1044 const TargetRegisterInfo *TRI,
1045 bool &NoSignFlag, bool &ClearsOverflowFlag) {
1046 if (!(CmpValDefInstr.getOpcode() == X86::SUBREG_TO_REG &&
1047 CmpInstr.getOpcode() == X86::TEST64rr) &&
1048 !(CmpValDefInstr.getOpcode() == X86::COPY &&
1049 CmpInstr.getOpcode() == X86::TEST16rr))
1050 return false;
1051
1052 // CmpInstr is a TEST16rr/TEST64rr instruction, and
1053 // `X86InstrInfo::analyzeCompare` guarantees that it's analyzable only if two
1054 // registers are identical.
1055 assert((CmpInstr.getOperand(0).getReg() == CmpInstr.getOperand(1).getReg()) &&
1056 "CmpInstr is an analyzable TEST16rr/TEST64rr, and "
1057 "`X86InstrInfo::analyzeCompare` requires two reg operands are the"
1058 "same.");
1059
1060 // Caller (`X86InstrInfo::optimizeCompareInstr`) guarantees that
1061 // `CmpValDefInstr` defines the value that's used by `CmpInstr`; in this case
1062 // if `CmpValDefInstr` sets the EFLAGS, it is likely that `CmpInstr` is
1063 // redundant.
1064 assert(
1065 (MRI->getVRegDef(CmpInstr.getOperand(0).getReg()) == &CmpValDefInstr) &&
1066 "Caller guarantees that TEST64rr is a user of SUBREG_TO_REG or TEST16rr "
1067 "is a user of COPY sub16bit.");
1068 MachineInstr *VregDefInstr = nullptr;
1069 if (CmpInstr.getOpcode() == X86::TEST16rr) {
1070 if (!CmpValDefInstr.getOperand(1).getReg().isVirtual())
1071 return false;
1072 VregDefInstr = MRI->getVRegDef(CmpValDefInstr.getOperand(1).getReg());
1073 if (!VregDefInstr)
1074 return false;
1075 // We can only remove test when AND32ri or AND64ri32 whose imm can fit 16bit
1076 // size, others 32/64 bit ops would test higher bits which test16rr don't
1077 // want to.
1078 if (!((VregDefInstr->getOpcode() == X86::AND32ri ||
1079 VregDefInstr->getOpcode() == X86::AND64ri32) &&
1080 isUInt<16>(VregDefInstr->getOperand(2).getImm())))
1081 return false;
1082 }
1083
1084 if (CmpInstr.getOpcode() == X86::TEST64rr) {
1085 // As seen in X86 td files, CmpValDefInstr.getOperand(1).getImm() is
1086 // typically 0.
1087 if (CmpValDefInstr.getOperand(1).getImm() != 0)
1088 return false;
1089
1090 // As seen in X86 td files, CmpValDefInstr.getOperand(3) is typically
1091 // sub_32bit or sub_xmm.
1092 if (CmpValDefInstr.getOperand(3).getImm() != X86::sub_32bit)
1093 return false;
1094
1095 VregDefInstr = MRI->getVRegDef(CmpValDefInstr.getOperand(2).getReg());
1096 }
1097
1098 assert(VregDefInstr && "Must have a definition (SSA)");
1099
1100 // Requires `CmpValDefInstr` and `VregDefInstr` are from the same MBB
1101 // to simplify the subsequent analysis.
1102 //
1103 // FIXME: If `VregDefInstr->getParent()` is the only predecessor of
1104 // `CmpValDefInstr.getParent()`, this could be handled.
1105 if (VregDefInstr->getParent() != CmpValDefInstr.getParent())
1106 return false;
1107
1108 if (X86::isAND(VregDefInstr->getOpcode())) {
1109 // Get a sequence of instructions like
1110 // %reg = and* ... // Set EFLAGS
1111 // ... // EFLAGS not changed
1112 // %extended_reg = subreg_to_reg 0, %reg, %subreg.sub_32bit
1113 // test64rr %extended_reg, %extended_reg, implicit-def $eflags
1114 // or
1115 // %reg = and32* ...
1116 // ... // EFLAGS not changed.
1117 // %src_reg = copy %reg.sub_16bit:gr32
1118 // test16rr %src_reg, %src_reg, implicit-def $eflags
1119 //
1120 // If subsequent readers use a subset of bits that don't change
1121 // after `and*` instructions, it's likely that the test64rr could
1122 // be optimized away.
1123 for (const MachineInstr &Instr :
1124 make_range(std::next(MachineBasicBlock::iterator(VregDefInstr)),
1125 MachineBasicBlock::iterator(CmpValDefInstr))) {
1126 // There are instructions between 'VregDefInstr' and
1127 // 'CmpValDefInstr' that modifies EFLAGS.
1128 if (Instr.modifiesRegister(X86::EFLAGS, TRI))
1129 return false;
1130 }
1131
1132 *AndInstr = VregDefInstr;
1133
1134 // AND instruction will essentially update SF and clear OF, so
1135 // NoSignFlag should be false in the sense that SF is modified by `AND`.
1136 //
1137 // However, the implementation artifically sets `NoSignFlag` to true
1138 // to poison the SF bit; that is to say, if SF is looked at later, the
1139 // optimization (to erase TEST64rr) will be disabled.
1140 //
1141 // The reason to poison SF bit is that SF bit value could be different
1142 // in the `AND` and `TEST` operation; signed bit is not known for `AND`,
1143 // and is known to be 0 as a result of `TEST64rr`.
1144 //
1145 // FIXME: As opposed to poisoning the SF bit directly, consider peeking into
1146 // the AND instruction and using the static information to guide peephole
1147 // optimization if possible. For example, it's possible to fold a
1148 // conditional move into a copy if the relevant EFLAG bits could be deduced
1149 // from an immediate operand of and operation.
1150 //
1151 NoSignFlag = true;
1152 // ClearsOverflowFlag is true for AND operation (no surprise).
1153 ClearsOverflowFlag = true;
1154 return true;
1155 }
1156 return false;
1157}
1158
1160 unsigned Opc, bool AllowSP, Register &NewSrc,
1161 bool &isKill, MachineOperand &ImplicitOp,
1162 LiveVariables *LV, LiveIntervals *LIS) const {
1163 MachineFunction &MF = *MI.getParent()->getParent();
1164 const TargetRegisterClass *RC;
1165 if (AllowSP) {
1166 RC = Opc != X86::LEA32r ? &X86::GR64RegClass : &X86::GR32RegClass;
1167 } else {
1168 RC = Opc != X86::LEA32r ? &X86::GR64_NOSPRegClass : &X86::GR32_NOSPRegClass;
1169 }
1170 Register SrcReg = Src.getReg();
1171 isKill = MI.killsRegister(SrcReg, /*TRI=*/nullptr);
1172
1173 // For both LEA64 and LEA32 the register already has essentially the right
1174 // type (32-bit or 64-bit) we may just need to forbid SP.
1175 if (Opc != X86::LEA64_32r) {
1176 NewSrc = SrcReg;
1177 assert(!Src.isUndef() && "Undef op doesn't need optimization");
1178
1179 if (NewSrc.isVirtual() && !MF.getRegInfo().constrainRegClass(NewSrc, RC))
1180 return false;
1181
1182 return true;
1183 }
1184
1185 // This is for an LEA64_32r and incoming registers are 32-bit. One way or
1186 // another we need to add 64-bit registers to the final MI.
1187 if (SrcReg.isPhysical()) {
1188 ImplicitOp = Src;
1189 ImplicitOp.setImplicit();
1190
1191 NewSrc = getX86SubSuperRegister(SrcReg, 64);
1192 assert(NewSrc.isValid() && "Invalid Operand");
1193 assert(!Src.isUndef() && "Undef op doesn't need optimization");
1194 } else {
1195 // Virtual register of the wrong class, we have to create a temporary 64-bit
1196 // vreg to feed into the LEA.
1197 NewSrc = MF.getRegInfo().createVirtualRegister(RC);
1198 MachineInstr *Copy =
1199 BuildMI(*MI.getParent(), MI, MI.getDebugLoc(), get(TargetOpcode::COPY))
1200 .addReg(NewSrc, RegState::Define | RegState::Undef, X86::sub_32bit)
1201 .addReg(SrcReg, getKillRegState(isKill));
1202
1203 // Which is obviously going to be dead after we're done with it.
1204 isKill = true;
1205
1206 if (LV)
1207 LV->replaceKillInstruction(SrcReg, MI, *Copy);
1208
1209 if (LIS) {
1210 SlotIndex CopyIdx = LIS->InsertMachineInstrInMaps(*Copy);
1212 LiveInterval &LI = LIS->getInterval(SrcReg);
1214 if (S->end.getBaseIndex() == Idx)
1215 S->end = CopyIdx.getRegSlot();
1216 }
1217 }
1218
1219 // We've set all the parameters without issue.
1220 return true;
1221}
1222
1223MachineInstr *X86InstrInfo::convertToThreeAddressWithLEA(unsigned MIOpc,
1225 LiveVariables *LV,
1226 LiveIntervals *LIS,
1227 bool Is8BitOp) const {
1228 // We handle 8-bit adds and various 16-bit opcodes in the switch below.
1229 MachineBasicBlock &MBB = *MI.getParent();
1231 assert((Is8BitOp ||
1233 *RegInfo.getRegClass(MI.getOperand(0).getReg())) == 16) &&
1234 "Unexpected type for LEA transform");
1235
1236 // TODO: For a 32-bit target, we need to adjust the LEA variables with
1237 // something like this:
1238 // Opcode = X86::LEA32r;
1239 // InRegLEA = RegInfo.createVirtualRegister(&X86::GR32_NOSPRegClass);
1240 // OutRegLEA =
1241 // Is8BitOp ? RegInfo.createVirtualRegister(&X86::GR32ABCD_RegClass)
1242 // : RegInfo.createVirtualRegister(&X86::GR32RegClass);
1243 if (!Subtarget.is64Bit())
1244 return nullptr;
1245
1246 unsigned Opcode = X86::LEA64_32r;
1247 Register InRegLEA = RegInfo.createVirtualRegister(&X86::GR64_NOSPRegClass);
1248 Register OutRegLEA = RegInfo.createVirtualRegister(&X86::GR32RegClass);
1249 Register InRegLEA2;
1250
1251 // Build and insert into an implicit UNDEF value. This is OK because
1252 // we will be shifting and then extracting the lower 8/16-bits.
1253 // This has the potential to cause partial register stall. e.g.
1254 // movw (%rbp,%rcx,2), %dx
1255 // leal -65(%rdx), %esi
1256 // But testing has shown this *does* help performance in 64-bit mode (at
1257 // least on modern x86 machines).
1258 MachineBasicBlock::iterator MBBI = MI.getIterator();
1259 Register Dest = MI.getOperand(0).getReg();
1260 Register Src = MI.getOperand(1).getReg();
1261 Register Src2;
1262 bool IsDead = MI.getOperand(0).isDead();
1263 bool IsKill = MI.getOperand(1).isKill();
1264 unsigned SubReg = Is8BitOp ? X86::sub_8bit : X86::sub_16bit;
1265 assert(!MI.getOperand(1).isUndef() && "Undef op doesn't need optimization");
1266 MachineInstr *ImpDef =
1267 BuildMI(MBB, MBBI, MI.getDebugLoc(), get(X86::IMPLICIT_DEF), InRegLEA);
1268 MachineInstr *InsMI =
1269 BuildMI(MBB, MBBI, MI.getDebugLoc(), get(TargetOpcode::COPY))
1270 .addReg(InRegLEA, RegState::Define, SubReg)
1271 .addReg(Src, getKillRegState(IsKill));
1272 MachineInstr *ImpDef2 = nullptr;
1273 MachineInstr *InsMI2 = nullptr;
1274
1276 BuildMI(MBB, MBBI, MI.getDebugLoc(), get(Opcode), OutRegLEA);
1277 switch (MIOpc) {
1278 default:
1279 llvm_unreachable("Unreachable!");
1280 case X86::SHL8ri:
1281 case X86::SHL16ri: {
1282 unsigned ShAmt = MI.getOperand(2).getImm();
1283 MIB.addReg(0)
1284 .addImm(1LL << ShAmt)
1285 .addReg(InRegLEA, RegState::Kill)
1286 .addImm(0)
1287 .addReg(0);
1288 break;
1289 }
1290 case X86::INC8r:
1291 case X86::INC16r:
1292 addRegOffset(MIB, InRegLEA, true, 1);
1293 break;
1294 case X86::DEC8r:
1295 case X86::DEC16r:
1296 addRegOffset(MIB, InRegLEA, true, -1);
1297 break;
1298 case X86::ADD8ri:
1299 case X86::ADD8ri_DB:
1300 case X86::ADD16ri:
1301 case X86::ADD16ri_DB:
1302 addRegOffset(MIB, InRegLEA, true, MI.getOperand(2).getImm());
1303 break;
1304 case X86::ADD8rr:
1305 case X86::ADD8rr_DB:
1306 case X86::ADD16rr:
1307 case X86::ADD16rr_DB: {
1308 Src2 = MI.getOperand(2).getReg();
1309 bool IsKill2 = MI.getOperand(2).isKill();
1310 assert(!MI.getOperand(2).isUndef() && "Undef op doesn't need optimization");
1311 if (Src == Src2) {
1312 // ADD8rr/ADD16rr killed %reg1028, %reg1028
1313 // just a single insert_subreg.
1314 addRegReg(MIB, InRegLEA, true, InRegLEA, false);
1315 } else {
1316 if (Subtarget.is64Bit())
1317 InRegLEA2 = RegInfo.createVirtualRegister(&X86::GR64_NOSPRegClass);
1318 else
1319 InRegLEA2 = RegInfo.createVirtualRegister(&X86::GR32_NOSPRegClass);
1320 // Build and insert into an implicit UNDEF value. This is OK because
1321 // we will be shifting and then extracting the lower 8/16-bits.
1322 ImpDef2 = BuildMI(MBB, &*MIB, MI.getDebugLoc(), get(X86::IMPLICIT_DEF),
1323 InRegLEA2);
1324 InsMI2 = BuildMI(MBB, &*MIB, MI.getDebugLoc(), get(TargetOpcode::COPY))
1325 .addReg(InRegLEA2, RegState::Define, SubReg)
1326 .addReg(Src2, getKillRegState(IsKill2));
1327 addRegReg(MIB, InRegLEA, true, InRegLEA2, true);
1328 }
1329 if (LV && IsKill2 && InsMI2)
1330 LV->replaceKillInstruction(Src2, MI, *InsMI2);
1331 break;
1332 }
1333 }
1334
1335 MachineInstr *NewMI = MIB;
1336 MachineInstr *ExtMI =
1337 BuildMI(MBB, MBBI, MI.getDebugLoc(), get(TargetOpcode::COPY))
1339 .addReg(OutRegLEA, RegState::Kill, SubReg);
1340
1341 if (LV) {
1342 // Update live variables.
1343 LV->getVarInfo(InRegLEA).Kills.push_back(NewMI);
1344 if (InRegLEA2)
1345 LV->getVarInfo(InRegLEA2).Kills.push_back(NewMI);
1346 LV->getVarInfo(OutRegLEA).Kills.push_back(ExtMI);
1347 if (IsKill)
1348 LV->replaceKillInstruction(Src, MI, *InsMI);
1349 if (IsDead)
1350 LV->replaceKillInstruction(Dest, MI, *ExtMI);
1351 }
1352
1353 if (LIS) {
1354 LIS->InsertMachineInstrInMaps(*ImpDef);
1355 SlotIndex InsIdx = LIS->InsertMachineInstrInMaps(*InsMI);
1356 if (ImpDef2)
1357 LIS->InsertMachineInstrInMaps(*ImpDef2);
1358 SlotIndex Ins2Idx;
1359 if (InsMI2)
1360 Ins2Idx = LIS->InsertMachineInstrInMaps(*InsMI2);
1361 SlotIndex NewIdx = LIS->ReplaceMachineInstrInMaps(MI, *NewMI);
1362 SlotIndex ExtIdx = LIS->InsertMachineInstrInMaps(*ExtMI);
1363 LIS->getInterval(InRegLEA);
1364 LIS->getInterval(OutRegLEA);
1365 if (InRegLEA2)
1366 LIS->getInterval(InRegLEA2);
1367
1368 // Move the use of Src up to InsMI.
1369 LiveInterval &SrcLI = LIS->getInterval(Src);
1370 LiveRange::Segment *SrcSeg = SrcLI.getSegmentContaining(NewIdx);
1371 if (SrcSeg->end == NewIdx.getRegSlot())
1372 SrcSeg->end = InsIdx.getRegSlot();
1373
1374 if (InsMI2) {
1375 // Move the use of Src2 up to InsMI2.
1376 LiveInterval &Src2LI = LIS->getInterval(Src2);
1377 LiveRange::Segment *Src2Seg = Src2LI.getSegmentContaining(NewIdx);
1378 if (Src2Seg->end == NewIdx.getRegSlot())
1379 Src2Seg->end = Ins2Idx.getRegSlot();
1380 }
1381
1382 // Move the definition of Dest down to ExtMI.
1383 LiveInterval &DestLI = LIS->getInterval(Dest);
1384 LiveRange::Segment *DestSeg =
1385 DestLI.getSegmentContaining(NewIdx.getRegSlot());
1386 assert(DestSeg->start == NewIdx.getRegSlot() &&
1387 DestSeg->valno->def == NewIdx.getRegSlot());
1388 DestSeg->start = ExtIdx.getRegSlot();
1389 DestSeg->valno->def = ExtIdx.getRegSlot();
1390 }
1391
1392 return ExtMI;
1393}
1394
1395/// This method must be implemented by targets that
1396/// set the M_CONVERTIBLE_TO_3_ADDR flag. When this flag is set, the target
1397/// may be able to convert a two-address instruction into a true
1398/// three-address instruction on demand. This allows the X86 target (for
1399/// example) to convert ADD and SHL instructions into LEA instructions if they
1400/// would require register copies due to two-addressness.
1401///
1402/// This method returns a null pointer if the transformation cannot be
1403/// performed, otherwise it returns the new instruction.
1404///
1406 LiveVariables *LV,
1407 LiveIntervals *LIS) const {
1408 // The following opcodes also sets the condition code register(s). Only
1409 // convert them to equivalent lea if the condition code register def's
1410 // are dead!
1412 return nullptr;
1413
1414 MachineFunction &MF = *MI.getParent()->getParent();
1415 // All instructions input are two-addr instructions. Get the known operands.
1416 const MachineOperand &Dest = MI.getOperand(0);
1417 const MachineOperand &Src = MI.getOperand(1);
1418
1419 // Ideally, operations with undef should be folded before we get here, but we
1420 // can't guarantee it. Bail out because optimizing undefs is a waste of time.
1421 // Without this, we have to forward undef state to new register operands to
1422 // avoid machine verifier errors.
1423 if (Src.isUndef())
1424 return nullptr;
1425 if (MI.getNumOperands() > 2)
1426 if (MI.getOperand(2).isReg() && MI.getOperand(2).isUndef())
1427 return nullptr;
1428
1429 MachineInstr *NewMI = nullptr;
1430 Register SrcReg, SrcReg2;
1431 bool Is64Bit = Subtarget.is64Bit();
1432
1433 bool Is8BitOp = false;
1434 unsigned NumRegOperands = 2;
1435 unsigned MIOpc = MI.getOpcode();
1436 switch (MIOpc) {
1437 default:
1438 llvm_unreachable("Unreachable!");
1439 case X86::SHL64ri: {
1440 assert(MI.getNumOperands() >= 3 && "Unknown shift instruction!");
1441 unsigned ShAmt = getTruncatedShiftCount(MI, 2);
1442 if (!isTruncatedShiftCountForLEA(ShAmt))
1443 return nullptr;
1444
1445 // LEA can't handle RSP.
1446 if (Src.getReg().isVirtual() && !MF.getRegInfo().constrainRegClass(
1447 Src.getReg(), &X86::GR64_NOSPRegClass))
1448 return nullptr;
1449
1450 NewMI = BuildMI(MF, MI.getDebugLoc(), get(X86::LEA64r))
1451 .add(Dest)
1452 .addReg(0)
1453 .addImm(1LL << ShAmt)
1454 .add(Src)
1455 .addImm(0)
1456 .addReg(0);
1457 break;
1458 }
1459 case X86::SHL32ri: {
1460 assert(MI.getNumOperands() >= 3 && "Unknown shift instruction!");
1461 unsigned ShAmt = getTruncatedShiftCount(MI, 2);
1462 if (!isTruncatedShiftCountForLEA(ShAmt))
1463 return nullptr;
1464
1465 unsigned Opc = Is64Bit ? X86::LEA64_32r : X86::LEA32r;
1466
1467 // LEA can't handle ESP.
1468 bool isKill;
1469 MachineOperand ImplicitOp = MachineOperand::CreateReg(0, false);
1470 if (!classifyLEAReg(MI, Src, Opc, /*AllowSP=*/false, SrcReg, isKill,
1471 ImplicitOp, LV, LIS))
1472 return nullptr;
1473
1474 MachineInstrBuilder MIB = BuildMI(MF, MI.getDebugLoc(), get(Opc))
1475 .add(Dest)
1476 .addReg(0)
1477 .addImm(1LL << ShAmt)
1478 .addReg(SrcReg, getKillRegState(isKill))
1479 .addImm(0)
1480 .addReg(0);
1481 if (ImplicitOp.getReg() != 0)
1482 MIB.add(ImplicitOp);
1483 NewMI = MIB;
1484
1485 // Add kills if classifyLEAReg created a new register.
1486 if (LV && SrcReg != Src.getReg())
1487 LV->getVarInfo(SrcReg).Kills.push_back(NewMI);
1488 break;
1489 }
1490 case X86::SHL8ri:
1491 Is8BitOp = true;
1492 [[fallthrough]];
1493 case X86::SHL16ri: {
1494 assert(MI.getNumOperands() >= 3 && "Unknown shift instruction!");
1495 unsigned ShAmt = getTruncatedShiftCount(MI, 2);
1496 if (!isTruncatedShiftCountForLEA(ShAmt))
1497 return nullptr;
1498 return convertToThreeAddressWithLEA(MIOpc, MI, LV, LIS, Is8BitOp);
1499 }
1500 case X86::INC64r:
1501 case X86::INC32r: {
1502 assert(MI.getNumOperands() >= 2 && "Unknown inc instruction!");
1503 unsigned Opc = MIOpc == X86::INC64r
1504 ? X86::LEA64r
1505 : (Is64Bit ? X86::LEA64_32r : X86::LEA32r);
1506 bool isKill;
1507 MachineOperand ImplicitOp = MachineOperand::CreateReg(0, false);
1508 if (!classifyLEAReg(MI, Src, Opc, /*AllowSP=*/false, SrcReg, isKill,
1509 ImplicitOp, LV, LIS))
1510 return nullptr;
1511
1512 MachineInstrBuilder MIB = BuildMI(MF, MI.getDebugLoc(), get(Opc))
1513 .add(Dest)
1514 .addReg(SrcReg, getKillRegState(isKill));
1515 if (ImplicitOp.getReg() != 0)
1516 MIB.add(ImplicitOp);
1517
1518 NewMI = addOffset(MIB, 1);
1519
1520 // Add kills if classifyLEAReg created a new register.
1521 if (LV && SrcReg != Src.getReg())
1522 LV->getVarInfo(SrcReg).Kills.push_back(NewMI);
1523 break;
1524 }
1525 case X86::DEC64r:
1526 case X86::DEC32r: {
1527 assert(MI.getNumOperands() >= 2 && "Unknown dec instruction!");
1528 unsigned Opc = MIOpc == X86::DEC64r
1529 ? X86::LEA64r
1530 : (Is64Bit ? X86::LEA64_32r : X86::LEA32r);
1531
1532 bool isKill;
1533 MachineOperand ImplicitOp = MachineOperand::CreateReg(0, false);
1534 if (!classifyLEAReg(MI, Src, Opc, /*AllowSP=*/false, SrcReg, isKill,
1535 ImplicitOp, LV, LIS))
1536 return nullptr;
1537
1538 MachineInstrBuilder MIB = BuildMI(MF, MI.getDebugLoc(), get(Opc))
1539 .add(Dest)
1540 .addReg(SrcReg, getKillRegState(isKill));
1541 if (ImplicitOp.getReg() != 0)
1542 MIB.add(ImplicitOp);
1543
1544 NewMI = addOffset(MIB, -1);
1545
1546 // Add kills if classifyLEAReg created a new register.
1547 if (LV && SrcReg != Src.getReg())
1548 LV->getVarInfo(SrcReg).Kills.push_back(NewMI);
1549 break;
1550 }
1551 case X86::DEC8r:
1552 case X86::INC8r:
1553 Is8BitOp = true;
1554 [[fallthrough]];
1555 case X86::DEC16r:
1556 case X86::INC16r:
1557 return convertToThreeAddressWithLEA(MIOpc, MI, LV, LIS, Is8BitOp);
1558 case X86::ADD64rr:
1559 case X86::ADD64rr_DB:
1560 case X86::ADD32rr:
1561 case X86::ADD32rr_DB: {
1562 assert(MI.getNumOperands() >= 3 && "Unknown add instruction!");
1563 unsigned Opc;
1564 if (MIOpc == X86::ADD64rr || MIOpc == X86::ADD64rr_DB)
1565 Opc = X86::LEA64r;
1566 else
1567 Opc = Is64Bit ? X86::LEA64_32r : X86::LEA32r;
1568
1569 const MachineOperand &Src2 = MI.getOperand(2);
1570 bool isKill2;
1571 MachineOperand ImplicitOp2 = MachineOperand::CreateReg(0, false);
1572 if (!classifyLEAReg(MI, Src2, Opc, /*AllowSP=*/false, SrcReg2, isKill2,
1573 ImplicitOp2, LV, LIS))
1574 return nullptr;
1575
1576 bool isKill;
1577 MachineOperand ImplicitOp = MachineOperand::CreateReg(0, false);
1578 if (Src.getReg() == Src2.getReg()) {
1579 // Don't call classify LEAReg a second time on the same register, in case
1580 // the first call inserted a COPY from Src2 and marked it as killed.
1581 isKill = isKill2;
1582 SrcReg = SrcReg2;
1583 } else {
1584 if (!classifyLEAReg(MI, Src, Opc, /*AllowSP=*/true, SrcReg, isKill,
1585 ImplicitOp, LV, LIS))
1586 return nullptr;
1587 }
1588
1589 MachineInstrBuilder MIB = BuildMI(MF, MI.getDebugLoc(), get(Opc)).add(Dest);
1590 if (ImplicitOp.getReg() != 0)
1591 MIB.add(ImplicitOp);
1592 if (ImplicitOp2.getReg() != 0)
1593 MIB.add(ImplicitOp2);
1594
1595 NewMI = addRegReg(MIB, SrcReg, isKill, SrcReg2, isKill2);
1596
1597 // Add kills if classifyLEAReg created a new register.
1598 if (LV) {
1599 if (SrcReg2 != Src2.getReg())
1600 LV->getVarInfo(SrcReg2).Kills.push_back(NewMI);
1601 if (SrcReg != SrcReg2 && SrcReg != Src.getReg())
1602 LV->getVarInfo(SrcReg).Kills.push_back(NewMI);
1603 }
1604 NumRegOperands = 3;
1605 break;
1606 }
1607 case X86::ADD8rr:
1608 case X86::ADD8rr_DB:
1609 Is8BitOp = true;
1610 [[fallthrough]];
1611 case X86::ADD16rr:
1612 case X86::ADD16rr_DB:
1613 return convertToThreeAddressWithLEA(MIOpc, MI, LV, LIS, Is8BitOp);
1614 case X86::ADD64ri32:
1615 case X86::ADD64ri32_DB:
1616 assert(MI.getNumOperands() >= 3 && "Unknown add instruction!");
1617 NewMI = addOffset(
1618 BuildMI(MF, MI.getDebugLoc(), get(X86::LEA64r)).add(Dest).add(Src),
1619 MI.getOperand(2));
1620 break;
1621 case X86::ADD32ri:
1622 case X86::ADD32ri_DB: {
1623 assert(MI.getNumOperands() >= 3 && "Unknown add instruction!");
1624 unsigned Opc = Is64Bit ? X86::LEA64_32r : X86::LEA32r;
1625
1626 bool isKill;
1627 MachineOperand ImplicitOp = MachineOperand::CreateReg(0, false);
1628 if (!classifyLEAReg(MI, Src, Opc, /*AllowSP=*/true, SrcReg, isKill,
1629 ImplicitOp, LV, LIS))
1630 return nullptr;
1631
1632 MachineInstrBuilder MIB = BuildMI(MF, MI.getDebugLoc(), get(Opc))
1633 .add(Dest)
1634 .addReg(SrcReg, getKillRegState(isKill));
1635 if (ImplicitOp.getReg() != 0)
1636 MIB.add(ImplicitOp);
1637
1638 NewMI = addOffset(MIB, MI.getOperand(2));
1639
1640 // Add kills if classifyLEAReg created a new register.
1641 if (LV && SrcReg != Src.getReg())
1642 LV->getVarInfo(SrcReg).Kills.push_back(NewMI);
1643 break;
1644 }
1645 case X86::ADD8ri:
1646 case X86::ADD8ri_DB:
1647 Is8BitOp = true;
1648 [[fallthrough]];
1649 case X86::ADD16ri:
1650 case X86::ADD16ri_DB:
1651 return convertToThreeAddressWithLEA(MIOpc, MI, LV, LIS, Is8BitOp);
1652 case X86::SUB8ri:
1653 case X86::SUB16ri:
1654 /// FIXME: Support these similar to ADD8ri/ADD16ri*.
1655 return nullptr;
1656 case X86::SUB32ri: {
1657 if (!MI.getOperand(2).isImm())
1658 return nullptr;
1659 int64_t Imm = MI.getOperand(2).getImm();
1660 if (!isInt<32>(-Imm))
1661 return nullptr;
1662
1663 assert(MI.getNumOperands() >= 3 && "Unknown add instruction!");
1664 unsigned Opc = Is64Bit ? X86::LEA64_32r : X86::LEA32r;
1665
1666 bool isKill;
1667 MachineOperand ImplicitOp = MachineOperand::CreateReg(0, false);
1668 if (!classifyLEAReg(MI, Src, Opc, /*AllowSP=*/true, SrcReg, isKill,
1669 ImplicitOp, LV, LIS))
1670 return nullptr;
1671
1672 MachineInstrBuilder MIB = BuildMI(MF, MI.getDebugLoc(), get(Opc))
1673 .add(Dest)
1674 .addReg(SrcReg, getKillRegState(isKill));
1675 if (ImplicitOp.getReg() != 0)
1676 MIB.add(ImplicitOp);
1677
1678 NewMI = addOffset(MIB, -Imm);
1679
1680 // Add kills if classifyLEAReg created a new register.
1681 if (LV && SrcReg != Src.getReg())
1682 LV->getVarInfo(SrcReg).Kills.push_back(NewMI);
1683 break;
1684 }
1685
1686 case X86::SUB64ri32: {
1687 if (!MI.getOperand(2).isImm())
1688 return nullptr;
1689 int64_t Imm = MI.getOperand(2).getImm();
1690 if (!isInt<32>(-Imm))
1691 return nullptr;
1692
1693 assert(MI.getNumOperands() >= 3 && "Unknown sub instruction!");
1694
1696 BuildMI(MF, MI.getDebugLoc(), get(X86::LEA64r)).add(Dest).add(Src);
1697 NewMI = addOffset(MIB, -Imm);
1698 break;
1699 }
1700
1701 case X86::VMOVDQU8Z128rmk:
1702 case X86::VMOVDQU8Z256rmk:
1703 case X86::VMOVDQU8Zrmk:
1704 case X86::VMOVDQU16Z128rmk:
1705 case X86::VMOVDQU16Z256rmk:
1706 case X86::VMOVDQU16Zrmk:
1707 case X86::VMOVDQU32Z128rmk:
1708 case X86::VMOVDQA32Z128rmk:
1709 case X86::VMOVDQU32Z256rmk:
1710 case X86::VMOVDQA32Z256rmk:
1711 case X86::VMOVDQU32Zrmk:
1712 case X86::VMOVDQA32Zrmk:
1713 case X86::VMOVDQU64Z128rmk:
1714 case X86::VMOVDQA64Z128rmk:
1715 case X86::VMOVDQU64Z256rmk:
1716 case X86::VMOVDQA64Z256rmk:
1717 case X86::VMOVDQU64Zrmk:
1718 case X86::VMOVDQA64Zrmk:
1719 case X86::VMOVUPDZ128rmk:
1720 case X86::VMOVAPDZ128rmk:
1721 case X86::VMOVUPDZ256rmk:
1722 case X86::VMOVAPDZ256rmk:
1723 case X86::VMOVUPDZrmk:
1724 case X86::VMOVAPDZrmk:
1725 case X86::VMOVUPSZ128rmk:
1726 case X86::VMOVAPSZ128rmk:
1727 case X86::VMOVUPSZ256rmk:
1728 case X86::VMOVAPSZ256rmk:
1729 case X86::VMOVUPSZrmk:
1730 case X86::VMOVAPSZrmk:
1731 case X86::VBROADCASTSDZ256rmk:
1732 case X86::VBROADCASTSDZrmk:
1733 case X86::VBROADCASTSSZ128rmk:
1734 case X86::VBROADCASTSSZ256rmk:
1735 case X86::VBROADCASTSSZrmk:
1736 case X86::VPBROADCASTDZ128rmk:
1737 case X86::VPBROADCASTDZ256rmk:
1738 case X86::VPBROADCASTDZrmk:
1739 case X86::VPBROADCASTQZ128rmk:
1740 case X86::VPBROADCASTQZ256rmk:
1741 case X86::VPBROADCASTQZrmk: {
1742 unsigned Opc;
1743 switch (MIOpc) {
1744 default:
1745 llvm_unreachable("Unreachable!");
1746 case X86::VMOVDQU8Z128rmk:
1747 Opc = X86::VPBLENDMBZ128rmk;
1748 break;
1749 case X86::VMOVDQU8Z256rmk:
1750 Opc = X86::VPBLENDMBZ256rmk;
1751 break;
1752 case X86::VMOVDQU8Zrmk:
1753 Opc = X86::VPBLENDMBZrmk;
1754 break;
1755 case X86::VMOVDQU16Z128rmk:
1756 Opc = X86::VPBLENDMWZ128rmk;
1757 break;
1758 case X86::VMOVDQU16Z256rmk:
1759 Opc = X86::VPBLENDMWZ256rmk;
1760 break;
1761 case X86::VMOVDQU16Zrmk:
1762 Opc = X86::VPBLENDMWZrmk;
1763 break;
1764 case X86::VMOVDQU32Z128rmk:
1765 Opc = X86::VPBLENDMDZ128rmk;
1766 break;
1767 case X86::VMOVDQU32Z256rmk:
1768 Opc = X86::VPBLENDMDZ256rmk;
1769 break;
1770 case X86::VMOVDQU32Zrmk:
1771 Opc = X86::VPBLENDMDZrmk;
1772 break;
1773 case X86::VMOVDQU64Z128rmk:
1774 Opc = X86::VPBLENDMQZ128rmk;
1775 break;
1776 case X86::VMOVDQU64Z256rmk:
1777 Opc = X86::VPBLENDMQZ256rmk;
1778 break;
1779 case X86::VMOVDQU64Zrmk:
1780 Opc = X86::VPBLENDMQZrmk;
1781 break;
1782 case X86::VMOVUPDZ128rmk:
1783 Opc = X86::VBLENDMPDZ128rmk;
1784 break;
1785 case X86::VMOVUPDZ256rmk:
1786 Opc = X86::VBLENDMPDZ256rmk;
1787 break;
1788 case X86::VMOVUPDZrmk:
1789 Opc = X86::VBLENDMPDZrmk;
1790 break;
1791 case X86::VMOVUPSZ128rmk:
1792 Opc = X86::VBLENDMPSZ128rmk;
1793 break;
1794 case X86::VMOVUPSZ256rmk:
1795 Opc = X86::VBLENDMPSZ256rmk;
1796 break;
1797 case X86::VMOVUPSZrmk:
1798 Opc = X86::VBLENDMPSZrmk;
1799 break;
1800 case X86::VMOVDQA32Z128rmk:
1801 Opc = X86::VPBLENDMDZ128rmk;
1802 break;
1803 case X86::VMOVDQA32Z256rmk:
1804 Opc = X86::VPBLENDMDZ256rmk;
1805 break;
1806 case X86::VMOVDQA32Zrmk:
1807 Opc = X86::VPBLENDMDZrmk;
1808 break;
1809 case X86::VMOVDQA64Z128rmk:
1810 Opc = X86::VPBLENDMQZ128rmk;
1811 break;
1812 case X86::VMOVDQA64Z256rmk:
1813 Opc = X86::VPBLENDMQZ256rmk;
1814 break;
1815 case X86::VMOVDQA64Zrmk:
1816 Opc = X86::VPBLENDMQZrmk;
1817 break;
1818 case X86::VMOVAPDZ128rmk:
1819 Opc = X86::VBLENDMPDZ128rmk;
1820 break;
1821 case X86::VMOVAPDZ256rmk:
1822 Opc = X86::VBLENDMPDZ256rmk;
1823 break;
1824 case X86::VMOVAPDZrmk:
1825 Opc = X86::VBLENDMPDZrmk;
1826 break;
1827 case X86::VMOVAPSZ128rmk:
1828 Opc = X86::VBLENDMPSZ128rmk;
1829 break;
1830 case X86::VMOVAPSZ256rmk:
1831 Opc = X86::VBLENDMPSZ256rmk;
1832 break;
1833 case X86::VMOVAPSZrmk:
1834 Opc = X86::VBLENDMPSZrmk;
1835 break;
1836 case X86::VBROADCASTSDZ256rmk:
1837 Opc = X86::VBLENDMPDZ256rmbk;
1838 break;
1839 case X86::VBROADCASTSDZrmk:
1840 Opc = X86::VBLENDMPDZrmbk;
1841 break;
1842 case X86::VBROADCASTSSZ128rmk:
1843 Opc = X86::VBLENDMPSZ128rmbk;
1844 break;
1845 case X86::VBROADCASTSSZ256rmk:
1846 Opc = X86::VBLENDMPSZ256rmbk;
1847 break;
1848 case X86::VBROADCASTSSZrmk:
1849 Opc = X86::VBLENDMPSZrmbk;
1850 break;
1851 case X86::VPBROADCASTDZ128rmk:
1852 Opc = X86::VPBLENDMDZ128rmbk;
1853 break;
1854 case X86::VPBROADCASTDZ256rmk:
1855 Opc = X86::VPBLENDMDZ256rmbk;
1856 break;
1857 case X86::VPBROADCASTDZrmk:
1858 Opc = X86::VPBLENDMDZrmbk;
1859 break;
1860 case X86::VPBROADCASTQZ128rmk:
1861 Opc = X86::VPBLENDMQZ128rmbk;
1862 break;
1863 case X86::VPBROADCASTQZ256rmk:
1864 Opc = X86::VPBLENDMQZ256rmbk;
1865 break;
1866 case X86::VPBROADCASTQZrmk:
1867 Opc = X86::VPBLENDMQZrmbk;
1868 break;
1869 }
1870
1871 NewMI = BuildMI(MF, MI.getDebugLoc(), get(Opc))
1872 .add(Dest)
1873 .add(MI.getOperand(2))
1874 .add(Src)
1875 .add(MI.getOperand(3))
1876 .add(MI.getOperand(4))
1877 .add(MI.getOperand(5))
1878 .add(MI.getOperand(6))
1879 .add(MI.getOperand(7));
1880 NumRegOperands = 4;
1881 break;
1882 }
1883
1884 case X86::VMOVDQU8Z128rrk:
1885 case X86::VMOVDQU8Z256rrk:
1886 case X86::VMOVDQU8Zrrk:
1887 case X86::VMOVDQU16Z128rrk:
1888 case X86::VMOVDQU16Z256rrk:
1889 case X86::VMOVDQU16Zrrk:
1890 case X86::VMOVDQU32Z128rrk:
1891 case X86::VMOVDQA32Z128rrk:
1892 case X86::VMOVDQU32Z256rrk:
1893 case X86::VMOVDQA32Z256rrk:
1894 case X86::VMOVDQU32Zrrk:
1895 case X86::VMOVDQA32Zrrk:
1896 case X86::VMOVDQU64Z128rrk:
1897 case X86::VMOVDQA64Z128rrk:
1898 case X86::VMOVDQU64Z256rrk:
1899 case X86::VMOVDQA64Z256rrk:
1900 case X86::VMOVDQU64Zrrk:
1901 case X86::VMOVDQA64Zrrk:
1902 case X86::VMOVUPDZ128rrk:
1903 case X86::VMOVAPDZ128rrk:
1904 case X86::VMOVUPDZ256rrk:
1905 case X86::VMOVAPDZ256rrk:
1906 case X86::VMOVUPDZrrk:
1907 case X86::VMOVAPDZrrk:
1908 case X86::VMOVUPSZ128rrk:
1909 case X86::VMOVAPSZ128rrk:
1910 case X86::VMOVUPSZ256rrk:
1911 case X86::VMOVAPSZ256rrk:
1912 case X86::VMOVUPSZrrk:
1913 case X86::VMOVAPSZrrk: {
1914 unsigned Opc;
1915 switch (MIOpc) {
1916 default:
1917 llvm_unreachable("Unreachable!");
1918 case X86::VMOVDQU8Z128rrk:
1919 Opc = X86::VPBLENDMBZ128rrk;
1920 break;
1921 case X86::VMOVDQU8Z256rrk:
1922 Opc = X86::VPBLENDMBZ256rrk;
1923 break;
1924 case X86::VMOVDQU8Zrrk:
1925 Opc = X86::VPBLENDMBZrrk;
1926 break;
1927 case X86::VMOVDQU16Z128rrk:
1928 Opc = X86::VPBLENDMWZ128rrk;
1929 break;
1930 case X86::VMOVDQU16Z256rrk:
1931 Opc = X86::VPBLENDMWZ256rrk;
1932 break;
1933 case X86::VMOVDQU16Zrrk:
1934 Opc = X86::VPBLENDMWZrrk;
1935 break;
1936 case X86::VMOVDQU32Z128rrk:
1937 Opc = X86::VPBLENDMDZ128rrk;
1938 break;
1939 case X86::VMOVDQU32Z256rrk:
1940 Opc = X86::VPBLENDMDZ256rrk;
1941 break;
1942 case X86::VMOVDQU32Zrrk:
1943 Opc = X86::VPBLENDMDZrrk;
1944 break;
1945 case X86::VMOVDQU64Z128rrk:
1946 Opc = X86::VPBLENDMQZ128rrk;
1947 break;
1948 case X86::VMOVDQU64Z256rrk:
1949 Opc = X86::VPBLENDMQZ256rrk;
1950 break;
1951 case X86::VMOVDQU64Zrrk:
1952 Opc = X86::VPBLENDMQZrrk;
1953 break;
1954 case X86::VMOVUPDZ128rrk:
1955 Opc = X86::VBLENDMPDZ128rrk;
1956 break;
1957 case X86::VMOVUPDZ256rrk:
1958 Opc = X86::VBLENDMPDZ256rrk;
1959 break;
1960 case X86::VMOVUPDZrrk:
1961 Opc = X86::VBLENDMPDZrrk;
1962 break;
1963 case X86::VMOVUPSZ128rrk:
1964 Opc = X86::VBLENDMPSZ128rrk;
1965 break;
1966 case X86::VMOVUPSZ256rrk:
1967 Opc = X86::VBLENDMPSZ256rrk;
1968 break;
1969 case X86::VMOVUPSZrrk:
1970 Opc = X86::VBLENDMPSZrrk;
1971 break;
1972 case X86::VMOVDQA32Z128rrk:
1973 Opc = X86::VPBLENDMDZ128rrk;
1974 break;
1975 case X86::VMOVDQA32Z256rrk:
1976 Opc = X86::VPBLENDMDZ256rrk;
1977 break;
1978 case X86::VMOVDQA32Zrrk:
1979 Opc = X86::VPBLENDMDZrrk;
1980 break;
1981 case X86::VMOVDQA64Z128rrk:
1982 Opc = X86::VPBLENDMQZ128rrk;
1983 break;
1984 case X86::VMOVDQA64Z256rrk:
1985 Opc = X86::VPBLENDMQZ256rrk;
1986 break;
1987 case X86::VMOVDQA64Zrrk:
1988 Opc = X86::VPBLENDMQZrrk;
1989 break;
1990 case X86::VMOVAPDZ128rrk:
1991 Opc = X86::VBLENDMPDZ128rrk;
1992 break;
1993 case X86::VMOVAPDZ256rrk:
1994 Opc = X86::VBLENDMPDZ256rrk;
1995 break;
1996 case X86::VMOVAPDZrrk:
1997 Opc = X86::VBLENDMPDZrrk;
1998 break;
1999 case X86::VMOVAPSZ128rrk:
2000 Opc = X86::VBLENDMPSZ128rrk;
2001 break;
2002 case X86::VMOVAPSZ256rrk:
2003 Opc = X86::VBLENDMPSZ256rrk;
2004 break;
2005 case X86::VMOVAPSZrrk:
2006 Opc = X86::VBLENDMPSZrrk;
2007 break;
2008 }
2009
2010 NewMI = BuildMI(MF, MI.getDebugLoc(), get(Opc))
2011 .add(Dest)
2012 .add(MI.getOperand(2))
2013 .add(Src)
2014 .add(MI.getOperand(3));
2015 NumRegOperands = 4;
2016 break;
2017 }
2018 }
2019
2020 if (!NewMI)
2021 return nullptr;
2022
2023 if (LV) { // Update live variables
2024 for (unsigned I = 0; I < NumRegOperands; ++I) {
2025 MachineOperand &Op = MI.getOperand(I);
2026 if (Op.isReg() && (Op.isDead() || Op.isKill()))
2027 LV->replaceKillInstruction(Op.getReg(), MI, *NewMI);
2028 }
2029 }
2030
2031 MachineBasicBlock &MBB = *MI.getParent();
2032 MBB.insert(MI.getIterator(), NewMI); // Insert the new inst
2033
2034 if (LIS) {
2035 LIS->ReplaceMachineInstrInMaps(MI, *NewMI);
2036 if (SrcReg)
2037 LIS->getInterval(SrcReg);
2038 if (SrcReg2)
2039 LIS->getInterval(SrcReg2);
2040 }
2041
2042 return NewMI;
2043}
2044
2045/// This determines which of three possible cases of a three source commute
2046/// the source indexes correspond to taking into account any mask operands.
2047/// All prevents commuting a passthru operand. Returns -1 if the commute isn't
2048/// possible.
2049/// Case 0 - Possible to commute the first and second operands.
2050/// Case 1 - Possible to commute the first and third operands.
2051/// Case 2 - Possible to commute the second and third operands.
2052static unsigned getThreeSrcCommuteCase(uint64_t TSFlags, unsigned SrcOpIdx1,
2053 unsigned SrcOpIdx2) {
2054 // Put the lowest index to SrcOpIdx1 to simplify the checks below.
2055 if (SrcOpIdx1 > SrcOpIdx2)
2056 std::swap(SrcOpIdx1, SrcOpIdx2);
2057
2058 unsigned Op1 = 1, Op2 = 2, Op3 = 3;
2059 if (X86II::isKMasked(TSFlags)) {
2060 Op2++;
2061 Op3++;
2062 }
2063
2064 if (SrcOpIdx1 == Op1 && SrcOpIdx2 == Op2)
2065 return 0;
2066 if (SrcOpIdx1 == Op1 && SrcOpIdx2 == Op3)
2067 return 1;
2068 if (SrcOpIdx1 == Op2 && SrcOpIdx2 == Op3)
2069 return 2;
2070 llvm_unreachable("Unknown three src commute case.");
2071}
2072
2074 const MachineInstr &MI, unsigned SrcOpIdx1, unsigned SrcOpIdx2,
2075 const X86InstrFMA3Group &FMA3Group) const {
2076
2077 unsigned Opc = MI.getOpcode();
2078
2079 // TODO: Commuting the 1st operand of FMA*_Int requires some additional
2080 // analysis. The commute optimization is legal only if all users of FMA*_Int
2081 // use only the lowest element of the FMA*_Int instruction. Such analysis are
2082 // not implemented yet. So, just return 0 in that case.
2083 // When such analysis are available this place will be the right place for
2084 // calling it.
2085 assert(!(FMA3Group.isIntrinsic() && (SrcOpIdx1 == 1 || SrcOpIdx2 == 1)) &&
2086 "Intrinsic instructions can't commute operand 1");
2087
2088 // Determine which case this commute is or if it can't be done.
2089 unsigned Case =
2090 getThreeSrcCommuteCase(MI.getDesc().TSFlags, SrcOpIdx1, SrcOpIdx2);
2091 assert(Case < 3 && "Unexpected case number!");
2092
2093 // Define the FMA forms mapping array that helps to map input FMA form
2094 // to output FMA form to preserve the operation semantics after
2095 // commuting the operands.
2096 const unsigned Form132Index = 0;
2097 const unsigned Form213Index = 1;
2098 const unsigned Form231Index = 2;
2099 static const unsigned FormMapping[][3] = {
2100 // 0: SrcOpIdx1 == 1 && SrcOpIdx2 == 2;
2101 // FMA132 A, C, b; ==> FMA231 C, A, b;
2102 // FMA213 B, A, c; ==> FMA213 A, B, c;
2103 // FMA231 C, A, b; ==> FMA132 A, C, b;
2104 {Form231Index, Form213Index, Form132Index},
2105 // 1: SrcOpIdx1 == 1 && SrcOpIdx2 == 3;
2106 // FMA132 A, c, B; ==> FMA132 B, c, A;
2107 // FMA213 B, a, C; ==> FMA231 C, a, B;
2108 // FMA231 C, a, B; ==> FMA213 B, a, C;
2109 {Form132Index, Form231Index, Form213Index},
2110 // 2: SrcOpIdx1 == 2 && SrcOpIdx2 == 3;
2111 // FMA132 a, C, B; ==> FMA213 a, B, C;
2112 // FMA213 b, A, C; ==> FMA132 b, C, A;
2113 // FMA231 c, A, B; ==> FMA231 c, B, A;
2114 {Form213Index, Form132Index, Form231Index}};
2115
2116 unsigned FMAForms[3];
2117 FMAForms[0] = FMA3Group.get132Opcode();
2118 FMAForms[1] = FMA3Group.get213Opcode();
2119 FMAForms[2] = FMA3Group.get231Opcode();
2120
2121 // Everything is ready, just adjust the FMA opcode and return it.
2122 for (unsigned FormIndex = 0; FormIndex < 3; FormIndex++)
2123 if (Opc == FMAForms[FormIndex])
2124 return FMAForms[FormMapping[Case][FormIndex]];
2125
2126 llvm_unreachable("Illegal FMA3 format");
2127}
2128
2129static void commuteVPTERNLOG(MachineInstr &MI, unsigned SrcOpIdx1,
2130 unsigned SrcOpIdx2) {
2131 // Determine which case this commute is or if it can't be done.
2132 unsigned Case =
2133 getThreeSrcCommuteCase(MI.getDesc().TSFlags, SrcOpIdx1, SrcOpIdx2);
2134 assert(Case < 3 && "Unexpected case value!");
2135
2136 // For each case we need to swap two pairs of bits in the final immediate.
2137 static const uint8_t SwapMasks[3][4] = {
2138 {0x04, 0x10, 0x08, 0x20}, // Swap bits 2/4 and 3/5.
2139 {0x02, 0x10, 0x08, 0x40}, // Swap bits 1/4 and 3/6.
2140 {0x02, 0x04, 0x20, 0x40}, // Swap bits 1/2 and 5/6.
2141 };
2142
2143 uint8_t Imm = MI.getOperand(MI.getNumOperands() - 1).getImm();
2144 // Clear out the bits we are swapping.
2145 uint8_t NewImm = Imm & ~(SwapMasks[Case][0] | SwapMasks[Case][1] |
2146 SwapMasks[Case][2] | SwapMasks[Case][3]);
2147 // If the immediate had a bit of the pair set, then set the opposite bit.
2148 if (Imm & SwapMasks[Case][0])
2149 NewImm |= SwapMasks[Case][1];
2150 if (Imm & SwapMasks[Case][1])
2151 NewImm |= SwapMasks[Case][0];
2152 if (Imm & SwapMasks[Case][2])
2153 NewImm |= SwapMasks[Case][3];
2154 if (Imm & SwapMasks[Case][3])
2155 NewImm |= SwapMasks[Case][2];
2156 MI.getOperand(MI.getNumOperands() - 1).setImm(NewImm);
2157}
2158
2159// Returns true if this is a VPERMI2 or VPERMT2 instruction that can be
2160// commuted.
2161static bool isCommutableVPERMV3Instruction(unsigned Opcode) {
2162#define VPERM_CASES(Suffix) \
2163 case X86::VPERMI2##Suffix##Z128rr: \
2164 case X86::VPERMT2##Suffix##Z128rr: \
2165 case X86::VPERMI2##Suffix##Z256rr: \
2166 case X86::VPERMT2##Suffix##Z256rr: \
2167 case X86::VPERMI2##Suffix##Zrr: \
2168 case X86::VPERMT2##Suffix##Zrr: \
2169 case X86::VPERMI2##Suffix##Z128rm: \
2170 case X86::VPERMT2##Suffix##Z128rm: \
2171 case X86::VPERMI2##Suffix##Z256rm: \
2172 case X86::VPERMT2##Suffix##Z256rm: \
2173 case X86::VPERMI2##Suffix##Zrm: \
2174 case X86::VPERMT2##Suffix##Zrm: \
2175 case X86::VPERMI2##Suffix##Z128rrkz: \
2176 case X86::VPERMT2##Suffix##Z128rrkz: \
2177 case X86::VPERMI2##Suffix##Z256rrkz: \
2178 case X86::VPERMT2##Suffix##Z256rrkz: \
2179 case X86::VPERMI2##Suffix##Zrrkz: \
2180 case X86::VPERMT2##Suffix##Zrrkz: \
2181 case X86::VPERMI2##Suffix##Z128rmkz: \
2182 case X86::VPERMT2##Suffix##Z128rmkz: \
2183 case X86::VPERMI2##Suffix##Z256rmkz: \
2184 case X86::VPERMT2##Suffix##Z256rmkz: \
2185 case X86::VPERMI2##Suffix##Zrmkz: \
2186 case X86::VPERMT2##Suffix##Zrmkz:
2187
2188#define VPERM_CASES_BROADCAST(Suffix) \
2189 VPERM_CASES(Suffix) \
2190 case X86::VPERMI2##Suffix##Z128rmb: \
2191 case X86::VPERMT2##Suffix##Z128rmb: \
2192 case X86::VPERMI2##Suffix##Z256rmb: \
2193 case X86::VPERMT2##Suffix##Z256rmb: \
2194 case X86::VPERMI2##Suffix##Zrmb: \
2195 case X86::VPERMT2##Suffix##Zrmb: \
2196 case X86::VPERMI2##Suffix##Z128rmbkz: \
2197 case X86::VPERMT2##Suffix##Z128rmbkz: \
2198 case X86::VPERMI2##Suffix##Z256rmbkz: \
2199 case X86::VPERMT2##Suffix##Z256rmbkz: \
2200 case X86::VPERMI2##Suffix##Zrmbkz: \
2201 case X86::VPERMT2##Suffix##Zrmbkz:
2202
2203 switch (Opcode) {
2204 default:
2205 return false;
2206 VPERM_CASES(B)
2211 VPERM_CASES(W)
2212 return true;
2213 }
2214#undef VPERM_CASES_BROADCAST
2215#undef VPERM_CASES
2216}
2217
2218// Returns commuted opcode for VPERMI2 and VPERMT2 instructions by switching
2219// from the I opcode to the T opcode and vice versa.
2220static unsigned getCommutedVPERMV3Opcode(unsigned Opcode) {
2221#define VPERM_CASES(Orig, New) \
2222 case X86::Orig##Z128rr: \
2223 return X86::New##Z128rr; \
2224 case X86::Orig##Z128rrkz: \
2225 return X86::New##Z128rrkz; \
2226 case X86::Orig##Z128rm: \
2227 return X86::New##Z128rm; \
2228 case X86::Orig##Z128rmkz: \
2229 return X86::New##Z128rmkz; \
2230 case X86::Orig##Z256rr: \
2231 return X86::New##Z256rr; \
2232 case X86::Orig##Z256rrkz: \
2233 return X86::New##Z256rrkz; \
2234 case X86::Orig##Z256rm: \
2235 return X86::New##Z256rm; \
2236 case X86::Orig##Z256rmkz: \
2237 return X86::New##Z256rmkz; \
2238 case X86::Orig##Zrr: \
2239 return X86::New##Zrr; \
2240 case X86::Orig##Zrrkz: \
2241 return X86::New##Zrrkz; \
2242 case X86::Orig##Zrm: \
2243 return X86::New##Zrm; \
2244 case X86::Orig##Zrmkz: \
2245 return X86::New##Zrmkz;
2246
2247#define VPERM_CASES_BROADCAST(Orig, New) \
2248 VPERM_CASES(Orig, New) \
2249 case X86::Orig##Z128rmb: \
2250 return X86::New##Z128rmb; \
2251 case X86::Orig##Z128rmbkz: \
2252 return X86::New##Z128rmbkz; \
2253 case X86::Orig##Z256rmb: \
2254 return X86::New##Z256rmb; \
2255 case X86::Orig##Z256rmbkz: \
2256 return X86::New##Z256rmbkz; \
2257 case X86::Orig##Zrmb: \
2258 return X86::New##Zrmb; \
2259 case X86::Orig##Zrmbkz: \
2260 return X86::New##Zrmbkz;
2261
2262 switch (Opcode) {
2263 VPERM_CASES(VPERMI2B, VPERMT2B)
2264 VPERM_CASES_BROADCAST(VPERMI2D, VPERMT2D)
2265 VPERM_CASES_BROADCAST(VPERMI2PD, VPERMT2PD)
2266 VPERM_CASES_BROADCAST(VPERMI2PS, VPERMT2PS)
2267 VPERM_CASES_BROADCAST(VPERMI2Q, VPERMT2Q)
2268 VPERM_CASES(VPERMI2W, VPERMT2W)
2269 VPERM_CASES(VPERMT2B, VPERMI2B)
2270 VPERM_CASES_BROADCAST(VPERMT2D, VPERMI2D)
2271 VPERM_CASES_BROADCAST(VPERMT2PD, VPERMI2PD)
2272 VPERM_CASES_BROADCAST(VPERMT2PS, VPERMI2PS)
2273 VPERM_CASES_BROADCAST(VPERMT2Q, VPERMI2Q)
2274 VPERM_CASES(VPERMT2W, VPERMI2W)
2275 }
2276
2277 llvm_unreachable("Unreachable!");
2278#undef VPERM_CASES_BROADCAST
2279#undef VPERM_CASES
2280}
2281
2283 unsigned OpIdx1,
2284 unsigned OpIdx2) const {
2285 auto CloneIfNew = [&](MachineInstr &MI) {
2286 return std::exchange(NewMI, false)
2287 ? MI.getParent()->getParent()->CloneMachineInstr(&MI)
2288 : &MI;
2289 };
2290 MachineInstr *WorkingMI = nullptr;
2291 unsigned Opc = MI.getOpcode();
2292
2293#define CASE_ND(OP) \
2294 case X86::OP: \
2295 case X86::OP##_ND:
2296
2297 switch (Opc) {
2298 // SHLD B, C, I <-> SHRD C, B, (BitWidth - I)
2299 CASE_ND(SHRD16rri8)
2300 CASE_ND(SHLD16rri8)
2301 CASE_ND(SHRD32rri8)
2302 CASE_ND(SHLD32rri8)
2303 CASE_ND(SHRD64rri8)
2304 CASE_ND(SHLD64rri8) {
2305 unsigned Size;
2306 switch (Opc) {
2307 default:
2308 llvm_unreachable("Unreachable!");
2309#define FROM_TO_SIZE(A, B, S) \
2310 case X86::A: \
2311 Opc = X86::B; \
2312 Size = S; \
2313 break; \
2314 case X86::A##_ND: \
2315 Opc = X86::B##_ND; \
2316 Size = S; \
2317 break; \
2318 case X86::B: \
2319 Opc = X86::A; \
2320 Size = S; \
2321 break; \
2322 case X86::B##_ND: \
2323 Opc = X86::A##_ND; \
2324 Size = S; \
2325 break;
2326
2327 FROM_TO_SIZE(SHRD16rri8, SHLD16rri8, 16)
2328 FROM_TO_SIZE(SHRD32rri8, SHLD32rri8, 32)
2329 FROM_TO_SIZE(SHRD64rri8, SHLD64rri8, 64)
2330#undef FROM_TO_SIZE
2331 }
2332 WorkingMI = CloneIfNew(MI);
2333 WorkingMI->setDesc(get(Opc));
2334 WorkingMI->getOperand(3).setImm(Size - MI.getOperand(3).getImm());
2335 break;
2336 }
2337 case X86::PFSUBrr:
2338 case X86::PFSUBRrr:
2339 // PFSUB x, y: x = x - y
2340 // PFSUBR x, y: x = y - x
2341 WorkingMI = CloneIfNew(MI);
2342 WorkingMI->setDesc(
2343 get(X86::PFSUBRrr == Opc ? X86::PFSUBrr : X86::PFSUBRrr));
2344 break;
2345 case X86::BLENDPDrri:
2346 case X86::BLENDPSrri:
2347 case X86::VBLENDPDrri:
2348 case X86::VBLENDPSrri:
2349 // If we're optimizing for size, try to use MOVSD/MOVSS.
2350 if (MI.getParent()->getParent()->getFunction().hasOptSize()) {
2351 unsigned Mask = (Opc == X86::BLENDPDrri || Opc == X86::VBLENDPDrri) ? 0x03: 0x0F;
2352 if ((MI.getOperand(3).getImm() ^ Mask) == 1) {
2353#define FROM_TO(FROM, TO) \
2354 case X86::FROM: \
2355 Opc = X86::TO; \
2356 break;
2357 switch (Opc) {
2358 default:
2359 llvm_unreachable("Unreachable!");
2360 FROM_TO(BLENDPDrri, MOVSDrr)
2361 FROM_TO(BLENDPSrri, MOVSSrr)
2362 FROM_TO(VBLENDPDrri, VMOVSDrr)
2363 FROM_TO(VBLENDPSrri, VMOVSSrr)
2364 }
2365 WorkingMI = CloneIfNew(MI);
2366 WorkingMI->setDesc(get(Opc));
2367 WorkingMI->removeOperand(3);
2368 break;
2369 }
2370#undef FROM_TO
2371 }
2372 [[fallthrough]];
2373 case X86::PBLENDWrri:
2374 case X86::VBLENDPDYrri:
2375 case X86::VBLENDPSYrri:
2376 case X86::VPBLENDDrri:
2377 case X86::VPBLENDWrri:
2378 case X86::VPBLENDDYrri:
2379 case X86::VPBLENDWYrri: {
2380 int8_t Mask;
2381 switch (Opc) {
2382 default:
2383 llvm_unreachable("Unreachable!");
2384 case X86::BLENDPDrri:
2385 Mask = (int8_t)0x03;
2386 break;
2387 case X86::BLENDPSrri:
2388 Mask = (int8_t)0x0F;
2389 break;
2390 case X86::PBLENDWrri:
2391 Mask = (int8_t)0xFF;
2392 break;
2393 case X86::VBLENDPDrri:
2394 Mask = (int8_t)0x03;
2395 break;
2396 case X86::VBLENDPSrri:
2397 Mask = (int8_t)0x0F;
2398 break;
2399 case X86::VBLENDPDYrri:
2400 Mask = (int8_t)0x0F;
2401 break;
2402 case X86::VBLENDPSYrri:
2403 Mask = (int8_t)0xFF;
2404 break;
2405 case X86::VPBLENDDrri:
2406 Mask = (int8_t)0x0F;
2407 break;
2408 case X86::VPBLENDWrri:
2409 Mask = (int8_t)0xFF;
2410 break;
2411 case X86::VPBLENDDYrri:
2412 Mask = (int8_t)0xFF;
2413 break;
2414 case X86::VPBLENDWYrri:
2415 Mask = (int8_t)0xFF;
2416 break;
2417 }
2418 // Only the least significant bits of Imm are used.
2419 // Using int8_t to ensure it will be sign extended to the int64_t that
2420 // setImm takes in order to match isel behavior.
2421 int8_t Imm = MI.getOperand(3).getImm() & Mask;
2422 WorkingMI = CloneIfNew(MI);
2423 WorkingMI->getOperand(3).setImm(Mask ^ Imm);
2424 break;
2425 }
2426 case X86::INSERTPSrri:
2427 case X86::VINSERTPSrri:
2428 case X86::VINSERTPSZrri: {
2429 unsigned Imm = MI.getOperand(MI.getNumOperands() - 1).getImm();
2430 unsigned ZMask = Imm & 15;
2431 unsigned DstIdx = (Imm >> 4) & 3;
2432 unsigned SrcIdx = (Imm >> 6) & 3;
2433
2434 // We can commute insertps if we zero 2 of the elements, the insertion is
2435 // "inline" and we don't override the insertion with a zero.
2436 if (DstIdx == SrcIdx && (ZMask & (1 << DstIdx)) == 0 &&
2437 llvm::popcount(ZMask) == 2) {
2438 unsigned AltIdx = llvm::countr_zero((ZMask | (1 << DstIdx)) ^ 15);
2439 assert(AltIdx < 4 && "Illegal insertion index");
2440 unsigned AltImm = (AltIdx << 6) | (AltIdx << 4) | ZMask;
2441 WorkingMI = CloneIfNew(MI);
2442 WorkingMI->getOperand(MI.getNumOperands() - 1).setImm(AltImm);
2443 break;
2444 }
2445 return nullptr;
2446 }
2447 case X86::MOVSDrr:
2448 case X86::MOVSSrr:
2449 case X86::VMOVSDrr:
2450 case X86::VMOVSSrr: {
2451 // On SSE41 or later we can commute a MOVSS/MOVSD to a BLENDPS/BLENDPD.
2452 if (Subtarget.hasSSE41()) {
2453 unsigned Mask;
2454 switch (Opc) {
2455 default:
2456 llvm_unreachable("Unreachable!");
2457 case X86::MOVSDrr:
2458 Opc = X86::BLENDPDrri;
2459 Mask = 0x02;
2460 break;
2461 case X86::MOVSSrr:
2462 Opc = X86::BLENDPSrri;
2463 Mask = 0x0E;
2464 break;
2465 case X86::VMOVSDrr:
2466 Opc = X86::VBLENDPDrri;
2467 Mask = 0x02;
2468 break;
2469 case X86::VMOVSSrr:
2470 Opc = X86::VBLENDPSrri;
2471 Mask = 0x0E;
2472 break;
2473 }
2474
2475 WorkingMI = CloneIfNew(MI);
2476 WorkingMI->setDesc(get(Opc));
2477 WorkingMI->addOperand(MachineOperand::CreateImm(Mask));
2478 break;
2479 }
2480
2481 WorkingMI = CloneIfNew(MI);
2482 WorkingMI->setDesc(get(X86::SHUFPDrri));
2483 WorkingMI->addOperand(MachineOperand::CreateImm(0x02));
2484 break;
2485 }
2486 case X86::SHUFPDrri: {
2487 // Commute to MOVSD.
2488 assert(MI.getOperand(3).getImm() == 0x02 && "Unexpected immediate!");
2489 WorkingMI = CloneIfNew(MI);
2490 WorkingMI->setDesc(get(X86::MOVSDrr));
2491 WorkingMI->removeOperand(3);
2492 break;
2493 }
2494 case X86::PCLMULQDQrri:
2495 case X86::VPCLMULQDQrri:
2496 case X86::VPCLMULQDQYrri:
2497 case X86::VPCLMULQDQZrri:
2498 case X86::VPCLMULQDQZ128rri:
2499 case X86::VPCLMULQDQZ256rri: {
2500 // SRC1 64bits = Imm[0] ? SRC1[127:64] : SRC1[63:0]
2501 // SRC2 64bits = Imm[4] ? SRC2[127:64] : SRC2[63:0]
2502 unsigned Imm = MI.getOperand(3).getImm();
2503 unsigned Src1Hi = Imm & 0x01;
2504 unsigned Src2Hi = Imm & 0x10;
2505 WorkingMI = CloneIfNew(MI);
2506 WorkingMI->getOperand(3).setImm((Src1Hi << 4) | (Src2Hi >> 4));
2507 break;
2508 }
2509 case X86::VPCMPBZ128rri:
2510 case X86::VPCMPUBZ128rri:
2511 case X86::VPCMPBZ256rri:
2512 case X86::VPCMPUBZ256rri:
2513 case X86::VPCMPBZrri:
2514 case X86::VPCMPUBZrri:
2515 case X86::VPCMPDZ128rri:
2516 case X86::VPCMPUDZ128rri:
2517 case X86::VPCMPDZ256rri:
2518 case X86::VPCMPUDZ256rri:
2519 case X86::VPCMPDZrri:
2520 case X86::VPCMPUDZrri:
2521 case X86::VPCMPQZ128rri:
2522 case X86::VPCMPUQZ128rri:
2523 case X86::VPCMPQZ256rri:
2524 case X86::VPCMPUQZ256rri:
2525 case X86::VPCMPQZrri:
2526 case X86::VPCMPUQZrri:
2527 case X86::VPCMPWZ128rri:
2528 case X86::VPCMPUWZ128rri:
2529 case X86::VPCMPWZ256rri:
2530 case X86::VPCMPUWZ256rri:
2531 case X86::VPCMPWZrri:
2532 case X86::VPCMPUWZrri:
2533 case X86::VPCMPBZ128rrik:
2534 case X86::VPCMPUBZ128rrik:
2535 case X86::VPCMPBZ256rrik:
2536 case X86::VPCMPUBZ256rrik:
2537 case X86::VPCMPBZrrik:
2538 case X86::VPCMPUBZrrik:
2539 case X86::VPCMPDZ128rrik:
2540 case X86::VPCMPUDZ128rrik:
2541 case X86::VPCMPDZ256rrik:
2542 case X86::VPCMPUDZ256rrik:
2543 case X86::VPCMPDZrrik:
2544 case X86::VPCMPUDZrrik:
2545 case X86::VPCMPQZ128rrik:
2546 case X86::VPCMPUQZ128rrik:
2547 case X86::VPCMPQZ256rrik:
2548 case X86::VPCMPUQZ256rrik:
2549 case X86::VPCMPQZrrik:
2550 case X86::VPCMPUQZrrik:
2551 case X86::VPCMPWZ128rrik:
2552 case X86::VPCMPUWZ128rrik:
2553 case X86::VPCMPWZ256rrik:
2554 case X86::VPCMPUWZ256rrik:
2555 case X86::VPCMPWZrrik:
2556 case X86::VPCMPUWZrrik:
2557 WorkingMI = CloneIfNew(MI);
2558 // Flip comparison mode immediate (if necessary).
2559 WorkingMI->getOperand(MI.getNumOperands() - 1)
2561 MI.getOperand(MI.getNumOperands() - 1).getImm() & 0x7));
2562 break;
2563 case X86::VPCOMBri:
2564 case X86::VPCOMUBri:
2565 case X86::VPCOMDri:
2566 case X86::VPCOMUDri:
2567 case X86::VPCOMQri:
2568 case X86::VPCOMUQri:
2569 case X86::VPCOMWri:
2570 case X86::VPCOMUWri:
2571 WorkingMI = CloneIfNew(MI);
2572 // Flip comparison mode immediate (if necessary).
2573 WorkingMI->getOperand(3).setImm(
2574 X86::getSwappedVPCOMImm(MI.getOperand(3).getImm() & 0x7));
2575 break;
2576 case X86::VCMPSDZrri:
2577 case X86::VCMPSSZrri:
2578 case X86::VCMPPDZrri:
2579 case X86::VCMPPSZrri:
2580 case X86::VCMPSHZrri:
2581 case X86::VCMPPHZrri:
2582 case X86::VCMPPHZ128rri:
2583 case X86::VCMPPHZ256rri:
2584 case X86::VCMPPDZ128rri:
2585 case X86::VCMPPSZ128rri:
2586 case X86::VCMPPDZ256rri:
2587 case X86::VCMPPSZ256rri:
2588 case X86::VCMPPDZrrik:
2589 case X86::VCMPPSZrrik:
2590 case X86::VCMPPDZ128rrik:
2591 case X86::VCMPPSZ128rrik:
2592 case X86::VCMPPDZ256rrik:
2593 case X86::VCMPPSZ256rrik:
2594 WorkingMI = CloneIfNew(MI);
2595 WorkingMI->getOperand(MI.getNumExplicitOperands() - 1)
2597 MI.getOperand(MI.getNumExplicitOperands() - 1).getImm() & 0x1f));
2598 break;
2599 case X86::VPERM2F128rri:
2600 case X86::VPERM2I128rri:
2601 // Flip permute source immediate.
2602 // Imm & 0x02: lo = if set, select Op1.lo/hi else Op0.lo/hi.
2603 // Imm & 0x20: hi = if set, select Op1.lo/hi else Op0.lo/hi.
2604 WorkingMI = CloneIfNew(MI);
2605 WorkingMI->getOperand(3).setImm((MI.getOperand(3).getImm() & 0xFF) ^ 0x22);
2606 break;
2607 case X86::MOVHLPSrr:
2608 case X86::UNPCKHPDrr:
2609 case X86::VMOVHLPSrr:
2610 case X86::VUNPCKHPDrr:
2611 case X86::VMOVHLPSZrr:
2612 case X86::VUNPCKHPDZ128rr:
2613 assert(Subtarget.hasSSE2() && "Commuting MOVHLP/UNPCKHPD requires SSE2!");
2614
2615 switch (Opc) {
2616 default:
2617 llvm_unreachable("Unreachable!");
2618 case X86::MOVHLPSrr:
2619 Opc = X86::UNPCKHPDrr;
2620 break;
2621 case X86::UNPCKHPDrr:
2622 Opc = X86::MOVHLPSrr;
2623 break;
2624 case X86::VMOVHLPSrr:
2625 Opc = X86::VUNPCKHPDrr;
2626 break;
2627 case X86::VUNPCKHPDrr:
2628 Opc = X86::VMOVHLPSrr;
2629 break;
2630 case X86::VMOVHLPSZrr:
2631 Opc = X86::VUNPCKHPDZ128rr;
2632 break;
2633 case X86::VUNPCKHPDZ128rr:
2634 Opc = X86::VMOVHLPSZrr;
2635 break;
2636 }
2637 WorkingMI = CloneIfNew(MI);
2638 WorkingMI->setDesc(get(Opc));
2639 break;
2640 CASE_ND(CMOV16rr)
2641 CASE_ND(CMOV32rr)
2642 CASE_ND(CMOV64rr) {
2643 WorkingMI = CloneIfNew(MI);
2644 unsigned OpNo = MI.getDesc().getNumOperands() - 1;
2645 X86::CondCode CC = static_cast<X86::CondCode>(MI.getOperand(OpNo).getImm());
2647 break;
2648 }
2649 case X86::VPTERNLOGDZrri:
2650 case X86::VPTERNLOGDZrmi:
2651 case X86::VPTERNLOGDZ128rri:
2652 case X86::VPTERNLOGDZ128rmi:
2653 case X86::VPTERNLOGDZ256rri:
2654 case X86::VPTERNLOGDZ256rmi:
2655 case X86::VPTERNLOGQZrri:
2656 case X86::VPTERNLOGQZrmi:
2657 case X86::VPTERNLOGQZ128rri:
2658 case X86::VPTERNLOGQZ128rmi:
2659 case X86::VPTERNLOGQZ256rri:
2660 case X86::VPTERNLOGQZ256rmi:
2661 case X86::VPTERNLOGDZrrik:
2662 case X86::VPTERNLOGDZ128rrik:
2663 case X86::VPTERNLOGDZ256rrik:
2664 case X86::VPTERNLOGQZrrik:
2665 case X86::VPTERNLOGQZ128rrik:
2666 case X86::VPTERNLOGQZ256rrik:
2667 case X86::VPTERNLOGDZrrikz:
2668 case X86::VPTERNLOGDZrmikz:
2669 case X86::VPTERNLOGDZ128rrikz:
2670 case X86::VPTERNLOGDZ128rmikz:
2671 case X86::VPTERNLOGDZ256rrikz:
2672 case X86::VPTERNLOGDZ256rmikz:
2673 case X86::VPTERNLOGQZrrikz:
2674 case X86::VPTERNLOGQZrmikz:
2675 case X86::VPTERNLOGQZ128rrikz:
2676 case X86::VPTERNLOGQZ128rmikz:
2677 case X86::VPTERNLOGQZ256rrikz:
2678 case X86::VPTERNLOGQZ256rmikz:
2679 case X86::VPTERNLOGDZ128rmbi:
2680 case X86::VPTERNLOGDZ256rmbi:
2681 case X86::VPTERNLOGDZrmbi:
2682 case X86::VPTERNLOGQZ128rmbi:
2683 case X86::VPTERNLOGQZ256rmbi:
2684 case X86::VPTERNLOGQZrmbi:
2685 case X86::VPTERNLOGDZ128rmbikz:
2686 case X86::VPTERNLOGDZ256rmbikz:
2687 case X86::VPTERNLOGDZrmbikz:
2688 case X86::VPTERNLOGQZ128rmbikz:
2689 case X86::VPTERNLOGQZ256rmbikz:
2690 case X86::VPTERNLOGQZrmbikz: {
2691 WorkingMI = CloneIfNew(MI);
2692 commuteVPTERNLOG(*WorkingMI, OpIdx1, OpIdx2);
2693 break;
2694 }
2695 default:
2697 WorkingMI = CloneIfNew(MI);
2698 WorkingMI->setDesc(get(getCommutedVPERMV3Opcode(Opc)));
2699 break;
2700 }
2701
2702 if (auto *FMA3Group = getFMA3Group(Opc, MI.getDesc().TSFlags)) {
2703 WorkingMI = CloneIfNew(MI);
2704 WorkingMI->setDesc(
2705 get(getFMA3OpcodeToCommuteOperands(MI, OpIdx1, OpIdx2, *FMA3Group)));
2706 break;
2707 }
2708 }
2709 return TargetInstrInfo::commuteInstructionImpl(MI, NewMI, OpIdx1, OpIdx2);
2710}
2711
2712bool X86InstrInfo::findThreeSrcCommutedOpIndices(const MachineInstr &MI,
2713 unsigned &SrcOpIdx1,
2714 unsigned &SrcOpIdx2,
2715 bool IsIntrinsic) const {
2716 uint64_t TSFlags = MI.getDesc().TSFlags;
2717
2718 unsigned FirstCommutableVecOp = 1;
2719 unsigned LastCommutableVecOp = 3;
2720 unsigned KMaskOp = -1U;
2721 if (X86II::isKMasked(TSFlags)) {
2722 // For k-zero-masked operations it is Ok to commute the first vector
2723 // operand. Unless this is an intrinsic instruction.
2724 // For regular k-masked operations a conservative choice is done as the
2725 // elements of the first vector operand, for which the corresponding bit
2726 // in the k-mask operand is set to 0, are copied to the result of the
2727 // instruction.
2728 // TODO/FIXME: The commute still may be legal if it is known that the
2729 // k-mask operand is set to either all ones or all zeroes.
2730 // It is also Ok to commute the 1st operand if all users of MI use only
2731 // the elements enabled by the k-mask operand. For example,
2732 // v4 = VFMADD213PSZrk v1, k, v2, v3; // v1[i] = k[i] ? v2[i]*v1[i]+v3[i]
2733 // : v1[i];
2734 // VMOVAPSZmrk <mem_addr>, k, v4; // this is the ONLY user of v4 ->
2735 // // Ok, to commute v1 in FMADD213PSZrk.
2736
2737 // The k-mask operand has index = 2 for masked and zero-masked operations.
2738 KMaskOp = 2;
2739
2740 // The operand with index = 1 is used as a source for those elements for
2741 // which the corresponding bit in the k-mask is set to 0.
2742 if (X86II::isKMergeMasked(TSFlags) || IsIntrinsic)
2743 FirstCommutableVecOp = 3;
2744
2745 LastCommutableVecOp++;
2746 } else if (IsIntrinsic) {
2747 // Commuting the first operand of an intrinsic instruction isn't possible
2748 // unless we can prove that only the lowest element of the result is used.
2749 FirstCommutableVecOp = 2;
2750 }
2751
2752 if (isMem(MI, LastCommutableVecOp))
2753 LastCommutableVecOp--;
2754
2755 // Only the first RegOpsNum operands are commutable.
2756 // Also, the value 'CommuteAnyOperandIndex' is valid here as it means
2757 // that the operand is not specified/fixed.
2758 if (SrcOpIdx1 != CommuteAnyOperandIndex &&
2759 (SrcOpIdx1 < FirstCommutableVecOp || SrcOpIdx1 > LastCommutableVecOp ||
2760 SrcOpIdx1 == KMaskOp))
2761 return false;
2762 if (SrcOpIdx2 != CommuteAnyOperandIndex &&
2763 (SrcOpIdx2 < FirstCommutableVecOp || SrcOpIdx2 > LastCommutableVecOp ||
2764 SrcOpIdx2 == KMaskOp))
2765 return false;
2766
2767 // Look for two different register operands assumed to be commutable
2768 // regardless of the FMA opcode. The FMA opcode is adjusted later.
2769 if (SrcOpIdx1 == CommuteAnyOperandIndex ||
2770 SrcOpIdx2 == CommuteAnyOperandIndex) {
2771 unsigned CommutableOpIdx2 = SrcOpIdx2;
2772
2773 // At least one of operands to be commuted is not specified and
2774 // this method is free to choose appropriate commutable operands.
2775 if (SrcOpIdx1 == SrcOpIdx2)
2776 // Both of operands are not fixed. By default set one of commutable
2777 // operands to the last register operand of the instruction.
2778 CommutableOpIdx2 = LastCommutableVecOp;
2779 else if (SrcOpIdx2 == CommuteAnyOperandIndex)
2780 // Only one of operands is not fixed.
2781 CommutableOpIdx2 = SrcOpIdx1;
2782
2783 // CommutableOpIdx2 is well defined now. Let's choose another commutable
2784 // operand and assign its index to CommutableOpIdx1.
2785 Register Op2Reg = MI.getOperand(CommutableOpIdx2).getReg();
2786
2787 unsigned CommutableOpIdx1;
2788 for (CommutableOpIdx1 = LastCommutableVecOp;
2789 CommutableOpIdx1 >= FirstCommutableVecOp; CommutableOpIdx1--) {
2790 // Just ignore and skip the k-mask operand.
2791 if (CommutableOpIdx1 == KMaskOp)
2792 continue;
2793
2794 // The commuted operands must have different registers.
2795 // Otherwise, the commute transformation does not change anything and
2796 // is useless then.
2797 if (Op2Reg != MI.getOperand(CommutableOpIdx1).getReg())
2798 break;
2799 }
2800
2801 // No appropriate commutable operands were found.
2802 if (CommutableOpIdx1 < FirstCommutableVecOp)
2803 return false;
2804
2805 // Assign the found pair of commutable indices to SrcOpIdx1 and SrcOpidx2
2806 // to return those values.
2807 if (!fixCommutedOpIndices(SrcOpIdx1, SrcOpIdx2, CommutableOpIdx1,
2808 CommutableOpIdx2))
2809 return false;
2810 }
2811
2812 return true;
2813}
2814
2816 unsigned &SrcOpIdx1,
2817 unsigned &SrcOpIdx2) const {
2818 const MCInstrDesc &Desc = MI.getDesc();
2819 if (!Desc.isCommutable())
2820 return false;
2821
2822 switch (MI.getOpcode()) {
2823 case X86::CMPSDrri:
2824 case X86::CMPSSrri:
2825 case X86::CMPPDrri:
2826 case X86::CMPPSrri:
2827 case X86::VCMPSDrri:
2828 case X86::VCMPSSrri:
2829 case X86::VCMPPDrri:
2830 case X86::VCMPPSrri:
2831 case X86::VCMPPDYrri:
2832 case X86::VCMPPSYrri:
2833 case X86::VCMPSDZrri:
2834 case X86::VCMPSSZrri:
2835 case X86::VCMPPDZrri:
2836 case X86::VCMPPSZrri:
2837 case X86::VCMPSHZrri:
2838 case X86::VCMPPHZrri:
2839 case X86::VCMPPHZ128rri:
2840 case X86::VCMPPHZ256rri:
2841 case X86::VCMPPDZ128rri:
2842 case X86::VCMPPSZ128rri:
2843 case X86::VCMPPDZ256rri:
2844 case X86::VCMPPSZ256rri:
2845 case X86::VCMPPDZrrik:
2846 case X86::VCMPPSZrrik:
2847 case X86::VCMPPDZ128rrik:
2848 case X86::VCMPPSZ128rrik:
2849 case X86::VCMPPDZ256rrik:
2850 case X86::VCMPPSZ256rrik: {
2851 unsigned OpOffset = X86II::isKMasked(Desc.TSFlags) ? 1 : 0;
2852
2853 // Float comparison can be safely commuted for
2854 // Ordered/Unordered/Equal/NotEqual tests
2855 unsigned Imm = MI.getOperand(3 + OpOffset).getImm() & 0x7;
2856 switch (Imm) {
2857 default:
2858 // EVEX versions can be commuted.
2859 if ((Desc.TSFlags & X86II::EncodingMask) == X86II::EVEX)
2860 break;
2861 return false;
2862 case 0x00: // EQUAL
2863 case 0x03: // UNORDERED
2864 case 0x04: // NOT EQUAL
2865 case 0x07: // ORDERED
2866 break;
2867 }
2868
2869 // The indices of the commutable operands are 1 and 2 (or 2 and 3
2870 // when masked).
2871 // Assign them to the returned operand indices here.
2872 return fixCommutedOpIndices(SrcOpIdx1, SrcOpIdx2, 1 + OpOffset,
2873 2 + OpOffset);
2874 }
2875 case X86::MOVSSrr:
2876 // X86::MOVSDrr is always commutable. MOVSS is only commutable if we can
2877 // form sse4.1 blend. We assume VMOVSSrr/VMOVSDrr is always commutable since
2878 // AVX implies sse4.1.
2879 if (Subtarget.hasSSE41())
2880 return TargetInstrInfo::findCommutedOpIndices(MI, SrcOpIdx1, SrcOpIdx2);
2881 return false;
2882 case X86::SHUFPDrri:
2883 // We can commute this to MOVSD.
2884 if (MI.getOperand(3).getImm() == 0x02)
2885 return TargetInstrInfo::findCommutedOpIndices(MI, SrcOpIdx1, SrcOpIdx2);
2886 return false;
2887 case X86::MOVHLPSrr:
2888 case X86::UNPCKHPDrr:
2889 case X86::VMOVHLPSrr:
2890 case X86::VUNPCKHPDrr:
2891 case X86::VMOVHLPSZrr:
2892 case X86::VUNPCKHPDZ128rr:
2893 if (Subtarget.hasSSE2())
2894 return TargetInstrInfo::findCommutedOpIndices(MI, SrcOpIdx1, SrcOpIdx2);
2895 return false;
2896 case X86::VPTERNLOGDZrri:
2897 case X86::VPTERNLOGDZrmi:
2898 case X86::VPTERNLOGDZ128rri:
2899 case X86::VPTERNLOGDZ128rmi:
2900 case X86::VPTERNLOGDZ256rri:
2901 case X86::VPTERNLOGDZ256rmi:
2902 case X86::VPTERNLOGQZrri:
2903 case X86::VPTERNLOGQZrmi:
2904 case X86::VPTERNLOGQZ128rri:
2905 case X86::VPTERNLOGQZ128rmi:
2906 case X86::VPTERNLOGQZ256rri:
2907 case X86::VPTERNLOGQZ256rmi:
2908 case X86::VPTERNLOGDZrrik:
2909 case X86::VPTERNLOGDZ128rrik:
2910 case X86::VPTERNLOGDZ256rrik:
2911 case X86::VPTERNLOGQZrrik:
2912 case X86::VPTERNLOGQZ128rrik:
2913 case X86::VPTERNLOGQZ256rrik:
2914 case X86::VPTERNLOGDZrrikz:
2915 case X86::VPTERNLOGDZrmikz:
2916 case X86::VPTERNLOGDZ128rrikz:
2917 case X86::VPTERNLOGDZ128rmikz:
2918 case X86::VPTERNLOGDZ256rrikz:
2919 case X86::VPTERNLOGDZ256rmikz:
2920 case X86::VPTERNLOGQZrrikz:
2921 case X86::VPTERNLOGQZrmikz:
2922 case X86::VPTERNLOGQZ128rrikz:
2923 case X86::VPTERNLOGQZ128rmikz:
2924 case X86::VPTERNLOGQZ256rrikz:
2925 case X86::VPTERNLOGQZ256rmikz:
2926 case X86::VPTERNLOGDZ128rmbi:
2927 case X86::VPTERNLOGDZ256rmbi:
2928 case X86::VPTERNLOGDZrmbi:
2929 case X86::VPTERNLOGQZ128rmbi:
2930 case X86::VPTERNLOGQZ256rmbi:
2931 case X86::VPTERNLOGQZrmbi:
2932 case X86::VPTERNLOGDZ128rmbikz:
2933 case X86::VPTERNLOGDZ256rmbikz:
2934 case X86::VPTERNLOGDZrmbikz:
2935 case X86::VPTERNLOGQZ128rmbikz:
2936 case X86::VPTERNLOGQZ256rmbikz:
2937 case X86::VPTERNLOGQZrmbikz:
2938 return findThreeSrcCommutedOpIndices(MI, SrcOpIdx1, SrcOpIdx2);
2939 case X86::VPDPWSSDYrr:
2940 case X86::VPDPWSSDrr:
2941 case X86::VPDPWSSDSYrr:
2942 case X86::VPDPWSSDSrr:
2943 case X86::VPDPWUUDrr:
2944 case X86::VPDPWUUDYrr:
2945 case X86::VPDPWUUDSrr:
2946 case X86::VPDPWUUDSYrr:
2947 case X86::VPDPBSSDSrr:
2948 case X86::VPDPBSSDSYrr:
2949 case X86::VPDPBSSDrr:
2950 case X86::VPDPBSSDYrr:
2951 case X86::VPDPBUUDSrr:
2952 case X86::VPDPBUUDSYrr:
2953 case X86::VPDPBUUDrr:
2954 case X86::VPDPBUUDYrr:
2955 case X86::VPDPBSSDSZ128r:
2956 case X86::VPDPBSSDSZ128rk:
2957 case X86::VPDPBSSDSZ128rkz:
2958 case X86::VPDPBSSDSZ256r:
2959 case X86::VPDPBSSDSZ256rk:
2960 case X86::VPDPBSSDSZ256rkz:
2961 case X86::VPDPBSSDSZr:
2962 case X86::VPDPBSSDSZrk:
2963 case X86::VPDPBSSDSZrkz:
2964 case X86::VPDPBSSDZ128r:
2965 case X86::VPDPBSSDZ128rk:
2966 case X86::VPDPBSSDZ128rkz:
2967 case X86::VPDPBSSDZ256r:
2968 case X86::VPDPBSSDZ256rk:
2969 case X86::VPDPBSSDZ256rkz:
2970 case X86::VPDPBSSDZr:
2971 case X86::VPDPBSSDZrk:
2972 case X86::VPDPBSSDZrkz:
2973 case X86::VPDPBUUDSZ128r:
2974 case X86::VPDPBUUDSZ128rk:
2975 case X86::VPDPBUUDSZ128rkz:
2976 case X86::VPDPBUUDSZ256r:
2977 case X86::VPDPBUUDSZ256rk:
2978 case X86::VPDPBUUDSZ256rkz:
2979 case X86::VPDPBUUDSZr:
2980 case X86::VPDPBUUDSZrk:
2981 case X86::VPDPBUUDSZrkz:
2982 case X86::VPDPBUUDZ128r:
2983 case X86::VPDPBUUDZ128rk:
2984 case X86::VPDPBUUDZ128rkz:
2985 case X86::VPDPBUUDZ256r:
2986 case X86::VPDPBUUDZ256rk:
2987 case X86::VPDPBUUDZ256rkz:
2988 case X86::VPDPBUUDZr:
2989 case X86::VPDPBUUDZrk:
2990 case X86::VPDPBUUDZrkz:
2991 case X86::VPDPWSSDZ128r:
2992 case X86::VPDPWSSDZ128rk:
2993 case X86::VPDPWSSDZ128rkz:
2994 case X86::VPDPWSSDZ256r:
2995 case X86::VPDPWSSDZ256rk:
2996 case X86::VPDPWSSDZ256rkz:
2997 case X86::VPDPWSSDZr:
2998 case X86::VPDPWSSDZrk:
2999 case X86::VPDPWSSDZrkz:
3000 case X86::VPDPWSSDSZ128r:
3001 case X86::VPDPWSSDSZ128rk:
3002 case X86::VPDPWSSDSZ128rkz:
3003 case X86::VPDPWSSDSZ256r:
3004 case X86::VPDPWSSDSZ256rk:
3005 case X86::VPDPWSSDSZ256rkz:
3006 case X86::VPDPWSSDSZr:
3007 case X86::VPDPWSSDSZrk:
3008 case X86::VPDPWSSDSZrkz:
3009 case X86::VPDPWUUDZ128r:
3010 case X86::VPDPWUUDZ128rk:
3011 case X86::VPDPWUUDZ128rkz:
3012 case X86::VPDPWUUDZ256r:
3013 case X86::VPDPWUUDZ256rk:
3014 case X86::VPDPWUUDZ256rkz:
3015 case X86::VPDPWUUDZr:
3016 case X86::VPDPWUUDZrk:
3017 case X86::VPDPWUUDZrkz:
3018 case X86::VPDPWUUDSZ128r:
3019 case X86::VPDPWUUDSZ128rk:
3020 case X86::VPDPWUUDSZ128rkz:
3021 case X86::VPDPWUUDSZ256r:
3022 case X86::VPDPWUUDSZ256rk:
3023 case X86::VPDPWUUDSZ256rkz:
3024 case X86::VPDPWUUDSZr:
3025 case X86::VPDPWUUDSZrk:
3026 case X86::VPDPWUUDSZrkz:
3027 case X86::VPMADD52HUQrr:
3028 case X86::VPMADD52HUQYrr:
3029 case X86::VPMADD52HUQZ128r:
3030 case X86::VPMADD52HUQZ128rk:
3031 case X86::VPMADD52HUQZ128rkz:
3032 case X86::VPMADD52HUQZ256r:
3033 case X86::VPMADD52HUQZ256rk:
3034 case X86::VPMADD52HUQZ256rkz:
3035 case X86::VPMADD52HUQZr:
3036 case X86::VPMADD52HUQZrk:
3037 case X86::VPMADD52HUQZrkz:
3038 case X86::VPMADD52LUQrr:
3039 case X86::VPMADD52LUQYrr:
3040 case X86::VPMADD52LUQZ128r:
3041 case X86::VPMADD52LUQZ128rk:
3042 case X86::VPMADD52LUQZ128rkz:
3043 case X86::VPMADD52LUQZ256r:
3044 case X86::VPMADD52LUQZ256rk:
3045 case X86::VPMADD52LUQZ256rkz:
3046 case X86::VPMADD52LUQZr:
3047 case X86::VPMADD52LUQZrk:
3048 case X86::VPMADD52LUQZrkz:
3049 case X86::VFMADDCPHZr:
3050 case X86::VFMADDCPHZrk:
3051 case X86::VFMADDCPHZrkz:
3052 case X86::VFMADDCPHZ128r:
3053 case X86::VFMADDCPHZ128rk:
3054 case X86::VFMADDCPHZ128rkz:
3055 case X86::VFMADDCPHZ256r:
3056 case X86::VFMADDCPHZ256rk:
3057 case X86::VFMADDCPHZ256rkz:
3058 case X86::VFMADDCSHZr:
3059 case X86::VFMADDCSHZrk:
3060 case X86::VFMADDCSHZrkz: {
3061 unsigned CommutableOpIdx1 = 2;
3062 unsigned CommutableOpIdx2 = 3;
3063 if (X86II::isKMasked(Desc.TSFlags)) {
3064 // Skip the mask register.
3065 ++CommutableOpIdx1;
3066 ++CommutableOpIdx2;
3067 }
3068 if (!fixCommutedOpIndices(SrcOpIdx1, SrcOpIdx2, CommutableOpIdx1,
3069 CommutableOpIdx2))
3070 return false;
3071 if (!MI.getOperand(SrcOpIdx1).isReg() || !MI.getOperand(SrcOpIdx2).isReg())
3072 // No idea.
3073 return false;
3074 return true;
3075 }
3076
3077 default:
3078 const X86InstrFMA3Group *FMA3Group =
3079 getFMA3Group(MI.getOpcode(), MI.getDesc().TSFlags);
3080 if (FMA3Group)
3081 return findThreeSrcCommutedOpIndices(MI, SrcOpIdx1, SrcOpIdx2,
3082 FMA3Group->isIntrinsic());
3083
3084 // Handled masked instructions since we need to skip over the mask input
3085 // and the preserved input.
3086 if (X86II::isKMasked(Desc.TSFlags)) {
3087 // First assume that the first input is the mask operand and skip past it.
3088 unsigned CommutableOpIdx1 = Desc.getNumDefs() + 1;
3089 unsigned CommutableOpIdx2 = Desc.getNumDefs() + 2;
3090 // Check if the first input is tied. If there isn't one then we only
3091 // need to skip the mask operand which we did above.
3092 if ((MI.getDesc().getOperandConstraint(Desc.getNumDefs(),
3093 MCOI::TIED_TO) != -1)) {
3094 // If this is zero masking instruction with a tied operand, we need to
3095 // move the first index back to the first input since this must
3096 // be a 3 input instruction and we want the first two non-mask inputs.
3097 // Otherwise this is a 2 input instruction with a preserved input and
3098 // mask, so we need to move the indices to skip one more input.
3099 if (X86II::isKMergeMasked(Desc.TSFlags)) {
3100 ++CommutableOpIdx1;
3101 ++CommutableOpIdx2;
3102 } else {
3103 --CommutableOpIdx1;
3104 }
3105 }
3106
3107 if (!fixCommutedOpIndices(SrcOpIdx1, SrcOpIdx2, CommutableOpIdx1,
3108 CommutableOpIdx2))
3109 return false;
3110
3111 if (!MI.getOperand(SrcOpIdx1).isReg() ||
3112 !MI.getOperand(SrcOpIdx2).isReg())
3113 // No idea.
3114 return false;
3115 return true;
3116 }
3117
3118 return TargetInstrInfo::findCommutedOpIndices(MI, SrcOpIdx1, SrcOpIdx2);
3119 }
3120 return false;
3121}
3122
3124 unsigned Opcode = MI->getOpcode();
3125 if (Opcode != X86::LEA32r && Opcode != X86::LEA64r &&
3126 Opcode != X86::LEA64_32r)
3127 return false;
3128
3129 const MachineOperand &Scale = MI->getOperand(1 + X86::AddrScaleAmt);
3130 const MachineOperand &Disp = MI->getOperand(1 + X86::AddrDisp);
3131 const MachineOperand &Segment = MI->getOperand(1 + X86::AddrSegmentReg);
3132
3133 if (Segment.getReg() != 0 || !Disp.isImm() || Disp.getImm() != 0 ||
3134 Scale.getImm() > 1)
3135 return false;
3136
3137 return true;
3138}
3139
3141 // Currently we're interested in following sequence only.
3142 // r3 = lea r1, r2
3143 // r5 = add r3, r4
3144 // Both r3 and r4 are killed in add, we hope the add instruction has the
3145 // operand order
3146 // r5 = add r4, r3
3147 // So later in X86FixupLEAs the lea instruction can be rewritten as add.
3148 unsigned Opcode = MI.getOpcode();
3149 if (Opcode != X86::ADD32rr && Opcode != X86::ADD64rr)
3150 return false;
3151
3152 const MachineRegisterInfo &MRI = MI.getParent()->getParent()->getRegInfo();
3153 Register Reg1 = MI.getOperand(1).getReg();
3154 Register Reg2 = MI.getOperand(2).getReg();
3155
3156 // Check if Reg1 comes from LEA in the same MBB.
3157 if (MachineInstr *Inst = MRI.getUniqueVRegDef(Reg1)) {
3158 if (isConvertibleLEA(Inst) && Inst->getParent() == MI.getParent()) {
3159 Commute = true;
3160 return true;
3161 }
3162 }
3163
3164 // Check if Reg2 comes from LEA in the same MBB.
3165 if (MachineInstr *Inst = MRI.getUniqueVRegDef(Reg2)) {
3166 if (isConvertibleLEA(Inst) && Inst->getParent() == MI.getParent()) {
3167 Commute = false;
3168 return true;
3169 }
3170 }
3171
3172 return false;
3173}
3174
3176 unsigned Opcode = MCID.getOpcode();
3177 if (!(X86::isJCC(Opcode) || X86::isSETCC(Opcode) || X86::isSETZUCC(Opcode) ||
3178 X86::isCMOVCC(Opcode) || X86::isCFCMOVCC(Opcode) ||
3179 X86::isCCMPCC(Opcode) || X86::isCTESTCC(Opcode)))
3180 return -1;
3181 // Assume that condition code is always the last use operand.
3182 unsigned NumUses = MCID.getNumOperands() - MCID.getNumDefs();
3183 return NumUses - 1;
3184}
3185
3187 const MCInstrDesc &MCID = MI.getDesc();
3188 int CondNo = getCondSrcNoFromDesc(MCID);
3189 if (CondNo < 0)
3190 return X86::COND_INVALID;
3191 CondNo += MCID.getNumDefs();
3192 return static_cast<X86::CondCode>(MI.getOperand(CondNo).getImm());
3193}
3194
3196 return X86::isJCC(MI.getOpcode()) ? X86::getCondFromMI(MI)
3198}
3199
3201 return X86::isSETCC(MI.getOpcode()) || X86::isSETZUCC(MI.getOpcode())
3204}
3205
3207 return X86::isCMOVCC(MI.getOpcode()) ? X86::getCondFromMI(MI)
3209}
3210
3212 return X86::isCFCMOVCC(MI.getOpcode()) ? X86::getCondFromMI(MI)
3214}
3215
3217 return X86::isCCMPCC(MI.getOpcode()) || X86::isCTESTCC(MI.getOpcode())
3220}
3221
3223 // CCMP/CTEST has two conditional operands:
3224 // - SCC: source conditonal code (same as CMOV)
3225 // - DCF: destination conditional flags, which has 4 valid bits
3226 //
3227 // +----+----+----+----+
3228 // | OF | SF | ZF | CF |
3229 // +----+----+----+----+
3230 //
3231 // If SCC(source conditional code) evaluates to false, CCMP/CTEST will updates
3232 // the conditional flags by as follows:
3233 //
3234 // OF = DCF.OF
3235 // SF = DCF.SF
3236 // ZF = DCF.ZF
3237 // CF = DCF.CF
3238 // PF = DCF.CF
3239 // AF = 0 (Auxiliary Carry Flag)
3240 //
3241 // Otherwise, the CMP or TEST is executed and it updates the
3242 // CSPAZO flags normally.
3243 //
3244 // NOTE:
3245 // If SCC = P, then SCC evaluates to true regardless of the CSPAZO value.
3246 // If SCC = NP, then SCC evaluates to false regardless of the CSPAZO value.
3247
3248 enum { CF = 1, ZF = 2, SF = 4, OF = 8, PF = CF };
3249
3250 switch (CC) {
3251 default:
3252 llvm_unreachable("Illegal condition code!");
3253 case X86::COND_NO:
3254 case X86::COND_NE:
3255 case X86::COND_GE:
3256 case X86::COND_G:
3257 case X86::COND_AE:
3258 case X86::COND_A:
3259 case X86::COND_NS:
3260 case X86::COND_NP:
3261 return 0;
3262 case X86::COND_O:
3263 return OF;
3264 case X86::COND_B:
3265 case X86::COND_BE:
3266 return CF;
3267 break;
3268 case X86::COND_E:
3269 case X86::COND_LE:
3270 return ZF;
3271 case X86::COND_S:
3272 case X86::COND_L:
3273 return SF;
3274 case X86::COND_P:
3275 return PF;
3276 }
3277}
3278
3279#define GET_X86_NF_TRANSFORM_TABLE
3280#define GET_X86_ND2NONND_TABLE
3281#include "X86GenInstrMapping.inc"
3282
3284 unsigned Opc) {
3285 const auto I = llvm::lower_bound(Table, Opc);
3286 return (I == Table.end() || I->OldOpc != Opc) ? 0U : I->NewOpc;
3287}
3288unsigned X86::getNFVariant(unsigned Opc) {
3289 return getNewOpcFromTable(X86NFTransformTable, Opc);
3290}
3291
3292unsigned X86::getNonNDVariant(unsigned Opc) {
3293 return getNewOpcFromTable(X86ND2NonNDTable, Opc);
3294}
3295
3296/// Return the inverse of the specified condition,
3297/// e.g. turning COND_E to COND_NE.
3299 switch (CC) {
3300 default:
3301 llvm_unreachable("Illegal condition code!");
3302 case X86::COND_E:
3303 return X86::COND_NE;
3304 case X86::COND_NE:
3305 return X86::COND_E;
3306 case X86::COND_L:
3307 return X86::COND_GE;
3308 case X86::COND_LE:
3309 return X86::COND_G;
3310 case X86::COND_G:
3311 return X86::COND_LE;
3312 case X86::COND_GE:
3313 return X86::COND_L;
3314 case X86::COND_B:
3315 return X86::COND_AE;
3316 case X86::COND_BE:
3317 return X86::COND_A;
3318 case X86::COND_A:
3319 return X86::COND_BE;
3320 case X86::COND_AE:
3321 return X86::COND_B;
3322 case X86::COND_S:
3323 return X86::COND_NS;
3324 case X86::COND_NS:
3325 return X86::COND_S;
3326 case X86::COND_P:
3327 return X86::COND_NP;
3328 case X86::COND_NP:
3329 return X86::COND_P;
3330 case X86::COND_O:
3331 return X86::COND_NO;
3332 case X86::COND_NO:
3333 return X86::COND_O;
3334 case X86::COND_NE_OR_P:
3335 return X86::COND_E_AND_NP;
3336 case X86::COND_E_AND_NP:
3337 return X86::COND_NE_OR_P;
3338 }
3339}
3340
3341/// Assuming the flags are set by MI(a,b), return the condition code if we
3342/// modify the instructions such that flags are set by MI(b,a).
3344 switch (CC) {
3345 default:
3346 return X86::COND_INVALID;
3347 case X86::COND_E:
3348 return X86::COND_E;
3349 case X86::COND_NE:
3350 return X86::COND_NE;
3351 case X86::COND_L:
3352 return X86::COND_G;
3353 case X86::COND_LE:
3354 return X86::COND_GE;
3355 case X86::COND_G:
3356 return X86::COND_L;
3357 case X86::COND_GE:
3358 return X86::COND_LE;
3359 case X86::COND_B:
3360 return X86::COND_A;
3361 case X86::COND_BE:
3362 return X86::COND_AE;
3363 case X86::COND_A:
3364 return X86::COND_B;
3365 case X86::COND_AE:
3366 return X86::COND_BE;
3367 }
3368}
3369
3370std::pair<X86::CondCode, bool>
3373 bool NeedSwap = false;
3374 switch (Predicate) {
3375 default:
3376 break;
3377 // Floating-point Predicates
3378 case CmpInst::FCMP_UEQ:
3379 CC = X86::COND_E;
3380 break;
3381 case CmpInst::FCMP_OLT:
3382 NeedSwap = true;
3383 [[fallthrough]];
3384 case CmpInst::FCMP_OGT:
3385 CC = X86::COND_A;
3386 break;
3387 case CmpInst::FCMP_OLE:
3388 NeedSwap = true;
3389 [[fallthrough]];
3390 case CmpInst::FCMP_OGE:
3391 CC = X86::COND_AE;
3392 break;
3393 case CmpInst::FCMP_UGT:
3394 NeedSwap = true;
3395 [[fallthrough]];
3396 case CmpInst::FCMP_ULT:
3397 CC = X86::COND_B;
3398 break;
3399 case CmpInst::FCMP_UGE:
3400 NeedSwap = true;
3401 [[fallthrough]];
3402 case CmpInst::FCMP_ULE:
3403 CC = X86::COND_BE;
3404 break;
3405 case CmpInst::FCMP_ONE:
3406 CC = X86::COND_NE;
3407 break;
3408 case CmpInst::FCMP_UNO:
3409 CC = X86::COND_P;
3410 break;
3411 case CmpInst::FCMP_ORD:
3412 CC = X86::COND_NP;
3413 break;
3414 case CmpInst::FCMP_OEQ:
3415 [[fallthrough]];
3416 case CmpInst::FCMP_UNE:
3418 break;
3419
3420 // Integer Predicates
3421 case CmpInst::ICMP_EQ:
3422 CC = X86::COND_E;
3423 break;
3424 case CmpInst::ICMP_NE:
3425 CC = X86::COND_NE;
3426 break;
3427 case CmpInst::ICMP_UGT:
3428 CC = X86::COND_A;
3429 break;
3430 case CmpInst::ICMP_UGE:
3431 CC = X86::COND_AE;
3432 break;
3433 case CmpInst::ICMP_ULT:
3434 CC = X86::COND_B;
3435 break;
3436 case CmpInst::ICMP_ULE:
3437 CC = X86::COND_BE;
3438 break;
3439 case CmpInst::ICMP_SGT:
3440 CC = X86::COND_G;
3441 break;
3442 case CmpInst::ICMP_SGE:
3443 CC = X86::COND_GE;
3444 break;
3445 case CmpInst::ICMP_SLT:
3446 CC = X86::COND_L;
3447 break;
3448 case CmpInst::ICMP_SLE:
3449 CC = X86::COND_LE;
3450 break;
3451 }
3452
3453 return std::make_pair(CC, NeedSwap);
3454}
3455
3456/// Return a cmov opcode for the given register size in bytes, and operand type.
3457unsigned X86::getCMovOpcode(unsigned RegBytes, bool HasMemoryOperand,
3458 bool HasNDD) {
3459 switch (RegBytes) {
3460 default:
3461 llvm_unreachable("Illegal register size!");
3462#define GET_ND_IF_ENABLED(OPC) (HasNDD ? OPC##_ND : OPC)
3463 case 2:
3464 return HasMemoryOperand ? GET_ND_IF_ENABLED(X86::CMOV16rm)
3465 : GET_ND_IF_ENABLED(X86::CMOV16rr);
3466 case 4:
3467 return HasMemoryOperand ? GET_ND_IF_ENABLED(X86::CMOV32rm)
3468 : GET_ND_IF_ENABLED(X86::CMOV32rr);
3469 case 8:
3470 return HasMemoryOperand ? GET_ND_IF_ENABLED(X86::CMOV64rm)
3471 : GET_ND_IF_ENABLED(X86::CMOV64rr);
3472 }
3473}
3474
3475/// Get the VPCMP immediate for the given condition.
3477 switch (CC) {
3478 default:
3479 llvm_unreachable("Unexpected SETCC condition");
3480 case ISD::SETNE:
3481 return 4;
3482 case ISD::SETEQ:
3483 return 0;
3484 case ISD::SETULT:
3485 case ISD::SETLT:
3486 return 1;
3487 case ISD::SETUGT:
3488 case ISD::SETGT:
3489 return 6;
3490 case ISD::SETUGE:
3491 case ISD::SETGE:
3492 return 5;
3493 case ISD::SETULE:
3494 case ISD::SETLE:
3495 return 2;
3496 }
3497}
3498
3499/// Get the VPCMP immediate if the operands are swapped.
3500unsigned X86::getSwappedVPCMPImm(unsigned Imm) {
3501 switch (Imm) {
3502 default:
3503 llvm_unreachable("Unreachable!");
3504 case 0x01:
3505 Imm = 0x06;
3506 break; // LT -> NLE
3507 case 0x02:
3508 Imm = 0x05;
3509 break; // LE -> NLT
3510 case 0x05:
3511 Imm = 0x02;
3512 break; // NLT -> LE
3513 case 0x06:
3514 Imm = 0x01;
3515 break; // NLE -> LT
3516 case 0x00: // EQ
3517 case 0x03: // FALSE
3518 case 0x04: // NE
3519 case 0x07: // TRUE
3520 break;
3521 }
3522
3523 return Imm;
3524}
3525
3526/// Get the VPCOM immediate if the operands are swapped.
3527unsigned X86::getSwappedVPCOMImm(unsigned Imm) {
3528 switch (Imm) {
3529 default:
3530 llvm_unreachable("Unreachable!");
3531 case 0x00:
3532 Imm = 0x02;
3533 break; // LT -> GT
3534 case 0x01:
3535 Imm = 0x03;
3536 break; // LE -> GE
3537 case 0x02:
3538 Imm = 0x00;
3539 break; // GT -> LT
3540 case 0x03:
3541 Imm = 0x01;
3542 break; // GE -> LE
3543 case 0x04: // EQ
3544 case 0x05: // NE
3545 case 0x06: // FALSE
3546 case 0x07: // TRUE
3547 break;
3548 }
3549
3550 return Imm;
3551}
3552
3553/// Get the VCMP immediate if the operands are swapped.
3554unsigned X86::getSwappedVCMPImm(unsigned Imm) {
3555 // Only need the lower 2 bits to distinquish.
3556 switch (Imm & 0x3) {
3557 default:
3558 llvm_unreachable("Unreachable!");
3559 case 0x00:
3560 case 0x03:
3561 // EQ/NE/TRUE/FALSE/ORD/UNORD don't change immediate when commuted.
3562 break;
3563 case 0x01:
3564 case 0x02:
3565 // Need to toggle bits 3:0. Bit 4 stays the same.
3566 Imm ^= 0xf;
3567 break;
3568 }
3569
3570 return Imm;
3571}
3572
3574 if (Info.RegClass == X86::VR128RegClassID ||
3575 Info.RegClass == X86::VR128XRegClassID)
3576 return 128;
3577 if (Info.RegClass == X86::VR256RegClassID ||
3578 Info.RegClass == X86::VR256XRegClassID)
3579 return 256;
3580 if (Info.RegClass == X86::VR512RegClassID)
3581 return 512;
3582 llvm_unreachable("Unknown register class!");
3583}
3584
3585/// Return true if the Reg is X87 register.
3586static bool isX87Reg(unsigned Reg) {
3587 return (Reg == X86::FPCW || Reg == X86::FPSW ||
3588 (Reg >= X86::ST0 && Reg <= X86::ST7));
3589}
3590
3591/// check if the instruction is X87 instruction
3593 // Call and inlineasm defs X87 register, so we special case it here because
3594 // otherwise calls are incorrectly flagged as x87 instructions
3595 // as a result.
3596 if (MI.isCall() || MI.isInlineAsm())
3597 return false;
3598 for (const MachineOperand &MO : MI.operands()) {
3599 if (!MO.isReg())
3600 continue;
3601 if (isX87Reg(MO.getReg()))
3602 return true;
3603 }
3604 return false;
3605}
3606
3608 auto IsMemOp = [](const MCOperandInfo &OpInfo) {
3609 return OpInfo.OperandType == MCOI::OPERAND_MEMORY;
3610 };
3611
3612 const MCInstrDesc &Desc = MI.getDesc();
3613
3614 // Directly invoke the MC-layer routine for real (i.e., non-pseudo)
3615 // instructions (fast case).
3616 if (!X86II::isPseudo(Desc.TSFlags)) {
3617 int MemRefIdx = X86II::getMemoryOperandNo(Desc.TSFlags);
3618 if (MemRefIdx >= 0)
3619 return MemRefIdx + X86II::getOperandBias(Desc);
3620#ifdef EXPENSIVE_CHECKS
3621 assert(none_of(Desc.operands(), IsMemOp) &&
3622 "Got false negative from X86II::getMemoryOperandNo()!");
3623#endif
3624 return -1;
3625 }
3626
3627 // Otherwise, handle pseudo instructions by examining the type of their
3628 // operands (slow case). An instruction cannot have a memory reference if it
3629 // has fewer than AddrNumOperands (= 5) explicit operands.
3630 unsigned NumOps = Desc.getNumOperands();
3631 if (NumOps < X86::AddrNumOperands) {
3632#ifdef EXPENSIVE_CHECKS
3633 assert(none_of(Desc.operands(), IsMemOp) &&
3634 "Expected no operands to have OPERAND_MEMORY type!");
3635#endif
3636 return -1;
3637 }
3638
3639 // The first operand with type OPERAND_MEMORY indicates the start of a memory
3640 // reference. We expect the following AddrNumOperand-1 operands to also have
3641 // OPERAND_MEMORY type.
3642 for (unsigned I = 0, E = NumOps - X86::AddrNumOperands; I != E; ++I) {
3643 if (IsMemOp(Desc.operands()[I])) {
3644#ifdef EXPENSIVE_CHECKS
3645 assert(std::all_of(Desc.operands().begin() + I,
3646 Desc.operands().begin() + I + X86::AddrNumOperands,
3647 IsMemOp) &&
3648 "Expected all five operands in the memory reference to have "
3649 "OPERAND_MEMORY type!");
3650#endif
3651 return I;
3652 }
3653 }
3654
3655 return -1;
3656}
3657
3659 unsigned OpNo) {
3660 assert(MI.getNumOperands() >= (OpNo + X86::AddrNumOperands) &&
3661 "Unexpected number of operands!");
3662
3663 const MachineOperand &Index = MI.getOperand(OpNo + X86::AddrIndexReg);
3664 if (!Index.isReg() || Index.getReg() != X86::NoRegister)
3665 return nullptr;
3666
3667 const MachineOperand &Disp = MI.getOperand(OpNo + X86::AddrDisp);
3668 if (!Disp.isCPI() || Disp.getOffset() != 0)
3669 return nullptr;
3670
3672 MI.getParent()->getParent()->getConstantPool()->getConstants();
3673 const MachineConstantPoolEntry &ConstantEntry = Constants[Disp.getIndex()];
3674
3675 // Bail if this is a machine constant pool entry, we won't be able to dig out
3676 // anything useful.
3677 if (ConstantEntry.isMachineConstantPoolEntry())
3678 return nullptr;
3679
3680 return ConstantEntry.Val.ConstVal;
3681}
3682
3684 switch (MI.getOpcode()) {
3685 case X86::TCRETURNdi:
3686 case X86::TCRETURNri:
3687 case X86::TCRETURNmi:
3688 case X86::TCRETURNdi64:
3689 case X86::TCRETURNri64:
3690 case X86::TCRETURNmi64:
3691 return true;
3692 default:
3693 return false;
3694 }
3695}
3696
3699 const MachineInstr &TailCall) const {
3700
3701 const MachineFunction *MF = TailCall.getMF();
3702
3703 if (MF->getTarget().getCodeModel() == CodeModel::Kernel) {
3704 // Kernel patches thunk calls in runtime, these should never be conditional.
3705 const MachineOperand &Target = TailCall.getOperand(0);
3706 if (Target.isSymbol()) {
3707 StringRef Symbol(Target.getSymbolName());
3708 // this is currently only relevant to r11/kernel indirect thunk.
3709 if (Symbol == "__x86_indirect_thunk_r11")
3710 return false;
3711 }
3712 }
3713
3714 if (TailCall.getOpcode() != X86::TCRETURNdi &&
3715 TailCall.getOpcode() != X86::TCRETURNdi64) {
3716 // Only direct calls can be done with a conditional branch.
3717 return false;
3718 }
3719
3720 if (Subtarget.isTargetWin64() && MF->hasWinCFI()) {
3721 // Conditional tail calls confuse the Win64 unwinder.
3722 return false;
3723 }
3724
3725 assert(BranchCond.size() == 1);
3726 if (BranchCond[0].getImm() > X86::LAST_VALID_COND) {
3727 // Can't make a conditional tail call with this condition.
3728 return false;
3729 }
3730
3732 if (X86FI->getTCReturnAddrDelta() != 0 ||
3733 TailCall.getOperand(1).getImm() != 0) {
3734 // A conditional tail call cannot do any stack adjustment.
3735 return false;
3736 }
3737
3738 return true;
3739}
3740
3743 const MachineInstr &TailCall) const {
3744 assert(canMakeTailCallConditional(BranchCond, TailCall));
3745
3747 while (I != MBB.begin()) {
3748 --I;
3749 if (I->isDebugInstr())
3750 continue;
3751 if (!I->isBranch())
3752 assert(0 && "Can't find the branch to replace!");
3753
3755 assert(BranchCond.size() == 1);
3756 if (CC != BranchCond[0].getImm())
3757 continue;
3758
3759 break;
3760 }
3761
3762 unsigned Opc = TailCall.getOpcode() == X86::TCRETURNdi ? X86::TCRETURNdicc
3763 : X86::TCRETURNdi64cc;
3764
3765 auto MIB = BuildMI(MBB, I, MBB.findDebugLoc(I), get(Opc));
3766 MIB->addOperand(TailCall.getOperand(0)); // Destination.
3767 MIB.addImm(0); // Stack offset (not used).
3768 MIB->addOperand(BranchCond[0]); // Condition.
3769 MIB.copyImplicitOps(TailCall); // Regmask and (imp-used) parameters.
3770
3771 // Add implicit uses and defs of all live regs potentially clobbered by the
3772 // call. This way they still appear live across the call.
3773 LivePhysRegs LiveRegs(getRegisterInfo());
3774 LiveRegs.addLiveOuts(MBB);
3776 LiveRegs.stepForward(*MIB, Clobbers);
3777 for (const auto &C : Clobbers) {
3778 MIB.addReg(C.first, RegState::Implicit);
3780 }
3781
3782 I->eraseFromParent();
3783}
3784
3785// Given a MBB and its TBB, find the FBB which was a fallthrough MBB (it may
3786// not be a fallthrough MBB now due to layout changes). Return nullptr if the
3787// fallthrough MBB cannot be identified.
3790 // Look for non-EHPad successors other than TBB. If we find exactly one, it
3791 // is the fallthrough MBB. If we find zero, then TBB is both the target MBB
3792 // and fallthrough MBB. If we find more than one, we cannot identify the
3793 // fallthrough MBB and should return nullptr.
3794 MachineBasicBlock *FallthroughBB = nullptr;
3795 for (MachineBasicBlock *Succ : MBB->successors()) {
3796 if (Succ->isEHPad() || (Succ == TBB && FallthroughBB))
3797 continue;
3798 // Return a nullptr if we found more than one fallthrough successor.
3799 if (FallthroughBB && FallthroughBB != TBB)
3800 return nullptr;
3801 FallthroughBB = Succ;
3802 }
3803 return FallthroughBB;
3804}
3805
3806bool X86InstrInfo::analyzeBranchImpl(
3809 SmallVectorImpl<MachineInstr *> &CondBranches, bool AllowModify) const {
3810
3811 // Start from the bottom of the block and work up, examining the
3812 // terminator instructions.
3814 MachineBasicBlock::iterator UnCondBrIter = MBB.end();
3815 while (I != MBB.begin()) {
3816 --I;
3817 if (I->isDebugInstr())
3818 continue;
3819
3820 // Working from the bottom, when we see a non-terminator instruction, we're
3821 // done.
3822 if (!isUnpredicatedTerminator(*I))
3823 break;
3824
3825 // A terminator that isn't a branch can't easily be handled by this
3826 // analysis.
3827 if (!I->isBranch())
3828 return true;
3829
3830 // Handle unconditional branches.
3831 if (I->getOpcode() == X86::JMP_1) {
3832 UnCondBrIter = I;
3833
3834 if (!AllowModify) {
3835 TBB = I->getOperand(0).getMBB();
3836 continue;
3837 }
3838
3839 // If the block has any instructions after a JMP, delete them.
3840 MBB.erase(std::next(I), MBB.end());
3841
3842 Cond.clear();
3843 FBB = nullptr;
3844
3845 // Delete the JMP if it's equivalent to a fall-through.
3846 if (MBB.isLayoutSuccessor(I->getOperand(0).getMBB())) {
3847 TBB = nullptr;
3848 I->eraseFromParent();
3849 I = MBB.end();
3850 UnCondBrIter = MBB.end();
3851 continue;
3852 }
3853
3854 // TBB is used to indicate the unconditional destination.
3855 TBB = I->getOperand(0).getMBB();
3856 continue;
3857 }
3858
3859 // Handle conditional branches.
3860 X86::CondCode BranchCode = X86::getCondFromBranch(*I);
3861 if (BranchCode == X86::COND_INVALID)
3862 return true; // Can't handle indirect branch.
3863
3864 // In practice we should never have an undef eflags operand, if we do
3865 // abort here as we are not prepared to preserve the flag.
3866 if (I->findRegisterUseOperand(X86::EFLAGS, /*TRI=*/nullptr)->isUndef())
3867 return true;
3868
3869 // Working from the bottom, handle the first conditional branch.
3870 if (Cond.empty()) {
3871 FBB = TBB;
3872 TBB = I->getOperand(0).getMBB();
3873 Cond.push_back(MachineOperand::CreateImm(BranchCode));
3874 CondBranches.push_back(&*I);
3875 continue;
3876 }
3877
3878 // Handle subsequent conditional branches. Only handle the case where all
3879 // conditional branches branch to the same destination and their condition
3880 // opcodes fit one of the special multi-branch idioms.
3881 assert(Cond.size() == 1);
3882 assert(TBB);
3883
3884 // If the conditions are the same, we can leave them alone.
3885 X86::CondCode OldBranchCode = (X86::CondCode)Cond[0].getImm();
3886 auto NewTBB = I->getOperand(0).getMBB();
3887 if (OldBranchCode == BranchCode && TBB == NewTBB)
3888 continue;
3889
3890 // If they differ, see if they fit one of the known patterns. Theoretically,
3891 // we could handle more patterns here, but we shouldn't expect to see them
3892 // if instruction selection has done a reasonable job.
3893 if (TBB == NewTBB &&
3894 ((OldBranchCode == X86::COND_P && BranchCode == X86::COND_NE) ||
3895 (OldBranchCode == X86::COND_NE && BranchCode == X86::COND_P))) {
3896 BranchCode = X86::COND_NE_OR_P;
3897 } else if ((OldBranchCode == X86::COND_NP && BranchCode == X86::COND_NE) ||
3898 (OldBranchCode == X86::COND_E && BranchCode == X86::COND_P)) {
3899 if (NewTBB != (FBB ? FBB : getFallThroughMBB(&MBB, TBB)))
3900 return true;
3901
3902 // X86::COND_E_AND_NP usually has two different branch destinations.
3903 //
3904 // JP B1
3905 // JE B2
3906 // JMP B1
3907 // B1:
3908 // B2:
3909 //
3910 // Here this condition branches to B2 only if NP && E. It has another
3911 // equivalent form:
3912 //
3913 // JNE B1
3914 // JNP B2
3915 // JMP B1
3916 // B1:
3917 // B2:
3918 //
3919 // Similarly it branches to B2 only if E && NP. That is why this condition
3920 // is named with COND_E_AND_NP.
3921 BranchCode = X86::COND_E_AND_NP;
3922 } else
3923 return true;
3924
3925 // Update the MachineOperand.
3926 Cond[0].setImm(BranchCode);
3927 CondBranches.push_back(&*I);
3928 }
3929
3930 return false;
3931}
3932
3935 MachineBasicBlock *&FBB,
3937 bool AllowModify) const {
3938 SmallVector<MachineInstr *, 4> CondBranches;
3939 return analyzeBranchImpl(MBB, TBB, FBB, Cond, CondBranches, AllowModify);
3940}
3941
3943 const MCInstrDesc &Desc = MI.getDesc();
3944 int MemRefBegin = X86II::getMemoryOperandNo(Desc.TSFlags);
3945 assert(MemRefBegin >= 0 && "instr should have memory operand");
3946 MemRefBegin += X86II::getOperandBias(Desc);
3947
3948 const MachineOperand &MO = MI.getOperand(MemRefBegin + X86::AddrDisp);
3949 if (!MO.isJTI())
3950 return -1;
3951
3952 return MO.getIndex();
3953}
3954
3956 Register Reg) {
3957 if (!Reg.isVirtual())
3958 return -1;
3959 MachineInstr *MI = MRI.getUniqueVRegDef(Reg);
3960 if (MI == nullptr)
3961 return -1;
3962 unsigned Opcode = MI->getOpcode();
3963 if (Opcode != X86::LEA64r && Opcode != X86::LEA32r)
3964 return -1;
3966}
3967
3969 unsigned Opcode = MI.getOpcode();
3970 // Switch-jump pattern for non-PIC code looks like:
3971 // JMP64m $noreg, 8, %X, %jump-table.X, $noreg
3972 if (Opcode == X86::JMP64m || Opcode == X86::JMP32m) {
3974 }
3975 // The pattern for PIC code looks like:
3976 // %0 = LEA64r $rip, 1, $noreg, %jump-table.X
3977 // %1 = MOVSX64rm32 %0, 4, XX, 0, $noreg
3978 // %2 = ADD64rr %1, %0
3979 // JMP64r %2
3980 if (Opcode == X86::JMP64r || Opcode == X86::JMP32r) {
3981 Register Reg = MI.getOperand(0).getReg();
3982 if (!Reg.isVirtual())
3983 return -1;
3984 const MachineFunction &MF = *MI.getParent()->getParent();
3985 const MachineRegisterInfo &MRI = MF.getRegInfo();
3986 MachineInstr *Add = MRI.getUniqueVRegDef(Reg);
3987 if (Add == nullptr)
3988 return -1;
3989 if (Add->getOpcode() != X86::ADD64rr && Add->getOpcode() != X86::ADD32rr)
3990 return -1;
3991 int JTI1 = getJumpTableIndexFromReg(MRI, Add->getOperand(1).getReg());
3992 if (JTI1 >= 0)
3993 return JTI1;
3994 int JTI2 = getJumpTableIndexFromReg(MRI, Add->getOperand(2).getReg());
3995 if (JTI2 >= 0)
3996 return JTI2;
3997 }
3998 return -1;
3999}
4000
4002 MachineBranchPredicate &MBP,
4003 bool AllowModify) const {
4004 using namespace std::placeholders;
4005
4007 SmallVector<MachineInstr *, 4> CondBranches;
4008 if (analyzeBranchImpl(MBB, MBP.TrueDest, MBP.FalseDest, Cond, CondBranches,
4009 AllowModify))
4010 return true;
4011
4012 if (Cond.size() != 1)
4013 return true;
4014
4015 assert(MBP.TrueDest && "expected!");
4016
4017 if (!MBP.FalseDest)
4018 MBP.FalseDest = MBB.getNextNode();
4019
4021
4022 MachineInstr *ConditionDef = nullptr;
4023 bool SingleUseCondition = true;
4024
4026 if (MI.modifiesRegister(X86::EFLAGS, TRI)) {
4027 ConditionDef = &MI;
4028 break;
4029 }
4030
4031 if (MI.readsRegister(X86::EFLAGS, TRI))
4032 SingleUseCondition = false;
4033 }
4034
4035 if (!ConditionDef)
4036 return true;
4037
4038 if (SingleUseCondition) {
4039 for (auto *Succ : MBB.successors())
4040 if (Succ->isLiveIn(X86::EFLAGS))
4041 SingleUseCondition = false;
4042 }
4043
4044 MBP.ConditionDef = ConditionDef;
4045 MBP.SingleUseCondition = SingleUseCondition;
4046
4047 // Currently we only recognize the simple pattern:
4048 //
4049 // test %reg, %reg
4050 // je %label
4051 //
4052 const unsigned TestOpcode =
4053 Subtarget.is64Bit() ? X86::TEST64rr : X86::TEST32rr;
4054
4055 if (ConditionDef->getOpcode() == TestOpcode &&
4056 ConditionDef->getNumOperands() == 3 &&
4057 ConditionDef->getOperand(0).isIdenticalTo(ConditionDef->getOperand(1)) &&
4058 (Cond[0].getImm() == X86::COND_NE || Cond[0].getImm() == X86::COND_E)) {
4059 MBP.LHS = ConditionDef->getOperand(0);
4060 MBP.RHS = MachineOperand::CreateImm(0);
4061 MBP.Predicate = Cond[0].getImm() == X86::COND_NE
4062 ? MachineBranchPredicate::PRED_NE
4063 : MachineBranchPredicate::PRED_EQ;
4064 return false;
4065 }
4066
4067 return true;
4068}
4069
4071 int *BytesRemoved) const {
4072 assert(!BytesRemoved && "code size not handled");
4073
4075 unsigned Count = 0;
4076
4077 while (I != MBB.begin()) {
4078 --I;
4079 if (I->isDebugInstr())
4080 continue;
4081 if (I->getOpcode() != X86::JMP_1 &&
4083 break;
4084 // Remove the branch.
4085 I->eraseFromParent();
4086 I = MBB.end();
4087 ++Count;
4088 }
4089
4090 return Count;
4091}
4092
4095 MachineBasicBlock *FBB,
4097 const DebugLoc &DL, int *BytesAdded) const {
4098 // Shouldn't be a fall through.
4099 assert(TBB && "insertBranch must not be told to insert a fallthrough");
4100 assert((Cond.size() == 1 || Cond.size() == 0) &&
4101 "X86 branch conditions have one component!");
4102 assert(!BytesAdded && "code size not handled");
4103
4104 if (Cond.empty()) {
4105 // Unconditional branch?
4106 assert(!FBB && "Unconditional branch with multiple successors!");
4107 BuildMI(&MBB, DL, get(X86::JMP_1)).addMBB(TBB);
4108 return 1;
4109 }
4110
4111 // If FBB is null, it is implied to be a fall-through block.
4112 bool FallThru = FBB == nullptr;
4113
4114 // Conditional branch.
4115 unsigned Count = 0;
4116 X86::CondCode CC = (X86::CondCode)Cond[0].getImm();
4117 switch (CC) {
4118 case X86::COND_NE_OR_P:
4119 // Synthesize NE_OR_P with two branches.
4120 BuildMI(&MBB, DL, get(X86::JCC_1)).addMBB(TBB).addImm(X86::COND_NE);
4121 ++Count;
4122 BuildMI(&MBB, DL, get(X86::JCC_1)).addMBB(TBB).addImm(X86::COND_P);
4123 ++Count;
4124 break;
4125 case X86::COND_E_AND_NP:
4126 // Use the next block of MBB as FBB if it is null.
4127 if (FBB == nullptr) {
4128 FBB = getFallThroughMBB(&MBB, TBB);
4129 assert(FBB && "MBB cannot be the last block in function when the false "
4130 "body is a fall-through.");
4131 }
4132 // Synthesize COND_E_AND_NP with two branches.
4133 BuildMI(&MBB, DL, get(X86::JCC_1)).addMBB(FBB).addImm(X86::COND_NE);
4134 ++Count;
4135 BuildMI(&MBB, DL, get(X86::JCC_1)).addMBB(TBB).addImm(X86::COND_NP);
4136 ++Count;
4137 break;
4138 default: {
4139 BuildMI(&MBB, DL, get(X86::JCC_1)).addMBB(TBB).addImm(CC);
4140 ++Count;
4141 }
4142 }
4143 if (!FallThru) {
4144 // Two-way Conditional branch. Insert the second branch.
4145 BuildMI(&MBB, DL, get(X86::JMP_1)).addMBB(FBB);
4146 ++Count;
4147 }
4148 return Count;
4149}
4150
4153 Register DstReg, Register TrueReg,
4154 Register FalseReg, int &CondCycles,
4155 int &TrueCycles, int &FalseCycles) const {
4156 // Not all subtargets have cmov instructions.
4157 if (!Subtarget.canUseCMOV())
4158 return false;
4159 if (Cond.size() != 1)
4160 return false;
4161 // We cannot do the composite conditions, at least not in SSA form.
4162 if ((X86::CondCode)Cond[0].getImm() > X86::LAST_VALID_COND)
4163 return false;
4164
4165 // Check register classes.
4167 const TargetRegisterClass *RC =
4168 RI.getCommonSubClass(MRI.getRegClass(TrueReg), MRI.getRegClass(FalseReg));
4169 if (!RC)
4170 return false;
4171
4172 // We have cmov instructions for 16, 32, and 64 bit general purpose registers.
4173 if (X86::GR16RegClass.hasSubClassEq(RC) ||
4174 X86::GR32RegClass.hasSubClassEq(RC) ||
4175 X86::GR64RegClass.hasSubClassEq(RC)) {
4176 // This latency applies to Pentium M, Merom, Wolfdale, Nehalem, and Sandy
4177 // Bridge. Probably Ivy Bridge as well.
4178 CondCycles = 2;
4179 TrueCycles = 2;
4180 FalseCycles = 2;
4181 return true;
4182 }
4183
4184 // Can't do vectors.
4185 return false;
4186}
4187
4190 const DebugLoc &DL, Register DstReg,
4192 Register FalseReg) const {
4194 const TargetRegisterInfo &TRI = *MRI.getTargetRegisterInfo();
4195 const TargetRegisterClass &RC = *MRI.getRegClass(DstReg);
4196 assert(Cond.size() == 1 && "Invalid Cond array");
4197 unsigned Opc =
4198 X86::getCMovOpcode(TRI.getRegSizeInBits(RC) / 8,
4199 false /*HasMemoryOperand*/, Subtarget.hasNDD());
4200 BuildMI(MBB, I, DL, get(Opc), DstReg)
4201 .addReg(FalseReg)
4202 .addReg(TrueReg)
4203 .addImm(Cond[0].getImm());
4204}
4205
4206/// Test if the given register is a physical h register.
4207static bool isHReg(unsigned Reg) {
4208 return X86::GR8_ABCD_HRegClass.contains(Reg);
4209}
4210
4211// Try and copy between VR128/VR64 and GR64 registers.
4212static unsigned CopyToFromAsymmetricReg(unsigned DestReg, unsigned SrcReg,
4213 const X86Subtarget &Subtarget) {
4214 bool HasAVX = Subtarget.hasAVX();
4215 bool HasAVX512 = Subtarget.hasAVX512();
4216 bool HasEGPR = Subtarget.hasEGPR();
4217
4218 // SrcReg(MaskReg) -> DestReg(GR64)
4219 // SrcReg(MaskReg) -> DestReg(GR32)
4220
4221 // All KMASK RegClasses hold the same k registers, can be tested against
4222 // anyone.
4223 if (X86::VK16RegClass.contains(SrcReg)) {
4224 if (X86::GR64RegClass.contains(DestReg)) {
4225 assert(Subtarget.hasBWI());
4226 return HasEGPR ? X86::KMOVQrk_EVEX : X86::KMOVQrk;
4227 }
4228 if (X86::GR32RegClass.contains(DestReg))
4229 return Subtarget.hasBWI() ? (HasEGPR ? X86::KMOVDrk_EVEX : X86::KMOVDrk)
4230 : (HasEGPR ? X86::KMOVWrk_EVEX : X86::KMOVWrk);
4231 }
4232
4233 // SrcReg(GR64) -> DestReg(MaskReg)
4234 // SrcReg(GR32) -> DestReg(MaskReg)
4235
4236 // All KMASK RegClasses hold the same k registers, can be tested against
4237 // anyone.
4238 if (X86::VK16RegClass.contains(DestReg)) {
4239 if (X86::GR64RegClass.contains(SrcReg)) {
4240 assert(Subtarget.hasBWI());
4241 return HasEGPR ? X86::KMOVQkr_EVEX : X86::KMOVQkr;
4242 }
4243 if (X86::GR32RegClass.contains(SrcReg))
4244 return Subtarget.hasBWI() ? (HasEGPR ? X86::KMOVDkr_EVEX : X86::KMOVDkr)
4245 : (HasEGPR ? X86::KMOVWkr_EVEX : X86::KMOVWkr);
4246 }
4247
4248 // SrcReg(VR128) -> DestReg(GR64)
4249 // SrcReg(VR64) -> DestReg(GR64)
4250 // SrcReg(GR64) -> DestReg(VR128)
4251 // SrcReg(GR64) -> DestReg(VR64)
4252
4253 if (X86::GR64RegClass.contains(DestReg)) {
4254 if (X86::VR128XRegClass.contains(SrcReg))
4255 // Copy from a VR128 register to a GR64 register.
4256 return HasAVX512 ? X86::VMOVPQIto64Zrr
4257 : HasAVX ? X86::VMOVPQIto64rr
4258 : X86::MOVPQIto64rr;
4259 if (X86::VR64RegClass.contains(SrcReg))
4260 // Copy from a VR64 register to a GR64 register.
4261 return X86::MMX_MOVD64from64rr;
4262 } else if (X86::GR64RegClass.contains(SrcReg)) {
4263 // Copy from a GR64 register to a VR128 register.
4264 if (X86::VR128XRegClass.contains(DestReg))
4265 return HasAVX512 ? X86::VMOV64toPQIZrr
4266 : HasAVX ? X86::VMOV64toPQIrr
4267 : X86::MOV64toPQIrr;
4268 // Copy from a GR64 register to a VR64 register.
4269 if (X86::VR64RegClass.contains(DestReg))
4270 return X86::MMX_MOVD64to64rr;
4271 }
4272
4273 // SrcReg(VR128) -> DestReg(GR32)
4274 // SrcReg(GR32) -> DestReg(VR128)
4275
4276 if (X86::GR32RegClass.contains(DestReg) &&
4277 X86::VR128XRegClass.contains(SrcReg))
4278 // Copy from a VR128 register to a GR32 register.
4279 return HasAVX512 ? X86::VMOVPDI2DIZrr
4280 : HasAVX ? X86::VMOVPDI2DIrr
4281 : X86::MOVPDI2DIrr;
4282
4283 if (X86::VR128XRegClass.contains(DestReg) &&
4284 X86::GR32RegClass.contains(SrcReg))
4285 // Copy from a VR128 register to a VR128 register.
4286 return HasAVX512 ? X86::VMOVDI2PDIZrr
4287 : HasAVX ? X86::VMOVDI2PDIrr
4288 : X86::MOVDI2PDIrr;
4289 return 0;
4290}
4291
4294 const DebugLoc &DL, MCRegister DestReg,
4295 MCRegister SrcReg, bool KillSrc,
4296 bool RenamableDest, bool RenamableSrc) const {
4297 // First deal with the normal symmetric copies.
4298 bool HasAVX = Subtarget.hasAVX();
4299 bool HasVLX = Subtarget.hasVLX();
4300 bool HasEGPR = Subtarget.hasEGPR();
4301 unsigned Opc = 0;
4302 if (X86::GR64RegClass.contains(DestReg, SrcReg))
4303 Opc = X86::MOV64rr;
4304 else if (X86::GR32RegClass.contains(DestReg, SrcReg))
4305 Opc = X86::MOV32rr;
4306 else if (X86::GR16RegClass.contains(DestReg, SrcReg))
4307 Opc = X86::MOV16rr;
4308 else if (X86::GR8RegClass.contains(DestReg, SrcReg)) {
4309 // Copying to or from a physical H register on x86-64 requires a NOREX
4310 // move. Otherwise use a normal move.
4311 if ((isHReg(DestReg) || isHReg(SrcReg)) && Subtarget.is64Bit()) {
4312 Opc = X86::MOV8rr_NOREX;
4313 // Both operands must be encodable without an REX prefix.
4314 assert(X86::GR8_NOREXRegClass.contains(SrcReg, DestReg) &&
4315 "8-bit H register can not be copied outside GR8_NOREX");
4316 } else
4317 Opc = X86::MOV8rr;
4318 } else if (X86::VR64RegClass.contains(DestReg, SrcReg))
4319 Opc = X86::MMX_MOVQ64rr;
4320 else if (X86::VR128XRegClass.contains(DestReg, SrcReg)) {
4321 if (HasVLX)
4322 Opc = X86::VMOVAPSZ128rr;
4323 else if (X86::VR128RegClass.contains(DestReg, SrcReg))
4324 Opc = HasAVX ? X86::VMOVAPSrr : X86::MOVAPSrr;
4325 else {
4326 // If this an extended register and we don't have VLX we need to use a
4327 // 512-bit move.
4328 Opc = X86::VMOVAPSZrr;
4330 DestReg =
4331 TRI->getMatchingSuperReg(DestReg, X86::sub_xmm, &X86::VR512RegClass);
4332 SrcReg =
4333 TRI->getMatchingSuperReg(SrcReg, X86::sub_xmm, &X86::VR512RegClass);
4334 }
4335 } else if (X86::VR256XRegClass.contains(DestReg, SrcReg)) {
4336 if (HasVLX)
4337 Opc = X86::VMOVAPSZ256rr;
4338 else if (X86::VR256RegClass.contains(DestReg, SrcReg))
4339 Opc = X86::VMOVAPSYrr;
4340 else {
4341 // If this an extended register and we don't have VLX we need to use a
4342 // 512-bit move.
4343 Opc = X86::VMOVAPSZrr;
4345 DestReg =
4346 TRI->getMatchingSuperReg(DestReg, X86::sub_ymm, &X86::VR512RegClass);
4347 SrcReg =
4348 TRI->getMatchingSuperReg(SrcReg, X86::sub_ymm, &X86::VR512RegClass);
4349 }
4350 } else if (X86::VR512RegClass.contains(DestReg, SrcReg))
4351 Opc = X86::VMOVAPSZrr;
4352 // All KMASK RegClasses hold the same k registers, can be tested against
4353 // anyone.
4354 else if (X86::VK16RegClass.contains(DestReg, SrcReg))
4355 Opc = Subtarget.hasBWI() ? (HasEGPR ? X86::KMOVQkk_EVEX : X86::KMOVQkk)
4356 : (HasEGPR ? X86::KMOVQkk_EVEX : X86::KMOVWkk);
4357 if (!Opc)
4358 Opc = CopyToFromAsymmetricReg(DestReg, SrcReg, Subtarget);
4359
4360 if (Opc) {
4361 BuildMI(MBB, MI, DL, get(Opc), DestReg)
4362 .addReg(SrcReg, getKillRegState(KillSrc));
4363 return;
4364 }
4365
4366 if (SrcReg == X86::EFLAGS || DestReg == X86::EFLAGS) {
4367 // FIXME: We use a fatal error here because historically LLVM has tried
4368 // lower some of these physreg copies and we want to ensure we get
4369 // reasonable bug reports if someone encounters a case no other testing
4370 // found. This path should be removed after the LLVM 7 release.
4371 report_fatal_error("Unable to copy EFLAGS physical register!");
4372 }
4373
4374 LLVM_DEBUG(dbgs() << "Cannot copy " << RI.getName(SrcReg) << " to "
4375 << RI.getName(DestReg) << '\n');
4376 report_fatal_error("Cannot emit physreg copy instruction");
4377}
4378
4379std::optional<DestSourcePair>
4381 if (MI.isMoveReg()) {
4382 // FIXME: Dirty hack for apparent invariant that doesn't hold when
4383 // subreg_to_reg is coalesced with ordinary copies, such that the bits that
4384 // were asserted as 0 are now undef.
4385 if (MI.getOperand(0).isUndef() && MI.getOperand(0).getSubReg())
4386 return std::nullopt;
4387
4388 return DestSourcePair{MI.getOperand(0), MI.getOperand(1)};
4389 }
4390 return std::nullopt;
4391}
4392
4393static unsigned getLoadStoreOpcodeForFP16(bool Load, const X86Subtarget &STI) {
4394 if (STI.hasFP16())
4395 return Load ? X86::VMOVSHZrm_alt : X86::VMOVSHZmr;
4396 if (Load)
4397 return STI.hasAVX512() ? X86::VMOVSSZrm
4398 : STI.hasAVX() ? X86::VMOVSSrm
4399 : X86::MOVSSrm;
4400 else
4401 return STI.hasAVX512() ? X86::VMOVSSZmr
4402 : STI.hasAVX() ? X86::VMOVSSmr
4403 : X86::MOVSSmr;
4404}
4405
4407 const TargetRegisterClass *RC,
4408 bool IsStackAligned,
4409 const X86Subtarget &STI, bool Load) {
4410 bool HasAVX = STI.hasAVX();
4411 bool HasAVX512 = STI.hasAVX512();
4412 bool HasVLX = STI.hasVLX();
4413 bool HasEGPR = STI.hasEGPR();
4414
4415 assert(RC != nullptr && "Invalid target register class");
4416 switch (STI.getRegisterInfo()->getSpillSize(*RC)) {
4417 default:
4418 llvm_unreachable("Unknown spill size");
4419 case 1:
4420 assert(X86::GR8RegClass.hasSubClassEq(RC) && "Unknown 1-byte regclass");
4421 if (STI.is64Bit())
4422 // Copying to or from a physical H register on x86-64 requires a NOREX
4423 // move. Otherwise use a normal move.
4424 if (isHReg(Reg) || X86::GR8_ABCD_HRegClass.hasSubClassEq(RC))
4425 return Load ? X86::MOV8rm_NOREX : X86::MOV8mr_NOREX;
4426 return Load ? X86::MOV8rm : X86::MOV8mr;
4427 case 2:
4428 if (X86::VK16RegClass.hasSubClassEq(RC))
4429 return Load ? (HasEGPR ? X86::KMOVWkm_EVEX : X86::KMOVWkm)
4430 : (HasEGPR ? X86::KMOVWmk_EVEX : X86::KMOVWmk);
4431 assert(X86::GR16RegClass.hasSubClassEq(RC) && "Unknown 2-byte regclass");
4432 return Load ? X86::MOV16rm : X86::MOV16mr;
4433 case 4:
4434 if (X86::GR32RegClass.hasSubClassEq(RC))
4435 return Load ? X86::MOV32rm : X86::MOV32mr;
4436 if (X86::FR32XRegClass.hasSubClassEq(RC))
4437 return Load ? (HasAVX512 ? X86::VMOVSSZrm_alt
4438 : HasAVX ? X86::VMOVSSrm_alt
4439 : X86::MOVSSrm_alt)
4440 : (HasAVX512 ? X86::VMOVSSZmr
4441 : HasAVX ? X86::VMOVSSmr
4442 : X86::MOVSSmr);
4443 if (X86::RFP32RegClass.hasSubClassEq(RC))
4444 return Load ? X86::LD_Fp32m : X86::ST_Fp32m;
4445 if (X86::VK32RegClass.hasSubClassEq(RC)) {
4446 assert(STI.hasBWI() && "KMOVD requires BWI");
4447 return Load ? (HasEGPR ? X86::KMOVDkm_EVEX : X86::KMOVDkm)
4448 : (HasEGPR ? X86::KMOVDmk_EVEX : X86::KMOVDmk);
4449 }
4450 // All of these mask pair classes have the same spill size, the same kind
4451 // of kmov instructions can be used with all of them.
4452 if (X86::VK1PAIRRegClass.hasSubClassEq(RC) ||
4453 X86::VK2PAIRRegClass.hasSubClassEq(RC) ||
4454 X86::VK4PAIRRegClass.hasSubClassEq(RC) ||
4455 X86::VK8PAIRRegClass.hasSubClassEq(RC) ||
4456 X86::VK16PAIRRegClass.hasSubClassEq(RC))
4457 return Load ? X86::MASKPAIR16LOAD : X86::MASKPAIR16STORE;
4458 if (X86::FR16RegClass.hasSubClassEq(RC) ||
4459 X86::FR16XRegClass.hasSubClassEq(RC))
4460 return getLoadStoreOpcodeForFP16(Load, STI);
4461 llvm_unreachable("Unknown 4-byte regclass");
4462 case 8:
4463 if (X86::GR64RegClass.hasSubClassEq(RC))
4464 return Load ? X86::MOV64rm : X86::MOV64mr;
4465 if (X86::FR64XRegClass.hasSubClassEq(RC))
4466 return Load ? (HasAVX512 ? X86::VMOVSDZrm_alt
4467 : HasAVX ? X86::VMOVSDrm_alt
4468 : X86::MOVSDrm_alt)
4469 : (HasAVX512 ? X86::VMOVSDZmr
4470 : HasAVX ? X86::VMOVSDmr
4471 : X86::MOVSDmr);
4472 if (X86::VR64RegClass.hasSubClassEq(RC))
4473 return Load ? X86::MMX_MOVQ64rm : X86::MMX_MOVQ64mr;
4474 if (X86::RFP64RegClass.hasSubClassEq(RC))
4475 return Load ? X86::LD_Fp64m : X86::ST_Fp64m;
4476 if (X86::VK64RegClass.hasSubClassEq(RC)) {
4477 assert(STI.hasBWI() && "KMOVQ requires BWI");
4478 return Load ? (HasEGPR ? X86::KMOVQkm_EVEX : X86::KMOVQkm)
4479 : (HasEGPR ? X86::KMOVQmk_EVEX : X86::KMOVQmk);
4480 }
4481 llvm_unreachable("Unknown 8-byte regclass");
4482 case 10:
4483 assert(X86::RFP80RegClass.hasSubClassEq(RC) && "Unknown 10-byte regclass");
4484 return Load ? X86::LD_Fp80m : X86::ST_FpP80m;
4485 case 16: {
4486 if (X86::VR128XRegClass.hasSubClassEq(RC)) {
4487 // If stack is realigned we can use aligned stores.
4488 if (IsStackAligned)
4489 return Load ? (HasVLX ? X86::VMOVAPSZ128rm
4490 : HasAVX512 ? X86::VMOVAPSZ128rm_NOVLX
4491 : HasAVX ? X86::VMOVAPSrm
4492 : X86::MOVAPSrm)
4493 : (HasVLX ? X86::VMOVAPSZ128mr
4494 : HasAVX512 ? X86::VMOVAPSZ128mr_NOVLX
4495 : HasAVX ? X86::VMOVAPSmr
4496 : X86::MOVAPSmr);
4497 else
4498 return Load ? (HasVLX ? X86::VMOVUPSZ128rm
4499 : HasAVX512 ? X86::VMOVUPSZ128rm_NOVLX
4500 : HasAVX ? X86::VMOVUPSrm
4501 : X86::MOVUPSrm)
4502 : (HasVLX ? X86::VMOVUPSZ128mr
4503 : HasAVX512 ? X86::VMOVUPSZ128mr_NOVLX
4504 : HasAVX ? X86::VMOVUPSmr
4505 : X86::MOVUPSmr);
4506 }
4507 llvm_unreachable("Unknown 16-byte regclass");
4508 }
4509 case 32:
4510 assert(X86::VR256XRegClass.hasSubClassEq(RC) && "Unknown 32-byte regclass");
4511 // If stack is realigned we can use aligned stores.
4512 if (IsStackAligned)
4513 return Load ? (HasVLX ? X86::VMOVAPSZ256rm
4514 : HasAVX512 ? X86::VMOVAPSZ256rm_NOVLX
4515 : X86::VMOVAPSYrm)
4516 : (HasVLX ? X86::VMOVAPSZ256mr
4517 : HasAVX512 ? X86::VMOVAPSZ256mr_NOVLX
4518 : X86::VMOVAPSYmr);
4519 else
4520 return Load ? (HasVLX ? X86::VMOVUPSZ256rm
4521 : HasAVX512 ? X86::VMOVUPSZ256rm_NOVLX
4522 : X86::VMOVUPSYrm)
4523 : (HasVLX ? X86::VMOVUPSZ256mr
4524 : HasAVX512 ? X86::VMOVUPSZ256mr_NOVLX
4525 : X86::VMOVUPSYmr);
4526 case 64:
4527 assert(X86::VR512RegClass.hasSubClassEq(RC) && "Unknown 64-byte regclass");
4528 assert(STI.hasAVX512() && "Using 512-bit register requires AVX512");
4529 if (IsStackAligned)
4530 return Load ? X86::VMOVAPSZrm : X86::VMOVAPSZmr;
4531 else
4532 return Load ? X86::VMOVUPSZrm : X86::VMOVUPSZmr;
4533 case 1024:
4534 assert(X86::TILERegClass.hasSubClassEq(RC) && "Unknown 1024-byte regclass");
4535 assert(STI.hasAMXTILE() && "Using 8*1024-bit register requires AMX-TILE");
4536#define GET_EGPR_IF_ENABLED(OPC) (STI.hasEGPR() ? OPC##_EVEX : OPC)
4537 return Load ? GET_EGPR_IF_ENABLED(X86::TILELOADD)
4538 : GET_EGPR_IF_ENABLED(X86::TILESTORED);
4539#undef GET_EGPR_IF_ENABLED
4540 case 2048:
4541 assert(X86::TILEPAIRRegClass.hasSubClassEq(RC) &&
4542 "Unknown 2048-byte regclass");
4543 assert(STI.hasAMXTILE() && "Using 2048-bit register requires AMX-TILE");
4544 return Load ? X86::PTILEPAIRLOAD : X86::PTILEPAIRSTORE;
4545 }
4546}
4547
4548std::optional<ExtAddrMode>
4550 const TargetRegisterInfo *TRI) const {
4551 const MCInstrDesc &Desc = MemI.getDesc();
4552 int MemRefBegin = X86II::getMemoryOperandNo(Desc.TSFlags);
4553 if (MemRefBegin < 0)
4554 return std::nullopt;
4555
4556 MemRefBegin += X86II::getOperandBias(Desc);
4557
4558 auto &BaseOp = MemI.getOperand(MemRefBegin + X86::AddrBaseReg);
4559 if (!BaseOp.isReg()) // Can be an MO_FrameIndex
4560 return std::nullopt;
4561
4562 const MachineOperand &DispMO = MemI.getOperand(MemRefBegin + X86::AddrDisp);
4563 // Displacement can be symbolic
4564 if (!DispMO.isImm())
4565 return std::nullopt;
4566
4567 ExtAddrMode AM;
4568 AM.BaseReg = BaseOp.getReg();
4569 AM.ScaledReg = MemI.getOperand(MemRefBegin + X86::AddrIndexReg).getReg();
4570 AM.Scale = MemI.getOperand(MemRefBegin + X86::AddrScaleAmt).getImm();
4571 AM.Displacement = DispMO.getImm();
4572 return AM;
4573}
4574
4576 StringRef &ErrInfo) const {
4577 std::optional<ExtAddrMode> AMOrNone = getAddrModeFromMemoryOp(MI, nullptr);
4578 if (!AMOrNone)
4579 return true;
4580
4581 ExtAddrMode AM = *AMOrNone;
4583 if (AM.ScaledReg != X86::NoRegister) {
4584 switch (AM.Scale) {
4585 case 1:
4586 case 2:
4587 case 4:
4588 case 8:
4589 break;
4590 default:
4591 ErrInfo = "Scale factor in address must be 1, 2, 4 or 8";
4592 return false;
4593 }
4594 }
4595 if (!isInt<32>(AM.Displacement)) {
4596 ErrInfo = "Displacement in address must fit into 32-bit signed "
4597 "integer";
4598 return false;
4599 }
4600
4601 return true;
4602}
4603
4605 const Register Reg,
4606 int64_t &ImmVal) const {
4607 Register MovReg = Reg;
4608 const MachineInstr *MovMI = &MI;
4609
4610 // Follow use-def for SUBREG_TO_REG to find the real move immediate
4611 // instruction. It is quite common for x86-64.
4612 if (MI.isSubregToReg()) {
4613 // We use following pattern to setup 64b immediate.
4614 // %8:gr32 = MOV32r0 implicit-def dead $eflags
4615 // %6:gr64 = SUBREG_TO_REG 0, killed %8:gr32, %subreg.sub_32bit
4616 if (!MI.getOperand(1).isImm())
4617 return false;
4618 unsigned FillBits = MI.getOperand(1).getImm();
4619 unsigned SubIdx = MI.getOperand(3).getImm();
4620 MovReg = MI.getOperand(2).getReg();
4621 if (SubIdx != X86::sub_32bit || FillBits != 0)
4622 return false;
4623 const MachineRegisterInfo &MRI = MI.getParent()->getParent()->getRegInfo();
4624 MovMI = MRI.getUniqueVRegDef(MovReg);
4625 if (!MovMI)
4626 return false;
4627 }
4628
4629 if (MovMI->getOpcode() == X86::MOV32r0 &&
4630 MovMI->getOperand(0).getReg() == MovReg) {
4631 ImmVal = 0;
4632 return true;
4633 }
4634
4635 if (MovMI->getOpcode() != X86::MOV32ri &&
4636 MovMI->getOpcode() != X86::MOV64ri &&
4637 MovMI->getOpcode() != X86::MOV32ri64 && MovMI->getOpcode() != X86::MOV8ri)
4638 return false;
4639 // Mov Src can be a global address.
4640 if (!MovMI->getOperand(1).isImm() || MovMI->getOperand(0).getReg() != MovReg)
4641 return false;
4642 ImmVal = MovMI->getOperand(1).getImm();
4643 return true;
4644}
4645
4647 const MachineInstr *MI, const Register NullValueReg,
4648 const TargetRegisterInfo *TRI) const {
4649 if (!MI->modifiesRegister(NullValueReg, TRI))
4650 return true;
4651 switch (MI->getOpcode()) {
4652 // Shift right/left of a null unto itself is still a null, i.e. rax = shl rax
4653 // X.
4654 case X86::SHR64ri:
4655 case X86::SHR32ri:
4656 case X86::SHL64ri:
4657 case X86::SHL32ri:
4658 assert(MI->getOperand(0).isDef() && MI->getOperand(1).isUse() &&
4659 "expected for shift opcode!");
4660 return MI->getOperand(0).getReg() == NullValueReg &&
4661 MI->getOperand(1).getReg() == NullValueReg;
4662 // Zero extend of a sub-reg of NullValueReg into itself does not change the
4663 // null value.
4664 case X86::MOV32rr:
4665 return llvm::all_of(MI->operands(), [&](const MachineOperand &MO) {
4666 return TRI->isSubRegisterEq(NullValueReg, MO.getReg());
4667 });
4668 default:
4669 return false;
4670 }
4671 llvm_unreachable("Should be handled above!");
4672}
4673
4676 int64_t &Offset, bool &OffsetIsScalable, LocationSize &Width,
4677 const TargetRegisterInfo *TRI) const {
4678 const MCInstrDesc &Desc = MemOp.getDesc();
4679 int MemRefBegin = X86II::getMemoryOperandNo(Desc.TSFlags);
4680 if (MemRefBegin < 0)
4681 return false;
4682
4683 MemRefBegin += X86II::getOperandBias(Desc);
4684
4685 const MachineOperand *BaseOp =
4686 &MemOp.getOperand(MemRefBegin + X86::AddrBaseReg);
4687 if (!BaseOp->isReg()) // Can be an MO_FrameIndex
4688 return false;
4689
4690 if (MemOp.getOperand(MemRefBegin + X86::AddrScaleAmt).getImm() != 1)
4691 return false;
4692
4693 if (MemOp.getOperand(MemRefBegin + X86::AddrIndexReg).getReg() !=
4694 X86::NoRegister)
4695 return false;
4696
4697 const MachineOperand &DispMO = MemOp.getOperand(MemRefBegin + X86::AddrDisp);
4698
4699 // Displacement can be symbolic
4700 if (!DispMO.isImm())
4701 return false;
4702
4703 Offset = DispMO.getImm();
4704
4705 if (!BaseOp->isReg())
4706 return false;
4707
4708 OffsetIsScalable = false;
4709 // FIXME: Relying on memoperands() may not be right thing to do here. Check
4710 // with X86 maintainers, and fix it accordingly. For now, it is ok, since
4711 // there is no use of `Width` for X86 back-end at the moment.
4712 Width =
4713 !MemOp.memoperands_empty() ? MemOp.memoperands().front()->getSize() : 0;
4714 BaseOps.push_back(BaseOp);
4715 return true;
4716}
4717
4718static unsigned getStoreRegOpcode(Register SrcReg,
4719 const TargetRegisterClass *RC,
4720 bool IsStackAligned,
4721 const X86Subtarget &STI) {
4722 return getLoadStoreRegOpcode(SrcReg, RC, IsStackAligned, STI, false);
4723}
4724
4725static unsigned getLoadRegOpcode(Register DestReg,
4726 const TargetRegisterClass *RC,
4727 bool IsStackAligned, const X86Subtarget &STI) {
4728 return getLoadStoreRegOpcode(DestReg, RC, IsStackAligned, STI, true);
4729}
4730
4731static bool isAMXOpcode(unsigned Opc) {
4732 switch (Opc) {
4733 default:
4734 return false;
4735 case X86::TILELOADD:
4736 case X86::TILESTORED:
4737 case X86::TILELOADD_EVEX:
4738 case X86::TILESTORED_EVEX:
4739 case X86::PTILEPAIRLOAD:
4740 case X86::PTILEPAIRSTORE:
4741 return true;
4742 }
4743}
4744
4747 unsigned Opc, Register Reg, int FrameIdx,
4748 bool isKill) const {
4749 switch (Opc) {
4750 default:
4751 llvm_unreachable("Unexpected special opcode!");
4752 case X86::TILESTORED:
4753 case X86::TILESTORED_EVEX:
4754 case X86::PTILEPAIRSTORE: {
4755 // tilestored %tmm, (%sp, %idx)
4757 Register VirtReg = RegInfo.createVirtualRegister(&X86::GR64_NOSPRegClass);
4758 BuildMI(MBB, MI, DebugLoc(), get(X86::MOV64ri), VirtReg).addImm(64);
4759 MachineInstr *NewMI =
4760 addFrameReference(BuildMI(MBB, MI, DebugLoc(), get(Opc)), FrameIdx)
4761 .addReg(Reg, getKillRegState(isKill));
4763 MO.setReg(VirtReg);
4764 MO.setIsKill(true);
4765 break;
4766 }
4767 case X86::TILELOADD:
4768 case X86::TILELOADD_EVEX:
4769 case X86::PTILEPAIRLOAD: {
4770 // tileloadd (%sp, %idx), %tmm
4772 Register VirtReg = RegInfo.createVirtualRegister(&X86::GR64_NOSPRegClass);
4773 BuildMI(MBB, MI, DebugLoc(), get(X86::MOV64ri), VirtReg).addImm(64);
4775 BuildMI(MBB, MI, DebugLoc(), get(Opc), Reg), FrameIdx);
4777 MO.setReg(VirtReg);
4778 MO.setIsKill(true);
4779 break;
4780 }
4781 }
4782}
4783
4786 bool isKill, int FrameIdx, const TargetRegisterClass *RC,
4787 const TargetRegisterInfo *TRI, Register VReg) const {
4788 const MachineFunction &MF = *MBB.getParent();
4789 const MachineFrameInfo &MFI = MF.getFrameInfo();
4790 assert(MFI.getObjectSize(FrameIdx) >= TRI->getSpillSize(*RC) &&
4791 "Stack slot too small for store");
4792
4793 unsigned Alignment = std::max<uint32_t>(TRI->getSpillSize(*RC), 16);
4794 bool isAligned =
4795 (Subtarget.getFrameLowering()->getStackAlign() >= Alignment) ||
4796 (RI.canRealignStack(MF) && !MFI.isFixedObjectIndex(FrameIdx));
4797
4798 unsigned Opc = getStoreRegOpcode(SrcReg, RC, isAligned, Subtarget);
4799 if (isAMXOpcode(Opc))
4800 loadStoreTileReg(MBB, MI, Opc, SrcReg, FrameIdx, isKill);
4801 else
4802 addFrameReference(BuildMI(MBB, MI, DebugLoc(), get(Opc)), FrameIdx)
4803 .addReg(SrcReg, getKillRegState(isKill));
4804}
4805
4808 Register DestReg, int FrameIdx,
4809 const TargetRegisterClass *RC,
4810 const TargetRegisterInfo *TRI,
4811 Register VReg) const {
4812 const MachineFunction &MF = *MBB.getParent();
4813 const MachineFrameInfo &MFI = MF.getFrameInfo();
4814 assert(MFI.getObjectSize(FrameIdx) >= TRI->getSpillSize(*RC) &&
4815 "Load size exceeds stack slot");
4816 unsigned Alignment = std::max<uint32_t>(TRI->getSpillSize(*RC), 16);
4817 bool isAligned =
4818 (Subtarget.getFrameLowering()->getStackAlign() >= Alignment) ||
4819 (RI.canRealignStack(MF) && !MFI.isFixedObjectIndex(FrameIdx));
4820
4821 unsigned Opc = getLoadRegOpcode(DestReg, RC, isAligned, Subtarget);
4822 if (isAMXOpcode(Opc))
4823 loadStoreTileReg(MBB, MI, Opc, DestReg, FrameIdx);
4824 else
4825 addFrameReference(BuildMI(MBB, MI, DebugLoc(), get(Opc), DestReg),
4826 FrameIdx);
4827}
4828
4830 Register &SrcReg2, int64_t &CmpMask,
4831 int64_t &CmpValue) const {
4832 switch (MI.getOpcode()) {
4833 default:
4834 break;
4835 case X86::CMP64ri32:
4836 case X86::CMP32ri:
4837 case X86::CMP16ri:
4838 case X86::CMP8ri:
4839 SrcReg = MI.getOperand(0).getReg();
4840 SrcReg2 = 0;
4841 if (MI.getOperand(1).isImm()) {
4842 CmpMask = ~0;
4843 CmpValue = MI.getOperand(1).getImm();
4844 } else {
4845 CmpMask = CmpValue = 0;
4846 }
4847 return true;
4848 // A SUB can be used to perform comparison.
4849 CASE_ND(SUB64rm)
4850 CASE_ND(SUB32rm)
4851 CASE_ND(SUB16rm)
4852 CASE_ND(SUB8rm)
4853 SrcReg = MI.getOperand(1).getReg();
4854 SrcReg2 = 0;
4855 CmpMask = 0;
4856 CmpValue = 0;
4857 return true;
4858 CASE_ND(SUB64rr)
4859 CASE_ND(SUB32rr)
4860 CASE_ND(SUB16rr)
4861 CASE_ND(SUB8rr)
4862 SrcReg = MI.getOperand(1).getReg();
4863 SrcReg2 = MI.getOperand(2).getReg();
4864 CmpMask = 0;
4865 CmpValue = 0;
4866 return true;
4867 CASE_ND(SUB64ri32)
4868 CASE_ND(SUB32ri)
4869 CASE_ND(SUB16ri)
4870 CASE_ND(SUB8ri)
4871 SrcReg = MI.getOperand(1).getReg();
4872 SrcReg2 = 0;
4873 if (MI.getOperand(2).isImm()) {
4874 CmpMask = ~0;
4875 CmpValue = MI.getOperand(2).getImm();
4876 } else {
4877 CmpMask = CmpValue = 0;
4878 }
4879 return true;
4880 case X86::CMP64rr:
4881 case X86::CMP32rr:
4882 case X86::CMP16rr:
4883 case X86::CMP8rr:
4884 SrcReg = MI.getOperand(0).getReg();
4885 SrcReg2 = MI.getOperand(1).getReg();
4886 CmpMask = 0;
4887 CmpValue = 0;
4888 return true;
4889 case X86::TEST8rr:
4890 case X86::TEST16rr:
4891 case X86::TEST32rr:
4892 case X86::TEST64rr:
4893 SrcReg = MI.getOperand(0).getReg();
4894 if (MI.getOperand(1).getReg() != SrcReg)
4895 return false;
4896 // Compare against zero.
4897 SrcReg2 = 0;
4898 CmpMask = ~0;
4899 CmpValue = 0;
4900 return true;
4901 }
4902 return false;
4903}
4904
4905bool X86InstrInfo::isRedundantFlagInstr(const MachineInstr &FlagI,
4906 Register SrcReg, Register SrcReg2,
4907 int64_t ImmMask, int64_t ImmValue,
4908 const MachineInstr &OI, bool *IsSwapped,
4909 int64_t *ImmDelta) const {
4910 switch (OI.getOpcode()) {
4911 case X86::CMP64rr:
4912 case X86::CMP32rr:
4913 case X86::CMP16rr:
4914 case X86::CMP8rr:
4915 CASE_ND(SUB64rr)
4916 CASE_ND(SUB32rr)
4917 CASE_ND(SUB16rr)
4918 CASE_ND(SUB8rr) {
4919 Register OISrcReg;
4920 Register OISrcReg2;
4921 int64_t OIMask;
4922 int64_t OIValue;
4923 if (!analyzeCompare(OI, OISrcReg, OISrcReg2, OIMask, OIValue) ||
4924 OIMask != ImmMask || OIValue != ImmValue)
4925 return false;
4926 if (SrcReg == OISrcReg && SrcReg2 == OISrcReg2) {
4927 *IsSwapped = false;
4928 return true;
4929 }
4930 if (SrcReg == OISrcReg2 && SrcReg2 == OISrcReg) {
4931 *IsSwapped = true;
4932 return true;
4933 }
4934 return false;
4935 }
4936 case X86::CMP64ri32:
4937 case X86::CMP32ri:
4938 case X86::CMP16ri:
4939 case X86::CMP8ri:
4940 CASE_ND(SUB64ri32)
4941 CASE_ND(SUB32ri)
4942 CASE_ND(SUB16ri)
4943 CASE_ND(SUB8ri)
4944 case X86::TEST64rr:
4945 case X86::TEST32rr:
4946 case X86::TEST16rr:
4947 case X86::TEST8rr: {
4948 if (ImmMask != 0) {
4949 Register OISrcReg;
4950 Register OISrcReg2;
4951 int64_t OIMask;
4952 int64_t OIValue;
4953 if (analyzeCompare(OI, OISrcReg, OISrcReg2, OIMask, OIValue) &&
4954 SrcReg == OISrcReg && ImmMask == OIMask) {
4955 if (OIValue == ImmValue) {
4956 *ImmDelta = 0;
4957 return true;
4958 } else if (static_cast<uint64_t>(ImmValue) ==
4959 static_cast<uint64_t>(OIValue) - 1) {
4960 *ImmDelta = -1;
4961 return true;
4962 } else if (static_cast<uint64_t>(ImmValue) ==
4963 static_cast<uint64_t>(OIValue) + 1) {
4964 *ImmDelta = 1;
4965 return true;
4966 } else {
4967 return false;
4968 }
4969 }
4970 }
4971 return FlagI.isIdenticalTo(OI);
4972 }
4973 default:
4974 return false;
4975 }
4976}
4977
4978/// Check whether the definition can be converted
4979/// to remove a comparison against zero.
4980inline static bool isDefConvertible(const MachineInstr &MI, bool &NoSignFlag,
4981 bool &ClearsOverflowFlag) {
4982 NoSignFlag = false;
4983 ClearsOverflowFlag = false;
4984
4985 // "ELF Handling for Thread-Local Storage" specifies that x86-64 GOTTPOFF, and
4986 // i386 GOTNTPOFF/INDNTPOFF relocations can convert an ADD to a LEA during
4987 // Initial Exec to Local Exec relaxation. In these cases, we must not depend
4988 // on the EFLAGS modification of ADD actually happening in the final binary.
4989 if (MI.getOpcode() == X86::ADD64rm || MI.getOpcode() == X86::ADD32rm) {
4990 unsigned Flags = MI.getOperand(5).getTargetFlags();
4991 if (Flags == X86II::MO_GOTTPOFF || Flags == X86II::MO_INDNTPOFF ||
4992 Flags == X86II::MO_GOTNTPOFF)
4993 return false;
4994 }
4995
4996 switch (MI.getOpcode()) {
4997 default:
4998 return false;
4999
5000 // The shift instructions only modify ZF if their shift count is non-zero.
5001 // N.B.: The processor truncates the shift count depending on the encoding.
5002 CASE_ND(SAR8ri)
5003 CASE_ND(SAR16ri)
5004 CASE_ND(SAR32ri)
5005 CASE_ND(SAR64ri)
5006 CASE_ND(SHR8ri)
5007 CASE_ND(SHR16ri)
5008 CASE_ND(SHR32ri)
5009 CASE_ND(SHR64ri)
5010 return getTruncatedShiftCount(MI, 2) != 0;
5011
5012 // Some left shift instructions can be turned into LEA instructions but only
5013 // if their flags aren't used. Avoid transforming such instructions.
5014 CASE_ND(SHL8ri)
5015 CASE_ND(SHL16ri)
5016 CASE_ND(SHL32ri)
5017 CASE_ND(SHL64ri) {
5018 unsigned ShAmt = getTruncatedShiftCount(MI, 2);
5019 if (isTruncatedShiftCountForLEA(ShAmt))
5020 return false;
5021 return ShAmt != 0;
5022 }
5023
5024 CASE_ND(SHRD16rri8)
5025 CASE_ND(SHRD32rri8)
5026 CASE_ND(SHRD64rri8)
5027 CASE_ND(SHLD16rri8)
5028 CASE_ND(SHLD32rri8)
5029 CASE_ND(SHLD64rri8)
5030 return getTruncatedShiftCount(MI, 3) != 0;
5031
5032 CASE_ND(SUB64ri32)
5033 CASE_ND(SUB32ri)
5034 CASE_ND(SUB16ri)
5035 CASE_ND(SUB8ri)
5036 CASE_ND(SUB64rr)
5037 CASE_ND(SUB32rr)
5038 CASE_ND(SUB16rr)
5039 CASE_ND(SUB8rr)
5040 CASE_ND(SUB64rm)
5041 CASE_ND(SUB32rm)
5042 CASE_ND(SUB16rm)
5043 CASE_ND(SUB8rm)
5044 CASE_ND(DEC64r)
5045 CASE_ND(DEC32r)
5046 CASE_ND(DEC16r)
5047 CASE_ND(DEC8r)
5048 CASE_ND(ADD64ri32)
5049 CASE_ND(ADD32ri)
5050 CASE_ND(ADD16ri)
5051 CASE_ND(ADD8ri)
5052 CASE_ND(ADD64rr)
5053 CASE_ND(ADD32rr)
5054 CASE_ND(ADD16rr)
5055 CASE_ND(ADD8rr)
5056 CASE_ND(ADD64rm)
5057 CASE_ND(ADD32rm)
5058 CASE_ND(ADD16rm)
5059 CASE_ND(ADD8rm)
5060 CASE_ND(INC64r)
5061 CASE_ND(INC32r)
5062 CASE_ND(INC16r)
5063 CASE_ND(INC8r)
5064 CASE_ND(ADC64ri32)
5065 CASE_ND(ADC32ri)
5066 CASE_ND(ADC16ri)
5067 CASE_ND(ADC8ri)
5068 CASE_ND(ADC64rr)
5069 CASE_ND(ADC32rr)
5070 CASE_ND(ADC16rr)
5071 CASE_ND(ADC8rr)
5072 CASE_ND(ADC64rm)
5073 CASE_ND(ADC32rm)
5074 CASE_ND(ADC16rm)
5075 CASE_ND(ADC8rm)
5076 CASE_ND(SBB64ri32)
5077 CASE_ND(SBB32ri)
5078 CASE_ND(SBB16ri)
5079 CASE_ND(SBB8ri)
5080 CASE_ND(SBB64rr)
5081 CASE_ND(SBB32rr)
5082 CASE_ND(SBB16rr)
5083 CASE_ND(SBB8rr)
5084 CASE_ND(SBB64rm)
5085 CASE_ND(SBB32rm)
5086 CASE_ND(SBB16rm)
5087 CASE_ND(SBB8rm)
5088 CASE_ND(NEG8r)
5089 CASE_ND(NEG16r)
5090 CASE_ND(NEG32r)
5091 CASE_ND(NEG64r)
5092 case X86::LZCNT16rr:
5093 case X86::LZCNT16rm:
5094 case X86::LZCNT32rr:
5095 case X86::LZCNT32rm:
5096 case X86::LZCNT64rr:
5097 case X86::LZCNT64rm:
5098 case X86::POPCNT16rr:
5099 case X86::POPCNT16rm:
5100 case X86::POPCNT32rr:
5101 case X86::POPCNT32rm:
5102 case X86::POPCNT64rr:
5103 case X86::POPCNT64rm:
5104 case X86::TZCNT16rr:
5105 case X86::TZCNT16rm:
5106 case X86::TZCNT32rr:
5107 case X86::TZCNT32rm:
5108 case X86::TZCNT64rr:
5109 case X86::TZCNT64rm:
5110 return true;
5111 CASE_ND(AND64ri32)
5112 CASE_ND(AND32ri)
5113 CASE_ND(AND16ri)
5114 CASE_ND(AND8ri)
5115 CASE_ND(AND64rr)
5116 CASE_ND(AND32rr)
5117 CASE_ND(AND16rr)
5118 CASE_ND(AND8rr)
5119 CASE_ND(AND64rm)
5120 CASE_ND(AND32rm)
5121 CASE_ND(AND16rm)
5122 CASE_ND(AND8rm)
5123 CASE_ND(XOR64ri32)
5124 CASE_ND(XOR32ri)
5125 CASE_ND(XOR16ri)
5126 CASE_ND(XOR8ri)
5127 CASE_ND(XOR64rr)
5128 CASE_ND(XOR32rr)
5129 CASE_ND(XOR16rr)
5130 CASE_ND(XOR8rr)
5131 CASE_ND(XOR64rm)
5132 CASE_ND(XOR32rm)
5133 CASE_ND(XOR16rm)
5134 CASE_ND(XOR8rm)
5135 CASE_ND(OR64ri32)
5136 CASE_ND(OR32ri)
5137 CASE_ND(OR16ri)
5138 CASE_ND(OR8ri)
5139 CASE_ND(OR64rr)
5140 CASE_ND(OR32rr)
5141 CASE_ND(OR16rr)
5142 CASE_ND(OR8rr)
5143 CASE_ND(OR64rm)
5144 CASE_ND(OR32rm)
5145 CASE_ND(OR16rm)
5146 CASE_ND(OR8rm)
5147 case X86::ANDN32rr:
5148 case X86::ANDN32rm:
5149 case X86::ANDN64rr:
5150 case X86::ANDN64rm:
5151 case X86::BLSI32rr:
5152 case X86::BLSI32rm:
5153 case X86::BLSI64rr:
5154 case X86::BLSI64rm:
5155 case X86::BLSMSK32rr:
5156 case X86::BLSMSK32rm:
5157 case X86::BLSMSK64rr:
5158 case X86::BLSMSK64rm:
5159 case X86::BLSR32rr:
5160 case X86::BLSR32rm:
5161 case X86::BLSR64rr:
5162 case X86::BLSR64rm:
5163 case X86::BLCFILL32rr:
5164 case X86::BLCFILL32rm:
5165 case X86::BLCFILL64rr:
5166 case X86::BLCFILL64rm:
5167 case X86::BLCI32rr:
5168 case X86::BLCI32rm:
5169 case X86::BLCI64rr:
5170 case X86::BLCI64rm:
5171 case X86::BLCIC32rr:
5172 case X86::BLCIC32rm:
5173 case X86::BLCIC64rr:
5174 case X86::BLCIC64rm:
5175 case X86::BLCMSK32rr:
5176 case X86::BLCMSK32rm:
5177 case X86::BLCMSK64rr:
5178 case X86::BLCMSK64rm:
5179 case X86::BLCS32rr:
5180 case X86::BLCS32rm:
5181 case X86::BLCS64rr:
5182 case X86::BLCS64rm:
5183 case X86::BLSFILL32rr:
5184 case X86::BLSFILL32rm:
5185 case X86::BLSFILL64rr:
5186 case X86::BLSFILL64rm:
5187 case X86::BLSIC32rr:
5188 case X86::BLSIC32rm:
5189 case X86::BLSIC64rr:
5190 case X86::BLSIC64rm:
5191 case X86::BZHI32rr:
5192 case X86::BZHI32rm:
5193 case X86::BZHI64rr:
5194 case X86::BZHI64rm:
5195 case X86::T1MSKC32rr:
5196 case X86::T1MSKC32rm:
5197 case X86::T1MSKC64rr:
5198 case X86::T1MSKC64rm:
5199 case X86::TZMSK32rr:
5200 case X86::TZMSK32rm:
5201 case X86::TZMSK64rr:
5202 case X86::TZMSK64rm:
5203 // These instructions clear the overflow flag just like TEST.
5204 // FIXME: These are not the only instructions in this switch that clear the
5205 // overflow flag.
5206 ClearsOverflowFlag = true;
5207 return true;
5208 case X86::BEXTR32rr:
5209 case X86::BEXTR64rr:
5210 case X86::BEXTR32rm:
5211 case X86::BEXTR64rm:
5212 case X86::BEXTRI32ri:
5213 case X86::BEXTRI32mi:
5214 case X86::BEXTRI64ri:
5215 case X86::BEXTRI64mi:
5216 // BEXTR doesn't update the sign flag so we can't use it. It does clear
5217 // the overflow flag, but that's not useful without the sign flag.
5218 NoSignFlag = true;
5219 return true;
5220 }
5221}
5222
5223/// Check whether the use can be converted to remove a comparison against zero.
5225 switch (MI.getOpcode()) {
5226 default:
5227 return X86::COND_INVALID;
5228 CASE_ND(NEG8r)
5229 CASE_ND(NEG16r)
5230 CASE_ND(NEG32r)
5231 CASE_ND(NEG64r)
5232 return X86::COND_AE;
5233 case X86::LZCNT16rr:
5234 case X86::LZCNT32rr:
5235 case X86::LZCNT64rr:
5236 return X86::COND_B;
5237 case X86::POPCNT16rr:
5238 case X86::POPCNT32rr:
5239 case X86::POPCNT64rr:
5240 return X86::COND_E;
5241 case X86::TZCNT16rr:
5242 case X86::TZCNT32rr:
5243 case X86::TZCNT64rr:
5244 return X86::COND_B;
5245 case X86::BSF16rr:
5246 case X86::BSF32rr:
5247 case X86::BSF64rr:
5248 case X86::BSR16rr:
5249 case X86::BSR32rr:
5250 case X86::BSR64rr:
5251 return X86::COND_E;
5252 case X86::BLSI32rr:
5253 case X86::BLSI64rr:
5254 return X86::COND_AE;
5255 case X86::BLSR32rr:
5256 case X86::BLSR64rr:
5257 case X86::BLSMSK32rr:
5258 case X86::BLSMSK64rr:
5259 return X86::COND_B;
5260 // TODO: TBM instructions.
5261 }
5262}
5263
5264/// Check if there exists an earlier instruction that
5265/// operates on the same source operands and sets flags in the same way as
5266/// Compare; remove Compare if possible.
5268 Register SrcReg2, int64_t CmpMask,
5269 int64_t CmpValue,
5270 const MachineRegisterInfo *MRI) const {
5271 // Check whether we can replace SUB with CMP.
5272 switch (CmpInstr.getOpcode()) {
5273 default:
5274 break;
5275 CASE_ND(SUB64ri32)
5276 CASE_ND(SUB32ri)
5277 CASE_ND(SUB16ri)
5278 CASE_ND(SUB8ri)
5279 CASE_ND(SUB64rm)
5280 CASE_ND(SUB32rm)
5281 CASE_ND(SUB16rm)
5282 CASE_ND(SUB8rm)
5283 CASE_ND(SUB64rr)
5284 CASE_ND(SUB32rr)
5285 CASE_ND(SUB16rr)
5286 CASE_ND(SUB8rr) {
5287 if (!MRI->use_nodbg_empty(CmpInstr.getOperand(0).getReg()))
5288 return false;
5289 // There is no use of the destination register, we can replace SUB with CMP.
5290 unsigned NewOpcode = 0;
5291#define FROM_TO(A, B) \
5292 CASE_ND(A) NewOpcode = X86::B; \
5293 break;
5294 switch (CmpInstr.getOpcode()) {
5295 default:
5296 llvm_unreachable("Unreachable!");
5297 FROM_TO(SUB64rm, CMP64rm)
5298 FROM_TO(SUB32rm, CMP32rm)
5299 FROM_TO(SUB16rm, CMP16rm)
5300 FROM_TO(SUB8rm, CMP8rm)
5301 FROM_TO(SUB64rr, CMP64rr)
5302 FROM_TO(SUB32rr, CMP32rr)
5303 FROM_TO(SUB16rr, CMP16rr)
5304 FROM_TO(SUB8rr, CMP8rr)
5305 FROM_TO(SUB64ri32, CMP64ri32)
5306 FROM_TO(SUB32ri, CMP32ri)
5307 FROM_TO(SUB16ri, CMP16ri)
5308 FROM_TO(SUB8ri, CMP8ri)
5309 }
5310#undef FROM_TO
5311 CmpInstr.setDesc(get(NewOpcode));
5312 CmpInstr.removeOperand(0);
5313 // Mutating this instruction invalidates any debug data associated with it.
5314 CmpInstr.dropDebugNumber();
5315 // Fall through to optimize Cmp if Cmp is CMPrr or CMPri.
5316 if (NewOpcode == X86::CMP64rm || NewOpcode == X86::CMP32rm ||
5317 NewOpcode == X86::CMP16rm || NewOpcode == X86::CMP8rm)
5318 return false;
5319 }
5320 }
5321
5322 // The following code tries to remove the comparison by re-using EFLAGS
5323 // from earlier instructions.
5324
5325 bool IsCmpZero = (CmpMask != 0 && CmpValue == 0);
5326
5327 // Transformation currently requires SSA values.
5328 if (SrcReg2.isPhysical())
5329 return false;
<