LLVM 19.0.0git
X86InstructionSelector.cpp
Go to the documentation of this file.
1//===- X86InstructionSelector.cpp -----------------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8/// \file
9/// This file implements the targeting of the InstructionSelector class for
10/// X86.
11/// \todo This should be generated by TableGen.
12//===----------------------------------------------------------------------===//
13
15#include "X86.h"
16#include "X86InstrBuilder.h"
17#include "X86InstrInfo.h"
18#include "X86RegisterBankInfo.h"
19#include "X86RegisterInfo.h"
20#include "X86Subtarget.h"
21#include "X86TargetMachine.h"
38#include "llvm/IR/DataLayout.h"
39#include "llvm/IR/InstrTypes.h"
40#include "llvm/IR/IntrinsicsX86.h"
43#include "llvm/Support/Debug.h"
47#include <cassert>
48#include <cstdint>
49#include <tuple>
50
51#define DEBUG_TYPE "X86-isel"
52
53using namespace llvm;
54
55namespace {
56
57#define GET_GLOBALISEL_PREDICATE_BITSET
58#include "X86GenGlobalISel.inc"
59#undef GET_GLOBALISEL_PREDICATE_BITSET
60
61class X86InstructionSelector : public InstructionSelector {
62public:
63 X86InstructionSelector(const X86TargetMachine &TM, const X86Subtarget &STI,
64 const X86RegisterBankInfo &RBI);
65
66 bool select(MachineInstr &I) override;
67 static const char *getName() { return DEBUG_TYPE; }
68
69private:
70 /// tblgen-erated 'select' implementation, used as the initial selector for
71 /// the patterns that don't require complex C++.
72 bool selectImpl(MachineInstr &I, CodeGenCoverage &CoverageInfo) const;
73
74 // TODO: remove after supported by Tablegen-erated instruction selection.
75 unsigned getLoadStoreOp(const LLT &Ty, const RegisterBank &RB, unsigned Opc,
76 Align Alignment) const;
77
79 MachineFunction &MF) const;
80 bool selectFrameIndexOrGep(MachineInstr &I, MachineRegisterInfo &MRI,
81 MachineFunction &MF) const;
82 bool selectGlobalValue(MachineInstr &I, MachineRegisterInfo &MRI,
83 MachineFunction &MF) const;
84 bool selectConstant(MachineInstr &I, MachineRegisterInfo &MRI,
85 MachineFunction &MF) const;
86 bool selectTruncOrPtrToInt(MachineInstr &I, MachineRegisterInfo &MRI,
87 MachineFunction &MF) const;
88 bool selectZext(MachineInstr &I, MachineRegisterInfo &MRI,
89 MachineFunction &MF) const;
90 bool selectAnyext(MachineInstr &I, MachineRegisterInfo &MRI,
91 MachineFunction &MF) const;
92 bool selectCmp(MachineInstr &I, MachineRegisterInfo &MRI,
93 MachineFunction &MF) const;
94 bool selectFCmp(MachineInstr &I, MachineRegisterInfo &MRI,
95 MachineFunction &MF) const;
96 bool selectUAddSub(MachineInstr &I, MachineRegisterInfo &MRI,
97 MachineFunction &MF) const;
101 MachineFunction &MF);
103 MachineFunction &MF);
104 bool selectInsert(MachineInstr &I, MachineRegisterInfo &MRI,
105 MachineFunction &MF) const;
106 bool selectExtract(MachineInstr &I, MachineRegisterInfo &MRI,
107 MachineFunction &MF) const;
108 bool selectCondBranch(MachineInstr &I, MachineRegisterInfo &MRI,
109 MachineFunction &MF) const;
110 bool selectTurnIntoCOPY(MachineInstr &I, MachineRegisterInfo &MRI,
111 const unsigned DstReg,
112 const TargetRegisterClass *DstRC,
113 const unsigned SrcReg,
114 const TargetRegisterClass *SrcRC) const;
115 bool materializeFP(MachineInstr &I, MachineRegisterInfo &MRI,
116 MachineFunction &MF) const;
117 bool selectImplicitDefOrPHI(MachineInstr &I, MachineRegisterInfo &MRI) const;
118 bool selectMulDivRem(MachineInstr &I, MachineRegisterInfo &MRI,
119 MachineFunction &MF) const;
120 bool selectSelect(MachineInstr &I, MachineRegisterInfo &MRI,
121 MachineFunction &MF) const;
122
123 // emit insert subreg instruction and insert it before MachineInstr &I
124 bool emitInsertSubreg(unsigned DstReg, unsigned SrcReg, MachineInstr &I,
126 // emit extract subreg instruction and insert it before MachineInstr &I
127 bool emitExtractSubreg(unsigned DstReg, unsigned SrcReg, MachineInstr &I,
129
130 const TargetRegisterClass *getRegClass(LLT Ty, const RegisterBank &RB) const;
131 const TargetRegisterClass *getRegClass(LLT Ty, unsigned Reg,
132 MachineRegisterInfo &MRI) const;
133
134 const X86TargetMachine &TM;
135 const X86Subtarget &STI;
136 const X86InstrInfo &TII;
137 const X86RegisterInfo &TRI;
138 const X86RegisterBankInfo &RBI;
139
140#define GET_GLOBALISEL_PREDICATES_DECL
141#include "X86GenGlobalISel.inc"
142#undef GET_GLOBALISEL_PREDICATES_DECL
143
144#define GET_GLOBALISEL_TEMPORARIES_DECL
145#include "X86GenGlobalISel.inc"
146#undef GET_GLOBALISEL_TEMPORARIES_DECL
147};
148
149} // end anonymous namespace
150
151#define GET_GLOBALISEL_IMPL
152#include "X86GenGlobalISel.inc"
153#undef GET_GLOBALISEL_IMPL
154
155X86InstructionSelector::X86InstructionSelector(const X86TargetMachine &TM,
156 const X86Subtarget &STI,
157 const X86RegisterBankInfo &RBI)
158 : TM(TM), STI(STI), TII(*STI.getInstrInfo()), TRI(*STI.getRegisterInfo()),
159 RBI(RBI),
161#include "X86GenGlobalISel.inc"
164#include "X86GenGlobalISel.inc"
166{
167}
168
169// FIXME: This should be target-independent, inferred from the types declared
170// for each class in the bank.
172X86InstructionSelector::getRegClass(LLT Ty, const RegisterBank &RB) const {
173 if (RB.getID() == X86::GPRRegBankID) {
174 if (Ty.getSizeInBits() <= 8)
175 return &X86::GR8RegClass;
176 if (Ty.getSizeInBits() == 16)
177 return &X86::GR16RegClass;
178 if (Ty.getSizeInBits() == 32)
179 return &X86::GR32RegClass;
180 if (Ty.getSizeInBits() == 64)
181 return &X86::GR64RegClass;
182 }
183 if (RB.getID() == X86::VECRRegBankID) {
184 if (Ty.getSizeInBits() == 16)
185 return STI.hasAVX512() ? &X86::FR16XRegClass : &X86::FR16RegClass;
186 if (Ty.getSizeInBits() == 32)
187 return STI.hasAVX512() ? &X86::FR32XRegClass : &X86::FR32RegClass;
188 if (Ty.getSizeInBits() == 64)
189 return STI.hasAVX512() ? &X86::FR64XRegClass : &X86::FR64RegClass;
190 if (Ty.getSizeInBits() == 128)
191 return STI.hasAVX512() ? &X86::VR128XRegClass : &X86::VR128RegClass;
192 if (Ty.getSizeInBits() == 256)
193 return STI.hasAVX512() ? &X86::VR256XRegClass : &X86::VR256RegClass;
194 if (Ty.getSizeInBits() == 512)
195 return &X86::VR512RegClass;
196 }
197
198 llvm_unreachable("Unknown RegBank!");
199}
200
202X86InstructionSelector::getRegClass(LLT Ty, unsigned Reg,
203 MachineRegisterInfo &MRI) const {
204 const RegisterBank &RegBank = *RBI.getRegBank(Reg, MRI, TRI);
205 return getRegClass(Ty, RegBank);
206}
207
208static unsigned getSubRegIndex(const TargetRegisterClass *RC) {
209 unsigned SubIdx = X86::NoSubRegister;
210 if (RC == &X86::GR32RegClass) {
211 SubIdx = X86::sub_32bit;
212 } else if (RC == &X86::GR16RegClass) {
213 SubIdx = X86::sub_16bit;
214 } else if (RC == &X86::GR8RegClass) {
215 SubIdx = X86::sub_8bit;
216 }
217
218 return SubIdx;
219}
220
222 assert(Reg.isPhysical());
223 if (X86::GR64RegClass.contains(Reg))
224 return &X86::GR64RegClass;
225 if (X86::GR32RegClass.contains(Reg))
226 return &X86::GR32RegClass;
227 if (X86::GR16RegClass.contains(Reg))
228 return &X86::GR16RegClass;
229 if (X86::GR8RegClass.contains(Reg))
230 return &X86::GR8RegClass;
231
232 llvm_unreachable("Unknown RegClass for PhysReg!");
233}
234
235// FIXME: We need some sort of API in RBI/TRI to allow generic code to
236// constrain operands of simple instructions given a TargetRegisterClass
237// and LLT
238bool X86InstructionSelector::selectDebugInstr(MachineInstr &I,
239 MachineRegisterInfo &MRI) const {
240 for (MachineOperand &MO : I.operands()) {
241 if (!MO.isReg())
242 continue;
243 Register Reg = MO.getReg();
244 if (!Reg)
245 continue;
246 if (Reg.isPhysical())
247 continue;
248 LLT Ty = MRI.getType(Reg);
249 const RegClassOrRegBank &RegClassOrBank = MRI.getRegClassOrRegBank(Reg);
250 const TargetRegisterClass *RC =
251 dyn_cast_if_present<const TargetRegisterClass *>(RegClassOrBank);
252 if (!RC) {
253 const RegisterBank &RB = *cast<const RegisterBank *>(RegClassOrBank);
254 RC = getRegClass(Ty, RB);
255 if (!RC) {
257 dbgs() << "Warning: DBG_VALUE operand has unexpected size/bank\n");
258 break;
259 }
260 }
261 RBI.constrainGenericRegister(Reg, *RC, MRI);
262 }
263
264 return true;
265}
266
267// Set X86 Opcode and constrain DestReg.
268bool X86InstructionSelector::selectCopy(MachineInstr &I,
269 MachineRegisterInfo &MRI) const {
270 Register DstReg = I.getOperand(0).getReg();
271 const unsigned DstSize = RBI.getSizeInBits(DstReg, MRI, TRI);
272 const RegisterBank &DstRegBank = *RBI.getRegBank(DstReg, MRI, TRI);
273
274 Register SrcReg = I.getOperand(1).getReg();
275 const unsigned SrcSize = RBI.getSizeInBits(SrcReg, MRI, TRI);
276 const RegisterBank &SrcRegBank = *RBI.getRegBank(SrcReg, MRI, TRI);
277
278 if (DstReg.isPhysical()) {
279 assert(I.isCopy() && "Generic operators do not allow physical registers");
280
281 if (DstSize > SrcSize && SrcRegBank.getID() == X86::GPRRegBankID &&
282 DstRegBank.getID() == X86::GPRRegBankID) {
283
284 const TargetRegisterClass *SrcRC =
285 getRegClass(MRI.getType(SrcReg), SrcRegBank);
286 const TargetRegisterClass *DstRC = getRegClassFromGRPhysReg(DstReg);
287
288 if (SrcRC != DstRC) {
289 // This case can be generated by ABI lowering, performe anyext
290 Register ExtSrc = MRI.createVirtualRegister(DstRC);
291 BuildMI(*I.getParent(), I, I.getDebugLoc(),
292 TII.get(TargetOpcode::SUBREG_TO_REG))
293 .addDef(ExtSrc)
294 .addImm(0)
295 .addReg(SrcReg)
296 .addImm(getSubRegIndex(SrcRC));
297
298 I.getOperand(1).setReg(ExtSrc);
299 }
300 }
301
302 return true;
303 }
304
305 assert((!SrcReg.isPhysical() || I.isCopy()) &&
306 "No phys reg on generic operators");
307 assert((DstSize == SrcSize ||
308 // Copies are a mean to setup initial types, the number of
309 // bits may not exactly match.
310 (SrcReg.isPhysical() &&
311 DstSize <= RBI.getSizeInBits(SrcReg, MRI, TRI))) &&
312 "Copy with different width?!");
313
314 const TargetRegisterClass *DstRC =
315 getRegClass(MRI.getType(DstReg), DstRegBank);
316
317 if (SrcRegBank.getID() == X86::GPRRegBankID &&
318 DstRegBank.getID() == X86::GPRRegBankID && SrcSize > DstSize &&
319 SrcReg.isPhysical()) {
320 // Change the physical register to performe truncate.
321
322 const TargetRegisterClass *SrcRC = getRegClassFromGRPhysReg(SrcReg);
323
324 if (DstRC != SrcRC) {
325 I.getOperand(1).setSubReg(getSubRegIndex(DstRC));
326 I.getOperand(1).substPhysReg(SrcReg, TRI);
327 }
328 }
329
330 // No need to constrain SrcReg. It will get constrained when
331 // we hit another of its use or its defs.
332 // Copies do not have constraints.
333 const TargetRegisterClass *OldRC = MRI.getRegClassOrNull(DstReg);
334 if (!OldRC || !DstRC->hasSubClassEq(OldRC)) {
335 if (!RBI.constrainGenericRegister(DstReg, *DstRC, MRI)) {
336 LLVM_DEBUG(dbgs() << "Failed to constrain " << TII.getName(I.getOpcode())
337 << " operand\n");
338 return false;
339 }
340 }
341 I.setDesc(TII.get(X86::COPY));
342 return true;
343}
344
345bool X86InstructionSelector::select(MachineInstr &I) {
346 assert(I.getParent() && "Instruction should be in a basic block!");
347 assert(I.getParent()->getParent() && "Instruction should be in a function!");
348
349 MachineBasicBlock &MBB = *I.getParent();
352
353 unsigned Opcode = I.getOpcode();
354 if (!isPreISelGenericOpcode(Opcode)) {
355 // Certain non-generic instructions also need some special handling.
356
357 if (Opcode == TargetOpcode::LOAD_STACK_GUARD)
358 return false;
359
360 if (I.isCopy())
361 return selectCopy(I, MRI);
362
363 if (I.isDebugInstr())
364 return selectDebugInstr(I, MRI);
365
366 return true;
367 }
368
369 assert(I.getNumOperands() == I.getNumExplicitOperands() &&
370 "Generic instruction has unexpected implicit operands\n");
371
372 if (selectImpl(I, *CoverageInfo))
373 return true;
374
375 LLVM_DEBUG(dbgs() << " C++ instruction selection: "; I.print(dbgs()));
376
377 // TODO: This should be implemented by tblgen.
378 switch (I.getOpcode()) {
379 default:
380 return false;
381 case TargetOpcode::G_STORE:
382 case TargetOpcode::G_LOAD:
383 return selectLoadStoreOp(I, MRI, MF);
384 case TargetOpcode::G_PTR_ADD:
385 case TargetOpcode::G_FRAME_INDEX:
386 return selectFrameIndexOrGep(I, MRI, MF);
387 case TargetOpcode::G_GLOBAL_VALUE:
388 return selectGlobalValue(I, MRI, MF);
389 case TargetOpcode::G_CONSTANT:
390 return selectConstant(I, MRI, MF);
391 case TargetOpcode::G_FCONSTANT:
392 return materializeFP(I, MRI, MF);
393 case TargetOpcode::G_PTRTOINT:
394 case TargetOpcode::G_TRUNC:
395 return selectTruncOrPtrToInt(I, MRI, MF);
396 case TargetOpcode::G_INTTOPTR:
397 return selectCopy(I, MRI);
398 case TargetOpcode::G_ZEXT:
399 return selectZext(I, MRI, MF);
400 case TargetOpcode::G_ANYEXT:
401 return selectAnyext(I, MRI, MF);
402 case TargetOpcode::G_ICMP:
403 return selectCmp(I, MRI, MF);
404 case TargetOpcode::G_FCMP:
405 return selectFCmp(I, MRI, MF);
406 case TargetOpcode::G_UADDE:
407 case TargetOpcode::G_UADDO:
408 case TargetOpcode::G_USUBE:
409 case TargetOpcode::G_USUBO:
410 return selectUAddSub(I, MRI, MF);
411 case TargetOpcode::G_UNMERGE_VALUES:
412 return selectUnmergeValues(I, MRI, MF);
413 case TargetOpcode::G_MERGE_VALUES:
414 case TargetOpcode::G_CONCAT_VECTORS:
415 return selectMergeValues(I, MRI, MF);
416 case TargetOpcode::G_EXTRACT:
417 return selectExtract(I, MRI, MF);
418 case TargetOpcode::G_INSERT:
419 return selectInsert(I, MRI, MF);
420 case TargetOpcode::G_BRCOND:
421 return selectCondBranch(I, MRI, MF);
422 case TargetOpcode::G_IMPLICIT_DEF:
423 case TargetOpcode::G_PHI:
424 return selectImplicitDefOrPHI(I, MRI);
425 case TargetOpcode::G_MUL:
426 case TargetOpcode::G_SMULH:
427 case TargetOpcode::G_UMULH:
428 case TargetOpcode::G_SDIV:
429 case TargetOpcode::G_UDIV:
430 case TargetOpcode::G_SREM:
431 case TargetOpcode::G_UREM:
432 return selectMulDivRem(I, MRI, MF);
433 case TargetOpcode::G_SELECT:
434 return selectSelect(I, MRI, MF);
435 }
436
437 return false;
438}
439
440unsigned X86InstructionSelector::getLoadStoreOp(const LLT &Ty,
441 const RegisterBank &RB,
442 unsigned Opc,
443 Align Alignment) const {
444 bool Isload = (Opc == TargetOpcode::G_LOAD);
445 bool HasAVX = STI.hasAVX();
446 bool HasAVX512 = STI.hasAVX512();
447 bool HasVLX = STI.hasVLX();
448
449 if (Ty == LLT::scalar(8)) {
450 if (X86::GPRRegBankID == RB.getID())
451 return Isload ? X86::MOV8rm : X86::MOV8mr;
452 } else if (Ty == LLT::scalar(16)) {
453 if (X86::GPRRegBankID == RB.getID())
454 return Isload ? X86::MOV16rm : X86::MOV16mr;
455 } else if (Ty == LLT::scalar(32) || Ty == LLT::pointer(0, 32)) {
456 if (X86::GPRRegBankID == RB.getID())
457 return Isload ? X86::MOV32rm : X86::MOV32mr;
458 if (X86::VECRRegBankID == RB.getID())
459 return Isload ? (HasAVX512 ? X86::VMOVSSZrm_alt :
460 HasAVX ? X86::VMOVSSrm_alt :
461 X86::MOVSSrm_alt)
462 : (HasAVX512 ? X86::VMOVSSZmr :
463 HasAVX ? X86::VMOVSSmr :
464 X86::MOVSSmr);
465 } else if (Ty == LLT::scalar(64) || Ty == LLT::pointer(0, 64)) {
466 if (X86::GPRRegBankID == RB.getID())
467 return Isload ? X86::MOV64rm : X86::MOV64mr;
468 if (X86::VECRRegBankID == RB.getID())
469 return Isload ? (HasAVX512 ? X86::VMOVSDZrm_alt :
470 HasAVX ? X86::VMOVSDrm_alt :
471 X86::MOVSDrm_alt)
472 : (HasAVX512 ? X86::VMOVSDZmr :
473 HasAVX ? X86::VMOVSDmr :
474 X86::MOVSDmr);
475 } else if (Ty.isVector() && Ty.getSizeInBits() == 128) {
476 if (Alignment >= Align(16))
477 return Isload ? (HasVLX ? X86::VMOVAPSZ128rm
478 : HasAVX512
479 ? X86::VMOVAPSZ128rm_NOVLX
480 : HasAVX ? X86::VMOVAPSrm : X86::MOVAPSrm)
481 : (HasVLX ? X86::VMOVAPSZ128mr
482 : HasAVX512
483 ? X86::VMOVAPSZ128mr_NOVLX
484 : HasAVX ? X86::VMOVAPSmr : X86::MOVAPSmr);
485 else
486 return Isload ? (HasVLX ? X86::VMOVUPSZ128rm
487 : HasAVX512
488 ? X86::VMOVUPSZ128rm_NOVLX
489 : HasAVX ? X86::VMOVUPSrm : X86::MOVUPSrm)
490 : (HasVLX ? X86::VMOVUPSZ128mr
491 : HasAVX512
492 ? X86::VMOVUPSZ128mr_NOVLX
493 : HasAVX ? X86::VMOVUPSmr : X86::MOVUPSmr);
494 } else if (Ty.isVector() && Ty.getSizeInBits() == 256) {
495 if (Alignment >= Align(32))
496 return Isload ? (HasVLX ? X86::VMOVAPSZ256rm
497 : HasAVX512 ? X86::VMOVAPSZ256rm_NOVLX
498 : X86::VMOVAPSYrm)
499 : (HasVLX ? X86::VMOVAPSZ256mr
500 : HasAVX512 ? X86::VMOVAPSZ256mr_NOVLX
501 : X86::VMOVAPSYmr);
502 else
503 return Isload ? (HasVLX ? X86::VMOVUPSZ256rm
504 : HasAVX512 ? X86::VMOVUPSZ256rm_NOVLX
505 : X86::VMOVUPSYrm)
506 : (HasVLX ? X86::VMOVUPSZ256mr
507 : HasAVX512 ? X86::VMOVUPSZ256mr_NOVLX
508 : X86::VMOVUPSYmr);
509 } else if (Ty.isVector() && Ty.getSizeInBits() == 512) {
510 if (Alignment >= Align(64))
511 return Isload ? X86::VMOVAPSZrm : X86::VMOVAPSZmr;
512 else
513 return Isload ? X86::VMOVUPSZrm : X86::VMOVUPSZmr;
514 }
515 return Opc;
516}
517
518// Fill in an address from the given instruction.
519static void X86SelectAddress(const MachineInstr &I,
521 X86AddressMode &AM) {
522 assert(I.getOperand(0).isReg() && "unsupported opperand.");
523 assert(MRI.getType(I.getOperand(0).getReg()).isPointer() &&
524 "unsupported type.");
525
526 if (I.getOpcode() == TargetOpcode::G_PTR_ADD) {
527 if (auto COff = getIConstantVRegSExtVal(I.getOperand(2).getReg(), MRI)) {
528 int64_t Imm = *COff;
529 if (isInt<32>(Imm)) { // Check for displacement overflow.
530 AM.Disp = static_cast<int32_t>(Imm);
531 AM.Base.Reg = I.getOperand(1).getReg();
532 return;
533 }
534 }
535 } else if (I.getOpcode() == TargetOpcode::G_FRAME_INDEX) {
536 AM.Base.FrameIndex = I.getOperand(1).getIndex();
538 return;
539 }
540
541 // Default behavior.
542 AM.Base.Reg = I.getOperand(0).getReg();
543}
544
545bool X86InstructionSelector::selectLoadStoreOp(MachineInstr &I,
547 MachineFunction &MF) const {
548 unsigned Opc = I.getOpcode();
549
550 assert((Opc == TargetOpcode::G_STORE || Opc == TargetOpcode::G_LOAD) &&
551 "Only G_STORE and G_LOAD are expected for selection");
552
553 const Register DefReg = I.getOperand(0).getReg();
554 LLT Ty = MRI.getType(DefReg);
555 const RegisterBank &RB = *RBI.getRegBank(DefReg, MRI, TRI);
556
557 assert(I.hasOneMemOperand());
558 auto &MemOp = **I.memoperands_begin();
559 if (MemOp.isAtomic()) {
560 // Note: for unordered operations, we rely on the fact the appropriate MMO
561 // is already on the instruction we're mutating, and thus we don't need to
562 // make any changes. So long as we select an opcode which is capable of
563 // loading or storing the appropriate size atomically, the rest of the
564 // backend is required to respect the MMO state.
565 if (!MemOp.isUnordered()) {
566 LLVM_DEBUG(dbgs() << "Atomic ordering not supported yet\n");
567 return false;
568 }
569 if (MemOp.getAlign() < Ty.getSizeInBits() / 8) {
570 LLVM_DEBUG(dbgs() << "Unaligned atomics not supported yet\n");
571 return false;
572 }
573 }
574
575 unsigned NewOpc = getLoadStoreOp(Ty, RB, Opc, MemOp.getAlign());
576 if (NewOpc == Opc)
577 return false;
578
579 I.setDesc(TII.get(NewOpc));
580 MachineInstrBuilder MIB(MF, I);
581 const MachineInstr *Ptr = MRI.getVRegDef(I.getOperand(1).getReg());
582
583 if (Ptr->getOpcode() == TargetOpcode::G_CONSTANT_POOL) {
584 assert(Opc == TargetOpcode::G_LOAD &&
585 "Only G_LOAD from constant pool is expected");
586 // TODO: Need a separate move for Large model
587 if (TM.getCodeModel() == CodeModel::Large)
588 return false;
589
590 unsigned char OpFlag = STI.classifyLocalReference(nullptr);
591 unsigned PICBase = 0;
592 if (OpFlag == X86II::MO_GOTOFF)
593 PICBase = TII.getGlobalBaseReg(&MF);
594 else if (STI.is64Bit())
595 PICBase = X86::RIP;
596
597 I.removeOperand(1);
598 addConstantPoolReference(MIB, Ptr->getOperand(1).getIndex(), PICBase,
599 OpFlag);
601 }
602
604 X86SelectAddress(*Ptr, MRI, AM);
605 if (Opc == TargetOpcode::G_LOAD) {
606 I.removeOperand(1);
607 addFullAddress(MIB, AM);
608 } else {
609 // G_STORE (VAL, Addr), X86Store instruction (Addr, VAL)
610 I.removeOperand(1);
611 I.removeOperand(0);
612 addFullAddress(MIB, AM).addUse(DefReg);
613 }
615}
616
617static unsigned getLeaOP(LLT Ty, const X86Subtarget &STI) {
618 if (Ty == LLT::pointer(0, 64))
619 return X86::LEA64r;
620 else if (Ty == LLT::pointer(0, 32))
621 return STI.isTarget64BitILP32() ? X86::LEA64_32r : X86::LEA32r;
622 else
623 llvm_unreachable("Can't get LEA opcode. Unsupported type.");
624}
625
626bool X86InstructionSelector::selectFrameIndexOrGep(MachineInstr &I,
628 MachineFunction &MF) const {
629 unsigned Opc = I.getOpcode();
630
631 assert((Opc == TargetOpcode::G_FRAME_INDEX || Opc == TargetOpcode::G_PTR_ADD) &&
632 "unexpected instruction");
633
634 const Register DefReg = I.getOperand(0).getReg();
635 LLT Ty = MRI.getType(DefReg);
636
637 // Use LEA to calculate frame index and GEP
638 unsigned NewOpc = getLeaOP(Ty, STI);
639 I.setDesc(TII.get(NewOpc));
640 MachineInstrBuilder MIB(MF, I);
641
642 if (Opc == TargetOpcode::G_FRAME_INDEX) {
643 addOffset(MIB, 0);
644 } else {
645 MachineOperand &InxOp = I.getOperand(2);
646 I.addOperand(InxOp); // set IndexReg
647 InxOp.ChangeToImmediate(1); // set Scale
648 MIB.addImm(0).addReg(0);
649 }
650
652}
653
654bool X86InstructionSelector::selectGlobalValue(MachineInstr &I,
656 MachineFunction &MF) const {
657 assert((I.getOpcode() == TargetOpcode::G_GLOBAL_VALUE) &&
658 "unexpected instruction");
659
660 auto GV = I.getOperand(1).getGlobal();
661 if (GV->isThreadLocal()) {
662 return false; // TODO: we don't support TLS yet.
663 }
664
665 // Can't handle alternate code models yet.
666 if (TM.getCodeModel() != CodeModel::Small)
667 return false;
668
670 AM.GV = GV;
671 AM.GVOpFlags = STI.classifyGlobalReference(GV);
672
673 // TODO: The ABI requires an extra load. not supported yet.
675 return false;
676
677 // TODO: This reference is relative to the pic base. not supported yet.
679 return false;
680
681 if (STI.isPICStyleRIPRel()) {
682 // Use rip-relative addressing.
683 assert(AM.Base.Reg == 0 && AM.IndexReg == 0);
684 AM.Base.Reg = X86::RIP;
685 }
686
687 const Register DefReg = I.getOperand(0).getReg();
688 LLT Ty = MRI.getType(DefReg);
689 unsigned NewOpc = getLeaOP(Ty, STI);
690
691 I.setDesc(TII.get(NewOpc));
692 MachineInstrBuilder MIB(MF, I);
693
694 I.removeOperand(1);
695 addFullAddress(MIB, AM);
696
698}
699
700bool X86InstructionSelector::selectConstant(MachineInstr &I,
702 MachineFunction &MF) const {
703 assert((I.getOpcode() == TargetOpcode::G_CONSTANT) &&
704 "unexpected instruction");
705
706 const Register DefReg = I.getOperand(0).getReg();
707 LLT Ty = MRI.getType(DefReg);
708
709 if (RBI.getRegBank(DefReg, MRI, TRI)->getID() != X86::GPRRegBankID)
710 return false;
711
712 uint64_t Val = 0;
713 if (I.getOperand(1).isCImm()) {
714 Val = I.getOperand(1).getCImm()->getZExtValue();
715 I.getOperand(1).ChangeToImmediate(Val);
716 } else if (I.getOperand(1).isImm()) {
717 Val = I.getOperand(1).getImm();
718 } else
719 llvm_unreachable("Unsupported operand type.");
720
721 unsigned NewOpc;
722 switch (Ty.getSizeInBits()) {
723 case 8:
724 NewOpc = X86::MOV8ri;
725 break;
726 case 16:
727 NewOpc = X86::MOV16ri;
728 break;
729 case 32:
730 NewOpc = X86::MOV32ri;
731 break;
732 case 64:
733 // TODO: in case isUInt<32>(Val), X86::MOV32ri can be used
734 if (isInt<32>(Val))
735 NewOpc = X86::MOV64ri32;
736 else
737 NewOpc = X86::MOV64ri;
738 break;
739 default:
740 llvm_unreachable("Can't select G_CONSTANT, unsupported type.");
741 }
742
743 I.setDesc(TII.get(NewOpc));
745}
746
747// Helper function for selectTruncOrPtrToInt and selectAnyext.
748// Returns true if DstRC lives on a floating register class and
749// SrcRC lives on a 128-bit vector class.
750static bool canTurnIntoCOPY(const TargetRegisterClass *DstRC,
751 const TargetRegisterClass *SrcRC) {
752 return (DstRC == &X86::FR32RegClass || DstRC == &X86::FR32XRegClass ||
753 DstRC == &X86::FR64RegClass || DstRC == &X86::FR64XRegClass) &&
754 (SrcRC == &X86::VR128RegClass || SrcRC == &X86::VR128XRegClass);
755}
756
757bool X86InstructionSelector::selectTurnIntoCOPY(
758 MachineInstr &I, MachineRegisterInfo &MRI, const unsigned DstReg,
759 const TargetRegisterClass *DstRC, const unsigned SrcReg,
760 const TargetRegisterClass *SrcRC) const {
761
762 if (!RBI.constrainGenericRegister(SrcReg, *SrcRC, MRI) ||
763 !RBI.constrainGenericRegister(DstReg, *DstRC, MRI)) {
764 LLVM_DEBUG(dbgs() << "Failed to constrain " << TII.getName(I.getOpcode())
765 << " operand\n");
766 return false;
767 }
768 I.setDesc(TII.get(X86::COPY));
769 return true;
770}
771
772bool X86InstructionSelector::selectTruncOrPtrToInt(MachineInstr &I,
774 MachineFunction &MF) const {
775 assert((I.getOpcode() == TargetOpcode::G_TRUNC ||
776 I.getOpcode() == TargetOpcode::G_PTRTOINT) &&
777 "unexpected instruction");
778
779 const Register DstReg = I.getOperand(0).getReg();
780 const Register SrcReg = I.getOperand(1).getReg();
781
782 const LLT DstTy = MRI.getType(DstReg);
783 const LLT SrcTy = MRI.getType(SrcReg);
784
785 const RegisterBank &DstRB = *RBI.getRegBank(DstReg, MRI, TRI);
786 const RegisterBank &SrcRB = *RBI.getRegBank(SrcReg, MRI, TRI);
787
788 if (DstRB.getID() != SrcRB.getID()) {
789 LLVM_DEBUG(dbgs() << TII.getName(I.getOpcode())
790 << " input/output on different banks\n");
791 return false;
792 }
793
794 const TargetRegisterClass *DstRC = getRegClass(DstTy, DstRB);
795 const TargetRegisterClass *SrcRC = getRegClass(SrcTy, SrcRB);
796
797 if (!DstRC || !SrcRC)
798 return false;
799
800 // If that's truncation of the value that lives on the vector class and goes
801 // into the floating class, just replace it with copy, as we are able to
802 // select it as a regular move.
803 if (canTurnIntoCOPY(DstRC, SrcRC))
804 return selectTurnIntoCOPY(I, MRI, DstReg, DstRC, SrcReg, SrcRC);
805
806 if (DstRB.getID() != X86::GPRRegBankID)
807 return false;
808
809 unsigned SubIdx;
810 if (DstRC == SrcRC) {
811 // Nothing to be done
812 SubIdx = X86::NoSubRegister;
813 } else if (DstRC == &X86::GR32RegClass) {
814 SubIdx = X86::sub_32bit;
815 } else if (DstRC == &X86::GR16RegClass) {
816 SubIdx = X86::sub_16bit;
817 } else if (DstRC == &X86::GR8RegClass) {
818 SubIdx = X86::sub_8bit;
819 } else {
820 return false;
821 }
822
823 SrcRC = TRI.getSubClassWithSubReg(SrcRC, SubIdx);
824
825 if (!RBI.constrainGenericRegister(SrcReg, *SrcRC, MRI) ||
826 !RBI.constrainGenericRegister(DstReg, *DstRC, MRI)) {
827 LLVM_DEBUG(dbgs() << "Failed to constrain " << TII.getName(I.getOpcode())
828 << "\n");
829 return false;
830 }
831
832 I.getOperand(1).setSubReg(SubIdx);
833
834 I.setDesc(TII.get(X86::COPY));
835 return true;
836}
837
838bool X86InstructionSelector::selectZext(MachineInstr &I,
840 MachineFunction &MF) const {
841 assert((I.getOpcode() == TargetOpcode::G_ZEXT) && "unexpected instruction");
842
843 const Register DstReg = I.getOperand(0).getReg();
844 const Register SrcReg = I.getOperand(1).getReg();
845
846 const LLT DstTy = MRI.getType(DstReg);
847 const LLT SrcTy = MRI.getType(SrcReg);
848
849 assert(!(SrcTy == LLT::scalar(8) && DstTy == LLT::scalar(16)) &&
850 "8=>16 Zext is handled by tablegen");
851 assert(!(SrcTy == LLT::scalar(8) && DstTy == LLT::scalar(32)) &&
852 "8=>32 Zext is handled by tablegen");
853 assert(!(SrcTy == LLT::scalar(16) && DstTy == LLT::scalar(32)) &&
854 "16=>32 Zext is handled by tablegen");
855 assert(!(SrcTy == LLT::scalar(8) && DstTy == LLT::scalar(64)) &&
856 "8=>64 Zext is handled by tablegen");
857 assert(!(SrcTy == LLT::scalar(16) && DstTy == LLT::scalar(64)) &&
858 "16=>64 Zext is handled by tablegen");
859 assert(!(SrcTy == LLT::scalar(32) && DstTy == LLT::scalar(64)) &&
860 "32=>64 Zext is handled by tablegen");
861
862 if (SrcTy != LLT::scalar(1))
863 return false;
864
865 unsigned AndOpc;
866 if (DstTy == LLT::scalar(8))
867 AndOpc = X86::AND8ri;
868 else if (DstTy == LLT::scalar(16))
869 AndOpc = X86::AND16ri;
870 else if (DstTy == LLT::scalar(32))
871 AndOpc = X86::AND32ri;
872 else if (DstTy == LLT::scalar(64))
873 AndOpc = X86::AND64ri32;
874 else
875 return false;
876
877 Register DefReg = SrcReg;
878 if (DstTy != LLT::scalar(8)) {
879 Register ImpDefReg =
880 MRI.createVirtualRegister(getRegClass(DstTy, DstReg, MRI));
881 BuildMI(*I.getParent(), I, I.getDebugLoc(),
882 TII.get(TargetOpcode::IMPLICIT_DEF), ImpDefReg);
883
884 DefReg = MRI.createVirtualRegister(getRegClass(DstTy, DstReg, MRI));
885 BuildMI(*I.getParent(), I, I.getDebugLoc(),
886 TII.get(TargetOpcode::INSERT_SUBREG), DefReg)
887 .addReg(ImpDefReg)
888 .addReg(SrcReg)
889 .addImm(X86::sub_8bit);
890 }
891
892 MachineInstr &AndInst =
893 *BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(AndOpc), DstReg)
894 .addReg(DefReg)
895 .addImm(1);
896
898
899 I.eraseFromParent();
900 return true;
901}
902
903bool X86InstructionSelector::selectAnyext(MachineInstr &I,
905 MachineFunction &MF) const {
906 assert((I.getOpcode() == TargetOpcode::G_ANYEXT) && "unexpected instruction");
907
908 const Register DstReg = I.getOperand(0).getReg();
909 const Register SrcReg = I.getOperand(1).getReg();
910
911 const LLT DstTy = MRI.getType(DstReg);
912 const LLT SrcTy = MRI.getType(SrcReg);
913
914 const RegisterBank &DstRB = *RBI.getRegBank(DstReg, MRI, TRI);
915 const RegisterBank &SrcRB = *RBI.getRegBank(SrcReg, MRI, TRI);
916
917 assert(DstRB.getID() == SrcRB.getID() &&
918 "G_ANYEXT input/output on different banks\n");
919
920 assert(DstTy.getSizeInBits() > SrcTy.getSizeInBits() &&
921 "G_ANYEXT incorrect operand size");
922
923 const TargetRegisterClass *DstRC = getRegClass(DstTy, DstRB);
924 const TargetRegisterClass *SrcRC = getRegClass(SrcTy, SrcRB);
925
926 // If that's ANY_EXT of the value that lives on the floating class and goes
927 // into the vector class, just replace it with copy, as we are able to select
928 // it as a regular move.
929 if (canTurnIntoCOPY(SrcRC, DstRC))
930 return selectTurnIntoCOPY(I, MRI, SrcReg, SrcRC, DstReg, DstRC);
931
932 if (DstRB.getID() != X86::GPRRegBankID)
933 return false;
934
935 if (!RBI.constrainGenericRegister(SrcReg, *SrcRC, MRI) ||
936 !RBI.constrainGenericRegister(DstReg, *DstRC, MRI)) {
937 LLVM_DEBUG(dbgs() << "Failed to constrain " << TII.getName(I.getOpcode())
938 << " operand\n");
939 return false;
940 }
941
942 if (SrcRC == DstRC) {
943 I.setDesc(TII.get(X86::COPY));
944 return true;
945 }
946
947 BuildMI(*I.getParent(), I, I.getDebugLoc(),
948 TII.get(TargetOpcode::SUBREG_TO_REG))
949 .addDef(DstReg)
950 .addImm(0)
951 .addReg(SrcReg)
952 .addImm(getSubRegIndex(SrcRC));
953
954 I.eraseFromParent();
955 return true;
956}
957
958bool X86InstructionSelector::selectCmp(MachineInstr &I,
960 MachineFunction &MF) const {
961 assert((I.getOpcode() == TargetOpcode::G_ICMP) && "unexpected instruction");
962
964 bool SwapArgs;
965 std::tie(CC, SwapArgs) = X86::getX86ConditionCode(
966 (CmpInst::Predicate)I.getOperand(1).getPredicate());
967
968 Register LHS = I.getOperand(2).getReg();
969 Register RHS = I.getOperand(3).getReg();
970
971 if (SwapArgs)
972 std::swap(LHS, RHS);
973
974 unsigned OpCmp;
975 LLT Ty = MRI.getType(LHS);
976
977 switch (Ty.getSizeInBits()) {
978 default:
979 return false;
980 case 8:
981 OpCmp = X86::CMP8rr;
982 break;
983 case 16:
984 OpCmp = X86::CMP16rr;
985 break;
986 case 32:
987 OpCmp = X86::CMP32rr;
988 break;
989 case 64:
990 OpCmp = X86::CMP64rr;
991 break;
992 }
993
995 *BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(OpCmp))
996 .addReg(LHS)
997 .addReg(RHS);
998
999 MachineInstr &SetInst = *BuildMI(*I.getParent(), I, I.getDebugLoc(),
1000 TII.get(X86::SETCCr), I.getOperand(0).getReg()).addImm(CC);
1001
1004
1005 I.eraseFromParent();
1006 return true;
1007}
1008
1009bool X86InstructionSelector::selectFCmp(MachineInstr &I,
1011 MachineFunction &MF) const {
1012 assert((I.getOpcode() == TargetOpcode::G_FCMP) && "unexpected instruction");
1013
1014 Register LhsReg = I.getOperand(2).getReg();
1015 Register RhsReg = I.getOperand(3).getReg();
1017 (CmpInst::Predicate)I.getOperand(1).getPredicate();
1018
1019 // FCMP_OEQ and FCMP_UNE cannot be checked with a single instruction.
1020 static const uint16_t SETFOpcTable[2][3] = {
1021 {X86::COND_E, X86::COND_NP, X86::AND8rr},
1022 {X86::COND_NE, X86::COND_P, X86::OR8rr}};
1023 const uint16_t *SETFOpc = nullptr;
1024 switch (Predicate) {
1025 default:
1026 break;
1027 case CmpInst::FCMP_OEQ:
1028 SETFOpc = &SETFOpcTable[0][0];
1029 break;
1030 case CmpInst::FCMP_UNE:
1031 SETFOpc = &SETFOpcTable[1][0];
1032 break;
1033 }
1034
1035 // Compute the opcode for the CMP instruction.
1036 unsigned OpCmp;
1037 LLT Ty = MRI.getType(LhsReg);
1038 switch (Ty.getSizeInBits()) {
1039 default:
1040 return false;
1041 case 32:
1042 OpCmp = X86::UCOMISSrr;
1043 break;
1044 case 64:
1045 OpCmp = X86::UCOMISDrr;
1046 break;
1047 }
1048
1049 Register ResultReg = I.getOperand(0).getReg();
1050 RBI.constrainGenericRegister(
1051 ResultReg,
1052 *getRegClass(LLT::scalar(8), *RBI.getRegBank(ResultReg, MRI, TRI)), MRI);
1053 if (SETFOpc) {
1055 *BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(OpCmp))
1056 .addReg(LhsReg)
1057 .addReg(RhsReg);
1058
1059 Register FlagReg1 = MRI.createVirtualRegister(&X86::GR8RegClass);
1060 Register FlagReg2 = MRI.createVirtualRegister(&X86::GR8RegClass);
1061 MachineInstr &Set1 = *BuildMI(*I.getParent(), I, I.getDebugLoc(),
1062 TII.get(X86::SETCCr), FlagReg1).addImm(SETFOpc[0]);
1063 MachineInstr &Set2 = *BuildMI(*I.getParent(), I, I.getDebugLoc(),
1064 TII.get(X86::SETCCr), FlagReg2).addImm(SETFOpc[1]);
1065 MachineInstr &Set3 = *BuildMI(*I.getParent(), I, I.getDebugLoc(),
1066 TII.get(SETFOpc[2]), ResultReg)
1067 .addReg(FlagReg1)
1068 .addReg(FlagReg2);
1073
1074 I.eraseFromParent();
1075 return true;
1076 }
1077
1079 bool SwapArgs;
1080 std::tie(CC, SwapArgs) = X86::getX86ConditionCode(Predicate);
1081 assert(CC <= X86::LAST_VALID_COND && "Unexpected condition code.");
1082
1083 if (SwapArgs)
1084 std::swap(LhsReg, RhsReg);
1085
1086 // Emit a compare of LHS/RHS.
1088 *BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(OpCmp))
1089 .addReg(LhsReg)
1090 .addReg(RhsReg);
1091
1092 MachineInstr &Set =
1093 *BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(X86::SETCCr), ResultReg).addImm(CC);
1096 I.eraseFromParent();
1097 return true;
1098}
1099
1100bool X86InstructionSelector::selectUAddSub(MachineInstr &I,
1102 MachineFunction &MF) const {
1103 assert((I.getOpcode() == TargetOpcode::G_UADDE ||
1104 I.getOpcode() == TargetOpcode::G_UADDO ||
1105 I.getOpcode() == TargetOpcode::G_USUBE ||
1106 I.getOpcode() == TargetOpcode::G_USUBO) &&
1107 "unexpected instruction");
1108
1109 const Register DstReg = I.getOperand(0).getReg();
1110 const Register CarryOutReg = I.getOperand(1).getReg();
1111 const Register Op0Reg = I.getOperand(2).getReg();
1112 const Register Op1Reg = I.getOperand(3).getReg();
1113 bool IsSub = I.getOpcode() == TargetOpcode::G_USUBE ||
1114 I.getOpcode() == TargetOpcode::G_USUBO;
1115 bool HasCarryIn = I.getOpcode() == TargetOpcode::G_UADDE ||
1116 I.getOpcode() == TargetOpcode::G_USUBE;
1117
1118 const LLT DstTy = MRI.getType(DstReg);
1119 assert(DstTy.isScalar() && "selectUAddSub only supported for scalar types");
1120
1121 // TODO: Handle immediate argument variants?
1122 unsigned OpADC, OpADD, OpSBB, OpSUB;
1123 switch (DstTy.getSizeInBits()) {
1124 case 8:
1125 OpADC = X86::ADC8rr;
1126 OpADD = X86::ADD8rr;
1127 OpSBB = X86::SBB8rr;
1128 OpSUB = X86::SUB8rr;
1129 break;
1130 case 16:
1131 OpADC = X86::ADC16rr;
1132 OpADD = X86::ADD16rr;
1133 OpSBB = X86::SBB16rr;
1134 OpSUB = X86::SUB16rr;
1135 break;
1136 case 32:
1137 OpADC = X86::ADC32rr;
1138 OpADD = X86::ADD32rr;
1139 OpSBB = X86::SBB32rr;
1140 OpSUB = X86::SUB32rr;
1141 break;
1142 case 64:
1143 OpADC = X86::ADC64rr;
1144 OpADD = X86::ADD64rr;
1145 OpSBB = X86::SBB64rr;
1146 OpSUB = X86::SUB64rr;
1147 break;
1148 default:
1149 llvm_unreachable("selectUAddSub unsupported type.");
1150 }
1151
1152 const RegisterBank &DstRB = *RBI.getRegBank(DstReg, MRI, TRI);
1153 const TargetRegisterClass *DstRC = getRegClass(DstTy, DstRB);
1154
1155 unsigned Opcode = IsSub ? OpSUB : OpADD;
1156
1157 // G_UADDE/G_USUBE - find CarryIn def instruction.
1158 if (HasCarryIn) {
1159 Register CarryInReg = I.getOperand(4).getReg();
1160 MachineInstr *Def = MRI.getVRegDef(CarryInReg);
1161 while (Def->getOpcode() == TargetOpcode::G_TRUNC) {
1162 CarryInReg = Def->getOperand(1).getReg();
1163 Def = MRI.getVRegDef(CarryInReg);
1164 }
1165
1166 // TODO - handle more CF generating instructions
1167 if (Def->getOpcode() == TargetOpcode::G_UADDE ||
1168 Def->getOpcode() == TargetOpcode::G_UADDO ||
1169 Def->getOpcode() == TargetOpcode::G_USUBE ||
1170 Def->getOpcode() == TargetOpcode::G_USUBO) {
1171 // carry set by prev ADD/SUB.
1172 BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(X86::COPY),
1173 X86::EFLAGS)
1174 .addReg(CarryInReg);
1175
1176 if (!RBI.constrainGenericRegister(CarryInReg, *DstRC, MRI))
1177 return false;
1178
1179 Opcode = IsSub ? OpSBB : OpADC;
1180 } else if (auto val = getIConstantVRegVal(CarryInReg, MRI)) {
1181 // carry is constant, support only 0.
1182 if (*val != 0)
1183 return false;
1184
1185 Opcode = IsSub ? OpSUB : OpADD;
1186 } else
1187 return false;
1188 }
1189
1190 MachineInstr &Inst =
1191 *BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(Opcode), DstReg)
1192 .addReg(Op0Reg)
1193 .addReg(Op1Reg);
1194
1195 BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(X86::COPY), CarryOutReg)
1196 .addReg(X86::EFLAGS);
1197
1198 if (!constrainSelectedInstRegOperands(Inst, TII, TRI, RBI) ||
1199 !RBI.constrainGenericRegister(CarryOutReg, *DstRC, MRI))
1200 return false;
1201
1202 I.eraseFromParent();
1203 return true;
1204}
1205
1206bool X86InstructionSelector::selectExtract(MachineInstr &I,
1208 MachineFunction &MF) const {
1209 assert((I.getOpcode() == TargetOpcode::G_EXTRACT) &&
1210 "unexpected instruction");
1211
1212 const Register DstReg = I.getOperand(0).getReg();
1213 const Register SrcReg = I.getOperand(1).getReg();
1214 int64_t Index = I.getOperand(2).getImm();
1215
1216 const LLT DstTy = MRI.getType(DstReg);
1217 const LLT SrcTy = MRI.getType(SrcReg);
1218
1219 // Meanwile handle vector type only.
1220 if (!DstTy.isVector())
1221 return false;
1222
1223 if (Index % DstTy.getSizeInBits() != 0)
1224 return false; // Not extract subvector.
1225
1226 if (Index == 0) {
1227 // Replace by extract subreg copy.
1228 if (!emitExtractSubreg(DstReg, SrcReg, I, MRI, MF))
1229 return false;
1230
1231 I.eraseFromParent();
1232 return true;
1233 }
1234
1235 bool HasAVX = STI.hasAVX();
1236 bool HasAVX512 = STI.hasAVX512();
1237 bool HasVLX = STI.hasVLX();
1238
1239 if (SrcTy.getSizeInBits() == 256 && DstTy.getSizeInBits() == 128) {
1240 if (HasVLX)
1241 I.setDesc(TII.get(X86::VEXTRACTF32x4Z256rr));
1242 else if (HasAVX)
1243 I.setDesc(TII.get(X86::VEXTRACTF128rr));
1244 else
1245 return false;
1246 } else if (SrcTy.getSizeInBits() == 512 && HasAVX512) {
1247 if (DstTy.getSizeInBits() == 128)
1248 I.setDesc(TII.get(X86::VEXTRACTF32x4Zrr));
1249 else if (DstTy.getSizeInBits() == 256)
1250 I.setDesc(TII.get(X86::VEXTRACTF64x4Zrr));
1251 else
1252 return false;
1253 } else
1254 return false;
1255
1256 // Convert to X86 VEXTRACT immediate.
1257 Index = Index / DstTy.getSizeInBits();
1258 I.getOperand(2).setImm(Index);
1259
1261}
1262
1263bool X86InstructionSelector::emitExtractSubreg(unsigned DstReg, unsigned SrcReg,
1264 MachineInstr &I,
1266 MachineFunction &MF) const {
1267 const LLT DstTy = MRI.getType(DstReg);
1268 const LLT SrcTy = MRI.getType(SrcReg);
1269 unsigned SubIdx = X86::NoSubRegister;
1270
1271 if (!DstTy.isVector() || !SrcTy.isVector())
1272 return false;
1273
1274 assert(SrcTy.getSizeInBits() > DstTy.getSizeInBits() &&
1275 "Incorrect Src/Dst register size");
1276
1277 if (DstTy.getSizeInBits() == 128)
1278 SubIdx = X86::sub_xmm;
1279 else if (DstTy.getSizeInBits() == 256)
1280 SubIdx = X86::sub_ymm;
1281 else
1282 return false;
1283
1284 const TargetRegisterClass *DstRC = getRegClass(DstTy, DstReg, MRI);
1285 const TargetRegisterClass *SrcRC = getRegClass(SrcTy, SrcReg, MRI);
1286
1287 SrcRC = TRI.getSubClassWithSubReg(SrcRC, SubIdx);
1288
1289 if (!RBI.constrainGenericRegister(SrcReg, *SrcRC, MRI) ||
1290 !RBI.constrainGenericRegister(DstReg, *DstRC, MRI)) {
1291 LLVM_DEBUG(dbgs() << "Failed to constrain EXTRACT_SUBREG\n");
1292 return false;
1293 }
1294
1295 BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(X86::COPY), DstReg)
1296 .addReg(SrcReg, 0, SubIdx);
1297
1298 return true;
1299}
1300
1301bool X86InstructionSelector::emitInsertSubreg(unsigned DstReg, unsigned SrcReg,
1302 MachineInstr &I,
1304 MachineFunction &MF) const {
1305 const LLT DstTy = MRI.getType(DstReg);
1306 const LLT SrcTy = MRI.getType(SrcReg);
1307 unsigned SubIdx = X86::NoSubRegister;
1308
1309 // TODO: support scalar types
1310 if (!DstTy.isVector() || !SrcTy.isVector())
1311 return false;
1312
1313 assert(SrcTy.getSizeInBits() < DstTy.getSizeInBits() &&
1314 "Incorrect Src/Dst register size");
1315
1316 if (SrcTy.getSizeInBits() == 128)
1317 SubIdx = X86::sub_xmm;
1318 else if (SrcTy.getSizeInBits() == 256)
1319 SubIdx = X86::sub_ymm;
1320 else
1321 return false;
1322
1323 const TargetRegisterClass *SrcRC = getRegClass(SrcTy, SrcReg, MRI);
1324 const TargetRegisterClass *DstRC = getRegClass(DstTy, DstReg, MRI);
1325
1326 if (!RBI.constrainGenericRegister(SrcReg, *SrcRC, MRI) ||
1327 !RBI.constrainGenericRegister(DstReg, *DstRC, MRI)) {
1328 LLVM_DEBUG(dbgs() << "Failed to constrain INSERT_SUBREG\n");
1329 return false;
1330 }
1331
1332 BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(X86::COPY))
1333 .addReg(DstReg, RegState::DefineNoRead, SubIdx)
1334 .addReg(SrcReg);
1335
1336 return true;
1337}
1338
1339bool X86InstructionSelector::selectInsert(MachineInstr &I,
1341 MachineFunction &MF) const {
1342 assert((I.getOpcode() == TargetOpcode::G_INSERT) && "unexpected instruction");
1343
1344 const Register DstReg = I.getOperand(0).getReg();
1345 const Register SrcReg = I.getOperand(1).getReg();
1346 const Register InsertReg = I.getOperand(2).getReg();
1347 int64_t Index = I.getOperand(3).getImm();
1348
1349 const LLT DstTy = MRI.getType(DstReg);
1350 const LLT InsertRegTy = MRI.getType(InsertReg);
1351
1352 // Meanwile handle vector type only.
1353 if (!DstTy.isVector())
1354 return false;
1355
1356 if (Index % InsertRegTy.getSizeInBits() != 0)
1357 return false; // Not insert subvector.
1358
1359 if (Index == 0 && MRI.getVRegDef(SrcReg)->isImplicitDef()) {
1360 // Replace by subreg copy.
1361 if (!emitInsertSubreg(DstReg, InsertReg, I, MRI, MF))
1362 return false;
1363
1364 I.eraseFromParent();
1365 return true;
1366 }
1367
1368 bool HasAVX = STI.hasAVX();
1369 bool HasAVX512 = STI.hasAVX512();
1370 bool HasVLX = STI.hasVLX();
1371
1372 if (DstTy.getSizeInBits() == 256 && InsertRegTy.getSizeInBits() == 128) {
1373 if (HasVLX)
1374 I.setDesc(TII.get(X86::VINSERTF32x4Z256rr));
1375 else if (HasAVX)
1376 I.setDesc(TII.get(X86::VINSERTF128rr));
1377 else
1378 return false;
1379 } else if (DstTy.getSizeInBits() == 512 && HasAVX512) {
1380 if (InsertRegTy.getSizeInBits() == 128)
1381 I.setDesc(TII.get(X86::VINSERTF32x4Zrr));
1382 else if (InsertRegTy.getSizeInBits() == 256)
1383 I.setDesc(TII.get(X86::VINSERTF64x4Zrr));
1384 else
1385 return false;
1386 } else
1387 return false;
1388
1389 // Convert to X86 VINSERT immediate.
1390 Index = Index / InsertRegTy.getSizeInBits();
1391
1392 I.getOperand(3).setImm(Index);
1393
1395}
1396
1397bool X86InstructionSelector::selectUnmergeValues(
1399 assert((I.getOpcode() == TargetOpcode::G_UNMERGE_VALUES) &&
1400 "unexpected instruction");
1401
1402 // Split to extracts.
1403 unsigned NumDefs = I.getNumOperands() - 1;
1404 Register SrcReg = I.getOperand(NumDefs).getReg();
1405 unsigned DefSize = MRI.getType(I.getOperand(0).getReg()).getSizeInBits();
1406
1407 for (unsigned Idx = 0; Idx < NumDefs; ++Idx) {
1408 MachineInstr &ExtrInst =
1409 *BuildMI(*I.getParent(), I, I.getDebugLoc(),
1410 TII.get(TargetOpcode::G_EXTRACT), I.getOperand(Idx).getReg())
1411 .addReg(SrcReg)
1412 .addImm(Idx * DefSize);
1413
1414 if (!select(ExtrInst))
1415 return false;
1416 }
1417
1418 I.eraseFromParent();
1419 return true;
1420}
1421
1422bool X86InstructionSelector::selectMergeValues(
1424 assert((I.getOpcode() == TargetOpcode::G_MERGE_VALUES ||
1425 I.getOpcode() == TargetOpcode::G_CONCAT_VECTORS) &&
1426 "unexpected instruction");
1427
1428 // Split to inserts.
1429 Register DstReg = I.getOperand(0).getReg();
1430 Register SrcReg0 = I.getOperand(1).getReg();
1431
1432 const LLT DstTy = MRI.getType(DstReg);
1433 const LLT SrcTy = MRI.getType(SrcReg0);
1434 unsigned SrcSize = SrcTy.getSizeInBits();
1435
1436 const RegisterBank &RegBank = *RBI.getRegBank(DstReg, MRI, TRI);
1437
1438 // For the first src use insertSubReg.
1439 Register DefReg = MRI.createGenericVirtualRegister(DstTy);
1440 MRI.setRegBank(DefReg, RegBank);
1441 if (!emitInsertSubreg(DefReg, I.getOperand(1).getReg(), I, MRI, MF))
1442 return false;
1443
1444 for (unsigned Idx = 2; Idx < I.getNumOperands(); ++Idx) {
1445 Register Tmp = MRI.createGenericVirtualRegister(DstTy);
1446 MRI.setRegBank(Tmp, RegBank);
1447
1448 MachineInstr &InsertInst = *BuildMI(*I.getParent(), I, I.getDebugLoc(),
1449 TII.get(TargetOpcode::G_INSERT), Tmp)
1450 .addReg(DefReg)
1451 .addReg(I.getOperand(Idx).getReg())
1452 .addImm((Idx - 1) * SrcSize);
1453
1454 DefReg = Tmp;
1455
1456 if (!select(InsertInst))
1457 return false;
1458 }
1459
1460 MachineInstr &CopyInst = *BuildMI(*I.getParent(), I, I.getDebugLoc(),
1461 TII.get(TargetOpcode::COPY), DstReg)
1462 .addReg(DefReg);
1463
1464 if (!select(CopyInst))
1465 return false;
1466
1467 I.eraseFromParent();
1468 return true;
1469}
1470
1471bool X86InstructionSelector::selectCondBranch(MachineInstr &I,
1473 MachineFunction &MF) const {
1474 assert((I.getOpcode() == TargetOpcode::G_BRCOND) && "unexpected instruction");
1475
1476 const Register CondReg = I.getOperand(0).getReg();
1477 MachineBasicBlock *DestMBB = I.getOperand(1).getMBB();
1478
1479 MachineInstr &TestInst =
1480 *BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(X86::TEST8ri))
1481 .addReg(CondReg)
1482 .addImm(1);
1483 BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(X86::JCC_1))
1484 .addMBB(DestMBB).addImm(X86::COND_NE);
1485
1486 constrainSelectedInstRegOperands(TestInst, TII, TRI, RBI);
1487
1488 I.eraseFromParent();
1489 return true;
1490}
1491
1492bool X86InstructionSelector::materializeFP(MachineInstr &I,
1494 MachineFunction &MF) const {
1495 assert((I.getOpcode() == TargetOpcode::G_FCONSTANT) &&
1496 "unexpected instruction");
1497
1498 // Can't handle alternate code models yet.
1499 CodeModel::Model CM = TM.getCodeModel();
1500 if (CM != CodeModel::Small && CM != CodeModel::Large)
1501 return false;
1502
1503 const Register DstReg = I.getOperand(0).getReg();
1504 const LLT DstTy = MRI.getType(DstReg);
1505 const RegisterBank &RegBank = *RBI.getRegBank(DstReg, MRI, TRI);
1506 Align Alignment = Align(DstTy.getSizeInBytes());
1507 const DebugLoc &DbgLoc = I.getDebugLoc();
1508
1509 unsigned Opc =
1510 getLoadStoreOp(DstTy, RegBank, TargetOpcode::G_LOAD, Alignment);
1511
1512 // Create the load from the constant pool.
1513 const ConstantFP *CFP = I.getOperand(1).getFPImm();
1514 unsigned CPI = MF.getConstantPool()->getConstantPoolIndex(CFP, Alignment);
1515 MachineInstr *LoadInst = nullptr;
1516 unsigned char OpFlag = STI.classifyLocalReference(nullptr);
1517
1518 if (CM == CodeModel::Large && STI.is64Bit()) {
1519 // Under X86-64 non-small code model, GV (and friends) are 64-bits, so
1520 // they cannot be folded into immediate fields.
1521
1522 Register AddrReg = MRI.createVirtualRegister(&X86::GR64RegClass);
1523 BuildMI(*I.getParent(), I, DbgLoc, TII.get(X86::MOV64ri), AddrReg)
1524 .addConstantPoolIndex(CPI, 0, OpFlag);
1525
1528 LLT::pointer(0, MF.getDataLayout().getPointerSizeInBits()), Alignment);
1529
1530 LoadInst =
1531 addDirectMem(BuildMI(*I.getParent(), I, DbgLoc, TII.get(Opc), DstReg),
1532 AddrReg)
1533 .addMemOperand(MMO);
1534
1535 } else if (CM == CodeModel::Small || !STI.is64Bit()) {
1536 // Handle the case when globals fit in our immediate field.
1537 // This is true for X86-32 always and X86-64 when in -mcmodel=small mode.
1538
1539 // x86-32 PIC requires a PIC base register for constant pools.
1540 unsigned PICBase = 0;
1541 if (OpFlag == X86II::MO_PIC_BASE_OFFSET || OpFlag == X86II::MO_GOTOFF) {
1542 // PICBase can be allocated by TII.getGlobalBaseReg(&MF).
1543 // In DAGISEL the code that initialize it generated by the CGBR pass.
1544 return false; // TODO support the mode.
1545 } else if (STI.is64Bit() && TM.getCodeModel() == CodeModel::Small)
1546 PICBase = X86::RIP;
1547
1549 BuildMI(*I.getParent(), I, DbgLoc, TII.get(Opc), DstReg), CPI, PICBase,
1550 OpFlag);
1551 } else
1552 return false;
1553
1555 I.eraseFromParent();
1556 return true;
1557}
1558
1559bool X86InstructionSelector::selectImplicitDefOrPHI(
1561 assert((I.getOpcode() == TargetOpcode::G_IMPLICIT_DEF ||
1562 I.getOpcode() == TargetOpcode::G_PHI) &&
1563 "unexpected instruction");
1564
1565 Register DstReg = I.getOperand(0).getReg();
1566
1567 if (!MRI.getRegClassOrNull(DstReg)) {
1568 const LLT DstTy = MRI.getType(DstReg);
1569 const TargetRegisterClass *RC = getRegClass(DstTy, DstReg, MRI);
1570
1571 if (!RBI.constrainGenericRegister(DstReg, *RC, MRI)) {
1572 LLVM_DEBUG(dbgs() << "Failed to constrain " << TII.getName(I.getOpcode())
1573 << " operand\n");
1574 return false;
1575 }
1576 }
1577
1578 if (I.getOpcode() == TargetOpcode::G_IMPLICIT_DEF)
1579 I.setDesc(TII.get(X86::IMPLICIT_DEF));
1580 else
1581 I.setDesc(TII.get(X86::PHI));
1582
1583 return true;
1584}
1585
1586bool X86InstructionSelector::selectMulDivRem(MachineInstr &I,
1588 MachineFunction &MF) const {
1589 // The implementation of this function is adapted from X86FastISel.
1590 assert((I.getOpcode() == TargetOpcode::G_MUL ||
1591 I.getOpcode() == TargetOpcode::G_SMULH ||
1592 I.getOpcode() == TargetOpcode::G_UMULH ||
1593 I.getOpcode() == TargetOpcode::G_SDIV ||
1594 I.getOpcode() == TargetOpcode::G_SREM ||
1595 I.getOpcode() == TargetOpcode::G_UDIV ||
1596 I.getOpcode() == TargetOpcode::G_UREM) &&
1597 "unexpected instruction");
1598
1599 const Register DstReg = I.getOperand(0).getReg();
1600 const Register Op1Reg = I.getOperand(1).getReg();
1601 const Register Op2Reg = I.getOperand(2).getReg();
1602
1603 const LLT RegTy = MRI.getType(DstReg);
1604 assert(RegTy == MRI.getType(Op1Reg) && RegTy == MRI.getType(Op2Reg) &&
1605 "Arguments and return value types must match");
1606
1607 const RegisterBank *RegRB = RBI.getRegBank(DstReg, MRI, TRI);
1608 if (!RegRB || RegRB->getID() != X86::GPRRegBankID)
1609 return false;
1610
1611 const static unsigned NumTypes = 4; // i8, i16, i32, i64
1612 const static unsigned NumOps = 7; // SDiv/SRem/UDiv/URem/Mul/SMulH/UMulh
1613 const static bool S = true; // IsSigned
1614 const static bool U = false; // !IsSigned
1615 const static unsigned Copy = TargetOpcode::COPY;
1616
1617 // For the X86 IDIV instruction, in most cases the dividend
1618 // (numerator) must be in a specific register pair highreg:lowreg,
1619 // producing the quotient in lowreg and the remainder in highreg.
1620 // For most data types, to set up the instruction, the dividend is
1621 // copied into lowreg, and lowreg is sign-extended into highreg. The
1622 // exception is i8, where the dividend is defined as a single register rather
1623 // than a register pair, and we therefore directly sign-extend the dividend
1624 // into lowreg, instead of copying, and ignore the highreg.
1625 const static struct MulDivRemEntry {
1626 // The following portion depends only on the data type.
1627 unsigned SizeInBits;
1628 unsigned LowInReg; // low part of the register pair
1629 unsigned HighInReg; // high part of the register pair
1630 // The following portion depends on both the data type and the operation.
1631 struct MulDivRemResult {
1632 unsigned OpMulDivRem; // The specific MUL/DIV opcode to use.
1633 unsigned OpSignExtend; // Opcode for sign-extending lowreg into
1634 // highreg, or copying a zero into highreg.
1635 unsigned OpCopy; // Opcode for copying dividend into lowreg, or
1636 // zero/sign-extending into lowreg for i8.
1637 unsigned ResultReg; // Register containing the desired result.
1638 bool IsOpSigned; // Whether to use signed or unsigned form.
1639 } ResultTable[NumOps];
1640 } OpTable[NumTypes] = {
1641 {8,
1642 X86::AX,
1643 0,
1644 {
1645 {X86::IDIV8r, 0, X86::MOVSX16rr8, X86::AL, S}, // SDiv
1646 {X86::IDIV8r, 0, X86::MOVSX16rr8, X86::AH, S}, // SRem
1647 {X86::DIV8r, 0, X86::MOVZX16rr8, X86::AL, U}, // UDiv
1648 {X86::DIV8r, 0, X86::MOVZX16rr8, X86::AH, U}, // URem
1649 {X86::IMUL8r, 0, X86::MOVSX16rr8, X86::AL, S}, // Mul
1650 {X86::IMUL8r, 0, X86::MOVSX16rr8, X86::AH, S}, // SMulH
1651 {X86::MUL8r, 0, X86::MOVZX16rr8, X86::AH, U}, // UMulH
1652 }}, // i8
1653 {16,
1654 X86::AX,
1655 X86::DX,
1656 {
1657 {X86::IDIV16r, X86::CWD, Copy, X86::AX, S}, // SDiv
1658 {X86::IDIV16r, X86::CWD, Copy, X86::DX, S}, // SRem
1659 {X86::DIV16r, X86::MOV32r0, Copy, X86::AX, U}, // UDiv
1660 {X86::DIV16r, X86::MOV32r0, Copy, X86::DX, U}, // URem
1661 {X86::IMUL16r, X86::MOV32r0, Copy, X86::AX, S}, // Mul
1662 {X86::IMUL16r, X86::MOV32r0, Copy, X86::DX, S}, // SMulH
1663 {X86::MUL16r, X86::MOV32r0, Copy, X86::DX, U}, // UMulH
1664 }}, // i16
1665 {32,
1666 X86::EAX,
1667 X86::EDX,
1668 {
1669 {X86::IDIV32r, X86::CDQ, Copy, X86::EAX, S}, // SDiv
1670 {X86::IDIV32r, X86::CDQ, Copy, X86::EDX, S}, // SRem
1671 {X86::DIV32r, X86::MOV32r0, Copy, X86::EAX, U}, // UDiv
1672 {X86::DIV32r, X86::MOV32r0, Copy, X86::EDX, U}, // URem
1673 {X86::IMUL32r, X86::MOV32r0, Copy, X86::EAX, S}, // Mul
1674 {X86::IMUL32r, X86::MOV32r0, Copy, X86::EDX, S}, // SMulH
1675 {X86::MUL32r, X86::MOV32r0, Copy, X86::EDX, U}, // UMulH
1676 }}, // i32
1677 {64,
1678 X86::RAX,
1679 X86::RDX,
1680 {
1681 {X86::IDIV64r, X86::CQO, Copy, X86::RAX, S}, // SDiv
1682 {X86::IDIV64r, X86::CQO, Copy, X86::RDX, S}, // SRem
1683 {X86::DIV64r, X86::MOV32r0, Copy, X86::RAX, U}, // UDiv
1684 {X86::DIV64r, X86::MOV32r0, Copy, X86::RDX, U}, // URem
1685 {X86::IMUL64r, X86::MOV32r0, Copy, X86::RAX, S}, // Mul
1686 {X86::IMUL64r, X86::MOV32r0, Copy, X86::RDX, S}, // SMulH
1687 {X86::MUL64r, X86::MOV32r0, Copy, X86::RDX, U}, // UMulH
1688 }}, // i64
1689 };
1690
1691 auto OpEntryIt = llvm::find_if(OpTable, [RegTy](const MulDivRemEntry &El) {
1692 return El.SizeInBits == RegTy.getSizeInBits();
1693 });
1694 if (OpEntryIt == std::end(OpTable))
1695 return false;
1696
1697 unsigned OpIndex;
1698 switch (I.getOpcode()) {
1699 default:
1700 llvm_unreachable("Unexpected mul/div/rem opcode");
1701 case TargetOpcode::G_SDIV:
1702 OpIndex = 0;
1703 break;
1704 case TargetOpcode::G_SREM:
1705 OpIndex = 1;
1706 break;
1707 case TargetOpcode::G_UDIV:
1708 OpIndex = 2;
1709 break;
1710 case TargetOpcode::G_UREM:
1711 OpIndex = 3;
1712 break;
1713 case TargetOpcode::G_MUL:
1714 OpIndex = 4;
1715 break;
1716 case TargetOpcode::G_SMULH:
1717 OpIndex = 5;
1718 break;
1719 case TargetOpcode::G_UMULH:
1720 OpIndex = 6;
1721 break;
1722 }
1723
1724 const MulDivRemEntry &TypeEntry = *OpEntryIt;
1725 const MulDivRemEntry::MulDivRemResult &OpEntry =
1726 TypeEntry.ResultTable[OpIndex];
1727
1728 const TargetRegisterClass *RegRC = getRegClass(RegTy, *RegRB);
1729 if (!RBI.constrainGenericRegister(Op1Reg, *RegRC, MRI) ||
1730 !RBI.constrainGenericRegister(Op2Reg, *RegRC, MRI) ||
1731 !RBI.constrainGenericRegister(DstReg, *RegRC, MRI)) {
1732 LLVM_DEBUG(dbgs() << "Failed to constrain " << TII.getName(I.getOpcode())
1733 << " operand\n");
1734 return false;
1735 }
1736
1737 // Move op1 into low-order input register.
1738 BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(OpEntry.OpCopy),
1739 TypeEntry.LowInReg)
1740 .addReg(Op1Reg);
1741
1742 // Zero-extend or sign-extend into high-order input register.
1743 if (OpEntry.OpSignExtend) {
1744 if (OpEntry.IsOpSigned)
1745 BuildMI(*I.getParent(), I, I.getDebugLoc(),
1746 TII.get(OpEntry.OpSignExtend));
1747 else {
1748 Register Zero32 = MRI.createVirtualRegister(&X86::GR32RegClass);
1749 BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(X86::MOV32r0),
1750 Zero32);
1751
1752 // Copy the zero into the appropriate sub/super/identical physical
1753 // register. Unfortunately the operations needed are not uniform enough
1754 // to fit neatly into the table above.
1755 if (RegTy.getSizeInBits() == 16) {
1756 BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(Copy),
1757 TypeEntry.HighInReg)
1758 .addReg(Zero32, 0, X86::sub_16bit);
1759 } else if (RegTy.getSizeInBits() == 32) {
1760 BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(Copy),
1761 TypeEntry.HighInReg)
1762 .addReg(Zero32);
1763 } else if (RegTy.getSizeInBits() == 64) {
1764 BuildMI(*I.getParent(), I, I.getDebugLoc(),
1765 TII.get(TargetOpcode::SUBREG_TO_REG), TypeEntry.HighInReg)
1766 .addImm(0)
1767 .addReg(Zero32)
1768 .addImm(X86::sub_32bit);
1769 }
1770 }
1771 }
1772
1773 // Generate the DIV/IDIV/MUL/IMUL instruction.
1774 BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(OpEntry.OpMulDivRem))
1775 .addReg(Op2Reg);
1776
1777 // For i8 remainder, we can't reference ah directly, as we'll end
1778 // up with bogus copies like %r9b = COPY %ah. Reference ax
1779 // instead to prevent ah references in a rex instruction.
1780 //
1781 // The current assumption of the fast register allocator is that isel
1782 // won't generate explicit references to the GR8_NOREX registers. If
1783 // the allocator and/or the backend get enhanced to be more robust in
1784 // that regard, this can be, and should be, removed.
1785 if (OpEntry.ResultReg == X86::AH && STI.is64Bit()) {
1786 Register SourceSuperReg = MRI.createVirtualRegister(&X86::GR16RegClass);
1787 Register ResultSuperReg = MRI.createVirtualRegister(&X86::GR16RegClass);
1788 BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(Copy), SourceSuperReg)
1789 .addReg(X86::AX);
1790
1791 // Shift AX right by 8 bits instead of using AH.
1792 BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(X86::SHR16ri),
1793 ResultSuperReg)
1794 .addReg(SourceSuperReg)
1795 .addImm(8);
1796
1797 // Now reference the 8-bit subreg of the result.
1798 BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(TargetOpcode::COPY),
1799 DstReg)
1800 .addReg(ResultSuperReg, 0, X86::sub_8bit);
1801 } else {
1802 BuildMI(*I.getParent(), I, I.getDebugLoc(), TII.get(TargetOpcode::COPY),
1803 DstReg)
1804 .addReg(OpEntry.ResultReg);
1805 }
1806 I.eraseFromParent();
1807
1808 return true;
1809}
1810
1811bool X86InstructionSelector::selectSelect(MachineInstr &I,
1813 MachineFunction &MF) const {
1814 GSelect &Sel = cast<GSelect>(I);
1815 unsigned DstReg = Sel.getReg(0);
1816 BuildMI(*Sel.getParent(), Sel, Sel.getDebugLoc(), TII.get(X86::TEST32rr))
1817 .addReg(Sel.getCondReg())
1818 .addReg(Sel.getCondReg());
1819
1820 unsigned OpCmp;
1821 LLT Ty = MRI.getType(DstReg);
1822 switch (Ty.getSizeInBits()) {
1823 default:
1824 return false;
1825 case 8:
1826 OpCmp = X86::CMOV_GR8;
1827 break;
1828 case 16:
1829 OpCmp = STI.canUseCMOV() ? X86::CMOV16rr : X86::CMOV_GR16;
1830 break;
1831 case 32:
1832 OpCmp = STI.canUseCMOV() ? X86::CMOV32rr : X86::CMOV_GR32;
1833 break;
1834 case 64:
1835 assert(STI.is64Bit() && STI.canUseCMOV());
1836 OpCmp = X86::CMOV64rr;
1837 break;
1838 }
1839 BuildMI(*Sel.getParent(), Sel, Sel.getDebugLoc(), TII.get(OpCmp), DstReg)
1840 .addReg(Sel.getTrueReg())
1841 .addReg(Sel.getFalseReg())
1843
1844 const TargetRegisterClass *DstRC = getRegClass(Ty, DstReg, MRI);
1845 if (!RBI.constrainGenericRegister(DstReg, *DstRC, MRI)) {
1846 LLVM_DEBUG(dbgs() << "Failed to constrain CMOV\n");
1847 return false;
1848 }
1849
1850 Sel.eraseFromParent();
1851 return true;
1852}
1853
1856 X86Subtarget &Subtarget,
1857 X86RegisterBankInfo &RBI) {
1858 return new X86InstructionSelector(TM, Subtarget, RBI);
1859}
unsigned const MachineRegisterInfo * MRI
static const TargetRegisterClass * getRegClass(const MachineInstr &MI, Register Reg)
static bool selectCopy(MachineInstr &I, const TargetInstrInfo &TII, MachineRegisterInfo &MRI, const TargetRegisterInfo &TRI, const RegisterBankInfo &RBI)
static bool selectDebugInstr(MachineInstr &I, MachineRegisterInfo &MRI, const RegisterBankInfo &RBI)
MachineBasicBlock & MBB
static bool selectMergeValues(MachineInstrBuilder &MIB, const ARMBaseInstrInfo &TII, MachineRegisterInfo &MRI, const TargetRegisterInfo &TRI, const RegisterBankInfo &RBI)
static bool selectUnmergeValues(MachineInstrBuilder &MIB, const ARMBaseInstrInfo &TII, MachineRegisterInfo &MRI, const TargetRegisterInfo &TRI, const RegisterBankInfo &RBI)
Atomic ordering constants.
Returns the sub type a function will return at a given Idx Should correspond to the result type of an ExtractValue instruction executed with just that one unsigned Idx
#define LLVM_DEBUG(X)
Definition: Debug.h:101
#define DEBUG_TYPE
Declares convenience wrapper classes for interpreting MachineInstr instances as specific generic oper...
const HexagonInstrInfo * TII
Implement a low-level type suitable for MachineInstr level instruction selection.
#define I(x, y, z)
Definition: MD5.cpp:58
This file declares the MachineConstantPool class which is an abstract constant pool to keep track of ...
unsigned const TargetRegisterInfo * TRI
static unsigned selectLoadStoreOp(unsigned GenericOpc, unsigned RegBankID, unsigned OpSize)
const char LLVMTargetMachineRef TM
static StringRef getName(Value *V)
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
unsigned OpIndex
static bool contains(SmallPtrSetImpl< ConstantExpr * > &Cache, ConstantExpr *Expr, Constant *C)
Definition: Value.cpp:469
#define GET_GLOBALISEL_PREDICATES_INIT
#define GET_GLOBALISEL_TEMPORARIES_INIT
static bool canTurnIntoCOPY(const TargetRegisterClass *DstRC, const TargetRegisterClass *SrcRC)
static unsigned getLeaOP(LLT Ty, const X86Subtarget &STI)
static const TargetRegisterClass * getRegClassFromGRPhysReg(Register Reg)
static void X86SelectAddress(const MachineInstr &I, const MachineRegisterInfo &MRI, X86AddressMode &AM)
Value * RHS
Value * LHS
This file declares the targeting of the RegisterBankInfo class for X86.
This class is the base class for the comparison instructions.
Definition: InstrTypes.h:983
Predicate
This enumeration lists the possible predicates for CmpInst subclasses.
Definition: InstrTypes.h:993
@ FCMP_OEQ
0 0 0 1 True if ordered and equal
Definition: InstrTypes.h:996
@ FCMP_UNE
1 1 1 0 True if unordered or not equal
Definition: InstrTypes.h:1009
ConstantFP - Floating Point Values [float, double].
Definition: Constants.h:269
unsigned getPointerSizeInBits(unsigned AS=0) const
Layout pointer size, in bits FIXME: The defaults need to be removed once all of the backends/clients ...
Definition: DataLayout.h:410
A debug info location.
Definition: DebugLoc.h:33
Represents a G_SELECT.
Register getCondReg() const
Register getFalseReg() const
Register getTrueReg() const
Register getReg(unsigned Idx) const
Access the Idx'th operand as a register and return it.
virtual bool select(MachineInstr &I)=0
Select the (possibly generic) instruction I to only use target-specific opcodes.
constexpr bool isScalar() const
Definition: LowLevelType.h:146
static constexpr LLT scalar(unsigned SizeInBits)
Get a low-level scalar or aggregate "bag of bits".
Definition: LowLevelType.h:42
constexpr bool isVector() const
Definition: LowLevelType.h:148
static constexpr LLT pointer(unsigned AddressSpace, unsigned SizeInBits)
Get a low-level pointer in the given address space.
Definition: LowLevelType.h:57
constexpr TypeSize getSizeInBits() const
Returns the total size of the type. Must only be called on sized types.
Definition: LowLevelType.h:193
constexpr TypeSize getSizeInBytes() const
Returns the total size of the type in bytes, i.e.
Definition: LowLevelType.h:203
An instruction for reading from memory.
Definition: Instructions.h:185
const MachineFunction * getParent() const
Return the MachineFunction containing this basic block.
unsigned getConstantPoolIndex(const Constant *C, Align Alignment)
getConstantPoolIndex - Create a new entry in the constant pool or return an existing one.
MachineMemOperand * getMachineMemOperand(MachinePointerInfo PtrInfo, MachineMemOperand::Flags f, LLT MemTy, Align base_alignment, const AAMDNodes &AAInfo=AAMDNodes(), const MDNode *Ranges=nullptr, SyncScope::ID SSID=SyncScope::System, AtomicOrdering Ordering=AtomicOrdering::NotAtomic, AtomicOrdering FailureOrdering=AtomicOrdering::NotAtomic)
getMachineMemOperand - Allocate a new MachineMemOperand.
MachineRegisterInfo & getRegInfo()
getRegInfo - Return information about the registers currently in use.
const DataLayout & getDataLayout() const
Return the DataLayout attached to the Module associated to this MF.
MachineConstantPool * getConstantPool()
getConstantPool - Return the constant pool object for the current function.
const MachineInstrBuilder & addImm(int64_t Val) const
Add a new immediate operand.
const MachineInstrBuilder & addConstantPoolIndex(unsigned Idx, int Offset=0, unsigned TargetFlags=0) const
const MachineInstrBuilder & addReg(Register RegNo, unsigned flags=0, unsigned SubReg=0) const
Add a new virtual register operand.
const MachineInstrBuilder & addMBB(MachineBasicBlock *MBB, unsigned TargetFlags=0) const
const MachineInstrBuilder & addUse(Register RegNo, unsigned Flags=0, unsigned SubReg=0) const
Add a virtual register use operand.
const MachineInstrBuilder & addMemOperand(MachineMemOperand *MMO) const
const MachineInstrBuilder & addDef(Register RegNo, unsigned Flags=0, unsigned SubReg=0) const
Add a virtual register definition operand.
Representation of each machine instruction.
Definition: MachineInstr.h:69
const MachineBasicBlock * getParent() const
Definition: MachineInstr.h:341
const DebugLoc & getDebugLoc() const
Returns the debug location id of this MachineInstr.
Definition: MachineInstr.h:493
void eraseFromParent()
Unlink 'this' from the containing basic block and delete it.
A description of a memory reference used in the backend.
@ MOLoad
The memory access reads data.
MachineOperand class - Representation of each machine instruction operand.
void ChangeToImmediate(int64_t ImmVal, unsigned TargetFlags=0)
ChangeToImmediate - Replace this operand with a new immediate operand of the specified value.
MachineRegisterInfo - Keep track of information for virtual and physical registers,...
This class implements the register bank concept.
Definition: RegisterBank.h:28
unsigned getID() const
Get the identifier of this register bank.
Definition: RegisterBank.h:45
Wrapper class representing virtual and physical registers.
Definition: Register.h:19
constexpr bool isPhysical() const
Return true if the specified register number is in the physical register namespace.
Definition: Register.h:95
bool hasSubClassEq(const TargetRegisterClass *RC) const
Returns true if RC is a sub-class of or equal to this class.
This class provides the information for the target register banks.
bool isTarget64BitILP32() const
Is this x86_64 with the ILP32 programming model (x32 ABI)?
Definition: X86Subtarget.h:180
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
Predicate
Predicate - These are "(BI << 5) | BO" for various predicates.
Definition: PPCPredicates.h:26
Reg
All possible values of the reg field in the ModR/M byte.
@ MO_GOTOFF
MO_GOTOFF - On a symbol operand this indicates that the immediate is the offset to the location of th...
Definition: X86BaseInfo.h:395
@ MO_PIC_BASE_OFFSET
MO_PIC_BASE_OFFSET - On a symbol operand this indicates that the immediate should get the value of th...
Definition: X86BaseInfo.h:385
@ LAST_VALID_COND
Definition: X86BaseInfo.h:92
std::pair< CondCode, bool > getX86ConditionCode(CmpInst::Predicate Predicate)
Return a pair of condition code for the given predicate and whether the instruction operands should b...
StringMapEntry< std::atomic< TypeEntryBody * > > TypeEntry
Definition: TypePool.h:27
NodeAddr< DefNode * > Def
Definition: RDFGraph.h:384
This is an optimization pass for GlobalISel generic memory operations.
Definition: AddressRanges.h:18
static bool isGlobalStubReference(unsigned char TargetFlag)
isGlobalStubReference - Return true if the specified TargetFlag operand is a reference to a stub for ...
Definition: X86InstrInfo.h:118
static bool isGlobalRelativeToPICBase(unsigned char TargetFlag)
isGlobalRelativeToPICBase - Return true if the specified global value reference is relative to a 32-b...
Definition: X86InstrInfo.h:136
MachineInstrBuilder BuildMI(MachineFunction &MF, const MIMetadata &MIMD, const MCInstrDesc &MCID)
Builder interface. Specify how to create the initial instruction itself.
std::optional< APInt > getIConstantVRegVal(Register VReg, const MachineRegisterInfo &MRI)
If VReg is defined by a G_CONSTANT, return the corresponding value.
Definition: Utils.cpp:295
bool constrainSelectedInstRegOperands(MachineInstr &I, const TargetInstrInfo &TII, const TargetRegisterInfo &TRI, const RegisterBankInfo &RBI)
Mutate the newly-selected instruction I to constrain its (possibly generic) virtual register operands...
Definition: Utils.cpp:155
bool isPreISelGenericOpcode(unsigned Opcode)
Check whether the given Opcode is a generic opcode that is not supposed to appear after ISel.
Definition: TargetOpcodes.h:30
static const MachineInstrBuilder & addFullAddress(const MachineInstrBuilder &MIB, const X86AddressMode &AM)
InstructionSelector * createX86InstructionSelector(const X86TargetMachine &TM, X86Subtarget &, X86RegisterBankInfo &)
std::optional< int64_t > getIConstantVRegSExtVal(Register VReg, const MachineRegisterInfo &MRI)
If VReg is defined by a G_CONSTANT fits in int64_t returns it.
Definition: Utils.cpp:307
static const MachineInstrBuilder & addConstantPoolReference(const MachineInstrBuilder &MIB, unsigned CPI, unsigned GlobalBaseReg, unsigned char OpFlags)
addConstantPoolReference - This function is used to add a reference to the base of a constant value s...
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
Definition: Debug.cpp:163
static const MachineInstrBuilder & addOffset(const MachineInstrBuilder &MIB, int Offset)
auto find_if(R &&Range, UnaryPredicate P)
Provide wrappers to std::find_if which take ranges instead of having to pass begin/end explicitly.
Definition: STLExtras.h:1749
static const MachineInstrBuilder & addDirectMem(const MachineInstrBuilder &MIB, unsigned Reg)
addDirectMem - This function is used to add a direct memory reference to the current instruction – th...
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
Definition: BitVector.h:860
This struct is a compact representation of a valid (non-zero power of two) alignment.
Definition: Alignment.h:39
static MachinePointerInfo getConstantPool(MachineFunction &MF)
Return a MachinePointerInfo record that refers to the constant pool.
X86AddressMode - This struct holds a generalized full x86 address mode.
enum llvm::X86AddressMode::@622 BaseType
union llvm::X86AddressMode::@623 Base
const GlobalValue * GV