LLVM 17.0.0git
X86MCInstLower.cpp
Go to the documentation of this file.
1//===-- X86MCInstLower.cpp - Convert X86 MachineInstr to an MCInst --------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file contains code to lower X86 MachineInstrs to their corresponding
10// MCInst records.
11//
12//===----------------------------------------------------------------------===//
13
19#include "X86AsmPrinter.h"
20#include "X86RegisterInfo.h"
22#include "X86Subtarget.h"
30#include "llvm/IR/DataLayout.h"
31#include "llvm/IR/GlobalValue.h"
32#include "llvm/IR/Mangler.h"
33#include "llvm/MC/MCAsmInfo.h"
35#include "llvm/MC/MCContext.h"
36#include "llvm/MC/MCExpr.h"
37#include "llvm/MC/MCFixup.h"
38#include "llvm/MC/MCInst.h"
40#include "llvm/MC/MCSection.h"
42#include "llvm/MC/MCStreamer.h"
43#include "llvm/MC/MCSymbol.h"
44#include "llvm/MC/MCSymbolELF.h"
50#include <string>
51
52using namespace llvm;
53
54namespace {
55
56/// X86MCInstLower - This class is used to lower an MachineInstr into an MCInst.
57class X86MCInstLower {
58 MCContext &Ctx;
59 const MachineFunction &MF;
60 const TargetMachine &TM;
61 const MCAsmInfo &MAI;
63
64public:
65 X86MCInstLower(const MachineFunction &MF, X86AsmPrinter &asmprinter);
66
67 std::optional<MCOperand> LowerMachineOperand(const MachineInstr *MI,
68 const MachineOperand &MO) const;
69 void Lower(const MachineInstr *MI, MCInst &OutMI) const;
70
73
74private:
75 MachineModuleInfoMachO &getMachOMMI() const;
76};
77
78} // end anonymous namespace
79
80/// A RAII helper which defines a region of instructions which can't have
81/// padding added between them for correctness.
86 : OS(OS), OldAllowAutoPadding(OS.getAllowAutoPadding()) {
87 changeAndComment(false);
88 }
90 void changeAndComment(bool b) {
91 if (b == OS.getAllowAutoPadding())
92 return;
94 if (b)
95 OS.emitRawComment("autopadding");
96 else
97 OS.emitRawComment("noautopadding");
98 }
99};
100
101// Emit a minimal sequence of nops spanning NumBytes bytes.
102static void emitX86Nops(MCStreamer &OS, unsigned NumBytes,
103 const X86Subtarget *Subtarget);
104
105void X86AsmPrinter::StackMapShadowTracker::count(MCInst &Inst,
106 const MCSubtargetInfo &STI,
107 MCCodeEmitter *CodeEmitter) {
108 if (InShadow) {
111 raw_svector_ostream VecOS(Code);
112 CodeEmitter->encodeInstruction(Inst, VecOS, Fixups, STI);
113 CurrentShadowSize += Code.size();
114 if (CurrentShadowSize >= RequiredShadowSize)
115 InShadow = false; // The shadow is big enough. Stop counting.
116 }
117}
118
119void X86AsmPrinter::StackMapShadowTracker::emitShadowPadding(
120 MCStreamer &OutStreamer, const MCSubtargetInfo &STI) {
121 if (InShadow && CurrentShadowSize < RequiredShadowSize) {
122 InShadow = false;
123 emitX86Nops(OutStreamer, RequiredShadowSize - CurrentShadowSize,
124 &MF->getSubtarget<X86Subtarget>());
125 }
126}
127
128void X86AsmPrinter::EmitAndCountInstruction(MCInst &Inst) {
129 OutStreamer->emitInstruction(Inst, getSubtargetInfo());
130 SMShadowTracker.count(Inst, getSubtargetInfo(), CodeEmitter.get());
131}
132
133X86MCInstLower::X86MCInstLower(const MachineFunction &mf,
134 X86AsmPrinter &asmprinter)
135 : Ctx(mf.getContext()), MF(mf), TM(mf.getTarget()), MAI(*TM.getMCAsmInfo()),
136 AsmPrinter(asmprinter) {}
137
138MachineModuleInfoMachO &X86MCInstLower::getMachOMMI() const {
139 return MF.getMMI().getObjFileInfo<MachineModuleInfoMachO>();
140}
141
142/// GetSymbolFromOperand - Lower an MO_GlobalAddress or MO_ExternalSymbol
143/// operand to an MCSymbol.
144MCSymbol *X86MCInstLower::GetSymbolFromOperand(const MachineOperand &MO) const {
145 const Triple &TT = TM.getTargetTriple();
146 if (MO.isGlobal() && TT.isOSBinFormatELF())
148
149 const DataLayout &DL = MF.getDataLayout();
150 assert((MO.isGlobal() || MO.isSymbol() || MO.isMBB()) &&
151 "Isn't a symbol reference");
152
153 MCSymbol *Sym = nullptr;
155 StringRef Suffix;
156
157 switch (MO.getTargetFlags()) {
159 // Handle dllimport linkage.
160 Name += "__imp_";
161 break;
163 Name += ".refptr.";
164 break;
167 Suffix = "$non_lazy_ptr";
168 break;
169 }
170
171 if (!Suffix.empty())
172 Name += DL.getPrivateGlobalPrefix();
173
174 if (MO.isGlobal()) {
175 const GlobalValue *GV = MO.getGlobal();
177 } else if (MO.isSymbol()) {
179 } else if (MO.isMBB()) {
180 assert(Suffix.empty());
181 Sym = MO.getMBB()->getSymbol();
182 }
183
184 Name += Suffix;
185 if (!Sym)
186 Sym = Ctx.getOrCreateSymbol(Name);
187
188 // If the target flags on the operand changes the name of the symbol, do that
189 // before we return the symbol.
190 switch (MO.getTargetFlags()) {
191 default:
192 break;
193 case X86II::MO_COFFSTUB: {
194 MachineModuleInfoCOFF &MMICOFF =
195 MF.getMMI().getObjFileInfo<MachineModuleInfoCOFF>();
197 if (!StubSym.getPointer()) {
198 assert(MO.isGlobal() && "Extern symbol not handled yet");
200 AsmPrinter.getSymbol(MO.getGlobal()), true);
201 }
202 break;
203 }
207 getMachOMMI().getGVStubEntry(Sym);
208 if (!StubSym.getPointer()) {
209 assert(MO.isGlobal() && "Extern symbol not handled yet");
213 }
214 break;
215 }
216 }
217
218 return Sym;
219}
220
221MCOperand X86MCInstLower::LowerSymbolOperand(const MachineOperand &MO,
222 MCSymbol *Sym) const {
223 // FIXME: We would like an efficient form for this, so we don't have to do a
224 // lot of extra uniquing.
225 const MCExpr *Expr = nullptr;
227
228 switch (MO.getTargetFlags()) {
229 default:
230 llvm_unreachable("Unknown target flag on GV operand");
231 case X86II::MO_NO_FLAG: // No flag.
232 // These affect the name of the symbol, not any suffix.
236 break;
237
238 case X86II::MO_TLVP:
239 RefKind = MCSymbolRefExpr::VK_TLVP;
240 break;
243 // Subtract the pic base.
245 Expr, MCSymbolRefExpr::create(MF.getPICBaseSymbol(), Ctx), Ctx);
246 break;
247 case X86II::MO_SECREL:
249 break;
250 case X86II::MO_TLSGD:
252 break;
253 case X86II::MO_TLSLD:
255 break;
256 case X86II::MO_TLSLDM:
258 break;
261 break;
264 break;
265 case X86II::MO_TPOFF:
267 break;
268 case X86II::MO_DTPOFF:
270 break;
271 case X86II::MO_NTPOFF:
273 break;
276 break;
279 break;
282 break;
283 case X86II::MO_GOT:
284 RefKind = MCSymbolRefExpr::VK_GOT;
285 break;
286 case X86II::MO_GOTOFF:
288 break;
289 case X86II::MO_PLT:
290 RefKind = MCSymbolRefExpr::VK_PLT;
291 break;
292 case X86II::MO_ABS8:
294 break;
297 Expr = MCSymbolRefExpr::create(Sym, Ctx);
298 // Subtract the pic base.
300 Expr, MCSymbolRefExpr::create(MF.getPICBaseSymbol(), Ctx), Ctx);
301 if (MO.isJTI()) {
302 assert(MAI.doesSetDirectiveSuppressReloc());
303 // If .set directive is supported, use it to reduce the number of
304 // relocations the assembler will generate for differences between
305 // local labels. This is only safe when the symbols are in the same
306 // section so we are restricting it to jumptable references.
307 MCSymbol *Label = Ctx.createTempSymbol();
308 AsmPrinter.OutStreamer->emitAssignment(Label, Expr);
309 Expr = MCSymbolRefExpr::create(Label, Ctx);
310 }
311 break;
312 }
313
314 if (!Expr)
315 Expr = MCSymbolRefExpr::create(Sym, RefKind, Ctx);
316
317 if (!MO.isJTI() && !MO.isMBB() && MO.getOffset())
319 Expr, MCConstantExpr::create(MO.getOffset(), Ctx), Ctx);
320 return MCOperand::createExpr(Expr);
321}
322
323/// Simplify FOO $imm, %{al,ax,eax,rax} to FOO $imm, for instruction with
324/// a short fixed-register form.
325static void SimplifyShortImmForm(MCInst &Inst, unsigned Opcode) {
326 unsigned ImmOp = Inst.getNumOperands() - 1;
327 assert(Inst.getOperand(0).isReg() &&
328 (Inst.getOperand(ImmOp).isImm() || Inst.getOperand(ImmOp).isExpr()) &&
329 ((Inst.getNumOperands() == 3 && Inst.getOperand(1).isReg() &&
330 Inst.getOperand(0).getReg() == Inst.getOperand(1).getReg()) ||
331 Inst.getNumOperands() == 2) &&
332 "Unexpected instruction!");
333
334 // Check whether the destination register can be fixed.
335 unsigned Reg = Inst.getOperand(0).getReg();
336 if (Reg != X86::AL && Reg != X86::AX && Reg != X86::EAX && Reg != X86::RAX)
337 return;
338
339 // If so, rewrite the instruction.
340 MCOperand Saved = Inst.getOperand(ImmOp);
341 Inst = MCInst();
342 Inst.setOpcode(Opcode);
343 Inst.addOperand(Saved);
344}
345
346/// If a movsx instruction has a shorter encoding for the used register
347/// simplify the instruction to use it instead.
348static void SimplifyMOVSX(MCInst &Inst) {
349 unsigned NewOpcode = 0;
350 unsigned Op0 = Inst.getOperand(0).getReg(), Op1 = Inst.getOperand(1).getReg();
351 switch (Inst.getOpcode()) {
352 default:
353 llvm_unreachable("Unexpected instruction!");
354 case X86::MOVSX16rr8: // movsbw %al, %ax --> cbtw
355 if (Op0 == X86::AX && Op1 == X86::AL)
356 NewOpcode = X86::CBW;
357 break;
358 case X86::MOVSX32rr16: // movswl %ax, %eax --> cwtl
359 if (Op0 == X86::EAX && Op1 == X86::AX)
360 NewOpcode = X86::CWDE;
361 break;
362 case X86::MOVSX64rr32: // movslq %eax, %rax --> cltq
363 if (Op0 == X86::RAX && Op1 == X86::EAX)
364 NewOpcode = X86::CDQE;
365 break;
366 }
367
368 if (NewOpcode != 0) {
369 Inst = MCInst();
370 Inst.setOpcode(NewOpcode);
371 }
372}
373
374/// Simplify things like MOV32rm to MOV32o32a.
375static void SimplifyShortMoveForm(X86AsmPrinter &Printer, MCInst &Inst,
376 unsigned Opcode) {
377 // Don't make these simplifications in 64-bit mode; other assemblers don't
378 // perform them because they make the code larger.
379 if (Printer.getSubtarget().is64Bit())
380 return;
381
382 bool IsStore = Inst.getOperand(0).isReg() && Inst.getOperand(1).isReg();
383 unsigned AddrBase = IsStore;
384 unsigned RegOp = IsStore ? 0 : 5;
385 unsigned AddrOp = AddrBase + 3;
386 assert(
387 Inst.getNumOperands() == 6 && Inst.getOperand(RegOp).isReg() &&
388 Inst.getOperand(AddrBase + X86::AddrBaseReg).isReg() &&
389 Inst.getOperand(AddrBase + X86::AddrScaleAmt).isImm() &&
390 Inst.getOperand(AddrBase + X86::AddrIndexReg).isReg() &&
391 Inst.getOperand(AddrBase + X86::AddrSegmentReg).isReg() &&
392 (Inst.getOperand(AddrOp).isExpr() || Inst.getOperand(AddrOp).isImm()) &&
393 "Unexpected instruction!");
394
395 // Check whether the destination register can be fixed.
396 unsigned Reg = Inst.getOperand(RegOp).getReg();
397 if (Reg != X86::AL && Reg != X86::AX && Reg != X86::EAX && Reg != X86::RAX)
398 return;
399
400 // Check whether this is an absolute address.
401 // FIXME: We know TLVP symbol refs aren't, but there should be a better way
402 // to do this here.
403 bool Absolute = true;
404 if (Inst.getOperand(AddrOp).isExpr()) {
405 const MCExpr *MCE = Inst.getOperand(AddrOp).getExpr();
406 if (const MCSymbolRefExpr *SRE = dyn_cast<MCSymbolRefExpr>(MCE))
407 if (SRE->getKind() == MCSymbolRefExpr::VK_TLVP)
408 Absolute = false;
409 }
410
411 if (Absolute &&
412 (Inst.getOperand(AddrBase + X86::AddrBaseReg).getReg() != 0 ||
413 Inst.getOperand(AddrBase + X86::AddrScaleAmt).getImm() != 1 ||
414 Inst.getOperand(AddrBase + X86::AddrIndexReg).getReg() != 0))
415 return;
416
417 // If so, rewrite the instruction.
418 MCOperand Saved = Inst.getOperand(AddrOp);
419 MCOperand Seg = Inst.getOperand(AddrBase + X86::AddrSegmentReg);
420 Inst = MCInst();
421 Inst.setOpcode(Opcode);
422 Inst.addOperand(Saved);
423 Inst.addOperand(Seg);
424}
425
426static unsigned getRetOpcode(const X86Subtarget &Subtarget) {
427 return Subtarget.is64Bit() ? X86::RET64 : X86::RET32;
428}
429
430std::optional<MCOperand>
431X86MCInstLower::LowerMachineOperand(const MachineInstr *MI,
432 const MachineOperand &MO) const {
433 switch (MO.getType()) {
434 default:
435 MI->print(errs());
436 llvm_unreachable("unknown operand type");
438 // Ignore all implicit register operands.
439 if (MO.isImplicit())
440 return std::nullopt;
441 return MCOperand::createReg(MO.getReg());
443 return MCOperand::createImm(MO.getImm());
449 return LowerSymbolOperand(MO, MO.getMCSymbol());
455 return LowerSymbolOperand(
458 // Ignore call clobbers.
459 return std::nullopt;
460 }
461}
462
463// Replace TAILJMP opcodes with their equivalent opcodes that have encoding
464// information.
465static unsigned convertTailJumpOpcode(unsigned Opcode) {
466 switch (Opcode) {
467 case X86::TAILJMPr:
468 Opcode = X86::JMP32r;
469 break;
470 case X86::TAILJMPm:
471 Opcode = X86::JMP32m;
472 break;
473 case X86::TAILJMPr64:
474 Opcode = X86::JMP64r;
475 break;
476 case X86::TAILJMPm64:
477 Opcode = X86::JMP64m;
478 break;
479 case X86::TAILJMPr64_REX:
480 Opcode = X86::JMP64r_REX;
481 break;
482 case X86::TAILJMPm64_REX:
483 Opcode = X86::JMP64m_REX;
484 break;
485 case X86::TAILJMPd:
486 case X86::TAILJMPd64:
487 Opcode = X86::JMP_1;
488 break;
489 case X86::TAILJMPd_CC:
490 case X86::TAILJMPd64_CC:
491 Opcode = X86::JCC_1;
492 break;
493 }
494
495 return Opcode;
496}
497
498void X86MCInstLower::Lower(const MachineInstr *MI, MCInst &OutMI) const {
499 OutMI.setOpcode(MI->getOpcode());
500
501 for (const MachineOperand &MO : MI->operands())
502 if (auto MaybeMCOp = LowerMachineOperand(MI, MO))
503 OutMI.addOperand(*MaybeMCOp);
504
505 // Handle a few special cases to eliminate operand modifiers.
506 switch (OutMI.getOpcode()) {
507 case X86::LEA64_32r:
508 case X86::LEA64r:
509 case X86::LEA16r:
510 case X86::LEA32r:
511 // LEA should have a segment register, but it must be empty.
513 "Unexpected # of LEA operands");
514 assert(OutMI.getOperand(1 + X86::AddrSegmentReg).getReg() == 0 &&
515 "LEA has segment specified!");
516 break;
517
518 case X86::MULX32Hrr:
519 case X86::MULX32Hrm:
520 case X86::MULX64Hrr:
521 case X86::MULX64Hrm: {
522 // Turn into regular MULX by duplicating the destination.
523 unsigned NewOpc;
524 switch (OutMI.getOpcode()) {
525 default: llvm_unreachable("Invalid opcode");
526 case X86::MULX32Hrr: NewOpc = X86::MULX32rr; break;
527 case X86::MULX32Hrm: NewOpc = X86::MULX32rm; break;
528 case X86::MULX64Hrr: NewOpc = X86::MULX64rr; break;
529 case X86::MULX64Hrm: NewOpc = X86::MULX64rm; break;
530 }
531 OutMI.setOpcode(NewOpc);
532 // Duplicate the destination.
533 unsigned DestReg = OutMI.getOperand(0).getReg();
534 OutMI.insert(OutMI.begin(), MCOperand::createReg(DestReg));
535 break;
536 }
537
538 // Commute operands to get a smaller encoding by using VEX.R instead of VEX.B
539 // if one of the registers is extended, but other isn't.
540 case X86::VMOVZPQILo2PQIrr:
541 case X86::VMOVAPDrr:
542 case X86::VMOVAPDYrr:
543 case X86::VMOVAPSrr:
544 case X86::VMOVAPSYrr:
545 case X86::VMOVDQArr:
546 case X86::VMOVDQAYrr:
547 case X86::VMOVDQUrr:
548 case X86::VMOVDQUYrr:
549 case X86::VMOVUPDrr:
550 case X86::VMOVUPDYrr:
551 case X86::VMOVUPSrr:
552 case X86::VMOVUPSYrr: {
555 unsigned NewOpc;
556 switch (OutMI.getOpcode()) {
557 default: llvm_unreachable("Invalid opcode");
558 case X86::VMOVZPQILo2PQIrr: NewOpc = X86::VMOVPQI2QIrr; break;
559 case X86::VMOVAPDrr: NewOpc = X86::VMOVAPDrr_REV; break;
560 case X86::VMOVAPDYrr: NewOpc = X86::VMOVAPDYrr_REV; break;
561 case X86::VMOVAPSrr: NewOpc = X86::VMOVAPSrr_REV; break;
562 case X86::VMOVAPSYrr: NewOpc = X86::VMOVAPSYrr_REV; break;
563 case X86::VMOVDQArr: NewOpc = X86::VMOVDQArr_REV; break;
564 case X86::VMOVDQAYrr: NewOpc = X86::VMOVDQAYrr_REV; break;
565 case X86::VMOVDQUrr: NewOpc = X86::VMOVDQUrr_REV; break;
566 case X86::VMOVDQUYrr: NewOpc = X86::VMOVDQUYrr_REV; break;
567 case X86::VMOVUPDrr: NewOpc = X86::VMOVUPDrr_REV; break;
568 case X86::VMOVUPDYrr: NewOpc = X86::VMOVUPDYrr_REV; break;
569 case X86::VMOVUPSrr: NewOpc = X86::VMOVUPSrr_REV; break;
570 case X86::VMOVUPSYrr: NewOpc = X86::VMOVUPSYrr_REV; break;
571 }
572 OutMI.setOpcode(NewOpc);
573 }
574 break;
575 }
576 case X86::VMOVSDrr:
577 case X86::VMOVSSrr: {
580 unsigned NewOpc;
581 switch (OutMI.getOpcode()) {
582 default: llvm_unreachable("Invalid opcode");
583 case X86::VMOVSDrr: NewOpc = X86::VMOVSDrr_REV; break;
584 case X86::VMOVSSrr: NewOpc = X86::VMOVSSrr_REV; break;
585 }
586 OutMI.setOpcode(NewOpc);
587 }
588 break;
589 }
590
591 case X86::VPCMPBZ128rmi: case X86::VPCMPBZ128rmik:
592 case X86::VPCMPBZ128rri: case X86::VPCMPBZ128rrik:
593 case X86::VPCMPBZ256rmi: case X86::VPCMPBZ256rmik:
594 case X86::VPCMPBZ256rri: case X86::VPCMPBZ256rrik:
595 case X86::VPCMPBZrmi: case X86::VPCMPBZrmik:
596 case X86::VPCMPBZrri: case X86::VPCMPBZrrik:
597 case X86::VPCMPDZ128rmi: case X86::VPCMPDZ128rmik:
598 case X86::VPCMPDZ128rmib: case X86::VPCMPDZ128rmibk:
599 case X86::VPCMPDZ128rri: case X86::VPCMPDZ128rrik:
600 case X86::VPCMPDZ256rmi: case X86::VPCMPDZ256rmik:
601 case X86::VPCMPDZ256rmib: case X86::VPCMPDZ256rmibk:
602 case X86::VPCMPDZ256rri: case X86::VPCMPDZ256rrik:
603 case X86::VPCMPDZrmi: case X86::VPCMPDZrmik:
604 case X86::VPCMPDZrmib: case X86::VPCMPDZrmibk:
605 case X86::VPCMPDZrri: case X86::VPCMPDZrrik:
606 case X86::VPCMPQZ128rmi: case X86::VPCMPQZ128rmik:
607 case X86::VPCMPQZ128rmib: case X86::VPCMPQZ128rmibk:
608 case X86::VPCMPQZ128rri: case X86::VPCMPQZ128rrik:
609 case X86::VPCMPQZ256rmi: case X86::VPCMPQZ256rmik:
610 case X86::VPCMPQZ256rmib: case X86::VPCMPQZ256rmibk:
611 case X86::VPCMPQZ256rri: case X86::VPCMPQZ256rrik:
612 case X86::VPCMPQZrmi: case X86::VPCMPQZrmik:
613 case X86::VPCMPQZrmib: case X86::VPCMPQZrmibk:
614 case X86::VPCMPQZrri: case X86::VPCMPQZrrik:
615 case X86::VPCMPWZ128rmi: case X86::VPCMPWZ128rmik:
616 case X86::VPCMPWZ128rri: case X86::VPCMPWZ128rrik:
617 case X86::VPCMPWZ256rmi: case X86::VPCMPWZ256rmik:
618 case X86::VPCMPWZ256rri: case X86::VPCMPWZ256rrik:
619 case X86::VPCMPWZrmi: case X86::VPCMPWZrmik:
620 case X86::VPCMPWZrri: case X86::VPCMPWZrrik: {
621 // Turn immediate 0 into the VPCMPEQ instruction.
622 if (OutMI.getOperand(OutMI.getNumOperands() - 1).getImm() == 0) {
623 unsigned NewOpc;
624 switch (OutMI.getOpcode()) {
625 default: llvm_unreachable("Invalid opcode");
626 case X86::VPCMPBZ128rmi: NewOpc = X86::VPCMPEQBZ128rm; break;
627 case X86::VPCMPBZ128rmik: NewOpc = X86::VPCMPEQBZ128rmk; break;
628 case X86::VPCMPBZ128rri: NewOpc = X86::VPCMPEQBZ128rr; break;
629 case X86::VPCMPBZ128rrik: NewOpc = X86::VPCMPEQBZ128rrk; break;
630 case X86::VPCMPBZ256rmi: NewOpc = X86::VPCMPEQBZ256rm; break;
631 case X86::VPCMPBZ256rmik: NewOpc = X86::VPCMPEQBZ256rmk; break;
632 case X86::VPCMPBZ256rri: NewOpc = X86::VPCMPEQBZ256rr; break;
633 case X86::VPCMPBZ256rrik: NewOpc = X86::VPCMPEQBZ256rrk; break;
634 case X86::VPCMPBZrmi: NewOpc = X86::VPCMPEQBZrm; break;
635 case X86::VPCMPBZrmik: NewOpc = X86::VPCMPEQBZrmk; break;
636 case X86::VPCMPBZrri: NewOpc = X86::VPCMPEQBZrr; break;
637 case X86::VPCMPBZrrik: NewOpc = X86::VPCMPEQBZrrk; break;
638 case X86::VPCMPDZ128rmi: NewOpc = X86::VPCMPEQDZ128rm; break;
639 case X86::VPCMPDZ128rmib: NewOpc = X86::VPCMPEQDZ128rmb; break;
640 case X86::VPCMPDZ128rmibk: NewOpc = X86::VPCMPEQDZ128rmbk; break;
641 case X86::VPCMPDZ128rmik: NewOpc = X86::VPCMPEQDZ128rmk; break;
642 case X86::VPCMPDZ128rri: NewOpc = X86::VPCMPEQDZ128rr; break;
643 case X86::VPCMPDZ128rrik: NewOpc = X86::VPCMPEQDZ128rrk; break;
644 case X86::VPCMPDZ256rmi: NewOpc = X86::VPCMPEQDZ256rm; break;
645 case X86::VPCMPDZ256rmib: NewOpc = X86::VPCMPEQDZ256rmb; break;
646 case X86::VPCMPDZ256rmibk: NewOpc = X86::VPCMPEQDZ256rmbk; break;
647 case X86::VPCMPDZ256rmik: NewOpc = X86::VPCMPEQDZ256rmk; break;
648 case X86::VPCMPDZ256rri: NewOpc = X86::VPCMPEQDZ256rr; break;
649 case X86::VPCMPDZ256rrik: NewOpc = X86::VPCMPEQDZ256rrk; break;
650 case X86::VPCMPDZrmi: NewOpc = X86::VPCMPEQDZrm; break;
651 case X86::VPCMPDZrmib: NewOpc = X86::VPCMPEQDZrmb; break;
652 case X86::VPCMPDZrmibk: NewOpc = X86::VPCMPEQDZrmbk; break;
653 case X86::VPCMPDZrmik: NewOpc = X86::VPCMPEQDZrmk; break;
654 case X86::VPCMPDZrri: NewOpc = X86::VPCMPEQDZrr; break;
655 case X86::VPCMPDZrrik: NewOpc = X86::VPCMPEQDZrrk; break;
656 case X86::VPCMPQZ128rmi: NewOpc = X86::VPCMPEQQZ128rm; break;
657 case X86::VPCMPQZ128rmib: NewOpc = X86::VPCMPEQQZ128rmb; break;
658 case X86::VPCMPQZ128rmibk: NewOpc = X86::VPCMPEQQZ128rmbk; break;
659 case X86::VPCMPQZ128rmik: NewOpc = X86::VPCMPEQQZ128rmk; break;
660 case X86::VPCMPQZ128rri: NewOpc = X86::VPCMPEQQZ128rr; break;
661 case X86::VPCMPQZ128rrik: NewOpc = X86::VPCMPEQQZ128rrk; break;
662 case X86::VPCMPQZ256rmi: NewOpc = X86::VPCMPEQQZ256rm; break;
663 case X86::VPCMPQZ256rmib: NewOpc = X86::VPCMPEQQZ256rmb; break;
664 case X86::VPCMPQZ256rmibk: NewOpc = X86::VPCMPEQQZ256rmbk; break;
665 case X86::VPCMPQZ256rmik: NewOpc = X86::VPCMPEQQZ256rmk; break;
666 case X86::VPCMPQZ256rri: NewOpc = X86::VPCMPEQQZ256rr; break;
667 case X86::VPCMPQZ256rrik: NewOpc = X86::VPCMPEQQZ256rrk; break;
668 case X86::VPCMPQZrmi: NewOpc = X86::VPCMPEQQZrm; break;
669 case X86::VPCMPQZrmib: NewOpc = X86::VPCMPEQQZrmb; break;
670 case X86::VPCMPQZrmibk: NewOpc = X86::VPCMPEQQZrmbk; break;
671 case X86::VPCMPQZrmik: NewOpc = X86::VPCMPEQQZrmk; break;
672 case X86::VPCMPQZrri: NewOpc = X86::VPCMPEQQZrr; break;
673 case X86::VPCMPQZrrik: NewOpc = X86::VPCMPEQQZrrk; break;
674 case X86::VPCMPWZ128rmi: NewOpc = X86::VPCMPEQWZ128rm; break;
675 case X86::VPCMPWZ128rmik: NewOpc = X86::VPCMPEQWZ128rmk; break;
676 case X86::VPCMPWZ128rri: NewOpc = X86::VPCMPEQWZ128rr; break;
677 case X86::VPCMPWZ128rrik: NewOpc = X86::VPCMPEQWZ128rrk; break;
678 case X86::VPCMPWZ256rmi: NewOpc = X86::VPCMPEQWZ256rm; break;
679 case X86::VPCMPWZ256rmik: NewOpc = X86::VPCMPEQWZ256rmk; break;
680 case X86::VPCMPWZ256rri: NewOpc = X86::VPCMPEQWZ256rr; break;
681 case X86::VPCMPWZ256rrik: NewOpc = X86::VPCMPEQWZ256rrk; break;
682 case X86::VPCMPWZrmi: NewOpc = X86::VPCMPEQWZrm; break;
683 case X86::VPCMPWZrmik: NewOpc = X86::VPCMPEQWZrmk; break;
684 case X86::VPCMPWZrri: NewOpc = X86::VPCMPEQWZrr; break;
685 case X86::VPCMPWZrrik: NewOpc = X86::VPCMPEQWZrrk; break;
686 }
687
688 OutMI.setOpcode(NewOpc);
689 OutMI.erase(&OutMI.getOperand(OutMI.getNumOperands() - 1));
690 break;
691 }
692
693 // Turn immediate 6 into the VPCMPGT instruction.
694 if (OutMI.getOperand(OutMI.getNumOperands() - 1).getImm() == 6) {
695 unsigned NewOpc;
696 switch (OutMI.getOpcode()) {
697 default: llvm_unreachable("Invalid opcode");
698 case X86::VPCMPBZ128rmi: NewOpc = X86::VPCMPGTBZ128rm; break;
699 case X86::VPCMPBZ128rmik: NewOpc = X86::VPCMPGTBZ128rmk; break;
700 case X86::VPCMPBZ128rri: NewOpc = X86::VPCMPGTBZ128rr; break;
701 case X86::VPCMPBZ128rrik: NewOpc = X86::VPCMPGTBZ128rrk; break;
702 case X86::VPCMPBZ256rmi: NewOpc = X86::VPCMPGTBZ256rm; break;
703 case X86::VPCMPBZ256rmik: NewOpc = X86::VPCMPGTBZ256rmk; break;
704 case X86::VPCMPBZ256rri: NewOpc = X86::VPCMPGTBZ256rr; break;
705 case X86::VPCMPBZ256rrik: NewOpc = X86::VPCMPGTBZ256rrk; break;
706 case X86::VPCMPBZrmi: NewOpc = X86::VPCMPGTBZrm; break;
707 case X86::VPCMPBZrmik: NewOpc = X86::VPCMPGTBZrmk; break;
708 case X86::VPCMPBZrri: NewOpc = X86::VPCMPGTBZrr; break;
709 case X86::VPCMPBZrrik: NewOpc = X86::VPCMPGTBZrrk; break;
710 case X86::VPCMPDZ128rmi: NewOpc = X86::VPCMPGTDZ128rm; break;
711 case X86::VPCMPDZ128rmib: NewOpc = X86::VPCMPGTDZ128rmb; break;
712 case X86::VPCMPDZ128rmibk: NewOpc = X86::VPCMPGTDZ128rmbk; break;
713 case X86::VPCMPDZ128rmik: NewOpc = X86::VPCMPGTDZ128rmk; break;
714 case X86::VPCMPDZ128rri: NewOpc = X86::VPCMPGTDZ128rr; break;
715 case X86::VPCMPDZ128rrik: NewOpc = X86::VPCMPGTDZ128rrk; break;
716 case X86::VPCMPDZ256rmi: NewOpc = X86::VPCMPGTDZ256rm; break;
717 case X86::VPCMPDZ256rmib: NewOpc = X86::VPCMPGTDZ256rmb; break;
718 case X86::VPCMPDZ256rmibk: NewOpc = X86::VPCMPGTDZ256rmbk; break;
719 case X86::VPCMPDZ256rmik: NewOpc = X86::VPCMPGTDZ256rmk; break;
720 case X86::VPCMPDZ256rri: NewOpc = X86::VPCMPGTDZ256rr; break;
721 case X86::VPCMPDZ256rrik: NewOpc = X86::VPCMPGTDZ256rrk; break;
722 case X86::VPCMPDZrmi: NewOpc = X86::VPCMPGTDZrm; break;
723 case X86::VPCMPDZrmib: NewOpc = X86::VPCMPGTDZrmb; break;
724 case X86::VPCMPDZrmibk: NewOpc = X86::VPCMPGTDZrmbk; break;
725 case X86::VPCMPDZrmik: NewOpc = X86::VPCMPGTDZrmk; break;
726 case X86::VPCMPDZrri: NewOpc = X86::VPCMPGTDZrr; break;
727 case X86::VPCMPDZrrik: NewOpc = X86::VPCMPGTDZrrk; break;
728 case X86::VPCMPQZ128rmi: NewOpc = X86::VPCMPGTQZ128rm; break;
729 case X86::VPCMPQZ128rmib: NewOpc = X86::VPCMPGTQZ128rmb; break;
730 case X86::VPCMPQZ128rmibk: NewOpc = X86::VPCMPGTQZ128rmbk; break;
731 case X86::VPCMPQZ128rmik: NewOpc = X86::VPCMPGTQZ128rmk; break;
732 case X86::VPCMPQZ128rri: NewOpc = X86::VPCMPGTQZ128rr; break;
733 case X86::VPCMPQZ128rrik: NewOpc = X86::VPCMPGTQZ128rrk; break;
734 case X86::VPCMPQZ256rmi: NewOpc = X86::VPCMPGTQZ256rm; break;
735 case X86::VPCMPQZ256rmib: NewOpc = X86::VPCMPGTQZ256rmb; break;
736 case X86::VPCMPQZ256rmibk: NewOpc = X86::VPCMPGTQZ256rmbk; break;
737 case X86::VPCMPQZ256rmik: NewOpc = X86::VPCMPGTQZ256rmk; break;
738 case X86::VPCMPQZ256rri: NewOpc = X86::VPCMPGTQZ256rr; break;
739 case X86::VPCMPQZ256rrik: NewOpc = X86::VPCMPGTQZ256rrk; break;
740 case X86::VPCMPQZrmi: NewOpc = X86::VPCMPGTQZrm; break;
741 case X86::VPCMPQZrmib: NewOpc = X86::VPCMPGTQZrmb; break;
742 case X86::VPCMPQZrmibk: NewOpc = X86::VPCMPGTQZrmbk; break;
743 case X86::VPCMPQZrmik: NewOpc = X86::VPCMPGTQZrmk; break;
744 case X86::VPCMPQZrri: NewOpc = X86::VPCMPGTQZrr; break;
745 case X86::VPCMPQZrrik: NewOpc = X86::VPCMPGTQZrrk; break;
746 case X86::VPCMPWZ128rmi: NewOpc = X86::VPCMPGTWZ128rm; break;
747 case X86::VPCMPWZ128rmik: NewOpc = X86::VPCMPGTWZ128rmk; break;
748 case X86::VPCMPWZ128rri: NewOpc = X86::VPCMPGTWZ128rr; break;
749 case X86::VPCMPWZ128rrik: NewOpc = X86::VPCMPGTWZ128rrk; break;
750 case X86::VPCMPWZ256rmi: NewOpc = X86::VPCMPGTWZ256rm; break;
751 case X86::VPCMPWZ256rmik: NewOpc = X86::VPCMPGTWZ256rmk; break;
752 case X86::VPCMPWZ256rri: NewOpc = X86::VPCMPGTWZ256rr; break;
753 case X86::VPCMPWZ256rrik: NewOpc = X86::VPCMPGTWZ256rrk; break;
754 case X86::VPCMPWZrmi: NewOpc = X86::VPCMPGTWZrm; break;
755 case X86::VPCMPWZrmik: NewOpc = X86::VPCMPGTWZrmk; break;
756 case X86::VPCMPWZrri: NewOpc = X86::VPCMPGTWZrr; break;
757 case X86::VPCMPWZrrik: NewOpc = X86::VPCMPGTWZrrk; break;
758 }
759
760 OutMI.setOpcode(NewOpc);
761 OutMI.erase(&OutMI.getOperand(OutMI.getNumOperands() - 1));
762 break;
763 }
764
765 break;
766 }
767
768 // CALL64r, CALL64pcrel32 - These instructions used to have
769 // register inputs modeled as normal uses instead of implicit uses. As such,
770 // they we used to truncate off all but the first operand (the callee). This
771 // issue seems to have been fixed at some point. This assert verifies that.
772 case X86::CALL64r:
773 case X86::CALL64pcrel32:
774 assert(OutMI.getNumOperands() == 1 && "Unexpected number of operands!");
775 break;
776
777 case X86::EH_RETURN:
778 case X86::EH_RETURN64: {
779 OutMI = MCInst();
780 OutMI.setOpcode(getRetOpcode(AsmPrinter.getSubtarget()));
781 break;
782 }
783
784 case X86::CLEANUPRET: {
785 // Replace CLEANUPRET with the appropriate RET.
786 OutMI = MCInst();
787 OutMI.setOpcode(getRetOpcode(AsmPrinter.getSubtarget()));
788 break;
789 }
790
791 case X86::CATCHRET: {
792 // Replace CATCHRET with the appropriate RET.
793 const X86Subtarget &Subtarget = AsmPrinter.getSubtarget();
794 unsigned ReturnReg = Subtarget.is64Bit() ? X86::RAX : X86::EAX;
795 OutMI = MCInst();
796 OutMI.setOpcode(getRetOpcode(Subtarget));
797 OutMI.addOperand(MCOperand::createReg(ReturnReg));
798 break;
799 }
800
801 // TAILJMPd, TAILJMPd64, TailJMPd_cc - Lower to the correct jump
802 // instruction.
803 case X86::TAILJMPr:
804 case X86::TAILJMPr64:
805 case X86::TAILJMPr64_REX:
806 case X86::TAILJMPd:
807 case X86::TAILJMPd64:
808 assert(OutMI.getNumOperands() == 1 && "Unexpected number of operands!");
810 break;
811
812 case X86::TAILJMPd_CC:
813 case X86::TAILJMPd64_CC:
814 assert(OutMI.getNumOperands() == 2 && "Unexpected number of operands!");
816 break;
817
818 case X86::TAILJMPm:
819 case X86::TAILJMPm64:
820 case X86::TAILJMPm64_REX:
822 "Unexpected number of operands!");
824 break;
825
826 case X86::DEC16r:
827 case X86::DEC32r:
828 case X86::INC16r:
829 case X86::INC32r:
830 // If we aren't in 64-bit mode we can use the 1-byte inc/dec instructions.
831 if (!AsmPrinter.getSubtarget().is64Bit()) {
832 unsigned Opcode;
833 switch (OutMI.getOpcode()) {
834 default: llvm_unreachable("Invalid opcode");
835 case X86::DEC16r: Opcode = X86::DEC16r_alt; break;
836 case X86::DEC32r: Opcode = X86::DEC32r_alt; break;
837 case X86::INC16r: Opcode = X86::INC16r_alt; break;
838 case X86::INC32r: Opcode = X86::INC32r_alt; break;
839 }
840 OutMI.setOpcode(Opcode);
841 }
842 break;
843
844 // We don't currently select the correct instruction form for instructions
845 // which have a short %eax, etc. form. Handle this by custom lowering, for
846 // now.
847 //
848 // Note, we are currently not handling the following instructions:
849 // MOV64ao8, MOV64o8a
850 // XCHG16ar, XCHG32ar, XCHG64ar
851 case X86::MOV8mr_NOREX:
852 case X86::MOV8mr:
853 case X86::MOV8rm_NOREX:
854 case X86::MOV8rm:
855 case X86::MOV16mr:
856 case X86::MOV16rm:
857 case X86::MOV32mr:
858 case X86::MOV32rm: {
859 unsigned NewOpc;
860 switch (OutMI.getOpcode()) {
861 default: llvm_unreachable("Invalid opcode");
862 case X86::MOV8mr_NOREX:
863 case X86::MOV8mr: NewOpc = X86::MOV8o32a; break;
864 case X86::MOV8rm_NOREX:
865 case X86::MOV8rm: NewOpc = X86::MOV8ao32; break;
866 case X86::MOV16mr: NewOpc = X86::MOV16o32a; break;
867 case X86::MOV16rm: NewOpc = X86::MOV16ao32; break;
868 case X86::MOV32mr: NewOpc = X86::MOV32o32a; break;
869 case X86::MOV32rm: NewOpc = X86::MOV32ao32; break;
870 }
871 SimplifyShortMoveForm(AsmPrinter, OutMI, NewOpc);
872 break;
873 }
874
875 case X86::ADC8ri: case X86::ADC16ri: case X86::ADC32ri: case X86::ADC64ri32:
876 case X86::ADD8ri: case X86::ADD16ri: case X86::ADD32ri: case X86::ADD64ri32:
877 case X86::AND8ri: case X86::AND16ri: case X86::AND32ri: case X86::AND64ri32:
878 case X86::CMP8ri: case X86::CMP16ri: case X86::CMP32ri: case X86::CMP64ri32:
879 case X86::OR8ri: case X86::OR16ri: case X86::OR32ri: case X86::OR64ri32:
880 case X86::SBB8ri: case X86::SBB16ri: case X86::SBB32ri: case X86::SBB64ri32:
881 case X86::SUB8ri: case X86::SUB16ri: case X86::SUB32ri: case X86::SUB64ri32:
882 case X86::TEST8ri:case X86::TEST16ri:case X86::TEST32ri:case X86::TEST64ri32:
883 case X86::XOR8ri: case X86::XOR16ri: case X86::XOR32ri: case X86::XOR64ri32: {
884 unsigned NewOpc;
885 switch (OutMI.getOpcode()) {
886 default: llvm_unreachable("Invalid opcode");
887 case X86::ADC8ri: NewOpc = X86::ADC8i8; break;
888 case X86::ADC16ri: NewOpc = X86::ADC16i16; break;
889 case X86::ADC32ri: NewOpc = X86::ADC32i32; break;
890 case X86::ADC64ri32: NewOpc = X86::ADC64i32; break;
891 case X86::ADD8ri: NewOpc = X86::ADD8i8; break;
892 case X86::ADD16ri: NewOpc = X86::ADD16i16; break;
893 case X86::ADD32ri: NewOpc = X86::ADD32i32; break;
894 case X86::ADD64ri32: NewOpc = X86::ADD64i32; break;
895 case X86::AND8ri: NewOpc = X86::AND8i8; break;
896 case X86::AND16ri: NewOpc = X86::AND16i16; break;
897 case X86::AND32ri: NewOpc = X86::AND32i32; break;
898 case X86::AND64ri32: NewOpc = X86::AND64i32; break;
899 case X86::CMP8ri: NewOpc = X86::CMP8i8; break;
900 case X86::CMP16ri: NewOpc = X86::CMP16i16; break;
901 case X86::CMP32ri: NewOpc = X86::CMP32i32; break;
902 case X86::CMP64ri32: NewOpc = X86::CMP64i32; break;
903 case X86::OR8ri: NewOpc = X86::OR8i8; break;
904 case X86::OR16ri: NewOpc = X86::OR16i16; break;
905 case X86::OR32ri: NewOpc = X86::OR32i32; break;
906 case X86::OR64ri32: NewOpc = X86::OR64i32; break;
907 case X86::SBB8ri: NewOpc = X86::SBB8i8; break;
908 case X86::SBB16ri: NewOpc = X86::SBB16i16; break;
909 case X86::SBB32ri: NewOpc = X86::SBB32i32; break;
910 case X86::SBB64ri32: NewOpc = X86::SBB64i32; break;
911 case X86::SUB8ri: NewOpc = X86::SUB8i8; break;
912 case X86::SUB16ri: NewOpc = X86::SUB16i16; break;
913 case X86::SUB32ri: NewOpc = X86::SUB32i32; break;
914 case X86::SUB64ri32: NewOpc = X86::SUB64i32; break;
915 case X86::TEST8ri: NewOpc = X86::TEST8i8; break;
916 case X86::TEST16ri: NewOpc = X86::TEST16i16; break;
917 case X86::TEST32ri: NewOpc = X86::TEST32i32; break;
918 case X86::TEST64ri32: NewOpc = X86::TEST64i32; break;
919 case X86::XOR8ri: NewOpc = X86::XOR8i8; break;
920 case X86::XOR16ri: NewOpc = X86::XOR16i16; break;
921 case X86::XOR32ri: NewOpc = X86::XOR32i32; break;
922 case X86::XOR64ri32: NewOpc = X86::XOR64i32; break;
923 }
924 SimplifyShortImmForm(OutMI, NewOpc);
925 break;
926 }
927
928 // Try to shrink some forms of movsx.
929 case X86::MOVSX16rr8:
930 case X86::MOVSX32rr16:
931 case X86::MOVSX64rr32:
932 SimplifyMOVSX(OutMI);
933 break;
934
935 case X86::VCMPPDrri:
936 case X86::VCMPPDYrri:
937 case X86::VCMPPSrri:
938 case X86::VCMPPSYrri:
939 case X86::VCMPSDrr:
940 case X86::VCMPSSrr: {
941 // Swap the operands if it will enable a 2 byte VEX encoding.
942 // FIXME: Change the immediate to improve opportunities?
945 unsigned Imm = MI->getOperand(3).getImm() & 0x7;
946 switch (Imm) {
947 default: break;
948 case 0x00: // EQUAL
949 case 0x03: // UNORDERED
950 case 0x04: // NOT EQUAL
951 case 0x07: // ORDERED
952 std::swap(OutMI.getOperand(1), OutMI.getOperand(2));
953 break;
954 }
955 }
956 break;
957 }
958
959 case X86::VMOVHLPSrr:
960 case X86::VUNPCKHPDrr:
961 // These are not truly commutable so hide them from the default case.
962 break;
963
964 case X86::MASKMOVDQU:
965 case X86::VMASKMOVDQU:
966 if (AsmPrinter.getSubtarget().is64Bit())
968 break;
969
970 default: {
971 // If the instruction is a commutable arithmetic instruction we might be
972 // able to commute the operands to get a 2 byte VEX prefix.
973 uint64_t TSFlags = MI->getDesc().TSFlags;
974 if (MI->getDesc().isCommutable() &&
979 OutMI.getNumOperands() == 3) {
982 std::swap(OutMI.getOperand(1), OutMI.getOperand(2));
983 }
984 // Add an REP prefix to BSF instructions so that new processors can
985 // recognize as TZCNT, which has better performance than BSF.
986 if (X86::isBSF(OutMI.getOpcode()) && !MF.getFunction().hasOptSize()) {
987 // BSF and TZCNT have different interpretations on ZF bit. So make sure
988 // it won't be used later.
989 const MachineOperand *FlagDef = MI->findRegisterDefOperand(X86::EFLAGS);
990 if (FlagDef && FlagDef->isDead())
992 }
993 break;
994 }
995 }
996}
997
998void X86AsmPrinter::LowerTlsAddr(X86MCInstLower &MCInstLowering,
999 const MachineInstr &MI) {
1000 NoAutoPaddingScope NoPadScope(*OutStreamer);
1001 bool Is64Bits = MI.getOpcode() != X86::TLS_addr32 &&
1002 MI.getOpcode() != X86::TLS_base_addr32;
1003 bool Is64BitsLP64 = MI.getOpcode() == X86::TLS_addr64 ||
1004 MI.getOpcode() == X86::TLS_base_addr64;
1005 MCContext &Ctx = OutStreamer->getContext();
1006
1008 switch (MI.getOpcode()) {
1009 case X86::TLS_addr32:
1010 case X86::TLS_addr64:
1011 case X86::TLS_addrX32:
1013 break;
1014 case X86::TLS_base_addr32:
1016 break;
1017 case X86::TLS_base_addr64:
1018 case X86::TLS_base_addrX32:
1020 break;
1021 default:
1022 llvm_unreachable("unexpected opcode");
1023 }
1024
1026 MCInstLowering.GetSymbolFromOperand(MI.getOperand(3)), SRVK, Ctx);
1027
1028 // As of binutils 2.32, ld has a bogus TLS relaxation error when the GD/LD
1029 // code sequence using R_X86_64_GOTPCREL (instead of R_X86_64_GOTPCRELX) is
1030 // attempted to be relaxed to IE/LE (binutils PR24784). Work around the bug by
1031 // only using GOT when GOTPCRELX is enabled.
1032 // TODO Delete the workaround when GOTPCRELX becomes commonplace.
1033 bool UseGot = MMI->getModule()->getRtLibUseGOT() &&
1035
1036 if (Is64Bits) {
1037 bool NeedsPadding = SRVK == MCSymbolRefExpr::VK_TLSGD;
1038 if (NeedsPadding && Is64BitsLP64)
1039 EmitAndCountInstruction(MCInstBuilder(X86::DATA16_PREFIX));
1040 EmitAndCountInstruction(MCInstBuilder(X86::LEA64r)
1041 .addReg(X86::RDI)
1042 .addReg(X86::RIP)
1043 .addImm(1)
1044 .addReg(0)
1045 .addExpr(Sym)
1046 .addReg(0));
1047 const MCSymbol *TlsGetAddr = Ctx.getOrCreateSymbol("__tls_get_addr");
1048 if (NeedsPadding) {
1049 if (!UseGot)
1050 EmitAndCountInstruction(MCInstBuilder(X86::DATA16_PREFIX));
1051 EmitAndCountInstruction(MCInstBuilder(X86::DATA16_PREFIX));
1052 EmitAndCountInstruction(MCInstBuilder(X86::REX64_PREFIX));
1053 }
1054 if (UseGot) {
1055 const MCExpr *Expr = MCSymbolRefExpr::create(
1056 TlsGetAddr, MCSymbolRefExpr::VK_GOTPCREL, Ctx);
1057 EmitAndCountInstruction(MCInstBuilder(X86::CALL64m)
1058 .addReg(X86::RIP)
1059 .addImm(1)
1060 .addReg(0)
1061 .addExpr(Expr)
1062 .addReg(0));
1063 } else {
1064 EmitAndCountInstruction(
1065 MCInstBuilder(X86::CALL64pcrel32)
1066 .addExpr(MCSymbolRefExpr::create(TlsGetAddr,
1068 }
1069 } else {
1070 if (SRVK == MCSymbolRefExpr::VK_TLSGD && !UseGot) {
1071 EmitAndCountInstruction(MCInstBuilder(X86::LEA32r)
1072 .addReg(X86::EAX)
1073 .addReg(0)
1074 .addImm(1)
1075 .addReg(X86::EBX)
1076 .addExpr(Sym)
1077 .addReg(0));
1078 } else {
1079 EmitAndCountInstruction(MCInstBuilder(X86::LEA32r)
1080 .addReg(X86::EAX)
1081 .addReg(X86::EBX)
1082 .addImm(1)
1083 .addReg(0)
1084 .addExpr(Sym)
1085 .addReg(0));
1086 }
1087
1088 const MCSymbol *TlsGetAddr = Ctx.getOrCreateSymbol("___tls_get_addr");
1089 if (UseGot) {
1090 const MCExpr *Expr =
1092 EmitAndCountInstruction(MCInstBuilder(X86::CALL32m)
1093 .addReg(X86::EBX)
1094 .addImm(1)
1095 .addReg(0)
1096 .addExpr(Expr)
1097 .addReg(0));
1098 } else {
1099 EmitAndCountInstruction(
1100 MCInstBuilder(X86::CALLpcrel32)
1101 .addExpr(MCSymbolRefExpr::create(TlsGetAddr,
1103 }
1104 }
1105}
1106
1107/// Emit the largest nop instruction smaller than or equal to \p NumBytes
1108/// bytes. Return the size of nop emitted.
1109static unsigned emitNop(MCStreamer &OS, unsigned NumBytes,
1110 const X86Subtarget *Subtarget) {
1111 // Determine the longest nop which can be efficiently decoded for the given
1112 // target cpu. 15-bytes is the longest single NOP instruction, but some
1113 // platforms can't decode the longest forms efficiently.
1114 unsigned MaxNopLength = 1;
1115 if (Subtarget->is64Bit()) {
1116 // FIXME: We can use NOOPL on 32-bit targets with FeatureNOPL, but the
1117 // IndexReg/BaseReg below need to be updated.
1118 if (Subtarget->hasFeature(X86::TuningFast7ByteNOP))
1119 MaxNopLength = 7;
1120 else if (Subtarget->hasFeature(X86::TuningFast15ByteNOP))
1121 MaxNopLength = 15;
1122 else if (Subtarget->hasFeature(X86::TuningFast11ByteNOP))
1123 MaxNopLength = 11;
1124 else
1125 MaxNopLength = 10;
1126 } if (Subtarget->is32Bit())
1127 MaxNopLength = 2;
1128
1129 // Cap a single nop emission at the profitable value for the target
1130 NumBytes = std::min(NumBytes, MaxNopLength);
1131
1132 unsigned NopSize;
1133 unsigned Opc, BaseReg, ScaleVal, IndexReg, Displacement, SegmentReg;
1134 IndexReg = Displacement = SegmentReg = 0;
1135 BaseReg = X86::RAX;
1136 ScaleVal = 1;
1137 switch (NumBytes) {
1138 case 0:
1139 llvm_unreachable("Zero nops?");
1140 break;
1141 case 1:
1142 NopSize = 1;
1143 Opc = X86::NOOP;
1144 break;
1145 case 2:
1146 NopSize = 2;
1147 Opc = X86::XCHG16ar;
1148 break;
1149 case 3:
1150 NopSize = 3;
1151 Opc = X86::NOOPL;
1152 break;
1153 case 4:
1154 NopSize = 4;
1155 Opc = X86::NOOPL;
1156 Displacement = 8;
1157 break;
1158 case 5:
1159 NopSize = 5;
1160 Opc = X86::NOOPL;
1161 Displacement = 8;
1162 IndexReg = X86::RAX;
1163 break;
1164 case 6:
1165 NopSize = 6;
1166 Opc = X86::NOOPW;
1167 Displacement = 8;
1168 IndexReg = X86::RAX;
1169 break;
1170 case 7:
1171 NopSize = 7;
1172 Opc = X86::NOOPL;
1173 Displacement = 512;
1174 break;
1175 case 8:
1176 NopSize = 8;
1177 Opc = X86::NOOPL;
1178 Displacement = 512;
1179 IndexReg = X86::RAX;
1180 break;
1181 case 9:
1182 NopSize = 9;
1183 Opc = X86::NOOPW;
1184 Displacement = 512;
1185 IndexReg = X86::RAX;
1186 break;
1187 default:
1188 NopSize = 10;
1189 Opc = X86::NOOPW;
1190 Displacement = 512;
1191 IndexReg = X86::RAX;
1192 SegmentReg = X86::CS;
1193 break;
1194 }
1195
1196 unsigned NumPrefixes = std::min(NumBytes - NopSize, 5U);
1197 NopSize += NumPrefixes;
1198 for (unsigned i = 0; i != NumPrefixes; ++i)
1199 OS.emitBytes("\x66");
1200
1201 switch (Opc) {
1202 default: llvm_unreachable("Unexpected opcode");
1203 case X86::NOOP:
1204 OS.emitInstruction(MCInstBuilder(Opc), *Subtarget);
1205 break;
1206 case X86::XCHG16ar:
1207 OS.emitInstruction(MCInstBuilder(Opc).addReg(X86::AX).addReg(X86::AX),
1208 *Subtarget);
1209 break;
1210 case X86::NOOPL:
1211 case X86::NOOPW:
1212 OS.emitInstruction(MCInstBuilder(Opc)
1213 .addReg(BaseReg)
1214 .addImm(ScaleVal)
1215 .addReg(IndexReg)
1216 .addImm(Displacement)
1217 .addReg(SegmentReg),
1218 *Subtarget);
1219 break;
1220 }
1221 assert(NopSize <= NumBytes && "We overemitted?");
1222 return NopSize;
1223}
1224
1225/// Emit the optimal amount of multi-byte nops on X86.
1226static void emitX86Nops(MCStreamer &OS, unsigned NumBytes,
1227 const X86Subtarget *Subtarget) {
1228 unsigned NopsToEmit = NumBytes;
1229 (void)NopsToEmit;
1230 while (NumBytes) {
1231 NumBytes -= emitNop(OS, NumBytes, Subtarget);
1232 assert(NopsToEmit >= NumBytes && "Emitted more than I asked for!");
1233 }
1234}
1235
1236void X86AsmPrinter::LowerSTATEPOINT(const MachineInstr &MI,
1237 X86MCInstLower &MCIL) {
1238 assert(Subtarget->is64Bit() && "Statepoint currently only supports X86-64");
1239
1240 NoAutoPaddingScope NoPadScope(*OutStreamer);
1241
1242 StatepointOpers SOpers(&MI);
1243 if (unsigned PatchBytes = SOpers.getNumPatchBytes()) {
1244 emitX86Nops(*OutStreamer, PatchBytes, Subtarget);
1245 } else {
1246 // Lower call target and choose correct opcode
1247 const MachineOperand &CallTarget = SOpers.getCallTarget();
1248 MCOperand CallTargetMCOp;
1249 unsigned CallOpcode;
1250 switch (CallTarget.getType()) {
1253 CallTargetMCOp = MCIL.LowerSymbolOperand(
1254 CallTarget, MCIL.GetSymbolFromOperand(CallTarget));
1255 CallOpcode = X86::CALL64pcrel32;
1256 // Currently, we only support relative addressing with statepoints.
1257 // Otherwise, we'll need a scratch register to hold the target
1258 // address. You'll fail asserts during load & relocation if this
1259 // symbol is to far away. (TODO: support non-relative addressing)
1260 break;
1262 CallTargetMCOp = MCOperand::createImm(CallTarget.getImm());
1263 CallOpcode = X86::CALL64pcrel32;
1264 // Currently, we only support relative addressing with statepoints.
1265 // Otherwise, we'll need a scratch register to hold the target
1266 // immediate. You'll fail asserts during load & relocation if this
1267 // address is to far away. (TODO: support non-relative addressing)
1268 break;
1270 // FIXME: Add retpoline support and remove this.
1271 if (Subtarget->useIndirectThunkCalls())
1272 report_fatal_error("Lowering register statepoints with thunks not "
1273 "yet implemented.");
1274 CallTargetMCOp = MCOperand::createReg(CallTarget.getReg());
1275 CallOpcode = X86::CALL64r;
1276 break;
1277 default:
1278 llvm_unreachable("Unsupported operand type in statepoint call target");
1279 break;
1280 }
1281
1282 // Emit call
1284 CallInst.setOpcode(CallOpcode);
1285 CallInst.addOperand(CallTargetMCOp);
1286 OutStreamer->emitInstruction(CallInst, getSubtargetInfo());
1287 }
1288
1289 // Record our statepoint node in the same section used by STACKMAP
1290 // and PATCHPOINT
1291 auto &Ctx = OutStreamer->getContext();
1292 MCSymbol *MILabel = Ctx.createTempSymbol();
1293 OutStreamer->emitLabel(MILabel);
1294 SM.recordStatepoint(*MILabel, MI);
1295}
1296
1297void X86AsmPrinter::LowerFAULTING_OP(const MachineInstr &FaultingMI,
1298 X86MCInstLower &MCIL) {
1299 // FAULTING_LOAD_OP <def>, <faltinf type>, <MBB handler>,
1300 // <opcode>, <operands>
1301
1302 NoAutoPaddingScope NoPadScope(*OutStreamer);
1303
1304 Register DefRegister = FaultingMI.getOperand(0).getReg();
1306 static_cast<FaultMaps::FaultKind>(FaultingMI.getOperand(1).getImm());
1307 MCSymbol *HandlerLabel = FaultingMI.getOperand(2).getMBB()->getSymbol();
1308 unsigned Opcode = FaultingMI.getOperand(3).getImm();
1309 unsigned OperandsBeginIdx = 4;
1310
1311 auto &Ctx = OutStreamer->getContext();
1312 MCSymbol *FaultingLabel = Ctx.createTempSymbol();
1313 OutStreamer->emitLabel(FaultingLabel);
1314
1315 assert(FK < FaultMaps::FaultKindMax && "Invalid Faulting Kind!");
1316 FM.recordFaultingOp(FK, FaultingLabel, HandlerLabel);
1317
1318 MCInst MI;
1319 MI.setOpcode(Opcode);
1320
1321 if (DefRegister != X86::NoRegister)
1322 MI.addOperand(MCOperand::createReg(DefRegister));
1323
1324 for (const MachineOperand &MO :
1325 llvm::drop_begin(FaultingMI.operands(), OperandsBeginIdx))
1326 if (auto MaybeOperand = MCIL.LowerMachineOperand(&FaultingMI, MO))
1327 MI.addOperand(*MaybeOperand);
1328
1329 OutStreamer->AddComment("on-fault: " + HandlerLabel->getName());
1330 OutStreamer->emitInstruction(MI, getSubtargetInfo());
1331}
1332
1333void X86AsmPrinter::LowerFENTRY_CALL(const MachineInstr &MI,
1334 X86MCInstLower &MCIL) {
1335 bool Is64Bits = Subtarget->is64Bit();
1336 MCContext &Ctx = OutStreamer->getContext();
1337 MCSymbol *fentry = Ctx.getOrCreateSymbol("__fentry__");
1338 const MCSymbolRefExpr *Op =
1340
1341 EmitAndCountInstruction(
1342 MCInstBuilder(Is64Bits ? X86::CALL64pcrel32 : X86::CALLpcrel32)
1343 .addExpr(Op));
1344}
1345
1346void X86AsmPrinter::LowerKCFI_CHECK(const MachineInstr &MI) {
1347 assert(std::next(MI.getIterator())->isCall() &&
1348 "KCFI_CHECK not followed by a call instruction");
1349
1350 // Adjust the offset for patchable-function-prefix. X86InstrInfo::getNop()
1351 // returns a 1-byte X86::NOOP, which means the offset is the same in
1352 // bytes. This assumes that patchable-function-prefix is the same for all
1353 // functions.
1354 const MachineFunction &MF = *MI.getMF();
1355 int64_t PrefixNops = 0;
1356 (void)MF.getFunction()
1357 .getFnAttribute("patchable-function-prefix")
1359 .getAsInteger(10, PrefixNops);
1360
1361 // KCFI allows indirect calls to any location that's preceded by a valid
1362 // type identifier. To avoid encoding the full constant into an instruction,
1363 // and thus emitting potential call target gadgets at each indirect call
1364 // site, load a negated constant to a register and compare that to the
1365 // expected value at the call target.
1366 const Register AddrReg = MI.getOperand(0).getReg();
1367 const uint32_t Type = MI.getOperand(1).getImm();
1368 // The check is immediately before the call. If the call target is in R10,
1369 // we can clobber R11 for the check instead.
1370 unsigned TempReg = AddrReg == X86::R10 ? X86::R11D : X86::R10D;
1371 EmitAndCountInstruction(
1372 MCInstBuilder(X86::MOV32ri).addReg(TempReg).addImm(-MaskKCFIType(Type)));
1373 EmitAndCountInstruction(MCInstBuilder(X86::ADD32rm)
1374 .addReg(X86::NoRegister)
1375 .addReg(TempReg)
1376 .addReg(AddrReg)
1377 .addImm(1)
1378 .addReg(X86::NoRegister)
1379 .addImm(-(PrefixNops + 4))
1380 .addReg(X86::NoRegister));
1381
1383 EmitAndCountInstruction(
1384 MCInstBuilder(X86::JCC_1)
1386 .addImm(X86::COND_E));
1387
1389 OutStreamer->emitLabel(Trap);
1390 EmitAndCountInstruction(MCInstBuilder(X86::TRAP));
1392 OutStreamer->emitLabel(Pass);
1393}
1394
1395void X86AsmPrinter::LowerASAN_CHECK_MEMACCESS(const MachineInstr &MI) {
1396 // FIXME: Make this work on non-ELF.
1398 report_fatal_error("llvm.asan.check.memaccess only supported on ELF");
1399 return;
1400 }
1401
1402 const auto &Reg = MI.getOperand(0).getReg();
1403 ASanAccessInfo AccessInfo(MI.getOperand(1).getImm());
1404
1405 uint64_t ShadowBase;
1406 int MappingScale;
1407 bool OrShadowOffset;
1409 AccessInfo.CompileKernel, &ShadowBase,
1410 &MappingScale, &OrShadowOffset);
1411
1412 StringRef Name = AccessInfo.IsWrite ? "store" : "load";
1413 StringRef Op = OrShadowOffset ? "or" : "add";
1414 std::string SymName = ("__asan_check_" + Name + "_" + Op + "_" +
1415 Twine(1ULL << AccessInfo.AccessSizeIndex) + "_" +
1416 TM.getMCRegisterInfo()->getName(Reg.asMCReg()))
1417 .str();
1418 if (OrShadowOffset)
1420 "OrShadowOffset is not supported with optimized callbacks");
1421
1422 EmitAndCountInstruction(
1423 MCInstBuilder(X86::CALL64pcrel32)
1424 .addExpr(MCSymbolRefExpr::create(
1426}
1427
1428void X86AsmPrinter::LowerPATCHABLE_OP(const MachineInstr &MI,
1429 X86MCInstLower &MCIL) {
1430 // PATCHABLE_OP minsize, opcode, operands
1431
1432 NoAutoPaddingScope NoPadScope(*OutStreamer);
1433
1434 unsigned MinSize = MI.getOperand(0).getImm();
1435 unsigned Opcode = MI.getOperand(1).getImm();
1436 // Opcode PATCHABLE_OP is a special case: there is no instruction to wrap,
1437 // simply emit a nop of size MinSize.
1438 bool EmptyInst = (Opcode == TargetOpcode::PATCHABLE_OP);
1439
1440 MCInst MCI;
1441 MCI.setOpcode(Opcode);
1442 for (auto &MO : drop_begin(MI.operands(), 2))
1443 if (auto MaybeOperand = MCIL.LowerMachineOperand(&MI, MO))
1444 MCI.addOperand(*MaybeOperand);
1445
1447 if (!EmptyInst) {
1449 raw_svector_ostream VecOS(Code);
1450 CodeEmitter->encodeInstruction(MCI, VecOS, Fixups, getSubtargetInfo());
1451 }
1452
1453 if (Code.size() < MinSize) {
1454 if (MinSize == 2 && Subtarget->is32Bit() &&
1455 Subtarget->isTargetWindowsMSVC() &&
1456 (Subtarget->getCPU().empty() || Subtarget->getCPU() == "pentium3")) {
1457 // For compatibility reasons, when targetting MSVC, is is important to
1458 // generate a 'legacy' NOP in the form of a 8B FF MOV EDI, EDI. Some tools
1459 // rely specifically on this pattern to be able to patch a function.
1460 // This is only for 32-bit targets, when using /arch:IA32 or /arch:SSE.
1461 OutStreamer->emitInstruction(
1462 MCInstBuilder(X86::MOV32rr_REV).addReg(X86::EDI).addReg(X86::EDI),
1463 *Subtarget);
1464 } else if (MinSize == 2 && Opcode == X86::PUSH64r) {
1465 // This is an optimization that lets us get away without emitting a nop in
1466 // many cases.
1467 //
1468 // NB! In some cases the encoding for PUSH64r (e.g. PUSH64r %r9) takes two
1469 // bytes too, so the check on MinSize is important.
1470 MCI.setOpcode(X86::PUSH64rmr);
1471 } else {
1472 unsigned NopSize = emitNop(*OutStreamer, MinSize, Subtarget);
1473 assert(NopSize == MinSize && "Could not implement MinSize!");
1474 (void)NopSize;
1475 }
1476 }
1477 if (!EmptyInst)
1478 OutStreamer->emitInstruction(MCI, getSubtargetInfo());
1479}
1480
1481// Lower a stackmap of the form:
1482// <id>, <shadowBytes>, ...
1483void X86AsmPrinter::LowerSTACKMAP(const MachineInstr &MI) {
1484 SMShadowTracker.emitShadowPadding(*OutStreamer, getSubtargetInfo());
1485
1486 auto &Ctx = OutStreamer->getContext();
1487 MCSymbol *MILabel = Ctx.createTempSymbol();
1488 OutStreamer->emitLabel(MILabel);
1489
1490 SM.recordStackMap(*MILabel, MI);
1491 unsigned NumShadowBytes = MI.getOperand(1).getImm();
1492 SMShadowTracker.reset(NumShadowBytes);
1493}
1494
1495// Lower a patchpoint of the form:
1496// [<def>], <id>, <numBytes>, <target>, <numArgs>, <cc>, ...
1497void X86AsmPrinter::LowerPATCHPOINT(const MachineInstr &MI,
1498 X86MCInstLower &MCIL) {
1499 assert(Subtarget->is64Bit() && "Patchpoint currently only supports X86-64");
1500
1501 SMShadowTracker.emitShadowPadding(*OutStreamer, getSubtargetInfo());
1502
1503 NoAutoPaddingScope NoPadScope(*OutStreamer);
1504
1505 auto &Ctx = OutStreamer->getContext();
1506 MCSymbol *MILabel = Ctx.createTempSymbol();
1507 OutStreamer->emitLabel(MILabel);
1508 SM.recordPatchPoint(*MILabel, MI);
1509
1510 PatchPointOpers opers(&MI);
1511 unsigned ScratchIdx = opers.getNextScratchIdx();
1512 unsigned EncodedBytes = 0;
1513 const MachineOperand &CalleeMO = opers.getCallTarget();
1514
1515 // Check for null target. If target is non-null (i.e. is non-zero or is
1516 // symbolic) then emit a call.
1517 if (!(CalleeMO.isImm() && !CalleeMO.getImm())) {
1518 MCOperand CalleeMCOp;
1519 switch (CalleeMO.getType()) {
1520 default:
1521 /// FIXME: Add a verifier check for bad callee types.
1522 llvm_unreachable("Unrecognized callee operand type.");
1524 if (CalleeMO.getImm())
1525 CalleeMCOp = MCOperand::createImm(CalleeMO.getImm());
1526 break;
1529 CalleeMCOp = MCIL.LowerSymbolOperand(CalleeMO,
1530 MCIL.GetSymbolFromOperand(CalleeMO));
1531 break;
1532 }
1533
1534 // Emit MOV to materialize the target address and the CALL to target.
1535 // This is encoded with 12-13 bytes, depending on which register is used.
1536 Register ScratchReg = MI.getOperand(ScratchIdx).getReg();
1537 if (X86II::isX86_64ExtendedReg(ScratchReg))
1538 EncodedBytes = 13;
1539 else
1540 EncodedBytes = 12;
1541
1542 EmitAndCountInstruction(
1543 MCInstBuilder(X86::MOV64ri).addReg(ScratchReg).addOperand(CalleeMCOp));
1544 // FIXME: Add retpoline support and remove this.
1545 if (Subtarget->useIndirectThunkCalls())
1547 "Lowering patchpoint with thunks not yet implemented.");
1548 EmitAndCountInstruction(MCInstBuilder(X86::CALL64r).addReg(ScratchReg));
1549 }
1550
1551 // Emit padding.
1552 unsigned NumBytes = opers.getNumPatchBytes();
1553 assert(NumBytes >= EncodedBytes &&
1554 "Patchpoint can't request size less than the length of a call.");
1555
1556 emitX86Nops(*OutStreamer, NumBytes - EncodedBytes, Subtarget);
1557}
1558
1559void X86AsmPrinter::LowerPATCHABLE_EVENT_CALL(const MachineInstr &MI,
1560 X86MCInstLower &MCIL) {
1561 assert(Subtarget->is64Bit() && "XRay custom events only supports X86-64");
1562
1563 NoAutoPaddingScope NoPadScope(*OutStreamer);
1564
1565 // We want to emit the following pattern, which follows the x86 calling
1566 // convention to prepare for the trampoline call to be patched in.
1567 //
1568 // .p2align 1, ...
1569 // .Lxray_event_sled_N:
1570 // jmp +N // jump across the instrumentation sled
1571 // ... // set up arguments in register
1572 // callq __xray_CustomEvent@plt // force dependency to symbol
1573 // ...
1574 // <jump here>
1575 //
1576 // After patching, it would look something like:
1577 //
1578 // nopw (2-byte nop)
1579 // ...
1580 // callq __xrayCustomEvent // already lowered
1581 // ...
1582 //
1583 // ---
1584 // First we emit the label and the jump.
1585 auto CurSled = OutContext.createTempSymbol("xray_event_sled_", true);
1586 OutStreamer->AddComment("# XRay Custom Event Log");
1587 OutStreamer->emitCodeAlignment(Align(2), &getSubtargetInfo());
1588 OutStreamer->emitLabel(CurSled);
1589
1590 // Use a two-byte `jmp`. This version of JMP takes an 8-bit relative offset as
1591 // an operand (computed as an offset from the jmp instruction).
1592 // FIXME: Find another less hacky way do force the relative jump.
1593 OutStreamer->emitBinaryData("\xeb\x0f");
1594
1595 // The default C calling convention will place two arguments into %rcx and
1596 // %rdx -- so we only work with those.
1597 const Register DestRegs[] = {X86::RDI, X86::RSI};
1598 bool UsedMask[] = {false, false};
1599 // Filled out in loop.
1600 Register SrcRegs[] = {0, 0};
1601
1602 // Then we put the operands in the %rdi and %rsi registers. We spill the
1603 // values in the register before we clobber them, and mark them as used in
1604 // UsedMask. In case the arguments are already in the correct register, we use
1605 // emit nops appropriately sized to keep the sled the same size in every
1606 // situation.
1607 for (unsigned I = 0; I < MI.getNumOperands(); ++I)
1608 if (auto Op = MCIL.LowerMachineOperand(&MI, MI.getOperand(I))) {
1609 assert(Op->isReg() && "Only support arguments in registers");
1610 SrcRegs[I] = getX86SubSuperRegister(Op->getReg(), 64);
1611 assert(SrcRegs[I].isValid() && "Invalid operand");
1612 if (SrcRegs[I] != DestRegs[I]) {
1613 UsedMask[I] = true;
1614 EmitAndCountInstruction(
1615 MCInstBuilder(X86::PUSH64r).addReg(DestRegs[I]));
1616 } else {
1617 emitX86Nops(*OutStreamer, 4, Subtarget);
1618 }
1619 }
1620
1621 // Now that the register values are stashed, mov arguments into place.
1622 // FIXME: This doesn't work if one of the later SrcRegs is equal to an
1623 // earlier DestReg. We will have already overwritten over the register before
1624 // we can copy from it.
1625 for (unsigned I = 0; I < MI.getNumOperands(); ++I)
1626 if (SrcRegs[I] != DestRegs[I])
1627 EmitAndCountInstruction(
1628 MCInstBuilder(X86::MOV64rr).addReg(DestRegs[I]).addReg(SrcRegs[I]));
1629
1630 // We emit a hard dependency on the __xray_CustomEvent symbol, which is the
1631 // name of the trampoline to be implemented by the XRay runtime.
1632 auto TSym = OutContext.getOrCreateSymbol("__xray_CustomEvent");
1636
1637 // Emit the call instruction.
1638 EmitAndCountInstruction(MCInstBuilder(X86::CALL64pcrel32)
1639 .addOperand(MCIL.LowerSymbolOperand(TOp, TSym)));
1640
1641 // Restore caller-saved and used registers.
1642 for (unsigned I = sizeof UsedMask; I-- > 0;)
1643 if (UsedMask[I])
1644 EmitAndCountInstruction(MCInstBuilder(X86::POP64r).addReg(DestRegs[I]));
1645 else
1646 emitX86Nops(*OutStreamer, 1, Subtarget);
1647
1648 OutStreamer->AddComment("xray custom event end.");
1649
1650 // Record the sled version. Version 0 of this sled was spelled differently, so
1651 // we let the runtime handle the different offsets we're using. Version 2
1652 // changed the absolute address to a PC-relative address.
1653 recordSled(CurSled, MI, SledKind::CUSTOM_EVENT, 2);
1654}
1655
1656void X86AsmPrinter::LowerPATCHABLE_TYPED_EVENT_CALL(const MachineInstr &MI,
1657 X86MCInstLower &MCIL) {
1658 assert(Subtarget->is64Bit() && "XRay typed events only supports X86-64");
1659
1660 NoAutoPaddingScope NoPadScope(*OutStreamer);
1661
1662 // We want to emit the following pattern, which follows the x86 calling
1663 // convention to prepare for the trampoline call to be patched in.
1664 //
1665 // .p2align 1, ...
1666 // .Lxray_event_sled_N:
1667 // jmp +N // jump across the instrumentation sled
1668 // ... // set up arguments in register
1669 // callq __xray_TypedEvent@plt // force dependency to symbol
1670 // ...
1671 // <jump here>
1672 //
1673 // After patching, it would look something like:
1674 //
1675 // nopw (2-byte nop)
1676 // ...
1677 // callq __xrayTypedEvent // already lowered
1678 // ...
1679 //
1680 // ---
1681 // First we emit the label and the jump.
1682 auto CurSled = OutContext.createTempSymbol("xray_typed_event_sled_", true);
1683 OutStreamer->AddComment("# XRay Typed Event Log");
1684 OutStreamer->emitCodeAlignment(Align(2), &getSubtargetInfo());
1685 OutStreamer->emitLabel(CurSled);
1686
1687 // Use a two-byte `jmp`. This version of JMP takes an 8-bit relative offset as
1688 // an operand (computed as an offset from the jmp instruction).
1689 // FIXME: Find another less hacky way do force the relative jump.
1690 OutStreamer->emitBinaryData("\xeb\x14");
1691
1692 // An x86-64 convention may place three arguments into %rcx, %rdx, and R8,
1693 // so we'll work with those. Or we may be called via SystemV, in which case
1694 // we don't have to do any translation.
1695 const Register DestRegs[] = {X86::RDI, X86::RSI, X86::RDX};
1696 bool UsedMask[] = {false, false, false};
1697
1698 // Will fill out src regs in the loop.
1699 Register SrcRegs[] = {0, 0, 0};
1700
1701 // Then we put the operands in the SystemV registers. We spill the values in
1702 // the registers before we clobber them, and mark them as used in UsedMask.
1703 // In case the arguments are already in the correct register, we emit nops
1704 // appropriately sized to keep the sled the same size in every situation.
1705 for (unsigned I = 0; I < MI.getNumOperands(); ++I)
1706 if (auto Op = MCIL.LowerMachineOperand(&MI, MI.getOperand(I))) {
1707 // TODO: Is register only support adequate?
1708 assert(Op->isReg() && "Only supports arguments in registers");
1709 SrcRegs[I] = getX86SubSuperRegister(Op->getReg(), 64);
1710 assert(SrcRegs[I].isValid() && "Invalid operand");
1711 if (SrcRegs[I] != DestRegs[I]) {
1712 UsedMask[I] = true;
1713 EmitAndCountInstruction(
1714 MCInstBuilder(X86::PUSH64r).addReg(DestRegs[I]));
1715 } else {
1716 emitX86Nops(*OutStreamer, 4, Subtarget);
1717 }
1718 }
1719
1720 // In the above loop we only stash all of the destination registers or emit
1721 // nops if the arguments are already in the right place. Doing the actually
1722 // moving is postponed until after all the registers are stashed so nothing
1723 // is clobbers. We've already added nops to account for the size of mov and
1724 // push if the register is in the right place, so we only have to worry about
1725 // emitting movs.
1726 // FIXME: This doesn't work if one of the later SrcRegs is equal to an
1727 // earlier DestReg. We will have already overwritten over the register before
1728 // we can copy from it.
1729 for (unsigned I = 0; I < MI.getNumOperands(); ++I)
1730 if (UsedMask[I])
1731 EmitAndCountInstruction(
1732 MCInstBuilder(X86::MOV64rr).addReg(DestRegs[I]).addReg(SrcRegs[I]));
1733
1734 // We emit a hard dependency on the __xray_TypedEvent symbol, which is the
1735 // name of the trampoline to be implemented by the XRay runtime.
1736 auto TSym = OutContext.getOrCreateSymbol("__xray_TypedEvent");
1740
1741 // Emit the call instruction.
1742 EmitAndCountInstruction(MCInstBuilder(X86::CALL64pcrel32)
1743 .addOperand(MCIL.LowerSymbolOperand(TOp, TSym)));
1744
1745 // Restore caller-saved and used registers.
1746 for (unsigned I = sizeof UsedMask; I-- > 0;)
1747 if (UsedMask[I])
1748 EmitAndCountInstruction(MCInstBuilder(X86::POP64r).addReg(DestRegs[I]));
1749 else
1750 emitX86Nops(*OutStreamer, 1, Subtarget);
1751
1752 OutStreamer->AddComment("xray typed event end.");
1753
1754 // Record the sled version.
1755 recordSled(CurSled, MI, SledKind::TYPED_EVENT, 2);
1756}
1757
1758void X86AsmPrinter::LowerPATCHABLE_FUNCTION_ENTER(const MachineInstr &MI,
1759 X86MCInstLower &MCIL) {
1760
1761 NoAutoPaddingScope NoPadScope(*OutStreamer);
1762
1763 const Function &F = MF->getFunction();
1764 if (F.hasFnAttribute("patchable-function-entry")) {
1765 unsigned Num;
1766 if (F.getFnAttribute("patchable-function-entry")
1767 .getValueAsString()
1768 .getAsInteger(10, Num))
1769 return;
1770 emitX86Nops(*OutStreamer, Num, Subtarget);
1771 return;
1772 }
1773 // We want to emit the following pattern:
1774 //
1775 // .p2align 1, ...
1776 // .Lxray_sled_N:
1777 // jmp .tmpN
1778 // # 9 bytes worth of noops
1779 //
1780 // We need the 9 bytes because at runtime, we'd be patching over the full 11
1781 // bytes with the following pattern:
1782 //
1783 // mov %r10, <function id, 32-bit> // 6 bytes
1784 // call <relative offset, 32-bits> // 5 bytes
1785 //
1786 auto CurSled = OutContext.createTempSymbol("xray_sled_", true);
1787 OutStreamer->emitCodeAlignment(Align(2), &getSubtargetInfo());
1788 OutStreamer->emitLabel(CurSled);
1789
1790 // Use a two-byte `jmp`. This version of JMP takes an 8-bit relative offset as
1791 // an operand (computed as an offset from the jmp instruction).
1792 // FIXME: Find another less hacky way do force the relative jump.
1793 OutStreamer->emitBytes("\xeb\x09");
1794 emitX86Nops(*OutStreamer, 9, Subtarget);
1796}
1797
1798void X86AsmPrinter::LowerPATCHABLE_RET(const MachineInstr &MI,
1799 X86MCInstLower &MCIL) {
1800 NoAutoPaddingScope NoPadScope(*OutStreamer);
1801
1802 // Since PATCHABLE_RET takes the opcode of the return statement as an
1803 // argument, we use that to emit the correct form of the RET that we want.
1804 // i.e. when we see this:
1805 //
1806 // PATCHABLE_RET X86::RET ...
1807 //
1808 // We should emit the RET followed by sleds.
1809 //
1810 // .p2align 1, ...
1811 // .Lxray_sled_N:
1812 // ret # or equivalent instruction
1813 // # 10 bytes worth of noops
1814 //
1815 // This just makes sure that the alignment for the next instruction is 2.
1816 auto CurSled = OutContext.createTempSymbol("xray_sled_", true);
1817 OutStreamer->emitCodeAlignment(Align(2), &getSubtargetInfo());
1818 OutStreamer->emitLabel(CurSled);
1819 unsigned OpCode = MI.getOperand(0).getImm();
1820 MCInst Ret;
1821 Ret.setOpcode(OpCode);
1822 for (auto &MO : drop_begin(MI.operands()))
1823 if (auto MaybeOperand = MCIL.LowerMachineOperand(&MI, MO))
1824 Ret.addOperand(*MaybeOperand);
1825 OutStreamer->emitInstruction(Ret, getSubtargetInfo());
1826 emitX86Nops(*OutStreamer, 10, Subtarget);
1828}
1829
1830void X86AsmPrinter::LowerPATCHABLE_TAIL_CALL(const MachineInstr &MI,
1831 X86MCInstLower &MCIL) {
1832 NoAutoPaddingScope NoPadScope(*OutStreamer);
1833
1834 // Like PATCHABLE_RET, we have the actual instruction in the operands to this
1835 // instruction so we lower that particular instruction and its operands.
1836 // Unlike PATCHABLE_RET though, we put the sled before the JMP, much like how
1837 // we do it for PATCHABLE_FUNCTION_ENTER. The sled should be very similar to
1838 // the PATCHABLE_FUNCTION_ENTER case, followed by the lowering of the actual
1839 // tail call much like how we have it in PATCHABLE_RET.
1840 auto CurSled = OutContext.createTempSymbol("xray_sled_", true);
1841 OutStreamer->emitCodeAlignment(Align(2), &getSubtargetInfo());
1842 OutStreamer->emitLabel(CurSled);
1844
1845 // Use a two-byte `jmp`. This version of JMP takes an 8-bit relative offset as
1846 // an operand (computed as an offset from the jmp instruction).
1847 // FIXME: Find another less hacky way do force the relative jump.
1848 OutStreamer->emitBytes("\xeb\x09");
1849 emitX86Nops(*OutStreamer, 9, Subtarget);
1850 OutStreamer->emitLabel(Target);
1851 recordSled(CurSled, MI, SledKind::TAIL_CALL, 2);
1852
1853 unsigned OpCode = MI.getOperand(0).getImm();
1854 OpCode = convertTailJumpOpcode(OpCode);
1855 MCInst TC;
1856 TC.setOpcode(OpCode);
1857
1858 // Before emitting the instruction, add a comment to indicate that this is
1859 // indeed a tail call.
1860 OutStreamer->AddComment("TAILCALL");
1861 for (auto &MO : drop_begin(MI.operands()))
1862 if (auto MaybeOperand = MCIL.LowerMachineOperand(&MI, MO))
1863 TC.addOperand(*MaybeOperand);
1864 OutStreamer->emitInstruction(TC, getSubtargetInfo());
1865}
1866
1867// Returns instruction preceding MBBI in MachineFunction.
1868// If MBBI is the first instruction of the first basic block, returns null.
1871 const MachineBasicBlock *MBB = MBBI->getParent();
1872 while (MBBI == MBB->begin()) {
1873 if (MBB == &MBB->getParent()->front())
1875 MBB = MBB->getPrevNode();
1876 MBBI = MBB->end();
1877 }
1878 --MBBI;
1879 return MBBI;
1880}
1881
1883 const MachineOperand &Op) {
1884 if (!Op.isCPI() || Op.getOffset() != 0)
1885 return nullptr;
1886
1888 MI.getParent()->getParent()->getConstantPool()->getConstants();
1889 const MachineConstantPoolEntry &ConstantEntry = Constants[Op.getIndex()];
1890
1891 // Bail if this is a machine constant pool entry, we won't be able to dig out
1892 // anything useful.
1893 if (ConstantEntry.isMachineConstantPoolEntry())
1894 return nullptr;
1895
1896 return ConstantEntry.Val.ConstVal;
1897}
1898
1899static std::string getShuffleComment(const MachineInstr *MI, unsigned SrcOp1Idx,
1900 unsigned SrcOp2Idx, ArrayRef<int> Mask) {
1901 std::string Comment;
1902
1903 // Compute the name for a register. This is really goofy because we have
1904 // multiple instruction printers that could (in theory) use different
1905 // names. Fortunately most people use the ATT style (outside of Windows)
1906 // and they actually agree on register naming here. Ultimately, this is
1907 // a comment, and so its OK if it isn't perfect.
1908 auto GetRegisterName = [](MCRegister Reg) -> StringRef {
1910 };
1911
1912 const MachineOperand &DstOp = MI->getOperand(0);
1913 const MachineOperand &SrcOp1 = MI->getOperand(SrcOp1Idx);
1914 const MachineOperand &SrcOp2 = MI->getOperand(SrcOp2Idx);
1915
1916 StringRef DstName = DstOp.isReg() ? GetRegisterName(DstOp.getReg()) : "mem";
1917 StringRef Src1Name =
1918 SrcOp1.isReg() ? GetRegisterName(SrcOp1.getReg()) : "mem";
1919 StringRef Src2Name =
1920 SrcOp2.isReg() ? GetRegisterName(SrcOp2.getReg()) : "mem";
1921
1922 // One source operand, fix the mask to print all elements in one span.
1923 SmallVector<int, 8> ShuffleMask(Mask);
1924 if (Src1Name == Src2Name)
1925 for (int i = 0, e = ShuffleMask.size(); i != e; ++i)
1926 if (ShuffleMask[i] >= e)
1927 ShuffleMask[i] -= e;
1928
1929 raw_string_ostream CS(Comment);
1930 CS << DstName;
1931
1932 // Handle AVX512 MASK/MASXZ write mask comments.
1933 // MASK: zmmX {%kY}
1934 // MASKZ: zmmX {%kY} {z}
1935 if (SrcOp1Idx > 1) {
1936 assert((SrcOp1Idx == 2 || SrcOp1Idx == 3) && "Unexpected writemask");
1937
1938 const MachineOperand &WriteMaskOp = MI->getOperand(SrcOp1Idx - 1);
1939 if (WriteMaskOp.isReg()) {
1940 CS << " {%" << GetRegisterName(WriteMaskOp.getReg()) << "}";
1941
1942 if (SrcOp1Idx == 2) {
1943 CS << " {z}";
1944 }
1945 }
1946 }
1947
1948 CS << " = ";
1949
1950 for (int i = 0, e = ShuffleMask.size(); i != e; ++i) {
1951 if (i != 0)
1952 CS << ",";
1953 if (ShuffleMask[i] == SM_SentinelZero) {
1954 CS << "zero";
1955 continue;
1956 }
1957
1958 // Otherwise, it must come from src1 or src2. Print the span of elements
1959 // that comes from this src.
1960 bool isSrc1 = ShuffleMask[i] < (int)e;
1961 CS << (isSrc1 ? Src1Name : Src2Name) << '[';
1962
1963 bool IsFirst = true;
1964 while (i != e && ShuffleMask[i] != SM_SentinelZero &&
1965 (ShuffleMask[i] < (int)e) == isSrc1) {
1966 if (!IsFirst)
1967 CS << ',';
1968 else
1969 IsFirst = false;
1970 if (ShuffleMask[i] == SM_SentinelUndef)
1971 CS << "u";
1972 else
1973 CS << ShuffleMask[i] % (int)e;
1974 ++i;
1975 }
1976 CS << ']';
1977 --i; // For loop increments element #.
1978 }
1979 CS.flush();
1980
1981 return Comment;
1982}
1983
1984static void printConstant(const APInt &Val, raw_ostream &CS) {
1985 if (Val.getBitWidth() <= 64) {
1986 CS << Val.getZExtValue();
1987 } else {
1988 // print multi-word constant as (w0,w1)
1989 CS << "(";
1990 for (int i = 0, N = Val.getNumWords(); i < N; ++i) {
1991 if (i > 0)
1992 CS << ",";
1993 CS << Val.getRawData()[i];
1994 }
1995 CS << ")";
1996 }
1997}
1998
1999static void printConstant(const APFloat &Flt, raw_ostream &CS) {
2000 SmallString<32> Str;
2001 // Force scientific notation to distinquish from integers.
2002 Flt.toString(Str, 0, 0);
2003 CS << Str;
2004}
2005
2006static void printConstant(const Constant *COp, raw_ostream &CS) {
2007 if (isa<UndefValue>(COp)) {
2008 CS << "u";
2009 } else if (auto *CI = dyn_cast<ConstantInt>(COp)) {
2010 printConstant(CI->getValue(), CS);
2011 } else if (auto *CF = dyn_cast<ConstantFP>(COp)) {
2012 printConstant(CF->getValueAPF(), CS);
2013 } else {
2014 CS << "?";
2015 }
2016}
2017
2018void X86AsmPrinter::EmitSEHInstruction(const MachineInstr *MI) {
2019 assert(MF->hasWinCFI() && "SEH_ instruction in function without WinCFI?");
2020 assert(getSubtarget().isOSWindows() && "SEH_ instruction Windows only");
2021
2022 // Use the .cv_fpo directives if we're emitting CodeView on 32-bit x86.
2023 if (EmitFPOData) {
2024 X86TargetStreamer *XTS =
2025 static_cast<X86TargetStreamer *>(OutStreamer->getTargetStreamer());
2026 switch (MI->getOpcode()) {
2027 case X86::SEH_PushReg:
2028 XTS->emitFPOPushReg(MI->getOperand(0).getImm());
2029 break;
2030 case X86::SEH_StackAlloc:
2031 XTS->emitFPOStackAlloc(MI->getOperand(0).getImm());
2032 break;
2033 case X86::SEH_StackAlign:
2034 XTS->emitFPOStackAlign(MI->getOperand(0).getImm());
2035 break;
2036 case X86::SEH_SetFrame:
2037 assert(MI->getOperand(1).getImm() == 0 &&
2038 ".cv_fpo_setframe takes no offset");
2039 XTS->emitFPOSetFrame(MI->getOperand(0).getImm());
2040 break;
2041 case X86::SEH_EndPrologue:
2042 XTS->emitFPOEndPrologue();
2043 break;
2044 case X86::SEH_SaveReg:
2045 case X86::SEH_SaveXMM:
2046 case X86::SEH_PushFrame:
2047 llvm_unreachable("SEH_ directive incompatible with FPO");
2048 break;
2049 default:
2050 llvm_unreachable("expected SEH_ instruction");
2051 }
2052 return;
2053 }
2054
2055 // Otherwise, use the .seh_ directives for all other Windows platforms.
2056 switch (MI->getOpcode()) {
2057 case X86::SEH_PushReg:
2058 OutStreamer->emitWinCFIPushReg(MI->getOperand(0).getImm());
2059 break;
2060
2061 case X86::SEH_SaveReg:
2062 OutStreamer->emitWinCFISaveReg(MI->getOperand(0).getImm(),
2063 MI->getOperand(1).getImm());
2064 break;
2065
2066 case X86::SEH_SaveXMM:
2067 OutStreamer->emitWinCFISaveXMM(MI->getOperand(0).getImm(),
2068 MI->getOperand(1).getImm());
2069 break;
2070
2071 case X86::SEH_StackAlloc:
2072 OutStreamer->emitWinCFIAllocStack(MI->getOperand(0).getImm());
2073 break;
2074
2075 case X86::SEH_SetFrame:
2076 OutStreamer->emitWinCFISetFrame(MI->getOperand(0).getImm(),
2077 MI->getOperand(1).getImm());
2078 break;
2079
2080 case X86::SEH_PushFrame:
2081 OutStreamer->emitWinCFIPushFrame(MI->getOperand(0).getImm());
2082 break;
2083
2084 case X86::SEH_EndPrologue:
2085 OutStreamer->emitWinCFIEndProlog();
2086 break;
2087
2088 default:
2089 llvm_unreachable("expected SEH_ instruction");
2090 }
2091}
2092
2093static unsigned getRegisterWidth(const MCOperandInfo &Info) {
2094 if (Info.RegClass == X86::VR128RegClassID ||
2095 Info.RegClass == X86::VR128XRegClassID)
2096 return 128;
2097 if (Info.RegClass == X86::VR256RegClassID ||
2098 Info.RegClass == X86::VR256XRegClassID)
2099 return 256;
2100 if (Info.RegClass == X86::VR512RegClassID)
2101 return 512;
2102 llvm_unreachable("Unknown register class!");
2103}
2104
2106 MCStreamer &OutStreamer) {
2107 switch (MI->getOpcode()) {
2108 // Lower PSHUFB and VPERMILP normally but add a comment if we can find
2109 // a constant shuffle mask. We won't be able to do this at the MC layer
2110 // because the mask isn't an immediate.
2111 case X86::PSHUFBrm:
2112 case X86::VPSHUFBrm:
2113 case X86::VPSHUFBYrm:
2114 case X86::VPSHUFBZ128rm:
2115 case X86::VPSHUFBZ128rmk:
2116 case X86::VPSHUFBZ128rmkz:
2117 case X86::VPSHUFBZ256rm:
2118 case X86::VPSHUFBZ256rmk:
2119 case X86::VPSHUFBZ256rmkz:
2120 case X86::VPSHUFBZrm:
2121 case X86::VPSHUFBZrmk:
2122 case X86::VPSHUFBZrmkz: {
2123 unsigned SrcIdx = 1;
2124 if (X86II::isKMasked(MI->getDesc().TSFlags)) {
2125 // Skip mask operand.
2126 ++SrcIdx;
2127 if (X86II::isKMergeMasked(MI->getDesc().TSFlags)) {
2128 // Skip passthru operand.
2129 ++SrcIdx;
2130 }
2131 }
2132 unsigned MaskIdx = SrcIdx + 1 + X86::AddrDisp;
2133
2134 assert(MI->getNumOperands() >= (SrcIdx + 1 + X86::AddrNumOperands) &&
2135 "Unexpected number of operands!");
2136
2137 const MachineOperand &MaskOp = MI->getOperand(MaskIdx);
2138 if (auto *C = getConstantFromPool(*MI, MaskOp)) {
2139 unsigned Width = getRegisterWidth(MI->getDesc().operands()[0]);
2141 DecodePSHUFBMask(C, Width, Mask);
2142 if (!Mask.empty())
2143 OutStreamer.AddComment(getShuffleComment(MI, SrcIdx, SrcIdx, Mask));
2144 }
2145 break;
2146 }
2147
2148 case X86::VPERMILPSrm:
2149 case X86::VPERMILPSYrm:
2150 case X86::VPERMILPSZ128rm:
2151 case X86::VPERMILPSZ128rmk:
2152 case X86::VPERMILPSZ128rmkz:
2153 case X86::VPERMILPSZ256rm:
2154 case X86::VPERMILPSZ256rmk:
2155 case X86::VPERMILPSZ256rmkz:
2156 case X86::VPERMILPSZrm:
2157 case X86::VPERMILPSZrmk:
2158 case X86::VPERMILPSZrmkz:
2159 case X86::VPERMILPDrm:
2160 case X86::VPERMILPDYrm:
2161 case X86::VPERMILPDZ128rm:
2162 case X86::VPERMILPDZ128rmk:
2163 case X86::VPERMILPDZ128rmkz:
2164 case X86::VPERMILPDZ256rm:
2165 case X86::VPERMILPDZ256rmk:
2166 case X86::VPERMILPDZ256rmkz:
2167 case X86::VPERMILPDZrm:
2168 case X86::VPERMILPDZrmk:
2169 case X86::VPERMILPDZrmkz: {
2170 unsigned ElSize;
2171 switch (MI->getOpcode()) {
2172 default: llvm_unreachable("Invalid opcode");
2173 case X86::VPERMILPSrm:
2174 case X86::VPERMILPSYrm:
2175 case X86::VPERMILPSZ128rm:
2176 case X86::VPERMILPSZ256rm:
2177 case X86::VPERMILPSZrm:
2178 case X86::VPERMILPSZ128rmkz:
2179 case X86::VPERMILPSZ256rmkz:
2180 case X86::VPERMILPSZrmkz:
2181 case X86::VPERMILPSZ128rmk:
2182 case X86::VPERMILPSZ256rmk:
2183 case X86::VPERMILPSZrmk:
2184 ElSize = 32;
2185 break;
2186 case X86::VPERMILPDrm:
2187 case X86::VPERMILPDYrm:
2188 case X86::VPERMILPDZ128rm:
2189 case X86::VPERMILPDZ256rm:
2190 case X86::VPERMILPDZrm:
2191 case X86::VPERMILPDZ128rmkz:
2192 case X86::VPERMILPDZ256rmkz:
2193 case X86::VPERMILPDZrmkz:
2194 case X86::VPERMILPDZ128rmk:
2195 case X86::VPERMILPDZ256rmk:
2196 case X86::VPERMILPDZrmk:
2197 ElSize = 64;
2198 break;
2199 }
2200
2201 unsigned SrcIdx = 1;
2202 if (X86II::isKMasked(MI->getDesc().TSFlags)) {
2203 // Skip mask operand.
2204 ++SrcIdx;
2205 if (X86II::isKMergeMasked(MI->getDesc().TSFlags)) {
2206 // Skip passthru operand.
2207 ++SrcIdx;
2208 }
2209 }
2210 unsigned MaskIdx = SrcIdx + 1 + X86::AddrDisp;
2211
2212 assert(MI->getNumOperands() >= (SrcIdx + 1 + X86::AddrNumOperands) &&
2213 "Unexpected number of operands!");
2214
2215 const MachineOperand &MaskOp = MI->getOperand(MaskIdx);
2216 if (auto *C = getConstantFromPool(*MI, MaskOp)) {
2217 unsigned Width = getRegisterWidth(MI->getDesc().operands()[0]);
2219 DecodeVPERMILPMask(C, ElSize, Width, Mask);
2220 if (!Mask.empty())
2221 OutStreamer.AddComment(getShuffleComment(MI, SrcIdx, SrcIdx, Mask));
2222 }
2223 break;
2224 }
2225
2226 case X86::VPERMIL2PDrm:
2227 case X86::VPERMIL2PSrm:
2228 case X86::VPERMIL2PDYrm:
2229 case X86::VPERMIL2PSYrm: {
2230 assert(MI->getNumOperands() >= (3 + X86::AddrNumOperands + 1) &&
2231 "Unexpected number of operands!");
2232
2233 const MachineOperand &CtrlOp = MI->getOperand(MI->getNumOperands() - 1);
2234 if (!CtrlOp.isImm())
2235 break;
2236
2237 unsigned ElSize;
2238 switch (MI->getOpcode()) {
2239 default: llvm_unreachable("Invalid opcode");
2240 case X86::VPERMIL2PSrm: case X86::VPERMIL2PSYrm: ElSize = 32; break;
2241 case X86::VPERMIL2PDrm: case X86::VPERMIL2PDYrm: ElSize = 64; break;
2242 }
2243
2244 const MachineOperand &MaskOp = MI->getOperand(3 + X86::AddrDisp);
2245 if (auto *C = getConstantFromPool(*MI, MaskOp)) {
2246 unsigned Width = getRegisterWidth(MI->getDesc().operands()[0]);
2248 DecodeVPERMIL2PMask(C, (unsigned)CtrlOp.getImm(), ElSize, Width, Mask);
2249 if (!Mask.empty())
2250 OutStreamer.AddComment(getShuffleComment(MI, 1, 2, Mask));
2251 }
2252 break;
2253 }
2254
2255 case X86::VPPERMrrm: {
2256 assert(MI->getNumOperands() >= (3 + X86::AddrNumOperands) &&
2257 "Unexpected number of operands!");
2258
2259 const MachineOperand &MaskOp = MI->getOperand(3 + X86::AddrDisp);
2260 if (auto *C = getConstantFromPool(*MI, MaskOp)) {
2261 unsigned Width = getRegisterWidth(MI->getDesc().operands()[0]);
2263 DecodeVPPERMMask(C, Width, Mask);
2264 if (!Mask.empty())
2265 OutStreamer.AddComment(getShuffleComment(MI, 1, 2, Mask));
2266 }
2267 break;
2268 }
2269
2270 case X86::MMX_MOVQ64rm: {
2271 assert(MI->getNumOperands() == (1 + X86::AddrNumOperands) &&
2272 "Unexpected number of operands!");
2273 if (auto *C = getConstantFromPool(*MI, MI->getOperand(1 + X86::AddrDisp))) {
2274 std::string Comment;
2275 raw_string_ostream CS(Comment);
2276 const MachineOperand &DstOp = MI->getOperand(0);
2278 if (auto *CF = dyn_cast<ConstantFP>(C)) {
2279 CS << "0x" << toString(CF->getValueAPF().bitcastToAPInt(), 16, false);
2280 OutStreamer.AddComment(CS.str());
2281 }
2282 }
2283 break;
2284 }
2285
2286#define MOV_CASE(Prefix, Suffix) \
2287 case X86::Prefix##MOVAPD##Suffix##rm: \
2288 case X86::Prefix##MOVAPS##Suffix##rm: \
2289 case X86::Prefix##MOVUPD##Suffix##rm: \
2290 case X86::Prefix##MOVUPS##Suffix##rm: \
2291 case X86::Prefix##MOVDQA##Suffix##rm: \
2292 case X86::Prefix##MOVDQU##Suffix##rm:
2293
2294#define MOV_AVX512_CASE(Suffix) \
2295 case X86::VMOVDQA64##Suffix##rm: \
2296 case X86::VMOVDQA32##Suffix##rm: \
2297 case X86::VMOVDQU64##Suffix##rm: \
2298 case X86::VMOVDQU32##Suffix##rm: \
2299 case X86::VMOVDQU16##Suffix##rm: \
2300 case X86::VMOVDQU8##Suffix##rm: \
2301 case X86::VMOVAPS##Suffix##rm: \
2302 case X86::VMOVAPD##Suffix##rm: \
2303 case X86::VMOVUPS##Suffix##rm: \
2304 case X86::VMOVUPD##Suffix##rm:
2305
2306#define CASE_ALL_MOV_RM() \
2307 MOV_CASE(, ) /* SSE */ \
2308 MOV_CASE(V, ) /* AVX-128 */ \
2309 MOV_CASE(V, Y) /* AVX-256 */ \
2310 MOV_AVX512_CASE(Z) \
2311 MOV_AVX512_CASE(Z256) \
2312 MOV_AVX512_CASE(Z128)
2313
2314 // For loads from a constant pool to a vector register, print the constant
2315 // loaded.
2317 case X86::VBROADCASTF128:
2318 case X86::VBROADCASTI128:
2319 case X86::VBROADCASTF32X4Z256rm:
2320 case X86::VBROADCASTF32X4rm:
2321 case X86::VBROADCASTF32X8rm:
2322 case X86::VBROADCASTF64X2Z128rm:
2323 case X86::VBROADCASTF64X2rm:
2324 case X86::VBROADCASTF64X4rm:
2325 case X86::VBROADCASTI32X4Z256rm:
2326 case X86::VBROADCASTI32X4rm:
2327 case X86::VBROADCASTI32X8rm:
2328 case X86::VBROADCASTI64X2Z128rm:
2329 case X86::VBROADCASTI64X2rm:
2330 case X86::VBROADCASTI64X4rm:
2331 assert(MI->getNumOperands() >= (1 + X86::AddrNumOperands) &&
2332 "Unexpected number of operands!");
2333 if (auto *C = getConstantFromPool(*MI, MI->getOperand(1 + X86::AddrDisp))) {
2334 int NumLanes = 1;
2335 // Override NumLanes for the broadcast instructions.
2336 switch (MI->getOpcode()) {
2337 case X86::VBROADCASTF128: NumLanes = 2; break;
2338 case X86::VBROADCASTI128: NumLanes = 2; break;
2339 case X86::VBROADCASTF32X4Z256rm: NumLanes = 2; break;
2340 case X86::VBROADCASTF32X4rm: NumLanes = 4; break;
2341 case X86::VBROADCASTF32X8rm: NumLanes = 2; break;
2342 case X86::VBROADCASTF64X2Z128rm: NumLanes = 2; break;
2343 case X86::VBROADCASTF64X2rm: NumLanes = 4; break;
2344 case X86::VBROADCASTF64X4rm: NumLanes = 2; break;
2345 case X86::VBROADCASTI32X4Z256rm: NumLanes = 2; break;
2346 case X86::VBROADCASTI32X4rm: NumLanes = 4; break;
2347 case X86::VBROADCASTI32X8rm: NumLanes = 2; break;
2348 case X86::VBROADCASTI64X2Z128rm: NumLanes = 2; break;
2349 case X86::VBROADCASTI64X2rm: NumLanes = 4; break;
2350 case X86::VBROADCASTI64X4rm: NumLanes = 2; break;
2351 }
2352
2353 std::string Comment;
2354 raw_string_ostream CS(Comment);
2355 const MachineOperand &DstOp = MI->getOperand(0);
2357 if (auto *CDS = dyn_cast<ConstantDataSequential>(C)) {
2358 CS << "[";
2359 for (int l = 0; l != NumLanes; ++l) {
2360 for (int i = 0, NumElements = CDS->getNumElements(); i < NumElements;
2361 ++i) {
2362 if (i != 0 || l != 0)
2363 CS << ",";
2364 if (CDS->getElementType()->isIntegerTy())
2365 printConstant(CDS->getElementAsAPInt(i), CS);
2366 else if (CDS->getElementType()->isHalfTy() ||
2367 CDS->getElementType()->isFloatTy() ||
2368 CDS->getElementType()->isDoubleTy())
2369 printConstant(CDS->getElementAsAPFloat(i), CS);
2370 else
2371 CS << "?";
2372 }
2373 }
2374 CS << "]";
2375 OutStreamer.AddComment(CS.str());
2376 } else if (auto *CV = dyn_cast<ConstantVector>(C)) {
2377 CS << "<";
2378 for (int l = 0; l != NumLanes; ++l) {
2379 for (int i = 0, NumOperands = CV->getNumOperands(); i < NumOperands;
2380 ++i) {
2381 if (i != 0 || l != 0)
2382 CS << ",";
2383 printConstant(CV->getOperand(i), CS);
2384 }
2385 }
2386 CS << ">";
2387 OutStreamer.AddComment(CS.str());
2388 }
2389 }
2390 break;
2391
2392 case X86::MOVDDUPrm:
2393 case X86::VMOVDDUPrm:
2394 case X86::VMOVDDUPZ128rm:
2395 case X86::VBROADCASTSSrm:
2396 case X86::VBROADCASTSSYrm:
2397 case X86::VBROADCASTSSZ128rm:
2398 case X86::VBROADCASTSSZ256rm:
2399 case X86::VBROADCASTSSZrm:
2400 case X86::VBROADCASTSDYrm:
2401 case X86::VBROADCASTSDZ256rm:
2402 case X86::VBROADCASTSDZrm:
2403 case X86::VPBROADCASTBrm:
2404 case X86::VPBROADCASTBYrm:
2405 case X86::VPBROADCASTBZ128rm:
2406 case X86::VPBROADCASTBZ256rm:
2407 case X86::VPBROADCASTBZrm:
2408 case X86::VPBROADCASTDrm:
2409 case X86::VPBROADCASTDYrm:
2410 case X86::VPBROADCASTDZ128rm:
2411 case X86::VPBROADCASTDZ256rm:
2412 case X86::VPBROADCASTDZrm:
2413 case X86::VPBROADCASTQrm:
2414 case X86::VPBROADCASTQYrm:
2415 case X86::VPBROADCASTQZ128rm:
2416 case X86::VPBROADCASTQZ256rm:
2417 case X86::VPBROADCASTQZrm:
2418 case X86::VPBROADCASTWrm:
2419 case X86::VPBROADCASTWYrm:
2420 case X86::VPBROADCASTWZ128rm:
2421 case X86::VPBROADCASTWZ256rm:
2422 case X86::VPBROADCASTWZrm:
2423 assert(MI->getNumOperands() >= (1 + X86::AddrNumOperands) &&
2424 "Unexpected number of operands!");
2425 if (auto *C = getConstantFromPool(*MI, MI->getOperand(1 + X86::AddrDisp))) {
2426 int NumElts;
2427 switch (MI->getOpcode()) {
2428 default: llvm_unreachable("Invalid opcode");
2429 case X86::MOVDDUPrm: NumElts = 2; break;
2430 case X86::VMOVDDUPrm: NumElts = 2; break;
2431 case X86::VMOVDDUPZ128rm: NumElts = 2; break;
2432 case X86::VBROADCASTSSrm: NumElts = 4; break;
2433 case X86::VBROADCASTSSYrm: NumElts = 8; break;
2434 case X86::VBROADCASTSSZ128rm: NumElts = 4; break;
2435 case X86::VBROADCASTSSZ256rm: NumElts = 8; break;
2436 case X86::VBROADCASTSSZrm: NumElts = 16; break;
2437 case X86::VBROADCASTSDYrm: NumElts = 4; break;
2438 case X86::VBROADCASTSDZ256rm: NumElts = 4; break;
2439 case X86::VBROADCASTSDZrm: NumElts = 8; break;
2440 case X86::VPBROADCASTBrm: NumElts = 16; break;
2441 case X86::VPBROADCASTBYrm: NumElts = 32; break;
2442 case X86::VPBROADCASTBZ128rm: NumElts = 16; break;
2443 case X86::VPBROADCASTBZ256rm: NumElts = 32; break;
2444 case X86::VPBROADCASTBZrm: NumElts = 64; break;
2445 case X86::VPBROADCASTDrm: NumElts = 4; break;
2446 case X86::VPBROADCASTDYrm: NumElts = 8; break;
2447 case X86::VPBROADCASTDZ128rm: NumElts = 4; break;
2448 case X86::VPBROADCASTDZ256rm: NumElts = 8; break;
2449 case X86::VPBROADCASTDZrm: NumElts = 16; break;
2450 case X86::VPBROADCASTQrm: NumElts = 2; break;
2451 case X86::VPBROADCASTQYrm: NumElts = 4; break;
2452 case X86::VPBROADCASTQZ128rm: NumElts = 2; break;
2453 case X86::VPBROADCASTQZ256rm: NumElts = 4; break;
2454 case X86::VPBROADCASTQZrm: NumElts = 8; break;
2455 case X86::VPBROADCASTWrm: NumElts = 8; break;
2456 case X86::VPBROADCASTWYrm: NumElts = 16; break;
2457 case X86::VPBROADCASTWZ128rm: NumElts = 8; break;
2458 case X86::VPBROADCASTWZ256rm: NumElts = 16; break;
2459 case X86::VPBROADCASTWZrm: NumElts = 32; break;
2460 }
2461
2462 std::string Comment;
2463 raw_string_ostream CS(Comment);
2464 const MachineOperand &DstOp = MI->getOperand(0);
2466 CS << "[";
2467 for (int i = 0; i != NumElts; ++i) {
2468 if (i != 0)
2469 CS << ",";
2470 printConstant(C, CS);
2471 }
2472 CS << "]";
2473 OutStreamer.AddComment(CS.str());
2474 }
2475 }
2476}
2477
2479 // FIXME: Enable feature predicate checks once all the test pass.
2480 // X86_MC::verifyInstructionPredicates(MI->getOpcode(),
2481 // Subtarget->getFeatureBits());
2482
2483 X86MCInstLower MCInstLowering(*MF, *this);
2484 const X86RegisterInfo *RI =
2485 MF->getSubtarget<X86Subtarget>().getRegisterInfo();
2486
2487 if (MI->getOpcode() == X86::OR64rm) {
2488 for (auto &Opd : MI->operands()) {
2489 if (Opd.isSymbol() && StringRef(Opd.getSymbolName()) ==
2490 "swift_async_extendedFramePointerFlags") {
2491 ShouldEmitWeakSwiftAsyncExtendedFramePointerFlags = true;
2492 }
2493 }
2494 }
2495
2496 // Add a comment about EVEX-2-VEX compression for AVX-512 instrs that
2497 // are compressed from EVEX encoding to VEX encoding.
2499 if (MI->getAsmPrinterFlags() & X86::AC_EVEX_2_VEX)
2500 OutStreamer->AddComment("EVEX TO VEX Compression ", false);
2501 }
2502
2503 // Add comments for values loaded from constant pool.
2504 if (OutStreamer->isVerboseAsm())
2506
2507 switch (MI->getOpcode()) {
2508 case TargetOpcode::DBG_VALUE:
2509 llvm_unreachable("Should be handled target independently");
2510
2511 case X86::EH_RETURN:
2512 case X86::EH_RETURN64: {
2513 // Lower these as normal, but add some comments.
2514 Register Reg = MI->getOperand(0).getReg();
2515 OutStreamer->AddComment(StringRef("eh_return, addr: %") +
2517 break;
2518 }
2519 case X86::CLEANUPRET: {
2520 // Lower these as normal, but add some comments.
2521 OutStreamer->AddComment("CLEANUPRET");
2522 break;
2523 }
2524
2525 case X86::CATCHRET: {
2526 // Lower these as normal, but add some comments.
2527 OutStreamer->AddComment("CATCHRET");
2528 break;
2529 }
2530
2531 case X86::ENDBR32:
2532 case X86::ENDBR64: {
2533 // CurrentPatchableFunctionEntrySym can be CurrentFnBegin only for
2534 // -fpatchable-function-entry=N,0. The entry MBB is guaranteed to be
2535 // non-empty. If MI is the initial ENDBR, place the
2536 // __patchable_function_entries label after ENDBR.
2539 MI == &MF->front().front()) {
2540 MCInst Inst;
2541 MCInstLowering.Lower(MI, Inst);
2542 EmitAndCountInstruction(Inst);
2545 return;
2546 }
2547 break;
2548 }
2549
2550 case X86::TAILJMPd64:
2551 if (IndCSPrefix && MI->hasRegisterImplicitUseOperand(X86::R11))
2552 EmitAndCountInstruction(MCInstBuilder(X86::CS_PREFIX));
2553 [[fallthrough]];
2554 case X86::TAILJMPr:
2555 case X86::TAILJMPm:
2556 case X86::TAILJMPd:
2557 case X86::TAILJMPd_CC:
2558 case X86::TAILJMPr64:
2559 case X86::TAILJMPm64:
2560 case X86::TAILJMPd64_CC:
2561 case X86::TAILJMPr64_REX:
2562 case X86::TAILJMPm64_REX:
2563 // Lower these as normal, but add some comments.
2564 OutStreamer->AddComment("TAILCALL");
2565 break;
2566
2567 case X86::TLS_addr32:
2568 case X86::TLS_addr64:
2569 case X86::TLS_addrX32:
2570 case X86::TLS_base_addr32:
2571 case X86::TLS_base_addr64:
2572 case X86::TLS_base_addrX32:
2573 return LowerTlsAddr(MCInstLowering, *MI);
2574
2575 case X86::MOVPC32r: {
2576 // This is a pseudo op for a two instruction sequence with a label, which
2577 // looks like:
2578 // call "L1$pb"
2579 // "L1$pb":
2580 // popl %esi
2581
2582 // Emit the call.
2583 MCSymbol *PICBase = MF->getPICBaseSymbol();
2584 // FIXME: We would like an efficient form for this, so we don't have to do a
2585 // lot of extra uniquing.
2586 EmitAndCountInstruction(
2587 MCInstBuilder(X86::CALLpcrel32)
2588 .addExpr(MCSymbolRefExpr::create(PICBase, OutContext)));
2589
2590 const X86FrameLowering *FrameLowering =
2591 MF->getSubtarget<X86Subtarget>().getFrameLowering();
2592 bool hasFP = FrameLowering->hasFP(*MF);
2593
2594 // TODO: This is needed only if we require precise CFA.
2595 bool HasActiveDwarfFrame = OutStreamer->getNumFrameInfos() &&
2596 !OutStreamer->getDwarfFrameInfos().back().End;
2597
2598 int stackGrowth = -RI->getSlotSize();
2599
2600 if (HasActiveDwarfFrame && !hasFP) {
2601 OutStreamer->emitCFIAdjustCfaOffset(-stackGrowth);
2602 }
2603
2604 // Emit the label.
2605 OutStreamer->emitLabel(PICBase);
2606
2607 // popl $reg
2608 EmitAndCountInstruction(
2609 MCInstBuilder(X86::POP32r).addReg(MI->getOperand(0).getReg()));
2610
2611 if (HasActiveDwarfFrame && !hasFP) {
2612 OutStreamer->emitCFIAdjustCfaOffset(stackGrowth);
2613 }
2614 return;
2615 }
2616
2617 case X86::ADD32ri: {
2618 // Lower the MO_GOT_ABSOLUTE_ADDRESS form of ADD32ri.
2619 if (MI->getOperand(2).getTargetFlags() != X86II::MO_GOT_ABSOLUTE_ADDRESS)
2620 break;
2621
2622 // Okay, we have something like:
2623 // EAX = ADD32ri EAX, MO_GOT_ABSOLUTE_ADDRESS(@MYGLOBAL)
2624
2625 // For this, we want to print something like:
2626 // MYGLOBAL + (. - PICBASE)
2627 // However, we can't generate a ".", so just emit a new label here and refer
2628 // to it.
2630 OutStreamer->emitLabel(DotSym);
2631
2632 // Now that we have emitted the label, lower the complex operand expression.
2633 MCSymbol *OpSym = MCInstLowering.GetSymbolFromOperand(MI->getOperand(2));
2634
2635 const MCExpr *DotExpr = MCSymbolRefExpr::create(DotSym, OutContext);
2636 const MCExpr *PICBase =
2638 DotExpr = MCBinaryExpr::createSub(DotExpr, PICBase, OutContext);
2639
2640 DotExpr = MCBinaryExpr::createAdd(
2642
2643 EmitAndCountInstruction(MCInstBuilder(X86::ADD32ri)
2644 .addReg(MI->getOperand(0).getReg())
2645 .addReg(MI->getOperand(1).getReg())
2646 .addExpr(DotExpr));
2647 return;
2648 }
2649 case TargetOpcode::STATEPOINT:
2650 return LowerSTATEPOINT(*MI, MCInstLowering);
2651
2652 case TargetOpcode::FAULTING_OP:
2653 return LowerFAULTING_OP(*MI, MCInstLowering);
2654
2655 case TargetOpcode::FENTRY_CALL:
2656 return LowerFENTRY_CALL(*MI, MCInstLowering);
2657
2658 case TargetOpcode::PATCHABLE_OP:
2659 return LowerPATCHABLE_OP(*MI, MCInstLowering);
2660
2661 case TargetOpcode::STACKMAP:
2662 return LowerSTACKMAP(*MI);
2663
2664 case TargetOpcode::PATCHPOINT:
2665 return LowerPATCHPOINT(*MI, MCInstLowering);
2666
2667 case TargetOpcode::PATCHABLE_FUNCTION_ENTER:
2668 return LowerPATCHABLE_FUNCTION_ENTER(*MI, MCInstLowering);
2669
2670 case TargetOpcode::PATCHABLE_RET:
2671 return LowerPATCHABLE_RET(*MI, MCInstLowering);
2672
2673 case TargetOpcode::PATCHABLE_TAIL_CALL:
2674 return LowerPATCHABLE_TAIL_CALL(*MI, MCInstLowering);
2675
2676 case TargetOpcode::PATCHABLE_EVENT_CALL:
2677 return LowerPATCHABLE_EVENT_CALL(*MI, MCInstLowering);
2678
2679 case TargetOpcode::PATCHABLE_TYPED_EVENT_CALL:
2680 return LowerPATCHABLE_TYPED_EVENT_CALL(*MI, MCInstLowering);
2681
2682 case X86::MORESTACK_RET:
2683 EmitAndCountInstruction(MCInstBuilder(getRetOpcode(*Subtarget)));
2684 return;
2685
2686 case X86::KCFI_CHECK:
2687 return LowerKCFI_CHECK(*MI);
2688
2689 case X86::ASAN_CHECK_MEMACCESS:
2690 return LowerASAN_CHECK_MEMACCESS(*MI);
2691
2692 case X86::MORESTACK_RET_RESTORE_R10:
2693 // Return, then restore R10.
2694 EmitAndCountInstruction(MCInstBuilder(getRetOpcode(*Subtarget)));
2695 EmitAndCountInstruction(
2696 MCInstBuilder(X86::MOV64rr).addReg(X86::R10).addReg(X86::RAX));
2697 return;
2698
2699 case X86::SEH_PushReg:
2700 case X86::SEH_SaveReg:
2701 case X86::SEH_SaveXMM:
2702 case X86::SEH_StackAlloc:
2703 case X86::SEH_StackAlign:
2704 case X86::SEH_SetFrame:
2705 case X86::SEH_PushFrame:
2706 case X86::SEH_EndPrologue:
2707 EmitSEHInstruction(MI);
2708 return;
2709
2710 case X86::SEH_Epilogue: {
2711 assert(MF->hasWinCFI() && "SEH_ instruction in function without WinCFI?");
2713 // Check if preceded by a call and emit nop if so.
2714 for (MBBI = PrevCrossBBInst(MBBI);
2717 // Pseudo instructions that aren't a call are assumed to not emit any
2718 // code. If they do, we worst case generate unnecessary noops after a
2719 // call.
2720 if (MBBI->isCall() || !MBBI->isPseudo()) {
2721 if (MBBI->isCall())
2722 EmitAndCountInstruction(MCInstBuilder(X86::NOOP));
2723 break;
2724 }
2725 }
2726 return;
2727 }
2728 case X86::UBSAN_UD1:
2729 EmitAndCountInstruction(MCInstBuilder(X86::UD1Lm)
2730 .addReg(X86::EAX)
2731 .addReg(X86::EAX)
2732 .addImm(1)
2733 .addReg(X86::NoRegister)
2734 .addImm(MI->getOperand(0).getImm())
2735 .addReg(X86::NoRegister));
2736 return;
2737 case X86::CALL64pcrel32:
2738 if (IndCSPrefix && MI->hasRegisterImplicitUseOperand(X86::R11))
2739 EmitAndCountInstruction(MCInstBuilder(X86::CS_PREFIX));
2740 break;
2741 }
2742
2743 MCInst TmpInst;
2744 MCInstLowering.Lower(MI, TmpInst);
2745
2746 // Stackmap shadows cannot include branch targets, so we can count the bytes
2747 // in a call towards the shadow, but must ensure that the no thread returns
2748 // in to the stackmap shadow. The only way to achieve this is if the call
2749 // is at the end of the shadow.
2750 if (MI->isCall()) {
2751 // Count then size of the call towards the shadow
2752 SMShadowTracker.count(TmpInst, getSubtargetInfo(), CodeEmitter.get());
2753 // Then flush the shadow so that we fill with nops before the call, not
2754 // after it.
2755 SMShadowTracker.emitShadowPadding(*OutStreamer, getSubtargetInfo());
2756 // Then emit the call
2757 OutStreamer->emitInstruction(TmpInst, getSubtargetInfo());
2758 return;
2759 }
2760
2761 EmitAndCountInstruction(TmpInst);
2762}
MachineBasicBlock & MBB
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
MachineBasicBlock MachineBasicBlock::iterator MBBI
static MCDisassembler::DecodeStatus addOperand(MCInst &Inst, const MCOperand &Opnd)
Analysis containing CSE Info
Definition: CSEInfo.cpp:27
std::string Name
Fixup Statepoint Caller Saved
IRTranslator LLVM IR MI
#define F(x, y, z)
Definition: MD5.cpp:55
#define I(x, y, z)
Definition: MD5.cpp:58
This file declares the MachineConstantPool class which is an abstract constant pool to keep track of ...
Memory true print Memory SSA Printer
Definition: MemorySSA.cpp:78
static MCSymbol * GetSymbolFromOperand(const MachineOperand &MO, AsmPrinter &AP)
const char LLVMTargetMachineRef TM
uint64_t TSFlags
static bool isValid(const char C)
Returns true if C is a valid mangled character: <0-9a-zA-Z_>.
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
raw_pwrite_stream & OS
This file defines the SmallString class.
static MCOperand LowerSymbolOperand(const MachineInstr *MI, const MachineOperand &MO, AsmPrinter &AP)
static void emitX86Nops(MCStreamer &OS, unsigned NumBytes, const X86Subtarget *Subtarget)
Emit the optimal amount of multi-byte nops on X86.
static unsigned getRetOpcode(const X86Subtarget &Subtarget)
static void printConstant(const APInt &Val, raw_ostream &CS)
static unsigned convertTailJumpOpcode(unsigned Opcode)
static const Constant * getConstantFromPool(const MachineInstr &MI, const MachineOperand &Op)
static void addConstantComments(const MachineInstr *MI, MCStreamer &OutStreamer)
static unsigned getRegisterWidth(const MCOperandInfo &Info)
static MachineBasicBlock::const_iterator PrevCrossBBInst(MachineBasicBlock::const_iterator MBBI)
static unsigned emitNop(MCStreamer &OS, unsigned NumBytes, const X86Subtarget *Subtarget)
Emit the largest nop instruction smaller than or equal to NumBytes bytes.
static void SimplifyShortImmForm(MCInst &Inst, unsigned Opcode)
Simplify FOO $imm, %{al,ax,eax,rax} to FOO $imm, for instruction with a short fixed-register form.
static void SimplifyMOVSX(MCInst &Inst)
If a movsx instruction has a shorter encoding for the used register simplify the instruction to use i...
#define CASE_ALL_MOV_RM()
static std::string getShuffleComment(const MachineInstr *MI, unsigned SrcOp1Idx, unsigned SrcOp2Idx, ArrayRef< int > Mask)
static void SimplifyShortMoveForm(X86AsmPrinter &Printer, MCInst &Inst, unsigned Opcode)
Simplify things like MOV32rm to MOV32o32a.
Class for arbitrary precision integers.
Definition: APInt.h:75
uint64_t getZExtValue() const
Get zero extended value.
Definition: APInt.h:1494
unsigned getBitWidth() const
Return the number of bits in the APInt.
Definition: APInt.h:1439
unsigned getNumWords() const
Get the number of words.
Definition: APInt.h:1446
const uint64_t * getRawData() const
This function returns a pointer to the internal storage of the APInt.
Definition: APInt.h:557
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
Definition: ArrayRef.h:41
This class is intended to be used as a driving class for all asm writers.
Definition: AsmPrinter.h:84
MCSymbol * getSymbol(const GlobalValue *GV) const
Definition: AsmPrinter.cpp:662
MCSymbol * CurrentFnBegin
Definition: AsmPrinter.h:200
TargetMachine & TM
Target machine description.
Definition: AsmPrinter.h:87
virtual MCSymbol * GetCPISymbol(unsigned CPID) const
Return the symbol for the specified constant pool entry.
void emitKCFITrapEntry(const MachineFunction &MF, const MCSymbol *Symbol)
MachineFunction * MF
The current machine function.
Definition: AsmPrinter.h:102
MCSymbol * GetJTISymbol(unsigned JTID, bool isLinkerPrivate=false) const
Return the symbol for the specified jump table entry.
void recordSled(MCSymbol *Sled, const MachineInstr &MI, SledKind Kind, uint8_t Version=0)
MCSymbol * getSymbolPreferLocal(const GlobalValue &GV) const
Similar to getSymbol() but preferred for references.
Definition: AsmPrinter.cpp:666
MachineModuleInfo * MMI
This is a pointer to the current MachineModuleInfo.
Definition: AsmPrinter.h:105
MCContext & OutContext
This is the context for the output file that we are streaming.
Definition: AsmPrinter.h:94
MCSymbol * createTempSymbol(const Twine &Name) const
bool isPositionIndependent() const
Definition: AsmPrinter.cpp:370
MCSymbol * CurrentPatchableFunctionEntrySym
The symbol for the entry in __patchable_function_entires.
Definition: AsmPrinter.h:117
std::unique_ptr< MCStreamer > OutStreamer
This is the MCStreamer object for the file we are generating.
Definition: AsmPrinter.h:99
void getNameWithPrefix(SmallVectorImpl< char > &Name, const GlobalValue *GV) const
Definition: AsmPrinter.cpp:657
StackMaps SM
Definition: AsmPrinter.h:210
MCSymbol * GetBlockAddressSymbol(const BlockAddress *BA) const
Return the MCSymbol used to satisfy BlockAddress uses of the specified basic block.
const MCSubtargetInfo & getSubtargetInfo() const
Return information about subtarget.
Definition: AsmPrinter.cpp:393
StringRef getValueAsString() const
Return the attribute's value as a string.
Definition: Attributes.cpp:317
This class represents a function call, abstracting a target machine's calling convention.
This is an important base class in LLVM.
Definition: Constant.h:41
A parsed version of the target data layout string in and methods for querying it.
Definition: DataLayout.h:110
Register getReg() const
void recordFaultingOp(FaultKind FaultTy, const MCSymbol *FaultingLabel, const MCSymbol *HandlerLabel)
Definition: FaultMaps.cpp:28
Attribute getFnAttribute(Attribute::AttrKind Kind) const
Return the attribute for the given attribute kind.
Definition: Function.cpp:670
bool hasInternalLinkage() const
Definition: GlobalValue.h:521
This class is intended to be used as a base class for asm properties and features specific to the tar...
Definition: MCAsmInfo.h:56
bool canRelaxRelocations() const
Definition: MCAsmInfo.h:879
static const MCBinaryExpr * createAdd(const MCExpr *LHS, const MCExpr *RHS, MCContext &Ctx)
Definition: MCExpr.h:525
static const MCBinaryExpr * createSub(const MCExpr *LHS, const MCExpr *RHS, MCContext &Ctx)
Definition: MCExpr.h:610
MCCodeEmitter - Generic instruction encoding interface.
Definition: MCCodeEmitter.h:21
virtual void encodeInstruction(const MCInst &Inst, raw_ostream &OS, SmallVectorImpl< MCFixup > &Fixups, const MCSubtargetInfo &STI) const =0
EncodeInstruction - Encode the given Inst to bytes on the output stream OS.
static const MCConstantExpr * create(int64_t Value, MCContext &Ctx, bool PrintInHex=false, unsigned SizeInBytes=0)
Definition: MCExpr.cpp:194
Context object for machine code objects.
Definition: MCContext.h:76
MCSymbol * createTempSymbol()
Create a temporary symbol with a unique name.
Definition: MCContext.cpp:318
const MCAsmInfo * getAsmInfo() const
Definition: MCContext.h:446
MCSymbol * getOrCreateSymbol(const Twine &Name)
Lookup the symbol inside with the specified Name.
Definition: MCContext.cpp:201
Base class for the full range of assembler expressions which are needed for parsing.
Definition: MCExpr.h:35
MCInstBuilder & addReg(unsigned Reg)
Add a new register operand.
Definition: MCInstBuilder.h:31
MCInstBuilder & addExpr(const MCExpr *Val)
Add a new MCExpr operand.
Definition: MCInstBuilder.h:55
Instances of this class represent a single low-level machine instruction.
Definition: MCInst.h:184
void erase(iterator I)
Definition: MCInst.h:216
unsigned getNumOperands() const
Definition: MCInst.h:208
unsigned getOpcode() const
Definition: MCInst.h:198
iterator insert(iterator I, const MCOperand &Op)
Definition: MCInst.h:224
void setFlags(unsigned F)
Definition: MCInst.h:200
void addOperand(const MCOperand Op)
Definition: MCInst.h:210
iterator begin()
Definition: MCInst.h:219
void setOpcode(unsigned Op)
Definition: MCInst.h:197
const MCOperand & getOperand(unsigned i) const
Definition: MCInst.h:206
This holds information about one operand of a machine instruction, indicating the register class for ...
Definition: MCInstrDesc.h:85
Instances of this class represent operands of the MCInst class.
Definition: MCInst.h:36
static MCOperand createReg(unsigned Reg)
Definition: MCInst.h:134
static MCOperand createExpr(const MCExpr *Val)
Definition: MCInst.h:162
int64_t getImm() const
Definition: MCInst.h:80
static MCOperand createImm(int64_t Val)
Definition: MCInst.h:141
bool isImm() const
Definition: MCInst.h:62
unsigned getReg() const
Returns the register number.
Definition: MCInst.h:69
bool isReg() const
Definition: MCInst.h:61
const MCExpr * getExpr() const
Definition: MCInst.h:114
bool isExpr() const
Definition: MCInst.h:65
const char * getName(MCRegister RegNo) const
Return the human-readable symbolic target-specific name for the specified physical register.
Wrapper class representing physical registers. Should be passed by value.
Definition: MCRegister.h:24
Streaming machine code generation interface.
Definition: MCStreamer.h:212
virtual void AddComment(const Twine &T, bool EOL=true)
Add a textual comment.
Definition: MCStreamer.h:359
virtual void emitRawComment(const Twine &T, bool TabPrefix=true)
Print T and prefix it with the comment string (normally #) and optionally a tab.
Definition: MCStreamer.cpp:121
void setAllowAutoPadding(bool v)
Definition: MCStreamer.h:308
bool getAllowAutoPadding() const
Definition: MCStreamer.h:309
Generic base class for all target subtargets.
Represent a reference to a symbol from inside an expression.
Definition: MCExpr.h:192
static const MCSymbolRefExpr * create(const MCSymbol *Symbol, MCContext &Ctx)
Definition: MCExpr.h:386
MCSymbol - Instances of this class represent a symbol name in the MC file, and MCSymbols are created ...
Definition: MCSymbol.h:41
StringRef getName() const
getName - Get the symbol name.
Definition: MCSymbol.h:203
MachineInstrBundleIterator< const MachineInstr > const_iterator
MCSymbol * getSymbol() const
Return the MCSymbol for this basic block.
const MachineFunction * getParent() const
Return the MachineFunction containing this basic block.
This class is a data container for one entry in a MachineConstantPool.
bool isMachineConstantPoolEntry() const
isMachineConstantPoolEntry - Return true if the MachineConstantPoolEntry is indeed a target specific ...
union llvm::MachineConstantPoolEntry::@193 Val
The constant itself.
MCSymbol * getPICBaseSymbol() const
getPICBaseSymbol - Return a function-local symbol to represent the PIC base.
const TargetSubtargetInfo & getSubtarget() const
getSubtarget - Return the subtarget for which this machine code is being compiled.
Function & getFunction()
Return the LLVM function that this machine code represents.
const MachineBasicBlock & front() const
Representation of each machine instruction.
Definition: MachineInstr.h:68
iterator_range< mop_iterator > operands()
Definition: MachineInstr.h:641
const MachineOperand & getOperand(unsigned i) const
Definition: MachineInstr.h:526
MachineModuleInfoCOFF - This is a MachineModuleInfoImpl implementation for COFF targets.
StubValueTy & getGVStubEntry(MCSymbol *Sym)
PointerIntPair< MCSymbol *, 1, bool > StubValueTy
MachineModuleInfoMachO - This is a MachineModuleInfoImpl implementation for MachO targets.
const Module * getModule() const
MachineOperand class - Representation of each machine instruction operand.
static MachineOperand CreateMCSymbol(MCSymbol *Sym, unsigned TargetFlags=0)
const GlobalValue * getGlobal() const
int64_t getImm() const
bool isImplicit() const
bool isReg() const
isReg - Tests if this is a MO_Register operand.
MachineBasicBlock * getMBB() const
bool isImm() const
isImm - Tests if this is a MO_Immediate operand.
bool isSymbol() const
isSymbol - Tests if this is a MO_ExternalSymbol operand.
bool isJTI() const
isJTI - Tests if this is a MO_JumpTableIndex operand.
const BlockAddress * getBlockAddress() const
unsigned getTargetFlags() const
bool isGlobal() const
isGlobal - Tests if this is a MO_GlobalAddress operand.
MachineOperandType getType() const
getType - Returns the MachineOperandType for this operand.
const char * getSymbolName() const
Register getReg() const
getReg - Returns the register number.
void setTargetFlags(unsigned F)
MCSymbol * getMCSymbol() const
@ MO_Immediate
Immediate operand.
@ MO_ConstantPoolIndex
Address of indexed Constant in Constant Pool.
@ MO_MCSymbol
MCSymbol reference (for debug/eh info)
@ MO_GlobalAddress
Address of a global value.
@ MO_RegisterMask
Mask of preserved registers.
@ MO_BlockAddress
Address of a basic block.
@ MO_MachineBasicBlock
MachineBasicBlock reference.
@ MO_Register
Register operand.
@ MO_ExternalSymbol
Name of external global symbol.
@ MO_JumpTableIndex
Address of indexed Jump Table for switch.
int64_t getOffset() const
Return the offset from the symbol in this operand.
bool isMBB() const
isMBB - Tests if this is a MO_MachineBasicBlock operand.
void getNameWithPrefix(raw_ostream &OS, const GlobalValue *GV, bool CannotUsePrivateLabel) const
Print the appropriate prefix and the specified global variable's name.
Definition: Mangler.cpp:119
bool getRtLibUseGOT() const
Returns true if PLT should be avoided for RTLib calls.
Definition: Module.cpp:666
Pass interface - Implemented by all 'passes'.
Definition: Pass.h:91
virtual void print(raw_ostream &OS, const Module *M) const
print - Print out the internal state of the pass.
Definition: Pass.cpp:130
MI-level patchpoint operands.
Definition: StackMaps.h:76
PointerIntPair - This class implements a pair of a pointer and small integer.
PointerTy getPointer() const
Wrapper class representing virtual and physical registers.
Definition: Register.h:19
SmallString - A SmallString is just a SmallVector with methods and accessors that make it work better...
Definition: SmallString.h:26
size_t size() const
Definition: SmallVector.h:91
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
Definition: SmallVector.h:1200
void recordStatepoint(const MCSymbol &L, const MachineInstr &MI)
Generate a stackmap record for a statepoint instruction.
Definition: StackMaps.cpp:572
void recordPatchPoint(const MCSymbol &L, const MachineInstr &MI)
Generate a stackmap record for a patchpoint instruction.
Definition: StackMaps.cpp:551
void recordStackMap(const MCSymbol &L, const MachineInstr &MI)
Generate a stackmap record for a stackmap instruction.
Definition: StackMaps.cpp:541
MI-level Statepoint operands.
Definition: StackMaps.h:158
StringRef - Represent a constant reference to a string, i.e.
Definition: StringRef.h:50
bool getAsInteger(unsigned Radix, T &Result) const
Parse the current string as an integer of the specified radix.
Definition: StringRef.h:468
constexpr bool empty() const
empty - Check if the string is empty.
Definition: StringRef.h:134
Primary interface to the complete machine description for the target machine.
Definition: TargetMachine.h:78
const Triple & getTargetTriple() const
TargetOptions Options
const MCRegisterInfo * getMCRegisterInfo() const
MCTargetOptions MCOptions
Machine level options.
Target - Wrapper for Target specific information.
Triple - Helper class for working with autoconf configuration names.
Definition: Triple.h:44
bool isOSBinFormatELF() const
Tests whether the OS uses the ELF binary format.
Definition: Triple.h:675
Twine - A lightweight data structure for efficiently representing the concatenation of temporary valu...
Definition: Twine.h:81
The instances of the Type class are immutable: once they are created, they are never changed.
Definition: Type.h:45
static const char * getRegisterName(MCRegister Reg)
void emitInstruction(const MachineInstr *MI) override
Targets should implement this to emit instructions.
const X86Subtarget & getSubtarget() const
bool hasFP(const MachineFunction &MF) const override
hasFP - Return true if the specified function should have a dedicated frame pointer register.
unsigned getSlotSize() const
bool isTargetWindowsMSVC() const
Definition: X86Subtarget.h:311
bool useIndirectThunkCalls() const
Definition: X86Subtarget.h:231
X86 target streamer implementing x86-only assembly directives.
virtual bool emitFPOPushReg(unsigned Reg, SMLoc L={})
virtual bool emitFPOSetFrame(unsigned Reg, SMLoc L={})
virtual bool emitFPOEndPrologue(SMLoc L={})
virtual bool emitFPOStackAlign(unsigned Align, SMLoc L={})
virtual bool emitFPOStackAlloc(unsigned StackAlloc, SMLoc L={})
This class implements an extremely fast bulk output stream that can only output to a stream.
Definition: raw_ostream.h:52
A raw_ostream that writes to an std::string.
Definition: raw_ostream.h:642
std::string & str()
Returns the string's reference.
Definition: raw_ostream.h:660
A raw_ostream that writes to an SmallVector or SmallString.
Definition: raw_ostream.h:672
This provides a very simple, boring adaptor for a begin and end iterator into a range type.
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
@ C
The default llvm calling convention, compatible with C.
Definition: CallingConv.h:34
Reg
All possible values of the reg field in the ModR/M byte.
bool isKMergeMasked(uint64_t TSFlags)
Definition: X86BaseInfo.h:1236
@ MRMSrcReg
MRMSrcReg - This form is used for instructions that use the Mod/RM byte to specify a source,...
Definition: X86BaseInfo.h:700
bool isX86_64ExtendedReg(unsigned RegNo)
Definition: X86BaseInfo.h:1191
@ MO_TLSLD
MO_TLSLD - On a symbol operand this indicates that the immediate is the offset of the GOT entry with ...
Definition: X86BaseInfo.h:473
@ MO_GOTPCREL_NORELAX
MO_GOTPCREL_NORELAX - Same as MO_GOTPCREL except that R_X86_64_GOTPCREL relocations are guaranteed to...
Definition: X86BaseInfo.h:447
@ MO_GOTOFF
MO_GOTOFF - On a symbol operand this indicates that the immediate is the offset to the location of th...
Definition: X86BaseInfo.h:434
@ MO_DARWIN_NONLAZY_PIC_BASE
MO_DARWIN_NONLAZY_PIC_BASE - On a symbol operand "FOO", this indicates that the reference is actually...
Definition: X86BaseInfo.h:547
@ MO_GOT_ABSOLUTE_ADDRESS
MO_GOT_ABSOLUTE_ADDRESS - On a symbol operand, this represents a relocation of: SYMBOL_LABEL + [.
Definition: X86BaseInfo.h:415
@ MO_COFFSTUB
MO_COFFSTUB - On a symbol operand "FOO", this indicates that the reference is actually to the "....
Definition: X86BaseInfo.h:575
@ MO_NTPOFF
MO_NTPOFF - On a symbol operand this indicates that the immediate is the negative thread-pointer offs...
Definition: X86BaseInfo.h:524
@ MO_DARWIN_NONLAZY
MO_DARWIN_NONLAZY - On a symbol operand "FOO", this indicates that the reference is actually to the "...
Definition: X86BaseInfo.h:542
@ MO_INDNTPOFF
MO_INDNTPOFF - On a symbol operand this indicates that the immediate is the absolute address of the G...
Definition: X86BaseInfo.h:500
@ MO_GOTNTPOFF
MO_GOTNTPOFF - On a symbol operand this indicates that the immediate is the offset of the GOT entry w...
Definition: X86BaseInfo.h:532
@ MO_TPOFF
MO_TPOFF - On a symbol operand this indicates that the immediate is the thread-pointer offset for the...
Definition: X86BaseInfo.h:508
@ MO_TLVP_PIC_BASE
MO_TLVP_PIC_BASE - On a symbol operand this indicates that the immediate is some TLS offset from the ...
Definition: X86BaseInfo.h:559
@ MO_GOT
MO_GOT - On a symbol operand this indicates that the immediate is the offset to the GOT entry for the...
Definition: X86BaseInfo.h:427
@ MO_ABS8
MO_ABS8 - On a symbol operand this indicates that the symbol is known to be an absolute symbol in ran...
Definition: X86BaseInfo.h:570
@ MO_PLT
MO_PLT - On a symbol operand this indicates that the immediate is offset to the PLT entry of symbol n...
Definition: X86BaseInfo.h:454
@ MO_TLSGD
MO_TLSGD - On a symbol operand this indicates that the immediate is the offset of the GOT entry with ...
Definition: X86BaseInfo.h:463
@ MO_TLVP
MO_TLVP - On a symbol operand this indicates that the immediate is some TLS offset.
Definition: X86BaseInfo.h:553
@ MO_DLLIMPORT
MO_DLLIMPORT - On a symbol operand "FOO", this indicates that the reference is actually to the "__imp...
Definition: X86BaseInfo.h:537
@ MO_GOTTPOFF
MO_GOTTPOFF - On a symbol operand this indicates that the immediate is the offset of the GOT entry wi...
Definition: X86BaseInfo.h:491
@ MO_SECREL
MO_SECREL - On a symbol operand this indicates that the immediate is the offset from beginning of sec...
Definition: X86BaseInfo.h:565
@ MO_DTPOFF
MO_DTPOFF - On a symbol operand this indicates that the immediate is the offset of the GOT entry with...
Definition: X86BaseInfo.h:516
@ MO_PIC_BASE_OFFSET
MO_PIC_BASE_OFFSET - On a symbol operand this indicates that the immediate should get the value of th...
Definition: X86BaseInfo.h:420
@ MO_TLSLDM
MO_TLSLDM - On a symbol operand this indicates that the immediate is the offset of the GOT entry with...
Definition: X86BaseInfo.h:483
@ MO_GOTPCREL
MO_GOTPCREL - On a symbol operand this indicates that the immediate is offset to the GOT entry for th...
Definition: X86BaseInfo.h:442
bool isKMasked(uint64_t TSFlags)
Definition: X86BaseInfo.h:1231
@ AddrScaleAmt
Definition: X86BaseInfo.h:33
@ AddrSegmentReg
AddrSegmentReg - The operand # of the segment in the memory operand.
Definition: X86BaseInfo.h:38
@ AddrIndexReg
Definition: X86BaseInfo.h:34
@ AddrNumOperands
AddrNumOperands - Total number of operands in a memory reference.
Definition: X86BaseInfo.h:41
@ IP_HAS_AD_SIZE
Definition: X86BaseInfo.h:59
@ IP_HAS_REPEAT
Definition: X86BaseInfo.h:61
std::optional< const char * > toString(const std::optional< DWARFFormValue > &V)
Take an optional DWARFFormValue and try to extract a string value from it.
This is an optimization pass for GlobalISel generic memory operations.
Definition: AddressRanges.h:18
auto drop_begin(T &&RangeOrContainer, size_t N=1)
Return a range covering RangeOrContainer with the first N elements excluded.
Definition: STLExtras.h:413
void DecodeVPERMILPMask(unsigned NumElts, unsigned ScalarBits, ArrayRef< uint64_t > RawMask, const APInt &UndefElts, SmallVectorImpl< int > &ShuffleMask)
Decode a VPERMILPD/VPERMILPS variable mask from a raw array of constants.
MCRegister getX86SubSuperRegister(MCRegister Reg, unsigned Size, bool High=false)
void DecodeVPERMIL2PMask(unsigned NumElts, unsigned ScalarBits, unsigned M2Z, ArrayRef< uint64_t > RawMask, const APInt &UndefElts, SmallVectorImpl< int > &ShuffleMask)
Decode a VPERMIL2PD/VPERMIL2PS variable mask from a raw array of constants.
void report_fatal_error(Error Err, bool gen_crash_diag=true)
Report a serious error, calling any installed error handler.
Definition: Error.cpp:145
@ SM_SentinelUndef
@ SM_SentinelZero
raw_fd_ostream & errs()
This returns a reference to a raw_ostream for standard error.
void DecodeVPPERMMask(ArrayRef< uint64_t > RawMask, const APInt &UndefElts, SmallVectorImpl< int > &ShuffleMask)
Decode a VPPERM mask from a raw array of constants such as from BUILD_VECTOR.
void getAddressSanitizerParams(const Triple &TargetTriple, int LongSize, bool IsKasan, uint64_t *ShadowBase, int *MappingScale, bool *OrShadowOffset)
void DecodePSHUFBMask(ArrayRef< uint64_t > RawMask, const APInt &UndefElts, SmallVectorImpl< int > &ShuffleMask)
Decode a PSHUFB mask from a raw array of constants such as from BUILD_VECTOR.
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
Definition: BitVector.h:860
#define N
A RAII helper which defines a region of instructions which can't have padding added between them for ...
void changeAndComment(bool b)
NoAutoPaddingScope(MCStreamer &OS)
const bool OldAllowAutoPadding
This struct is a compact representation of a valid (non-zero power of two) alignment.
Definition: Alignment.h:39