LLVM 18.0.0git
X86EvexToVex.cpp
Go to the documentation of this file.
1//===- X86EvexToVex.cpp ---------------------------------------------------===//
2// Compress EVEX instructions to VEX encoding when possible to reduce code size
3//
4// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
5// See https://llvm.org/LICENSE.txt for license information.
6// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
7//
8//===----------------------------------------------------------------------===//
9//
10/// \file
11/// This file defines the pass that goes over all AVX-512 instructions which
12/// are encoded using the EVEX prefix and if possible replaces them by their
13/// corresponding VEX encoding which is usually shorter by 2 bytes.
14/// EVEX instructions may be encoded via the VEX prefix when the AVX-512
15/// instruction has a corresponding AVX/AVX2 opcode, when vector length
16/// accessed by instruction is less than 512 bits and when it does not use
17// the xmm or the mask registers or xmm/ymm registers with indexes higher
18// than 15.
19/// The pass applies code reduction on the generated code for AVX-512 instrs.
20//
21//===----------------------------------------------------------------------===//
22
25#include "X86.h"
26#include "X86InstrInfo.h"
27#include "X86Subtarget.h"
28#include "llvm/ADT/StringRef.h"
33#include "llvm/MC/MCInstrDesc.h"
34#include "llvm/Pass.h"
35#include <atomic>
36#include <cassert>
37#include <cstdint>
38
39using namespace llvm;
40
41// Including the generated EVEX2VEX tables.
45
47 return EvexOpc < RHS.EvexOpc;
48 }
49
51 unsigned Opc) {
52 return TE.EvexOpc < Opc;
53 }
54};
55#include "X86GenEVEX2VEXTables.inc"
56
57#define EVEX2VEX_DESC "Compressing EVEX instrs to VEX encoding when possible"
58#define EVEX2VEX_NAME "x86-evex-to-vex-compress"
59
60#define DEBUG_TYPE EVEX2VEX_NAME
61
62namespace {
63
64class EvexToVexInstPass : public MachineFunctionPass {
65public:
66 static char ID;
67 EvexToVexInstPass() : MachineFunctionPass(ID) {}
68 StringRef getPassName() const override { return EVEX2VEX_DESC; }
69
70 /// Loop over all of the basic blocks, replacing EVEX instructions
71 /// by equivalent VEX instructions when possible for reducing code size.
72 bool runOnMachineFunction(MachineFunction &MF) override;
73
74 // This pass runs after regalloc and doesn't support VReg operands.
77 MachineFunctionProperties::Property::NoVRegs);
78 }
79};
80
81} // end anonymous namespace
82
83char EvexToVexInstPass::ID = 0;
84
86 auto isHiRegIdx = [](unsigned Reg) {
87 // Check for XMM register with indexes between 16 - 31.
88 if (Reg >= X86::XMM16 && Reg <= X86::XMM31)
89 return true;
90 // Check for YMM register with indexes between 16 - 31.
91 if (Reg >= X86::YMM16 && Reg <= X86::YMM31)
92 return true;
93 // Check for GPR with indexes between 16 - 31.
95 return true;
96 return false;
97 };
98
99 // Check that operands are not ZMM regs or
100 // XMM/YMM regs with hi indexes between 16 - 31.
101 for (const MachineOperand &MO : MI.explicit_operands()) {
102 if (!MO.isReg())
103 continue;
104
105 Register Reg = MO.getReg();
106 assert(!X86II::isZMMReg(Reg) &&
107 "ZMM instructions should not be in the EVEX->VEX tables");
108 if (isHiRegIdx(Reg))
109 return true;
110 }
111
112 return false;
113}
114
115static bool checkVEXInstPredicate(unsigned EvexOpc, const X86Subtarget &ST) {
116 switch (EvexOpc) {
117 default:
118 return true;
119 case X86::VCVTNEPS2BF16Z128rm:
120 case X86::VCVTNEPS2BF16Z128rr:
121 case X86::VCVTNEPS2BF16Z256rm:
122 case X86::VCVTNEPS2BF16Z256rr:
123 return ST.hasAVXNECONVERT();
124 case X86::VPDPBUSDSZ128m:
125 case X86::VPDPBUSDSZ128r:
126 case X86::VPDPBUSDSZ256m:
127 case X86::VPDPBUSDSZ256r:
128 case X86::VPDPBUSDZ128m:
129 case X86::VPDPBUSDZ128r:
130 case X86::VPDPBUSDZ256m:
131 case X86::VPDPBUSDZ256r:
132 case X86::VPDPWSSDSZ128m:
133 case X86::VPDPWSSDSZ128r:
134 case X86::VPDPWSSDSZ256m:
135 case X86::VPDPWSSDSZ256r:
136 case X86::VPDPWSSDZ128m:
137 case X86::VPDPWSSDZ128r:
138 case X86::VPDPWSSDZ256m:
139 case X86::VPDPWSSDZ256r:
140 return ST.hasAVXVNNI();
141 case X86::VPMADD52HUQZ128m:
142 case X86::VPMADD52HUQZ128r:
143 case X86::VPMADD52HUQZ256m:
144 case X86::VPMADD52HUQZ256r:
145 case X86::VPMADD52LUQZ128m:
146 case X86::VPMADD52LUQZ128r:
147 case X86::VPMADD52LUQZ256m:
148 case X86::VPMADD52LUQZ256r:
149 return ST.hasAVXIFMA();
150 }
151}
152
153// Do any custom cleanup needed to finalize the conversion.
154static bool performCustomAdjustments(MachineInstr &MI, unsigned VexOpc) {
155 (void)VexOpc;
156 unsigned Opc = MI.getOpcode();
157 switch (Opc) {
158 case X86::VALIGNDZ128rri:
159 case X86::VALIGNDZ128rmi:
160 case X86::VALIGNQZ128rri:
161 case X86::VALIGNQZ128rmi: {
162 assert((VexOpc == X86::VPALIGNRrri || VexOpc == X86::VPALIGNRrmi) &&
163 "Unexpected new opcode!");
164 unsigned Scale =
165 (Opc == X86::VALIGNQZ128rri || Opc == X86::VALIGNQZ128rmi) ? 8 : 4;
166 MachineOperand &Imm = MI.getOperand(MI.getNumExplicitOperands() - 1);
167 Imm.setImm(Imm.getImm() * Scale);
168 break;
169 }
170 case X86::VSHUFF32X4Z256rmi:
171 case X86::VSHUFF32X4Z256rri:
172 case X86::VSHUFF64X2Z256rmi:
173 case X86::VSHUFF64X2Z256rri:
174 case X86::VSHUFI32X4Z256rmi:
175 case X86::VSHUFI32X4Z256rri:
176 case X86::VSHUFI64X2Z256rmi:
177 case X86::VSHUFI64X2Z256rri: {
178 assert((VexOpc == X86::VPERM2F128rr || VexOpc == X86::VPERM2I128rr ||
179 VexOpc == X86::VPERM2F128rm || VexOpc == X86::VPERM2I128rm) &&
180 "Unexpected new opcode!");
181 MachineOperand &Imm = MI.getOperand(MI.getNumExplicitOperands() - 1);
182 int64_t ImmVal = Imm.getImm();
183 // Set bit 5, move bit 1 to bit 4, copy bit 0.
184 Imm.setImm(0x20 | ((ImmVal & 2) << 3) | (ImmVal & 1));
185 break;
186 }
187 case X86::VRNDSCALEPDZ128rri:
188 case X86::VRNDSCALEPDZ128rmi:
189 case X86::VRNDSCALEPSZ128rri:
190 case X86::VRNDSCALEPSZ128rmi:
191 case X86::VRNDSCALEPDZ256rri:
192 case X86::VRNDSCALEPDZ256rmi:
193 case X86::VRNDSCALEPSZ256rri:
194 case X86::VRNDSCALEPSZ256rmi:
195 case X86::VRNDSCALESDZr:
196 case X86::VRNDSCALESDZm:
197 case X86::VRNDSCALESSZr:
198 case X86::VRNDSCALESSZm:
199 case X86::VRNDSCALESDZr_Int:
200 case X86::VRNDSCALESDZm_Int:
201 case X86::VRNDSCALESSZr_Int:
202 case X86::VRNDSCALESSZm_Int:
203 const MachineOperand &Imm = MI.getOperand(MI.getNumExplicitOperands()-1);
204 int64_t ImmVal = Imm.getImm();
205 // Ensure that only bits 3:0 of the immediate are used.
206 if ((ImmVal & 0xf) != ImmVal)
207 return false;
208 break;
209 }
210
211 return true;
212}
213
214// For EVEX instructions that can be encoded using VEX encoding
215// replace them by the VEX encoding in order to reduce size.
217 // VEX format.
218 // # of bytes: 0,2,3 1 1 0,1 0,1,2,4 0,1
219 // [Prefixes] [VEX] OPCODE ModR/M [SIB] [DISP] [IMM]
220 //
221 // EVEX format.
222 // # of bytes: 4 1 1 1 4 / 1 1
223 // [Prefixes] EVEX Opcode ModR/M [SIB] [Disp32] / [Disp8*N] [Immediate]
224 const MCInstrDesc &Desc = MI.getDesc();
225
226 // Check for EVEX instructions only.
227 if ((Desc.TSFlags & X86II::EncodingMask) != X86II::EVEX)
228 return false;
229
230 // Check for EVEX instructions with mask or broadcast as in these cases
231 // the EVEX prefix is needed in order to carry this information
232 // thus preventing the transformation to VEX encoding.
233 if (Desc.TSFlags & (X86II::EVEX_K | X86II::EVEX_B))
234 return false;
235
236 // Check for EVEX instructions with L2 set. These instructions are 512-bits
237 // and can't be converted to VEX.
238 if (Desc.TSFlags & X86II::EVEX_L2)
239 return false;
240
241 // Use the VEX.L bit to select the 128 or 256-bit table.
243 (Desc.TSFlags & X86II::VEX_L) ? ArrayRef(X86EvexToVex256CompressTable)
244 : ArrayRef(X86EvexToVex128CompressTable);
245
246 unsigned EvexOpc = MI.getOpcode();
247 const auto *I = llvm::lower_bound(Table, EvexOpc);
248 if (I == Table.end() || I->EvexOpc != EvexOpc)
249 return false;
250
252 return false;
253 if (!checkVEXInstPredicate(EvexOpc, ST))
254 return false;
255 if (!performCustomAdjustments(MI, I->VexOpc))
256 return false;
257
258 MI.setDesc(ST.getInstrInfo()->get(I->VexOpc));
259 MI.setAsmPrinterFlag(X86::AC_EVEX_2_VEX);
260 return true;
261}
262
263bool EvexToVexInstPass::runOnMachineFunction(MachineFunction &MF) {
264#ifndef NDEBUG
265 // Make sure the tables are sorted.
266 static std::atomic<bool> TableChecked(false);
267 if (!TableChecked.load(std::memory_order_relaxed)) {
268 assert(llvm::is_sorted(X86EvexToVex128CompressTable) &&
269 "X86EvexToVex128CompressTable is not sorted!");
270 assert(llvm::is_sorted(X86EvexToVex256CompressTable) &&
271 "X86EvexToVex256CompressTable is not sorted!");
272 TableChecked.store(true, std::memory_order_relaxed);
273 }
274#endif
276 if (!ST.hasAVX512())
277 return false;
278
279 bool Changed = false;
280
281 /// Go over all basic blocks in function and replace
282 /// EVEX encoded instrs by VEX encoding when possible.
283 for (MachineBasicBlock &MBB : MF) {
284 // Traverse the basic block.
285 for (MachineInstr &MI : MBB)
286 Changed |= CompressEvexToVexImpl(MI, ST);
287 }
288
289 return Changed;
290}
291
292INITIALIZE_PASS(EvexToVexInstPass, EVEX2VEX_NAME, EVEX2VEX_DESC, false, false)
293
295 return new EvexToVexInstPass();
296}
MachineBasicBlock & MBB
IRTranslator LLVM IR MI
#define I(x, y, z)
Definition: MD5.cpp:58
#define INITIALIZE_PASS(passName, arg, name, cfg, analysis)
Definition: PassSupport.h:38
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
static bool checkVEXInstPredicate(unsigned EvexOpc, const X86Subtarget &ST)
static bool performCustomAdjustments(MachineInstr &MI, unsigned VexOpc)
#define EVEX2VEX_NAME
static bool CompressEvexToVexImpl(MachineInstr &MI, const X86Subtarget &ST)
#define EVEX2VEX_DESC
static bool usesExtendedRegister(const MachineInstr &MI)
Value * RHS
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
Definition: ArrayRef.h:41
iterator end() const
Definition: ArrayRef.h:154
FunctionPass class - This class is used to implement most global optimizations.
Definition: Pass.h:311
Describe properties that are true of each instruction in the target description file.
Definition: MCInstrDesc.h:198
MachineFunctionPass - This class adapts the FunctionPass interface to allow convenient creation of pa...
virtual bool runOnMachineFunction(MachineFunction &MF)=0
runOnMachineFunction - This method must be overloaded to perform the desired machine code transformat...
virtual MachineFunctionProperties getRequiredProperties() const
Properties which a MachineFunction may have at a given point in time.
MachineFunctionProperties & set(Property P)
const TargetSubtargetInfo & getSubtarget() const
getSubtarget - Return the subtarget for which this machine code is being compiled.
Representation of each machine instruction.
Definition: MachineInstr.h:68
MachineOperand class - Representation of each machine instruction operand.
virtual StringRef getPassName() const
getPassName - Return a nice clean name for a pass.
Definition: Pass.cpp:81
Wrapper class representing virtual and physical registers.
Definition: Register.h:19
StringRef - Represent a constant reference to a string, i.e.
Definition: StringRef.h:50
unsigned ID
LLVM IR allows to use arbitrary numbers as calling convention identifiers.
Definition: CallingConv.h:24
bool isZMMReg(unsigned RegNo)
Definition: X86BaseInfo.h:1165
@ EVEX
EVEX - Specifies that this instruction use EVEX form which provides syntax support up to 32 512-bit r...
Definition: X86BaseInfo.h:829
bool isApxExtendedReg(unsigned RegNo)
Definition: X86BaseInfo.h:1171
This is an optimization pass for GlobalISel generic memory operations.
Definition: AddressRanges.h:18
FunctionPass * createX86EvexToVexInsts()
This pass replaces EVEX encoded of AVX-512 instructiosn by VEX encoding when possible in order to red...
bool is_sorted(R &&Range, Compare C)
Wrapper function around std::is_sorted to check if elements in a range R are sorted with respect to a...
Definition: STLExtras.h:1906
auto lower_bound(R &&Range, T &&Value)
Provide wrappers to std::lower_bound which take ranges instead of having to pass begin/end explicitly...
Definition: STLExtras.h:1945
friend bool operator<(const X86EvexToVexCompressTableEntry &TE, unsigned Opc)
bool operator<(const X86EvexToVexCompressTableEntry &RHS) const
Description of the encoding of one expression Op.