LLVM 18.0.0git
X86ISelLowering.cpp
Go to the documentation of this file.
1//===-- X86ISelLowering.cpp - X86 DAG Lowering Implementation -------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file defines the interfaces that X86 uses to lower LLVM code into a
10// selection DAG.
11//
12//===----------------------------------------------------------------------===//
13
14#include "X86ISelLowering.h"
15#include "MCTargetDesc/X86ShuffleDecode.h"
16#include "X86.h"
17#include "X86CallingConv.h"
18#include "X86FrameLowering.h"
19#include "X86InstrBuilder.h"
20#include "X86IntrinsicsInfo.h"
21#include "X86MachineFunctionInfo.h"
22#include "X86TargetMachine.h"
23#include "X86TargetObjectFile.h"
24#include "llvm/ADT/SmallBitVector.h"
25#include "llvm/ADT/SmallSet.h"
26#include "llvm/ADT/Statistic.h"
27#include "llvm/ADT/StringExtras.h"
28#include "llvm/ADT/StringSwitch.h"
29#include "llvm/Analysis/BlockFrequencyInfo.h"
30#include "llvm/Analysis/ObjCARCUtil.h"
31#include "llvm/Analysis/ProfileSummaryInfo.h"
32#include "llvm/Analysis/VectorUtils.h"
33#include "llvm/CodeGen/IntrinsicLowering.h"
34#include "llvm/CodeGen/MachineFrameInfo.h"
35#include "llvm/CodeGen/MachineFunction.h"
36#include "llvm/CodeGen/MachineInstrBuilder.h"
37#include "llvm/CodeGen/MachineJumpTableInfo.h"
38#include "llvm/CodeGen/MachineLoopInfo.h"
39#include "llvm/CodeGen/MachineModuleInfo.h"
40#include "llvm/CodeGen/MachineRegisterInfo.h"
41#include "llvm/CodeGen/TargetLowering.h"
42#include "llvm/CodeGen/WinEHFuncInfo.h"
43#include "llvm/IR/CallingConv.h"
44#include "llvm/IR/Constants.h"
45#include "llvm/IR/DerivedTypes.h"
46#include "llvm/IR/EHPersonalities.h"
47#include "llvm/IR/Function.h"
48#include "llvm/IR/GlobalAlias.h"
49#include "llvm/IR/GlobalVariable.h"
50#include "llvm/IR/IRBuilder.h"
51#include "llvm/IR/Instructions.h"
52#include "llvm/IR/Intrinsics.h"
53#include "llvm/IR/PatternMatch.h"
54#include "llvm/MC/MCAsmInfo.h"
55#include "llvm/MC/MCContext.h"
56#include "llvm/MC/MCExpr.h"
57#include "llvm/MC/MCSymbol.h"
58#include "llvm/Support/CommandLine.h"
59#include "llvm/Support/Debug.h"
60#include "llvm/Support/ErrorHandling.h"
61#include "llvm/Support/KnownBits.h"
62#include "llvm/Support/MathExtras.h"
63#include "llvm/Target/TargetOptions.h"
64#include <algorithm>
65#include <bitset>
66#include <cctype>
67#include <numeric>
68using namespace llvm;
69
70#define DEBUG_TYPE "x86-isel"
71
72static cl::opt<int> ExperimentalPrefInnermostLoopAlignment(
73 "x86-experimental-pref-innermost-loop-alignment", cl::init(4),
74 cl::desc(
75 "Sets the preferable loop alignment for experiments (as log2 bytes) "
76 "for innermost loops only. If specified, this option overrides "
77 "alignment set by x86-experimental-pref-loop-alignment."),
78 cl::Hidden);
79
80static cl::opt<bool> MulConstantOptimization(
81 "mul-constant-optimization", cl::init(true),
82 cl::desc("Replace 'mul x, Const' with more effective instructions like "
83 "SHIFT, LEA, etc."),
84 cl::Hidden);
85
86static cl::opt<bool> ExperimentalUnorderedISEL(
87 "x86-experimental-unordered-atomic-isel", cl::init(false),
88 cl::desc("Use LoadSDNode and StoreSDNode instead of "
89 "AtomicSDNode for unordered atomic loads and "
90 "stores respectively."),
91 cl::Hidden);
92
93X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
94 const X86Subtarget &STI)
95 : TargetLowering(TM), Subtarget(STI) {
96 bool UseX87 = !Subtarget.useSoftFloat() && Subtarget.hasX87();
97 MVT PtrVT = MVT::getIntegerVT(TM.getPointerSizeInBits(0));
98
99 // Set up the TargetLowering object.
100
101 // X86 is weird. It always uses i8 for shift amounts and setcc results.
102 setBooleanContents(ZeroOrOneBooleanContent);
103 // X86-SSE is even stranger. It uses -1 or 0 for vector masks.
104 setBooleanVectorContents(ZeroOrNegativeOneBooleanContent);
105
106 // For 64-bit, since we have so many registers, use the ILP scheduler.
107 // For 32-bit, use the register pressure specific scheduling.
108 // For Atom, always use ILP scheduling.
109 if (Subtarget.isAtom())
110 setSchedulingPreference(Sched::ILP);
111 else if (Subtarget.is64Bit())
112 setSchedulingPreference(Sched::ILP);
113 else
114 setSchedulingPreference(Sched::RegPressure);
115 const X86RegisterInfo *RegInfo = Subtarget.getRegisterInfo();
116 setStackPointerRegisterToSaveRestore(RegInfo->getStackRegister());
117
118 // Bypass expensive divides and use cheaper ones.
119 if (TM.getOptLevel() >= CodeGenOptLevel::Default) {
120 if (Subtarget.hasSlowDivide32())
121 addBypassSlowDiv(32, 8);
122 if (Subtarget.hasSlowDivide64() && Subtarget.is64Bit())
123 addBypassSlowDiv(64, 32);
124 }
125
126 // Setup Windows compiler runtime calls.
127 if (Subtarget.isTargetWindowsMSVC() || Subtarget.isTargetWindowsItanium()) {
128 static const struct {
129 const RTLIB::Libcall Op;
130 const char * const Name;
131 const CallingConv::ID CC;
132 } LibraryCalls[] = {
133 { RTLIB::SDIV_I64, "_alldiv", CallingConv::X86_StdCall },
134 { RTLIB::UDIV_I64, "_aulldiv", CallingConv::X86_StdCall },
135 { RTLIB::SREM_I64, "_allrem", CallingConv::X86_StdCall },
136 { RTLIB::UREM_I64, "_aullrem", CallingConv::X86_StdCall },
137 { RTLIB::MUL_I64, "_allmul", CallingConv::X86_StdCall },
138 };
139
140 for (const auto &LC : LibraryCalls) {
141 setLibcallName(LC.Op, LC.Name);
142 setLibcallCallingConv(LC.Op, LC.CC);
143 }
144 }
145
146 if (Subtarget.getTargetTriple().isOSMSVCRT()) {
147 // MSVCRT doesn't have powi; fall back to pow
148 setLibcallName(RTLIB::POWI_F32, nullptr);
149 setLibcallName(RTLIB::POWI_F64, nullptr);
150 }
151
152 // If we don't have cmpxchg8b(meaing this is a 386/486), limit atomic size to
153 // 32 bits so the AtomicExpandPass will expand it so we don't need cmpxchg8b.
154 // FIXME: Should we be limiting the atomic size on other configs? Default is
155 // 1024.
156 if (!Subtarget.canUseCMPXCHG8B())
157 setMaxAtomicSizeInBitsSupported(32);
158
159 setMaxDivRemBitWidthSupported(Subtarget.is64Bit() ? 128 : 64);
160
161 setMaxLargeFPConvertBitWidthSupported(128);
162
163 // Set up the register classes.
164 addRegisterClass(MVT::i8, &X86::GR8RegClass);
165 addRegisterClass(MVT::i16, &X86::GR16RegClass);
166 addRegisterClass(MVT::i32, &X86::GR32RegClass);
167 if (Subtarget.is64Bit())
168 addRegisterClass(MVT::i64, &X86::GR64RegClass);
169
170 for (MVT VT : MVT::integer_valuetypes())
171 setLoadExtAction(ISD::SEXTLOAD, VT, MVT::i1, Promote);
172
173 // We don't accept any truncstore of integer registers.
174 setTruncStoreAction(MVT::i64, MVT::i32, Expand);
175 setTruncStoreAction(MVT::i64, MVT::i16, Expand);
176 setTruncStoreAction(MVT::i64, MVT::i8 , Expand);
177 setTruncStoreAction(MVT::i32, MVT::i16, Expand);
178 setTruncStoreAction(MVT::i32, MVT::i8 , Expand);
179 setTruncStoreAction(MVT::i16, MVT::i8, Expand);
180
181 setTruncStoreAction(MVT::f64, MVT::f32, Expand);
182
183 // SETOEQ and SETUNE require checking two conditions.
184 for (auto VT : {MVT::f32, MVT::f64, MVT::f80}) {
185 setCondCodeAction(ISD::SETOEQ, VT, Expand);
186 setCondCodeAction(ISD::SETUNE, VT, Expand);
187 }
188
189 // Integer absolute.
190 if (Subtarget.canUseCMOV()) {
191 setOperationAction(ISD::ABS , MVT::i16 , Custom);
192 setOperationAction(ISD::ABS , MVT::i32 , Custom);
193 if (Subtarget.is64Bit())
194 setOperationAction(ISD::ABS , MVT::i64 , Custom);
195 }
196
197 // Absolute difference.
198 for (auto Op : {ISD::ABDS, ISD::ABDU}) {
199 setOperationAction(Op , MVT::i8 , Custom);
200 setOperationAction(Op , MVT::i16 , Custom);
201 setOperationAction(Op , MVT::i32 , Custom);
202 if (Subtarget.is64Bit())
203 setOperationAction(Op , MVT::i64 , Custom);
204 }
205
206 // Signed saturation subtraction.
207 setOperationAction(ISD::SSUBSAT , MVT::i8 , Custom);
208 setOperationAction(ISD::SSUBSAT , MVT::i16 , Custom);
209 setOperationAction(ISD::SSUBSAT , MVT::i32 , Custom);
210 if (Subtarget.is64Bit())
211 setOperationAction(ISD::SSUBSAT , MVT::i64 , Custom);
212
213 // Funnel shifts.
214 for (auto ShiftOp : {ISD::FSHL, ISD::FSHR}) {
215 // For slow shld targets we only lower for code size.
216 LegalizeAction ShiftDoubleAction = Subtarget.isSHLDSlow() ? Custom : Legal;
217
218 setOperationAction(ShiftOp , MVT::i8 , Custom);
219 setOperationAction(ShiftOp , MVT::i16 , Custom);
220 setOperationAction(ShiftOp , MVT::i32 , ShiftDoubleAction);
221 if (Subtarget.is64Bit())
222 setOperationAction(ShiftOp , MVT::i64 , ShiftDoubleAction);
223 }
224
225 if (!Subtarget.useSoftFloat()) {
226 // Promote all UINT_TO_FP to larger SINT_TO_FP's, as X86 doesn't have this
227 // operation.
228 setOperationAction(ISD::UINT_TO_FP, MVT::i8, Promote);
229 setOperationAction(ISD::STRICT_UINT_TO_FP, MVT::i8, Promote);
230 setOperationAction(ISD::UINT_TO_FP, MVT::i16, Promote);
231 setOperationAction(ISD::STRICT_UINT_TO_FP, MVT::i16, Promote);
232 // We have an algorithm for SSE2, and we turn this into a 64-bit
233 // FILD or VCVTUSI2SS/SD for other targets.
234 setOperationAction(ISD::UINT_TO_FP, MVT::i32, Custom);
235 setOperationAction(ISD::STRICT_UINT_TO_FP, MVT::i32, Custom);
236 // We have an algorithm for SSE2->double, and we turn this into a
237 // 64-bit FILD followed by conditional FADD for other targets.
238 setOperationAction(ISD::UINT_TO_FP, MVT::i64, Custom);
239 setOperationAction(ISD::STRICT_UINT_TO_FP, MVT::i64, Custom);
240
241 // Promote i8 SINT_TO_FP to larger SINT_TO_FP's, as X86 doesn't have
242 // this operation.
243 setOperationAction(ISD::SINT_TO_FP, MVT::i8, Promote);
244 setOperationAction(ISD::STRICT_SINT_TO_FP, MVT::i8, Promote);
245 // SSE has no i16 to fp conversion, only i32. We promote in the handler
246 // to allow f80 to use i16 and f64 to use i16 with sse1 only
247 setOperationAction(ISD::SINT_TO_FP, MVT::i16, Custom);
248 setOperationAction(ISD::STRICT_SINT_TO_FP, MVT::i16, Custom);
249 // f32 and f64 cases are Legal with SSE1/SSE2, f80 case is not
250 setOperationAction(ISD::SINT_TO_FP, MVT::i32, Custom);
251 setOperationAction(ISD::STRICT_SINT_TO_FP, MVT::i32, Custom);
252 // In 32-bit mode these are custom lowered. In 64-bit mode F32 and F64
253 // are Legal, f80 is custom lowered.
254 setOperationAction(ISD::SINT_TO_FP, MVT::i64, Custom);
255 setOperationAction(ISD::STRICT_SINT_TO_FP, MVT::i64, Custom);
256
257 // Promote i8 FP_TO_SINT to larger FP_TO_SINTS's, as X86 doesn't have
258 // this operation.
259 setOperationAction(ISD::FP_TO_SINT, MVT::i8, Promote);
260 // FIXME: This doesn't generate invalid exception when it should. PR44019.
261 setOperationAction(ISD::STRICT_FP_TO_SINT, MVT::i8, Promote);
262 setOperationAction(ISD::FP_TO_SINT, MVT::i16, Custom);
263 setOperationAction(ISD::STRICT_FP_TO_SINT, MVT::i16, Custom);
264 setOperationAction(ISD::FP_TO_SINT, MVT::i32, Custom);
265 setOperationAction(ISD::STRICT_FP_TO_SINT, MVT::i32, Custom);
266 // In 32-bit mode these are custom lowered. In 64-bit mode F32 and F64
267 // are Legal, f80 is custom lowered.
268 setOperationAction(ISD::FP_TO_SINT, MVT::i64, Custom);
269 setOperationAction(ISD::STRICT_FP_TO_SINT, MVT::i64, Custom);
270
271 // Handle FP_TO_UINT by promoting the destination to a larger signed
272 // conversion.
273 setOperationAction(ISD::FP_TO_UINT, MVT::i8, Promote);
274 // FIXME: This doesn't generate invalid exception when it should. PR44019.
275 setOperationAction(ISD::STRICT_FP_TO_UINT, MVT::i8, Promote);
276 setOperationAction(ISD::FP_TO_UINT, MVT::i16, Promote);
277 // FIXME: This doesn't generate invalid exception when it should. PR44019.
278 setOperationAction(ISD::STRICT_FP_TO_UINT, MVT::i16, Promote);
279 setOperationAction(ISD::FP_TO_UINT, MVT::i32, Custom);
280 setOperationAction(ISD::STRICT_FP_TO_UINT, MVT::i32, Custom);
281 setOperationAction(ISD::FP_TO_UINT, MVT::i64, Custom);
282 setOperationAction(ISD::STRICT_FP_TO_UINT, MVT::i64, Custom);
283
284 setOperationAction(ISD::LRINT, MVT::f32, Custom);
285 setOperationAction(ISD::LRINT, MVT::f64, Custom);
286 setOperationAction(ISD::LLRINT, MVT::f32, Custom);
287 setOperationAction(ISD::LLRINT, MVT::f64, Custom);
288
289 if (!Subtarget.is64Bit()) {
290 setOperationAction(ISD::LRINT, MVT::i64, Custom);
291 setOperationAction(ISD::LLRINT, MVT::i64, Custom);
292 }
293 }
294
295 if (Subtarget.hasSSE2()) {
296 // Custom lowering for saturating float to int conversions.
297 // We handle promotion to larger result types manually.
298 for (MVT VT : { MVT::i8, MVT::i16, MVT::i32 }) {
299 setOperationAction(ISD::FP_TO_UINT_SAT, VT, Custom);
300 setOperationAction(ISD::FP_TO_SINT_SAT, VT, Custom);
301 }
302 if (Subtarget.is64Bit()) {
303 setOperationAction(ISD::FP_TO_UINT_SAT, MVT::i64, Custom);
304 setOperationAction(ISD::FP_TO_SINT_SAT, MVT::i64, Custom);
305 }
306 }
307
308 // Handle address space casts between mixed sized pointers.
309 setOperationAction(ISD::ADDRSPACECAST, MVT::i32, Custom);
310 setOperationAction(ISD::ADDRSPACECAST, MVT::i64, Custom);
311
312 // TODO: when we have SSE, these could be more efficient, by using movd/movq.
313 if (!Subtarget.hasSSE2()) {
314 setOperationAction(ISD::BITCAST , MVT::f32 , Expand);
315 setOperationAction(ISD::BITCAST , MVT::i32 , Expand);
316 if (Subtarget.is64Bit()) {
317 setOperationAction(ISD::BITCAST , MVT::f64 , Expand);
318 // Without SSE, i64->f64 goes through memory.
319 setOperationAction(ISD::BITCAST , MVT::i64 , Expand);
320 }
321 } else if (!Subtarget.is64Bit())
322 setOperationAction(ISD::BITCAST , MVT::i64 , Custom);
323
324 // Scalar integer divide and remainder are lowered to use operations that
325 // produce two results, to match the available instructions. This exposes
326 // the two-result form to trivial CSE, which is able to combine x/y and x%y
327 // into a single instruction.
328 //
329 // Scalar integer multiply-high is also lowered to use two-result
330 // operations, to match the available instructions. However, plain multiply
331 // (low) operations are left as Legal, as there are single-result
332 // instructions for this in x86. Using the two-result multiply instructions
333 // when both high and low results are needed must be arranged by dagcombine.
334 for (auto VT : { MVT::i8, MVT::i16, MVT::i32, MVT::i64 }) {
335 setOperationAction(ISD::MULHS, VT, Expand);
336 setOperationAction(ISD::MULHU, VT, Expand);
337 setOperationAction(ISD::SDIV, VT, Expand);
338 setOperationAction(ISD::UDIV, VT, Expand);
339 setOperationAction(ISD::SREM, VT, Expand);
340 setOperationAction(ISD::UREM, VT, Expand);
341 }
342
343 setOperationAction(ISD::BR_JT , MVT::Other, Expand);
344 setOperationAction(ISD::BRCOND , MVT::Other, Custom);
345 for (auto VT : { MVT::f32, MVT::f64, MVT::f80, MVT::f128,
346 MVT::i8, MVT::i16, MVT::i32, MVT::i64 }) {
347 setOperationAction(ISD::BR_CC, VT, Expand);
348 setOperationAction(ISD::SELECT_CC, VT, Expand);
349 }
350 if (Subtarget.is64Bit())
351 setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i32, Legal);
352 setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i16 , Legal);
353 setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i8 , Legal);
354 setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i1 , Expand);
355
356 setOperationAction(ISD::FREM , MVT::f32 , Expand);
357 setOperationAction(ISD::FREM , MVT::f64 , Expand);
358 setOperationAction(ISD::FREM , MVT::f80 , Expand);
359 setOperationAction(ISD::FREM , MVT::f128 , Expand);
360
361 if (!Subtarget.useSoftFloat() && Subtarget.hasX87()) {
362 setOperationAction(ISD::GET_ROUNDING , MVT::i32 , Custom);
363 setOperationAction(ISD::SET_ROUNDING , MVT::Other, Custom);
364 setOperationAction(ISD::GET_FPENV_MEM , MVT::Other, Custom);
365 setOperationAction(ISD::SET_FPENV_MEM , MVT::Other, Custom);
366 setOperationAction(ISD::RESET_FPENV , MVT::Other, Custom);
367 }
368
369 // Promote the i8 variants and force them on up to i32 which has a shorter
370 // encoding.
371 setOperationPromotedToType(ISD::CTTZ , MVT::i8 , MVT::i32);
372 setOperationPromotedToType(ISD::CTTZ_ZERO_UNDEF, MVT::i8 , MVT::i32);
373 // Promoted i16. tzcntw has a false dependency on Intel CPUs. For BSF, we emit
374 // a REP prefix to encode it as TZCNT for modern CPUs so it makes sense to
375 // promote that too.
376 setOperationPromotedToType(ISD::CTTZ , MVT::i16 , MVT::i32);
377 setOperationPromotedToType(ISD::CTTZ_ZERO_UNDEF, MVT::i16 , MVT::i32);
378
379 if (!Subtarget.hasBMI()) {
380 setOperationAction(ISD::CTTZ , MVT::i32 , Custom);
381 setOperationAction(ISD::CTTZ_ZERO_UNDEF, MVT::i32 , Legal);
382 if (Subtarget.is64Bit()) {
383 setOperationAction(ISD::CTTZ , MVT::i64 , Custom);
384 setOperationAction(ISD::CTTZ_ZERO_UNDEF, MVT::i64, Legal);
385 }
386 }
387
388 if (Subtarget.hasLZCNT()) {
389 // When promoting the i8 variants, force them to i32 for a shorter
390 // encoding.
391 setOperationPromotedToType(ISD::CTLZ , MVT::i8 , MVT::i32);
392 setOperationPromotedToType(ISD::CTLZ_ZERO_UNDEF, MVT::i8 , MVT::i32);
393 } else {
394 for (auto VT : {MVT::i8, MVT::i16, MVT::i32, MVT::i64}) {
395 if (VT == MVT::i64 && !Subtarget.is64Bit())
396 continue;
397 setOperationAction(ISD::CTLZ , VT, Custom);
398 setOperationAction(ISD::CTLZ_ZERO_UNDEF, VT, Custom);
399 }
400 }
401
402 for (auto Op : {ISD::FP16_TO_FP, ISD::STRICT_FP16_TO_FP, ISD::FP_TO_FP16,
403 ISD::STRICT_FP_TO_FP16}) {
404 // Special handling for half-precision floating point conversions.
405 // If we don't have F16C support, then lower half float conversions
406 // into library calls.
407 setOperationAction(
408 Op, MVT::f32,
409 (!Subtarget.useSoftFloat() && Subtarget.hasF16C()) ? Custom : Expand);
410 // There's never any support for operations beyond MVT::f32.
411 setOperationAction(Op, MVT::f64, Expand);
412 setOperationAction(Op, MVT::f80, Expand);
413 setOperationAction(Op, MVT::f128, Expand);
414 }
415
416 for (MVT VT : {MVT::f32, MVT::f64, MVT::f80, MVT::f128}) {
417 setLoadExtAction(ISD::EXTLOAD, VT, MVT::f16, Expand);
418 setLoadExtAction(ISD::EXTLOAD, VT, MVT::bf16, Expand);
419 setTruncStoreAction(VT, MVT::f16, Expand);
420 setTruncStoreAction(VT, MVT::bf16, Expand);
421
422 setOperationAction(ISD::BF16_TO_FP, VT, Expand);
423 setOperationAction(ISD::FP_TO_BF16, VT, Custom);
424 }
425
426 setOperationAction(ISD::PARITY, MVT::i8, Custom);
427 setOperationAction(ISD::PARITY, MVT::i16, Custom);
428 setOperationAction(ISD::PARITY, MVT::i32, Custom);
429 if (Subtarget.is64Bit())
430 setOperationAction(ISD::PARITY, MVT::i64, Custom);
431 if (Subtarget.hasPOPCNT()) {
432 setOperationPromotedToType(ISD::CTPOP, MVT::i8, MVT::i32);
433 // popcntw is longer to encode than popcntl and also has a false dependency
434 // on the dest that popcntl hasn't had since Cannon Lake.
435 setOperationPromotedToType(ISD::CTPOP, MVT::i16, MVT::i32);
436 } else {
437 setOperationAction(ISD::CTPOP , MVT::i8 , Expand);
438 setOperationAction(ISD::CTPOP , MVT::i16 , Expand);
439 setOperationAction(ISD::CTPOP , MVT::i32 , Expand);
440 if (Subtarget.is64Bit())
441 setOperationAction(ISD::CTPOP , MVT::i64 , Expand);
442 else
443 setOperationAction(ISD::CTPOP , MVT::i64 , Custom);
444 }
445
446 setOperationAction(ISD::READCYCLECOUNTER , MVT::i64 , Custom);
447
448 if (!Subtarget.hasMOVBE())
449 setOperationAction(ISD::BSWAP , MVT::i16 , Expand);
450
451 // X86 wants to expand cmov itself.
452 for (auto VT : { MVT::f32, MVT::f64, MVT::f80, MVT::f128 }) {
453 setOperationAction(ISD::SELECT, VT, Custom);
454 setOperationAction(ISD::SETCC, VT, Custom);
455 setOperationAction(ISD::STRICT_FSETCC, VT, Custom);
456 setOperationAction(ISD::STRICT_FSETCCS, VT, Custom);
457 }
458 for (auto VT : { MVT::i8, MVT::i16, MVT::i32, MVT::i64 }) {
459 if (VT == MVT::i64 && !Subtarget.is64Bit())
460 continue;
461 setOperationAction(ISD::SELECT, VT, Custom);
462 setOperationAction(ISD::SETCC, VT, Custom);
463 }
464
465 // Custom action for SELECT MMX and expand action for SELECT_CC MMX
466 setOperationAction(ISD::SELECT, MVT::x86mmx, Custom);
467 setOperationAction(ISD::SELECT_CC, MVT::x86mmx, Expand);
468
469 setOperationAction(ISD::EH_RETURN , MVT::Other, Custom);
470 // NOTE: EH_SJLJ_SETJMP/_LONGJMP are not recommended, since
471 // LLVM/Clang supports zero-cost DWARF and SEH exception handling.
472 setOperationAction(ISD::EH_SJLJ_SETJMP, MVT::i32, Custom);
473 setOperationAction(ISD::EH_SJLJ_LONGJMP, MVT::Other, Custom);
474 setOperationAction(ISD::EH_SJLJ_SETUP_DISPATCH, MVT::Other, Custom);
475 if (TM.Options.ExceptionModel == ExceptionHandling::SjLj)
476 setLibcallName(RTLIB::UNWIND_RESUME, "_Unwind_SjLj_Resume");
477
478 // Darwin ABI issue.
479 for (auto VT : { MVT::i32, MVT::i64 }) {
480 if (VT == MVT::i64 && !Subtarget.is64Bit())
481 continue;
482 setOperationAction(ISD::ConstantPool , VT, Custom);
483 setOperationAction(ISD::JumpTable , VT, Custom);
484 setOperationAction(ISD::GlobalAddress , VT, Custom);
485 setOperationAction(ISD::GlobalTLSAddress, VT, Custom);
486 setOperationAction(ISD::ExternalSymbol , VT, Custom);
487 setOperationAction(ISD::BlockAddress , VT, Custom);
488 }
489
490 // 64-bit shl, sra, srl (iff 32-bit x86)
491 for (auto VT : { MVT::i32, MVT::i64 }) {
492 if (VT == MVT::i64 && !Subtarget.is64Bit())
493 continue;
494 setOperationAction(ISD::SHL_PARTS, VT, Custom);
495 setOperationAction(ISD::SRA_PARTS, VT, Custom);
496 setOperationAction(ISD::SRL_PARTS, VT, Custom);
497 }
498
499 if (Subtarget.hasSSEPrefetch() || Subtarget.hasThreeDNow())
500 setOperationAction(ISD::PREFETCH , MVT::Other, Custom);
501
502 setOperationAction(ISD::ATOMIC_FENCE , MVT::Other, Custom);
503
504 // Expand certain atomics
505 for (auto VT : { MVT::i8, MVT::i16, MVT::i32, MVT::i64 }) {
506 setOperationAction(ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS, VT, Custom);
507 setOperationAction(ISD::ATOMIC_LOAD_SUB, VT, Custom);
508 setOperationAction(ISD::ATOMIC_LOAD_ADD, VT, Custom);
509 setOperationAction(ISD::ATOMIC_LOAD_OR, VT, Custom);
510 setOperationAction(ISD::ATOMIC_LOAD_XOR, VT, Custom);
511 setOperationAction(ISD::ATOMIC_LOAD_AND, VT, Custom);
512 setOperationAction(ISD::ATOMIC_STORE, VT, Custom);
513 }
514
515 if (!Subtarget.is64Bit())
516 setOperationAction(ISD::ATOMIC_LOAD, MVT::i64, Custom);
517
518 if (Subtarget.canUseCMPXCHG16B())
519 setOperationAction(ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS, MVT::i128, Custom);
520
521 // FIXME - use subtarget debug flags
522 if (!Subtarget.isTargetDarwin() && !Subtarget.isTargetELF() &&
523 !Subtarget.isTargetCygMing() && !Subtarget.isTargetWin64() &&
524 TM.Options.ExceptionModel != ExceptionHandling::SjLj) {
525 setOperationAction(ISD::EH_LABEL, MVT::Other, Expand);
526 }
527
528 setOperationAction(ISD::FRAME_TO_ARGS_OFFSET, MVT::i32, Custom);
529 setOperationAction(ISD::FRAME_TO_ARGS_OFFSET, MVT::i64, Custom);
530
531 setOperationAction(ISD::INIT_TRAMPOLINE, MVT::Other, Custom);
532 setOperationAction(ISD::ADJUST_TRAMPOLINE, MVT::Other, Custom);
533
534 setOperationAction(ISD::TRAP, MVT::Other, Legal);
535 setOperationAction(ISD::DEBUGTRAP, MVT::Other, Legal);
536 if (Subtarget.isTargetPS())
537 setOperationAction(ISD::UBSANTRAP, MVT::Other, Expand);
538 else
539 setOperationAction(ISD::UBSANTRAP, MVT::Other, Legal);
540
541 // VASTART needs to be custom lowered to use the VarArgsFrameIndex
542 setOperationAction(ISD::VASTART , MVT::Other, Custom);
543 setOperationAction(ISD::VAEND , MVT::Other, Expand);
544 bool Is64Bit = Subtarget.is64Bit();
545 setOperationAction(ISD::VAARG, MVT::Other, Is64Bit ? Custom : Expand);
546 setOperationAction(ISD::VACOPY, MVT::Other, Is64Bit ? Custom : Expand);
547
548 setOperationAction(ISD::STACKSAVE, MVT::Other, Expand);
549 setOperationAction(ISD::STACKRESTORE, MVT::Other, Expand);
550
551 setOperationAction(ISD::DYNAMIC_STACKALLOC, PtrVT, Custom);
552
553 // GC_TRANSITION_START and GC_TRANSITION_END need custom lowering.
554 setOperationAction(ISD::GC_TRANSITION_START, MVT::Other, Custom);
555 setOperationAction(ISD::GC_TRANSITION_END, MVT::Other, Custom);
556
557 setOperationAction(ISD::STRICT_FP_EXTEND, MVT::f64, Legal);
558
559 auto setF16Action = [&] (MVT VT, LegalizeAction Action) {
560 setOperationAction(ISD::FABS, VT, Action);
561 setOperationAction(ISD::FNEG, VT, Action);
562 setOperationAction(ISD::FCOPYSIGN, VT, Expand);
563 setOperationAction(ISD::FREM, VT, Action);
564 setOperationAction(ISD::FMA, VT, Action);
565 setOperationAction(ISD::FMINNUM, VT, Action);
566 setOperationAction(ISD::FMAXNUM, VT, Action);
567 setOperationAction(ISD::FMINIMUM, VT, Action);
568 setOperationAction(ISD::FMAXIMUM, VT, Action);
569 setOperationAction(ISD::FSIN, VT, Action);
570 setOperationAction(ISD::FCOS, VT, Action);
571 setOperationAction(ISD::FSINCOS, VT, Action);
572 setOperationAction(ISD::FSQRT, VT, Action);
573 setOperationAction(ISD::FPOW, VT, Action);
574 setOperationAction(ISD::FLOG, VT, Action);
575 setOperationAction(ISD::FLOG2, VT, Action);
576 setOperationAction(ISD::FLOG10, VT, Action);
577 setOperationAction(ISD::FEXP, VT, Action);
578 setOperationAction(ISD::FEXP2, VT, Action);
579 setOperationAction(ISD::FEXP10, VT, Action);
580 setOperationAction(ISD::FCEIL, VT, Action);
581 setOperationAction(ISD::FFLOOR, VT, Action);
582 setOperationAction(ISD::FNEARBYINT, VT, Action);
583 setOperationAction(ISD::FRINT, VT, Action);
584 setOperationAction(ISD::BR_CC, VT, Action);
585 setOperationAction(ISD::SETCC, VT, Action);
586 setOperationAction(ISD::SELECT, VT, Custom);
587 setOperationAction(ISD::SELECT_CC, VT, Action);
588 setOperationAction(ISD::FROUND, VT, Action);
589 setOperationAction(ISD::FROUNDEVEN, VT, Action);
590 setOperationAction(ISD::FTRUNC, VT, Action);
591 setOperationAction(ISD::FLDEXP, VT, Action);
592 };
593
594 if (!Subtarget.useSoftFloat() && Subtarget.hasSSE2()) {
595 // f16, f32 and f64 use SSE.
596 // Set up the FP register classes.
597 addRegisterClass(MVT::f16, Subtarget.hasAVX512() ? &X86::FR16XRegClass
598 : &X86::FR16RegClass);
599 addRegisterClass(MVT::f32, Subtarget.hasAVX512() ? &X86::FR32XRegClass
600 : &X86::FR32RegClass);
601 addRegisterClass(MVT::f64, Subtarget.hasAVX512() ? &X86::FR64XRegClass
602 : &X86::FR64RegClass);
603
604 // Disable f32->f64 extload as we can only generate this in one instruction
605 // under optsize. So its easier to pattern match (fpext (load)) for that
606 // case instead of needing to emit 2 instructions for extload in the
607 // non-optsize case.
608 setLoadExtAction(ISD::EXTLOAD, MVT::f64, MVT::f32, Expand);
609
610 for (auto VT : { MVT::f32, MVT::f64 }) {
611 // Use ANDPD to simulate FABS.
612 setOperationAction(ISD::FABS, VT, Custom);
613
614 // Use XORP to simulate FNEG.
615 setOperationAction(ISD::FNEG, VT, Custom);
616
617 // Use ANDPD and ORPD to simulate FCOPYSIGN.
618 setOperationAction(ISD::FCOPYSIGN, VT, Custom);
619
620 // These might be better off as horizontal vector ops.
621 setOperationAction(ISD::FADD, VT, Custom);
622 setOperationAction(ISD::FSUB, VT, Custom);
623
624 // We don't support sin/cos/fmod
625 setOperationAction(ISD::FSIN , VT, Expand);
626 setOperationAction(ISD::FCOS , VT, Expand);
627 setOperationAction(ISD::FSINCOS, VT, Expand);
628 }
629
630 // Half type will be promoted by default.
631 setF16Action(MVT::f16, Promote);
632 setOperationAction(ISD::FADD, MVT::f16, Promote);
633 setOperationAction(ISD::FSUB, MVT::f16, Promote);
634 setOperationAction(ISD::FMUL, MVT::f16, Promote);
635 setOperationAction(ISD::FDIV, MVT::f16, Promote);
636 setOperationAction(ISD::FP_ROUND, MVT::f16, Custom);
637 setOperationAction(ISD::FP_EXTEND, MVT::f32, Custom);
638 setOperationAction(ISD::FP_EXTEND, MVT::f64, Custom);
639
640 setOperationAction(ISD::STRICT_FADD, MVT::f16, Promote);
641 setOperationAction(ISD::STRICT_FSUB, MVT::f16, Promote);
642 setOperationAction(ISD::STRICT_FMUL, MVT::f16, Promote);
643 setOperationAction(ISD::STRICT_FDIV, MVT::f16, Promote);
644 setOperationAction(ISD::STRICT_FMA, MVT::f16, Promote);
645 setOperationAction(ISD::STRICT_FMINNUM, MVT::f16, Promote);
646 setOperationAction(ISD::STRICT_FMAXNUM, MVT::f16, Promote);
647 setOperationAction(ISD::STRICT_FMINIMUM, MVT::f16, Promote);
648 setOperationAction(ISD::STRICT_FMAXIMUM, MVT::f16, Promote);
649 setOperationAction(ISD::STRICT_FSQRT, MVT::f16, Promote);
650 setOperationAction(ISD::STRICT_FPOW, MVT::f16, Promote);
651 setOperationAction(ISD::STRICT_FLDEXP, MVT::f16, Promote);
652 setOperationAction(ISD::STRICT_FLOG, MVT::f16, Promote);
653 setOperationAction(ISD::STRICT_FLOG2, MVT::f16, Promote);
654 setOperationAction(ISD::STRICT_FLOG10, MVT::f16, Promote);
655 setOperationAction(ISD::STRICT_FEXP, MVT::f16, Promote);
656 setOperationAction(ISD::STRICT_FEXP2, MVT::f16, Promote);
657 setOperationAction(ISD::STRICT_FCEIL, MVT::f16, Promote);
658 setOperationAction(ISD::STRICT_FFLOOR, MVT::f16, Promote);
659 setOperationAction(ISD::STRICT_FNEARBYINT, MVT::f16, Promote);
660 setOperationAction(ISD::STRICT_FRINT, MVT::f16, Promote);
661 setOperationAction(ISD::STRICT_FSETCC, MVT::f16, Promote);
662 setOperationAction(ISD::STRICT_FSETCCS, MVT::f16, Promote);
663 setOperationAction(ISD::STRICT_FROUND, MVT::f16, Promote);
664 setOperationAction(ISD::STRICT_FROUNDEVEN, MVT::f16, Promote);
665 setOperationAction(ISD::STRICT_FTRUNC, MVT::f16, Promote);
666 setOperationAction(ISD::STRICT_FP_ROUND, MVT::f16, Custom);
667 setOperationAction(ISD::STRICT_FP_EXTEND, MVT::f32, Custom);
668 setOperationAction(ISD::STRICT_FP_EXTEND, MVT::f64, Custom);
669
670 setLibcallName(RTLIB::FPROUND_F32_F16, "__truncsfhf2");
671 setLibcallName(RTLIB::FPEXT_F16_F32, "__extendhfsf2");
672
673 // Lower this to MOVMSK plus an AND.
674 setOperationAction(ISD::FGETSIGN, MVT::i64, Custom);
675 setOperationAction(ISD::FGETSIGN, MVT::i32, Custom);
676
677 } else if (!Subtarget.useSoftFloat() && Subtarget.hasSSE1() &&
678 (UseX87 || Is64Bit)) {
679 // Use SSE for f32, x87 for f64.
680 // Set up the FP register classes.
681 addRegisterClass(MVT::f32, &X86::FR32RegClass);
682 if (UseX87)
683 addRegisterClass(MVT::f64, &X86::RFP64RegClass);
684
685 // Use ANDPS to simulate FABS.
686 setOperationAction(ISD::FABS , MVT::f32, Custom);
687
688 // Use XORP to simulate FNEG.
689 setOperationAction(ISD::FNEG , MVT::f32, Custom);
690
691 if (UseX87)
692 setOperationAction(ISD::UNDEF, MVT::f64, Expand);
693
694 // Use ANDPS and ORPS to simulate FCOPYSIGN.
695 if (UseX87)
696 setOperationAction(ISD::FCOPYSIGN, MVT::f64, Expand);
697 setOperationAction(ISD::FCOPYSIGN, MVT::f32, Custom);
698
699 // We don't support sin/cos/fmod
700 setOperationAction(ISD::FSIN , MVT::f32, Expand);
701 setOperationAction(ISD::FCOS , MVT::f32, Expand);
702 setOperationAction(ISD::FSINCOS, MVT::f32, Expand);
703
704 if (UseX87) {
705 // Always expand sin/cos functions even though x87 has an instruction.
706 setOperationAction(ISD::FSIN, MVT::f64, Expand);
707 setOperationAction(ISD::FCOS, MVT::f64, Expand);
708 setOperationAction(ISD::FSINCOS, MVT::f64, Expand);
709 }
710 } else if (UseX87) {
711 // f32 and f64 in x87.
712 // Set up the FP register classes.
713 addRegisterClass(MVT::f64, &X86::RFP64RegClass);
714 addRegisterClass(MVT::f32, &X86::RFP32RegClass);
715
716 for (auto VT : { MVT::f32, MVT::f64 }) {
717 setOperationAction(ISD::UNDEF, VT, Expand);
718 setOperationAction(ISD::FCOPYSIGN, VT, Expand);
719
720 // Always expand sin/cos functions even though x87 has an instruction.
721 setOperationAction(ISD::FSIN , VT, Expand);
722 setOperationAction(ISD::FCOS , VT, Expand);
723 setOperationAction(ISD::FSINCOS, VT, Expand);
724 }
725 }
726
727 // Expand FP32 immediates into loads from the stack, save special cases.
728 if (isTypeLegal(MVT::f32)) {
729 if (UseX87 && (getRegClassFor(MVT::f32) == &X86::RFP32RegClass)) {
730 addLegalFPImmediate(APFloat(+0.0f)); // FLD0
731 addLegalFPImmediate(APFloat(+1.0f)); // FLD1
732 addLegalFPImmediate(APFloat(-0.0f)); // FLD0/FCHS
733 addLegalFPImmediate(APFloat(-1.0f)); // FLD1/FCHS
734 } else // SSE immediates.
735 addLegalFPImmediate(APFloat(+0.0f)); // xorps
736 }
737 // Expand FP64 immediates into loads from the stack, save special cases.
738 if (isTypeLegal(MVT::f64)) {
739 if (UseX87 && getRegClassFor(MVT::f64) == &X86::RFP64RegClass) {
740 addLegalFPImmediate(APFloat(+0.0)); // FLD0
741 addLegalFPImmediate(APFloat(+1.0)); // FLD1
742 addLegalFPImmediate(APFloat(-0.0)); // FLD0/FCHS
743 addLegalFPImmediate(APFloat(-1.0)); // FLD1/FCHS
744 } else // SSE immediates.
745 addLegalFPImmediate(APFloat(+0.0)); // xorpd
746 }
747 // Support fp16 0 immediate.
748 if (isTypeLegal(MVT::f16))
749 addLegalFPImmediate(APFloat::getZero(APFloat::IEEEhalf()));
750
751 // Handle constrained floating-point operations of scalar.
752 setOperationAction(ISD::STRICT_FADD, MVT::f32, Legal);
753 setOperationAction(ISD::STRICT_FADD, MVT::f64, Legal);
754 setOperationAction(ISD::STRICT_FSUB, MVT::f32, Legal);
755 setOperationAction(ISD::STRICT_FSUB, MVT::f64, Legal);
756 setOperationAction(ISD::STRICT_FMUL, MVT::f32, Legal);
757 setOperationAction(ISD::STRICT_FMUL, MVT::f64, Legal);
758 setOperationAction(ISD::STRICT_FDIV, MVT::f32, Legal);
759 setOperationAction(ISD::STRICT_FDIV, MVT::f64, Legal);
760 setOperationAction(ISD::STRICT_FP_ROUND, MVT::f32, Legal);
761 setOperationAction(ISD::STRICT_FP_ROUND, MVT::f64, Legal);
762 setOperationAction(ISD::STRICT_FSQRT, MVT::f32, Legal);
763 setOperationAction(ISD::STRICT_FSQRT, MVT::f64, Legal);
764
765 // We don't support FMA.
766 setOperationAction(ISD::FMA, MVT::f64, Expand);
767 setOperationAction(ISD::FMA, MVT::f32, Expand);
768
769 // f80 always uses X87.
770 if (UseX87) {
771 addRegisterClass(MVT::f80, &X86::RFP80RegClass);
772 setOperationAction(ISD::UNDEF, MVT::f80, Expand);
773 setOperationAction(ISD::FCOPYSIGN, MVT::f80, Expand);
774 {
775 APFloat TmpFlt = APFloat::getZero(APFloat::x87DoubleExtended());
776 addLegalFPImmediate(TmpFlt); // FLD0
777 TmpFlt.changeSign();
778 addLegalFPImmediate(TmpFlt); // FLD0/FCHS
779
780 bool ignored;
781 APFloat TmpFlt2(+1.0);
782 TmpFlt2.convert(APFloat::x87DoubleExtended(), APFloat::rmNearestTiesToEven,
783 &ignored);
784 addLegalFPImmediate(TmpFlt2); // FLD1
785 TmpFlt2.changeSign();
786 addLegalFPImmediate(TmpFlt2); // FLD1/FCHS
787 }
788
789 // Always expand sin/cos functions even though x87 has an instruction.
790 setOperationAction(ISD::FSIN , MVT::f80, Expand);
791 setOperationAction(ISD::FCOS , MVT::f80, Expand);
792 setOperationAction(ISD::FSINCOS, MVT::f80, Expand);
793
794 setOperationAction(ISD::FFLOOR, MVT::f80, Expand);
795 setOperationAction(ISD::FCEIL, MVT::f80, Expand);
796 setOperationAction(ISD::FTRUNC, MVT::f80, Expand);
797 setOperationAction(ISD::FRINT, MVT::f80, Expand);
798 setOperationAction(ISD::FNEARBYINT, MVT::f80, Expand);
799 setOperationAction(ISD::FMA, MVT::f80, Expand);
800 setOperationAction(ISD::LROUND, MVT::f80, Expand);
801 setOperationAction(ISD::LLROUND, MVT::f80, Expand);
802 setOperationAction(ISD::LRINT, MVT::f80, Custom);
803 setOperationAction(ISD::LLRINT, MVT::f80, Custom);
804
805 // Handle constrained floating-point operations of scalar.
806 setOperationAction(ISD::STRICT_FADD , MVT::f80, Legal);
807 setOperationAction(ISD::STRICT_FSUB , MVT::f80, Legal);
808 setOperationAction(ISD::STRICT_FMUL , MVT::f80, Legal);
809 setOperationAction(ISD::STRICT_FDIV , MVT::f80, Legal);
810 setOperationAction(ISD::STRICT_FSQRT , MVT::f80, Legal);
811 if (isTypeLegal(MVT::f16)) {
812 setOperationAction(ISD::FP_EXTEND, MVT::f80, Custom);
813 setOperationAction(ISD::STRICT_FP_EXTEND, MVT::f80, Custom);
814 } else {
815 setOperationAction(ISD::STRICT_FP_EXTEND, MVT::f80, Legal);
816 }
817 // FIXME: When the target is 64-bit, STRICT_FP_ROUND will be overwritten
818 // as Custom.
819 setOperationAction(ISD::STRICT_FP_ROUND, MVT::f80, Legal);
820 }
821
822 // f128 uses xmm registers, but most operations require libcalls.
823 if (!Subtarget.useSoftFloat() && Subtarget.is64Bit() && Subtarget.hasSSE1()) {
824 addRegisterClass(MVT::f128, Subtarget.hasVLX() ? &X86::VR128XRegClass
825 : &X86::VR128RegClass);
826
827 addLegalFPImmediate(APFloat::getZero(APFloat::IEEEquad())); // xorps
828
829 setOperationAction(ISD::FADD, MVT::f128, LibCall);
830 setOperationAction(ISD::STRICT_FADD, MVT::f128, LibCall);
831 setOperationAction(ISD::FSUB, MVT::f128, LibCall);
832 setOperationAction(ISD::STRICT_FSUB, MVT::f128, LibCall);
833 setOperationAction(ISD::FDIV, MVT::f128, LibCall);
834 setOperationAction(ISD::STRICT_FDIV, MVT::f128, LibCall);
835 setOperationAction(ISD::FMUL, MVT::f128, LibCall);
836 setOperationAction(ISD::STRICT_FMUL, MVT::f128, LibCall);
837 setOperationAction(ISD::FMA, MVT::f128, LibCall);
838 setOperationAction(ISD::STRICT_FMA, MVT::f128, LibCall);
839
840 setOperationAction(ISD::FABS, MVT::f128, Custom);
841 setOperationAction(ISD::FNEG, MVT::f128, Custom);
842 setOperationAction(ISD::FCOPYSIGN, MVT::f128, Custom);
843
844 setOperationAction(ISD::FSIN, MVT::f128, LibCall);
845 setOperationAction(ISD::STRICT_FSIN, MVT::f128, LibCall);
846 setOperationAction(ISD::FCOS, MVT::f128, LibCall);
847 setOperationAction(ISD::STRICT_FCOS, MVT::f128, LibCall);
848 setOperationAction(ISD::FSINCOS, MVT::f128, LibCall);
849 // No STRICT_FSINCOS
850 setOperationAction(ISD::FSQRT, MVT::f128, LibCall);
851 setOperationAction(ISD::STRICT_FSQRT, MVT::f128, LibCall);
852
853 setOperationAction(ISD::FP_EXTEND, MVT::f128, Custom);
854 setOperationAction(ISD::STRICT_FP_EXTEND, MVT::f128, Custom);
855 // We need to custom handle any FP_ROUND with an f128 input, but
856 // LegalizeDAG uses the result type to know when to run a custom handler.
857 // So we have to list all legal floating point result types here.
858 if (isTypeLegal(MVT::f32)) {
859 setOperationAction(ISD::FP_ROUND, MVT::f32, Custom);
860 setOperationAction(ISD::STRICT_FP_ROUND, MVT::f32, Custom);
861 }
862 if (isTypeLegal(MVT::f64)) {
863 setOperationAction(ISD::FP_ROUND, MVT::f64, Custom);
864 setOperationAction(ISD::STRICT_FP_ROUND, MVT::f64, Custom);
865 }
866 if (isTypeLegal(MVT::f80)) {
867 setOperationAction(ISD::FP_ROUND, MVT::f80, Custom);
868 setOperationAction(ISD::STRICT_FP_ROUND, MVT::f80, Custom);
869 }
870
871 setOperationAction(ISD::SETCC, MVT::f128, Custom);
872
873 setLoadExtAction(ISD::EXTLOAD, MVT::f128, MVT::f32, Expand);
874 setLoadExtAction(ISD::EXTLOAD, MVT::f128, MVT::f64, Expand);
875 setLoadExtAction(ISD::EXTLOAD, MVT::f128, MVT::f80, Expand);
876 setTruncStoreAction(MVT::f128, MVT::f32, Expand);
877 setTruncStoreAction(MVT::f128, MVT::f64, Expand);
878 setTruncStoreAction(MVT::f128, MVT::f80, Expand);
879 }
880
881 // Always use a library call for pow.
882 setOperationAction(ISD::FPOW , MVT::f32 , Expand);
883 setOperationAction(ISD::FPOW , MVT::f64 , Expand);
884 setOperationAction(ISD::FPOW , MVT::f80 , Expand);
885 setOperationAction(ISD::FPOW , MVT::f128 , Expand);
886
887 setOperationAction(ISD::FLOG, MVT::f80, Expand);
888 setOperationAction(ISD::FLOG2, MVT::f80, Expand);
889 setOperationAction(ISD::FLOG10, MVT::f80, Expand);
890 setOperationAction(ISD::FEXP, MVT::f80, Expand);
891 setOperationAction(ISD::FEXP2, MVT::f80, Expand);
892 setOperationAction(ISD::FEXP10, MVT::f80, Expand);
893 setOperationAction(ISD::FMINNUM, MVT::f80, Expand);
894 setOperationAction(ISD::FMAXNUM, MVT::f80, Expand);
895
896 // Some FP actions are always expanded for vector types.
897 for (auto VT : { MVT::v8f16, MVT::v16f16, MVT::v32f16,
898 MVT::v4f32, MVT::v8f32, MVT::v16f32,
899 MVT::v2f64, MVT::v4f64, MVT::v8f64 }) {
900 setOperationAction(ISD::FSIN, VT, Expand);
901 setOperationAction(ISD::FSINCOS, VT, Expand);
902 setOperationAction(ISD::FCOS, VT, Expand);
903 setOperationAction(ISD::FREM, VT, Expand);
904 setOperationAction(ISD::FCOPYSIGN, VT, Expand);
905 setOperationAction(ISD::FPOW, VT, Expand);
906 setOperationAction(ISD::FLOG, VT, Expand);
907 setOperationAction(ISD::FLOG2, VT, Expand);
908 setOperationAction(ISD::FLOG10, VT, Expand);
909 setOperationAction(ISD::FEXP, VT, Expand);
910 setOperationAction(ISD::FEXP2, VT, Expand);
911 setOperationAction(ISD::FEXP10, VT, Expand);
912 }
913
914 // First set operation action for all vector types to either promote
915 // (for widening) or expand (for scalarization). Then we will selectively
916 // turn on ones that can be effectively codegen'd.
917 for (MVT VT : MVT::fixedlen_vector_valuetypes()) {
918 setOperationAction(ISD::SDIV, VT, Expand);
919 setOperationAction(ISD::UDIV, VT, Expand);
920 setOperationAction(ISD::SREM, VT, Expand);
921 setOperationAction(ISD::UREM, VT, Expand);
922 setOperationAction(ISD::EXTRACT_VECTOR_ELT, VT,Expand);
923 setOperationAction(ISD::INSERT_VECTOR_ELT, VT, Expand);
924 setOperationAction(ISD::EXTRACT_SUBVECTOR, VT,Expand);
925 setOperationAction(ISD::INSERT_SUBVECTOR, VT,Expand);
926 setOperationAction(ISD::FMA, VT, Expand);
927 setOperationAction(ISD::FFLOOR, VT, Expand);
928 setOperationAction(ISD::FCEIL, VT, Expand);
929 setOperationAction(ISD::FTRUNC, VT, Expand);
930 setOperationAction(ISD::FRINT, VT, Expand);
931 setOperationAction(ISD::FNEARBYINT, VT, Expand);
932 setOperationAction(ISD::SMUL_LOHI, VT, Expand);
933 setOperationAction(ISD::MULHS, VT, Expand);
934 setOperationAction(ISD::UMUL_LOHI, VT, Expand);
935 setOperationAction(ISD::MULHU, VT, Expand);
936 setOperationAction(ISD::SDIVREM, VT, Expand);
937 setOperationAction(ISD::UDIVREM, VT, Expand);
938 setOperationAction(ISD::CTPOP, VT, Expand);
939 setOperationAction(ISD::CTTZ, VT, Expand);
940 setOperationAction(ISD::CTLZ, VT, Expand);
941 setOperationAction(ISD::ROTL, VT, Expand);
942 setOperationAction(ISD::ROTR, VT, Expand);
943 setOperationAction(ISD::BSWAP, VT, Expand);
944 setOperationAction(ISD::SETCC, VT, Expand);
945 setOperationAction(ISD::FP_TO_UINT, VT, Expand);
946 setOperationAction(ISD::FP_TO_SINT, VT, Expand);
947 setOperationAction(ISD::UINT_TO_FP, VT, Expand);
948 setOperationAction(ISD::SINT_TO_FP, VT, Expand);
949 setOperationAction(ISD::SIGN_EXTEND_INREG, VT,Expand);
950 setOperationAction(ISD::TRUNCATE, VT, Expand);
951 setOperationAction(ISD::SIGN_EXTEND, VT, Expand);
952 setOperationAction(ISD::ZERO_EXTEND, VT, Expand);
953 setOperationAction(ISD::ANY_EXTEND, VT, Expand);
954 setOperationAction(ISD::SELECT_CC, VT, Expand);
955 for (MVT InnerVT : MVT::fixedlen_vector_valuetypes()) {
956 setTruncStoreAction(InnerVT, VT, Expand);
957
958 setLoadExtAction(ISD::SEXTLOAD, InnerVT, VT, Expand);
959 setLoadExtAction(ISD::ZEXTLOAD, InnerVT, VT, Expand);
960
961 // N.b. ISD::EXTLOAD legality is basically ignored except for i1-like
962 // types, we have to deal with them whether we ask for Expansion or not.
963 // Setting Expand causes its own optimisation problems though, so leave
964 // them legal.
965 if (VT.getVectorElementType() == MVT::i1)
966 setLoadExtAction(ISD::EXTLOAD, InnerVT, VT, Expand);
967
968 // EXTLOAD for MVT::f16 vectors is not legal because f16 vectors are
969 // split/scalarized right now.
970 if (VT.getVectorElementType() == MVT::f16 ||
971 VT.getVectorElementType() == MVT::bf16)
972 setLoadExtAction(ISD::EXTLOAD, InnerVT, VT, Expand);
973 }
974 }
975
976 // FIXME: In order to prevent SSE instructions being expanded to MMX ones
977 // with -msoft-float, disable use of MMX as well.
978 if (!Subtarget.useSoftFloat() && Subtarget.hasMMX()) {
979 addRegisterClass(MVT::x86mmx, &X86::VR64RegClass);
980 // No operations on x86mmx supported, everything uses intrinsics.
981 }
982
983 if (!Subtarget.useSoftFloat() && Subtarget.hasSSE1()) {
984 addRegisterClass(MVT::v4f32, Subtarget.hasVLX() ? &X86::VR128XRegClass
985 : &X86::VR128RegClass);
986
987 setOperationAction(ISD::FMAXIMUM, MVT::f32, Custom);
988 setOperationAction(ISD::FMINIMUM, MVT::f32, Custom);
989
990 setOperationAction(ISD::FNEG, MVT::v4f32, Custom);
991 setOperationAction(ISD::FABS, MVT::v4f32, Custom);
992 setOperationAction(ISD::FCOPYSIGN, MVT::v4f32, Custom);
993 setOperationAction(ISD::BUILD_VECTOR, MVT::v4f32, Custom);
994 setOperationAction(ISD::VECTOR_SHUFFLE, MVT::v4f32, Custom);
995 setOperationAction(ISD::VSELECT, MVT::v4f32, Custom);
996 setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v4f32, Custom);
997 setOperationAction(ISD::SELECT, MVT::v4f32, Custom);
998
999 setOperationAction(ISD::LOAD, MVT::v2f32, Custom);
1000 setOperationAction(ISD::STORE, MVT::v2f32, Custom);
1001
1002 setOperationAction(ISD::STRICT_FADD, MVT::v4f32, Legal);
1003 setOperationAction(ISD::STRICT_FSUB, MVT::v4f32, Legal);
1004 setOperationAction(ISD::STRICT_FMUL, MVT::v4f32, Legal);
1005 setOperationAction(ISD::STRICT_FDIV, MVT::v4f32, Legal);
1006 setOperationAction(ISD::STRICT_FSQRT, MVT::v4f32, Legal);
1007 }
1008
1009 if (!Subtarget.useSoftFloat() && Subtarget.hasSSE2()) {
1010 addRegisterClass(MVT::v2f64, Subtarget.hasVLX() ? &X86::VR128XRegClass
1011 : &X86::VR128RegClass);
1012
1013 // FIXME: Unfortunately, -soft-float and -no-implicit-float mean XMM
1014 // registers cannot be used even for integer operations.
1015 addRegisterClass(MVT::v16i8, Subtarget.hasVLX() ? &X86::VR128XRegClass
1016 : &X86::VR128RegClass);
1017 addRegisterClass(MVT::v8i16, Subtarget.hasVLX() ? &X86::VR128XRegClass
1018 : &X86::VR128RegClass);
1019 addRegisterClass(MVT::v8f16, Subtarget.hasVLX() ? &X86::VR128XRegClass
1020 : &X86::VR128RegClass);
1021 addRegisterClass(MVT::v4i32, Subtarget.hasVLX() ? &X86::VR128XRegClass
1022 : &X86::VR128RegClass);
1023 addRegisterClass(MVT::v2i64, Subtarget.hasVLX() ? &X86::VR128XRegClass
1024 : &X86::VR128RegClass);
1025
1026 for (auto VT : { MVT::f64, MVT::v4f32, MVT::v2f64 }) {
1027 setOperationAction(ISD::FMAXIMUM, VT, Custom);
1028 setOperationAction(ISD::FMINIMUM, VT, Custom);
1029 }
1030
1031 for (auto VT : { MVT::v2i8, MVT::v4i8, MVT::v8i8,
1032 MVT::v2i16, MVT::v4i16, MVT::v2i32 }) {
1033 setOperationAction(ISD::SDIV, VT, Custom);
1034 setOperationAction(ISD::SREM, VT, Custom);
1035 setOperationAction(ISD::UDIV, VT, Custom);
1036 setOperationAction(ISD::UREM, VT, Custom);
1037 }
1038
1039 setOperationAction(ISD::MUL, MVT::v2i8, Custom);
1040 setOperationAction(ISD::MUL, MVT::v4i8, Custom);
1041 setOperationAction(ISD::MUL, MVT::v8i8, Custom);
1042
1043 setOperationAction(ISD::MUL, MVT::v16i8, Custom);
1044 setOperationAction(ISD::MUL, MVT::v4i32, Custom);
1045 setOperationAction(ISD::MUL, MVT::v2i64, Custom);
1046 setOperationAction(ISD::MULHU, MVT::v4i32, Custom);
1047 setOperationAction(ISD::MULHS, MVT::v4i32, Custom);
1048 setOperationAction(ISD::MULHU, MVT::v16i8, Custom);
1049 setOperationAction(ISD::MULHS, MVT::v16i8, Custom);
1050 setOperationAction(ISD::MULHU, MVT::v8i16, Legal);
1051 setOperationAction(ISD::MULHS, MVT::v8i16, Legal);
1052 setOperationAction(ISD::MUL, MVT::v8i16, Legal);
1053 setOperationAction(ISD::AVGCEILU, MVT::v16i8, Legal);
1054 setOperationAction(ISD::AVGCEILU, MVT::v8i16, Legal);
1055
1056 setOperationAction(ISD::SMULO, MVT::v16i8, Custom);
1057 setOperationAction(ISD::UMULO, MVT::v16i8, Custom);
1058 setOperationAction(ISD::UMULO, MVT::v2i32, Custom);
1059
1060 setOperationAction(ISD::FNEG, MVT::v2f64, Custom);
1061 setOperationAction(ISD::FABS, MVT::v2f64, Custom);
1062 setOperationAction(ISD::FCOPYSIGN, MVT::v2f64, Custom);
1063
1064 for (auto VT : { MVT::v16i8, MVT::v8i16, MVT::v4i32, MVT::v2i64 }) {
1065 setOperationAction(ISD::SMAX, VT, VT == MVT::v8i16 ? Legal : Custom);
1066 setOperationAction(ISD::SMIN, VT, VT == MVT::v8i16 ? Legal : Custom);
1067 setOperationAction(ISD::UMAX, VT, VT == MVT::v16i8 ? Legal : Custom);
1068 setOperationAction(ISD::UMIN, VT, VT == MVT::v16i8 ? Legal : Custom);
1069 }
1070
1071 setOperationAction(ISD::ABDU, MVT::v16i8, Custom);
1072 setOperationAction(ISD::ABDS, MVT::v16i8, Custom);
1073 setOperationAction(ISD::ABDU, MVT::v8i16, Custom);
1074 setOperationAction(ISD::ABDS, MVT::v8i16, Custom);
1075 setOperationAction(ISD::ABDU, MVT::v4i32, Custom);
1076 setOperationAction(ISD::ABDS, MVT::v4i32, Custom);
1077
1078 setOperationAction(ISD::UADDSAT, MVT::v16i8, Legal);
1079 setOperationAction(ISD::SADDSAT, MVT::v16i8, Legal);
1080 setOperationAction(ISD::USUBSAT, MVT::v16i8, Legal);
1081 setOperationAction(ISD::SSUBSAT, MVT::v16i8, Legal);
1082 setOperationAction(ISD::UADDSAT, MVT::v8i16, Legal);
1083 setOperationAction(ISD::SADDSAT, MVT::v8i16, Legal);
1084 setOperationAction(ISD::USUBSAT, MVT::v8i16, Legal);
1085 setOperationAction(ISD::SSUBSAT, MVT::v8i16, Legal);
1086 setOperationAction(ISD::USUBSAT, MVT::v4i32, Custom);
1087 setOperationAction(ISD::USUBSAT, MVT::v2i64, Custom);
1088
1089 setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v16i8, Custom);
1090 setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v8i16, Custom);
1091 setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v4i32, Custom);
1092 setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v4f32, Custom);
1093
1094 for (auto VT : { MVT::v16i8, MVT::v8i16, MVT::v4i32, MVT::v2i64 }) {
1095 setOperationAction(ISD::SETCC, VT, Custom);
1096 setOperationAction(ISD::CTPOP, VT, Custom);
1097 setOperationAction(ISD::ABS, VT, Custom);
1098
1099 // The condition codes aren't legal in SSE/AVX and under AVX512 we use
1100 // setcc all the way to isel and prefer SETGT in some isel patterns.
1101 setCondCodeAction(ISD::SETLT, VT, Custom);
1102 setCondCodeAction(ISD::SETLE, VT, Custom);
1103 }
1104
1105 setOperationAction(ISD::SETCC, MVT::v2f64, Custom);
1106 setOperationAction(ISD::SETCC, MVT::v4f32, Custom);
1107 setOperationAction(ISD::STRICT_FSETCC, MVT::v2f64, Custom);
1108 setOperationAction(ISD::STRICT_FSETCC, MVT::v4f32, Custom);
1109 setOperationAction(ISD::STRICT_FSETCCS, MVT::v2f64, Custom);
1110 setOperationAction(ISD::STRICT_FSETCCS, MVT::v4f32, Custom);
1111
1112 for (auto VT : { MVT::v16i8, MVT::v8i16, MVT::v4i32 }) {
1113 setOperationAction(ISD::SCALAR_TO_VECTOR, VT, Custom);
1114 setOperationAction(ISD::BUILD_VECTOR, VT, Custom);
1115 setOperationAction(ISD::VECTOR_SHUFFLE, VT, Custom);
1116 setOperationAction(ISD::VSELECT, VT, Custom);
1117 setOperationAction(ISD::EXTRACT_VECTOR_ELT, VT, Custom);
1118 }
1119
1120 for (auto VT : { MVT::v8f16, MVT::v2f64, MVT::v2i64 }) {
1121 setOperationAction(ISD::BUILD_VECTOR, VT, Custom);
1122 setOperationAction(ISD::VECTOR_SHUFFLE, VT, Custom);
1123 setOperationAction(ISD::VSELECT, VT, Custom);
1124
1125 if (VT == MVT::v2i64 && !Subtarget.is64Bit())
1126 continue;
1127
1128 setOperationAction(ISD::INSERT_VECTOR_ELT, VT, Custom);
1129 setOperationAction(ISD::EXTRACT_VECTOR_ELT, VT, Custom);
1130 }
1131 setF16Action(MVT::v8f16, Expand);
1132 setOperationAction(ISD::FADD, MVT::v8f16, Expand);
1133 setOperationAction(ISD::FSUB, MVT::v8f16, Expand);
1134 setOperationAction(ISD::FMUL, MVT::v8f16, Expand);
1135 setOperationAction(ISD::FDIV, MVT::v8f16, Expand);
1136
1137 // Custom lower v2i64 and v2f64 selects.
1138 setOperationAction(ISD::SELECT, MVT::v2f64, Custom);
1139 setOperationAction(ISD::SELECT, MVT::v2i64, Custom);
1140 setOperationAction(ISD::SELECT, MVT::v4i32, Custom);
1141 setOperationAction(ISD::SELECT, MVT::v8i16, Custom);
1142 setOperationAction(ISD::SELECT, MVT::v8f16, Custom);
1143 setOperationAction(ISD::SELECT, MVT::v16i8, Custom);
1144
1145 setOperationAction(ISD::FP_TO_SINT, MVT::v4i32, Custom);
1146 setOperationAction(ISD::FP_TO_UINT, MVT::v4i32, Custom);
1147 setOperationAction(ISD::FP_TO_SINT, MVT::v2i32, Custom);
1148 setOperationAction(ISD::FP_TO_UINT, MVT::v2i32, Custom);
1149 setOperationAction(ISD::STRICT_FP_TO_SINT, MVT::v4i32, Custom);
1150 setOperationAction(ISD::STRICT_FP_TO_SINT, MVT::v2i32, Custom);
1151
1152 // Custom legalize these to avoid over promotion or custom promotion.
1153 for (auto VT : {MVT::v2i8, MVT::v4i8, MVT::v8i8, MVT::v2i16, MVT::v4i16}) {
1154 setOperationAction(ISD::FP_TO_SINT, VT, Custom);
1155 setOperationAction(ISD::FP_TO_UINT, VT, Custom);
1156 setOperationAction(ISD::STRICT_FP_TO_SINT, VT, Custom);
1157 setOperationAction(ISD::STRICT_FP_TO_UINT, VT, Custom);
1158 }
1159
1160 setOperationAction(ISD::SINT_TO_FP, MVT::v4i32, Custom);
1161 setOperationAction(ISD::STRICT_SINT_TO_FP, MVT::v4i32, Custom);
1162 setOperationAction(ISD::SINT_TO_FP, MVT::v2i32, Custom);
1163 setOperationAction(ISD::STRICT_SINT_TO_FP, MVT::v2i32, Custom);
1164
1165 setOperationAction(ISD::UINT_TO_FP, MVT::v2i32, Custom);
1166 setOperationAction(ISD::STRICT_UINT_TO_FP, MVT::v2i32, Custom);
1167
1168 setOperationAction(ISD::UINT_TO_FP, MVT::v4i32, Custom);
1169 setOperationAction(ISD::STRICT_UINT_TO_FP, MVT::v4i32, Custom);
1170
1171 // Fast v2f32 UINT_TO_FP( v2i32 ) custom conversion.
1172 setOperationAction(ISD::SINT_TO_FP, MVT::v2f32, Custom);
1173 setOperationAction(ISD::STRICT_SINT_TO_FP, MVT::v2f32, Custom);
1174 setOperationAction(ISD::UINT_TO_FP, MVT::v2f32, Custom);
1175 setOperationAction(ISD::STRICT_UINT_TO_FP, MVT::v2f32, Custom);
1176
1177 setOperationAction(ISD::FP_EXTEND, MVT::v2f32, Custom);
1178 setOperationAction(ISD::STRICT_FP_EXTEND, MVT::v2f32, Custom);
1179 setOperationAction(ISD::FP_ROUND, MVT::v2f32, Custom);
1180 setOperationAction(ISD::STRICT_FP_ROUND, MVT::v2f32, Custom);
1181
1182 // We want to legalize this to an f64 load rather than an i64 load on
1183 // 64-bit targets and two 32-bit loads on a 32-bit target. Similar for
1184 // store.
1185 setOperationAction(ISD::LOAD, MVT::v2i32, Custom);
1186 setOperationAction(ISD::LOAD, MVT::v4i16, Custom);
1187 setOperationAction(ISD::LOAD, MVT::v8i8, Custom);
1188 setOperationAction(ISD::STORE, MVT::v2i32, Custom);
1189 setOperationAction(ISD::STORE, MVT::v4i16, Custom);
1190 setOperationAction(ISD::STORE, MVT::v8i8, Custom);
1191
1192 // Add 32-bit vector stores to help vectorization opportunities.
1193 setOperationAction(ISD::STORE, MVT::v2i16, Custom);
1194 setOperationAction(ISD::STORE, MVT::v4i8, Custom);
1195
1196 setOperationAction(ISD::BITCAST, MVT::v2i32, Custom);
1197 setOperationAction(ISD::BITCAST, MVT::v4i16, Custom);
1198 setOperationAction(ISD::BITCAST, MVT::v8i8, Custom);
1199 if (!Subtarget.hasAVX512())
1200 setOperationAction(ISD::BITCAST, MVT::v16i1, Custom);
1201
1202 setOperationAction(ISD::SIGN_EXTEND_VECTOR_INREG, MVT::v2i64, Custom);
1203 setOperationAction(ISD::SIGN_EXTEND_VECTOR_INREG, MVT::v4i32, Custom);
1204 setOperationAction(ISD::SIGN_EXTEND_VECTOR_INREG, MVT::v8i16, Custom);
1205
1206 setOperationAction(ISD::SIGN_EXTEND, MVT::v4i64, Custom);
1207
1208 setOperationAction(ISD::TRUNCATE, MVT::v2i8, Custom);
1209 setOperationAction(ISD::TRUNCATE, MVT::v2i16, Custom);
1210 setOperationAction(ISD::TRUNCATE, MVT::v2i32, Custom);
1211 setOperationAction(ISD::TRUNCATE, MVT::v2i64, Custom);
1212 setOperationAction(ISD::TRUNCATE, MVT::v4i8, Custom);
1213 setOperationAction(ISD::TRUNCATE, MVT::v4i16, Custom);
1214 setOperationAction(ISD::TRUNCATE, MVT::v4i32, Custom);
1215 setOperationAction(ISD::TRUNCATE, MVT::v4i64, Custom);
1216 setOperationAction(ISD::TRUNCATE, MVT::v8i8, Custom);
1217 setOperationAction(ISD::TRUNCATE, MVT::v8i16, Custom);
1218 setOperationAction(ISD::TRUNCATE, MVT::v8i32, Custom);
1219 setOperationAction(ISD::TRUNCATE, MVT::v8i64, Custom);
1220 setOperationAction(ISD::TRUNCATE, MVT::v16i8, Custom);
1221 setOperationAction(ISD::TRUNCATE, MVT::v16i16, Custom);
1222 setOperationAction(ISD::TRUNCATE, MVT::v16i32, Custom);
1223 setOperationAction(ISD::TRUNCATE, MVT::v16i64, Custom);
1224
1225 // In the customized shift lowering, the legal v4i32/v2i64 cases
1226 // in AVX2 will be recognized.
1227 for (auto VT : { MVT::v16i8, MVT::v8i16, MVT::v4i32, MVT::v2i64 }) {
1228 setOperationAction(ISD::SRL, VT, Custom);
1229 setOperationAction(ISD::SHL, VT, Custom);
1230 setOperationAction(ISD::SRA, VT, Custom);
1231 if (VT == MVT::v2i64) continue;
1232 setOperationAction(ISD::ROTL, VT, Custom);
1233 setOperationAction(ISD::ROTR, VT, Custom);
1234 setOperationAction(ISD::FSHL, VT, Custom);
1235 setOperationAction(ISD::FSHR, VT, Custom);
1236 }
1237
1238 setOperationAction(ISD::STRICT_FSQRT, MVT::v2f64, Legal);
1239 setOperationAction(ISD::STRICT_FADD, MVT::v2f64, Legal);
1240 setOperationAction(ISD::STRICT_FSUB, MVT::v2f64, Legal);
1241 setOperationAction(ISD::STRICT_FMUL, MVT::v2f64, Legal);
1242 setOperationAction(ISD::STRICT_FDIV, MVT::v2f64, Legal);
1243 }
1244
1245 if (!Subtarget.useSoftFloat() && Subtarget.hasSSSE3()) {
1246 setOperationAction(ISD::ABS, MVT::v16i8, Legal);
1247 setOperationAction(ISD::ABS, MVT::v8i16, Legal);
1248 setOperationAction(ISD::ABS, MVT::v4i32, Legal);
1249 setOperationAction(ISD::BITREVERSE, MVT::v16i8, Custom);
1250 setOperationAction(ISD::CTLZ, MVT::v16i8, Custom);
1251 setOperationAction(ISD::CTLZ, MVT::v8i16, Custom);
1252 setOperationAction(ISD::CTLZ, MVT::v4i32, Custom);
1253 setOperationAction(ISD::CTLZ, MVT::v2i64, Custom);
1254
1255 // These might be better off as horizontal vector ops.
1256 setOperationAction(ISD::ADD, MVT::i16, Custom);
1257 setOperationAction(ISD::ADD, MVT::i32, Custom);
1258 setOperationAction(ISD::SUB, MVT::i16, Custom);
1259 setOperationAction(ISD::SUB, MVT::i32, Custom);
1260 }
1261
1262 if (!Subtarget.useSoftFloat() && Subtarget.hasSSE41()) {
1263 for (MVT RoundedTy : {MVT::f32, MVT::f64, MVT::v4f32, MVT::v2f64}) {
1264 setOperationAction(ISD::FFLOOR, RoundedTy, Legal);
1265 setOperationAction(ISD::STRICT_FFLOOR, RoundedTy, Legal);
1266 setOperationAction(ISD::FCEIL, RoundedTy, Legal);
1267 setOperationAction(ISD::STRICT_FCEIL, RoundedTy, Legal);
1268 setOperationAction(ISD::FTRUNC, RoundedTy, Legal);
1269 setOperationAction(ISD::STRICT_FTRUNC, RoundedTy, Legal);
1270 setOperationAction(ISD::FRINT, RoundedTy, Legal);
1271 setOperationAction(ISD::STRICT_FRINT, RoundedTy, Legal);
1272 setOperationAction(ISD::FNEARBYINT, RoundedTy, Legal);
1273 setOperationAction(ISD::STRICT_FNEARBYINT, RoundedTy, Legal);
1274 setOperationAction(ISD::FROUNDEVEN, RoundedTy, Legal);
1275 setOperationAction(ISD::STRICT_FROUNDEVEN, RoundedTy, Legal);
1276
1277 setOperationAction(ISD::FROUND, RoundedTy, Custom);
1278 }
1279
1280 setOperationAction(ISD::SMAX, MVT::v16i8, Legal);
1281 setOperationAction(ISD::SMAX, MVT::v4i32, Legal);
1282 setOperationAction(ISD::UMAX, MVT::v8i16, Legal);
1283 setOperationAction(ISD::UMAX, MVT::v4i32, Legal);
1284 setOperationAction(ISD::SMIN, MVT::v16i8, Legal);
1285 setOperationAction(ISD::SMIN, MVT::v4i32, Legal);
1286 setOperationAction(ISD::UMIN, MVT::v8i16, Legal);
1287 setOperationAction(ISD::UMIN, MVT::v4i32, Legal);
1288
1289 for (auto VT : {MVT::v16i8, MVT::v8i16, MVT::v4i32, MVT::v2i64}) {
1290 setOperationAction(ISD::ABDS, VT, Custom);
1291 setOperationAction(ISD::ABDU, VT, Custom);
1292 }
1293
1294 setOperationAction(ISD::UADDSAT, MVT::v4i32, Custom);
1295 setOperationAction(ISD::SADDSAT, MVT::v2i64, Custom);
1296 setOperationAction(ISD::SSUBSAT, MVT::v2i64, Custom);
1297
1298 // FIXME: Do we need to handle scalar-to-vector here?
1299 setOperationAction(ISD::MUL, MVT::v4i32, Legal);
1300 setOperationAction(ISD::SMULO, MVT::v2i32, Custom);
1301
1302 // We directly match byte blends in the backend as they match the VSELECT
1303 // condition form.
1304 setOperationAction(ISD::VSELECT, MVT::v16i8, Legal);
1305
1306 // SSE41 brings specific instructions for doing vector sign extend even in
1307 // cases where we don't have SRA.
1308 for (auto VT : { MVT::v8i16, MVT::v4i32, MVT::v2i64 }) {
1309 setOperationAction(ISD::SIGN_EXTEND_VECTOR_INREG, VT, Legal);
1310 setOperationAction(ISD::ZERO_EXTEND_VECTOR_INREG, VT, Legal);
1311 }
1312
1313 // SSE41 also has vector sign/zero extending loads, PMOV[SZ]X
1314 for (auto LoadExtOp : { ISD::SEXTLOAD, ISD::ZEXTLOAD }) {
1315 setLoadExtAction(LoadExtOp, MVT::v8i16, MVT::v8i8, Legal);
1316 setLoadExtAction(LoadExtOp, MVT::v4i32, MVT::v4i8, Legal);
1317 setLoadExtAction(LoadExtOp, MVT::v2i64, MVT::v2i8, Legal);
1318 setLoadExtAction(LoadExtOp, MVT::v4i32, MVT::v4i16, Legal);
1319 setLoadExtAction(LoadExtOp, MVT::v2i64, MVT::v2i16, Legal);
1320 setLoadExtAction(LoadExtOp, MVT::v2i64, MVT::v2i32, Legal);
1321 }
1322
1323 if (Subtarget.is64Bit() && !Subtarget.hasAVX512()) {
1324 // We need to scalarize v4i64->v432 uint_to_fp using cvtsi2ss, but we can
1325 // do the pre and post work in the vector domain.
1326 setOperationAction(ISD::UINT_TO_FP, MVT::v4i64, Custom);
1327 setOperationAction(ISD::STRICT_UINT_TO_FP, MVT::v4i64, Custom);
1328 // We need to mark SINT_TO_FP as Custom even though we want to expand it
1329 // so that DAG combine doesn't try to turn it into uint_to_fp.
1330 setOperationAction(ISD::SINT_TO_FP, MVT::v4i64, Custom);
1331 setOperationAction(ISD::STRICT_SINT_TO_FP, MVT::v4i64, Custom);
1332 }
1333 }
1334
1335 if (!Subtarget.useSoftFloat() && Subtarget.hasSSE42()) {
1336 setOperationAction(ISD::UADDSAT, MVT::v2i64, Custom);
1337 }
1338
1339 if (!Subtarget.useSoftFloat() && Subtarget.hasXOP()) {
1340 for (auto VT : { MVT::v16i8, MVT::v8i16, MVT::v4i32, MVT::v2i64,
1341 MVT::v32i8, MVT::v16i16, MVT::v8i32, MVT::v4i64 }) {
1342 setOperationAction(ISD::ROTL, VT, Custom);
1343 setOperationAction(ISD::ROTR, VT, Custom);
1344 }
1345
1346 // XOP can efficiently perform BITREVERSE with VPPERM.
1347 for (auto VT : { MVT::i8, MVT::i16, MVT::i32, MVT::i64 })
1348 setOperationAction(ISD::BITREVERSE, VT, Custom);
1349
1350 for (auto VT : { MVT::v16i8, MVT::v8i16, MVT::v4i32, MVT::v2i64,
1351 MVT::v32i8, MVT::v16i16, MVT::v8i32, MVT::v4i64 })
1352 setOperationAction(ISD::BITREVERSE, VT, Custom);
1353 }
1354
1355 if (!Subtarget.useSoftFloat() && Subtarget.hasAVX()) {
1356 bool HasInt256 = Subtarget.hasInt256();
1357
1358 addRegisterClass(MVT::v32i8, Subtarget.hasVLX() ? &X86::VR256XRegClass
1359 : &X86::VR256RegClass);
1360 addRegisterClass(MVT::v16i16, Subtarget.hasVLX() ? &X86::VR256XRegClass
1361 : &X86::VR256RegClass);
1362 addRegisterClass(MVT::v16f16, Subtarget.hasVLX() ? &X86::VR256XRegClass
1363 : &X86::VR256RegClass);
1364 addRegisterClass(MVT::v8i32, Subtarget.hasVLX() ? &X86::VR256XRegClass
1365 : &X86::VR256RegClass);
1366 addRegisterClass(MVT::v8f32, Subtarget.hasVLX() ? &X86::VR256XRegClass
1367 : &X86::VR256RegClass);
1368 addRegisterClass(MVT::v4i64, Subtarget.hasVLX() ? &X86::VR256XRegClass
1369 : &X86::VR256RegClass);
1370 addRegisterClass(MVT::v4f64, Subtarget.hasVLX() ? &X86::VR256XRegClass
1371 : &X86::VR256RegClass);
1372
1373 for (auto VT : { MVT::v8f32, MVT::v4f64 }) {
1374 setOperationAction(ISD::FFLOOR, VT, Legal);
1375 setOperationAction(ISD::STRICT_FFLOOR, VT, Legal);
1376 setOperationAction(ISD::FCEIL, VT, Legal);
1377 setOperationAction(ISD::STRICT_FCEIL, VT, Legal);
1378 setOperationAction(ISD::FTRUNC, VT, Legal);
1379 setOperationAction(ISD::STRICT_FTRUNC, VT, Legal);
1380 setOperationAction(ISD::FRINT, VT, Legal);
1381 setOperationAction(ISD::STRICT_FRINT, VT, Legal);
1382 setOperationAction(ISD::FNEARBYINT, VT, Legal);
1383 setOperationAction(ISD::STRICT_FNEARBYINT, VT, Legal);
1384 setOperationAction(ISD::FROUNDEVEN, VT, Legal);
1385 setOperationAction(ISD::STRICT_FROUNDEVEN, VT, Legal);
1386
1387 setOperationAction(ISD::FROUND, VT, Custom);
1388
1389 setOperationAction(ISD::FNEG, VT, Custom);
1390 setOperationAction(ISD::FABS, VT, Custom);
1391 setOperationAction(ISD::FCOPYSIGN, VT, Custom);
1392
1393 setOperationAction(ISD::FMAXIMUM, VT, Custom);
1394 setOperationAction(ISD::FMINIMUM, VT, Custom);
1395 }
1396
1397 // (fp_to_int:v8i16 (v8f32 ..)) requires the result type to be promoted
1398 // even though v8i16 is a legal type.
1399 setOperationPromotedToType(ISD::FP_TO_SINT, MVT::v8i16, MVT::v8i32);
1400 setOperationPromotedToType(ISD::FP_TO_UINT, MVT::v8i16, MVT::v8i32);
1401 setOperationPromotedToType(ISD::STRICT_FP_TO_SINT, MVT::v8i16, MVT::v8i32);
1402 setOperationPromotedToType(ISD::STRICT_FP_TO_UINT, MVT::v8i16, MVT::v8i32);
1403 setOperationAction(ISD::FP_TO_SINT, MVT::v8i32, Custom);
1404 setOperationAction(ISD::FP_TO_UINT, MVT::v8i32, Custom);
1405 setOperationAction(ISD::STRICT_FP_TO_SINT, MVT::v8i32, Custom);
1406
1407 setOperationAction(ISD::SINT_TO_FP, MVT::v8i32, Custom);
1408 setOperationAction(ISD::STRICT_SINT_TO_FP, MVT::v8i32, Custom);
1409 setOperationAction(ISD::FP_EXTEND, MVT::v8f32, Expand);
1410 setOperationAction(ISD::FP_ROUND, MVT::v8f16, Expand);
1411 setOperationAction(ISD::FP_EXTEND, MVT::v4f64, Custom);
1412 setOperationAction(ISD::STRICT_FP_EXTEND, MVT::v4f64, Custom);
1413
1414 setOperationAction(ISD::STRICT_FP_ROUND, MVT::v4f32, Legal);
1415 setOperationAction(ISD::STRICT_FADD, MVT::v8f32, Legal);
1416 setOperationAction(ISD::STRICT_FADD, MVT::v4f64, Legal);
1417 setOperationAction(ISD::STRICT_FSUB, MVT::v8f32, Legal);
1418 setOperationAction(ISD::STRICT_FSUB, MVT::v4f64, Legal);
1419 setOperationAction(ISD::STRICT_FMUL, MVT::v8f32, Legal);
1420 setOperationAction(ISD::STRICT_FMUL, MVT::v4f64, Legal);
1421 setOperationAction(ISD::STRICT_FDIV, MVT::v8f32, Legal);
1422 setOperationAction(ISD::STRICT_FDIV, MVT::v4f64, Legal);
1423 setOperationAction(ISD::STRICT_FSQRT, MVT::v8f32, Legal);
1424 setOperationAction(ISD::STRICT_FSQRT, MVT::v4f64, Legal);
1425
1426 if (!Subtarget.hasAVX512())
1427 setOperationAction(ISD::BITCAST, MVT::v32i1, Custom);
1428
1429 // In the customized shift lowering, the legal v8i32/v4i64 cases
1430 // in AVX2 will be recognized.
1431 for (auto VT : { MVT::v32i8, MVT::v16i16, MVT::v8i32, MVT::v4i64 }) {
1432 setOperationAction(ISD::SRL, VT, Custom);
1433 setOperationAction(ISD::SHL, VT, Custom);
1434 setOperationAction(ISD::SRA, VT, Custom);
1435 setOperationAction(ISD::ABDS, VT, Custom);
1436 setOperationAction(ISD::ABDU, VT, Custom);
1437 if (VT == MVT::v4i64) continue;
1438 setOperationAction(ISD::ROTL, VT, Custom);
1439 setOperationAction(ISD::ROTR, VT, Custom);
1440 setOperationAction(ISD::FSHL, VT, Custom);
1441 setOperationAction(ISD::FSHR, VT, Custom);
1442 }
1443
1444 // These types need custom splitting if their input is a 128-bit vector.
1445 setOperationAction(ISD::SIGN_EXTEND, MVT::v8i64, Custom);
1446 setOperationAction(ISD::SIGN_EXTEND, MVT::v16i32, Custom);
1447 setOperationAction(ISD::ZERO_EXTEND, MVT::v8i64, Custom);
1448 setOperationAction(ISD::ZERO_EXTEND, MVT::v16i32, Custom);
1449
1450 setOperationAction(ISD::SELECT, MVT::v4f64, Custom);
1451 setOperationAction(ISD::SELECT, MVT::v4i64, Custom);
1452 setOperationAction(ISD::SELECT, MVT::v8i32, Custom);
1453 setOperationAction(ISD::SELECT, MVT::v16i16, Custom);
1454 setOperationAction(ISD::SELECT, MVT::v16f16, Custom);
1455 setOperationAction(ISD::SELECT, MVT::v32i8, Custom);
1456 setOperationAction(ISD::SELECT, MVT::v8f32, Custom);
1457
1458 for (auto VT : { MVT::v16i16, MVT::v8i32, MVT::v4i64 }) {
1459 setOperationAction(ISD::SIGN_EXTEND, VT, Custom);
1460 setOperationAction(ISD::ZERO_EXTEND, VT, Custom);
1461 setOperationAction(ISD::ANY_EXTEND, VT, Custom);
1462 }
1463
1464 setOperationAction(ISD::TRUNCATE, MVT::v32i8, Custom);
1465 setOperationAction(ISD::TRUNCATE, MVT::v32i16, Custom);
1466 setOperationAction(ISD::TRUNCATE, MVT::v32i32, Custom);
1467 setOperationAction(ISD::TRUNCATE, MVT::v32i64, Custom);
1468
1469 setOperationAction(ISD::BITREVERSE, MVT::v32i8, Custom);
1470
1471 for (auto VT : { MVT::v32i8, MVT::v16i16, MVT::v8i32, MVT::v4i64 }) {
1472 setOperationAction(ISD::SETCC, VT, Custom);
1473 setOperationAction(ISD::CTPOP, VT, Custom);
1474 setOperationAction(ISD::CTLZ, VT, Custom);
1475
1476 // The condition codes aren't legal in SSE/AVX and under AVX512 we use
1477 // setcc all the way to isel and prefer SETGT in some isel patterns.
1478 setCondCodeAction(ISD::SETLT, VT, Custom);
1479 setCondCodeAction(ISD::SETLE, VT, Custom);
1480 }
1481
1482 setOperationAction(ISD::SETCC, MVT::v4f64, Custom);
1483 setOperationAction(ISD::SETCC, MVT::v8f32, Custom);
1484 setOperationAction(ISD::STRICT_FSETCC, MVT::v4f64, Custom);
1485 setOperationAction(ISD::STRICT_FSETCC, MVT::v8f32, Custom);
1486 setOperationAction(ISD::STRICT_FSETCCS, MVT::v4f64, Custom);
1487 setOperationAction(ISD::STRICT_FSETCCS, MVT::v8f32, Custom);
1488
1489 if (Subtarget.hasAnyFMA()) {
1490 for (auto VT : { MVT::f32, MVT::f64, MVT::v4f32, MVT::v8f32,
1491 MVT::v2f64, MVT::v4f64 }) {
1492 setOperationAction(ISD::FMA, VT, Legal);
1493 setOperationAction(ISD::STRICT_FMA, VT, Legal);
1494 }
1495 }
1496
1497 for (auto VT : { MVT::v32i8, MVT::v16i16, MVT::v8i32, MVT::v4i64 }) {
1498 setOperationAction(ISD::ADD, VT, HasInt256 ? Legal : Custom);
1499 setOperationAction(ISD::SUB, VT, HasInt256 ? Legal : Custom);
1500 }
1501
1502 setOperationAction(ISD::MUL, MVT::v4i64, Custom);
1503 setOperationAction(ISD::MUL, MVT::v8i32, HasInt256 ? Legal : Custom);
1504 setOperationAction(ISD::MUL, MVT::v16i16, HasInt256 ? Legal : Custom);
1505 setOperationAction(ISD::MUL, MVT::v32i8, Custom);
1506
1507 setOperationAction(ISD::MULHU, MVT::v8i32, Custom);
1508 setOperationAction(ISD::MULHS, MVT::v8i32, Custom);
1509 setOperationAction(ISD::MULHU, MVT::v16i16, HasInt256 ? Legal : Custom);
1510 setOperationAction(ISD::MULHS, MVT::v16i16, HasInt256 ? Legal : Custom);
1511 setOperationAction(ISD::MULHU, MVT::v32i8, Custom);
1512 setOperationAction(ISD::MULHS, MVT::v32i8, Custom);
1513 setOperationAction(ISD::AVGCEILU, MVT::v16i16, HasInt256 ? Legal : Custom);
1514 setOperationAction(ISD::AVGCEILU, MVT::v32i8, HasInt256 ? Legal : Custom);
1515
1516 setOperationAction(ISD::SMULO, MVT::v32i8, Custom);
1517 setOperationAction(ISD::UMULO, MVT::v32i8, Custom);
1518
1519 setOperationAction(ISD::ABS, MVT::v4i64, Custom);
1520 setOperationAction(ISD::SMAX, MVT::v4i64, Custom);
1521 setOperationAction(ISD::UMAX, MVT::v4i64, Custom);
1522 setOperationAction(ISD::SMIN, MVT::v4i64, Custom);
1523 setOperationAction(ISD::UMIN, MVT::v4i64, Custom);
1524
1525 setOperationAction(ISD::UADDSAT, MVT::v32i8, HasInt256 ? Legal : Custom);
1526 setOperationAction(ISD::SADDSAT, MVT::v32i8, HasInt256 ? Legal : Custom);
1527 setOperationAction(ISD::USUBSAT, MVT::v32i8, HasInt256 ? Legal : Custom);
1528 setOperationAction(ISD::SSUBSAT, MVT::v32i8, HasInt256 ? Legal : Custom);
1529 setOperationAction(ISD::UADDSAT, MVT::v16i16, HasInt256 ? Legal : Custom);
1530 setOperationAction(ISD::SADDSAT, MVT::v16i16, HasInt256 ? Legal : Custom);
1531 setOperationAction(ISD::USUBSAT, MVT::v16i16, HasInt256 ? Legal : Custom);
1532 setOperationAction(ISD::SSUBSAT, MVT::v16i16, HasInt256 ? Legal : Custom);
1533 setOperationAction(ISD::UADDSAT, MVT::v8i32, Custom);
1534 setOperationAction(ISD::USUBSAT, MVT::v8i32, Custom);
1535 setOperationAction(ISD::UADDSAT, MVT::v4i64, Custom);
1536 setOperationAction(ISD::USUBSAT, MVT::v4i64, Custom);
1537
1538 for (auto VT : { MVT::v32i8, MVT::v16i16, MVT::v8i32 }) {
1539 setOperationAction(ISD::ABS, VT, HasInt256 ? Legal : Custom);
1540 setOperationAction(ISD::SMAX, VT, HasInt256 ? Legal : Custom);
1541 setOperationAction(ISD::UMAX, VT, HasInt256 ? Legal : Custom);
1542 setOperationAction(ISD::SMIN, VT, HasInt256 ? Legal : Custom);
1543 setOperationAction(ISD::UMIN, VT, HasInt256 ? Legal : Custom);
1544 }
1545
1546 for (auto VT : {MVT::v16i16, MVT::v8i32, MVT::v4i64}) {
1547 setOperationAction(ISD::SIGN_EXTEND_VECTOR_INREG, VT, Custom);
1548 setOperationAction(ISD::ZERO_EXTEND_VECTOR_INREG, VT, Custom);
1549 }
1550
1551 if (HasInt256) {
1552 // The custom lowering for UINT_TO_FP for v8i32 becomes interesting
1553 // when we have a 256bit-wide blend with immediate.
1554 setOperationAction(ISD::UINT_TO_FP, MVT::v8i32, Custom);
1555 setOperationAction(ISD::STRICT_UINT_TO_FP, MVT::v8i32, Custom);
1556
1557 // AVX2 also has wider vector sign/zero extending loads, VPMOV[SZ]X
1558 for (auto LoadExtOp : { ISD::SEXTLOAD, ISD::ZEXTLOAD }) {
1559 setLoadExtAction(LoadExtOp, MVT::v16i16, MVT::v16i8, Legal);
1560 setLoadExtAction(LoadExtOp, MVT::v8i32, MVT::v8i8, Legal);
1561 setLoadExtAction(LoadExtOp, MVT::v4i64, MVT::v4i8, Legal);
1562 setLoadExtAction(LoadExtOp, MVT::v8i32, MVT::v8i16, Legal);
1563 setLoadExtAction(LoadExtOp, MVT::v4i64, MVT::v4i16, Legal);
1564 setLoadExtAction(LoadExtOp, MVT::v4i64, MVT::v4i32, Legal);
1565 }
1566 }
1567
1568 for (auto VT : { MVT::v4i32, MVT::v8i32, MVT::v2i64, MVT::v4i64,
1569 MVT::v4f32, MVT::v8f32, MVT::v2f64, MVT::v4f64 }) {
1570 setOperationAction(ISD::MLOAD, VT, Subtarget.hasVLX() ? Legal : Custom);
1571 setOperationAction(ISD::MSTORE, VT, Legal);
1572 }
1573
1574 // Extract subvector is special because the value type
1575 // (result) is 128-bit but the source is 256-bit wide.
1576 for (auto VT : { MVT::v16i8, MVT::v8i16, MVT::v4i32, MVT::v2i64,
1577 MVT::v8f16, MVT::v4f32, MVT::v2f64 }) {
1578 setOperationAction(ISD::EXTRACT_SUBVECTOR, VT, Legal);
1579 }
1580
1581 // Custom lower several nodes for 256-bit types.
1582 for (MVT VT : { MVT::v32i8, MVT::v16i16, MVT::v8i32, MVT::v4i64,
1583 MVT::v16f16, MVT::v8f32, MVT::v4f64 }) {
1584 setOperationAction(ISD::BUILD_VECTOR, VT, Custom);
1585 setOperationAction(ISD::VECTOR_SHUFFLE, VT, Custom);
1586 setOperationAction(ISD::VSELECT, VT, Custom);
1587 setOperationAction(ISD::INSERT_VECTOR_ELT, VT, Custom);
1588 setOperationAction(ISD::EXTRACT_VECTOR_ELT, VT, Custom);
1589 setOperationAction(ISD::SCALAR_TO_VECTOR, VT, Custom);
1590 setOperationAction(ISD::INSERT_SUBVECTOR, VT, Legal);
1591 setOperationAction(ISD::CONCAT_VECTORS, VT, Custom);
1592 setOperationAction(ISD::STORE, VT, Custom);
1593 }
1594 setF16Action(MVT::v16f16, Expand);
1595 setOperationAction(ISD::FADD, MVT::v16f16, Expand);
1596 setOperationAction(ISD::FSUB, MVT::v16f16, Expand);
1597 setOperationAction(ISD::FMUL, MVT::v16f16, Expand);
1598 setOperationAction(ISD::FDIV, MVT::v16f16, Expand);
1599
1600 if (HasInt256) {
1601 setOperationAction(ISD::VSELECT, MVT::v32i8, Legal);
1602
1603 // Custom legalize 2x32 to get a little better code.
1604 setOperationAction(ISD::MGATHER, MVT::v2f32, Custom);
1605 setOperationAction(ISD::MGATHER, MVT::v2i32, Custom);
1606
1607 for (auto VT : { MVT::v4i32, MVT::v8i32, MVT::v2i64, MVT::v4i64,
1608 MVT::v4f32, MVT::v8f32, MVT::v2f64, MVT::v4f64 })
1609 setOperationAction(ISD::MGATHER, VT, Custom);
1610 }
1611 }
1612
1613 if (!Subtarget.useSoftFloat() && !Subtarget.hasFP16() &&
1614 Subtarget.hasF16C()) {
1615 for (MVT VT : { MVT::f16, MVT::v2f16, MVT::v4f16, MVT::v8f16 }) {
1616 setOperationAction(ISD::FP_ROUND, VT, Custom);
1617 setOperationAction(ISD::STRICT_FP_ROUND, VT, Custom);
1618 }
1619 for (MVT VT : { MVT::f32, MVT::v2f32, MVT::v4f32, MVT::v8f32 }) {
1620 setOperationAction(ISD::FP_EXTEND, VT, Custom);
1621 setOperationAction(ISD::STRICT_FP_EXTEND, VT, Custom);
1622 }
1623 for (unsigned Opc : {ISD::FADD, ISD::FSUB, ISD::FMUL, ISD::FDIV}) {
1624 setOperationPromotedToType(Opc, MVT::v8f16, MVT::v8f32);
1625 setOperationPromotedToType(Opc, MVT::v16f16, MVT::v16f32);
1626 }
1627 }
1628
1629 // This block controls legalization of the mask vector sizes that are
1630 // available with AVX512. 512-bit vectors are in a separate block controlled
1631 // by useAVX512Regs.
1632 if (!Subtarget.useSoftFloat() && Subtarget.hasAVX512()) {
1633 addRegisterClass(MVT::v1i1, &X86::VK1RegClass);
1634 addRegisterClass(MVT::v2i1, &X86::VK2RegClass);
1635 addRegisterClass(MVT::v4i1, &X86::VK4RegClass);
1636 addRegisterClass(MVT::v8i1, &X86::VK8RegClass);
1637 addRegisterClass(MVT::v16i1, &X86::VK16RegClass);
1638
1639 setOperationAction(ISD::SELECT, MVT::v1i1, Custom);
1640 setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v1i1, Custom);
1641 setOperationAction(ISD::BUILD_VECTOR, MVT::v1i1, Custom);
1642
1643 setOperationPromotedToType(ISD::FP_TO_SINT, MVT::v8i1, MVT::v8i32);
1644 setOperationPromotedToType(ISD::FP_TO_UINT, MVT::v8i1, MVT::v8i32);
1645 setOperationPromotedToType(ISD::FP_TO_SINT, MVT::v4i1, MVT::v4i32);
1646 setOperationPromotedToType(ISD::FP_TO_UINT, MVT::v4i1, MVT::v4i32);
1647 setOperationPromotedToType(ISD::STRICT_FP_TO_SINT, MVT::v8i1, MVT::v8i32);
1648 setOperationPromotedToType(ISD::STRICT_FP_TO_UINT, MVT::v8i1, MVT::v8i32);
1649 setOperationPromotedToType(ISD::STRICT_FP_TO_SINT, MVT::v4i1, MVT::v4i32);
1650 setOperationPromotedToType(ISD::STRICT_FP_TO_UINT, MVT::v4i1, MVT::v4i32);
1651 setOperationAction(ISD::FP_TO_SINT, MVT::v2i1, Custom);
1652 setOperationAction(ISD::FP_TO_UINT, MVT::v2i1, Custom);
1653 setOperationAction(ISD::STRICT_FP_TO_SINT, MVT::v2i1, Custom);
1654 setOperationAction(ISD::STRICT_FP_TO_UINT, MVT::v2i1, Custom);
1655
1656 // There is no byte sized k-register load or store without AVX512DQ.
1657 if (!Subtarget.hasDQI()) {
1658 setOperationAction(ISD::LOAD, MVT::v1i1, Custom);
1659 setOperationAction(ISD::LOAD, MVT::v2i1, Custom);
1660 setOperationAction(ISD::LOAD, MVT::v4i1, Custom);
1661 setOperationAction(ISD::LOAD, MVT::v8i1, Custom);
1662
1663 setOperationAction(ISD::STORE, MVT::v1i1, Custom);
1664 setOperationAction(ISD::STORE, MVT::v2i1, Custom);
1665 setOperationAction(ISD::STORE, MVT::v4i1, Custom);
1666 setOperationAction(ISD::STORE, MVT::v8i1, Custom);
1667 }
1668
1669 // Extends of v16i1/v8i1/v4i1/v2i1 to 128-bit vectors.
1670 for (auto VT : { MVT::v16i8, MVT::v8i16, MVT::v4i32, MVT::v2i64 }) {
1671 setOperationAction(ISD::SIGN_EXTEND, VT, Custom);
1672 setOperationAction(ISD::ZERO_EXTEND, VT, Custom);
1673 setOperationAction(ISD::ANY_EXTEND, VT, Custom);
1674 }
1675
1676 for (auto VT : { MVT::v1i1, MVT::v2i1, MVT::v4i1, MVT::v8i1, MVT::v16i1 })
1677 setOperationAction(ISD::VSELECT, VT, Expand);
1678
1679 for (auto VT : { MVT::v2i1, MVT::v4i1, MVT::v8i1, MVT::v16i1 }) {
1680 setOperationAction(ISD::SETCC, VT, Custom);
1681 setOperationAction(ISD::SELECT, VT, Custom);
1682 setOperationAction(ISD::TRUNCATE, VT, Custom);
1683
1684 setOperationAction(ISD::BUILD_VECTOR, VT, Custom);
1685 setOperationAction(ISD::CONCAT_VECTORS, VT, Custom);
1686 setOperationAction(ISD::EXTRACT_VECTOR_ELT, VT, Custom);
1687 setOperationAction(ISD::INSERT_SUBVECTOR, VT, Custom);
1688 setOperationAction(ISD::INSERT_VECTOR_ELT, VT, Custom);
1689 setOperationAction(ISD::VECTOR_SHUFFLE, VT, Custom);
1690 }
1691
1692 for (auto VT : { MVT::v1i1, MVT::v2i1, MVT::v4i1, MVT::v8i1 })
1693 setOperationAction(ISD::EXTRACT_SUBVECTOR, VT, Custom);
1694 }
1695
1696 // This block controls legalization for 512-bit operations with 8/16/32/64 bit
1697 // elements. 512-bits can be disabled based on prefer-vector-width and
1698 // required-vector-width function attributes.
1699 if (!Subtarget.useSoftFloat() && Subtarget.useAVX512Regs()) {
1700 bool HasBWI = Subtarget.hasBWI();
1701
1702 addRegisterClass(MVT::v16i32, &X86::VR512RegClass);
1703 addRegisterClass(MVT::v16f32, &X86::VR512RegClass);
1704 addRegisterClass(MVT::v8i64, &X86::VR512RegClass);
1705 addRegisterClass(MVT::v8f64, &X86::VR512RegClass);
1706 addRegisterClass(MVT::v32i16, &X86::VR512RegClass);
1707 addRegisterClass(MVT::v32f16, &X86::VR512RegClass);
1708 addRegisterClass(MVT::v64i8, &X86::VR512RegClass);
1709
1710 for (auto ExtType : {ISD::ZEXTLOAD, ISD::SEXTLOAD}) {
1711 setLoadExtAction(ExtType, MVT::v16i32, MVT::v16i8, Legal);
1712 setLoadExtAction(ExtType, MVT::v16i32, MVT::v16i16, Legal);
1713 setLoadExtAction(ExtType, MVT::v8i64, MVT::v8i8, Legal);
1714 setLoadExtAction(ExtType, MVT::v8i64, MVT::v8i16, Legal);
1715 setLoadExtAction(ExtType, MVT::v8i64, MVT::v8i32, Legal);
1716 if (HasBWI)
1717 setLoadExtAction(ExtType, MVT::v32i16, MVT::v32i8, Legal);
1718 }
1719
1720 for (MVT VT : { MVT::v16f32, MVT::v8f64 }) {
1721 setOperationAction(ISD::FMAXIMUM, VT, Custom);
1722 setOperationAction(ISD::FMINIMUM, VT, Custom);
1723 setOperationAction(ISD::FNEG, VT, Custom);
1724 setOperationAction(ISD::FABS, VT, Custom);
1725 setOperationAction(ISD::FMA, VT, Legal);
1726 setOperationAction(ISD::STRICT_FMA, VT, Legal);
1727 setOperationAction(ISD::FCOPYSIGN, VT, Custom);
1728 }
1729
1730 for (MVT VT : { MVT::v16i1, MVT::v16i8 }) {
1731 setOperationPromotedToType(ISD::FP_TO_SINT , VT, MVT::v16i32);
1732 setOperationPromotedToType(ISD::FP_TO_UINT , VT, MVT::v16i32);
1733 setOperationPromotedToType(ISD::STRICT_FP_TO_SINT, VT, MVT::v16i32);
1734 setOperationPromotedToType(ISD::STRICT_FP_TO_UINT, VT, MVT::v16i32);
1735 }
1736
1737 for (MVT VT : { MVT::v16i16, MVT::v16i32 }) {
1738 setOperationAction(ISD::FP_TO_SINT, VT, Custom);
1739 setOperationAction(ISD::FP_TO_UINT, VT, Custom);
1740 setOperationAction(ISD::STRICT_FP_TO_SINT, VT, Custom);
1741 setOperationAction(ISD::STRICT_FP_TO_UINT, VT, Custom);
1742 }
1743
1744 setOperationAction(ISD::SINT_TO_FP, MVT::v16i32, Custom);
1745 setOperationAction(ISD::UINT_TO_FP, MVT::v16i32, Custom);
1746 setOperationAction(ISD::STRICT_SINT_TO_FP, MVT::v16i32, Custom);
1747 setOperationAction(ISD::STRICT_UINT_TO_FP, MVT::v16i32, Custom);
1748 setOperationAction(ISD::FP_EXTEND, MVT::v8f64, Custom);
1749 setOperationAction(ISD::STRICT_FP_EXTEND, MVT::v8f64, Custom);
1750
1751 setOperationAction(ISD::STRICT_FADD, MVT::v16f32, Legal);
1752 setOperationAction(ISD::STRICT_FADD, MVT::v8f64, Legal);
1753 setOperationAction(ISD::STRICT_FSUB, MVT::v16f32, Legal);
1754 setOperationAction(ISD::STRICT_FSUB, MVT::v8f64, Legal);
1755 setOperationAction(ISD::STRICT_FMUL, MVT::v16f32, Legal);
1756 setOperationAction(ISD::STRICT_FMUL, MVT::v8f64, Legal);
1757 setOperationAction(ISD::STRICT_FDIV, MVT::v16f32, Legal);
1758 setOperationAction(ISD::STRICT_FDIV, MVT::v8f64, Legal);
1759 setOperationAction(ISD::STRICT_FSQRT, MVT::v16f32, Legal);
1760 setOperationAction(ISD::STRICT_FSQRT, MVT::v8f64, Legal);
1761 setOperationAction(ISD::STRICT_FP_ROUND, MVT::v8f32, Legal);
1762
1763 setTruncStoreAction(MVT::v8i64, MVT::v8i8, Legal);
1764 setTruncStoreAction(MVT::v8i64, MVT::v8i16, Legal);
1765 setTruncStoreAction(MVT::v8i64, MVT::v8i32, Legal);
1766 setTruncStoreAction(MVT::v16i32, MVT::v16i8, Legal);
1767 setTruncStoreAction(MVT::v16i32, MVT::v16i16, Legal);
1768 if (HasBWI)
1769 setTruncStoreAction(MVT::v32i16, MVT::v32i8, Legal);
1770
1771 // With 512-bit vectors and no VLX, we prefer to widen MLOAD/MSTORE
1772 // to 512-bit rather than use the AVX2 instructions so that we can use
1773 // k-masks.
1774 if (!Subtarget.hasVLX()) {
1775 for (auto VT : {MVT::v4i32, MVT::v8i32, MVT::v2i64, MVT::v4i64,
1776 MVT::v4f32, MVT::v8f32, MVT::v2f64, MVT::v4f64}) {
1777 setOperationAction(ISD::MLOAD, VT, Custom);
1778 setOperationAction(ISD::MSTORE, VT, Custom);
1779 }
1780 }
1781
1782 setOperationAction(ISD::TRUNCATE, MVT::v8i32, Legal);
1783 setOperationAction(ISD::TRUNCATE, MVT::v16i16, Legal);
1784 setOperationAction(ISD::TRUNCATE, MVT::v32i8, HasBWI ? Legal : Custom);
1785 setOperationAction(ISD::ZERO_EXTEND, MVT::v32i16, Custom);
1786 setOperationAction(ISD::ZERO_EXTEND, MVT::v16i32, Custom);
1787 setOperationAction(ISD::ZERO_EXTEND, MVT::v8i64, Custom);
1788 setOperationAction(ISD::ANY_EXTEND, MVT::v32i16, Custom);
1789 setOperationAction(ISD::ANY_EXTEND, MVT::v16i32, Custom);
1790 setOperationAction(ISD::ANY_EXTEND, MVT::v8i64, Custom);
1791 setOperationAction(ISD::SIGN_EXTEND, MVT::v32i16, Custom);
1792 setOperationAction(ISD::SIGN_EXTEND, MVT::v16i32, Custom);
1793 setOperationAction(ISD::SIGN_EXTEND, MVT::v8i64, Custom);
1794
1795 if (HasBWI) {
1796 // Extends from v64i1 masks to 512-bit vectors.
1797 setOperationAction(ISD::SIGN_EXTEND, MVT::v64i8, Custom);
1798 setOperationAction(ISD::ZERO_EXTEND, MVT::v64i8, Custom);
1799 setOperationAction(ISD::ANY_EXTEND, MVT::v64i8, Custom);
1800 }
1801
1802 for (auto VT : { MVT::v16f32, MVT::v8f64 }) {
1803 setOperationAction(ISD::FFLOOR, VT, Legal);
1804 setOperationAction(ISD::STRICT_FFLOOR, VT, Legal);
1805 setOperationAction(ISD::FCEIL, VT, Legal);
1806 setOperationAction(ISD::STRICT_FCEIL, VT, Legal);
1807 setOperationAction(ISD::FTRUNC, VT, Legal);
1808 setOperationAction(ISD::STRICT_FTRUNC, VT, Legal);
1809 setOperationAction(ISD::FRINT, VT, Legal);
1810 setOperationAction(ISD::STRICT_FRINT, VT, Legal);
1811 setOperationAction(ISD::FNEARBYINT, VT, Legal);
1812 setOperationAction(ISD::STRICT_FNEARBYINT, VT, Legal);
1813 setOperationAction(ISD::FROUNDEVEN, VT, Legal);
1814 setOperationAction(ISD::STRICT_FROUNDEVEN, VT, Legal);
1815
1816 setOperationAction(ISD::FROUND, VT, Custom);
1817 }
1818
1819 for (auto VT : {MVT::v32i16, MVT::v16i32, MVT::v8i64}) {
1820 setOperationAction(ISD::SIGN_EXTEND_VECTOR_INREG, VT, Custom);
1821 setOperationAction(ISD::ZERO_EXTEND_VECTOR_INREG, VT, Custom);
1822 }
1823
1824 setOperationAction(ISD::ADD, MVT::v32i16, HasBWI ? Legal : Custom);
1825 setOperationAction(ISD::SUB, MVT::v32i16, HasBWI ? Legal : Custom);
1826 setOperationAction(ISD::ADD, MVT::v64i8, HasBWI ? Legal : Custom);
1827 setOperationAction(ISD::SUB, MVT::v64i8, HasBWI ? Legal : Custom);
1828
1829 setOperationAction(ISD::MUL, MVT::v8i64, Custom);
1830 setOperationAction(ISD::MUL, MVT::v16i32, Legal);
1831 setOperationAction(ISD::MUL, MVT::v32i16, HasBWI ? Legal : Custom);
1832 setOperationAction(ISD::MUL, MVT::v64i8, Custom);
1833
1834 setOperationAction(ISD::MULHU, MVT::v16i32, Custom);
1835 setOperationAction(ISD::MULHS, MVT::v16i32, Custom);
1836 setOperationAction(ISD::MULHS, MVT::v32i16, HasBWI ? Legal : Custom);
1837 setOperationAction(ISD::MULHU, MVT::v32i16, HasBWI ? Legal : Custom);
1838 setOperationAction(ISD::MULHS, MVT::v64i8, Custom);
1839 setOperationAction(ISD::MULHU, MVT::v64i8, Custom);
1840 setOperationAction(ISD::AVGCEILU, MVT::v32i16, HasBWI ? Legal : Custom);
1841 setOperationAction(ISD::AVGCEILU, MVT::v64i8, HasBWI ? Legal : Custom);
1842
1843 setOperationAction(ISD::SMULO, MVT::v64i8, Custom);
1844 setOperationAction(ISD::UMULO, MVT::v64i8, Custom);
1845
1846 setOperationAction(ISD::BITREVERSE, MVT::v64i8, Custom);
1847
1848 for (auto VT : { MVT::v64i8, MVT::v32i16, MVT::v16i32, MVT::v8i64 }) {
1849 setOperationAction(ISD::SRL, VT, Custom);
1850 setOperationAction(ISD::SHL, VT, Custom);
1851 setOperationAction(ISD::SRA, VT, Custom);
1852 setOperationAction(ISD::ROTL, VT, Custom);
1853 setOperationAction(ISD::ROTR, VT, Custom);
1854 setOperationAction(ISD::SETCC, VT, Custom);
1855 setOperationAction(ISD::ABDS, VT, Custom);
1856 setOperationAction(ISD::ABDU, VT, Custom);
1857
1858 // The condition codes aren't legal in SSE/AVX and under AVX512 we use
1859 // setcc all the way to isel and prefer SETGT in some isel patterns.
1860 setCondCodeAction(ISD::SETLT, VT, Custom);
1861 setCondCodeAction(ISD::SETLE, VT, Custom);
1862 }
1863
1864 setOperationAction(ISD::SETCC, MVT::v8f64, Custom);
1865 setOperationAction(ISD::SETCC, MVT::v16f32, Custom);
1866 setOperationAction(ISD::STRICT_FSETCC, MVT::v8f64, Custom);
1867 setOperationAction(ISD::STRICT_FSETCC, MVT::v16f32, Custom);
1868 setOperationAction(ISD::STRICT_FSETCCS, MVT::v8f64, Custom);
1869 setOperationAction(ISD::STRICT_FSETCCS, MVT::v16f32, Custom);
1870
1871 for (auto VT : { MVT::v16i32, MVT::v8i64 }) {
1872 setOperationAction(ISD::SMAX, VT, Legal);
1873 setOperationAction(ISD::UMAX, VT, Legal);
1874 setOperationAction(ISD::SMIN, VT, Legal);
1875 setOperationAction(ISD::UMIN, VT, Legal);
1876 setOperationAction(ISD::ABS, VT, Legal);
1877 setOperationAction(ISD::CTPOP, VT, Custom);
1878 }
1879
1880 for (auto VT : { MVT::v64i8, MVT::v32i16 }) {
1881 setOperationAction(ISD::ABS, VT, HasBWI ? Legal : Custom);
1882 setOperationAction(ISD::CTPOP, VT, Subtarget.hasBITALG() ? Legal : Custom);
1883 setOperationAction(ISD::CTLZ, VT, Custom);
1884 setOperationAction(ISD::SMAX, VT, HasBWI ? Legal : Custom);
1885 setOperationAction(ISD::UMAX, VT, HasBWI ? Legal : Custom);
1886 setOperationAction(ISD::SMIN, VT, HasBWI ? Legal : Custom);
1887 setOperationAction(ISD::UMIN, VT, HasBWI ? Legal : Custom);
1888 setOperationAction(ISD::UADDSAT, VT, HasBWI ? Legal : Custom);
1889 setOperationAction(ISD::SADDSAT, VT, HasBWI ? Legal : Custom);
1890 setOperationAction(ISD::USUBSAT, VT, HasBWI ? Legal : Custom);
1891 setOperationAction(ISD::SSUBSAT, VT, HasBWI ? Legal : Custom);
1892 }
1893
1894 setOperationAction(ISD::FSHL, MVT::v64i8, Custom);
1895 setOperationAction(ISD::FSHR, MVT::v64i8, Custom);
1896 setOperationAction(ISD::FSHL, MVT::v32i16, Custom);
1897 setOperationAction(ISD::FSHR, MVT::v32i16, Custom);
1898 setOperationAction(ISD::FSHL, MVT::v16i32, Custom);
1899 setOperationAction(ISD::FSHR, MVT::v16i32, Custom);
1900
1901 if (Subtarget.hasDQI()) {
1902 for (auto Opc : {ISD::SINT_TO_FP, ISD::UINT_TO_FP, ISD::STRICT_SINT_TO_FP,
1903 ISD::STRICT_UINT_TO_FP, ISD::FP_TO_SINT, ISD::FP_TO_UINT,
1904 ISD::STRICT_FP_TO_SINT, ISD::STRICT_FP_TO_UINT})
1905 setOperationAction(Opc, MVT::v8i64, Custom);
1906 setOperationAction(ISD::MUL, MVT::v8i64, Legal);
1907 }
1908
1909 if (Subtarget.hasCDI()) {
1910 // NonVLX sub-targets extend 128/256 vectors to use the 512 version.
1911 for (auto VT : { MVT::v16i32, MVT::v8i64} ) {
1912 setOperationAction(ISD::CTLZ, VT, Legal);
1913 }
1914 } // Subtarget.hasCDI()
1915
1916 if (Subtarget.hasVPOPCNTDQ()) {
1917 for (auto VT : { MVT::v16i32, MVT::v8i64 })
1918 setOperationAction(ISD::CTPOP, VT, Legal);
1919 }
1920
1921 // Extract subvector is special because the value type
1922 // (result) is 256-bit but the source is 512-bit wide.
1923 // 128-bit was made Legal under AVX1.
1924 for (auto VT : { MVT::v32i8, MVT::v16i16, MVT::v8i32, MVT::v4i64,
1925 MVT::v16f16, MVT::v8f32, MVT::v4f64 })
1926 setOperationAction(ISD::EXTRACT_SUBVECTOR, VT, Legal);
1927
1928 for (auto VT : { MVT::v64i8, MVT::v32i16, MVT::v16i32, MVT::v8i64,
1929 MVT::v32f16, MVT::v16f32, MVT::v8f64 }) {
1930 setOperationAction(ISD::CONCAT_VECTORS, VT, Custom);
1931 setOperationAction(ISD::INSERT_SUBVECTOR, VT, Legal);
1932 setOperationAction(ISD::SELECT, VT, Custom);
1933 setOperationAction(ISD::VSELECT, VT, Custom);
1934 setOperationAction(ISD::BUILD_VECTOR, VT, Custom);
1935 setOperationAction(ISD::EXTRACT_VECTOR_ELT, VT, Custom);
1936 setOperationAction(ISD::VECTOR_SHUFFLE, VT, Custom);
1937 setOperationAction(ISD::SCALAR_TO_VECTOR, VT, Custom);
1938 setOperationAction(ISD::INSERT_VECTOR_ELT, VT, Custom);
1939 }
1940 setF16Action(MVT::v32f16, Expand);
1941 setOperationAction(ISD::FP_ROUND, MVT::v16f16, Custom);
1942 setOperationAction(ISD::STRICT_FP_ROUND, MVT::v16f16, Custom);
1943 setOperationAction(ISD::FP_EXTEND, MVT::v16f32, Custom);
1944 setOperationAction(ISD::STRICT_FP_EXTEND, MVT::v16f32, Custom);
1945 for (unsigned Opc : {ISD::FADD, ISD::FSUB, ISD::FMUL, ISD::FDIV}) {
1946 setOperationPromotedToType(Opc, MVT::v16f16, MVT::v16f32);
1947 setOperationPromotedToType(Opc, MVT::v32f16, MVT::v32f32);
1948 }
1949
1950 for (auto VT : { MVT::v16i32, MVT::v8i64, MVT::v16f32, MVT::v8f64 }) {
1951 setOperationAction(ISD::MLOAD, VT, Legal);
1952 setOperationAction(ISD::MSTORE, VT, Legal);
1953 setOperationAction(ISD::MGATHER, VT, Custom);
1954 setOperationAction(ISD::MSCATTER, VT, Custom);
1955 }
1956 if (HasBWI) {
1957 for (auto VT : { MVT::v64i8, MVT::v32i16 }) {
1958 setOperationAction(ISD::MLOAD, VT, Legal);
1959 setOperationAction(ISD::MSTORE, VT, Legal);
1960 }
1961 } else {
1962 setOperationAction(ISD::STORE, MVT::v32i16, Custom);
1963 setOperationAction(ISD::STORE, MVT::v64i8, Custom);
1964 }
1965
1966 if (Subtarget.hasVBMI2()) {
1967 for (auto VT : { MVT::v8i16, MVT::v4i32, MVT::v2i64,
1968 MVT::v16i16, MVT::v8i32, MVT::v4i64,
1969 MVT::v32i16, MVT::v16i32, MVT::v8i64 }) {
1970 setOperationAction(ISD::FSHL, VT, Custom);
1971 setOperationAction(ISD::FSHR, VT, Custom);
1972 }
1973
1974 setOperationAction(ISD::ROTL, MVT::v32i16, Custom);
1975 setOperationAction(ISD::ROTR, MVT::v8i16, Custom);
1976 setOperationAction(ISD::ROTR, MVT::v16i16, Custom);
1977 setOperationAction(ISD::ROTR, MVT::v32i16, Custom);
1978 }
1979 }// useAVX512Regs
1980
1981 // This block controls legalization for operations that don't have
1982 // pre-AVX512 equivalents. Without VLX we use 512-bit operations for
1983 // narrower widths.
1984 if (!Subtarget.useSoftFloat() && Subtarget.hasAVX512()) {
1985 // These operations are handled on non-VLX by artificially widening in
1986 // isel patterns.
1987
1988 setOperationAction(ISD::STRICT_FP_TO_UINT, MVT::v8i32, Custom);
1989 setOperationAction(ISD::STRICT_FP_TO_UINT, MVT::v4i32, Custom);
1990 setOperationAction(ISD::STRICT_FP_TO_UINT, MVT::v2i32, Custom);
1991
1992 if (Subtarget.hasDQI()) {
1993 // Fast v2f32 SINT_TO_FP( v2i64 ) custom conversion.
1994 // v2f32 UINT_TO_FP is already custom under SSE2.
1995 assert(isOperationCustom(ISD::UINT_TO_FP, MVT::v2f32) &&
1996 isOperationCustom(ISD::STRICT_UINT_TO_FP, MVT::v2f32) &&
1997 "Unexpected operation action!");
1998 // v2i64 FP_TO_S/UINT(v2f32) custom conversion.
1999 setOperationAction(ISD::FP_TO_SINT, MVT::v2f32, Custom);
2000 setOperationAction(ISD::FP_TO_UINT, MVT::v2f32, Custom);
2001 setOperationAction(ISD::STRICT_FP_TO_SINT, MVT::v2f32, Custom);
2002 setOperationAction(ISD::STRICT_FP_TO_UINT, MVT::v2f32, Custom);
2003 }
2004
2005 for (auto VT : { MVT::v2i64, MVT::v4i64 }) {
2006 setOperationAction(ISD::SMAX, VT, Legal);
2007 setOperationAction(ISD::UMAX, VT, Legal);
2008 setOperationAction(ISD::SMIN, VT, Legal);
2009 setOperationAction(ISD::UMIN, VT, Legal);
2010 setOperationAction(ISD::ABS, VT, Legal);
2011 }
2012
2013 for (auto VT : { MVT::v4i32, MVT::v8i32, MVT::v2i64, MVT::v4i64 }) {
2014 setOperationAction(ISD::ROTL, VT, Custom);
2015 setOperationAction(ISD::ROTR, VT, Custom);
2016 }
2017
2018 // Custom legalize 2x32 to get a little better code.
2019 setOperationAction(ISD::MSCATTER, MVT::v2f32, Custom);
2020 setOperationAction(ISD::MSCATTER, MVT::v2i32, Custom);
2021
2022 for (auto VT : { MVT::v4i32, MVT::v8i32, MVT::v2i64, MVT::v4i64,
2023 MVT::v4f32, MVT::v8f32, MVT::v2f64, MVT::v4f64 })
2024 setOperationAction(ISD::MSCATTER, VT, Custom);
2025
2026 if (Subtarget.hasDQI()) {
2027 for (auto Opc : {ISD::SINT_TO_FP, ISD::UINT_TO_FP, ISD::STRICT_SINT_TO_FP,
2028 ISD::STRICT_UINT_TO_FP, ISD::FP_TO_SINT, ISD::FP_TO_UINT,
2029 ISD::STRICT_FP_TO_SINT, ISD::STRICT_FP_TO_UINT}) {
2030 setOperationAction(Opc, MVT::v2i64, Custom);
2031 setOperationAction(Opc, MVT::v4i64, Custom);
2032 }
2033 setOperationAction(ISD::MUL, MVT::v2i64, Legal);
2034 setOperationAction(ISD::MUL, MVT::v4i64, Legal);
2035 }
2036
2037 if (Subtarget.hasCDI()) {
2038 for (auto VT : { MVT::v4i32, MVT::v8i32, MVT::v2i64, MVT::v4i64 }) {
2039 setOperationAction(ISD::CTLZ, VT, Legal);
2040 }
2041 } // Subtarget.hasCDI()
2042
2043 if (Subtarget.hasVPOPCNTDQ()) {
2044 for (auto VT : { MVT::v4i32, MVT::v8i32, MVT::v2i64, MVT::v4i64 })
2045 setOperationAction(ISD::CTPOP, VT, Legal);
2046 }
2047 }
2048
2049 // This block control legalization of v32i1/v64i1 which are available with
2050 // AVX512BW..
2051 if (!Subtarget.useSoftFloat() && Subtarget.hasBWI()) {
2052 addRegisterClass(MVT::v32i1, &X86::VK32RegClass);
2053 addRegisterClass(MVT::v64i1, &X86::VK64RegClass);
2054
2055 for (auto VT : { MVT::v32i1, MVT::v64i1 }) {
2056 setOperationAction(ISD::VSELECT, VT, Expand);
2057 setOperationAction(ISD::TRUNCATE, VT, Custom);
2058 setOperationAction(ISD::SETCC, VT, Custom);
2059 setOperationAction(ISD::EXTRACT_VECTOR_ELT, VT, Custom);
2060 setOperationAction(ISD::INSERT_VECTOR_ELT, VT, Custom);
2061 setOperationAction(ISD::SELECT, VT, Custom);
2062 setOperationAction(ISD::BUILD_VECTOR, VT, Custom);
2063 setOperationAction(ISD::VECTOR_SHUFFLE, VT, Custom);
2064 setOperationAction(ISD::CONCAT_VECTORS, VT, Custom);
2065 setOperationAction(ISD::INSERT_SUBVECTOR, VT, Custom);
2066 }
2067
2068 for (auto VT : { MVT::v16i1, MVT::v32i1 })
2069 setOperationAction(ISD::EXTRACT_SUBVECTOR, VT, Custom);
2070
2071 // Extends from v32i1 masks to 256-bit vectors.
2072 setOperationAction(ISD::SIGN_EXTEND, MVT::v32i8, Custom);
2073 setOperationAction(ISD::ZERO_EXTEND, MVT::v32i8, Custom);
2074 setOperationAction(ISD::ANY_EXTEND, MVT::v32i8, Custom);
2075
2076 for (auto VT : { MVT::v32i8, MVT::v16i8, MVT::v16i16, MVT::v8i16 }) {
2077 setOperationAction(ISD::MLOAD, VT, Subtarget.hasVLX() ? Legal : Custom);
2078 setOperationAction(ISD::MSTORE, VT, Subtarget.hasVLX() ? Legal : Custom);
2079 }
2080
2081 // These operations are handled on non-VLX by artificially widening in
2082 // isel patterns.
2083 // TODO: Custom widen in lowering on non-VLX and drop the isel patterns?
2084
2085 if (Subtarget.hasBITALG()) {
2086 for (auto VT : { MVT::v16i8, MVT::v32i8, MVT::v8i16, MVT::v16i16 })
2087 setOperationAction(ISD::CTPOP, VT, Legal);
2088 }
2089 }
2090
2091 if (!Subtarget.useSoftFloat() && Subtarget.hasFP16()) {
2092 auto setGroup = [&] (MVT VT) {
2093 setOperationAction(ISD::FADD, VT, Legal);
2094 setOperationAction(ISD::STRICT_FADD, VT, Legal);
2095 setOperationAction(ISD::FSUB, VT, Legal);
2096 setOperationAction(ISD::STRICT_FSUB, VT, Legal);
2097 setOperationAction(ISD::FMUL, VT, Legal);
2098 setOperationAction(ISD::STRICT_FMUL, VT, Legal);
2099 setOperationAction(ISD::FDIV, VT, Legal);
2100 setOperationAction(ISD::STRICT_FDIV, VT, Legal);
2101 setOperationAction(ISD::FSQRT, VT, Legal);
2102 setOperationAction(ISD::STRICT_FSQRT, VT, Legal);
2103
2104 setOperationAction(ISD::FFLOOR, VT, Legal);
2105 setOperationAction(ISD::STRICT_FFLOOR, VT, Legal);
2106 setOperationAction(ISD::FCEIL, VT, Legal);
2107 setOperationAction(ISD::STRICT_FCEIL, VT, Legal);
2108 setOperationAction(ISD::FTRUNC, VT, Legal);
2109 setOperationAction(ISD::STRICT_FTRUNC, VT, Legal);
2110 setOperationAction(ISD::FRINT, VT, Legal);
2111 setOperationAction(ISD::STRICT_FRINT, VT, Legal);
2112 setOperationAction(ISD::FNEARBYINT, VT, Legal);
2113 setOperationAction(ISD::STRICT_FNEARBYINT, VT, Legal);
2114
2115 setOperationAction(ISD::FROUND, VT, Custom);
2116
2117 setOperationAction(ISD::LOAD, VT, Legal);
2118 setOperationAction(ISD::STORE, VT, Legal);
2119
2120 setOperationAction(ISD::FMA, VT, Legal);
2121 setOperationAction(ISD::STRICT_FMA, VT, Legal);
2122 setOperationAction(ISD::VSELECT, VT, Legal);
2123 setOperationAction(ISD::BUILD_VECTOR, VT, Custom);
2124 setOperationAction(ISD::SELECT, VT, Custom);
2125
2126 setOperationAction(ISD::FNEG, VT, Custom);
2127 setOperationAction(ISD::FABS, VT, Custom);
2128 setOperationAction(ISD::FCOPYSIGN, VT, Custom);
2129 setOperationAction(ISD::EXTRACT_VECTOR_ELT, VT, Custom);
2130 setOperationAction(ISD::VECTOR_SHUFFLE, VT, Custom);
2131
2132 setOperationAction(ISD::SETCC, VT, Custom);
2133 setOperationAction(ISD::STRICT_FSETCC, VT, Custom);
2134 setOperationAction(ISD::STRICT_FSETCCS, VT, Custom);
2135 };
2136
2137 // AVX512_FP16 scalar operations
2138 setGroup(MVT::f16);
2139 setOperationAction(ISD::FREM, MVT::f16, Promote);
2140 setOperationAction(ISD::STRICT_FREM, MVT::f16, Promote);
2141 setOperationAction(ISD::SELECT_CC, MVT::f16, Expand);
2142 setOperationAction(ISD::BR_CC, MVT::f16, Expand);
2143 setOperationAction(ISD::STRICT_FROUND, MVT::f16, Promote);
2144 setOperationAction(ISD::FROUNDEVEN, MVT::f16, Legal);
2145 setOperationAction(ISD::STRICT_FROUNDEVEN, MVT::f16, Legal);
2146 setOperationAction(ISD::FP_ROUND, MVT::f16, Custom);
2147 setOperationAction(ISD::STRICT_FP_ROUND, MVT::f16, Custom);
2148 setOperationAction(ISD::FMAXIMUM, MVT::f16, Custom);
2149 setOperationAction(ISD::FMINIMUM, MVT::f16, Custom);
2150 setOperationAction(ISD::FP_EXTEND, MVT::f32, Legal);
2151 setOperationAction(ISD::STRICT_FP_EXTEND, MVT::f32, Legal);
2152
2153 setCondCodeAction(ISD::SETOEQ, MVT::f16, Expand);
2154 setCondCodeAction(ISD::SETUNE, MVT::f16, Expand);
2155
2156 if (Subtarget.useAVX512Regs()) {
2157 setGroup(MVT::v32f16);
2158 setOperationAction(ISD::SCALAR_TO_VECTOR, MVT::v32f16, Custom);
2159 setOperationAction(ISD::SINT_TO_FP, MVT::v32i16, Legal);
2160 setOperationAction(ISD::STRICT_SINT_TO_FP, MVT::v32i16, Legal);
2161 setOperationAction(ISD::UINT_TO_FP, MVT::v32i16, Legal);
2162 setOperationAction(ISD::STRICT_UINT_TO_FP, MVT::v32i16, Legal);
2163 setOperationAction(ISD::FP_ROUND, MVT::v16f16, Legal);
2164 setOperationAction(ISD::STRICT_FP_ROUND, MVT::v16f16, Legal);
2165 setOperationAction(ISD::FP_EXTEND, MVT::v16f32, Custom);
2166 setOperationAction(ISD::STRICT_FP_EXTEND, MVT::v16f32, Legal);
2167 setOperationAction(ISD::FP_EXTEND, MVT::v8f64, Custom);
2168 setOperationAction(ISD::STRICT_FP_EXTEND, MVT::v8f64, Legal);
2169 setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v32f16, Custom);
2170
2171 setOperationAction(ISD::FP_TO_SINT, MVT::v32i16, Custom);
2172 setOperationAction(ISD::STRICT_FP_TO_SINT, MVT::v32i16, Custom);
2173 setOperationAction(ISD::FP_TO_UINT, MVT::v32i16, Custom);
2174 setOperationAction(ISD::STRICT_FP_TO_UINT, MVT::v32i16, Custom);
2175 setOperationPromotedToType(ISD::FP_TO_SINT, MVT::v32i8, MVT::v32i16);
2176 setOperationPromotedToType(ISD::STRICT_FP_TO_SINT, MVT::v32i8,
2177 MVT::v32i16);
2178 setOperationPromotedToType(ISD::FP_TO_UINT, MVT::v32i8, MVT::v32i16);
2179 setOperationPromotedToType(ISD::STRICT_FP_TO_UINT, MVT::v32i8,
2180 MVT::v32i16);
2181 setOperationPromotedToType(ISD::FP_TO_SINT, MVT::v32i1, MVT::v32i16);
2182 setOperationPromotedToType(ISD::STRICT_FP_TO_SINT, MVT::v32i1,
2183 MVT::v32i16);
2184 setOperationPromotedToType(ISD::FP_TO_UINT, MVT::v32i1, MVT::v32i16);
2185 setOperationPromotedToType(ISD::STRICT_FP_TO_UINT, MVT::v32i1,
2186 MVT::v32i16);
2187
2188 setOperationAction(ISD::EXTRACT_SUBVECTOR, MVT::v16f16, Legal);
2189 setOperationAction(ISD::INSERT_SUBVECTOR, MVT::v32f16, Legal);
2190 setOperationAction(ISD::CONCAT_VECTORS, MVT::v32f16, Custom);
2191
2192 setLoadExtAction(ISD::EXTLOAD, MVT::v8f64, MVT::v8f16, Legal);
2193 setLoadExtAction(ISD::EXTLOAD, MVT::v16f32, MVT::v16f16, Legal);
2194 }
2195
2196 if (Subtarget.hasVLX()) {
2197 setGroup(MVT::v8f16);
2198 setGroup(MVT::v16f16);
2199
2200 setOperationAction(ISD::SCALAR_TO_VECTOR, MVT::v8f16, Legal);
2201 setOperationAction(ISD::SCALAR_TO_VECTOR, MVT::v16f16, Custom);
2202 setOperationAction(ISD::SINT_TO_FP, MVT::v16i16, Legal);
2203 setOperationAction(ISD::STRICT_SINT_TO_FP, MVT::v16i16, Legal);
2204 setOperationAction(ISD::SINT_TO_FP, MVT::v8i16, Legal);
2205 setOperationAction(ISD::STRICT_SINT_TO_FP, MVT::v8i16, Legal);
2206 setOperationAction(ISD::UINT_TO_FP, MVT::v16i16, Legal);
2207 setOperationAction(ISD::STRICT_UINT_TO_FP, MVT::v16i16, Legal);
2208 setOperationAction(ISD::UINT_TO_FP, MVT::v8i16, Legal);
2209 setOperationAction(ISD::STRICT_UINT_TO_FP, MVT::v8i16, Legal);
2210
2211 setOperationAction(ISD::FP_TO_SINT, MVT::v8i16, Custom);
2212 setOperationAction(ISD::STRICT_FP_TO_SINT, MVT::v8i16, Custom);
2213 setOperationAction(ISD::FP_TO_UINT, MVT::v8i16, Custom);
2214 setOperationAction(ISD::STRICT_FP_TO_UINT, MVT::v8i16, Custom);
2215 setOperationAction(ISD::FP_ROUND, MVT::v8f16, Legal);
2216 setOperationAction(ISD::STRICT_FP_ROUND, MVT::v8f16, Legal);
2217 setOperationAction(ISD::FP_EXTEND, MVT::v8f32, Custom);
2218 setOperationAction(ISD::STRICT_FP_EXTEND, MVT::v8f32, Legal);
2219 setOperationAction(ISD::FP_EXTEND, MVT::v4f64, Custom);
2220 setOperationAction(ISD::STRICT_FP_EXTEND, MVT::v4f64, Legal);
2221
2222 // INSERT_VECTOR_ELT v8f16 extended to VECTOR_SHUFFLE
2223 setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v8f16, Custom);
2224 setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v16f16, Custom);
2225
2226 setOperationAction(ISD::EXTRACT_SUBVECTOR, MVT::v8f16, Legal);
2227 setOperationAction(ISD::INSERT_SUBVECTOR, MVT::v16f16, Legal);
2228 setOperationAction(ISD::CONCAT_VECTORS, MVT::v16f16, Custom);
2229
2230 setLoadExtAction(ISD::EXTLOAD, MVT::v4f64, MVT::v4f16, Legal);
2231 setLoadExtAction(ISD::EXTLOAD, MVT::v2f64, MVT::v2f16, Legal);
2232 setLoadExtAction(ISD::EXTLOAD, MVT::v8f32, MVT::v8f16, Legal);
2233 setLoadExtAction(ISD::EXTLOAD, MVT::v4f32, MVT::v4f16, Legal);
2234
2235 // Need to custom widen these to prevent scalarization.
2236 setOperationAction(ISD::LOAD, MVT::v4f16, Custom);
2237 setOperationAction(ISD::STORE, MVT::v4f16, Custom);
2238 }
2239 }
2240
2241 if (!Subtarget.useSoftFloat() &&
2242 (Subtarget.hasAVXNECONVERT() || Subtarget.hasBF16())) {
2243 addRegisterClass(MVT::v8bf16, Subtarget.hasAVX512() ? &X86::VR128XRegClass
2244 : &X86::VR128RegClass);
2245 addRegisterClass(MVT::v16bf16, Subtarget.hasAVX512() ? &X86::VR256XRegClass
2246 : &X86::VR256RegClass);
2247 // We set the type action of bf16 to TypeSoftPromoteHalf, but we don't
2248 // provide the method to promote BUILD_VECTOR and INSERT_VECTOR_ELT.
2249 // Set the operation action Custom to do the customization later.
2250 setOperationAction(ISD::BUILD_VECTOR, MVT::bf16, Custom);
2251 setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::bf16, Custom);
2252 for (auto VT : {MVT::v8bf16, MVT::v16bf16}) {
2253 setF16Action(VT, Expand);
2254 setOperationAction(ISD::FADD, VT, Expand);
2255 setOperationAction(ISD::FSUB, VT, Expand);
2256 setOperationAction(ISD::FMUL, VT, Expand);
2257 setOperationAction(ISD::FDIV, VT, Expand);
2258 setOperationAction(ISD::BUILD_VECTOR, VT, Custom);
2259 setOperationAction(ISD::VECTOR_SHUFFLE, VT, Custom);
2260 }
2261 setOperationAction(ISD::FP_ROUND, MVT::v8bf16, Custom);
2262 addLegalFPImmediate(APFloat::getZero(APFloat::BFloat()));
2263 }
2264
2265 if (!Subtarget.useSoftFloat() && Subtarget.hasBF16()) {
2266 addRegisterClass(MVT::v32bf16, &X86::VR512RegClass);
2267 setF16Action(MVT::v32bf16, Expand);
2268 setOperationAction(ISD::FADD, MVT::v32bf16, Expand);
2269 setOperationAction(ISD::FSUB, MVT::v32bf16, Expand);
2270 setOperationAction(ISD::FMUL, MVT::v32bf16, Expand);
2271 setOperationAction(ISD::FDIV, MVT::v32bf16, Expand);
2272 setOperationAction(ISD::BUILD_VECTOR, MVT::v32bf16, Custom);
2273 setOperationAction(ISD::FP_ROUND, MVT::v16bf16, Custom);
2274 setOperationAction(ISD::VECTOR_SHUFFLE, MVT::v32bf16, Custom);
2275 }
2276
2277 if (!Subtarget.useSoftFloat() && Subtarget.hasVLX()) {
2278 setTruncStoreAction(MVT::v4i64, MVT::v4i8, Legal);
2279 setTruncStoreAction(MVT::v4i64, MVT::v4i16, Legal);
2280 setTruncStoreAction(MVT::v4i64, MVT::v4i32, Legal);
2281 setTruncStoreAction(MVT::v8i32, MVT::v8i8, Legal);
2282 setTruncStoreAction(MVT::v8i32, MVT::v8i16, Legal);
2283
2284 setTruncStoreAction(MVT::v2i64, MVT::v2i8, Legal);
2285 setTruncStoreAction(MVT::v2i64, MVT::v2i16, Legal);
2286 setTruncStoreAction(MVT::v2i64, MVT::v2i32, Legal);
2287 setTruncStoreAction(MVT::v4i32, MVT::v4i8, Legal);
2288 setTruncStoreAction(MVT::v4i32, MVT::v4i16, Legal);
2289
2290 if (Subtarget.hasBWI()) {
2291 setTruncStoreAction(MVT::v16i16, MVT::v16i8, Legal);
2292 setTruncStoreAction(MVT::v8i16, MVT::v8i8, Legal);
2293 }
2294
2295 if (Subtarget.hasFP16()) {
2296 // vcvttph2[u]dq v4f16 -> v4i32/64, v2f16 -> v2i32/64
2297 setOperationAction(ISD::FP_TO_SINT, MVT::v2f16, Custom);
2298 setOperationAction(ISD::STRICT_FP_TO_SINT, MVT::v2f16, Custom);
2299 setOperationAction(ISD::FP_TO_UINT, MVT::v2f16, Custom);
2300 setOperationAction(ISD::STRICT_FP_TO_UINT, MVT::v2f16, Custom);
2301 setOperationAction(ISD::FP_TO_SINT, MVT::v4f16, Custom);
2302 setOperationAction(ISD::STRICT_FP_TO_SINT, MVT::v4f16, Custom);
2303 setOperationAction(ISD::FP_TO_UINT, MVT::v4f16, Custom);
2304 setOperationAction(ISD::STRICT_FP_TO_UINT, MVT::v4f16, Custom);
2305 // vcvt[u]dq2ph v4i32/64 -> v4f16, v2i32/64 -> v2f16
2306 setOperationAction(ISD::SINT_TO_FP, MVT::v2f16, Custom);
2307 setOperationAction(ISD::STRICT_SINT_TO_FP, MVT::v2f16, Custom);
2308 setOperationAction(ISD::UINT_TO_FP, MVT::v2f16, Custom);
2309 setOperationAction(ISD::STRICT_UINT_TO_FP, MVT::v2f16, Custom);
2310 setOperationAction(ISD::SINT_TO_FP, MVT::v4f16, Custom);
2311 setOperationAction(ISD::STRICT_SINT_TO_FP, MVT::v4f16, Custom);
2312 setOperationAction(ISD::UINT_TO_FP, MVT::v4f16, Custom);
2313 setOperationAction(ISD::STRICT_UINT_TO_FP, MVT::v4f16, Custom);
2314 // vcvtps2phx v4f32 -> v4f16, v2f32 -> v2f16
2315 setOperationAction(ISD::FP_ROUND, MVT::v2f16, Custom);
2316 setOperationAction(ISD::STRICT_FP_ROUND, MVT::v2f16, Custom);
2317 setOperationAction(ISD::FP_ROUND, MVT::v4f16, Custom);
2318 setOperationAction(ISD::STRICT_FP_ROUND, MVT::v4f16, Custom);
2319 // vcvtph2psx v4f16 -> v4f32, v2f16 -> v2f32
2320 setOperationAction(ISD::FP_EXTEND, MVT::v2f16, Custom);
2321 setOperationAction(ISD::STRICT_FP_EXTEND, MVT::v2f16, Custom);
2322 setOperationAction(ISD::FP_EXTEND, MVT::v4f16, Custom);
2323 setOperationAction(ISD::STRICT_FP_EXTEND, MVT::v4f16, Custom);
2324 }
2325 }
2326
2327 if (Subtarget.hasAMXTILE()) {
2328 addRegisterClass(MVT::x86amx, &X86::TILERegClass);
2329 }
2330
2331 // We want to custom lower some of our intrinsics.
2332 setOperationAction(ISD::INTRINSIC_WO_CHAIN, MVT::Other, Custom);
2333 setOperationAction(ISD::INTRINSIC_W_CHAIN, MVT::Other, Custom);
2334 setOperationAction(ISD::INTRINSIC_VOID, MVT::Other, Custom);
2335 if (!Subtarget.is64Bit()) {
2336 setOperationAction(ISD::INTRINSIC_W_CHAIN, MVT::i64, Custom);
2337 }
2338
2339 // Only custom-lower 64-bit SADDO and friends on 64-bit because we don't
2340 // handle type legalization for these operations here.
2341 //
2342 // FIXME: We really should do custom legalization for addition and
2343 // subtraction on x86-32 once PR3203 is fixed. We really can't do much better
2344 // than generic legalization for 64-bit multiplication-with-overflow, though.
2345 for (auto VT : { MVT::i8, MVT::i16, MVT::i32, MVT::i64 }) {
2346 if (VT == MVT::i64 && !Subtarget.is64Bit())
2347 continue;
2348 // Add/Sub/Mul with overflow operations are custom lowered.
2349 setOperationAction(ISD::SADDO, VT, Custom);
2350 setOperationAction(ISD::UADDO, VT, Custom);
2351 setOperationAction(ISD::SSUBO, VT, Custom);
2352 setOperationAction(ISD::USUBO, VT, Custom);
2353 setOperationAction(ISD::SMULO, VT, Custom);
2354 setOperationAction(ISD::UMULO, VT, Custom);
2355
2356 // Support carry in as value rather than glue.
2357 setOperationAction(ISD::UADDO_CARRY, VT, Custom);
2358 setOperationAction(ISD::USUBO_CARRY, VT, Custom);
2359 setOperationAction(ISD::SETCCCARRY, VT, Custom);
2360 setOperationAction(ISD::SADDO_CARRY, VT, Custom);
2361 setOperationAction(ISD::SSUBO_CARRY, VT, Custom);
2362 }
2363
2364 if (!Subtarget.is64Bit()) {
2365 // These libcalls are not available in 32-bit.
2366 setLibcallName(RTLIB::SHL_I128, nullptr);
2367 setLibcallName(RTLIB::SRL_I128, nullptr);
2368 setLibcallName(RTLIB::SRA_I128, nullptr);
2369 setLibcallName(RTLIB::MUL_I128, nullptr);
2370 // The MULO libcall is not part of libgcc, only compiler-rt.
2371 setLibcallName(RTLIB::MULO_I64, nullptr);
2372 }
2373 // The MULO libcall is not part of libgcc, only compiler-rt.
2374 setLibcallName(RTLIB::MULO_I128, nullptr);
2375
2376 // Combine sin / cos into _sincos_stret if it is available.
2377 if (getLibcallName(RTLIB::SINCOS_STRET_F32) != nullptr &&
2378 getLibcallName(RTLIB::SINCOS_STRET_F64) != nullptr) {
2379 setOperationAction(ISD::FSINCOS, MVT::f64, Custom);
2380 setOperationAction(ISD::FSINCOS, MVT::f32, Custom);
2381 }
2382
2383 if (Subtarget.isTargetWin64()) {
2384 setOperationAction(ISD::SDIV, MVT::i128, Custom);
2385 setOperationAction(ISD::UDIV, MVT::i128, Custom);
2386 setOperationAction(ISD::SREM, MVT::i128, Custom);
2387 setOperationAction(ISD::UREM, MVT::i128, Custom);
2388 setOperationAction(ISD::FP_TO_SINT, MVT::i128, Custom);
2389 setOperationAction(ISD::FP_TO_UINT, MVT::i128, Custom);
2390 setOperationAction(ISD::SINT_TO_FP, MVT::i128, Custom);
2391 setOperationAction(ISD::UINT_TO_FP, MVT::i128, Custom);
2392 setOperationAction(ISD::STRICT_FP_TO_SINT, MVT::i128, Custom);
2393 setOperationAction(ISD::STRICT_FP_TO_UINT, MVT::i128, Custom);
2394 setOperationAction(ISD::STRICT_SINT_TO_FP, MVT::i128, Custom);
2395 setOperationAction(ISD::STRICT_UINT_TO_FP, MVT::i128, Custom);
2396 }
2397
2398 // On 32 bit MSVC, `fmodf(f32)` is not defined - only `fmod(f64)`
2399 // is. We should promote the value to 64-bits to solve this.
2400 // This is what the CRT headers do - `fmodf` is an inline header
2401 // function casting to f64 and calling `fmod`.
2402 if (Subtarget.is32Bit() &&
2403 (Subtarget.isTargetWindowsMSVC() || Subtarget.isTargetWindowsItanium()))
2404 for (ISD::NodeType Op :
2405 {ISD::FCEIL, ISD::STRICT_FCEIL,
2406 ISD::FCOS, ISD::STRICT_FCOS,
2407 ISD::FEXP, ISD::STRICT_FEXP,
2408 ISD::FFLOOR, ISD::STRICT_FFLOOR,
2409 ISD::FREM, ISD::STRICT_FREM,
2410 ISD::FLOG, ISD::STRICT_FLOG,
2411 ISD::FLOG10, ISD::STRICT_FLOG10,
2412 ISD::FPOW, ISD::STRICT_FPOW,
2413 ISD::FSIN, ISD::STRICT_FSIN})
2414 if (isOperationExpand(Op, MVT::f32))
2415 setOperationAction(Op, MVT::f32, Promote);
2416
2417 // We have target-specific dag combine patterns for the following nodes:
2418 setTargetDAGCombine({ISD::VECTOR_SHUFFLE,
2419 ISD::SCALAR_TO_VECTOR,
2420 ISD::INSERT_VECTOR_ELT,
2421 ISD::EXTRACT_VECTOR_ELT,
2422 ISD::CONCAT_VECTORS,
2423 ISD::INSERT_SUBVECTOR,
2424 ISD::EXTRACT_SUBVECTOR,
2425 ISD::BITCAST,
2426 ISD::VSELECT,
2427 ISD::SELECT,
2428 ISD::SHL,
2429 ISD::SRA,
2430 ISD::SRL,
2431 ISD::OR,
2432 ISD::AND,
2433 ISD::ADD,
2434 ISD::FADD,
2435 ISD::FSUB,
2436 ISD::FNEG,
2437 ISD::FMA,
2438 ISD::STRICT_FMA,
2439 ISD::FMINNUM,
2440 ISD::FMAXNUM,
2441 ISD::SUB,
2442 ISD::LOAD,
2443 ISD::MLOAD,
2444 ISD::STORE,
2445 ISD::MSTORE,
2446 ISD::TRUNCATE,
2447 ISD::ZERO_EXTEND,
2448 ISD::ANY_EXTEND,
2449 ISD::SIGN_EXTEND,
2450 ISD::SIGN_EXTEND_INREG,
2451 ISD::ANY_EXTEND_VECTOR_INREG,
2452 ISD::SIGN_EXTEND_VECTOR_INREG,
2453 ISD::ZERO_EXTEND_VECTOR_INREG,
2454 ISD::SINT_TO_FP,
2455 ISD::UINT_TO_FP,
2456 ISD::STRICT_SINT_TO_FP,
2457 ISD::STRICT_UINT_TO_FP,
2458 ISD::SETCC,
2459 ISD::MUL,
2460 ISD::XOR,
2461 ISD::MSCATTER,
2462 ISD::MGATHER,
2463 ISD::FP16_TO_FP,
2464 ISD::FP_EXTEND,
2465 ISD::STRICT_FP_EXTEND,
2466 ISD::FP_ROUND,
2467 ISD::STRICT_FP_ROUND});
2468
2469 computeRegisterProperties(Subtarget.getRegisterInfo());
2470
2471 MaxStoresPerMemset = 16; // For @llvm.memset -> sequence of stores
2472 MaxStoresPerMemsetOptSize = 8;
2473 MaxStoresPerMemcpy = 8; // For @llvm.memcpy -> sequence of stores
2474 MaxStoresPerMemcpyOptSize = 4;
2475 MaxStoresPerMemmove = 8; // For @llvm.memmove -> sequence of stores
2476 MaxStoresPerMemmoveOptSize = 4;
2477
2478 // TODO: These control memcmp expansion in CGP and could be raised higher, but
2479 // that needs to benchmarked and balanced with the potential use of vector
2480 // load/store types (PR33329, PR33914).
2481 MaxLoadsPerMemcmp = 2;
2482 MaxLoadsPerMemcmpOptSize = 2;
2483
2484 // Default loop alignment, which can be overridden by -align-loops.
2485 setPrefLoopAlignment(Align(16));
2486
2487 // An out-of-order CPU can speculatively execute past a predictable branch,
2488 // but a conditional move could be stalled by an expensive earlier operation.
2489 PredictableSelectIsExpensive = Subtarget.getSchedModel().isOutOfOrder();
2490 EnableExtLdPromotion = true;
2491 setPrefFunctionAlignment(Align(16));
2492
2493 verifyIntrinsicTables();
2494
2495 // Default to having -disable-strictnode-mutation on
2496 IsStrictFPEnabled = true;
2497}
2498
2499// This has so far only been implemented for 64-bit MachO.
2500bool X86TargetLowering::useLoadStackGuardNode() const {
2501 return Subtarget.isTargetMachO() && Subtarget.is64Bit();
2502}
2503
2504bool X86TargetLowering::useStackGuardXorFP() const {
2505 // Currently only MSVC CRTs XOR the frame pointer into the stack guard value.
2506 return Subtarget.getTargetTriple().isOSMSVCRT() && !Subtarget.isTargetMachO();
2507}
2508
2509SDValue X86TargetLowering::emitStackGuardXorFP(SelectionDAG &DAG, SDValue Val,
2510 const SDLoc &DL) const {
2511 EVT PtrTy = getPointerTy(DAG.getDataLayout());
2512 unsigned XorOp = Subtarget.is64Bit() ? X86::XOR64_FP : X86::XOR32_FP;
2513 MachineSDNode *Node = DAG.getMachineNode(XorOp, DL, PtrTy, Val);
2514 return SDValue(Node, 0);
2515}
2516
2517TargetLoweringBase::LegalizeTypeAction
2518X86TargetLowering::getPreferredVectorAction(MVT VT) const {
2519 if ((VT == MVT::v32i1 || VT == MVT::v64i1) && Subtarget.hasAVX512() &&
2520 !Subtarget.hasBWI())
2521 return TypeSplitVector;
2522
2523 if (!VT.isScalableVector() && VT.getVectorNumElements() != 1 &&
2524 !Subtarget.hasF16C() && VT.getVectorElementType() == MVT::f16)
2525 return TypeSplitVector;
2526
2527 if (!VT.isScalableVector() && VT.getVectorNumElements() != 1 &&
2528 VT.getVectorElementType() != MVT::i1)
2529 return TypeWidenVector;
2530
2531 return TargetLoweringBase::getPreferredVectorAction(VT);
2532}
2533
2534FastISel *
2535X86TargetLowering::createFastISel(FunctionLoweringInfo &funcInfo,
2536 const TargetLibraryInfo *libInfo) const {
2537 return X86::createFastISel(funcInfo, libInfo);
2538}
2539
2540//===----------------------------------------------------------------------===//
2541// Other Lowering Hooks
2542//===----------------------------------------------------------------------===//
2543
2544bool X86::mayFoldLoad(SDValue Op, const X86Subtarget &Subtarget,
2545 bool AssumeSingleUse) {
2546 if (!AssumeSingleUse && !Op.hasOneUse())
2547 return false;
2548 if (!ISD::isNormalLoad(Op.getNode()))
2549 return false;
2550
2551 // If this is an unaligned vector, make sure the target supports folding it.
2552 auto *Ld = cast<LoadSDNode>(Op.getNode());
2553 if (!Subtarget.hasAVX() && !Subtarget.hasSSEUnalignedMem() &&
2554 Ld->getValueSizeInBits(0) == 128 && Ld->getAlign() < Align(16))
2555 return false;
2556
2557 // TODO: If this is a non-temporal load and the target has an instruction
2558 // for it, it should not be folded. See "useNonTemporalLoad()".
2559
2560 return true;
2561}
2562
2563bool X86::mayFoldLoadIntoBroadcastFromMem(SDValue Op, MVT EltVT,
2564 const X86Subtarget &Subtarget,
2565 bool AssumeSingleUse) {
2566 assert(Subtarget.hasAVX() && "Expected AVX for broadcast from memory");
2567 if (!X86::mayFoldLoad(Op, Subtarget, AssumeSingleUse))
2568 return false;
2569
2570 // We can not replace a wide volatile load with a broadcast-from-memory,
2571 // because that would narrow the load, which isn't legal for volatiles.
2572 auto *Ld = cast<LoadSDNode>(Op.getNode());
2573 return !Ld->isVolatile() ||
2574 Ld->getValueSizeInBits(0) == EltVT.getScalarSizeInBits();
2575}
2576
2577bool X86::mayFoldIntoStore(SDValue Op) {
2578 return Op.hasOneUse() && ISD::isNormalStore(*Op.getNode()->use_begin());
2579}
2580
2581bool X86::mayFoldIntoZeroExtend(SDValue Op) {
2582 if (Op.hasOneUse()) {
2583 unsigned Opcode = Op.getNode()->use_begin()->getOpcode();
2584 return (ISD::ZERO_EXTEND == Opcode);
2585 }
2586 return false;
2587}
2588
2589static bool isTargetShuffle(unsigned Opcode) {
2590 switch(Opcode) {
2591 default: return false;
2592 case X86ISD::BLENDI:
2593 case X86ISD::PSHUFB:
2594 case X86ISD::PSHUFD:
2595 case X86ISD::PSHUFHW:
2596 case X86ISD::PSHUFLW:
2597 case X86ISD::SHUFP:
2598 case X86ISD::INSERTPS:
2599 case X86ISD::EXTRQI:
2600 case X86ISD::INSERTQI:
2601 case X86ISD::VALIGN:
2602 case X86ISD::PALIGNR:
2603 case X86ISD::VSHLDQ:
2604 case X86ISD::VSRLDQ:
2605 case X86ISD::MOVLHPS:
2606 case X86ISD::MOVHLPS:
2607 case X86ISD::MOVSHDUP:
2608 case X86ISD::MOVSLDUP:
2609 case X86ISD::MOVDDUP:
2610 case X86ISD::MOVSS:
2611 case X86ISD::MOVSD:
2612 case X86ISD::MOVSH:
2613 case X86ISD::UNPCKL:
2614 case X86ISD::UNPCKH:
2615 case X86ISD::VBROADCAST:
2616 case X86ISD::VPERMILPI:
2617 case X86ISD::VPERMILPV:
2618 case X86ISD::VPERM2X128:
2619 case X86ISD::SHUF128:
2620 case X86ISD::VPERMIL2:
2621 case X86ISD::VPERMI:
2622 case X86ISD::VPPERM:
2623 case X86ISD::VPERMV:
2624 case X86ISD::VPERMV3:
2625 case X86ISD::VZEXT_MOVL:
2626 return true;
2627 }
2628}
2629
2630static bool isTargetShuffleVariableMask(unsigned Opcode) {
2631 switch (Opcode) {
2632 default: return false;
2633 // Target Shuffles.
2634 case X86ISD::PSHUFB:
2635 case X86ISD::VPERMILPV:
2636 case X86ISD::VPERMIL2:
2637 case X86ISD::VPPERM:
2638 case X86ISD::VPERMV:
2639 case X86ISD::VPERMV3:
2640 return true;
2641 // 'Faux' Target Shuffles.
2642 case ISD::OR:
2643 case ISD::AND:
2644 case X86ISD::ANDNP:
2645 return true;
2646 }
2647}
2648
2649SDValue X86TargetLowering::getReturnAddressFrameIndex(SelectionDAG &DAG) const {
2650 MachineFunction &MF = DAG.getMachineFunction();
2651 const X86RegisterInfo *RegInfo = Subtarget.getRegisterInfo();
2652 X86MachineFunctionInfo *FuncInfo = MF.getInfo<X86MachineFunctionInfo>();
2653 int ReturnAddrIndex = FuncInfo->getRAIndex();
2654
2655 if (ReturnAddrIndex == 0) {
2656 // Set up a frame object for the return address.
2657 unsigned SlotSize = RegInfo->getSlotSize();
2658 ReturnAddrIndex = MF.getFrameInfo().CreateFixedObject(SlotSize,
2659 -(int64_t)SlotSize,
2660 false);
2661 FuncInfo->setRAIndex(ReturnAddrIndex);
2662 }
2663
2664 return DAG.getFrameIndex(ReturnAddrIndex, getPointerTy(DAG.getDataLayout()));
2665}
2666
2667bool X86::isOffsetSuitableForCodeModel(int64_t Offset, CodeModel::Model M,
2668 bool hasSymbolicDisplacement) {
2669 // Offset should fit into 32 bit immediate field.
2670 if (!isInt<32>(Offset))
2671 return false;
2672
2673 // If we don't have a symbolic displacement - we don't have any extra
2674 // restrictions.
2675 if (!hasSymbolicDisplacement)
2676 return true;
2677
2678 // FIXME: Some tweaks might be needed for medium code model.
2679 if (M != CodeModel::Small && M != CodeModel::Kernel)
2680 return false;
2681
2682 // For small code model we assume that latest object is 16MB before end of 31
2683 // bits boundary. We may also accept pretty large negative constants knowing
2684 // that all objects are in the positive half of address space.
2685 if (M == CodeModel::Small && Offset < 16*1024*1024)
2686 return true;
2687
2688 // For kernel code model we know that all object resist in the negative half
2689 // of 32bits address space. We may not accept negative offsets, since they may
2690 // be just off and we may accept pretty large positive ones.
2691 if (M == CodeModel::Kernel && Offset >= 0)
2692 return true;
2693
2694 return false;
2695}
2696
2697/// Return true if the condition is an signed comparison operation.
2698static bool isX86CCSigned(unsigned X86CC) {
2699 switch (X86CC) {
2700 default:
2701 llvm_unreachable("Invalid integer condition!");
2702 case X86::COND_E:
2703 case X86::COND_NE:
2704 case X86::COND_B:
2705 case X86::COND_A:
2706 case X86::COND_BE:
2707 case X86::COND_AE:
2708 return false;
2709 case X86::COND_G:
2710 case X86::COND_GE:
2711 case X86::COND_L:
2712 case X86::COND_LE:
2713 return true;
2714 }
2715}
2716
2717static X86::CondCode TranslateIntegerX86CC(ISD::CondCode SetCCOpcode) {
2718 switch (SetCCOpcode) {
2719 default: llvm_unreachable("Invalid integer condition!");
2720 case ISD::SETEQ: return X86::COND_E;
2721 case ISD::SETGT: return X86::COND_G;
2722 case ISD::SETGE: return X86::COND_GE;
2723 case ISD::SETLT: return X86::COND_L;
2724 case ISD::SETLE: return X86::COND_LE;
2725 case ISD::SETNE: return X86::COND_NE;
2726 case ISD::SETULT: return X86::COND_B;
2727 case ISD::SETUGT: return X86::COND_A;
2728 case ISD::SETULE: return X86::COND_BE;
2729 case ISD::SETUGE: return X86::COND_AE;
2730 }
2731}
2732
2733/// Do a one-to-one translation of a ISD::CondCode to the X86-specific
2734/// condition code, returning the condition code and the LHS/RHS of the
2735/// comparison to make.
2736static X86::CondCode TranslateX86CC(ISD::CondCode SetCCOpcode, const SDLoc &DL,
2737 bool isFP, SDValue &LHS, SDValue &RHS,
2738 SelectionDAG &DAG) {
2739 if (!isFP) {
2740 if (ConstantSDNode *RHSC = dyn_cast<ConstantSDNode>(RHS)) {
2741 if (SetCCOpcode == ISD::SETGT && RHSC->isAllOnes()) {
2742 // X > -1 -> X == 0, jump !sign.
2743 RHS = DAG.getConstant(0, DL, RHS.getValueType());
2744 return X86::COND_NS;
2745 }
2746 if (SetCCOpcode == ISD::SETLT && RHSC->isZero()) {
2747 // X < 0 -> X == 0, jump on sign.
2748 return X86::COND_S;
2749 }
2750 if (SetCCOpcode == ISD::SETGE && RHSC->isZero()) {
2751 // X >= 0 -> X == 0, jump on !sign.
2752 return X86::COND_NS;
2753 }
2754 if (SetCCOpcode == ISD::SETLT && RHSC->isOne()) {
2755 // X < 1 -> X <= 0
2756 RHS = DAG.getConstant(0, DL, RHS.getValueType());
2757 return X86::COND_LE;
2758 }
2759 }
2760
2761 return TranslateIntegerX86CC(SetCCOpcode);
2762 }
2763
2764 // First determine if it is required or is profitable to flip the operands.
2765
2766 // If LHS is a foldable load, but RHS is not, flip the condition.
2767 if (ISD::isNON_EXTLoad(LHS.getNode()) &&
2768 !ISD::isNON_EXTLoad(RHS.getNode())) {
2769 SetCCOpcode = getSetCCSwappedOperands(SetCCOpcode);
2770 std::swap(LHS, RHS);
2771 }
2772
2773 switch (SetCCOpcode) {
2774 default: break;
2775 case ISD::SETOLT:
2776 case ISD::SETOLE:
2777 case ISD::SETUGT:
2778 case ISD::SETUGE:
2779 std::swap(LHS, RHS);
2780 break;
2781 }
2782
2783 // On a floating point condition, the flags are set as follows:
2784 // ZF PF CF op
2785 // 0 | 0 | 0 | X > Y
2786 // 0 | 0 | 1 | X < Y
2787 // 1 | 0 | 0 | X == Y
2788 // 1 | 1 | 1 | unordered
2789 switch (SetCCOpcode) {
2790 default: llvm_unreachable("Condcode should be pre-legalized away");
2791 case ISD::SETUEQ:
2792 case ISD::SETEQ: return X86::COND_E;
2793 case ISD::SETOLT: // flipped
2794 case ISD::SETOGT:
2795 case ISD::SETGT: return X86::COND_A;
2796 case ISD::SETOLE: // flipped
2797 case ISD::SETOGE:
2798 case ISD::SETGE: return X86::COND_AE;
2799 case ISD::SETUGT: // flipped
2800 case ISD::SETULT:
2801 case ISD::SETLT: return X86::COND_B;
2802 case ISD::SETUGE: // flipped
2803 case ISD::SETULE:
2804 case ISD::SETLE: return X86::COND_BE;
2805 case ISD::SETONE:
2806 case ISD::SETNE: return X86::COND_NE;
2807 case ISD::SETUO: return X86::COND_P;
2808 case ISD::SETO: return X86::COND_NP;
2809 case ISD::SETOEQ:
2810 case ISD::SETUNE: return X86::COND_INVALID;
2811 }
2812}
2813
2814/// Is there a floating point cmov for the specific X86 condition code?
2815/// Current x86 isa includes the following FP cmov instructions:
2816/// fcmovb, fcomvbe, fcomve, fcmovu, fcmovae, fcmova, fcmovne, fcmovnu.
2817static bool hasFPCMov(unsigned X86CC) {
2818 switch (X86CC) {
2819 default:
2820 return false;
2821 case X86::COND_B:
2822 case X86::COND_BE:
2823 case X86::COND_E:
2824 case X86::COND_P:
2825 case X86::COND_A:
2826 case X86::COND_AE:
2827 case X86::COND_NE:
2828 case X86::COND_NP:
2829 return true;
2830 }
2831}
2832
2833static bool useVPTERNLOG(const X86Subtarget &Subtarget, MVT VT) {
2834 return Subtarget.hasVLX() || Subtarget.canExtendTo512DQ() ||
2835 VT.is512BitVector();
2836}
2837
2838bool X86TargetLowering::getTgtMemIntrinsic(IntrinsicInfo &Info,
2839 const CallInst &I,
2840 MachineFunction &MF,
2841 unsigned Intrinsic) const {
2842 Info.flags = MachineMemOperand::MONone;
2843 Info.offset = 0;
2844
2845 const IntrinsicData* IntrData = getIntrinsicWithChain(Intrinsic);
2846 if (!IntrData) {
2847 switch (Intrinsic) {
2848 case Intrinsic::x86_aesenc128kl:
2849 case Intrinsic::x86_aesdec128kl:
2850 Info.opc = ISD::INTRINSIC_W_CHAIN;
2851 Info.ptrVal = I.getArgOperand(1);
2852 Info.memVT = EVT::getIntegerVT(I.getType()->getContext(), 48);
2853 Info.align = Align(1);
2854 Info.flags |= MachineMemOperand::MOLoad;
2855 return true;
2856 case Intrinsic::x86_aesenc256kl:
2857 case Intrinsic::x86_aesdec256kl:
2858 Info.opc = ISD::INTRINSIC_W_CHAIN;
2859 Info.ptrVal = I.getArgOperand(1);
2860 Info.memVT = EVT::getIntegerVT(I.getType()->getContext(), 64);
2861 Info.align = Align(1);
2862 Info.flags |= MachineMemOperand::MOLoad;
2863 return true;
2864 case Intrinsic::x86_aesencwide128kl:
2865 case Intrinsic::x86_aesdecwide128kl:
2866 Info.opc = ISD::INTRINSIC_W_CHAIN;
2867 Info.ptrVal = I.getArgOperand(0);
2868 Info.memVT = EVT::getIntegerVT(I.getType()->getContext(), 48);
2869 Info.align = Align(1);
2870 Info.flags |= MachineMemOperand::MOLoad;
2871 return true;
2872 case Intrinsic::x86_aesencwide256kl:
2873 case Intrinsic::x86_aesdecwide256kl:
2874 Info.opc = ISD::INTRINSIC_W_CHAIN;
2875 Info.ptrVal = I.getArgOperand(0);
2876 Info.memVT = EVT::getIntegerVT(I.getType()->getContext(), 64);
2877 Info.align = Align(1);
2878 Info.flags |= MachineMemOperand::MOLoad;
2879 return true;
2880 case Intrinsic::x86_cmpccxadd32:
2881 case Intrinsic::x86_cmpccxadd64:
2882 case Intrinsic::x86_atomic_bts:
2883 case Intrinsic::x86_atomic_btc:
2884 case Intrinsic::x86_atomic_btr: {
2885 Info.opc = ISD::INTRINSIC_W_CHAIN;
2886 Info.ptrVal = I.getArgOperand(0);
2887 unsigned Size = I.getType()->getScalarSizeInBits();
2888 Info.memVT = EVT::getIntegerVT(I.getType()->getContext(), Size);
2889 Info.align = Align(Size);
2890 Info.flags |= MachineMemOperand::MOLoad | MachineMemOperand::MOStore |
2891 MachineMemOperand::MOVolatile;
2892 return true;
2893 }
2894 case Intrinsic::x86_atomic_bts_rm:
2895 case Intrinsic::x86_atomic_btc_rm:
2896 case Intrinsic::x86_atomic_btr_rm: {
2897 Info.opc = ISD::INTRINSIC_W_CHAIN;
2898 Info.ptrVal = I.getArgOperand(0);
2899 unsigned Size = I.getArgOperand(1)->getType()->getScalarSizeInBits();
2900 Info.memVT = EVT::getIntegerVT(I.getType()->getContext(), Size);
2901 Info.align = Align(Size);
2902 Info.flags |= MachineMemOperand::MOLoad | MachineMemOperand::MOStore |
2903 MachineMemOperand::MOVolatile;
2904 return true;
2905 }
2906 case Intrinsic::x86_aadd32:
2907 case Intrinsic::x86_aadd64:
2908 case Intrinsic::x86_aand32:
2909 case Intrinsic::x86_aand64:
2910 case Intrinsic::x86_aor32:
2911 case Intrinsic::x86_aor64:
2912 case Intrinsic::x86_axor32:
2913 case Intrinsic::x86_axor64:
2914 case Intrinsic::x86_atomic_add_cc:
2915 case Intrinsic::x86_atomic_sub_cc:
2916 case Intrinsic::x86_atomic_or_cc:
2917 case Intrinsic::x86_atomic_and_cc:
2918 case Intrinsic::x86_atomic_xor_cc: {
2919 Info.opc = ISD::INTRINSIC_W_CHAIN;
2920 Info.ptrVal = I.getArgOperand(0);
2921 unsigned Size = I.getArgOperand(1)->getType()->getScalarSizeInBits();
2922 Info.memVT = EVT::getIntegerVT(I.getType()->getContext(), Size);
2923 Info.align = Align(Size);
2924 Info.flags |= MachineMemOperand::MOLoad | MachineMemOperand::MOStore |
2925 MachineMemOperand::MOVolatile;
2926 return true;
2927 }
2928 }
2929 return false;
2930 }
2931
2932 switch (IntrData->Type) {
2933 case TRUNCATE_TO_MEM_VI8:
2934 case TRUNCATE_TO_MEM_VI16:
2935 case TRUNCATE_TO_MEM_VI32: {
2936 Info.opc = ISD::INTRINSIC_VOID;
2937 Info.ptrVal = I.getArgOperand(0);
2938 MVT VT = MVT::getVT(I.getArgOperand(1)->getType());
2939 MVT ScalarVT = MVT::INVALID_SIMPLE_VALUE_TYPE;
2940 if (IntrData->Type == TRUNCATE_TO_MEM_VI8)
2941 ScalarVT = MVT::i8;
2942 else if (IntrData->Type == TRUNCATE_TO_MEM_VI16)
2943 ScalarVT = MVT::i16;
2944 else if (IntrData->Type == TRUNCATE_TO_MEM_VI32)
2945 ScalarVT = MVT::i32;
2946
2947 Info.memVT = MVT::getVectorVT(ScalarVT, VT.getVectorNumElements());
2948 Info.align = Align(1);
2949 Info.flags |= MachineMemOperand::MOStore;
2950 break;
2951 }
2952 case GATHER:
2953 case GATHER_AVX2: {
2954 Info.opc = ISD::INTRINSIC_W_CHAIN;
2955 Info.ptrVal = nullptr;
2956 MVT DataVT = MVT::getVT(I.getType());
2957 MVT IndexVT = MVT::getVT(I.getArgOperand(2)->getType());
2958 unsigned NumElts = std::min(DataVT.getVectorNumElements(),
2959 IndexVT.getVectorNumElements());
2960 Info.memVT = MVT::getVectorVT(DataVT.getVectorElementType(), NumElts);
2961 Info.align = Align(1);
2962 Info.flags |= MachineMemOperand::MOLoad;
2963 break;
2964 }
2965 case SCATTER: {
2966 Info.opc = ISD::INTRINSIC_VOID;
2967 Info.ptrVal = nullptr;
2968 MVT DataVT = MVT::getVT(I.getArgOperand(3)->getType());
2969 MVT IndexVT = MVT::getVT(I.getArgOperand(2)->getType());
2970 unsigned NumElts = std::min(DataVT.getVectorNumElements(),
2971 IndexVT.getVectorNumElements());
2972 Info.memVT = MVT::getVectorVT(DataVT.getVectorElementType(), NumElts);
2973 Info.align = Align(1);
2974 Info.flags |= MachineMemOperand::MOStore;
2975 break;
2976 }
2977 default:
2978 return false;
2979 }
2980
2981 return true;
2982}
2983
2984/// Returns true if the target can instruction select the
2985/// specified FP immediate natively. If false, the legalizer will
2986/// materialize the FP immediate as a load from a constant pool.
2987bool X86TargetLowering::isFPImmLegal(const APFloat &Imm, EVT VT,
2988 bool ForCodeSize) const {
2989 for (const APFloat &FPImm : LegalFPImmediates)
2990 if (Imm.bitwiseIsEqual(FPImm))
2991 return true;
2992 return false;
2993}
2994
2995bool X86TargetLowering::shouldReduceLoadWidth(SDNode *Load,
2996 ISD::LoadExtType ExtTy,
2997 EVT NewVT) const {
2998 assert(cast<LoadSDNode>(Load)->isSimple() && "illegal to narrow");
2999
3000 // "ELF Handling for Thread-Local Storage" specifies that R_X86_64_GOTTPOFF
3001 // relocation target a movq or addq instruction: don't let the load shrink.
3002 SDValue BasePtr = cast<LoadSDNode>(Load)->getBasePtr();
3003 if (BasePtr.getOpcode() == X86ISD::WrapperRIP)
3004 if (const auto *GA = dyn_cast<GlobalAddressSDNode>(BasePtr.getOperand(0)))
3005 return GA->getTargetFlags() != X86II::MO_GOTTPOFF;
3006
3007 // If this is an (1) AVX vector load with (2) multiple uses and (3) all of
3008 // those uses are extracted directly into a store, then the extract + store
3009 // can be store-folded. Therefore, it's probably not worth splitting the load.
3010 EVT VT = Load->getValueType(0);
3011 if ((VT.is256BitVector() || VT.is512BitVector()) && !Load->hasOneUse()) {
3012 for (auto UI = Load->use_begin(), UE = Load->use_end(); UI != UE; ++UI) {
3013 // Skip uses of the chain value. Result 0 of the node is the load value.
3014 if (UI.getUse().getResNo() != 0)
3015 continue;
3016
3017 // If this use is not an extract + store, it's probably worth splitting.
3018 if (UI->getOpcode() != ISD::EXTRACT_SUBVECTOR || !UI->hasOneUse() ||
3019 UI->use_begin()->getOpcode() != ISD::STORE)
3020 return true;
3021 }
3022 // All non-chain uses are extract + store.
3023 return false;
3024 }
3025
3026 return true;
3027}
3028
3029/// Returns true if it is beneficial to convert a load of a constant
3030/// to just the constant itself.
3031bool X86TargetLowering::shouldConvertConstantLoadToIntImm(const APInt &Imm,
3032 Type *Ty) const {
3033 assert(Ty->isIntegerTy());
3034
3035 unsigned BitSize = Ty->getPrimitiveSizeInBits();
3036 if (BitSize == 0 || BitSize > 64)
3037 return false;
3038 return true;
3039}
3040
3041bool X86TargetLowering::reduceSelectOfFPConstantLoads(EVT CmpOpVT) const {
3042 // If we are using XMM registers in the ABI and the condition of the select is
3043 // a floating-point compare and we have blendv or conditional move, then it is
3044 // cheaper to select instead of doing a cross-register move and creating a
3045 // load that depends on the compare result.
3046 bool IsFPSetCC = CmpOpVT.isFloatingPoint() && CmpOpVT != MVT::f128;
3047 return !IsFPSetCC || !Subtarget.isTarget64BitLP64() || !Subtarget.hasAVX();
3048}
3049
3050bool X86TargetLowering::convertSelectOfConstantsToMath(EVT VT) const {
3051 // TODO: It might be a win to ease or lift this restriction, but the generic
3052 // folds in DAGCombiner conflict with vector folds for an AVX512 target.
3053 if (VT.isVector() && Subtarget.hasAVX512())
3054 return false;
3055
3056 return true;
3057}
3058
3059bool X86TargetLowering::decomposeMulByConstant(LLVMContext &Context, EVT VT,
3060 SDValue C) const {
3061 // TODO: We handle scalars using custom code, but generic combining could make
3062 // that unnecessary.
3063 APInt MulC;
3064 if (!ISD::isConstantSplatVector(C.getNode(), MulC))
3065 return false;
3066
3067 // Find the type this will be legalized too. Otherwise we might prematurely
3068 // convert this to shl+add/sub and then still have to type legalize those ops.
3069 // Another choice would be to defer the decision for illegal types until
3070 // after type legalization. But constant splat vectors of i64 can't make it
3071 // through type legalization on 32-bit targets so we would need to special
3072 // case vXi64.
3073 while (getTypeAction(Context, VT) != TypeLegal)
3074 VT = getTypeToTransformTo(Context, VT);
3075
3076 // If vector multiply is legal, assume that's faster than shl + add/sub.
3077 // Multiply is a complex op with higher latency and lower throughput in
3078 // most implementations, sub-vXi32 vector multiplies are always fast,
3079 // vXi32 mustn't have a SlowMULLD implementation, and anything larger (vXi64)
3080 // is always going to be slow.
3081 unsigned EltSizeInBits = VT.getScalarSizeInBits();
3082 if (isOperationLegal(ISD::MUL, VT) && EltSizeInBits <= 32 &&
3083 (EltSizeInBits != 32 || !Subtarget.isPMULLDSlow()))
3084 return false;
3085
3086 // shl+add, shl+sub, shl+add+neg
3087 return (MulC + 1).isPowerOf2() || (MulC - 1).isPowerOf2() ||
3088 (1 - MulC).isPowerOf2() || (-(MulC + 1)).isPowerOf2();
3089}
3090
3091bool X86TargetLowering::isExtractSubvectorCheap(EVT ResVT, EVT SrcVT,
3092 unsigned Index) const {
3093 if (!isOperationLegalOrCustom(ISD::EXTRACT_SUBVECTOR, ResVT))
3094 return false;
3095
3096 // Mask vectors support all subregister combinations and operations that
3097 // extract half of vector.
3098 if (ResVT.getVectorElementType() == MVT::i1)
3099 return Index == 0 || ((ResVT.getSizeInBits() == SrcVT.getSizeInBits()*2) &&
3100 (Index == ResVT.getVectorNumElements()));
3101
3102 return (Index % ResVT.getVectorNumElements()) == 0;
3103}
3104
3105bool X86TargetLowering::shouldScalarizeBinop(SDValue VecOp) const {
3106 unsigned Opc = VecOp.getOpcode();
3107
3108 // Assume target opcodes can't be scalarized.
3109 // TODO - do we have any exceptions?
3110 if (Opc >= ISD::BUILTIN_OP_END)
3111 return false;
3112
3113 // If the vector op is not supported, try to convert to scalar.
3114 EVT VecVT = VecOp.getValueType();
3115 if (!isOperationLegalOrCustomOrPromote(Opc, VecVT))
3116 return true;
3117
3118 // If the vector op is supported, but the scalar op is not, the transform may
3119 // not be worthwhile.
3120 EVT ScalarVT = VecVT.getScalarType();
3121 return isOperationLegalOrCustomOrPromote(Opc, ScalarVT);
3122}
3123
3124bool X86TargetLowering::shouldFormOverflowOp(unsigned Opcode, EVT VT,
3125 bool) const {
3126 // TODO: Allow vectors?
3127 if (VT.isVector())
3128 return false;
3129 return VT.isSimple() || !isOperationExpand(Opcode, VT);
3130}
3131
3132bool X86TargetLowering::isCheapToSpeculateCttz(Type *Ty) const {
3133 // Speculate cttz only if we can directly use TZCNT or can promote to i32.
3134 return Subtarget.hasBMI() ||
3135 (!Ty->isVectorTy() && Ty->getScalarSizeInBits() < 32);
3136}
3137
3138bool X86TargetLowering::isCheapToSpeculateCtlz(Type *Ty) const {
3139 // Speculate ctlz only if we can directly use LZCNT.
3140 return Subtarget.hasLZCNT();
3141}
3142
3143bool X86TargetLowering::ShouldShrinkFPConstant(EVT VT) const {
3144 // Don't shrink FP constpool if SSE2 is available since cvtss2sd is more
3145 // expensive than a straight movsd. On the other hand, it's important to
3146 // shrink long double fp constant since fldt is very slow.
3147 return !Subtarget.hasSSE2() || VT == MVT::f80;
3148}
3149
3150bool X86TargetLowering::isScalarFPTypeInSSEReg(EVT VT) const {
3151 return (VT == MVT::f64 && Subtarget.hasSSE2()) ||
3152 (VT == MVT::f32 && Subtarget.hasSSE1()) || VT == MVT::f16;
3153}
3154
3155bool X86TargetLowering::isLoadBitCastBeneficial(EVT LoadVT, EVT BitcastVT,
3156 const SelectionDAG &DAG,
3157 const MachineMemOperand &MMO) const {
3158 if (!Subtarget.hasAVX512() && !LoadVT.isVector() && BitcastVT.isVector() &&
3159 BitcastVT.getVectorElementType() == MVT::i1)
3160 return false;
3161
3162 if (!Subtarget.hasDQI() && BitcastVT == MVT::v8i1 && LoadVT == MVT::i8)
3163 return false;
3164
3165 // If both types are legal vectors, it's always ok to convert them.
3166 if (LoadVT.isVector() && BitcastVT.isVector() &&
3167 isTypeLegal(LoadVT) && isTypeLegal(BitcastVT))
3168 return true;
3169
3170 return TargetLowering::isLoadBitCastBeneficial(LoadVT, BitcastVT, DAG, MMO);
3171}
3172
3173bool X86TargetLowering::canMergeStoresTo(unsigned AddressSpace, EVT MemVT,
3174 const MachineFunction &MF) const {
3175 // Do not merge to float value size (128 bytes) if no implicit
3176 // float attribute is set.
3177 bool NoFloat = MF.getFunction().hasFnAttribute(Attribute::NoImplicitFloat);
3178
3179 if (NoFloat) {
3180 unsigned MaxIntSize = Subtarget.is64Bit() ? 64 : 32;
3181 return (MemVT.getSizeInBits() <= MaxIntSize);
3182 }
3183 // Make sure we don't merge greater than our preferred vector
3184 // width.
3185 if (MemVT.getSizeInBits() > Subtarget.getPreferVectorWidth())
3186