LLVM 20.0.0git
X86ISelLowering.cpp
Go to the documentation of this file.
1//===-- X86ISelLowering.cpp - X86 DAG Lowering Implementation -------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file defines the interfaces that X86 uses to lower LLVM code into a
10// selection DAG.
11//
12//===----------------------------------------------------------------------===//
13
14#include "X86ISelLowering.h"
15#include "MCTargetDesc/X86ShuffleDecode.h"
16#include "X86.h"
17#include "X86CallingConv.h"
18#include "X86FrameLowering.h"
19#include "X86InstrBuilder.h"
20#include "X86IntrinsicsInfo.h"
21#include "X86MachineFunctionInfo.h"
22#include "X86TargetMachine.h"
23#include "X86TargetObjectFile.h"
24#include "llvm/ADT/SmallBitVector.h"
25#include "llvm/ADT/SmallSet.h"
26#include "llvm/ADT/Statistic.h"
27#include "llvm/ADT/StringExtras.h"
28#include "llvm/ADT/StringSwitch.h"
29#include "llvm/Analysis/BlockFrequencyInfo.h"
30#include "llvm/Analysis/ObjCARCUtil.h"
31#include "llvm/Analysis/ProfileSummaryInfo.h"
32#include "llvm/Analysis/VectorUtils.h"
33#include "llvm/CodeGen/IntrinsicLowering.h"
34#include "llvm/CodeGen/MachineFrameInfo.h"
35#include "llvm/CodeGen/MachineFunction.h"
36#include "llvm/CodeGen/MachineInstrBuilder.h"
37#include "llvm/CodeGen/MachineJumpTableInfo.h"
38#include "llvm/CodeGen/MachineLoopInfo.h"
39#include "llvm/CodeGen/MachineModuleInfo.h"
40#include "llvm/CodeGen/MachineRegisterInfo.h"
41#include "llvm/CodeGen/SDPatternMatch.h"
42#include "llvm/CodeGen/TargetLowering.h"
43#include "llvm/CodeGen/WinEHFuncInfo.h"
44#include "llvm/IR/CallingConv.h"
45#include "llvm/IR/Constants.h"
46#include "llvm/IR/DerivedTypes.h"
47#include "llvm/IR/EHPersonalities.h"
48#include "llvm/IR/Function.h"
49#include "llvm/IR/GlobalAlias.h"
50#include "llvm/IR/GlobalVariable.h"
51#include "llvm/IR/IRBuilder.h"
52#include "llvm/IR/Instructions.h"
53#include "llvm/IR/Intrinsics.h"
54#include "llvm/IR/PatternMatch.h"
55#include "llvm/MC/MCAsmInfo.h"
56#include "llvm/MC/MCContext.h"
57#include "llvm/MC/MCExpr.h"
58#include "llvm/MC/MCSymbol.h"
59#include "llvm/Support/CommandLine.h"
60#include "llvm/Support/Debug.h"
61#include "llvm/Support/ErrorHandling.h"
62#include "llvm/Support/KnownBits.h"
63#include "llvm/Support/MathExtras.h"
64#include "llvm/Target/TargetOptions.h"
65#include <algorithm>
66#include <bitset>
67#include <cctype>
68#include <numeric>
69using namespace llvm;
70
71#define DEBUG_TYPE "x86-isel"
72
73static cl::opt<int> ExperimentalPrefInnermostLoopAlignment(
74 "x86-experimental-pref-innermost-loop-alignment", cl::init(4),
75 cl::desc(
76 "Sets the preferable loop alignment for experiments (as log2 bytes) "
77 "for innermost loops only. If specified, this option overrides "
78 "alignment set by x86-experimental-pref-loop-alignment."),
79 cl::Hidden);
80
81static cl::opt<int> BrMergingBaseCostThresh(
82 "x86-br-merging-base-cost", cl::init(2),
83 cl::desc(
84 "Sets the cost threshold for when multiple conditionals will be merged "
85 "into one branch versus be split in multiple branches. Merging "
86 "conditionals saves branches at the cost of additional instructions. "
87 "This value sets the instruction cost limit, below which conditionals "
88 "will be merged, and above which conditionals will be split. Set to -1 "
89 "to never merge branches."),
90 cl::Hidden);
91
92static cl::opt<int> BrMergingCcmpBias(
93 "x86-br-merging-ccmp-bias", cl::init(6),
94 cl::desc("Increases 'x86-br-merging-base-cost' in cases that the target "
95 "supports conditional compare instructions."),
96 cl::Hidden);
97
98static cl::opt<int> BrMergingLikelyBias(
99 "x86-br-merging-likely-bias", cl::init(0),
100 cl::desc("Increases 'x86-br-merging-base-cost' in cases that it is likely "
101 "that all conditionals will be executed. For example for merging "
102 "the conditionals (a == b && c > d), if its known that a == b is "
103 "likely, then it is likely that if the conditionals are split "
104 "both sides will be executed, so it may be desirable to increase "
105 "the instruction cost threshold. Set to -1 to never merge likely "
106 "branches."),
107 cl::Hidden);
108
109static cl::opt<int> BrMergingUnlikelyBias(
110 "x86-br-merging-unlikely-bias", cl::init(-1),
111 cl::desc(
112 "Decreases 'x86-br-merging-base-cost' in cases that it is unlikely "
113 "that all conditionals will be executed. For example for merging "
114 "the conditionals (a == b && c > d), if its known that a == b is "
115 "unlikely, then it is unlikely that if the conditionals are split "
116 "both sides will be executed, so it may be desirable to decrease "
117 "the instruction cost threshold. Set to -1 to never merge unlikely "
118 "branches."),
119 cl::Hidden);
120
121static cl::opt<bool> MulConstantOptimization(
122 "mul-constant-optimization", cl::init(true),
123 cl::desc("Replace 'mul x, Const' with more effective instructions like "
124 "SHIFT, LEA, etc."),
125 cl::Hidden);
126
127X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
128 const X86Subtarget &STI)
129 : TargetLowering(TM), Subtarget(STI) {
130 bool UseX87 = !Subtarget.useSoftFloat() && Subtarget.hasX87();
131 MVT PtrVT = MVT::getIntegerVT(TM.getPointerSizeInBits(0));
132
133 // Set up the TargetLowering object.
134
135 // X86 is weird. It always uses i8 for shift amounts and setcc results.
136 setBooleanContents(ZeroOrOneBooleanContent);
137 // X86-SSE is even stranger. It uses -1 or 0 for vector masks.
138 setBooleanVectorContents(ZeroOrNegativeOneBooleanContent);
139
140 // X86 instruction cache is coherent with its data cache so we can use the
141 // default expansion to a no-op.
142 setOperationAction(ISD::CLEAR_CACHE, MVT::Other, Expand);
143
144 // For 64-bit, since we have so many registers, use the ILP scheduler.
145 // For 32-bit, use the register pressure specific scheduling.
146 // For Atom, always use ILP scheduling.
147 if (Subtarget.isAtom())
148 setSchedulingPreference(Sched::ILP);
149 else if (Subtarget.is64Bit())
150 setSchedulingPreference(Sched::ILP);
151 else
152 setSchedulingPreference(Sched::RegPressure);
153 const X86RegisterInfo *RegInfo = Subtarget.getRegisterInfo();
154 setStackPointerRegisterToSaveRestore(RegInfo->getStackRegister());
155
156 // Bypass expensive divides and use cheaper ones.
157 if (TM.getOptLevel() >= CodeGenOptLevel::Default) {
158 if (Subtarget.hasSlowDivide32())
159 addBypassSlowDiv(32, 8);
160 if (Subtarget.hasSlowDivide64() && Subtarget.is64Bit())
161 addBypassSlowDiv(64, 32);
162 }
163
164 // Setup Windows compiler runtime calls.
165 if (Subtarget.isTargetWindowsMSVC() || Subtarget.isTargetWindowsItanium()) {
166 static const struct {
167 const RTLIB::Libcall Op;
168 const char * const Name;
169 const CallingConv::ID CC;
170 } LibraryCalls[] = {
171 { RTLIB::SDIV_I64, "_alldiv", CallingConv::X86_StdCall },
172 { RTLIB::UDIV_I64, "_aulldiv", CallingConv::X86_StdCall },
173 { RTLIB::SREM_I64, "_allrem", CallingConv::X86_StdCall },
174 { RTLIB::UREM_I64, "_aullrem", CallingConv::X86_StdCall },
175 { RTLIB::MUL_I64, "_allmul", CallingConv::X86_StdCall },
176 };
177
178 for (const auto &LC : LibraryCalls) {
179 setLibcallName(LC.Op, LC.Name);
180 setLibcallCallingConv(LC.Op, LC.CC);
181 }
182 }
183
184 if (Subtarget.canUseCMPXCHG16B())
185 setMaxAtomicSizeInBitsSupported(128);
186 else if (Subtarget.canUseCMPXCHG8B())
187 setMaxAtomicSizeInBitsSupported(64);
188 else
189 setMaxAtomicSizeInBitsSupported(32);
190
191 setMaxDivRemBitWidthSupported(Subtarget.is64Bit() ? 128 : 64);
192
193 setMaxLargeFPConvertBitWidthSupported(128);
194
195 // Set up the register classes.
196 addRegisterClass(MVT::i8, &X86::GR8RegClass);
197 addRegisterClass(MVT::i16, &X86::GR16RegClass);
198 addRegisterClass(MVT::i32, &X86::GR32RegClass);
199 if (Subtarget.is64Bit())
200 addRegisterClass(MVT::i64, &X86::GR64RegClass);
201
202 for (MVT VT : MVT::integer_valuetypes())
203 setLoadExtAction(ISD::SEXTLOAD, VT, MVT::i1, Promote);
204
205 // We don't accept any truncstore of integer registers.
206 setTruncStoreAction(MVT::i64, MVT::i32, Expand);
207 setTruncStoreAction(MVT::i64, MVT::i16, Expand);
208 setTruncStoreAction(MVT::i64, MVT::i8 , Expand);
209 setTruncStoreAction(MVT::i32, MVT::i16, Expand);
210 setTruncStoreAction(MVT::i32, MVT::i8 , Expand);
211 setTruncStoreAction(MVT::i16, MVT::i8, Expand);
212
213 setTruncStoreAction(MVT::f64, MVT::f32, Expand);
214
215 // SETOEQ and SETUNE require checking two conditions.
216 for (auto VT : {MVT::f32, MVT::f64, MVT::f80}) {
217 setCondCodeAction(ISD::SETOEQ, VT, Expand);
218 setCondCodeAction(ISD::SETUNE, VT, Expand);
219 }
220
221 // Integer absolute.
222 if (Subtarget.canUseCMOV()) {
223 setOperationAction(ISD::ABS , MVT::i16 , Custom);
224 setOperationAction(ISD::ABS , MVT::i32 , Custom);
225 if (Subtarget.is64Bit())
226 setOperationAction(ISD::ABS , MVT::i64 , Custom);
227 }
228
229 // Absolute difference.
230 for (auto Op : {ISD::ABDS, ISD::ABDU}) {
231 setOperationAction(Op , MVT::i8 , Custom);
232 setOperationAction(Op , MVT::i16 , Custom);
233 setOperationAction(Op , MVT::i32 , Custom);
234 if (Subtarget.is64Bit())
235 setOperationAction(Op , MVT::i64 , Custom);
236 }
237
238 // Signed saturation subtraction.
239 setOperationAction(ISD::SSUBSAT , MVT::i8 , Custom);
240 setOperationAction(ISD::SSUBSAT , MVT::i16 , Custom);
241 setOperationAction(ISD::SSUBSAT , MVT::i32 , Custom);
242 if (Subtarget.is64Bit())
243 setOperationAction(ISD::SSUBSAT , MVT::i64 , Custom);
244
245 // Funnel shifts.
246 for (auto ShiftOp : {ISD::FSHL, ISD::FSHR}) {
247 // For slow shld targets we only lower for code size.
248 LegalizeAction ShiftDoubleAction = Subtarget.isSHLDSlow() ? Custom : Legal;
249
250 setOperationAction(ShiftOp , MVT::i8 , Custom);
251 setOperationAction(ShiftOp , MVT::i16 , Custom);
252 setOperationAction(ShiftOp , MVT::i32 , ShiftDoubleAction);
253 if (Subtarget.is64Bit())
254 setOperationAction(ShiftOp , MVT::i64 , ShiftDoubleAction);
255 }
256
257 if (!Subtarget.useSoftFloat()) {
258 // Promote all UINT_TO_FP to larger SINT_TO_FP's, as X86 doesn't have this
259 // operation.
260 setOperationAction(ISD::UINT_TO_FP, MVT::i8, Promote);
261 setOperationAction(ISD::STRICT_UINT_TO_FP, MVT::i8, Promote);
262 setOperationAction(ISD::UINT_TO_FP, MVT::i16, Promote);
263 setOperationAction(ISD::STRICT_UINT_TO_FP, MVT::i16, Promote);
264 // We have an algorithm for SSE2, and we turn this into a 64-bit
265 // FILD or VCVTUSI2SS/SD for other targets.
266 setOperationAction(ISD::UINT_TO_FP, MVT::i32, Custom);
267 setOperationAction(ISD::STRICT_UINT_TO_FP, MVT::i32, Custom);
268 // We have an algorithm for SSE2->double, and we turn this into a
269 // 64-bit FILD followed by conditional FADD for other targets.
270 setOperationAction(ISD::UINT_TO_FP, MVT::i64, Custom);
271 setOperationAction(ISD::STRICT_UINT_TO_FP, MVT::i64, Custom);
272
273 // Promote i8 SINT_TO_FP to larger SINT_TO_FP's, as X86 doesn't have
274 // this operation.
275 setOperationAction(ISD::SINT_TO_FP, MVT::i8, Promote);
276 setOperationAction(ISD::STRICT_SINT_TO_FP, MVT::i8, Promote);
277 // SSE has no i16 to fp conversion, only i32. We promote in the handler
278 // to allow f80 to use i16 and f64 to use i16 with sse1 only
279 setOperationAction(ISD::SINT_TO_FP, MVT::i16, Custom);
280 setOperationAction(ISD::STRICT_SINT_TO_FP, MVT::i16, Custom);
281 // f32 and f64 cases are Legal with SSE1/SSE2, f80 case is not
282 setOperationAction(ISD::SINT_TO_FP, MVT::i32, Custom);
283 setOperationAction(ISD::STRICT_SINT_TO_FP, MVT::i32, Custom);
284 // In 32-bit mode these are custom lowered. In 64-bit mode F32 and F64
285 // are Legal, f80 is custom lowered.
286 setOperationAction(ISD::SINT_TO_FP, MVT::i64, Custom);
287 setOperationAction(ISD::STRICT_SINT_TO_FP, MVT::i64, Custom);
288
289 // Promote i8 FP_TO_SINT to larger FP_TO_SINTS's, as X86 doesn't have
290 // this operation.
291 setOperationAction(ISD::FP_TO_SINT, MVT::i8, Promote);
292 // FIXME: This doesn't generate invalid exception when it should. PR44019.
293 setOperationAction(ISD::STRICT_FP_TO_SINT, MVT::i8, Promote);
294 setOperationAction(ISD::FP_TO_SINT, MVT::i16, Custom);
295 setOperationAction(ISD::STRICT_FP_TO_SINT, MVT::i16, Custom);
296 setOperationAction(ISD::FP_TO_SINT, MVT::i32, Custom);
297 setOperationAction(ISD::STRICT_FP_TO_SINT, MVT::i32, Custom);
298 // In 32-bit mode these are custom lowered. In 64-bit mode F32 and F64
299 // are Legal, f80 is custom lowered.
300 setOperationAction(ISD::FP_TO_SINT, MVT::i64, Custom);
301 setOperationAction(ISD::STRICT_FP_TO_SINT, MVT::i64, Custom);
302
303 // Handle FP_TO_UINT by promoting the destination to a larger signed
304 // conversion.
305 setOperationAction(ISD::FP_TO_UINT, MVT::i8, Promote);
306 // FIXME: This doesn't generate invalid exception when it should. PR44019.
307 setOperationAction(ISD::STRICT_FP_TO_UINT, MVT::i8, Promote);
308 setOperationAction(ISD::FP_TO_UINT, MVT::i16, Promote);
309 // FIXME: This doesn't generate invalid exception when it should. PR44019.
310 setOperationAction(ISD::STRICT_FP_TO_UINT, MVT::i16, Promote);
311 setOperationAction(ISD::FP_TO_UINT, MVT::i32, Custom);
312 setOperationAction(ISD::STRICT_FP_TO_UINT, MVT::i32, Custom);
313 setOperationAction(ISD::FP_TO_UINT, MVT::i64, Custom);
314 setOperationAction(ISD::STRICT_FP_TO_UINT, MVT::i64, Custom);
315
316 setOperationAction(ISD::LRINT, MVT::f32, Custom);
317 setOperationAction(ISD::LRINT, MVT::f64, Custom);
318 setOperationAction(ISD::LLRINT, MVT::f32, Custom);
319 setOperationAction(ISD::LLRINT, MVT::f64, Custom);
320
321 if (!Subtarget.is64Bit()) {
322 setOperationAction(ISD::LRINT, MVT::i64, Custom);
323 setOperationAction(ISD::LLRINT, MVT::i64, Custom);
324 }
325 }
326
327 if (Subtarget.hasSSE2()) {
328 // Custom lowering for saturating float to int conversions.
329 // We handle promotion to larger result types manually.
330 for (MVT VT : { MVT::i8, MVT::i16, MVT::i32 }) {
331 setOperationAction(ISD::FP_TO_UINT_SAT, VT, Custom);
332 setOperationAction(ISD::FP_TO_SINT_SAT, VT, Custom);
333 }
334 if (Subtarget.is64Bit()) {
335 setOperationAction(ISD::FP_TO_UINT_SAT, MVT::i64, Custom);
336 setOperationAction(ISD::FP_TO_SINT_SAT, MVT::i64, Custom);
337 }
338 }
339
340 // Handle address space casts between mixed sized pointers.
341 setOperationAction(ISD::ADDRSPACECAST, MVT::i32, Custom);
342 setOperationAction(ISD::ADDRSPACECAST, MVT::i64, Custom);
343
344 // TODO: when we have SSE, these could be more efficient, by using movd/movq.
345 if (!Subtarget.hasSSE2()) {
346 setOperationAction(ISD::BITCAST , MVT::f32 , Expand);
347 setOperationAction(ISD::BITCAST , MVT::i32 , Expand);
348 if (Subtarget.is64Bit()) {
349 setOperationAction(ISD::BITCAST , MVT::f64 , Expand);
350 // Without SSE, i64->f64 goes through memory.
351 setOperationAction(ISD::BITCAST , MVT::i64 , Expand);
352 }
353 } else if (!Subtarget.is64Bit())
354 setOperationAction(ISD::BITCAST , MVT::i64 , Custom);
355
356 // Scalar integer divide and remainder are lowered to use operations that
357 // produce two results, to match the available instructions. This exposes
358 // the two-result form to trivial CSE, which is able to combine x/y and x%y
359 // into a single instruction.
360 //
361 // Scalar integer multiply-high is also lowered to use two-result
362 // operations, to match the available instructions. However, plain multiply
363 // (low) operations are left as Legal, as there are single-result
364 // instructions for this in x86. Using the two-result multiply instructions
365 // when both high and low results are needed must be arranged by dagcombine.
366 for (auto VT : { MVT::i8, MVT::i16, MVT::i32, MVT::i64 }) {
367 setOperationAction(ISD::MULHS, VT, Expand);
368 setOperationAction(ISD::MULHU, VT, Expand);
369 setOperationAction(ISD::SDIV, VT, Expand);
370 setOperationAction(ISD::UDIV, VT, Expand);
371 setOperationAction(ISD::SREM, VT, Expand);
372 setOperationAction(ISD::UREM, VT, Expand);
373 }
374
375 setOperationAction(ISD::BR_JT , MVT::Other, Expand);
376 setOperationAction(ISD::BRCOND , MVT::Other, Custom);
377 for (auto VT : { MVT::f32, MVT::f64, MVT::f80, MVT::f128,
378 MVT::i8, MVT::i16, MVT::i32, MVT::i64 }) {
379 setOperationAction(ISD::BR_CC, VT, Expand);
380 setOperationAction(ISD::SELECT_CC, VT, Expand);
381 }
382 if (Subtarget.is64Bit())
383 setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i32, Legal);
384 setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i16 , Legal);
385 setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i8 , Legal);
386 setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i1 , Expand);
387
388 setOperationAction(ISD::FREM , MVT::f32 , Expand);
389 setOperationAction(ISD::FREM , MVT::f64 , Expand);
390 setOperationAction(ISD::FREM , MVT::f80 , Expand);
391 setOperationAction(ISD::FREM , MVT::f128 , Expand);
392
393 if (!Subtarget.useSoftFloat() && Subtarget.hasX87()) {
394 setOperationAction(ISD::GET_ROUNDING , MVT::i32 , Custom);
395 setOperationAction(ISD::SET_ROUNDING , MVT::Other, Custom);
396 setOperationAction(ISD::GET_FPENV_MEM , MVT::Other, Custom);
397 setOperationAction(ISD::SET_FPENV_MEM , MVT::Other, Custom);
398 setOperationAction(ISD::RESET_FPENV , MVT::Other, Custom);
399 }
400
401 // Promote the i8 variants and force them on up to i32 which has a shorter
402 // encoding.
403 setOperationPromotedToType(ISD::CTTZ , MVT::i8 , MVT::i32);
404 setOperationPromotedToType(ISD::CTTZ_ZERO_UNDEF, MVT::i8 , MVT::i32);
405 // Promoted i16. tzcntw has a false dependency on Intel CPUs. For BSF, we emit
406 // a REP prefix to encode it as TZCNT for modern CPUs so it makes sense to
407 // promote that too.
408 setOperationPromotedToType(ISD::CTTZ , MVT::i16 , MVT::i32);
409 setOperationPromotedToType(ISD::CTTZ_ZERO_UNDEF, MVT::i16 , MVT::i32);
410
411 if (!Subtarget.hasBMI()) {
412 setOperationAction(ISD::CTTZ , MVT::i32 , Custom);
413 setOperationAction(ISD::CTTZ_ZERO_UNDEF, MVT::i32 , Legal);
414 if (Subtarget.is64Bit()) {
415 setOperationAction(ISD::CTTZ , MVT::i64 , Custom);
416 setOperationAction(ISD::CTTZ_ZERO_UNDEF, MVT::i64, Legal);
417 }
418 }
419
420 if (Subtarget.hasLZCNT()) {
421 // When promoting the i8 variants, force them to i32 for a shorter
422 // encoding.
423 setOperationPromotedToType(ISD::CTLZ , MVT::i8 , MVT::i32);
424 setOperationPromotedToType(ISD::CTLZ_ZERO_UNDEF, MVT::i8 , MVT::i32);
425 } else {
426 for (auto VT : {MVT::i8, MVT::i16, MVT::i32, MVT::i64}) {
427 if (VT == MVT::i64 && !Subtarget.is64Bit())
428 continue;
429 setOperationAction(ISD::CTLZ , VT, Custom);
430 setOperationAction(ISD::CTLZ_ZERO_UNDEF, VT, Custom);
431 }
432 }
433
434 for (auto Op : {ISD::FP16_TO_FP, ISD::STRICT_FP16_TO_FP, ISD::FP_TO_FP16,
435 ISD::STRICT_FP_TO_FP16}) {
436 // Special handling for half-precision floating point conversions.
437 // If we don't have F16C support, then lower half float conversions
438 // into library calls.
439 setOperationAction(
440 Op, MVT::f32,
441 (!Subtarget.useSoftFloat() && Subtarget.hasF16C()) ? Custom : Expand);
442 // There's never any support for operations beyond MVT::f32.
443 setOperationAction(Op, MVT::f64, Expand);
444 setOperationAction(Op, MVT::f80, Expand);
445 setOperationAction(Op, MVT::f128, Expand);
446 }
447
448 for (auto VT : {MVT::f32, MVT::f64, MVT::f80, MVT::f128}) {
449 setOperationAction(ISD::STRICT_FP_TO_BF16, VT, Expand);
450 setOperationAction(ISD::STRICT_BF16_TO_FP, VT, Expand);
451 }
452
453 for (MVT VT : {MVT::f32, MVT::f64, MVT::f80, MVT::f128}) {
454 setLoadExtAction(ISD::EXTLOAD, VT, MVT::f16, Expand);
455 setLoadExtAction(ISD::EXTLOAD, VT, MVT::bf16, Expand);
456 setTruncStoreAction(VT, MVT::f16, Expand);
457 setTruncStoreAction(VT, MVT::bf16, Expand);
458
459 setOperationAction(ISD::BF16_TO_FP, VT, Expand);
460 setOperationAction(ISD::FP_TO_BF16, VT, Custom);
461 }
462
463 setOperationAction(ISD::PARITY, MVT::i8, Custom);
464 setOperationAction(ISD::PARITY, MVT::i16, Custom);
465 setOperationAction(ISD::PARITY, MVT::i32, Custom);
466 if (Subtarget.is64Bit())
467 setOperationAction(ISD::PARITY, MVT::i64, Custom);
468 if (Subtarget.hasPOPCNT()) {
469 setOperationPromotedToType(ISD::CTPOP, MVT::i8, MVT::i32);
470 // popcntw is longer to encode than popcntl and also has a false dependency
471 // on the dest that popcntl hasn't had since Cannon Lake.
472 setOperationPromotedToType(ISD::CTPOP, MVT::i16, MVT::i32);
473 } else {
474 setOperationAction(ISD::CTPOP , MVT::i8 , Custom);
475 setOperationAction(ISD::CTPOP , MVT::i16 , Custom);
476 setOperationAction(ISD::CTPOP , MVT::i32 , Custom);
477 setOperationAction(ISD::CTPOP , MVT::i64 , Custom);
478 }
479
480 setOperationAction(ISD::READCYCLECOUNTER , MVT::i64 , Custom);
481
482 if (!Subtarget.hasMOVBE())
483 setOperationAction(ISD::BSWAP , MVT::i16 , Expand);
484
485 // X86 wants to expand cmov itself.
486 for (auto VT : { MVT::f32, MVT::f64, MVT::f80, MVT::f128 }) {
487 setOperationAction(ISD::SELECT, VT, Custom);
488 setOperationAction(ISD::SETCC, VT, Custom);
489 setOperationAction(ISD::STRICT_FSETCC, VT, Custom);
490 setOperationAction(ISD::STRICT_FSETCCS, VT, Custom);
491 }
492 for (auto VT : { MVT::i8, MVT::i16, MVT::i32, MVT::i64 }) {
493 if (VT == MVT::i64 && !Subtarget.is64Bit())
494 continue;
495 setOperationAction(ISD::SELECT, VT, Custom);
496 setOperationAction(ISD::SETCC, VT, Custom);
497 }
498
499 // Custom action for SELECT MMX and expand action for SELECT_CC MMX
500 setOperationAction(ISD::SELECT, MVT::x86mmx, Custom);
501 setOperationAction(ISD::SELECT_CC, MVT::x86mmx, Expand);
502
503 setOperationAction(ISD::EH_RETURN , MVT::Other, Custom);
504 // NOTE: EH_SJLJ_SETJMP/_LONGJMP are not recommended, since
505 // LLVM/Clang supports zero-cost DWARF and SEH exception handling.
506 setOperationAction(ISD::EH_SJLJ_SETJMP, MVT::i32, Custom);
507 setOperationAction(ISD::EH_SJLJ_LONGJMP, MVT::Other, Custom);
508 setOperationAction(ISD::EH_SJLJ_SETUP_DISPATCH, MVT::Other, Custom);
509 if (TM.Options.ExceptionModel == ExceptionHandling::SjLj)
510 setLibcallName(RTLIB::UNWIND_RESUME, "_Unwind_SjLj_Resume");
511
512 // Darwin ABI issue.
513 for (auto VT : { MVT::i32, MVT::i64 }) {
514 if (VT == MVT::i64 && !Subtarget.is64Bit())
515 continue;
516 setOperationAction(ISD::ConstantPool , VT, Custom);
517 setOperationAction(ISD::JumpTable , VT, Custom);
518 setOperationAction(ISD::GlobalAddress , VT, Custom);
519 setOperationAction(ISD::GlobalTLSAddress, VT, Custom);
520 setOperationAction(ISD::ExternalSymbol , VT, Custom);
521 setOperationAction(ISD::BlockAddress , VT, Custom);
522 }
523
524 // 64-bit shl, sra, srl (iff 32-bit x86)
525 for (auto VT : { MVT::i32, MVT::i64 }) {
526 if (VT == MVT::i64 && !Subtarget.is64Bit())
527 continue;
528 setOperationAction(ISD::SHL_PARTS, VT, Custom);
529 setOperationAction(ISD::SRA_PARTS, VT, Custom);
530 setOperationAction(ISD::SRL_PARTS, VT, Custom);
531 }
532
533 if (Subtarget.hasSSEPrefetch())
534 setOperationAction(ISD::PREFETCH , MVT::Other, Custom);
535
536 setOperationAction(ISD::ATOMIC_FENCE , MVT::Other, Custom);
537
538 // Expand certain atomics
539 for (auto VT : { MVT::i8, MVT::i16, MVT::i32, MVT::i64 }) {
540 setOperationAction(ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS, VT, Custom);
541 setOperationAction(ISD::ATOMIC_LOAD_SUB, VT, Custom);
542 setOperationAction(ISD::ATOMIC_LOAD_ADD, VT, Custom);
543 setOperationAction(ISD::ATOMIC_LOAD_OR, VT, Custom);
544 setOperationAction(ISD::ATOMIC_LOAD_XOR, VT, Custom);
545 setOperationAction(ISD::ATOMIC_LOAD_AND, VT, Custom);
546 setOperationAction(ISD::ATOMIC_STORE, VT, Custom);
547 }
548
549 if (!Subtarget.is64Bit())
550 setOperationAction(ISD::ATOMIC_LOAD, MVT::i64, Custom);
551
552 if (Subtarget.is64Bit() && Subtarget.hasAVX()) {
553 // All CPUs supporting AVX will atomically load/store aligned 128-bit
554 // values, so we can emit [V]MOVAPS/[V]MOVDQA.
555 setOperationAction(ISD::ATOMIC_LOAD, MVT::i128, Custom);
556 setOperationAction(ISD::ATOMIC_STORE, MVT::i128, Custom);
557 }
558
559 if (Subtarget.canUseCMPXCHG16B())
560 setOperationAction(ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS, MVT::i128, Custom);
561
562 // FIXME - use subtarget debug flags
563 if (!Subtarget.isTargetDarwin() && !Subtarget.isTargetELF() &&
564 !Subtarget.isTargetCygMing() && !Subtarget.isTargetWin64() &&
565 TM.Options.ExceptionModel != ExceptionHandling::SjLj) {
566 setOperationAction(ISD::EH_LABEL, MVT::Other, Expand);
567 }
568
569 setOperationAction(ISD::FRAME_TO_ARGS_OFFSET, MVT::i32, Custom);
570 setOperationAction(ISD::FRAME_TO_ARGS_OFFSET, MVT::i64, Custom);
571
572 setOperationAction(ISD::INIT_TRAMPOLINE, MVT::Other, Custom);
573 setOperationAction(ISD::ADJUST_TRAMPOLINE, MVT::Other, Custom);
574
575 setOperationAction(ISD::TRAP, MVT::Other, Legal);
576 setOperationAction(ISD::DEBUGTRAP, MVT::Other, Legal);
577 if (Subtarget.isTargetPS())
578 setOperationAction(ISD::UBSANTRAP, MVT::Other, Expand);
579 else
580 setOperationAction(ISD::UBSANTRAP, MVT::Other, Legal);
581
582 // VASTART needs to be custom lowered to use the VarArgsFrameIndex
583 setOperationAction(ISD::VASTART , MVT::Other, Custom);
584 setOperationAction(ISD::VAEND , MVT::Other, Expand);
585 bool Is64Bit = Subtarget.is64Bit();
586 setOperationAction(ISD::VAARG, MVT::Other, Is64Bit ? Custom : Expand);
587 setOperationAction(ISD::VACOPY, MVT::Other, Is64Bit ? Custom : Expand);
588
589 setOperationAction(ISD::STACKSAVE, MVT::Other, Expand);
590 setOperationAction(ISD::STACKRESTORE, MVT::Other, Expand);
591
592 setOperationAction(ISD::DYNAMIC_STACKALLOC, PtrVT, Custom);
593
594 // GC_TRANSITION_START and GC_TRANSITION_END need custom lowering.
595 setOperationAction(ISD::GC_TRANSITION_START, MVT::Other, Custom);
596 setOperationAction(ISD::GC_TRANSITION_END, MVT::Other, Custom);
597
598 setOperationAction(ISD::STRICT_FP_EXTEND, MVT::f64, Legal);
599
600 auto setF16Action = [&] (MVT VT, LegalizeAction Action) {
601 setOperationAction(ISD::FABS, VT, Action);
602 setOperationAction(ISD::FNEG, VT, Action);
603 setOperationAction(ISD::FCOPYSIGN, VT, Expand);
604 setOperationAction(ISD::FREM, VT, Action);
605 setOperationAction(ISD::FMA, VT, Action);
606 setOperationAction(ISD::FMINNUM, VT, Action);
607 setOperationAction(ISD::FMAXNUM, VT, Action);
608 setOperationAction(ISD::FMINIMUM, VT, Action);
609 setOperationAction(ISD::FMAXIMUM, VT, Action);
610 setOperationAction(ISD::FSIN, VT, Action);
611 setOperationAction(ISD::FCOS, VT, Action);
612 setOperationAction(ISD::FSINCOS, VT, Action);
613 setOperationAction(ISD::FTAN, VT, Action);
614 setOperationAction(ISD::FSQRT, VT, Action);
615 setOperationAction(ISD::FPOW, VT, Action);
616 setOperationAction(ISD::FLOG, VT, Action);
617 setOperationAction(ISD::FLOG2, VT, Action);
618 setOperationAction(ISD::FLOG10, VT, Action);
619 setOperationAction(ISD::FEXP, VT, Action);
620 setOperationAction(ISD::FEXP2, VT, Action);
621 setOperationAction(ISD::FEXP10, VT, Action);
622 setOperationAction(ISD::FCEIL, VT, Action);
623 setOperationAction(ISD::FFLOOR, VT, Action);
624 setOperationAction(ISD::FNEARBYINT, VT, Action);
625 setOperationAction(ISD::FRINT, VT, Action);
626 setOperationAction(ISD::BR_CC, VT, Action);
627 setOperationAction(ISD::SETCC, VT, Action);
628 setOperationAction(ISD::SELECT, VT, Custom);
629 setOperationAction(ISD::SELECT_CC, VT, Action);
630 setOperationAction(ISD::FROUND, VT, Action);
631 setOperationAction(ISD::FROUNDEVEN, VT, Action);
632 setOperationAction(ISD::FTRUNC, VT, Action);
633 setOperationAction(ISD::FLDEXP, VT, Action);
634 };
635
636 if (!Subtarget.useSoftFloat() && Subtarget.hasSSE2()) {
637 // f16, f32 and f64 use SSE.
638 // Set up the FP register classes.
639 addRegisterClass(MVT::f16, Subtarget.hasAVX512() ? &X86::FR16XRegClass
640 : &X86::FR16RegClass);
641 addRegisterClass(MVT::f32, Subtarget.hasAVX512() ? &X86::FR32XRegClass
642 : &X86::FR32RegClass);
643 addRegisterClass(MVT::f64, Subtarget.hasAVX512() ? &X86::FR64XRegClass
644 : &X86::FR64RegClass);
645
646 // Disable f32->f64 extload as we can only generate this in one instruction
647 // under optsize. So its easier to pattern match (fpext (load)) for that
648 // case instead of needing to emit 2 instructions for extload in the
649 // non-optsize case.
650 setLoadExtAction(ISD::EXTLOAD, MVT::f64, MVT::f32, Expand);
651
652 for (auto VT : { MVT::f32, MVT::f64 }) {
653 // Use ANDPD to simulate FABS.
654 setOperationAction(ISD::FABS, VT, Custom);
655
656 // Use XORP to simulate FNEG.
657 setOperationAction(ISD::FNEG, VT, Custom);
658
659 // Use ANDPD and ORPD to simulate FCOPYSIGN.
660 setOperationAction(ISD::FCOPYSIGN, VT, Custom);
661
662 // These might be better off as horizontal vector ops.
663 setOperationAction(ISD::FADD, VT, Custom);
664 setOperationAction(ISD::FSUB, VT, Custom);
665
666 // We don't support sin/cos/fmod
667 setOperationAction(ISD::FSIN , VT, Expand);
668 setOperationAction(ISD::FCOS , VT, Expand);
669 setOperationAction(ISD::FSINCOS, VT, Expand);
670 }
671
672 // Half type will be promoted by default.
673 setF16Action(MVT::f16, Promote);
674 setOperationAction(ISD::FADD, MVT::f16, Promote);
675 setOperationAction(ISD::FSUB, MVT::f16, Promote);
676 setOperationAction(ISD::FMUL, MVT::f16, Promote);
677 setOperationAction(ISD::FDIV, MVT::f16, Promote);
678 setOperationAction(ISD::FP_ROUND, MVT::f16, Custom);
679 setOperationAction(ISD::FP_EXTEND, MVT::f32, Custom);
680 setOperationAction(ISD::FP_EXTEND, MVT::f64, Custom);
681
682 setOperationAction(ISD::STRICT_FADD, MVT::f16, Promote);
683 setOperationAction(ISD::STRICT_FSUB, MVT::f16, Promote);
684 setOperationAction(ISD::STRICT_FMUL, MVT::f16, Promote);
685 setOperationAction(ISD::STRICT_FDIV, MVT::f16, Promote);
686 setOperationAction(ISD::STRICT_FMA, MVT::f16, Promote);
687 setOperationAction(ISD::STRICT_FMINNUM, MVT::f16, Promote);
688 setOperationAction(ISD::STRICT_FMAXNUM, MVT::f16, Promote);
689 setOperationAction(ISD::STRICT_FMINIMUM, MVT::f16, Promote);
690 setOperationAction(ISD::STRICT_FMAXIMUM, MVT::f16, Promote);
691 setOperationAction(ISD::STRICT_FSQRT, MVT::f16, Promote);
692 setOperationAction(ISD::STRICT_FPOW, MVT::f16, Promote);
693 setOperationAction(ISD::STRICT_FLDEXP, MVT::f16, Promote);
694 setOperationAction(ISD::STRICT_FLOG, MVT::f16, Promote);
695 setOperationAction(ISD::STRICT_FLOG2, MVT::f16, Promote);
696 setOperationAction(ISD::STRICT_FLOG10, MVT::f16, Promote);
697 setOperationAction(ISD::STRICT_FEXP, MVT::f16, Promote);
698 setOperationAction(ISD::STRICT_FEXP2, MVT::f16, Promote);
699 setOperationAction(ISD::STRICT_FCEIL, MVT::f16, Promote);
700 setOperationAction(ISD::STRICT_FFLOOR, MVT::f16, Promote);
701 setOperationAction(ISD::STRICT_FNEARBYINT, MVT::f16, Promote);
702 setOperationAction(ISD::STRICT_FRINT, MVT::f16, Promote);
703 setOperationAction(ISD::STRICT_FSETCC, MVT::f16, Promote);
704 setOperationAction(ISD::STRICT_FSETCCS, MVT::f16, Promote);
705 setOperationAction(ISD::STRICT_FROUND, MVT::f16, Promote);
706 setOperationAction(ISD::STRICT_FROUNDEVEN, MVT::f16, Promote);
707 setOperationAction(ISD::STRICT_FTRUNC, MVT::f16, Promote);
708 setOperationAction(ISD::STRICT_FP_ROUND, MVT::f16, Custom);
709 setOperationAction(ISD::STRICT_FP_EXTEND, MVT::f32, Custom);
710 setOperationAction(ISD::STRICT_FP_EXTEND, MVT::f64, Custom);
711
712 setLibcallName(RTLIB::FPROUND_F32_F16, "__truncsfhf2");
713 setLibcallName(RTLIB::FPEXT_F16_F32, "__extendhfsf2");
714
715 // Lower this to MOVMSK plus an AND.
716 setOperationAction(ISD::FGETSIGN, MVT::i64, Custom);
717 setOperationAction(ISD::FGETSIGN, MVT::i32, Custom);
718
719 } else if (!Subtarget.useSoftFloat() && Subtarget.hasSSE1() &&
720 (UseX87 || Is64Bit)) {
721 // Use SSE for f32, x87 for f64.
722 // Set up the FP register classes.
723 addRegisterClass(MVT::f32, &X86::FR32RegClass);
724 if (UseX87)
725 addRegisterClass(MVT::f64, &X86::RFP64RegClass);
726
727 // Use ANDPS to simulate FABS.
728 setOperationAction(ISD::FABS , MVT::f32, Custom);
729
730 // Use XORP to simulate FNEG.
731 setOperationAction(ISD::FNEG , MVT::f32, Custom);
732
733 if (UseX87)
734 setOperationAction(ISD::UNDEF, MVT::f64, Expand);
735
736 // Use ANDPS and ORPS to simulate FCOPYSIGN.
737 if (UseX87)
738 setOperationAction(ISD::FCOPYSIGN, MVT::f64, Expand);
739 setOperationAction(ISD::FCOPYSIGN, MVT::f32, Custom);
740
741 // We don't support sin/cos/fmod
742 setOperationAction(ISD::FSIN , MVT::f32, Expand);
743 setOperationAction(ISD::FCOS , MVT::f32, Expand);
744 setOperationAction(ISD::FSINCOS, MVT::f32, Expand);
745
746 if (UseX87) {
747 // Always expand sin/cos functions even though x87 has an instruction.
748 setOperationAction(ISD::FSIN, MVT::f64, Expand);
749 setOperationAction(ISD::FCOS, MVT::f64, Expand);
750 setOperationAction(ISD::FSINCOS, MVT::f64, Expand);
751 }
752 } else if (UseX87) {
753 // f32 and f64 in x87.
754 // Set up the FP register classes.
755 addRegisterClass(MVT::f64, &X86::RFP64RegClass);
756 addRegisterClass(MVT::f32, &X86::RFP32RegClass);
757
758 for (auto VT : { MVT::f32, MVT::f64 }) {
759 setOperationAction(ISD::UNDEF, VT, Expand);
760 setOperationAction(ISD::FCOPYSIGN, VT, Expand);
761
762 // Always expand sin/cos functions even though x87 has an instruction.
763 setOperationAction(ISD::FSIN , VT, Expand);
764 setOperationAction(ISD::FCOS , VT, Expand);
765 setOperationAction(ISD::FSINCOS, VT, Expand);
766 }
767 }
768
769 // Expand FP32 immediates into loads from the stack, save special cases.
770 if (isTypeLegal(MVT::f32)) {
771 if (UseX87 && (getRegClassFor(MVT::f32) == &X86::RFP32RegClass)) {
772 addLegalFPImmediate(APFloat(+0.0f)); // FLD0
773 addLegalFPImmediate(APFloat(+1.0f)); // FLD1
774 addLegalFPImmediate(APFloat(-0.0f)); // FLD0/FCHS
775 addLegalFPImmediate(APFloat(-1.0f)); // FLD1/FCHS
776 } else // SSE immediates.
777 addLegalFPImmediate(APFloat(+0.0f)); // xorps
778 }
779 // Expand FP64 immediates into loads from the stack, save special cases.
780 if (isTypeLegal(MVT::f64)) {
781 if (UseX87 && getRegClassFor(MVT::f64) == &X86::RFP64RegClass) {
782 addLegalFPImmediate(APFloat(+0.0)); // FLD0
783 addLegalFPImmediate(APFloat(+1.0)); // FLD1
784 addLegalFPImmediate(APFloat(-0.0)); // FLD0/FCHS
785 addLegalFPImmediate(APFloat(-1.0)); // FLD1/FCHS
786 } else // SSE immediates.
787 addLegalFPImmediate(APFloat(+0.0)); // xorpd
788 }
789 // Support fp16 0 immediate.
790 if (isTypeLegal(MVT::f16))
791 addLegalFPImmediate(APFloat::getZero(APFloat::IEEEhalf()));
792
793 // Handle constrained floating-point operations of scalar.
794 setOperationAction(ISD::STRICT_FADD, MVT::f32, Legal);
795 setOperationAction(ISD::STRICT_FADD, MVT::f64, Legal);
796 setOperationAction(ISD::STRICT_FSUB, MVT::f32, Legal);
797 setOperationAction(ISD::STRICT_FSUB, MVT::f64, Legal);
798 setOperationAction(ISD::STRICT_FMUL, MVT::f32, Legal);
799 setOperationAction(ISD::STRICT_FMUL, MVT::f64, Legal);
800 setOperationAction(ISD::STRICT_FDIV, MVT::f32, Legal);
801 setOperationAction(ISD::STRICT_FDIV, MVT::f64, Legal);
802 setOperationAction(ISD::STRICT_FP_ROUND, MVT::f32, Legal);
803 setOperationAction(ISD::STRICT_FP_ROUND, MVT::f64, Legal);
804 setOperationAction(ISD::STRICT_FSQRT, MVT::f32, Legal);
805 setOperationAction(ISD::STRICT_FSQRT, MVT::f64, Legal);
806
807 // We don't support FMA.
808 setOperationAction(ISD::FMA, MVT::f64, Expand);
809 setOperationAction(ISD::FMA, MVT::f32, Expand);
810
811 // f80 always uses X87.
812 if (UseX87) {
813 addRegisterClass(MVT::f80, &X86::RFP80RegClass);
814 setOperationAction(ISD::UNDEF, MVT::f80, Expand);
815 setOperationAction(ISD::FCOPYSIGN, MVT::f80, Expand);
816 {
817 APFloat TmpFlt = APFloat::getZero(APFloat::x87DoubleExtended());
818 addLegalFPImmediate(TmpFlt); // FLD0
819 TmpFlt.changeSign();
820 addLegalFPImmediate(TmpFlt); // FLD0/FCHS
821
822 bool ignored;
823 APFloat TmpFlt2(+1.0);
824 TmpFlt2.convert(APFloat::x87DoubleExtended(), APFloat::rmNearestTiesToEven,
825 &ignored);
826 addLegalFPImmediate(TmpFlt2); // FLD1
827 TmpFlt2.changeSign();
828 addLegalFPImmediate(TmpFlt2); // FLD1/FCHS
829 }
830
831 // Always expand sin/cos functions even though x87 has an instruction.
832 // clang-format off
833 setOperationAction(ISD::FSIN , MVT::f80, Expand);
834 setOperationAction(ISD::FCOS , MVT::f80, Expand);
835 setOperationAction(ISD::FSINCOS, MVT::f80, Expand);
836 setOperationAction(ISD::FTAN , MVT::f80, Expand);
837 setOperationAction(ISD::FASIN , MVT::f80, Expand);
838 setOperationAction(ISD::FACOS , MVT::f80, Expand);
839 setOperationAction(ISD::FATAN , MVT::f80, Expand);
840 setOperationAction(ISD::FSINH , MVT::f80, Expand);
841 setOperationAction(ISD::FCOSH , MVT::f80, Expand);
842 setOperationAction(ISD::FTANH , MVT::f80, Expand);
843 // clang-format on
844
845 setOperationAction(ISD::FFLOOR, MVT::f80, Expand);
846 setOperationAction(ISD::FCEIL, MVT::f80, Expand);
847 setOperationAction(ISD::FTRUNC, MVT::f80, Expand);
848 setOperationAction(ISD::FRINT, MVT::f80, Expand);
849 setOperationAction(ISD::FNEARBYINT, MVT::f80, Expand);
850 setOperationAction(ISD::FROUNDEVEN, MVT::f80, Expand);
851 setOperationAction(ISD::FMA, MVT::f80, Expand);
852 setOperationAction(ISD::LROUND, MVT::f80, LibCall);
853 setOperationAction(ISD::LLROUND, MVT::f80, LibCall);
854 setOperationAction(ISD::LRINT, MVT::f80, Custom);
855 setOperationAction(ISD::LLRINT, MVT::f80, Custom);
856
857 // Handle constrained floating-point operations of scalar.
858 setOperationAction(ISD::STRICT_FADD , MVT::f80, Legal);
859 setOperationAction(ISD::STRICT_FSUB , MVT::f80, Legal);
860 setOperationAction(ISD::STRICT_FMUL , MVT::f80, Legal);
861 setOperationAction(ISD::STRICT_FDIV , MVT::f80, Legal);
862 setOperationAction(ISD::STRICT_FSQRT , MVT::f80, Legal);
863 if (isTypeLegal(MVT::f16)) {
864 setOperationAction(ISD::FP_EXTEND, MVT::f80, Custom);
865 setOperationAction(ISD::STRICT_FP_EXTEND, MVT::f80, Custom);
866 } else {
867 setOperationAction(ISD::STRICT_FP_EXTEND, MVT::f80, Legal);
868 }
869 // FIXME: When the target is 64-bit, STRICT_FP_ROUND will be overwritten
870 // as Custom.
871 setOperationAction(ISD::STRICT_FP_ROUND, MVT::f80, Legal);
872 }
873
874 // f128 uses xmm registers, but most operations require libcalls.
875 if (!Subtarget.useSoftFloat() && Subtarget.is64Bit() && Subtarget.hasSSE1()) {
876 addRegisterClass(MVT::f128, Subtarget.hasVLX() ? &X86::VR128XRegClass
877 : &X86::VR128RegClass);
878
879 addLegalFPImmediate(APFloat::getZero(APFloat::IEEEquad())); // xorps
880
881 setOperationAction(ISD::FADD, MVT::f128, LibCall);
882 setOperationAction(ISD::STRICT_FADD, MVT::f128, LibCall);
883 setOperationAction(ISD::FSUB, MVT::f128, LibCall);
884 setOperationAction(ISD::STRICT_FSUB, MVT::f128, LibCall);
885 setOperationAction(ISD::FDIV, MVT::f128, LibCall);
886 setOperationAction(ISD::STRICT_FDIV, MVT::f128, LibCall);
887 setOperationAction(ISD::FMUL, MVT::f128, LibCall);
888 setOperationAction(ISD::STRICT_FMUL, MVT::f128, LibCall);
889 setOperationAction(ISD::FMA, MVT::f128, LibCall);
890 setOperationAction(ISD::STRICT_FMA, MVT::f128, LibCall);
891
892 setOperationAction(ISD::FABS, MVT::f128, Custom);
893 setOperationAction(ISD::FNEG, MVT::f128, Custom);
894 setOperationAction(ISD::FCOPYSIGN, MVT::f128, Custom);
895
896 // clang-format off
897 setOperationAction(ISD::FSIN, MVT::f128, LibCall);
898 setOperationAction(ISD::STRICT_FSIN, MVT::f128, LibCall);
899 setOperationAction(ISD::FCOS, MVT::f128, LibCall);
900 setOperationAction(ISD::STRICT_FCOS, MVT::f128, LibCall);
901 setOperationAction(ISD::FSINCOS, MVT::f128, LibCall);
902 setOperationAction(ISD::FTAN, MVT::f128, LibCall);
903 setOperationAction(ISD::STRICT_FTAN, MVT::f128, LibCall);
904 // clang-format on
905 // No STRICT_FSINCOS
906 setOperationAction(ISD::FSQRT, MVT::f128, LibCall);
907 setOperationAction(ISD::STRICT_FSQRT, MVT::f128, LibCall);
908
909 setOperationAction(ISD::FP_EXTEND, MVT::f128, Custom);
910 setOperationAction(ISD::STRICT_FP_EXTEND, MVT::f128, Custom);
911 // We need to custom handle any FP_ROUND with an f128 input, but
912 // LegalizeDAG uses the result type to know when to run a custom handler.
913 // So we have to list all legal floating point result types here.
914 if (isTypeLegal(MVT::f32)) {
915 setOperationAction(ISD::FP_ROUND, MVT::f32, Custom);
916 setOperationAction(ISD::STRICT_FP_ROUND, MVT::f32, Custom);
917 }
918 if (isTypeLegal(MVT::f64)) {
919 setOperationAction(ISD::FP_ROUND, MVT::f64, Custom);
920 setOperationAction(ISD::STRICT_FP_ROUND, MVT::f64, Custom);
921 }
922 if (isTypeLegal(MVT::f80)) {
923 setOperationAction(ISD::FP_ROUND, MVT::f80, Custom);
924 setOperationAction(ISD::STRICT_FP_ROUND, MVT::f80, Custom);
925 }
926
927 setOperationAction(ISD::SETCC, MVT::f128, Custom);
928
929 setLoadExtAction(ISD::EXTLOAD, MVT::f128, MVT::f32, Expand);
930 setLoadExtAction(ISD::EXTLOAD, MVT::f128, MVT::f64, Expand);
931 setLoadExtAction(ISD::EXTLOAD, MVT::f128, MVT::f80, Expand);
932 setTruncStoreAction(MVT::f128, MVT::f32, Expand);
933 setTruncStoreAction(MVT::f128, MVT::f64, Expand);
934 setTruncStoreAction(MVT::f128, MVT::f80, Expand);
935 }
936
937 // Always use a library call for pow.
938 setOperationAction(ISD::FPOW , MVT::f32 , Expand);
939 setOperationAction(ISD::FPOW , MVT::f64 , Expand);
940 setOperationAction(ISD::FPOW , MVT::f80 , Expand);
941 setOperationAction(ISD::FPOW , MVT::f128 , Expand);
942
943 setOperationAction(ISD::FLOG, MVT::f80, Expand);
944 setOperationAction(ISD::FLOG2, MVT::f80, Expand);
945 setOperationAction(ISD::FLOG10, MVT::f80, Expand);
946 setOperationAction(ISD::FEXP, MVT::f80, Expand);
947 setOperationAction(ISD::FEXP2, MVT::f80, Expand);
948 setOperationAction(ISD::FEXP10, MVT::f80, Expand);
949 setOperationAction(ISD::FMINNUM, MVT::f80, Expand);
950 setOperationAction(ISD::FMAXNUM, MVT::f80, Expand);
951
952 // Some FP actions are always expanded for vector types.
953 for (auto VT : { MVT::v8f16, MVT::v16f16, MVT::v32f16,
954 MVT::v4f32, MVT::v8f32, MVT::v16f32,
955 MVT::v2f64, MVT::v4f64, MVT::v8f64 }) {
956 // clang-format off
957 setOperationAction(ISD::FSIN, VT, Expand);
958 setOperationAction(ISD::FSINCOS, VT, Expand);
959 setOperationAction(ISD::FCOS, VT, Expand);
960 setOperationAction(ISD::FTAN, VT, Expand);
961 setOperationAction(ISD::FREM, VT, Expand);
962 setOperationAction(ISD::FCOPYSIGN, VT, Expand);
963 setOperationAction(ISD::FPOW, VT, Expand);
964 setOperationAction(ISD::FLOG, VT, Expand);
965 setOperationAction(ISD::FLOG2, VT, Expand);
966 setOperationAction(ISD::FLOG10, VT, Expand);
967 setOperationAction(ISD::FEXP, VT, Expand);
968 setOperationAction(ISD::FEXP2, VT, Expand);
969 setOperationAction(ISD::FEXP10, VT, Expand);
970 // clang-format on
971 }
972
973 // First set operation action for all vector types to either promote
974 // (for widening) or expand (for scalarization). Then we will selectively
975 // turn on ones that can be effectively codegen'd.
976 for (MVT VT : MVT::fixedlen_vector_valuetypes()) {
977 setOperationAction(ISD::SDIV, VT, Expand);
978 setOperationAction(ISD::UDIV, VT, Expand);
979 setOperationAction(ISD::SREM, VT, Expand);
980 setOperationAction(ISD::UREM, VT, Expand);
981 setOperationAction(ISD::EXTRACT_VECTOR_ELT, VT,Expand);
982 setOperationAction(ISD::INSERT_VECTOR_ELT, VT, Expand);
983 setOperationAction(ISD::EXTRACT_SUBVECTOR, VT,Expand);
984 setOperationAction(ISD::INSERT_SUBVECTOR, VT,Expand);
985 setOperationAction(ISD::FMA, VT, Expand);
986 setOperationAction(ISD::FFLOOR, VT, Expand);
987 setOperationAction(ISD::FCEIL, VT, Expand);
988 setOperationAction(ISD::FTRUNC, VT, Expand);
989 setOperationAction(ISD::FRINT, VT, Expand);
990 setOperationAction(ISD::FNEARBYINT, VT, Expand);
991 setOperationAction(ISD::FROUNDEVEN, VT, Expand);
992 setOperationAction(ISD::SMUL_LOHI, VT, Expand);
993 setOperationAction(ISD::MULHS, VT, Expand);
994 setOperationAction(ISD::UMUL_LOHI, VT, Expand);
995 setOperationAction(ISD::MULHU, VT, Expand);
996 setOperationAction(ISD::SDIVREM, VT, Expand);
997 setOperationAction(ISD::UDIVREM, VT, Expand);
998 setOperationAction(ISD::CTPOP, VT, Expand);
999 setOperationAction(ISD::CTTZ, VT, Expand);
1000 setOperationAction(ISD::CTLZ, VT, Expand);
1001 setOperationAction(ISD::ROTL, VT, Expand);
1002 setOperationAction(ISD::ROTR, VT, Expand);
1003 setOperationAction(ISD::BSWAP, VT, Expand);
1004 setOperationAction(ISD::SETCC, VT, Expand);
1005 setOperationAction(ISD::FP_TO_UINT, VT, Expand);
1006 setOperationAction(ISD::FP_TO_SINT, VT, Expand);
1007 setOperationAction(ISD::UINT_TO_FP, VT, Expand);
1008 setOperationAction(ISD::SINT_TO_FP, VT, Expand);
1009 setOperationAction(ISD::SIGN_EXTEND_INREG, VT,Expand);
1010 setOperationAction(ISD::TRUNCATE, VT, Expand);
1011 setOperationAction(ISD::SIGN_EXTEND, VT, Expand);
1012 setOperationAction(ISD::ZERO_EXTEND, VT, Expand);
1013 setOperationAction(ISD::ANY_EXTEND, VT, Expand);
1014 setOperationAction(ISD::SELECT_CC, VT, Expand);
1015 for (MVT InnerVT : MVT::fixedlen_vector_valuetypes()) {
1016 setTruncStoreAction(InnerVT, VT, Expand);
1017
1018 setLoadExtAction(ISD::SEXTLOAD, InnerVT, VT, Expand);
1019 setLoadExtAction(ISD::ZEXTLOAD, InnerVT, VT, Expand);
1020
1021 // N.b. ISD::EXTLOAD legality is basically ignored except for i1-like
1022 // types, we have to deal with them whether we ask for Expansion or not.
1023 // Setting Expand causes its own optimisation problems though, so leave
1024 // them legal.
1025 if (VT.getVectorElementType() == MVT::i1)
1026 setLoadExtAction(ISD::EXTLOAD, InnerVT, VT, Expand);
1027
1028 // EXTLOAD for MVT::f16 vectors is not legal because f16 vectors are
1029 // split/scalarized right now.
1030 if (VT.getVectorElementType() == MVT::f16 ||
1031 VT.getVectorElementType() == MVT::bf16)
1032 setLoadExtAction(ISD::EXTLOAD, InnerVT, VT, Expand);
1033 }
1034 }
1035
1036 // FIXME: In order to prevent SSE instructions being expanded to MMX ones
1037 // with -msoft-float, disable use of MMX as well.
1038 if (!Subtarget.useSoftFloat() && Subtarget.hasMMX()) {
1039 addRegisterClass(MVT::x86mmx, &X86::VR64RegClass);
1040 // No operations on x86mmx supported, everything uses intrinsics.
1041 }
1042
1043 if (!Subtarget.useSoftFloat() && Subtarget.hasSSE1()) {
1044 addRegisterClass(MVT::v4f32, Subtarget.hasVLX() ? &X86::VR128XRegClass
1045 : &X86::VR128RegClass);
1046
1047 setOperationAction(ISD::FMAXIMUM, MVT::f32, Custom);
1048 setOperationAction(ISD::FMINIMUM, MVT::f32, Custom);
1049
1050 setOperationAction(ISD::FNEG, MVT::v4f32, Custom);
1051 setOperationAction(ISD::FABS, MVT::v4f32, Custom);
1052 setOperationAction(ISD::FCOPYSIGN, MVT::v4f32, Custom);
1053 setOperationAction(ISD::BUILD_VECTOR, MVT::v4f32, Custom);
1054 setOperationAction(ISD::VECTOR_SHUFFLE, MVT::v4f32, Custom);
1055 setOperationAction(ISD::VSELECT, MVT::v4f32, Custom);
1056 setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v4f32, Custom);
1057 setOperationAction(ISD::SELECT, MVT::v4f32, Custom);
1058
1059 setOperationAction(ISD::LOAD, MVT::v2f32, Custom);
1060 setOperationAction(ISD::STORE, MVT::v2f32, Custom);
1061
1062 setOperationAction(ISD::STRICT_FADD, MVT::v4f32, Legal);
1063 setOperationAction(ISD::STRICT_FSUB, MVT::v4f32, Legal);
1064 setOperationAction(ISD::STRICT_FMUL, MVT::v4f32, Legal);
1065 setOperationAction(ISD::STRICT_FDIV, MVT::v4f32, Legal);
1066 setOperationAction(ISD::STRICT_FSQRT, MVT::v4f32, Legal);
1067 }
1068
1069 if (!Subtarget.useSoftFloat() && Subtarget.hasSSE2()) {
1070 addRegisterClass(MVT::v2f64, Subtarget.hasVLX() ? &X86::VR128XRegClass
1071 : &X86::VR128RegClass);
1072
1073 // FIXME: Unfortunately, -soft-float and -no-implicit-float mean XMM
1074 // registers cannot be used even for integer operations.
1075 addRegisterClass(MVT::v16i8, Subtarget.hasVLX() ? &X86::VR128XRegClass
1076 : &X86::VR128RegClass);
1077 addRegisterClass(MVT::v8i16, Subtarget.hasVLX() ? &X86::VR128XRegClass
1078 : &X86::VR128RegClass);
1079 addRegisterClass(MVT::v8f16, Subtarget.hasVLX() ? &X86::VR128XRegClass
1080 : &X86::VR128RegClass);
1081 addRegisterClass(MVT::v4i32, Subtarget.hasVLX() ? &X86::VR128XRegClass
1082 : &X86::VR128RegClass);
1083 addRegisterClass(MVT::v2i64, Subtarget.hasVLX() ? &X86::VR128XRegClass
1084 : &X86::VR128RegClass);
1085
1086 for (auto VT : { MVT::f64, MVT::v4f32, MVT::v2f64 }) {
1087 setOperationAction(ISD::FMAXIMUM, VT, Custom);
1088 setOperationAction(ISD::FMINIMUM, VT, Custom);
1089 }
1090
1091 for (auto VT : { MVT::v2i8, MVT::v4i8, MVT::v8i8,
1092 MVT::v2i16, MVT::v4i16, MVT::v2i32 }) {
1093 setOperationAction(ISD::SDIV, VT, Custom);
1094 setOperationAction(ISD::SREM, VT, Custom);
1095 setOperationAction(ISD::UDIV, VT, Custom);
1096 setOperationAction(ISD::UREM, VT, Custom);
1097 }
1098
1099 setOperationAction(ISD::MUL, MVT::v2i8, Custom);
1100 setOperationAction(ISD::MUL, MVT::v4i8, Custom);
1101 setOperationAction(ISD::MUL, MVT::v8i8, Custom);
1102
1103 setOperationAction(ISD::MUL, MVT::v16i8, Custom);
1104 setOperationAction(ISD::MUL, MVT::v4i32, Custom);
1105 setOperationAction(ISD::MUL, MVT::v2i64, Custom);
1106 setOperationAction(ISD::MULHU, MVT::v4i32, Custom);
1107 setOperationAction(ISD::MULHS, MVT::v4i32, Custom);
1108 setOperationAction(ISD::MULHU, MVT::v16i8, Custom);
1109 setOperationAction(ISD::MULHS, MVT::v16i8, Custom);
1110 setOperationAction(ISD::MULHU, MVT::v8i16, Legal);
1111 setOperationAction(ISD::MULHS, MVT::v8i16, Legal);
1112 setOperationAction(ISD::MUL, MVT::v8i16, Legal);
1113 setOperationAction(ISD::AVGCEILU, MVT::v16i8, Legal);
1114 setOperationAction(ISD::AVGCEILU, MVT::v8i16, Legal);
1115
1116 setOperationAction(ISD::SMULO, MVT::v16i8, Custom);
1117 setOperationAction(ISD::UMULO, MVT::v16i8, Custom);
1118 setOperationAction(ISD::UMULO, MVT::v2i32, Custom);
1119
1120 setOperationAction(ISD::FNEG, MVT::v2f64, Custom);
1121 setOperationAction(ISD::FABS, MVT::v2f64, Custom);
1122 setOperationAction(ISD::FCOPYSIGN, MVT::v2f64, Custom);
1123
1124 setOperationAction(ISD::LRINT, MVT::v4f32, Custom);
1125
1126 for (auto VT : { MVT::v16i8, MVT::v8i16, MVT::v4i32, MVT::v2i64 }) {
1127 setOperationAction(ISD::SMAX, VT, VT == MVT::v8i16 ? Legal : Custom);
1128 setOperationAction(ISD::SMIN, VT, VT == MVT::v8i16 ? Legal : Custom);
1129 setOperationAction(ISD::UMAX, VT, VT == MVT::v16i8 ? Legal : Custom);
1130 setOperationAction(ISD::UMIN, VT, VT == MVT::v16i8 ? Legal : Custom);
1131 }
1132
1133 setOperationAction(ISD::UADDSAT, MVT::v16i8, Legal);
1134 setOperationAction(ISD::SADDSAT, MVT::v16i8, Legal);
1135 setOperationAction(ISD::USUBSAT, MVT::v16i8, Legal);
1136 setOperationAction(ISD::SSUBSAT, MVT::v16i8, Legal);
1137 setOperationAction(ISD::UADDSAT, MVT::v8i16, Legal);
1138 setOperationAction(ISD::SADDSAT, MVT::v8i16, Legal);
1139 setOperationAction(ISD::USUBSAT, MVT::v8i16, Legal);
1140 setOperationAction(ISD::SSUBSAT, MVT::v8i16, Legal);
1141 setOperationAction(ISD::USUBSAT, MVT::v4i32, Custom);
1142 setOperationAction(ISD::USUBSAT, MVT::v2i64, Custom);
1143
1144 setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v16i8, Custom);
1145 setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v8i16, Custom);
1146 setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v4i32, Custom);
1147 setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v4f32, Custom);
1148
1149 for (auto VT : { MVT::v16i8, MVT::v8i16, MVT::v4i32, MVT::v2i64 }) {
1150 setOperationAction(ISD::SETCC, VT, Custom);
1151 setOperationAction(ISD::CTPOP, VT, Custom);
1152 setOperationAction(ISD::ABS, VT, Custom);
1153 setOperationAction(ISD::ABDS, VT, Custom);
1154 setOperationAction(ISD::ABDU, VT, Custom);
1155
1156 // The condition codes aren't legal in SSE/AVX and under AVX512 we use
1157 // setcc all the way to isel and prefer SETGT in some isel patterns.
1158 setCondCodeAction(ISD::SETLT, VT, Custom);
1159 setCondCodeAction(ISD::SETLE, VT, Custom);
1160 }
1161
1162 setOperationAction(ISD::SETCC, MVT::v2f64, Custom);
1163 setOperationAction(ISD::SETCC, MVT::v4f32, Custom);
1164 setOperationAction(ISD::STRICT_FSETCC, MVT::v2f64, Custom);
1165 setOperationAction(ISD::STRICT_FSETCC, MVT::v4f32, Custom);
1166 setOperationAction(ISD::STRICT_FSETCCS, MVT::v2f64, Custom);
1167 setOperationAction(ISD::STRICT_FSETCCS, MVT::v4f32, Custom);
1168
1169 for (auto VT : { MVT::v16i8, MVT::v8i16, MVT::v4i32 }) {
1170 setOperationAction(ISD::SCALAR_TO_VECTOR, VT, Custom);
1171 setOperationAction(ISD::BUILD_VECTOR, VT, Custom);
1172 setOperationAction(ISD::VECTOR_SHUFFLE, VT, Custom);
1173 setOperationAction(ISD::VSELECT, VT, Custom);
1174 setOperationAction(ISD::EXTRACT_VECTOR_ELT, VT, Custom);
1175 }
1176
1177 for (auto VT : { MVT::v8f16, MVT::v2f64, MVT::v2i64 }) {
1178 setOperationAction(ISD::BUILD_VECTOR, VT, Custom);
1179 setOperationAction(ISD::VECTOR_SHUFFLE, VT, Custom);
1180 setOperationAction(ISD::VSELECT, VT, Custom);
1181
1182 if (VT == MVT::v2i64 && !Subtarget.is64Bit())
1183 continue;
1184
1185 setOperationAction(ISD::INSERT_VECTOR_ELT, VT, Custom);
1186 setOperationAction(ISD::EXTRACT_VECTOR_ELT, VT, Custom);
1187 }
1188 setF16Action(MVT::v8f16, Expand);
1189 setOperationAction(ISD::FADD, MVT::v8f16, Expand);
1190 setOperationAction(ISD::FSUB, MVT::v8f16, Expand);
1191 setOperationAction(ISD::FMUL, MVT::v8f16, Expand);
1192 setOperationAction(ISD::FDIV, MVT::v8f16, Expand);
1193 setOperationAction(ISD::FNEG, MVT::v8f16, Custom);
1194 setOperationAction(ISD::FABS, MVT::v8f16, Custom);
1195 setOperationAction(ISD::FCOPYSIGN, MVT::v8f16, Custom);
1196
1197 // Custom lower v2i64 and v2f64 selects.
1198 setOperationAction(ISD::SELECT, MVT::v2f64, Custom);
1199 setOperationAction(ISD::SELECT, MVT::v2i64, Custom);
1200 setOperationAction(ISD::SELECT, MVT::v4i32, Custom);
1201 setOperationAction(ISD::SELECT, MVT::v8i16, Custom);
1202 setOperationAction(ISD::SELECT, MVT::v8f16, Custom);
1203 setOperationAction(ISD::SELECT, MVT::v16i8, Custom);
1204
1205 setOperationAction(ISD::FP_TO_SINT, MVT::v4i32, Custom);
1206 setOperationAction(ISD::FP_TO_UINT, MVT::v4i32, Custom);
1207 setOperationAction(ISD::FP_TO_SINT, MVT::v2i32, Custom);
1208 setOperationAction(ISD::FP_TO_UINT, MVT::v2i32, Custom);
1209 setOperationAction(ISD::STRICT_FP_TO_SINT, MVT::v4i32, Custom);
1210 setOperationAction(ISD::STRICT_FP_TO_SINT, MVT::v2i32, Custom);
1211
1212 // Custom legalize these to avoid over promotion or custom promotion.
1213 for (auto VT : {MVT::v2i8, MVT::v4i8, MVT::v8i8, MVT::v2i16, MVT::v4i16}) {
1214 setOperationAction(ISD::FP_TO_SINT, VT, Custom);
1215 setOperationAction(ISD::FP_TO_UINT, VT, Custom);
1216 setOperationAction(ISD::STRICT_FP_TO_SINT, VT, Custom);
1217 setOperationAction(ISD::STRICT_FP_TO_UINT, VT, Custom);
1218 }
1219
1220 setOperationAction(ISD::SINT_TO_FP, MVT::v4i32, Custom);
1221 setOperationAction(ISD::STRICT_SINT_TO_FP, MVT::v4i32, Custom);
1222 setOperationAction(ISD::SINT_TO_FP, MVT::v2i32, Custom);
1223 setOperationAction(ISD::STRICT_SINT_TO_FP, MVT::v2i32, Custom);
1224
1225 setOperationAction(ISD::UINT_TO_FP, MVT::v2i32, Custom);
1226 setOperationAction(ISD::STRICT_UINT_TO_FP, MVT::v2i32, Custom);
1227
1228 setOperationAction(ISD::UINT_TO_FP, MVT::v4i32, Custom);
1229 setOperationAction(ISD::STRICT_UINT_TO_FP, MVT::v4i32, Custom);
1230
1231 // Fast v2f32 UINT_TO_FP( v2i32 ) custom conversion.
1232 setOperationAction(ISD::SINT_TO_FP, MVT::v2f32, Custom);
1233 setOperationAction(ISD::STRICT_SINT_TO_FP, MVT::v2f32, Custom);
1234 setOperationAction(ISD::UINT_TO_FP, MVT::v2f32, Custom);
1235 setOperationAction(ISD::STRICT_UINT_TO_FP, MVT::v2f32, Custom);
1236
1237 setOperationAction(ISD::FP_EXTEND, MVT::v2f32, Custom);
1238 setOperationAction(ISD::STRICT_FP_EXTEND, MVT::v2f32, Custom);
1239 setOperationAction(ISD::FP_ROUND, MVT::v2f32, Custom);
1240 setOperationAction(ISD::STRICT_FP_ROUND, MVT::v2f32, Custom);
1241
1242 // We want to legalize this to an f64 load rather than an i64 load on
1243 // 64-bit targets and two 32-bit loads on a 32-bit target. Similar for
1244 // store.
1245 setOperationAction(ISD::LOAD, MVT::v2i32, Custom);
1246 setOperationAction(ISD::LOAD, MVT::v4i16, Custom);
1247 setOperationAction(ISD::LOAD, MVT::v8i8, Custom);
1248 setOperationAction(ISD::STORE, MVT::v2i32, Custom);
1249 setOperationAction(ISD::STORE, MVT::v4i16, Custom);
1250 setOperationAction(ISD::STORE, MVT::v8i8, Custom);
1251
1252 // Add 32-bit vector stores to help vectorization opportunities.
1253 setOperationAction(ISD::STORE, MVT::v2i16, Custom);
1254 setOperationAction(ISD::STORE, MVT::v4i8, Custom);
1255
1256 setOperationAction(ISD::BITCAST, MVT::v2i32, Custom);
1257 setOperationAction(ISD::BITCAST, MVT::v4i16, Custom);
1258 setOperationAction(ISD::BITCAST, MVT::v8i8, Custom);
1259 if (!Subtarget.hasAVX512())
1260 setOperationAction(ISD::BITCAST, MVT::v16i1, Custom);
1261
1262 setOperationAction(ISD::SIGN_EXTEND_VECTOR_INREG, MVT::v2i64, Custom);
1263 setOperationAction(ISD::SIGN_EXTEND_VECTOR_INREG, MVT::v4i32, Custom);
1264 setOperationAction(ISD::SIGN_EXTEND_VECTOR_INREG, MVT::v8i16, Custom);
1265
1266 setOperationAction(ISD::SIGN_EXTEND, MVT::v4i64, Custom);
1267
1268 setOperationAction(ISD::TRUNCATE, MVT::v2i8, Custom);
1269 setOperationAction(ISD::TRUNCATE, MVT::v2i16, Custom);
1270 setOperationAction(ISD::TRUNCATE, MVT::v2i32, Custom);
1271 setOperationAction(ISD::TRUNCATE, MVT::v2i64, Custom);
1272 setOperationAction(ISD::TRUNCATE, MVT::v4i8, Custom);
1273 setOperationAction(ISD::TRUNCATE, MVT::v4i16, Custom);
1274 setOperationAction(ISD::TRUNCATE, MVT::v4i32, Custom);
1275 setOperationAction(ISD::TRUNCATE, MVT::v4i64, Custom);
1276 setOperationAction(ISD::TRUNCATE, MVT::v8i8, Custom);
1277 setOperationAction(ISD::TRUNCATE, MVT::v8i16, Custom);
1278 setOperationAction(ISD::TRUNCATE, MVT::v8i32, Custom);
1279 setOperationAction(ISD::TRUNCATE, MVT::v8i64, Custom);
1280 setOperationAction(ISD::TRUNCATE, MVT::v16i8, Custom);
1281 setOperationAction(ISD::TRUNCATE, MVT::v16i16, Custom);
1282 setOperationAction(ISD::TRUNCATE, MVT::v16i32, Custom);
1283 setOperationAction(ISD::TRUNCATE, MVT::v16i64, Custom);
1284
1285 // In the customized shift lowering, the legal v4i32/v2i64 cases
1286 // in AVX2 will be recognized.
1287 for (auto VT : { MVT::v16i8, MVT::v8i16, MVT::v4i32, MVT::v2i64 }) {
1288 setOperationAction(ISD::SRL, VT, Custom);
1289 setOperationAction(ISD::SHL, VT, Custom);
1290 setOperationAction(ISD::SRA, VT, Custom);
1291 if (VT == MVT::v2i64) continue;
1292 setOperationAction(ISD::ROTL, VT, Custom);
1293 setOperationAction(ISD::ROTR, VT, Custom);
1294 setOperationAction(ISD::FSHL, VT, Custom);
1295 setOperationAction(ISD::FSHR, VT, Custom);
1296 }
1297
1298 setOperationAction(ISD::STRICT_FSQRT, MVT::v2f64, Legal);
1299 setOperationAction(ISD::STRICT_FADD, MVT::v2f64, Legal);
1300 setOperationAction(ISD::STRICT_FSUB, MVT::v2f64, Legal);
1301 setOperationAction(ISD::STRICT_FMUL, MVT::v2f64, Legal);
1302 setOperationAction(ISD::STRICT_FDIV, MVT::v2f64, Legal);
1303 }
1304
1305 if (Subtarget.hasGFNI()) {
1306 setOperationAction(ISD::BITREVERSE, MVT::i8, Custom);
1307 setOperationAction(ISD::BITREVERSE, MVT::i16, Custom);
1308 setOperationAction(ISD::BITREVERSE, MVT::i32, Custom);
1309 setOperationAction(ISD::BITREVERSE, MVT::i64, Custom);
1310 }
1311
1312 if (!Subtarget.useSoftFloat() && Subtarget.hasSSSE3()) {
1313 setOperationAction(ISD::ABS, MVT::v16i8, Legal);
1314 setOperationAction(ISD::ABS, MVT::v8i16, Legal);
1315 setOperationAction(ISD::ABS, MVT::v4i32, Legal);
1316
1317 for (auto VT : {MVT::v16i8, MVT::v8i16, MVT::v4i32, MVT::v2i64}) {
1318 setOperationAction(ISD::BITREVERSE, VT, Custom);
1319 setOperationAction(ISD::CTLZ, VT, Custom);
1320 }
1321
1322 // These might be better off as horizontal vector ops.
1323 setOperationAction(ISD::ADD, MVT::i16, Custom);
1324 setOperationAction(ISD::ADD, MVT::i32, Custom);
1325 setOperationAction(ISD::SUB, MVT::i16, Custom);
1326 setOperationAction(ISD::SUB, MVT::i32, Custom);
1327 }
1328
1329 if (!Subtarget.useSoftFloat() && Subtarget.hasSSE41()) {
1330 for (MVT RoundedTy : {MVT::f32, MVT::f64, MVT::v4f32, MVT::v2f64}) {
1331 setOperationAction(ISD::FFLOOR, RoundedTy, Legal);
1332 setOperationAction(ISD::STRICT_FFLOOR, RoundedTy, Legal);
1333 setOperationAction(ISD::FCEIL, RoundedTy, Legal);
1334 setOperationAction(ISD::STRICT_FCEIL, RoundedTy, Legal);
1335 setOperationAction(ISD::FTRUNC, RoundedTy, Legal);
1336 setOperationAction(ISD::STRICT_FTRUNC, RoundedTy, Legal);
1337 setOperationAction(ISD::FRINT, RoundedTy, Legal);
1338 setOperationAction(ISD::STRICT_FRINT, RoundedTy, Legal);
1339 setOperationAction(ISD::FNEARBYINT, RoundedTy, Legal);
1340 setOperationAction(ISD::STRICT_FNEARBYINT, RoundedTy, Legal);
1341 setOperationAction(ISD::FROUNDEVEN, RoundedTy, Legal);
1342 setOperationAction(ISD::STRICT_FROUNDEVEN, RoundedTy, Legal);
1343
1344 setOperationAction(ISD::FROUND, RoundedTy, Custom);
1345 }
1346
1347 setOperationAction(ISD::SMAX, MVT::v16i8, Legal);
1348 setOperationAction(ISD::SMAX, MVT::v4i32, Legal);
1349 setOperationAction(ISD::UMAX, MVT::v8i16, Legal);
1350 setOperationAction(ISD::UMAX, MVT::v4i32, Legal);
1351 setOperationAction(ISD::SMIN, MVT::v16i8, Legal);
1352 setOperationAction(ISD::SMIN, MVT::v4i32, Legal);
1353 setOperationAction(ISD::UMIN, MVT::v8i16, Legal);
1354 setOperationAction(ISD::UMIN, MVT::v4i32, Legal);
1355
1356 setOperationAction(ISD::UADDSAT, MVT::v4i32, Custom);
1357 setOperationAction(ISD::SADDSAT, MVT::v2i64, Custom);
1358 setOperationAction(ISD::SSUBSAT, MVT::v2i64, Custom);
1359
1360 // FIXME: Do we need to handle scalar-to-vector here?
1361 setOperationAction(ISD::MUL, MVT::v4i32, Legal);
1362 setOperationAction(ISD::SMULO, MVT::v2i32, Custom);
1363
1364 // We directly match byte blends in the backend as they match the VSELECT
1365 // condition form.
1366 setOperationAction(ISD::VSELECT, MVT::v16i8, Legal);
1367
1368 // SSE41 brings specific instructions for doing vector sign extend even in
1369 // cases where we don't have SRA.
1370 for (auto VT : { MVT::v8i16, MVT::v4i32, MVT::v2i64 }) {
1371 setOperationAction(ISD::SIGN_EXTEND_VECTOR_INREG, VT, Legal);
1372 setOperationAction(ISD::ZERO_EXTEND_VECTOR_INREG, VT, Legal);
1373 }
1374
1375 // SSE41 also has vector sign/zero extending loads, PMOV[SZ]X
1376 for (auto LoadExtOp : { ISD::SEXTLOAD, ISD::ZEXTLOAD }) {
1377 setLoadExtAction(LoadExtOp, MVT::v8i16, MVT::v8i8, Legal);
1378 setLoadExtAction(LoadExtOp, MVT::v4i32, MVT::v4i8, Legal);
1379 setLoadExtAction(LoadExtOp, MVT::v2i64, MVT::v2i8, Legal);
1380 setLoadExtAction(LoadExtOp, MVT::v4i32, MVT::v4i16, Legal);
1381 setLoadExtAction(LoadExtOp, MVT::v2i64, MVT::v2i16, Legal);
1382 setLoadExtAction(LoadExtOp, MVT::v2i64, MVT::v2i32, Legal);
1383 }
1384
1385 if (Subtarget.is64Bit() && !Subtarget.hasAVX512()) {
1386 // We need to scalarize v4i64->v432 uint_to_fp using cvtsi2ss, but we can
1387 // do the pre and post work in the vector domain.
1388 setOperationAction(ISD::UINT_TO_FP, MVT::v4i64, Custom);
1389 setOperationAction(ISD::STRICT_UINT_TO_FP, MVT::v4i64, Custom);
1390 // We need to mark SINT_TO_FP as Custom even though we want to expand it
1391 // so that DAG combine doesn't try to turn it into uint_to_fp.
1392 setOperationAction(ISD::SINT_TO_FP, MVT::v4i64, Custom);
1393 setOperationAction(ISD::STRICT_SINT_TO_FP, MVT::v4i64, Custom);
1394 }
1395 }
1396
1397 if (!Subtarget.useSoftFloat() && Subtarget.hasSSE42()) {
1398 setOperationAction(ISD::UADDSAT, MVT::v2i64, Custom);
1399 }
1400
1401 if (!Subtarget.useSoftFloat() && Subtarget.hasXOP()) {
1402 for (auto VT : { MVT::v16i8, MVT::v8i16, MVT::v4i32, MVT::v2i64,
1403 MVT::v32i8, MVT::v16i16, MVT::v8i32, MVT::v4i64 }) {
1404 setOperationAction(ISD::ROTL, VT, Custom);
1405 setOperationAction(ISD::ROTR, VT, Custom);
1406 }
1407
1408 // XOP can efficiently perform BITREVERSE with VPPERM.
1409 for (auto VT : { MVT::i8, MVT::i16, MVT::i32, MVT::i64 })
1410 setOperationAction(ISD::BITREVERSE, VT, Custom);
1411 }
1412
1413 if (!Subtarget.useSoftFloat() && Subtarget.hasAVX()) {
1414 bool HasInt256 = Subtarget.hasInt256();
1415
1416 addRegisterClass(MVT::v32i8, Subtarget.hasVLX() ? &X86::VR256XRegClass
1417 : &X86::VR256RegClass);
1418 addRegisterClass(MVT::v16i16, Subtarget.hasVLX() ? &X86::VR256XRegClass
1419 : &X86::VR256RegClass);
1420 addRegisterClass(MVT::v16f16, Subtarget.hasVLX() ? &X86::VR256XRegClass
1421 : &X86::VR256RegClass);
1422 addRegisterClass(MVT::v8i32, Subtarget.hasVLX() ? &X86::VR256XRegClass
1423 : &X86::VR256RegClass);
1424 addRegisterClass(MVT::v8f32, Subtarget.hasVLX() ? &X86::VR256XRegClass
1425 : &X86::VR256RegClass);
1426 addRegisterClass(MVT::v4i64, Subtarget.hasVLX() ? &X86::VR256XRegClass
1427 : &X86::VR256RegClass);
1428 addRegisterClass(MVT::v4f64, Subtarget.hasVLX() ? &X86::VR256XRegClass
1429 : &X86::VR256RegClass);
1430
1431 for (auto VT : { MVT::v8f32, MVT::v4f64 }) {
1432 setOperationAction(ISD::FFLOOR, VT, Legal);
1433 setOperationAction(ISD::STRICT_FFLOOR, VT, Legal);
1434 setOperationAction(ISD::FCEIL, VT, Legal);
1435 setOperationAction(ISD::STRICT_FCEIL, VT, Legal);
1436 setOperationAction(ISD::FTRUNC, VT, Legal);
1437 setOperationAction(ISD::STRICT_FTRUNC, VT, Legal);
1438 setOperationAction(ISD::FRINT, VT, Legal);
1439 setOperationAction(ISD::STRICT_FRINT, VT, Legal);
1440 setOperationAction(ISD::FNEARBYINT, VT, Legal);
1441 setOperationAction(ISD::STRICT_FNEARBYINT, VT, Legal);
1442 setOperationAction(ISD::FROUNDEVEN, VT, Legal);
1443 setOperationAction(ISD::STRICT_FROUNDEVEN, VT, Legal);
1444
1445 setOperationAction(ISD::FROUND, VT, Custom);
1446
1447 setOperationAction(ISD::FNEG, VT, Custom);
1448 setOperationAction(ISD::FABS, VT, Custom);
1449 setOperationAction(ISD::FCOPYSIGN, VT, Custom);
1450
1451 setOperationAction(ISD::FMAXIMUM, VT, Custom);
1452 setOperationAction(ISD::FMINIMUM, VT, Custom);
1453 }
1454
1455 setOperationAction(ISD::LRINT, MVT::v8f32, Custom);
1456 setOperationAction(ISD::LRINT, MVT::v4f64, Custom);
1457
1458 // (fp_to_int:v8i16 (v8f32 ..)) requires the result type to be promoted
1459 // even though v8i16 is a legal type.
1460 setOperationPromotedToType(ISD::FP_TO_SINT, MVT::v8i16, MVT::v8i32);
1461 setOperationPromotedToType(ISD::FP_TO_UINT, MVT::v8i16, MVT::v8i32);
1462 setOperationPromotedToType(ISD::STRICT_FP_TO_SINT, MVT::v8i16, MVT::v8i32);
1463 setOperationPromotedToType(ISD::STRICT_FP_TO_UINT, MVT::v8i16, MVT::v8i32);
1464 setOperationAction(ISD::FP_TO_SINT, MVT::v8i32, Custom);
1465 setOperationAction(ISD::FP_TO_UINT, MVT::v8i32, Custom);
1466 setOperationAction(ISD::STRICT_FP_TO_SINT, MVT::v8i32, Custom);
1467
1468 setOperationAction(ISD::SINT_TO_FP, MVT::v8i32, Custom);
1469 setOperationAction(ISD::STRICT_SINT_TO_FP, MVT::v8i32, Custom);
1470 setOperationAction(ISD::FP_EXTEND, MVT::v8f32, Expand);
1471 setOperationAction(ISD::FP_ROUND, MVT::v8f16, Expand);
1472 setOperationAction(ISD::FP_EXTEND, MVT::v4f64, Custom);
1473 setOperationAction(ISD::STRICT_FP_EXTEND, MVT::v4f64, Custom);
1474
1475 setOperationAction(ISD::STRICT_FP_ROUND, MVT::v4f32, Legal);
1476 setOperationAction(ISD::STRICT_FADD, MVT::v8f32, Legal);
1477 setOperationAction(ISD::STRICT_FADD, MVT::v4f64, Legal);
1478 setOperationAction(ISD::STRICT_FSUB, MVT::v8f32, Legal);
1479 setOperationAction(ISD::STRICT_FSUB, MVT::v4f64, Legal);
1480 setOperationAction(ISD::STRICT_FMUL, MVT::v8f32, Legal);
1481 setOperationAction(ISD::STRICT_FMUL, MVT::v4f64, Legal);
1482 setOperationAction(ISD::STRICT_FDIV, MVT::v8f32, Legal);
1483 setOperationAction(ISD::STRICT_FDIV, MVT::v4f64, Legal);
1484 setOperationAction(ISD::STRICT_FSQRT, MVT::v8f32, Legal);
1485 setOperationAction(ISD::STRICT_FSQRT, MVT::v4f64, Legal);
1486
1487 if (!Subtarget.hasAVX512())
1488 setOperationAction(ISD::BITCAST, MVT::v32i1, Custom);
1489
1490 // In the customized shift lowering, the legal v8i32/v4i64 cases
1491 // in AVX2 will be recognized.
1492 for (auto VT : { MVT::v32i8, MVT::v16i16, MVT::v8i32, MVT::v4i64 }) {
1493 setOperationAction(ISD::SRL, VT, Custom);
1494 setOperationAction(ISD::SHL, VT, Custom);
1495 setOperationAction(ISD::SRA, VT, Custom);
1496 setOperationAction(ISD::ABDS, VT, Custom);
1497 setOperationAction(ISD::ABDU, VT, Custom);
1498 if (VT == MVT::v4i64) continue;
1499 setOperationAction(ISD::ROTL, VT, Custom);
1500 setOperationAction(ISD::ROTR, VT, Custom);
1501 setOperationAction(ISD::FSHL, VT, Custom);
1502 setOperationAction(ISD::FSHR, VT, Custom);
1503 }
1504
1505 // These types need custom splitting if their input is a 128-bit vector.
1506 setOperationAction(ISD::SIGN_EXTEND, MVT::v8i64, Custom);
1507 setOperationAction(ISD::SIGN_EXTEND, MVT::v16i32, Custom);
1508 setOperationAction(ISD::ZERO_EXTEND, MVT::v8i64, Custom);
1509 setOperationAction(ISD::ZERO_EXTEND, MVT::v16i32, Custom);
1510
1511 setOperationAction(ISD::SELECT, MVT::v4f64, Custom);
1512 setOperationAction(ISD::SELECT, MVT::v4i64, Custom);
1513 setOperationAction(ISD::SELECT, MVT::v8i32, Custom);
1514 setOperationAction(ISD::SELECT, MVT::v16i16, Custom);
1515 setOperationAction(ISD::SELECT, MVT::v16f16, Custom);
1516 setOperationAction(ISD::SELECT, MVT::v32i8, Custom);
1517 setOperationAction(ISD::SELECT, MVT::v8f32, Custom);
1518
1519 for (auto VT : { MVT::v16i16, MVT::v8i32, MVT::v4i64 }) {
1520 setOperationAction(ISD::SIGN_EXTEND, VT, Custom);
1521 setOperationAction(ISD::ZERO_EXTEND, VT, Custom);
1522 setOperationAction(ISD::ANY_EXTEND, VT, Custom);
1523 }
1524
1525 setOperationAction(ISD::TRUNCATE, MVT::v32i8, Custom);
1526 setOperationAction(ISD::TRUNCATE, MVT::v32i16, Custom);
1527 setOperationAction(ISD::TRUNCATE, MVT::v32i32, Custom);
1528 setOperationAction(ISD::TRUNCATE, MVT::v32i64, Custom);
1529
1530 for (auto VT : { MVT::v32i8, MVT::v16i16, MVT::v8i32, MVT::v4i64 }) {
1531 setOperationAction(ISD::SETCC, VT, Custom);
1532 setOperationAction(ISD::CTPOP, VT, Custom);
1533 setOperationAction(ISD::CTLZ, VT, Custom);
1534 setOperationAction(ISD::BITREVERSE, VT, Custom);
1535
1536 // The condition codes aren't legal in SSE/AVX and under AVX512 we use
1537 // setcc all the way to isel and prefer SETGT in some isel patterns.
1538 setCondCodeAction(ISD::SETLT, VT, Custom);
1539 setCondCodeAction(ISD::SETLE, VT, Custom);
1540 }
1541
1542 setOperationAction(ISD::SETCC, MVT::v4f64, Custom);
1543 setOperationAction(ISD::SETCC, MVT::v8f32, Custom);
1544 setOperationAction(ISD::STRICT_FSETCC, MVT::v4f64, Custom);
1545 setOperationAction(ISD::STRICT_FSETCC, MVT::v8f32, Custom);
1546 setOperationAction(ISD::STRICT_FSETCCS, MVT::v4f64, Custom);
1547 setOperationAction(ISD::STRICT_FSETCCS, MVT::v8f32, Custom);
1548
1549 if (Subtarget.hasAnyFMA()) {
1550 for (auto VT : { MVT::f32, MVT::f64, MVT::v4f32, MVT::v8f32,
1551 MVT::v2f64, MVT::v4f64 }) {
1552 setOperationAction(ISD::FMA, VT, Legal);
1553 setOperationAction(ISD::STRICT_FMA, VT, Legal);
1554 }
1555 }
1556
1557 for (auto VT : { MVT::v32i8, MVT::v16i16, MVT::v8i32, MVT::v4i64 }) {
1558 setOperationAction(ISD::ADD, VT, HasInt256 ? Legal : Custom);
1559 setOperationAction(ISD::SUB, VT, HasInt256 ? Legal : Custom);
1560 }
1561
1562 setOperationAction(ISD::MUL, MVT::v4i64, Custom);
1563 setOperationAction(ISD::MUL, MVT::v8i32, HasInt256 ? Legal : Custom);
1564 setOperationAction(ISD::MUL, MVT::v16i16, HasInt256 ? Legal : Custom);
1565 setOperationAction(ISD::MUL, MVT::v32i8, Custom);
1566
1567 setOperationAction(ISD::MULHU, MVT::v8i32, Custom);
1568 setOperationAction(ISD::MULHS, MVT::v8i32, Custom);
1569 setOperationAction(ISD::MULHU, MVT::v16i16, HasInt256 ? Legal : Custom);
1570 setOperationAction(ISD::MULHS, MVT::v16i16, HasInt256 ? Legal : Custom);
1571 setOperationAction(ISD::MULHU, MVT::v32i8, Custom);
1572 setOperationAction(ISD::MULHS, MVT::v32i8, Custom);
1573 setOperationAction(ISD::AVGCEILU, MVT::v16i16, HasInt256 ? Legal : Custom);
1574 setOperationAction(ISD::AVGCEILU, MVT::v32i8, HasInt256 ? Legal : Custom);
1575
1576 setOperationAction(ISD::SMULO, MVT::v32i8, Custom);
1577 setOperationAction(ISD::UMULO, MVT::v32i8, Custom);
1578
1579 setOperationAction(ISD::ABS, MVT::v4i64, Custom);
1580 setOperationAction(ISD::SMAX, MVT::v4i64, Custom);
1581 setOperationAction(ISD::UMAX, MVT::v4i64, Custom);
1582 setOperationAction(ISD::SMIN, MVT::v4i64, Custom);
1583 setOperationAction(ISD::UMIN, MVT::v4i64, Custom);
1584
1585 setOperationAction(ISD::UADDSAT, MVT::v32i8, HasInt256 ? Legal : Custom);
1586 setOperationAction(ISD::SADDSAT, MVT::v32i8, HasInt256 ? Legal : Custom);
1587 setOperationAction(ISD::USUBSAT, MVT::v32i8, HasInt256 ? Legal : Custom);
1588 setOperationAction(ISD::SSUBSAT, MVT::v32i8, HasInt256 ? Legal : Custom);
1589 setOperationAction(ISD::UADDSAT, MVT::v16i16, HasInt256 ? Legal : Custom);
1590 setOperationAction(ISD::SADDSAT, MVT::v16i16, HasInt256 ? Legal : Custom);
1591 setOperationAction(ISD::USUBSAT, MVT::v16i16, HasInt256 ? Legal : Custom);
1592 setOperationAction(ISD::SSUBSAT, MVT::v16i16, HasInt256 ? Legal : Custom);
1593 setOperationAction(ISD::UADDSAT, MVT::v8i32, Custom);
1594 setOperationAction(ISD::USUBSAT, MVT::v8i32, Custom);
1595 setOperationAction(ISD::UADDSAT, MVT::v4i64, Custom);
1596 setOperationAction(ISD::USUBSAT, MVT::v4i64, Custom);
1597
1598 for (auto VT : { MVT::v32i8, MVT::v16i16, MVT::v8i32 }) {
1599 setOperationAction(ISD::ABS, VT, HasInt256 ? Legal : Custom);
1600 setOperationAction(ISD::SMAX, VT, HasInt256 ? Legal : Custom);
1601 setOperationAction(ISD::UMAX, VT, HasInt256 ? Legal : Custom);
1602 setOperationAction(ISD::SMIN, VT, HasInt256 ? Legal : Custom);
1603 setOperationAction(ISD::UMIN, VT, HasInt256 ? Legal : Custom);
1604 }
1605
1606 for (auto VT : {MVT::v16i16, MVT::v8i32, MVT::v4i64}) {
1607 setOperationAction(ISD::SIGN_EXTEND_VECTOR_INREG, VT, Custom);
1608 setOperationAction(ISD::ZERO_EXTEND_VECTOR_INREG, VT, Custom);
1609 }
1610
1611 if (HasInt256) {
1612 // The custom lowering for UINT_TO_FP for v8i32 becomes interesting
1613 // when we have a 256bit-wide blend with immediate.
1614 setOperationAction(ISD::UINT_TO_FP, MVT::v8i32, Custom);
1615 setOperationAction(ISD::STRICT_UINT_TO_FP, MVT::v8i32, Custom);
1616
1617 // AVX2 also has wider vector sign/zero extending loads, VPMOV[SZ]X
1618 for (auto LoadExtOp : { ISD::SEXTLOAD, ISD::ZEXTLOAD }) {
1619 setLoadExtAction(LoadExtOp, MVT::v16i16, MVT::v16i8, Legal);
1620 setLoadExtAction(LoadExtOp, MVT::v8i32, MVT::v8i8, Legal);
1621 setLoadExtAction(LoadExtOp, MVT::v4i64, MVT::v4i8, Legal);
1622 setLoadExtAction(LoadExtOp, MVT::v8i32, MVT::v8i16, Legal);
1623 setLoadExtAction(LoadExtOp, MVT::v4i64, MVT::v4i16, Legal);
1624 setLoadExtAction(LoadExtOp, MVT::v4i64, MVT::v4i32, Legal);
1625 }
1626 }
1627
1628 for (auto VT : { MVT::v4i32, MVT::v8i32, MVT::v2i64, MVT::v4i64,
1629 MVT::v4f32, MVT::v8f32, MVT::v2f64, MVT::v4f64 }) {
1630 setOperationAction(ISD::MLOAD, VT, Subtarget.hasVLX() ? Legal : Custom);
1631 setOperationAction(ISD::MSTORE, VT, Legal);
1632 }
1633
1634 // Extract subvector is special because the value type
1635 // (result) is 128-bit but the source is 256-bit wide.
1636 for (auto VT : { MVT::v16i8, MVT::v8i16, MVT::v4i32, MVT::v2i64,
1637 MVT::v8f16, MVT::v4f32, MVT::v2f64 }) {
1638 setOperationAction(ISD::EXTRACT_SUBVECTOR, VT, Legal);
1639 }
1640
1641 // Custom lower several nodes for 256-bit types.
1642 for (MVT VT : { MVT::v32i8, MVT::v16i16, MVT::v8i32, MVT::v4i64,
1643 MVT::v16f16, MVT::v8f32, MVT::v4f64 }) {
1644 setOperationAction(ISD::BUILD_VECTOR, VT, Custom);
1645 setOperationAction(ISD::VECTOR_SHUFFLE, VT, Custom);
1646 setOperationAction(ISD::VSELECT, VT, Custom);
1647 setOperationAction(ISD::INSERT_VECTOR_ELT, VT, Custom);
1648 setOperationAction(ISD::EXTRACT_VECTOR_ELT, VT, Custom);
1649 setOperationAction(ISD::SCALAR_TO_VECTOR, VT, Custom);
1650 setOperationAction(ISD::INSERT_SUBVECTOR, VT, Legal);
1651 setOperationAction(ISD::CONCAT_VECTORS, VT, Custom);
1652 setOperationAction(ISD::STORE, VT, Custom);
1653 }
1654 setF16Action(MVT::v16f16, Expand);
1655 setOperationAction(ISD::FNEG, MVT::v16f16, Custom);
1656 setOperationAction(ISD::FABS, MVT::v16f16, Custom);
1657 setOperationAction(ISD::FCOPYSIGN, MVT::v16f16, Custom);
1658 setOperationAction(ISD::FADD, MVT::v16f16, Expand);
1659 setOperationAction(ISD::FSUB, MVT::v16f16, Expand);
1660 setOperationAction(ISD::FMUL, MVT::v16f16, Expand);
1661 setOperationAction(ISD::FDIV, MVT::v16f16, Expand);
1662
1663 if (HasInt256) {
1664 setOperationAction(ISD::VSELECT, MVT::v32i8, Legal);
1665
1666 // Custom legalize 2x32 to get a little better code.
1667 setOperationAction(ISD::MGATHER, MVT::v2f32, Custom);
1668 setOperationAction(ISD::MGATHER, MVT::v2i32, Custom);
1669
1670 for (auto VT : { MVT::v4i32, MVT::v8i32, MVT::v2i64, MVT::v4i64,
1671 MVT::v4f32, MVT::v8f32, MVT::v2f64, MVT::v4f64 })
1672 setOperationAction(ISD::MGATHER, VT, Custom);
1673 }
1674 }
1675
1676 if (!Subtarget.useSoftFloat() && !Subtarget.hasFP16() &&
1677 Subtarget.hasF16C()) {
1678 for (MVT VT : { MVT::f16, MVT::v2f16, MVT::v4f16, MVT::v8f16 }) {
1679 setOperationAction(ISD::FP_ROUND, VT, Custom);
1680 setOperationAction(ISD::STRICT_FP_ROUND, VT, Custom);
1681 }
1682 for (MVT VT : { MVT::f32, MVT::v2f32, MVT::v4f32, MVT::v8f32 }) {
1683 setOperationAction(ISD::FP_EXTEND, VT, Custom);
1684 setOperationAction(ISD::STRICT_FP_EXTEND, VT, Custom);
1685 }
1686 for (unsigned Opc : {ISD::FADD, ISD::FSUB, ISD::FMUL, ISD::FDIV}) {
1687 setOperationPromotedToType(Opc, MVT::v8f16, MVT::v8f32);
1688 setOperationPromotedToType(Opc, MVT::v16f16, MVT::v16f32);
1689 }
1690 }
1691
1692 // This block controls legalization of the mask vector sizes that are
1693 // available with AVX512. 512-bit vectors are in a separate block controlled
1694 // by useAVX512Regs.
1695 if (!Subtarget.useSoftFloat() && Subtarget.hasAVX512()) {
1696 addRegisterClass(MVT::v1i1, &X86::VK1RegClass);
1697 addRegisterClass(MVT::v2i1, &X86::VK2RegClass);
1698 addRegisterClass(MVT::v4i1, &X86::VK4RegClass);
1699 addRegisterClass(MVT::v8i1, &X86::VK8RegClass);
1700 addRegisterClass(MVT::v16i1, &X86::VK16RegClass);
1701
1702 setOperationAction(ISD::SELECT, MVT::v1i1, Custom);
1703 setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v1i1, Custom);
1704 setOperationAction(ISD::BUILD_VECTOR, MVT::v1i1, Custom);
1705
1706 setOperationPromotedToType(ISD::FP_TO_SINT, MVT::v8i1, MVT::v8i32);
1707 setOperationPromotedToType(ISD::FP_TO_UINT, MVT::v8i1, MVT::v8i32);
1708 setOperationPromotedToType(ISD::FP_TO_SINT, MVT::v4i1, MVT::v4i32);
1709 setOperationPromotedToType(ISD::FP_TO_UINT, MVT::v4i1, MVT::v4i32);
1710 setOperationPromotedToType(ISD::STRICT_FP_TO_SINT, MVT::v8i1, MVT::v8i32);
1711 setOperationPromotedToType(ISD::STRICT_FP_TO_UINT, MVT::v8i1, MVT::v8i32);
1712 setOperationPromotedToType(ISD::STRICT_FP_TO_SINT, MVT::v4i1, MVT::v4i32);
1713 setOperationPromotedToType(ISD::STRICT_FP_TO_UINT, MVT::v4i1, MVT::v4i32);
1714 setOperationAction(ISD::FP_TO_SINT, MVT::v2i1, Custom);
1715 setOperationAction(ISD::FP_TO_UINT, MVT::v2i1, Custom);
1716 setOperationAction(ISD::STRICT_FP_TO_SINT, MVT::v2i1, Custom);
1717 setOperationAction(ISD::STRICT_FP_TO_UINT, MVT::v2i1, Custom);
1718
1719 // There is no byte sized k-register load or store without AVX512DQ.
1720 if (!Subtarget.hasDQI()) {
1721 setOperationAction(ISD::LOAD, MVT::v1i1, Custom);
1722 setOperationAction(ISD::LOAD, MVT::v2i1, Custom);
1723 setOperationAction(ISD::LOAD, MVT::v4i1, Custom);
1724 setOperationAction(ISD::LOAD, MVT::v8i1, Custom);
1725
1726 setOperationAction(ISD::STORE, MVT::v1i1, Custom);
1727 setOperationAction(ISD::STORE, MVT::v2i1, Custom);
1728 setOperationAction(ISD::STORE, MVT::v4i1, Custom);
1729 setOperationAction(ISD::STORE, MVT::v8i1, Custom);
1730 }
1731
1732 // Extends of v16i1/v8i1/v4i1/v2i1 to 128-bit vectors.
1733 for (auto VT : { MVT::v16i8, MVT::v8i16, MVT::v4i32, MVT::v2i64 }) {
1734 setOperationAction(ISD::SIGN_EXTEND, VT, Custom);
1735 setOperationAction(ISD::ZERO_EXTEND, VT, Custom);
1736 setOperationAction(ISD::ANY_EXTEND, VT, Custom);
1737 }
1738
1739 for (auto VT : { MVT::v1i1, MVT::v2i1, MVT::v4i1, MVT::v8i1, MVT::v16i1 })
1740 setOperationAction(ISD::VSELECT, VT, Expand);
1741
1742 for (auto VT : { MVT::v2i1, MVT::v4i1, MVT::v8i1, MVT::v16i1 }) {
1743 setOperationAction(ISD::SETCC, VT, Custom);
1744 setOperationAction(ISD::SELECT, VT, Custom);
1745 setOperationAction(ISD::TRUNCATE, VT, Custom);
1746
1747 setOperationAction(ISD::BUILD_VECTOR, VT, Custom);
1748 setOperationAction(ISD::CONCAT_VECTORS, VT, Custom);
1749 setOperationAction(ISD::EXTRACT_VECTOR_ELT, VT, Custom);
1750 setOperationAction(ISD::INSERT_SUBVECTOR, VT, Custom);
1751 setOperationAction(ISD::INSERT_VECTOR_ELT, VT, Custom);
1752 setOperationAction(ISD::VECTOR_SHUFFLE, VT, Custom);
1753 }
1754
1755 for (auto VT : { MVT::v1i1, MVT::v2i1, MVT::v4i1, MVT::v8i1 })
1756 setOperationAction(ISD::EXTRACT_SUBVECTOR, VT, Custom);
1757 }
1758 if (Subtarget.hasDQI() && Subtarget.hasVLX()) {
1759 for (MVT VT : {MVT::v4f32, MVT::v8f32, MVT::v2f64, MVT::v4f64}) {
1760 setOperationAction(ISD::LRINT, VT, Legal);
1761 setOperationAction(ISD::LLRINT, VT, Legal);
1762 }
1763 }
1764
1765 // This block controls legalization for 512-bit operations with 8/16/32/64 bit
1766 // elements. 512-bits can be disabled based on prefer-vector-width and
1767 // required-vector-width function attributes.
1768 if (!Subtarget.useSoftFloat() && Subtarget.useAVX512Regs()) {
1769 bool HasBWI = Subtarget.hasBWI();
1770
1771 addRegisterClass(MVT::v16i32, &X86::VR512RegClass);
1772 addRegisterClass(MVT::v16f32, &X86::VR512RegClass);
1773 addRegisterClass(MVT::v8i64, &X86::VR512RegClass);
1774 addRegisterClass(MVT::v8f64, &X86::VR512RegClass);
1775 addRegisterClass(MVT::v32i16, &X86::VR512RegClass);
1776 addRegisterClass(MVT::v32f16, &X86::VR512RegClass);
1777 addRegisterClass(MVT::v64i8, &X86::VR512RegClass);
1778
1779 for (auto ExtType : {ISD::ZEXTLOAD, ISD::SEXTLOAD}) {
1780 setLoadExtAction(ExtType, MVT::v16i32, MVT::v16i8, Legal);
1781 setLoadExtAction(ExtType, MVT::v16i32, MVT::v16i16, Legal);
1782 setLoadExtAction(ExtType, MVT::v8i64, MVT::v8i8, Legal);
1783 setLoadExtAction(ExtType, MVT::v8i64, MVT::v8i16, Legal);
1784 setLoadExtAction(ExtType, MVT::v8i64, MVT::v8i32, Legal);
1785 if (HasBWI)
1786 setLoadExtAction(ExtType, MVT::v32i16, MVT::v32i8, Legal);
1787 }
1788
1789 for (MVT VT : { MVT::v16f32, MVT::v8f64 }) {
1790 setOperationAction(ISD::FMAXIMUM, VT, Custom);
1791 setOperationAction(ISD::FMINIMUM, VT, Custom);
1792 setOperationAction(ISD::FNEG, VT, Custom);
1793 setOperationAction(ISD::FABS, VT, Custom);
1794 setOperationAction(ISD::FMA, VT, Legal);
1795 setOperationAction(ISD::STRICT_FMA, VT, Legal);
1796 setOperationAction(ISD::FCOPYSIGN, VT, Custom);
1797 }
1798 setOperationAction(ISD::LRINT, MVT::v16f32,
1799 Subtarget.hasDQI() ? Legal : Custom);
1800 setOperationAction(ISD::LRINT, MVT::v8f64,
1801 Subtarget.hasDQI() ? Legal : Custom);
1802 if (Subtarget.hasDQI())
1803 setOperationAction(ISD::LLRINT, MVT::v8f64, Legal);
1804
1805 for (MVT VT : { MVT::v16i1, MVT::v16i8 }) {
1806 setOperationPromotedToType(ISD::FP_TO_SINT , VT, MVT::v16i32);
1807 setOperationPromotedToType(ISD::FP_TO_UINT , VT, MVT::v16i32);
1808 setOperationPromotedToType(ISD::STRICT_FP_TO_SINT, VT, MVT::v16i32);
1809 setOperationPromotedToType(ISD::STRICT_FP_TO_UINT, VT, MVT::v16i32);
1810 }
1811
1812 for (MVT VT : { MVT::v16i16, MVT::v16i32 }) {
1813 setOperationAction(ISD::FP_TO_SINT, VT, Custom);
1814 setOperationAction(ISD::FP_TO_UINT, VT, Custom);
1815 setOperationAction(ISD::STRICT_FP_TO_SINT, VT, Custom);
1816 setOperationAction(ISD::STRICT_FP_TO_UINT, VT, Custom);
1817 }
1818
1819 setOperationAction(ISD::SINT_TO_FP, MVT::v16i32, Custom);
1820 setOperationAction(ISD::UINT_TO_FP, MVT::v16i32, Custom);
1821 setOperationAction(ISD::STRICT_SINT_TO_FP, MVT::v16i32, Custom);
1822 setOperationAction(ISD::STRICT_UINT_TO_FP, MVT::v16i32, Custom);
1823 setOperationAction(ISD::FP_EXTEND, MVT::v8f64, Custom);
1824 setOperationAction(ISD::STRICT_FP_EXTEND, MVT::v8f64, Custom);
1825
1826 setOperationAction(ISD::STRICT_FADD, MVT::v16f32, Legal);
1827 setOperationAction(ISD::STRICT_FADD, MVT::v8f64, Legal);
1828 setOperationAction(ISD::STRICT_FSUB, MVT::v16f32, Legal);
1829 setOperationAction(ISD::STRICT_FSUB, MVT::v8f64, Legal);
1830 setOperationAction(ISD::STRICT_FMUL, MVT::v16f32, Legal);
1831 setOperationAction(ISD::STRICT_FMUL, MVT::v8f64, Legal);
1832 setOperationAction(ISD::STRICT_FDIV, MVT::v16f32, Legal);
1833 setOperationAction(ISD::STRICT_FDIV, MVT::v8f64, Legal);
1834 setOperationAction(ISD::STRICT_FSQRT, MVT::v16f32, Legal);
1835 setOperationAction(ISD::STRICT_FSQRT, MVT::v8f64, Legal);
1836 setOperationAction(ISD::STRICT_FP_ROUND, MVT::v8f32, Legal);
1837
1838 setTruncStoreAction(MVT::v8i64, MVT::v8i8, Legal);
1839 setTruncStoreAction(MVT::v8i64, MVT::v8i16, Legal);
1840 setTruncStoreAction(MVT::v8i64, MVT::v8i32, Legal);
1841 setTruncStoreAction(MVT::v16i32, MVT::v16i8, Legal);
1842 setTruncStoreAction(MVT::v16i32, MVT::v16i16, Legal);
1843 if (HasBWI)
1844 setTruncStoreAction(MVT::v32i16, MVT::v32i8, Legal);
1845
1846 // With 512-bit vectors and no VLX, we prefer to widen MLOAD/MSTORE
1847 // to 512-bit rather than use the AVX2 instructions so that we can use
1848 // k-masks.
1849 if (!Subtarget.hasVLX()) {
1850 for (auto VT : {MVT::v4i32, MVT::v8i32, MVT::v2i64, MVT::v4i64,
1851 MVT::v4f32, MVT::v8f32, MVT::v2f64, MVT::v4f64}) {
1852 setOperationAction(ISD::MLOAD, VT, Custom);
1853 setOperationAction(ISD::MSTORE, VT, Custom);
1854 }
1855 }
1856
1857 setOperationAction(ISD::TRUNCATE, MVT::v8i32, Legal);
1858 setOperationAction(ISD::TRUNCATE, MVT::v16i16, Legal);
1859 setOperationAction(ISD::TRUNCATE, MVT::v32i8, HasBWI ? Legal : Custom);
1860 setOperationAction(ISD::ZERO_EXTEND, MVT::v32i16, Custom);
1861 setOperationAction(ISD::ZERO_EXTEND, MVT::v16i32, Custom);
1862 setOperationAction(ISD::ZERO_EXTEND, MVT::v8i64, Custom);
1863 setOperationAction(ISD::ANY_EXTEND, MVT::v32i16, Custom);
1864 setOperationAction(ISD::ANY_EXTEND, MVT::v16i32, Custom);
1865 setOperationAction(ISD::ANY_EXTEND, MVT::v8i64, Custom);
1866 setOperationAction(ISD::SIGN_EXTEND, MVT::v32i16, Custom);
1867 setOperationAction(ISD::SIGN_EXTEND, MVT::v16i32, Custom);
1868 setOperationAction(ISD::SIGN_EXTEND, MVT::v8i64, Custom);
1869
1870 if (HasBWI) {
1871 // Extends from v64i1 masks to 512-bit vectors.
1872 setOperationAction(ISD::SIGN_EXTEND, MVT::v64i8, Custom);
1873 setOperationAction(ISD::ZERO_EXTEND, MVT::v64i8, Custom);
1874 setOperationAction(ISD::ANY_EXTEND, MVT::v64i8, Custom);
1875 }
1876
1877 for (auto VT : { MVT::v16f32, MVT::v8f64 }) {
1878 setOperationAction(ISD::FFLOOR, VT, Legal);
1879 setOperationAction(ISD::STRICT_FFLOOR, VT, Legal);
1880 setOperationAction(ISD::FCEIL, VT, Legal);
1881 setOperationAction(ISD::STRICT_FCEIL, VT, Legal);
1882 setOperationAction(ISD::FTRUNC, VT, Legal);
1883 setOperationAction(ISD::STRICT_FTRUNC, VT, Legal);
1884 setOperationAction(ISD::FRINT, VT, Legal);
1885 setOperationAction(ISD::STRICT_FRINT, VT, Legal);
1886 setOperationAction(ISD::FNEARBYINT, VT, Legal);
1887 setOperationAction(ISD::STRICT_FNEARBYINT, VT, Legal);
1888 setOperationAction(ISD::FROUNDEVEN, VT, Legal);
1889 setOperationAction(ISD::STRICT_FROUNDEVEN, VT, Legal);
1890
1891 setOperationAction(ISD::FROUND, VT, Custom);
1892 }
1893
1894 for (auto VT : {MVT::v32i16, MVT::v16i32, MVT::v8i64}) {
1895 setOperationAction(ISD::SIGN_EXTEND_VECTOR_INREG, VT, Custom);
1896 setOperationAction(ISD::ZERO_EXTEND_VECTOR_INREG, VT, Custom);
1897 }
1898
1899 setOperationAction(ISD::ADD, MVT::v32i16, HasBWI ? Legal : Custom);
1900 setOperationAction(ISD::SUB, MVT::v32i16, HasBWI ? Legal : Custom);
1901 setOperationAction(ISD::ADD, MVT::v64i8, HasBWI ? Legal : Custom);
1902 setOperationAction(ISD::SUB, MVT::v64i8, HasBWI ? Legal : Custom);
1903
1904 setOperationAction(ISD::MUL, MVT::v8i64, Custom);
1905 setOperationAction(ISD::MUL, MVT::v16i32, Legal);
1906 setOperationAction(ISD::MUL, MVT::v32i16, HasBWI ? Legal : Custom);
1907 setOperationAction(ISD::MUL, MVT::v64i8, Custom);
1908
1909 setOperationAction(ISD::MULHU, MVT::v16i32, Custom);
1910 setOperationAction(ISD::MULHS, MVT::v16i32, Custom);
1911 setOperationAction(ISD::MULHS, MVT::v32i16, HasBWI ? Legal : Custom);
1912 setOperationAction(ISD::MULHU, MVT::v32i16, HasBWI ? Legal : Custom);
1913 setOperationAction(ISD::MULHS, MVT::v64i8, Custom);
1914 setOperationAction(ISD::MULHU, MVT::v64i8, Custom);
1915 setOperationAction(ISD::AVGCEILU, MVT::v32i16, HasBWI ? Legal : Custom);
1916 setOperationAction(ISD::AVGCEILU, MVT::v64i8, HasBWI ? Legal : Custom);
1917
1918 setOperationAction(ISD::SMULO, MVT::v64i8, Custom);
1919 setOperationAction(ISD::UMULO, MVT::v64i8, Custom);
1920
1921 for (auto VT : { MVT::v64i8, MVT::v32i16, MVT::v16i32, MVT::v8i64 }) {
1922 setOperationAction(ISD::SRL, VT, Custom);
1923 setOperationAction(ISD::SHL, VT, Custom);
1924 setOperationAction(ISD::SRA, VT, Custom);
1925 setOperationAction(ISD::ROTL, VT, Custom);
1926 setOperationAction(ISD::ROTR, VT, Custom);
1927 setOperationAction(ISD::SETCC, VT, Custom);
1928 setOperationAction(ISD::ABDS, VT, Custom);
1929 setOperationAction(ISD::ABDU, VT, Custom);
1930 setOperationAction(ISD::BITREVERSE, VT, Custom);
1931
1932 // The condition codes aren't legal in SSE/AVX and under AVX512 we use
1933 // setcc all the way to isel and prefer SETGT in some isel patterns.
1934 setCondCodeAction(ISD::SETLT, VT, Custom);
1935 setCondCodeAction(ISD::SETLE, VT, Custom);
1936 }
1937
1938 setOperationAction(ISD::SETCC, MVT::v8f64, Custom);
1939 setOperationAction(ISD::SETCC, MVT::v16f32, Custom);
1940 setOperationAction(ISD::STRICT_FSETCC, MVT::v8f64, Custom);
1941 setOperationAction(ISD::STRICT_FSETCC, MVT::v16f32, Custom);
1942 setOperationAction(ISD::STRICT_FSETCCS, MVT::v8f64, Custom);
1943 setOperationAction(ISD::STRICT_FSETCCS, MVT::v16f32, Custom);
1944
1945 for (auto VT : { MVT::v16i32, MVT::v8i64 }) {
1946 setOperationAction(ISD::SMAX, VT, Legal);
1947 setOperationAction(ISD::UMAX, VT, Legal);
1948 setOperationAction(ISD::SMIN, VT, Legal);
1949 setOperationAction(ISD::UMIN, VT, Legal);
1950 setOperationAction(ISD::ABS, VT, Legal);
1951 setOperationAction(ISD::CTPOP, VT, Custom);
1952 }
1953
1954 for (auto VT : { MVT::v64i8, MVT::v32i16 }) {
1955 setOperationAction(ISD::ABS, VT, HasBWI ? Legal : Custom);
1956 setOperationAction(ISD::CTPOP, VT, Subtarget.hasBITALG() ? Legal : Custom);
1957 setOperationAction(ISD::CTLZ, VT, Custom);
1958 setOperationAction(ISD::SMAX, VT, HasBWI ? Legal : Custom);
1959 setOperationAction(ISD::UMAX, VT, HasBWI ? Legal : Custom);
1960 setOperationAction(ISD::SMIN, VT, HasBWI ? Legal : Custom);
1961 setOperationAction(ISD::UMIN, VT, HasBWI ? Legal : Custom);
1962 setOperationAction(ISD::UADDSAT, VT, HasBWI ? Legal : Custom);
1963 setOperationAction(ISD::SADDSAT, VT, HasBWI ? Legal : Custom);
1964 setOperationAction(ISD::USUBSAT, VT, HasBWI ? Legal : Custom);
1965 setOperationAction(ISD::SSUBSAT, VT, HasBWI ? Legal : Custom);
1966 }
1967
1968 setOperationAction(ISD::FSHL, MVT::v64i8, Custom);
1969 setOperationAction(ISD::FSHR, MVT::v64i8, Custom);
1970 setOperationAction(ISD::FSHL, MVT::v32i16, Custom);
1971 setOperationAction(ISD::FSHR, MVT::v32i16, Custom);
1972 setOperationAction(ISD::FSHL, MVT::v16i32, Custom);
1973 setOperationAction(ISD::FSHR, MVT::v16i32, Custom);
1974
1975 if (Subtarget.hasDQI()) {
1976 for (auto Opc : {ISD::SINT_TO_FP, ISD::UINT_TO_FP, ISD::STRICT_SINT_TO_FP,
1977 ISD::STRICT_UINT_TO_FP, ISD::FP_TO_SINT, ISD::FP_TO_UINT,
1978 ISD::STRICT_FP_TO_SINT, ISD::STRICT_FP_TO_UINT})
1979 setOperationAction(Opc, MVT::v8i64, Custom);
1980 setOperationAction(ISD::MUL, MVT::v8i64, Legal);
1981 }
1982
1983 if (Subtarget.hasCDI()) {
1984 // NonVLX sub-targets extend 128/256 vectors to use the 512 version.
1985 for (auto VT : { MVT::v16i32, MVT::v8i64} ) {
1986 setOperationAction(ISD::CTLZ, VT, Legal);
1987 }
1988 } // Subtarget.hasCDI()
1989
1990 if (Subtarget.hasVPOPCNTDQ()) {
1991 for (auto VT : { MVT::v16i32, MVT::v8i64 })
1992 setOperationAction(ISD::CTPOP, VT, Legal);
1993 }
1994
1995 // Extract subvector is special because the value type
1996 // (result) is 256-bit but the source is 512-bit wide.
1997 // 128-bit was made Legal under AVX1.
1998 for (auto VT : { MVT::v32i8, MVT::v16i16, MVT::v8i32, MVT::v4i64,
1999 MVT::v16f16, MVT::v8f32, MVT::v4f64 })
2000 setOperationAction(ISD::EXTRACT_SUBVECTOR, VT, Legal);
2001
2002 for (auto VT : { MVT::v64i8, MVT::v32i16, MVT::v16i32, MVT::v8i64,
2003 MVT::v32f16, MVT::v16f32, MVT::v8f64 }) {
2004 setOperationAction(ISD::CONCAT_VECTORS, VT, Custom);
2005 setOperationAction(ISD::INSERT_SUBVECTOR, VT, Legal);
2006 setOperationAction(ISD::SELECT, VT, Custom);
2007 setOperationAction(ISD::VSELECT, VT, Custom);
2008 setOperationAction(ISD::BUILD_VECTOR, VT, Custom);
2009 setOperationAction(ISD::EXTRACT_VECTOR_ELT, VT, Custom);
2010 setOperationAction(ISD::VECTOR_SHUFFLE, VT, Custom);
2011 setOperationAction(ISD::SCALAR_TO_VECTOR, VT, Custom);
2012 setOperationAction(ISD::INSERT_VECTOR_ELT, VT, Custom);
2013 }
2014 setF16Action(MVT::v32f16, Expand);
2015 setOperationAction(ISD::FP_ROUND, MVT::v16f16, Custom);
2016 setOperationAction(ISD::STRICT_FP_ROUND, MVT::v16f16, Custom);
2017 setOperationAction(ISD::FP_EXTEND, MVT::v16f32, Custom);
2018 setOperationAction(ISD::STRICT_FP_EXTEND, MVT::v16f32, Custom);
2019 for (unsigned Opc : {ISD::FADD, ISD::FSUB, ISD::FMUL, ISD::FDIV})
2020 setOperationPromotedToType(Opc, MVT::v32f16, MVT::v32f32);
2021
2022 for (auto VT : { MVT::v16i32, MVT::v8i64, MVT::v16f32, MVT::v8f64 }) {
2023 setOperationAction(ISD::MLOAD, VT, Legal);
2024 setOperationAction(ISD::MSTORE, VT, Legal);
2025 setOperationAction(ISD::MGATHER, VT, Custom);
2026 setOperationAction(ISD::MSCATTER, VT, Custom);
2027 }
2028 if (HasBWI) {
2029 for (auto VT : { MVT::v64i8, MVT::v32i16 }) {
2030 setOperationAction(ISD::MLOAD, VT, Legal);
2031 setOperationAction(ISD::MSTORE, VT, Legal);
2032 }
2033 } else {
2034 setOperationAction(ISD::STORE, MVT::v32i16, Custom);
2035 setOperationAction(ISD::STORE, MVT::v64i8, Custom);
2036 }
2037
2038 if (Subtarget.hasVBMI2()) {
2039 for (auto VT : {MVT::v32i16, MVT::v16i32, MVT::v8i64}) {
2040 setOperationAction(ISD::FSHL, VT, Custom);
2041 setOperationAction(ISD::FSHR, VT, Custom);
2042 }
2043
2044 setOperationAction(ISD::ROTL, MVT::v32i16, Custom);
2045 setOperationAction(ISD::ROTR, MVT::v32i16, Custom);
2046 }
2047
2048 setOperationAction(ISD::FNEG, MVT::v32f16, Custom);
2049 setOperationAction(ISD::FABS, MVT::v32f16, Custom);
2050 setOperationAction(ISD::FCOPYSIGN, MVT::v32f16, Custom);
2051 }// useAVX512Regs
2052
2053 if (!Subtarget.useSoftFloat() && Subtarget.hasVBMI2()) {
2054 for (auto VT : {MVT::v8i16, MVT::v4i32, MVT::v2i64, MVT::v16i16, MVT::v8i32,
2055 MVT::v4i64}) {
2056 setOperationAction(ISD::FSHL, VT, Custom);
2057 setOperationAction(ISD::FSHR, VT, Custom);
2058 }
2059 }
2060
2061 // This block controls legalization for operations that don't have
2062 // pre-AVX512 equivalents. Without VLX we use 512-bit operations for
2063 // narrower widths.
2064 if (!Subtarget.useSoftFloat() && Subtarget.hasAVX512()) {
2065 // These operations are handled on non-VLX by artificially widening in
2066 // isel patterns.
2067
2068 setOperationAction(ISD::STRICT_FP_TO_UINT, MVT::v8i32, Custom);
2069 setOperationAction(ISD::STRICT_FP_TO_UINT, MVT::v4i32, Custom);
2070 setOperationAction(ISD::STRICT_FP_TO_UINT, MVT::v2i32, Custom);
2071
2072 if (Subtarget.hasDQI()) {
2073 // Fast v2f32 SINT_TO_FP( v2i64 ) custom conversion.
2074 // v2f32 UINT_TO_FP is already custom under SSE2.
2075 assert(isOperationCustom(ISD::UINT_TO_FP, MVT::v2f32) &&
2076 isOperationCustom(ISD::STRICT_UINT_TO_FP, MVT::v2f32) &&
2077 "Unexpected operation action!");
2078 // v2i64 FP_TO_S/UINT(v2f32) custom conversion.
2079 setOperationAction(ISD::FP_TO_SINT, MVT::v2f32, Custom);
2080 setOperationAction(ISD::FP_TO_UINT, MVT::v2f32, Custom);
2081 setOperationAction(ISD::STRICT_FP_TO_SINT, MVT::v2f32, Custom);
2082 setOperationAction(ISD::STRICT_FP_TO_UINT, MVT::v2f32, Custom);
2083 }
2084
2085 for (auto VT : { MVT::v2i64, MVT::v4i64 }) {
2086 setOperationAction(ISD::SMAX, VT, Legal);
2087 setOperationAction(ISD::UMAX, VT, Legal);
2088 setOperationAction(ISD::SMIN, VT, Legal);
2089 setOperationAction(ISD::UMIN, VT, Legal);
2090 setOperationAction(ISD::ABS, VT, Legal);
2091 }
2092
2093 for (auto VT : { MVT::v4i32, MVT::v8i32, MVT::v2i64, MVT::v4i64 }) {
2094 setOperationAction(ISD::ROTL, VT, Custom);
2095 setOperationAction(ISD::ROTR, VT, Custom);
2096 }
2097
2098 // Custom legalize 2x32 to get a little better code.
2099 setOperationAction(ISD::MSCATTER, MVT::v2f32, Custom);
2100 setOperationAction(ISD::MSCATTER, MVT::v2i32, Custom);
2101
2102 for (auto VT : { MVT::v4i32, MVT::v8i32, MVT::v2i64, MVT::v4i64,
2103 MVT::v4f32, MVT::v8f32, MVT::v2f64, MVT::v4f64 })
2104 setOperationAction(ISD::MSCATTER, VT, Custom);
2105
2106 if (Subtarget.hasDQI()) {
2107 for (auto Opc : {ISD::SINT_TO_FP, ISD::UINT_TO_FP, ISD::STRICT_SINT_TO_FP,
2108 ISD::STRICT_UINT_TO_FP, ISD::FP_TO_SINT, ISD::FP_TO_UINT,
2109 ISD::STRICT_FP_TO_SINT, ISD::STRICT_FP_TO_UINT}) {
2110 setOperationAction(Opc, MVT::v2i64, Custom);
2111 setOperationAction(Opc, MVT::v4i64, Custom);
2112 }
2113 setOperationAction(ISD::MUL, MVT::v2i64, Legal);
2114 setOperationAction(ISD::MUL, MVT::v4i64, Legal);
2115 }
2116
2117 if (Subtarget.hasCDI()) {
2118 for (auto VT : { MVT::v4i32, MVT::v8i32, MVT::v2i64, MVT::v4i64 }) {
2119 setOperationAction(ISD::CTLZ, VT, Legal);
2120 }
2121 } // Subtarget.hasCDI()
2122
2123 if (Subtarget.hasVPOPCNTDQ()) {
2124 for (auto VT : { MVT::v4i32, MVT::v8i32, MVT::v2i64, MVT::v4i64 })
2125 setOperationAction(ISD::CTPOP, VT, Legal);
2126 }
2127 }
2128
2129 // This block control legalization of v32i1/v64i1 which are available with
2130 // AVX512BW..
2131 if (!Subtarget.useSoftFloat() && Subtarget.hasBWI()) {
2132 addRegisterClass(MVT::v32i1, &X86::VK32RegClass);
2133 addRegisterClass(MVT::v64i1, &X86::VK64RegClass);
2134
2135 for (auto VT : { MVT::v32i1, MVT::v64i1 }) {
2136 setOperationAction(ISD::VSELECT, VT, Expand);
2137 setOperationAction(ISD::TRUNCATE, VT, Custom);
2138 setOperationAction(ISD::SETCC, VT, Custom);
2139 setOperationAction(ISD::EXTRACT_VECTOR_ELT, VT, Custom);
2140 setOperationAction(ISD::INSERT_VECTOR_ELT, VT, Custom);
2141 setOperationAction(ISD::SELECT, VT, Custom);
2142 setOperationAction(ISD::BUILD_VECTOR, VT, Custom);
2143 setOperationAction(ISD::VECTOR_SHUFFLE, VT, Custom);
2144 setOperationAction(ISD::CONCAT_VECTORS, VT, Custom);
2145 setOperationAction(ISD::INSERT_SUBVECTOR, VT, Custom);
2146 }
2147
2148 for (auto VT : { MVT::v16i1, MVT::v32i1 })
2149 setOperationAction(ISD::EXTRACT_SUBVECTOR, VT, Custom);
2150
2151 // Extends from v32i1 masks to 256-bit vectors.
2152 setOperationAction(ISD::SIGN_EXTEND, MVT::v32i8, Custom);
2153 setOperationAction(ISD::ZERO_EXTEND, MVT::v32i8, Custom);
2154 setOperationAction(ISD::ANY_EXTEND, MVT::v32i8, Custom);
2155
2156 for (auto VT : { MVT::v32i8, MVT::v16i8, MVT::v16i16, MVT::v8i16 }) {
2157 setOperationAction(ISD::MLOAD, VT, Subtarget.hasVLX() ? Legal : Custom);
2158 setOperationAction(ISD::MSTORE, VT, Subtarget.hasVLX() ? Legal : Custom);
2159 }
2160
2161 // These operations are handled on non-VLX by artificially widening in
2162 // isel patterns.
2163 // TODO: Custom widen in lowering on non-VLX and drop the isel patterns?
2164
2165 if (Subtarget.hasBITALG()) {
2166 for (auto VT : { MVT::v16i8, MVT::v32i8, MVT::v8i16, MVT::v16i16 })
2167 setOperationAction(ISD::CTPOP, VT, Legal);
2168 }
2169 }
2170
2171 if (!Subtarget.useSoftFloat() && Subtarget.hasFP16()) {
2172 auto setGroup = [&] (MVT VT) {
2173 setOperationAction(ISD::FADD, VT, Legal);
2174 setOperationAction(ISD::STRICT_FADD, VT, Legal);
2175 setOperationAction(ISD::FSUB, VT, Legal);
2176 setOperationAction(ISD::STRICT_FSUB, VT, Legal);
2177 setOperationAction(ISD::FMUL, VT, Legal);
2178 setOperationAction(ISD::STRICT_FMUL, VT, Legal);
2179 setOperationAction(ISD::FDIV, VT, Legal);
2180 setOperationAction(ISD::STRICT_FDIV, VT, Legal);
2181 setOperationAction(ISD::FSQRT, VT, Legal);
2182 setOperationAction(ISD::STRICT_FSQRT, VT, Legal);
2183
2184 setOperationAction(ISD::FFLOOR, VT, Legal);
2185 setOperationAction(ISD::STRICT_FFLOOR, VT, Legal);
2186 setOperationAction(ISD::FCEIL, VT, Legal);
2187 setOperationAction(ISD::STRICT_FCEIL, VT, Legal);
2188 setOperationAction(ISD::FTRUNC, VT, Legal);
2189 setOperationAction(ISD::STRICT_FTRUNC, VT, Legal);
2190 setOperationAction(ISD::FRINT, VT, Legal);
2191 setOperationAction(ISD::STRICT_FRINT, VT, Legal);
2192 setOperationAction(ISD::FNEARBYINT, VT, Legal);
2193 setOperationAction(ISD::STRICT_FNEARBYINT, VT, Legal);
2194 setOperationAction(ISD::FROUNDEVEN, VT, Legal);
2195 setOperationAction(ISD::STRICT_FROUNDEVEN, VT, Legal);
2196
2197 setOperationAction(ISD::FROUND, VT, Custom);
2198
2199 setOperationAction(ISD::LOAD, VT, Legal);
2200 setOperationAction(ISD::STORE, VT, Legal);
2201
2202 setOperationAction(ISD::FMA, VT, Legal);
2203 setOperationAction(ISD::STRICT_FMA, VT, Legal);
2204 setOperationAction(ISD::VSELECT, VT, Legal);
2205 setOperationAction(ISD::BUILD_VECTOR, VT, Custom);
2206 setOperationAction(ISD::SELECT, VT, Custom);
2207
2208 setOperationAction(ISD::FNEG, VT, Custom);
2209 setOperationAction(ISD::FABS, VT, Custom);
2210 setOperationAction(ISD::FCOPYSIGN, VT, Custom);
2211 setOperationAction(ISD::EXTRACT_VECTOR_ELT, VT, Custom);
2212 setOperationAction(ISD::VECTOR_SHUFFLE, VT, Custom);
2213
2214 setOperationAction(ISD::SETCC, VT, Custom);
2215 setOperationAction(ISD::STRICT_FSETCC, VT, Custom);
2216 setOperationAction(ISD::STRICT_FSETCCS, VT, Custom);
2217 };
2218
2219 // AVX512_FP16 scalar operations
2220 setGroup(MVT::f16);
2221 setOperationAction(ISD::FREM, MVT::f16, Promote);
2222 setOperationAction(ISD::STRICT_FREM, MVT::f16, Promote);
2223 setOperationAction(ISD::SELECT_CC, MVT::f16, Expand);
2224 setOperationAction(ISD::BR_CC, MVT::f16, Expand);
2225 setOperationAction(ISD::STRICT_FROUND, MVT::f16, Promote);
2226 setOperationAction(ISD::FROUNDEVEN, MVT::f16, Legal);
2227 setOperationAction(ISD::STRICT_FROUNDEVEN, MVT::f16, Legal);
2228 setOperationAction(ISD::FP_ROUND, MVT::f16, Custom);
2229 setOperationAction(ISD::STRICT_FP_ROUND, MVT::f16, Custom);
2230 setOperationAction(ISD::FMAXIMUM, MVT::f16, Custom);
2231 setOperationAction(ISD::FMINIMUM, MVT::f16, Custom);
2232 setOperationAction(ISD::FP_EXTEND, MVT::f32, Legal);
2233 setOperationAction(ISD::STRICT_FP_EXTEND, MVT::f32, Legal);
2234
2235 setCondCodeAction(ISD::SETOEQ, MVT::f16, Expand);
2236 setCondCodeAction(ISD::SETUNE, MVT::f16, Expand);
2237
2238 if (Subtarget.useAVX512Regs()) {
2239 setGroup(MVT::v32f16);
2240 setOperationAction(ISD::SCALAR_TO_VECTOR, MVT::v32f16, Custom);
2241 setOperationAction(ISD::SINT_TO_FP, MVT::v32i16, Legal);
2242 setOperationAction(ISD::STRICT_SINT_TO_FP, MVT::v32i16, Legal);
2243 setOperationAction(ISD::UINT_TO_FP, MVT::v32i16, Legal);
2244 setOperationAction(ISD::STRICT_UINT_TO_FP, MVT::v32i16, Legal);
2245 setOperationAction(ISD::FP_ROUND, MVT::v16f16, Legal);
2246 setOperationAction(ISD::STRICT_FP_ROUND, MVT::v16f16, Legal);
2247 setOperationAction(ISD::FP_EXTEND, MVT::v16f32, Custom);
2248 setOperationAction(ISD::STRICT_FP_EXTEND, MVT::v16f32, Legal);
2249 setOperationAction(ISD::FP_EXTEND, MVT::v8f64, Custom);
2250 setOperationAction(ISD::STRICT_FP_EXTEND, MVT::v8f64, Legal);
2251 setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v32f16, Custom);
2252
2253 setOperationAction(ISD::FP_TO_SINT, MVT::v32i16, Custom);
2254 setOperationAction(ISD::STRICT_FP_TO_SINT, MVT::v32i16, Custom);
2255 setOperationAction(ISD::FP_TO_UINT, MVT::v32i16, Custom);
2256 setOperationAction(ISD::STRICT_FP_TO_UINT, MVT::v32i16, Custom);
2257 setOperationPromotedToType(ISD::FP_TO_SINT, MVT::v32i8, MVT::v32i16);
2258 setOperationPromotedToType(ISD::STRICT_FP_TO_SINT, MVT::v32i8,
2259 MVT::v32i16);
2260 setOperationPromotedToType(ISD::FP_TO_UINT, MVT::v32i8, MVT::v32i16);
2261 setOperationPromotedToType(ISD::STRICT_FP_TO_UINT, MVT::v32i8,
2262 MVT::v32i16);
2263 setOperationPromotedToType(ISD::FP_TO_SINT, MVT::v32i1, MVT::v32i16);
2264 setOperationPromotedToType(ISD::STRICT_FP_TO_SINT, MVT::v32i1,
2265 MVT::v32i16);
2266 setOperationPromotedToType(ISD::FP_TO_UINT, MVT::v32i1, MVT::v32i16);
2267 setOperationPromotedToType(ISD::STRICT_FP_TO_UINT, MVT::v32i1,
2268 MVT::v32i16);
2269
2270 setOperationAction(ISD::EXTRACT_SUBVECTOR, MVT::v16f16, Legal);
2271 setOperationAction(ISD::INSERT_SUBVECTOR, MVT::v32f16, Legal);
2272 setOperationAction(ISD::CONCAT_VECTORS, MVT::v32f16, Custom);
2273
2274 setLoadExtAction(ISD::EXTLOAD, MVT::v8f64, MVT::v8f16, Legal);
2275 setLoadExtAction(ISD::EXTLOAD, MVT::v16f32, MVT::v16f16, Legal);
2276 }
2277
2278 if (Subtarget.hasVLX()) {
2279 setGroup(MVT::v8f16);
2280 setGroup(MVT::v16f16);
2281
2282 setOperationAction(ISD::SCALAR_TO_VECTOR, MVT::v8f16, Legal);
2283 setOperationAction(ISD::SCALAR_TO_VECTOR, MVT::v16f16, Custom);
2284 setOperationAction(ISD::SINT_TO_FP, MVT::v16i16, Legal);
2285 setOperationAction(ISD::STRICT_SINT_TO_FP, MVT::v16i16, Legal);
2286 setOperationAction(ISD::SINT_TO_FP, MVT::v8i16, Legal);
2287 setOperationAction(ISD::STRICT_SINT_TO_FP, MVT::v8i16, Legal);
2288 setOperationAction(ISD::UINT_TO_FP, MVT::v16i16, Legal);
2289 setOperationAction(ISD::STRICT_UINT_TO_FP, MVT::v16i16, Legal);
2290 setOperationAction(ISD::UINT_TO_FP, MVT::v8i16, Legal);
2291 setOperationAction(ISD::STRICT_UINT_TO_FP, MVT::v8i16, Legal);
2292
2293 setOperationAction(ISD::FP_TO_SINT, MVT::v8i16, Custom);
2294 setOperationAction(ISD::STRICT_FP_TO_SINT, MVT::v8i16, Custom);
2295 setOperationAction(ISD::FP_TO_UINT, MVT::v8i16, Custom);
2296 setOperationAction(ISD::STRICT_FP_TO_UINT, MVT::v8i16, Custom);
2297 setOperationAction(ISD::FP_ROUND, MVT::v8f16, Legal);
2298 setOperationAction(ISD::STRICT_FP_ROUND, MVT::v8f16, Legal);
2299 setOperationAction(ISD::FP_EXTEND, MVT::v8f32, Custom);
2300 setOperationAction(ISD::STRICT_FP_EXTEND, MVT::v8f32, Legal);
2301 setOperationAction(ISD::FP_EXTEND, MVT::v4f64, Custom);
2302 setOperationAction(ISD::STRICT_FP_EXTEND, MVT::v4f64, Legal);
2303
2304 // INSERT_VECTOR_ELT v8f16 extended to VECTOR_SHUFFLE
2305 setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v8f16, Custom);
2306 setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v16f16, Custom);
2307
2308 setOperationAction(ISD::EXTRACT_SUBVECTOR, MVT::v8f16, Legal);
2309 setOperationAction(ISD::INSERT_SUBVECTOR, MVT::v16f16, Legal);
2310 setOperationAction(ISD::CONCAT_VECTORS, MVT::v16f16, Custom);
2311
2312 setLoadExtAction(ISD::EXTLOAD, MVT::v4f64, MVT::v4f16, Legal);
2313 setLoadExtAction(ISD::EXTLOAD, MVT::v2f64, MVT::v2f16, Legal);
2314 setLoadExtAction(ISD::EXTLOAD, MVT::v8f32, MVT::v8f16, Legal);
2315 setLoadExtAction(ISD::EXTLOAD, MVT::v4f32, MVT::v4f16, Legal);
2316
2317 // Need to custom widen these to prevent scalarization.
2318 setOperationAction(ISD::LOAD, MVT::v4f16, Custom);
2319 setOperationAction(ISD::STORE, MVT::v4f16, Custom);
2320 }
2321 }
2322
2323 if (!Subtarget.useSoftFloat() &&
2324 (Subtarget.hasAVXNECONVERT() || Subtarget.hasBF16())) {
2325 addRegisterClass(MVT::v8bf16, Subtarget.hasAVX512() ? &X86::VR128XRegClass
2326 : &X86::VR128RegClass);
2327 addRegisterClass(MVT::v16bf16, Subtarget.hasAVX512() ? &X86::VR256XRegClass
2328 : &X86::VR256RegClass);
2329 // We set the type action of bf16 to TypeSoftPromoteHalf, but we don't
2330 // provide the method to promote BUILD_VECTOR and INSERT_VECTOR_ELT.
2331 // Set the operation action Custom to do the customization later.
2332 setOperationAction(ISD::BUILD_VECTOR, MVT::bf16, Custom);
2333 setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::bf16, Custom);
2334 for (auto VT : {MVT::v8bf16, MVT::v16bf16}) {
2335 setF16Action(VT, Expand);
2336 setOperationAction(ISD::BUILD_VECTOR, VT, Custom);
2337 setOperationAction(ISD::VECTOR_SHUFFLE, VT, Custom);
2338 setOperationAction(ISD::INSERT_SUBVECTOR, VT, Legal);
2339 setOperationAction(ISD::CONCAT_VECTORS, VT, Custom);
2340 }
2341 for (unsigned Opc : {ISD::FADD, ISD::FSUB, ISD::FMUL, ISD::FDIV}) {
2342 setOperationPromotedToType(Opc, MVT::v8bf16, MVT::v8f32);
2343 setOperationPromotedToType(Opc, MVT::v16bf16, MVT::v16f32);
2344 }
2345 setOperationAction(ISD::FP_ROUND, MVT::v8bf16, Custom);
2346 addLegalFPImmediate(APFloat::getZero(APFloat::BFloat()));
2347 }
2348
2349 if (!Subtarget.useSoftFloat() && Subtarget.hasBF16()) {
2350 addRegisterClass(MVT::v32bf16, &X86::VR512RegClass);
2351 setF16Action(MVT::v32bf16, Expand);
2352 for (unsigned Opc : {ISD::FADD, ISD::FSUB, ISD::FMUL, ISD::FDIV})
2353 setOperationPromotedToType(Opc, MVT::v32bf16, MVT::v32f32);
2354 setOperationAction(ISD::BUILD_VECTOR, MVT::v32bf16, Custom);
2355 setOperationAction(ISD::FP_ROUND, MVT::v16bf16, Custom);
2356 setOperationAction(ISD::VECTOR_SHUFFLE, MVT::v32bf16, Custom);
2357 setOperationAction(ISD::INSERT_SUBVECTOR, MVT::v32bf16, Legal);
2358 setOperationAction(ISD::CONCAT_VECTORS, MVT::v32bf16, Custom);
2359 }
2360
2361 if (!Subtarget.useSoftFloat() && Subtarget.hasVLX()) {
2362 setTruncStoreAction(MVT::v4i64, MVT::v4i8, Legal);
2363 setTruncStoreAction(MVT::v4i64, MVT::v4i16, Legal);
2364 setTruncStoreAction(MVT::v4i64, MVT::v4i32, Legal);
2365 setTruncStoreAction(MVT::v8i32, MVT::v8i8, Legal);
2366 setTruncStoreAction(MVT::v8i32, MVT::v8i16, Legal);
2367
2368 setTruncStoreAction(MVT::v2i64, MVT::v2i8, Legal);
2369 setTruncStoreAction(MVT::v2i64, MVT::v2i16, Legal);
2370 setTruncStoreAction(MVT::v2i64, MVT::v2i32, Legal);
2371 setTruncStoreAction(MVT::v4i32, MVT::v4i8, Legal);
2372 setTruncStoreAction(MVT::v4i32, MVT::v4i16, Legal);
2373
2374 if (Subtarget.hasBWI()) {
2375 setTruncStoreAction(MVT::v16i16, MVT::v16i8, Legal);
2376 setTruncStoreAction(MVT::v8i16, MVT::v8i8, Legal);
2377 }
2378
2379 if (Subtarget.hasFP16()) {
2380 // vcvttph2[u]dq v4f16 -> v4i32/64, v2f16 -> v2i32/64
2381 setOperationAction(ISD::FP_TO_SINT, MVT::v2f16, Custom);
2382 setOperationAction(ISD::STRICT_FP_TO_SINT, MVT::v2f16, Custom);
2383 setOperationAction(ISD::FP_TO_UINT, MVT::v2f16, Custom);
2384 setOperationAction(ISD::STRICT_FP_TO_UINT, MVT::v2f16, Custom);
2385 setOperationAction(ISD::FP_TO_SINT, MVT::v4f16, Custom);
2386 setOperationAction(ISD::STRICT_FP_TO_SINT, MVT::v4f16, Custom);
2387 setOperationAction(ISD::FP_TO_UINT, MVT::v4f16, Custom);
2388 setOperationAction(ISD::STRICT_FP_TO_UINT, MVT::v4f16, Custom);
2389 // vcvt[u]dq2ph v4i32/64 -> v4f16, v2i32/64 -> v2f16
2390 setOperationAction(ISD::SINT_TO_FP, MVT::v2f16, Custom);
2391 setOperationAction(ISD::STRICT_SINT_TO_FP, MVT::v2f16, Custom);
2392 setOperationAction(ISD::UINT_TO_FP, MVT::v2f16, Custom);
2393 setOperationAction(ISD::STRICT_UINT_TO_FP, MVT::v2f16, Custom);
2394 setOperationAction(ISD::SINT_TO_FP, MVT::v4f16, Custom);
2395 setOperationAction(ISD::STRICT_SINT_TO_FP, MVT::v4f16, Custom);
2396 setOperationAction(ISD::UINT_TO_FP, MVT::v4f16, Custom);
2397 setOperationAction(ISD::STRICT_UINT_TO_FP, MVT::v4f16, Custom);
2398 // vcvtps2phx v4f32 -> v4f16, v2f32 -> v2f16
2399 setOperationAction(ISD::FP_ROUND, MVT::v2f16, Custom);
2400 setOperationAction(ISD::STRICT_FP_ROUND, MVT::v2f16, Custom);
2401 setOperationAction(ISD::FP_ROUND, MVT::v4f16, Custom);
2402 setOperationAction(ISD::STRICT_FP_ROUND, MVT::v4f16, Custom);
2403 // vcvtph2psx v4f16 -> v4f32, v2f16 -> v2f32
2404 setOperationAction(ISD::FP_EXTEND, MVT::v2f16, Custom);
2405 setOperationAction(ISD::STRICT_FP_EXTEND, MVT::v2f16, Custom);
2406 setOperationAction(ISD::FP_EXTEND, MVT::v4f16, Custom);
2407 setOperationAction(ISD::STRICT_FP_EXTEND, MVT::v4f16, Custom);
2408 }
2409 }
2410
2411 if (!Subtarget.useSoftFloat() && Subtarget.hasAMXTILE()) {
2412 addRegisterClass(MVT::x86amx, &X86::TILERegClass);
2413 }
2414
2415 // We want to custom lower some of our intrinsics.
2416 setOperationAction(ISD::INTRINSIC_WO_CHAIN, MVT::Other, Custom);
2417 setOperationAction(ISD::INTRINSIC_W_CHAIN, MVT::Other, Custom);
2418 setOperationAction(ISD::INTRINSIC_VOID, MVT::Other, Custom);
2419 if (!Subtarget.is64Bit()) {
2420 setOperationAction(ISD::INTRINSIC_W_CHAIN, MVT::i64, Custom);
2421 }
2422
2423 // Only custom-lower 64-bit SADDO and friends on 64-bit because we don't
2424 // handle type legalization for these operations here.
2425 //
2426 // FIXME: We really should do custom legalization for addition and
2427 // subtraction on x86-32 once PR3203 is fixed. We really can't do much better
2428 // than generic legalization for 64-bit multiplication-with-overflow, though.
2429 for (auto VT : { MVT::i8, MVT::i16, MVT::i32, MVT::i64 }) {
2430 if (VT == MVT::i64 && !Subtarget.is64Bit())
2431 continue;
2432 // Add/Sub/Mul with overflow operations are custom lowered.
2433 setOperationAction(ISD::SADDO, VT, Custom);
2434 setOperationAction(ISD::UADDO, VT, Custom);
2435 setOperationAction(ISD::SSUBO, VT, Custom);
2436 setOperationAction(ISD::USUBO, VT, Custom);
2437 setOperationAction(ISD::SMULO, VT, Custom);
2438 setOperationAction(ISD::UMULO, VT, Custom);
2439
2440 // Support carry in as value rather than glue.
2441 setOperationAction(ISD::UADDO_CARRY, VT, Custom);
2442 setOperationAction(ISD::USUBO_CARRY, VT, Custom);
2443 setOperationAction(ISD::SETCCCARRY, VT, Custom);
2444 setOperationAction(ISD::SADDO_CARRY, VT, Custom);
2445 setOperationAction(ISD::SSUBO_CARRY, VT, Custom);
2446 }
2447
2448 // Combine sin / cos into _sincos_stret if it is available.
2449 if (getLibcallName(RTLIB::SINCOS_STRET_F32) != nullptr &&
2450 getLibcallName(RTLIB::SINCOS_STRET_F64) != nullptr) {
2451 setOperationAction(ISD::FSINCOS, MVT::f64, Custom);
2452 setOperationAction(ISD::FSINCOS, MVT::f32, Custom);
2453 }
2454
2455 if (Subtarget.isTargetWin64()) {
2456 setOperationAction(ISD::SDIV, MVT::i128, Custom);
2457 setOperationAction(ISD::UDIV, MVT::i128, Custom);
2458 setOperationAction(ISD::SREM, MVT::i128, Custom);
2459 setOperationAction(ISD::UREM, MVT::i128, Custom);
2460 setOperationAction(ISD::FP_TO_SINT, MVT::i128, Custom);
2461 setOperationAction(ISD::FP_TO_UINT, MVT::i128, Custom);
2462 setOperationAction(ISD::SINT_TO_FP, MVT::i128, Custom);
2463 setOperationAction(ISD::UINT_TO_FP, MVT::i128, Custom);
2464 setOperationAction(ISD::STRICT_FP_TO_SINT, MVT::i128, Custom);
2465 setOperationAction(ISD::STRICT_FP_TO_UINT, MVT::i128, Custom);
2466 setOperationAction(ISD::STRICT_SINT_TO_FP, MVT::i128, Custom);
2467 setOperationAction(ISD::STRICT_UINT_TO_FP, MVT::i128, Custom);
2468 }
2469
2470 // On 32 bit MSVC, `fmodf(f32)` is not defined - only `fmod(f64)`
2471 // is. We should promote the value to 64-bits to solve this.
2472 // This is what the CRT headers do - `fmodf` is an inline header
2473 // function casting to f64 and calling `fmod`.
2474 if (Subtarget.is32Bit() &&
2475 (Subtarget.isTargetWindowsMSVC() || Subtarget.isTargetWindowsItanium()))
2476 // clang-format off
2477 for (ISD::NodeType Op :
2478 {ISD::FACOS, ISD::STRICT_FACOS,
2479 ISD::FASIN, ISD::STRICT_FASIN,
2480 ISD::FATAN, ISD::STRICT_FATAN,
2481 ISD::FCEIL, ISD::STRICT_FCEIL,
2482 ISD::FCOS, ISD::STRICT_FCOS,
2483 ISD::FCOSH, ISD::STRICT_FCOSH,
2484 ISD::FEXP, ISD::STRICT_FEXP,
2485 ISD::FFLOOR, ISD::STRICT_FFLOOR,
2486 ISD::FREM, ISD::STRICT_FREM,
2487 ISD::FLOG, ISD::STRICT_FLOG,
2488 ISD::FLOG10, ISD::STRICT_FLOG10,
2489 ISD::FPOW, ISD::STRICT_FPOW,
2490 ISD::FSIN, ISD::STRICT_FSIN,
2491 ISD::FSINH, ISD::STRICT_FSINH,
2492 ISD::FTAN, ISD::STRICT_FTAN,
2493 ISD::FTANH, ISD::STRICT_FTANH})
2494 if (isOperationExpand(Op, MVT::f32))
2495 setOperationAction(Op, MVT::f32, Promote);
2496 // clang-format on
2497
2498 // On MSVC, both 32-bit and 64-bit, ldexpf(f32) is not defined. MinGW has
2499 // it, but it's just a wrapper around ldexp.
2500 if (Subtarget.isOSWindows()) {
2501 for (ISD::NodeType Op : {ISD::FLDEXP, ISD::STRICT_FLDEXP, ISD::FFREXP})
2502 if (isOperationExpand(Op, MVT::f32))
2503 setOperationAction(Op, MVT::f32, Promote);
2504 }
2505
2506 // We have target-specific dag combine patterns for the following nodes:
2507 setTargetDAGCombine({ISD::VECTOR_SHUFFLE,
2508 ISD::SCALAR_TO_VECTOR,
2509 ISD::INSERT_VECTOR_ELT,
2510 ISD::EXTRACT_VECTOR_ELT,
2511 ISD::CONCAT_VECTORS,
2512 ISD::INSERT_SUBVECTOR,
2513 ISD::EXTRACT_SUBVECTOR,
2514 ISD::BITCAST,
2515 ISD::VSELECT,
2516 ISD::SELECT,
2517 ISD::SHL,
2518 ISD::SRA,
2519 ISD::SRL,
2520 ISD::OR,
2521 ISD::AND,
2522 ISD::AVGCEILS,
2523 ISD::AVGCEILU,
2524 ISD::AVGFLOORS,
2525 ISD::AVGFLOORU,
2526 ISD::BITREVERSE,
2527 ISD::ADD,
2528 ISD::FADD,
2529 ISD::FSUB,
2530 ISD::FNEG,
2531 ISD::FMA,
2532 ISD::STRICT_FMA,
2533 ISD::FMINNUM,
2534 ISD::FMAXNUM,
2535 ISD::SUB,
2536 ISD::LOAD,
2537 ISD::LRINT,
2538 ISD::LLRINT,
2539 ISD::MLOAD,
2540 ISD::STORE,
2541 ISD::MSTORE,
2542 ISD::TRUNCATE,
2543 ISD::ZERO_EXTEND,
2544 ISD::ANY_EXTEND,
2545 ISD::SIGN_EXTEND,
2546 ISD::SIGN_EXTEND_INREG,
2547 ISD::ANY_EXTEND_VECTOR_INREG,
2548 ISD::SIGN_EXTEND_VECTOR_INREG,
2549 ISD::ZERO_EXTEND_VECTOR_INREG,
2550 ISD::SINT_TO_FP,
2551 ISD::UINT_TO_FP,
2552 ISD::STRICT_SINT_TO_FP,
2553 ISD::STRICT_UINT_TO_FP,
2554 ISD::SETCC,
2555 ISD::MUL,
2556 ISD::XOR,
2557 ISD::MSCATTER,
2558 ISD::MGATHER,
2559 ISD::FP16_TO_FP,
2560 ISD::FP_EXTEND,
2561 ISD::STRICT_FP_EXTEND,
2562 ISD::FP_ROUND,
2563 ISD::STRICT_FP_ROUND,
2564 ISD::INTRINSIC_VOID,
2565 ISD::INTRINSIC_WO_CHAIN,
2566 ISD::INTRINSIC_W_CHAIN});
2567
2568 computeRegisterProperties(Subtarget.getRegisterInfo());
2569
2570 MaxStoresPerMemset = 16; // For @llvm.memset -> sequence of stores
2571 MaxStoresPerMemsetOptSize = 8;
2572 MaxStoresPerMemcpy = 8; // For @llvm.memcpy -> sequence of stores
2573 MaxStoresPerMemcpyOptSize = 4;
2574 MaxStoresPerMemmove = 8; // For @llvm.memmove -> sequence of stores
2575 MaxStoresPerMemmoveOptSize = 4;
2576
2577 // TODO: These control memcmp expansion in CGP and could be raised higher, but
2578 // that needs to benchmarked and balanced with the potential use of vector
2579 // load/store types (PR33329, PR33914).
2580 MaxLoadsPerMemcmp = 2;
2581 MaxLoadsPerMemcmpOptSize = 2;
2582
2583 // Default loop alignment, which can be overridden by -align-loops.
2584 setPrefLoopAlignment(Align(16));
2585
2586 // An out-of-order CPU can speculatively execute past a predictable branch,
2587 // but a conditional move could be stalled by an expensive earlier operation.
2588 PredictableSelectIsExpensive = Subtarget.getSchedModel().isOutOfOrder();
2589 EnableExtLdPromotion = true;
2590 setPrefFunctionAlignment(Align(16));
2591
2592 verifyIntrinsicTables();
2593
2594 // Default to having -disable-strictnode-mutation on
2595 IsStrictFPEnabled = true;
2596}
2597
2598// This has so far only been implemented for 64-bit MachO.
2599bool X86TargetLowering::useLoadStackGuardNode() const {
2600 return Subtarget.isTargetMachO() && Subtarget.is64Bit();
2601}
2602
2603bool X86TargetLowering::useStackGuardXorFP() const {
2604 // Currently only MSVC CRTs XOR the frame pointer into the stack guard value.
2605 return Subtarget.getTargetTriple().isOSMSVCRT() && !Subtarget.isTargetMachO();
2606}
2607
2608SDValue X86TargetLowering::emitStackGuardXorFP(SelectionDAG &DAG, SDValue Val,
2609 const SDLoc &DL) const {
2610 EVT PtrTy = getPointerTy(DAG.getDataLayout());
2611 unsigned XorOp = Subtarget.is64Bit() ? X86::XOR64_FP : X86::XOR32_FP;
2612 MachineSDNode *Node = DAG.getMachineNode(XorOp, DL, PtrTy, Val);
2613 return SDValue(Node, 0);
2614}
2615
2616TargetLoweringBase::LegalizeTypeAction
2617X86TargetLowering::getPreferredVectorAction(MVT VT) const {
2618 if ((VT == MVT::v32i1 || VT == MVT::v64i1) && Subtarget.hasAVX512() &&
2619 !Subtarget.hasBWI())
2620 return TypeSplitVector;
2621
2622 if (!VT.isScalableVector() && VT.getVectorNumElements() != 1 &&
2623 !Subtarget.hasF16C() && VT.getVectorElementType() == MVT::f16)
2624 return TypeSplitVector;
2625
2626 if (!VT.isScalableVector() && VT.getVectorNumElements() != 1 &&
2627 VT.getVectorElementType() != MVT::i1)
2628 return TypeWidenVector;
2629
2630 return TargetLoweringBase::getPreferredVectorAction(VT);
2631}
2632
2633FastISel *
2634X86TargetLowering::createFastISel(FunctionLoweringInfo &funcInfo,
2635 const TargetLibraryInfo *libInfo) const {
2636 return X86::createFastISel(funcInfo, libInfo);
2637}
2638
2639//===----------------------------------------------------------------------===//
2640// Other Lowering Hooks
2641//===----------------------------------------------------------------------===//
2642
2643bool X86::mayFoldLoad(SDValue Op, const X86Subtarget &Subtarget,
2644 bool AssumeSingleUse) {
2645 if (!AssumeSingleUse && !Op.hasOneUse())
2646 return false;
2647 if (!ISD::isNormalLoad(Op.getNode()))
2648 return false;
2649
2650 // If this is an unaligned vector, make sure the target supports folding it.
2651 auto *Ld = cast<LoadSDNode>(Op.getNode());
2652 if (!Subtarget.hasAVX() && !Subtarget.hasSSEUnalignedMem() &&
2653 Ld->getValueSizeInBits(0) == 128 && Ld->getAlign() < Align(16))
2654 return false;
2655
2656 // TODO: If this is a non-temporal load and the target has an instruction
2657 // for it, it should not be folded. See "useNonTemporalLoad()".
2658
2659 return true;
2660}
2661
2662bool X86::mayFoldLoadIntoBroadcastFromMem(SDValue Op, MVT EltVT,
2663 const X86Subtarget &Subtarget,
2664 bool AssumeSingleUse) {
2665 assert(Subtarget.hasAVX() && "Expected AVX for broadcast from memory");
2666 if (!X86::mayFoldLoad(Op, Subtarget, AssumeSingleUse))
2667 return false;
2668
2669 // We can not replace a wide volatile load with a broadcast-from-memory,
2670 // because that would narrow the load, which isn't legal for volatiles.
2671 auto *Ld = cast<LoadSDNode>(Op.getNode());
2672 return !Ld->isVolatile() ||
2673 Ld->getValueSizeInBits(0) == EltVT.getScalarSizeInBits();
2674}
2675
2676bool X86::mayFoldIntoStore(SDValue Op) {
2677 return Op.hasOneUse() && ISD::isNormalStore(*Op.getNode()->use_begin());
2678}
2679
2680bool X86::mayFoldIntoZeroExtend(SDValue Op) {
2681 if (Op.hasOneUse()) {
2682 unsigned Opcode = Op.getNode()->use_begin()->getOpcode();
2683 return (ISD::ZERO_EXTEND == Opcode);
2684 }
2685 return false;
2686}
2687
2688static bool isLogicOp(unsigned Opcode) {
2689 // TODO: Add support for X86ISD::FAND/FOR/FXOR/FANDN with test coverage.
2690 return ISD::isBitwiseLogicOp(Opcode) || X86ISD::ANDNP == Opcode;
2691}
2692
2693static bool isTargetShuffle(unsigned Opcode) {
2694 switch(Opcode) {
2695 default: return false;
2696 case X86ISD::BLENDI:
2697 case X86ISD::PSHUFB:
2698 case X86ISD::PSHUFD:
2699 case X86ISD::PSHUFHW:
2700 case X86ISD::PSHUFLW:
2701 case X86ISD::SHUFP:
2702 case X86ISD::INSERTPS:
2703 case X86ISD::EXTRQI:
2704 case X86ISD::INSERTQI:
2705 case X86ISD::VALIGN:
2706 case X86ISD::PALIGNR:
2707 case X86ISD::VSHLDQ:
2708 case X86ISD::VSRLDQ:
2709 case X86ISD::MOVLHPS:
2710 case X86ISD::MOVHLPS:
2711 case X86ISD::MOVSHDUP:
2712 case X86ISD::MOVSLDUP:
2713 case X86ISD::MOVDDUP:
2714 case X86ISD::MOVSS:
2715 case X86ISD::MOVSD:
2716 case X86ISD::MOVSH:
2717 case X86ISD::UNPCKL:
2718 case X86ISD::UNPCKH:
2719 case X86ISD::VBROADCAST:
2720 case X86ISD::VPERMILPI:
2721 case X86ISD::VPERMILPV:
2722 case X86ISD::VPERM2X128:
2723 case X86ISD::SHUF128:
2724 case X86ISD::VPERMIL2:
2725 case X86ISD::VPERMI:
2726 case X86ISD::VPPERM:
2727 case X86ISD::VPERMV:
2728 case X86ISD::VPERMV3:
2729 case X86ISD::VZEXT_MOVL:
2730 return true;
2731 }
2732}
2733
2734static bool isTargetShuffleVariableMask(unsigned Opcode) {
2735 switch (Opcode) {
2736 default: return false;
2737 // Target Shuffles.
2738 case X86ISD::PSHUFB:
2739 case X86ISD::VPERMILPV:
2740 case X86ISD::VPERMIL2:
2741 case X86ISD::VPPERM:
2742 case X86ISD::VPERMV:
2743 case X86ISD::VPERMV3:
2744 return true;
2745 // 'Faux' Target Shuffles.
2746 case ISD::OR:
2747 case ISD::AND:
2748 case X86ISD::ANDNP:
2749 return true;
2750 }
2751}
2752
2753SDValue X86TargetLowering::getReturnAddressFrameIndex(SelectionDAG &DAG) const {
2754 MachineFunction &MF = DAG.getMachineFunction();
2755 const X86RegisterInfo *RegInfo = Subtarget.getRegisterInfo();
2756 X86MachineFunctionInfo *FuncInfo = MF.getInfo<X86MachineFunctionInfo>();
2757 int ReturnAddrIndex = FuncInfo->getRAIndex();
2758
2759 if (ReturnAddrIndex == 0) {
2760 // Set up a frame object for the return address.
2761 unsigned SlotSize = RegInfo->getSlotSize();
2762 ReturnAddrIndex = MF.getFrameInfo().CreateFixedObject(SlotSize,
2763 -(int64_t)SlotSize,
2764 false);
2765 FuncInfo->setRAIndex(ReturnAddrIndex);
2766 }
2767
2768 return DAG.getFrameIndex(ReturnAddrIndex, getPointerTy(DAG.getDataLayout()));
2769}
2770
2771bool X86::isOffsetSuitableForCodeModel(int64_t Offset, CodeModel::Model CM,
2772 bool HasSymbolicDisplacement) {
2773 // Offset should fit into 32 bit immediate field.
2774 if (!isInt<32>(Offset))
2775 return false;
2776
2777 // If we don't have a symbolic displacement - we don't have any extra
2778 // restrictions.
2779 if (!HasSymbolicDisplacement)
2780 return true;
2781
2782 // We can fold large offsets in the large code model because we always use
2783 // 64-bit offsets.
2784 if (CM == CodeModel::Large)
2785 return true;
2786
2787 // For kernel code model we know that all object resist in the negative half
2788 // of 32bits address space. We may not accept negative offsets, since they may
2789 // be just off and we may accept pretty large positive ones.
2790 if (CM == CodeModel::Kernel)
2791 return Offset >= 0;
2792
2793 // For other non-large code models we assume that latest small object is 16MB
2794 // before end of 31 bits boundary. We may also accept pretty large negative
2795 // constants knowing that all objects are in the positive half of address
2796 // space.
2797 return Offset < 16 * 1024 * 1024;
2798}
2799
2800/// Return true if the condition is an signed comparison operation.
2801static bool isX86CCSigned(unsigned X86CC) {
2802 switch (X86CC) {
2803 default:
2804 llvm_unreachable("Invalid integer condition!");
2805 case X86::COND_E:
2806 case X86::COND_NE:
2807 case X86::COND_B:
2808 case X86::COND_A:
2809 case X86::COND_BE:
2810 case X86::COND_AE:
2811 return false;
2812 case X86::COND_G:
2813 case X86::COND_GE:
2814 case X86::COND_L:
2815 case X86::COND_LE:
2816 return true;
2817 }
2818}
2819
2820static X86::CondCode TranslateIntegerX86CC(ISD::CondCode SetCCOpcode) {
2821 switch (SetCCOpcode) {
2822 // clang-format off
2823 default: llvm_unreachable("Invalid integer condition!");
2824 case ISD::SETEQ: return X86::COND_E;
2825 case ISD::SETGT: return X86::COND_G;
2826 case ISD::SETGE: return X86::COND_GE;
2827 case ISD::SETLT: return X86::COND_L;
2828 case ISD::SETLE: return X86::COND_LE;
2829 case ISD::SETNE: return X86::COND_NE;
2830 case ISD::SETULT: return X86::COND_B;
2831 case ISD::SETUGT: return X86::COND_A;
2832 case ISD::SETULE: return X86::COND_BE;
2833 case ISD::SETUGE: return X86::COND_AE;
2834 // clang-format on
2835 }
2836}
2837
2838/// Do a one-to-one translation of a ISD::CondCode to the X86-specific
2839/// condition code, returning the condition code and the LHS/RHS of the
2840/// comparison to make.
2841static X86::CondCode TranslateX86CC(ISD::CondCode SetCCOpcode, const SDLoc &DL,
2842 bool isFP, SDValue &LHS, SDValue &RHS,
2843 SelectionDAG &DAG) {
2844 if (!isFP) {
2845 if (ConstantSDNode *RHSC = dyn_cast<ConstantSDNode>(RHS)) {
2846 if (SetCCOpcode == ISD::SETGT && RHSC->isAllOnes()) {
2847 // X > -1 -> X == 0, jump !sign.
2848 RHS = DAG.getConstant(0, DL, RHS.getValueType());
2849 return X86::COND_NS;
2850 }
2851 if (SetCCOpcode == ISD::SETLT && RHSC->isZero()) {
2852 // X < 0 -> X == 0, jump on sign.
2853 return X86::COND_S;
2854 }
2855 if (SetCCOpcode == ISD::SETGE && RHSC->isZero()) {
2856 // X >= 0 -> X == 0, jump on !sign.
2857 return X86::COND_NS;
2858 }
2859 if (SetCCOpcode == ISD::SETLT && RHSC->isOne()) {
2860 // X < 1 -> X <= 0
2861 RHS = DAG.getConstant(0, DL, RHS.getValueType());
2862 return X86::COND_LE;
2863 }
2864 }
2865
2866 return TranslateIntegerX86CC(SetCCOpcode);
2867 }
2868
2869 // First determine if it is required or is profitable to flip the operands.
2870
2871 // If LHS is a foldable load, but RHS is not, flip the condition.
2872 if (ISD::isNON_EXTLoad(LHS.getNode()) &&
2873 !ISD::isNON_EXTLoad(RHS.getNode())) {
2874 SetCCOpcode = getSetCCSwappedOperands(SetCCOpcode);
2875 std::swap(LHS, RHS);
2876 }
2877
2878 switch (SetCCOpcode) {
2879 default: break;
2880 case ISD::SETOLT:
2881 case ISD::SETOLE:
2882 case ISD::SETUGT:
2883 case ISD::SETUGE:
2884 std::swap(LHS, RHS);
2885 break;
2886 }
2887
2888 // On a floating point condition, the flags are set as follows:
2889 // ZF PF CF op
2890 // 0 | 0 | 0 | X > Y
2891 // 0 | 0 | 1 | X < Y
2892 // 1 | 0 | 0 | X == Y
2893 // 1 | 1 | 1 | unordered
2894 switch (SetCCOpcode) {
2895 // clang-format off
2896 default: llvm_unreachable("Condcode should be pre-legalized away");
2897 case ISD::SETUEQ:
2898 case ISD::SETEQ: return X86::COND_E;
2899 case ISD::SETOLT: // flipped
2900 case ISD::SETOGT:
2901 case ISD::SETGT: return X86::COND_A;
2902 case ISD::SETOLE: // flipped
2903 case ISD::SETOGE:
2904 case ISD::SETGE: return X86::COND_AE;
2905 case ISD::SETUGT: // flipped
2906 case ISD::SETULT:
2907 case ISD::SETLT: return X86::COND_B;
2908 case ISD::SETUGE: // flipped
2909 case ISD::SETULE:
2910 case ISD::SETLE: return X86::COND_BE;
2911 case ISD::SETONE:
2912 case ISD::SETNE: return X86::COND_NE;
2913 case ISD::SETUO: return X86::COND_P;
2914 case ISD::SETO: return X86::COND_NP;
2915 case ISD::SETOEQ:
2916 case ISD::SETUNE: return X86::COND_INVALID;
2917 // clang-format on
2918 }
2919}
2920
2921/// Is there a floating point cmov for the specific X86 condition code?
2922/// Current x86 isa includes the following FP cmov instructions:
2923/// fcmovb, fcomvbe, fcomve, fcmovu, fcmovae, fcmova, fcmovne, fcmovnu.
2924static bool hasFPCMov(unsigned X86CC) {
2925 switch (X86CC) {
2926 default:
2927 return false;
2928 case X86::COND_B:
2929 case X86::COND_BE:
2930 case X86::COND_E:
2931 case X86::COND_P:
2932 case X86::COND_A:
2933 case X86::COND_AE:
2934 case X86::COND_NE:
2935 case X86::COND_NP:
2936 return true;
2937 }
2938}
2939
2940static bool useVPTERNLOG(const X86Subtarget &Subtarget, MVT VT) {
2941 return Subtarget.hasVLX() || Subtarget.canExtendTo512DQ() ||
2942 VT.is512BitVector();
2943}
2944
2945bool X86TargetLowering::getTgtMemIntrinsic(IntrinsicInfo &Info,
2946 const CallInst &I,
2947 MachineFunction &MF,
2948 unsigned Intrinsic) const {
2949 Info.flags = MachineMemOperand::MONone;
2950 Info.offset = 0;
2951
2952 const IntrinsicData* IntrData = getIntrinsicWithChain(Intrinsic);
2953 if (!IntrData) {
2954 switch (Intrinsic) {
2955 case Intrinsic::x86_aesenc128kl:
2956 case Intrinsic::x86_aesdec128kl:
2957 Info.opc = ISD::INTRINSIC_W_CHAIN;
2958 Info.ptrVal = I.getArgOperand(1);
2959 Info.memVT = EVT::getIntegerVT(I.getType()->getContext(), 48);
2960 Info.align = Align(1);
2961 Info.flags |= MachineMemOperand::MOLoad;
2962 return true;
2963 case Intrinsic::x86_aesenc256kl:
2964 case Intrinsic::x86_aesdec256kl:
2965 Info.opc = ISD::INTRINSIC_W_CHAIN;
2966 Info.ptrVal = I.getArgOperand(1);
2967 Info.memVT = EVT::getIntegerVT(I.getType()->getContext(), 64);
2968 Info.align = Align(1);
2969 Info.flags |= MachineMemOperand::MOLoad;
2970 return true;
2971 case Intrinsic::x86_aesencwide128kl:
2972 case Intrinsic::x86_aesdecwide128kl:
2973 Info.opc = ISD::INTRINSIC_W_CHAIN;
2974 Info.ptrVal = I.getArgOperand(0);
2975 Info.memVT = EVT::getIntegerVT(I.getType()->getContext(), 48);
2976 Info.align = Align(1);
2977 Info.flags |= MachineMemOperand::MOLoad;
2978 return true;
2979 case Intrinsic::x86_aesencwide256kl:
2980 case Intrinsic::x86_aesdecwide256kl:
2981 Info.opc = ISD::INTRINSIC_W_CHAIN;
2982 Info.ptrVal = I.getArgOperand(0);
2983 Info.memVT = EVT::getIntegerVT(I.getType()->getContext(), 64);
2984 Info.align = Align(1);
2985 Info.flags |= MachineMemOperand::MOLoad;
2986 return true;
2987 case Intrinsic::x86_cmpccxadd32:
2988 case Intrinsic::x86_cmpccxadd64:
2989 case Intrinsic::x86_atomic_bts:
2990 case Intrinsic::x86_atomic_btc:
2991 case Intrinsic::x86_atomic_btr: {
2992 Info.opc = ISD::INTRINSIC_W_CHAIN;
2993 Info.ptrVal = I.getArgOperand(0);
2994 unsigned Size = I.getType()->getScalarSizeInBits();
2995 Info.memVT = EVT::getIntegerVT(I.getType()->getContext(), Size);
2996 Info.align = Align(Size);
2997 Info.flags |= MachineMemOperand::MOLoad | MachineMemOperand::MOStore |
2998 MachineMemOperand::MOVolatile;
2999 return true;
3000 }
3001 case Intrinsic::x86_atomic_bts_rm:
3002 case Intrinsic::x86_atomic_btc_rm:
3003 case Intrinsic::x86_atomic_btr_rm: {
3004 Info.opc = ISD::INTRINSIC_W_CHAIN;
3005 Info.ptrVal = I.getArgOperand(0);
3006 unsigned Size = I.getArgOperand(1)->getType()->getScalarSizeInBits();
3007 Info.memVT = EVT::getIntegerVT(I.getType()->getContext(), Size);
3008 Info.align = Align(Size);
3009 Info.flags |= MachineMemOperand::MOLoad | MachineMemOperand::MOStore |
3010 MachineMemOperand::MOVolatile;
3011 return true;
3012 }
3013 case Intrinsic::x86_aadd32:
3014 case Intrinsic::x86_aadd64:
3015 case Intrinsic::x86_aand32:
3016 case Intrinsic::x86_aand64:
3017 case Intrinsic::x86_aor32:
3018 case Intrinsic::x86_aor64:
3019 case Intrinsic::x86_axor32:
3020 case Intrinsic::x86_axor64:
3021 case Intrinsic::x86_atomic_add_cc:
3022 case Intrinsic::x86_atomic_sub_cc:
3023 case Intrinsic::x86_atomic_or_cc:
3024 case Intrinsic::x86_atomic_and_cc:
3025 case Intrinsic::x86_atomic_xor_cc: {
3026 Info.opc = ISD::INTRINSIC_W_CHAIN;
3027 Info.ptrVal = I.getArgOperand(0);
3028 unsigned Size = I.getArgOperand(1)->getType()->getScalarSizeInBits();
3029 Info.memVT = EVT::getIntegerVT(I.getType()->getContext(), Size);
3030 Info.align = Align(Size);
3031 Info.flags |= MachineMemOperand::MOLoad | MachineMemOperand::MOStore |
3032 MachineMemOperand::MOVolatile;
3033 return true;
3034 }
3035 }
3036 return false;
3037 }
3038
3039 switch (IntrData->Type) {
3040 case TRUNCATE_TO_MEM_VI8:
3041 case TRUNCATE_TO_MEM_VI16:
3042 case TRUNCATE_TO_MEM_VI32: {
3043 Info.opc = ISD::INTRINSIC_VOID;
3044 Info.ptrVal = I.getArgOperand(0);
3045 MVT VT = MVT::getVT(I.getArgOperand(1)->getType());
3046 MVT ScalarVT = MVT::INVALID_SIMPLE_VALUE_TYPE;
3047 if (IntrData->Type == TRUNCATE_TO_MEM_VI8)
3048 ScalarVT = MVT::i8;
3049 else if (IntrData->Type == TRUNCATE_TO_MEM_VI16)
3050 ScalarVT = MVT::i16;
3051 else if (IntrData->Type == TRUNCATE_TO_MEM_VI32)
3052 ScalarVT = MVT::i32;
3053
3054 Info.memVT = MVT::getVectorVT(ScalarVT, VT.getVectorNumElements());
3055 Info.align = Align(1);
3056 Info.flags |= MachineMemOperand::MOStore;
3057 break;
3058 }
3059 case GATHER:
3060 case GATHER_AVX2: {
3061 Info.opc = ISD::INTRINSIC_W_CHAIN;
3062 Info.ptrVal = nullptr;
3063 MVT DataVT = MVT::getVT(I.getType());
3064 MVT IndexVT = MVT::getVT(I.getArgOperand(2)->getType());
3065 unsigned NumElts = std::min(DataVT.getVectorNumElements(),
3066 IndexVT.getVectorNumElements());
3067 Info.memVT = MVT::getVectorVT(DataVT.getVectorElementType(), NumElts);
3068 Info.align = Align(1);
3069 Info.flags |= MachineMemOperand::MOLoad;
3070 break;
3071 }
3072 case SCATTER: {
3073 Info.opc = ISD::INTRINSIC_VOID;
3074 Info.ptrVal = nullptr;
3075 MVT DataVT = MVT::getVT(I.getArgOperand(3)->getType());
3076 MVT IndexVT = MVT::getVT(I.getArgOperand(2)->getType());
3077 unsigned NumElts = std::min(DataVT.getVectorNumElements(),
3078 IndexVT.getVectorNumElements());
3079 Info.memVT = MVT::getVectorVT(DataVT.getVectorElementType(), NumElts);
3080 Info.align = Align(1);
3081 Info.flags |= MachineMemOperand::MOStore;
3082 break;
3083 }
3084 default:
3085 return false;
3086 }
3087
3088 return true;
3089}
3090
3091/// Returns true if the target can instruction select the
3092/// specified FP immediate natively. If false, the legalizer will
3093/// materialize the FP immediate as a load from a constant pool.
3094bool X86TargetLowering::isFPImmLegal(const APFloat &Imm, EVT VT,
3095 bool ForCodeSize) const {
3096 for (const APFloat &FPImm : LegalFPImmediates)
3097 if (Imm.bitwiseIsEqual(FPImm))
3098 return true;
3099 return false;
3100}
3101
3102bool X86TargetLowering::shouldReduceLoadWidth(SDNode *Load,
3103 ISD::LoadExtType ExtTy,
3104 EVT NewVT) const {
3105 assert(cast<LoadSDNode>(Load)->isSimple() && "illegal to narrow");
3106
3107 // "ELF Handling for Thread-Local Storage" specifies that R_X86_64_GOTTPOFF
3108 // relocation target a movq or addq instruction: don't let the load shrink.
3109 SDValue BasePtr = cast<LoadSDNode>(Load)->getBasePtr();
3110 if (BasePtr.getOpcode() == X86ISD::WrapperRIP)
3111 if (const auto *GA = dyn_cast<GlobalAddressSDNode>(BasePtr.getOperand(0)))
3112 return GA->getTargetFlags() != X86II::MO_GOTTPOFF;
3113
3114 // If this is an (1) AVX vector load with (2) multiple uses and (3) all of
3115 // those uses are extracted directly into a store, then the extract + store
3116 // can be store-folded. Therefore, it's probably not worth splitting the load.
3117 EVT VT = Load->getValueType(0);
3118 if ((VT.is256BitVector() || VT.is512BitVector()) && !Load->hasOneUse()) {
3119 for (auto UI = Load->use_begin(), UE = Load->use_end(); UI != UE; ++UI) {
3120 // Skip uses of the chain value. Result 0 of the node is the load value.
3121 if (UI.getUse().getResNo() != 0)
3122 continue;
3123
3124 // If this use is not an extract + store, it's probably worth splitting.
3125 if (UI->getOpcode() != ISD::EXTRACT_SUBVECTOR || !UI->hasOneUse() ||
3126 UI->use_begin()->getOpcode() != ISD::STORE)
3127 return true;
3128 }
3129 // All non-chain uses are extract + store.
3130 return false;
3131 }
3132
3133 return true;
3134}
3135
3136/// Returns true if it is beneficial to convert a load of a constant
3137/// to just the constant itself.
3138bool X86TargetLowering::shouldConvertConstantLoadToIntImm(const APInt &Imm,
3139 Type *Ty) const {
3140 assert(Ty->isIntegerTy());
3141
3142 unsigned BitSize = Ty->getPrimitiveSizeInBits();
3143 if (BitSize == 0 || BitSize > 64)
3144 return false;
3145 return true;
3146}
3147