LLVM 20.0.0git
AArch64ISelLowering.cpp
Go to the documentation of this file.
1//===-- AArch64ISelLowering.cpp - AArch64 DAG Lowering Implementation ----===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements the AArch64TargetLowering class.
10//
11//===----------------------------------------------------------------------===//
12
13#include "AArch64ISelLowering.h"
14#include "AArch64CallingConvention.h"
15#include "AArch64ExpandImm.h"
16#include "AArch64MachineFunctionInfo.h"
17#include "AArch64PerfectShuffle.h"
18#include "AArch64RegisterInfo.h"
19#include "AArch64Subtarget.h"
20#include "MCTargetDesc/AArch64AddressingModes.h"
21#include "Utils/AArch64BaseInfo.h"
22#include "llvm/ADT/APFloat.h"
23#include "llvm/ADT/APInt.h"
24#include "llvm/ADT/ArrayRef.h"
25#include "llvm/ADT/STLExtras.h"
26#include "llvm/ADT/SmallSet.h"
27#include "llvm/ADT/SmallVector.h"
28#include "llvm/ADT/Statistic.h"
29#include "llvm/ADT/StringRef.h"
30#include "llvm/ADT/Twine.h"
31#include "llvm/Analysis/LoopInfo.h"
32#include "llvm/Analysis/MemoryLocation.h"
33#include "llvm/Analysis/ObjCARCUtil.h"
34#include "llvm/Analysis/OptimizationRemarkEmitter.h"
35#include "llvm/Analysis/TargetTransformInfo.h"
36#include "llvm/Analysis/ValueTracking.h"
37#include "llvm/Analysis/VectorUtils.h"
38#include "llvm/CodeGen/Analysis.h"
39#include "llvm/CodeGen/CallingConvLower.h"
40#include "llvm/CodeGen/ComplexDeinterleavingPass.h"
41#include "llvm/CodeGen/GlobalISel/GenericMachineInstrs.h"
42#include "llvm/CodeGen/GlobalISel/Utils.h"
43#include "llvm/CodeGen/ISDOpcodes.h"
44#include "llvm/CodeGen/MachineBasicBlock.h"
45#include "llvm/CodeGen/MachineFrameInfo.h"
46#include "llvm/CodeGen/MachineFunction.h"
47#include "llvm/CodeGen/MachineInstr.h"
48#include "llvm/CodeGen/MachineInstrBuilder.h"
49#include "llvm/CodeGen/MachineMemOperand.h"
50#include "llvm/CodeGen/MachineRegisterInfo.h"
51#include "llvm/CodeGen/RuntimeLibcallUtil.h"
52#include "llvm/CodeGen/SelectionDAG.h"
53#include "llvm/CodeGen/SelectionDAGNodes.h"
54#include "llvm/CodeGen/TargetCallingConv.h"
55#include "llvm/CodeGen/TargetInstrInfo.h"
56#include "llvm/CodeGen/TargetOpcodes.h"
57#include "llvm/CodeGen/ValueTypes.h"
58#include "llvm/CodeGenTypes/MachineValueType.h"
59#include "llvm/IR/Attributes.h"
60#include "llvm/IR/Constants.h"
61#include "llvm/IR/DataLayout.h"
62#include "llvm/IR/DebugLoc.h"
63#include "llvm/IR/DerivedTypes.h"
64#include "llvm/IR/Function.h"
65#include "llvm/IR/GetElementPtrTypeIterator.h"
66#include "llvm/IR/GlobalValue.h"
67#include "llvm/IR/IRBuilder.h"
68#include "llvm/IR/Instruction.h"
69#include "llvm/IR/Instructions.h"
70#include "llvm/IR/IntrinsicInst.h"
71#include "llvm/IR/Intrinsics.h"
72#include "llvm/IR/IntrinsicsAArch64.h"
73#include "llvm/IR/Module.h"
74#include "llvm/IR/PatternMatch.h"
75#include "llvm/IR/Type.h"
76#include "llvm/IR/Use.h"
77#include "llvm/IR/Value.h"
78#include "llvm/MC/MCRegisterInfo.h"
79#include "llvm/Support/AtomicOrdering.h"
80#include "llvm/Support/Casting.h"
81#include "llvm/Support/CodeGen.h"
82#include "llvm/Support/CommandLine.h"
83#include "llvm/Support/Debug.h"
84#include "llvm/Support/ErrorHandling.h"
85#include "llvm/Support/InstructionCost.h"
86#include "llvm/Support/KnownBits.h"
87#include "llvm/Support/MathExtras.h"
88#include "llvm/Support/SipHash.h"
89#include "llvm/Support/raw_ostream.h"
90#include "llvm/Target/TargetMachine.h"
91#include "llvm/Target/TargetOptions.h"
92#include "llvm/TargetParser/Triple.h"
93#include <algorithm>
94#include <bitset>
95#include <cassert>
96#include <cctype>
97#include <cstdint>
98#include <cstdlib>
99#include <iterator>
100#include <limits>
101#include <optional>
102#include <tuple>
103#include <utility>
104#include <vector>
105
106using namespace llvm;
107using namespace llvm::PatternMatch;
108
109#define DEBUG_TYPE "aarch64-lower"
110
111STATISTIC(NumTailCalls, "Number of tail calls");
112STATISTIC(NumShiftInserts, "Number of vector shift inserts");
113STATISTIC(NumOptimizedImms, "Number of times immediates were optimized");
114
115// FIXME: The necessary dtprel relocations don't seem to be supported
116// well in the GNU bfd and gold linkers at the moment. Therefore, by
117// default, for now, fall back to GeneralDynamic code generation.
118cl::opt<bool> EnableAArch64ELFLocalDynamicTLSGeneration(
119 "aarch64-elf-ldtls-generation", cl::Hidden,
120 cl::desc("Allow AArch64 Local Dynamic TLS code generation"),
121 cl::init(false));
122
123static cl::opt<bool>
124EnableOptimizeLogicalImm("aarch64-enable-logical-imm", cl::Hidden,
125 cl::desc("Enable AArch64 logical imm instruction "
126 "optimization"),
127 cl::init(true));
128
129// Temporary option added for the purpose of testing functionality added
130// to DAGCombiner.cpp in D92230. It is expected that this can be removed
131// in future when both implementations will be based off MGATHER rather
132// than the GLD1 nodes added for the SVE gather load intrinsics.
133static cl::opt<bool>
134EnableCombineMGatherIntrinsics("aarch64-enable-mgather-combine", cl::Hidden,
135 cl::desc("Combine extends of AArch64 masked "
136 "gather intrinsics"),
137 cl::init(true));
138
139static cl::opt<bool> EnableExtToTBL("aarch64-enable-ext-to-tbl", cl::Hidden,
140 cl::desc("Combine ext and trunc to TBL"),
141 cl::init(true));
142
143// All of the XOR, OR and CMP use ALU ports, and data dependency will become the
144// bottleneck after this transform on high end CPU. So this max leaf node
145// limitation is guard cmp+ccmp will be profitable.
146static cl::opt<unsigned> MaxXors("aarch64-max-xors", cl::init(16), cl::Hidden,
147 cl::desc("Maximum of xors"));
148
149// By turning this on, we will not fallback to DAG ISel when encountering
150// scalable vector types for all instruction, even if SVE is not yet supported
151// with some instructions.
152// See [AArch64TargetLowering::fallbackToDAGISel] for implementation details.
153cl::opt<bool> EnableSVEGISel(
154 "aarch64-enable-gisel-sve", cl::Hidden,
155 cl::desc("Enable / disable SVE scalable vectors in Global ISel"),
156 cl::init(false));
157
158/// Value type used for condition codes.
159static const MVT MVT_CC = MVT::i32;
160
161static const MCPhysReg GPRArgRegs[] = {AArch64::X0, AArch64::X1, AArch64::X2,
162 AArch64::X3, AArch64::X4, AArch64::X5,
163 AArch64::X6, AArch64::X7};
164static const MCPhysReg FPRArgRegs[] = {AArch64::Q0, AArch64::Q1, AArch64::Q2,
165 AArch64::Q3, AArch64::Q4, AArch64::Q5,
166 AArch64::Q6, AArch64::Q7};
167
168ArrayRef<MCPhysReg> llvm::AArch64::getGPRArgRegs() { return GPRArgRegs; }
169
170ArrayRef<MCPhysReg> llvm::AArch64::getFPRArgRegs() { return FPRArgRegs; }
171
172static inline EVT getPackedSVEVectorVT(EVT VT) {
173 switch (VT.getSimpleVT().SimpleTy) {
174 default:
175 llvm_unreachable("unexpected element type for vector");
176 case MVT::i8:
177 return MVT::nxv16i8;
178 case MVT::i16:
179 return MVT::nxv8i16;
180 case MVT::i32:
181 return MVT::nxv4i32;
182 case MVT::i64:
183 return MVT::nxv2i64;
184 case MVT::f16:
185 return MVT::nxv8f16;
186 case MVT::f32:
187 return MVT::nxv4f32;
188 case MVT::f64:
189 return MVT::nxv2f64;
190 case MVT::bf16:
191 return MVT::nxv8bf16;
192 }
193}
194
195// NOTE: Currently there's only a need to return integer vector types. If this
196// changes then just add an extra "type" parameter.
197static inline EVT getPackedSVEVectorVT(ElementCount EC) {
198 switch (EC.getKnownMinValue()) {
199 default:
200 llvm_unreachable("unexpected element count for vector");
201 case 16:
202 return MVT::nxv16i8;
203 case 8:
204 return MVT::nxv8i16;
205 case 4:
206 return MVT::nxv4i32;
207 case 2:
208 return MVT::nxv2i64;
209 }
210}
211
212static inline EVT getPromotedVTForPredicate(EVT VT) {
213 assert(VT.isScalableVector() && (VT.getVectorElementType() == MVT::i1) &&
214 "Expected scalable predicate vector type!");
215 switch (VT.getVectorMinNumElements()) {
216 default:
217 llvm_unreachable("unexpected element count for vector");
218 case 2:
219 return MVT::nxv2i64;
220 case 4:
221 return MVT::nxv4i32;
222 case 8:
223 return MVT::nxv8i16;
224 case 16:
225 return MVT::nxv16i8;
226 }
227}
228
229/// Returns true if VT's elements occupy the lowest bit positions of its
230/// associated register class without any intervening space.
231///
232/// For example, nxv2f16, nxv4f16 and nxv8f16 are legal types that belong to the
233/// same register class, but only nxv8f16 can be treated as a packed vector.
234static inline bool isPackedVectorType(EVT VT, SelectionDAG &DAG) {
235 assert(VT.isVector() && DAG.getTargetLoweringInfo().isTypeLegal(VT) &&
236 "Expected legal vector type!");
237 return VT.isFixedLengthVector() ||
238 VT.getSizeInBits().getKnownMinValue() == AArch64::SVEBitsPerBlock;
239}
240
241// Returns true for ####_MERGE_PASSTHRU opcodes, whose operands have a leading
242// predicate and end with a passthru value matching the result type.
243static bool isMergePassthruOpcode(unsigned Opc) {
244 switch (Opc) {
245 default:
246 return false;
247 case AArch64ISD::BITREVERSE_MERGE_PASSTHRU:
248 case AArch64ISD::BSWAP_MERGE_PASSTHRU:
249 case AArch64ISD::REVH_MERGE_PASSTHRU:
250 case AArch64ISD::REVW_MERGE_PASSTHRU:
251 case AArch64ISD::REVD_MERGE_PASSTHRU:
252 case AArch64ISD::CTLZ_MERGE_PASSTHRU:
253 case AArch64ISD::CTPOP_MERGE_PASSTHRU:
254 case AArch64ISD::DUP_MERGE_PASSTHRU:
255 case AArch64ISD::ABS_MERGE_PASSTHRU:
256 case AArch64ISD::NEG_MERGE_PASSTHRU:
257 case AArch64ISD::FNEG_MERGE_PASSTHRU:
258 case AArch64ISD::SIGN_EXTEND_INREG_MERGE_PASSTHRU:
259 case AArch64ISD::ZERO_EXTEND_INREG_MERGE_PASSTHRU:
260 case AArch64ISD::FCEIL_MERGE_PASSTHRU:
261 case AArch64ISD::FFLOOR_MERGE_PASSTHRU:
262 case AArch64ISD::FNEARBYINT_MERGE_PASSTHRU:
263 case AArch64ISD::FRINT_MERGE_PASSTHRU:
264 case AArch64ISD::FROUND_MERGE_PASSTHRU:
265 case AArch64ISD::FROUNDEVEN_MERGE_PASSTHRU:
266 case AArch64ISD::FTRUNC_MERGE_PASSTHRU:
267 case AArch64ISD::FP_ROUND_MERGE_PASSTHRU:
268 case AArch64ISD::FP_EXTEND_MERGE_PASSTHRU:
269 case AArch64ISD::SINT_TO_FP_MERGE_PASSTHRU:
270 case AArch64ISD::UINT_TO_FP_MERGE_PASSTHRU:
271 case AArch64ISD::FCVTZU_MERGE_PASSTHRU:
272 case AArch64ISD::FCVTZS_MERGE_PASSTHRU:
273 case AArch64ISD::FSQRT_MERGE_PASSTHRU:
274 case AArch64ISD::FRECPX_MERGE_PASSTHRU:
275 case AArch64ISD::FABS_MERGE_PASSTHRU:
276 return true;
277 }
278}
279
280// Returns true if inactive lanes are known to be zeroed by construction.
281static bool isZeroingInactiveLanes(SDValue Op) {
282 switch (Op.getOpcode()) {
283 default:
284 return false;
285 // We guarantee i1 splat_vectors to zero the other lanes
286 case ISD::SPLAT_VECTOR:
287 case AArch64ISD::PTRUE:
288 case AArch64ISD::SETCC_MERGE_ZERO:
289 return true;
290 case ISD::INTRINSIC_WO_CHAIN:
291 switch (Op.getConstantOperandVal(0)) {
292 default:
293 return false;
294 case Intrinsic::aarch64_sve_ptrue:
295 case Intrinsic::aarch64_sve_pnext:
296 case Intrinsic::aarch64_sve_cmpeq:
297 case Intrinsic::aarch64_sve_cmpne:
298 case Intrinsic::aarch64_sve_cmpge:
299 case Intrinsic::aarch64_sve_cmpgt:
300 case Intrinsic::aarch64_sve_cmphs:
301 case Intrinsic::aarch64_sve_cmphi:
302 case Intrinsic::aarch64_sve_cmpeq_wide:
303 case Intrinsic::aarch64_sve_cmpne_wide:
304 case Intrinsic::aarch64_sve_cmpge_wide:
305 case Intrinsic::aarch64_sve_cmpgt_wide:
306 case Intrinsic::aarch64_sve_cmplt_wide:
307 case Intrinsic::aarch64_sve_cmple_wide:
308 case Intrinsic::aarch64_sve_cmphs_wide:
309 case Intrinsic::aarch64_sve_cmphi_wide:
310 case Intrinsic::aarch64_sve_cmplo_wide:
311 case Intrinsic::aarch64_sve_cmpls_wide:
312 case Intrinsic::aarch64_sve_fcmpeq:
313 case Intrinsic::aarch64_sve_fcmpne:
314 case Intrinsic::aarch64_sve_fcmpge:
315 case Intrinsic::aarch64_sve_fcmpgt:
316 case Intrinsic::aarch64_sve_fcmpuo:
317 case Intrinsic::aarch64_sve_facgt:
318 case Intrinsic::aarch64_sve_facge:
319 case Intrinsic::aarch64_sve_whilege:
320 case Intrinsic::aarch64_sve_whilegt:
321 case Intrinsic::aarch64_sve_whilehi:
322 case Intrinsic::aarch64_sve_whilehs:
323 case Intrinsic::aarch64_sve_whilele:
324 case Intrinsic::aarch64_sve_whilelo:
325 case Intrinsic::aarch64_sve_whilels:
326 case Intrinsic::aarch64_sve_whilelt:
327 case Intrinsic::aarch64_sve_match:
328 case Intrinsic::aarch64_sve_nmatch:
329 case Intrinsic::aarch64_sve_whilege_x2:
330 case Intrinsic::aarch64_sve_whilegt_x2:
331 case Intrinsic::aarch64_sve_whilehi_x2:
332 case Intrinsic::aarch64_sve_whilehs_x2:
333 case Intrinsic::aarch64_sve_whilele_x2:
334 case Intrinsic::aarch64_sve_whilelo_x2:
335 case Intrinsic::aarch64_sve_whilels_x2:
336 case Intrinsic::aarch64_sve_whilelt_x2:
337 return true;
338 }
339 }
340}
341
342static std::tuple<SDValue, SDValue>
343extractPtrauthBlendDiscriminators(SDValue Disc, SelectionDAG *DAG) {
344 SDLoc DL(Disc);
345 SDValue AddrDisc;
346 SDValue ConstDisc;
347
348 // If this is a blend, remember the constant and address discriminators.
349 // Otherwise, it's either a constant discriminator, or a non-blended
350 // address discriminator.
351 if (Disc->getOpcode() == ISD::INTRINSIC_WO_CHAIN &&
352 Disc->getConstantOperandVal(0) == Intrinsic::ptrauth_blend) {
353 AddrDisc = Disc->getOperand(1);
354 ConstDisc = Disc->getOperand(2);
355 } else {
356 ConstDisc = Disc;
357 }
358
359 // If the constant discriminator (either the blend RHS, or the entire
360 // discriminator value) isn't a 16-bit constant, bail out, and let the
361 // discriminator be computed separately.
362 const auto *ConstDiscN = dyn_cast<ConstantSDNode>(ConstDisc);
363 if (!ConstDiscN || !isUInt<16>(ConstDiscN->getZExtValue()))
364 return std::make_tuple(DAG->getTargetConstant(0, DL, MVT::i64), Disc);
365
366 // If there's no address discriminator, use NoRegister, which we'll later
367 // replace with XZR, or directly use a Z variant of the inst. when available.
368 if (!AddrDisc)
369 AddrDisc = DAG->getRegister(AArch64::NoRegister, MVT::i64);
370
371 return std::make_tuple(
372 DAG->getTargetConstant(ConstDiscN->getZExtValue(), DL, MVT::i64),
373 AddrDisc);
374}
375
376AArch64TargetLowering::AArch64TargetLowering(const TargetMachine &TM,
377 const AArch64Subtarget &STI)
378 : TargetLowering(TM), Subtarget(&STI) {
379 // AArch64 doesn't have comparisons which set GPRs or setcc instructions, so
380 // we have to make something up. Arbitrarily, choose ZeroOrOne.
381 setBooleanContents(ZeroOrOneBooleanContent);
382 // When comparing vectors the result sets the different elements in the
383 // vector to all-one or all-zero.
384 setBooleanVectorContents(ZeroOrNegativeOneBooleanContent);
385
386 // Set up the register classes.
387 addRegisterClass(MVT::i32, &AArch64::GPR32allRegClass);
388 addRegisterClass(MVT::i64, &AArch64::GPR64allRegClass);
389
390 if (Subtarget->hasLS64()) {
391 addRegisterClass(MVT::i64x8, &AArch64::GPR64x8ClassRegClass);
392 setOperationAction(ISD::LOAD, MVT::i64x8, Custom);
393 setOperationAction(ISD::STORE, MVT::i64x8, Custom);
394 }
395
396 if (Subtarget->hasFPARMv8()) {
397 addRegisterClass(MVT::f16, &AArch64::FPR16RegClass);
398 addRegisterClass(MVT::bf16, &AArch64::FPR16RegClass);
399 addRegisterClass(MVT::f32, &AArch64::FPR32RegClass);
400 addRegisterClass(MVT::f64, &AArch64::FPR64RegClass);
401 addRegisterClass(MVT::f128, &AArch64::FPR128RegClass);
402 }
403
404 if (Subtarget->hasNEON()) {
405 addRegisterClass(MVT::v16i8, &AArch64::FPR8RegClass);
406 addRegisterClass(MVT::v8i16, &AArch64::FPR16RegClass);
407
408 addDRType(MVT::v2f32);
409 addDRType(MVT::v8i8);
410 addDRType(MVT::v4i16);
411 addDRType(MVT::v2i32);
412 addDRType(MVT::v1i64);
413 addDRType(MVT::v1f64);
414 addDRType(MVT::v4f16);
415 addDRType(MVT::v4bf16);
416
417 addQRType(MVT::v4f32);
418 addQRType(MVT::v2f64);
419 addQRType(MVT::v16i8);
420 addQRType(MVT::v8i16);
421 addQRType(MVT::v4i32);
422 addQRType(MVT::v2i64);
423 addQRType(MVT::v8f16);
424 addQRType(MVT::v8bf16);
425 }
426
427 if (Subtarget->isSVEorStreamingSVEAvailable()) {
428 // Add legal sve predicate types
429 addRegisterClass(MVT::nxv1i1, &AArch64::PPRRegClass);
430 addRegisterClass(MVT::nxv2i1, &AArch64::PPRRegClass);
431 addRegisterClass(MVT::nxv4i1, &AArch64::PPRRegClass);
432 addRegisterClass(MVT::nxv8i1, &AArch64::PPRRegClass);
433 addRegisterClass(MVT::nxv16i1, &AArch64::PPRRegClass);
434
435 // Add legal sve data types
436 addRegisterClass(MVT::nxv16i8, &AArch64::ZPRRegClass);
437 addRegisterClass(MVT::nxv8i16, &AArch64::ZPRRegClass);
438 addRegisterClass(MVT::nxv4i32, &AArch64::ZPRRegClass);
439 addRegisterClass(MVT::nxv2i64, &AArch64::ZPRRegClass);
440
441 addRegisterClass(MVT::nxv2f16, &AArch64::ZPRRegClass);
442 addRegisterClass(MVT::nxv4f16, &AArch64::ZPRRegClass);
443 addRegisterClass(MVT::nxv8f16, &AArch64::ZPRRegClass);
444 addRegisterClass(MVT::nxv2f32, &AArch64::ZPRRegClass);
445 addRegisterClass(MVT::nxv4f32, &AArch64::ZPRRegClass);
446 addRegisterClass(MVT::nxv2f64, &AArch64::ZPRRegClass);
447
448 addRegisterClass(MVT::nxv2bf16, &AArch64::ZPRRegClass);
449 addRegisterClass(MVT::nxv4bf16, &AArch64::ZPRRegClass);
450 addRegisterClass(MVT::nxv8bf16, &AArch64::ZPRRegClass);
451
452 if (Subtarget->useSVEForFixedLengthVectors()) {
453 for (MVT VT : MVT::integer_fixedlen_vector_valuetypes())
454 if (useSVEForFixedLengthVectorVT(VT))
455 addRegisterClass(VT, &AArch64::ZPRRegClass);
456
457 for (MVT VT : MVT::fp_fixedlen_vector_valuetypes())
458 if (useSVEForFixedLengthVectorVT(VT))
459 addRegisterClass(VT, &AArch64::ZPRRegClass);
460 }
461 }
462
463 if (Subtarget->hasSVE2p1() || Subtarget->hasSME2()) {
464 addRegisterClass(MVT::aarch64svcount, &AArch64::PPRRegClass);
465 setOperationPromotedToType(ISD::LOAD, MVT::aarch64svcount, MVT::nxv16i1);
466 setOperationPromotedToType(ISD::STORE, MVT::aarch64svcount, MVT::nxv16i1);
467
468 setOperationAction(ISD::SELECT, MVT::aarch64svcount, Custom);
469 setOperationAction(ISD::SELECT_CC, MVT::aarch64svcount, Expand);
470 }
471
472 // Compute derived properties from the register classes
473 computeRegisterProperties(Subtarget->getRegisterInfo());
474
475 // Provide all sorts of operation actions
476 setOperationAction(ISD::GlobalAddress, MVT::i64, Custom);
477 setOperationAction(ISD::GlobalTLSAddress, MVT::i64, Custom);
478 setOperationAction(ISD::SETCC, MVT::i32, Custom);
479 setOperationAction(ISD::SETCC, MVT::i64, Custom);
480 setOperationAction(ISD::SETCC, MVT::bf16, Custom);
481 setOperationAction(ISD::SETCC, MVT::f16, Custom);
482 setOperationAction(ISD::SETCC, MVT::f32, Custom);
483 setOperationAction(ISD::SETCC, MVT::f64, Custom);
484 setOperationAction(ISD::STRICT_FSETCC, MVT::bf16, Custom);
485 setOperationAction(ISD::STRICT_FSETCC, MVT::f16, Custom);
486 setOperationAction(ISD::STRICT_FSETCC, MVT::f32, Custom);
487 setOperationAction(ISD::STRICT_FSETCC, MVT::f64, Custom);
488 setOperationAction(ISD::STRICT_FSETCCS, MVT::f16, Custom);
489 setOperationAction(ISD::STRICT_FSETCCS, MVT::f32, Custom);
490 setOperationAction(ISD::STRICT_FSETCCS, MVT::f64, Custom);
491 setOperationAction(ISD::BITREVERSE, MVT::i32, Legal);
492 setOperationAction(ISD::BITREVERSE, MVT::i64, Legal);
493 setOperationAction(ISD::BRCOND, MVT::Other, Custom);
494 setOperationAction(ISD::BR_CC, MVT::i32, Custom);
495 setOperationAction(ISD::BR_CC, MVT::i64, Custom);
496 setOperationAction(ISD::BR_CC, MVT::f16, Custom);
497 setOperationAction(ISD::BR_CC, MVT::f32, Custom);
498 setOperationAction(ISD::BR_CC, MVT::f64, Custom);
499 setOperationAction(ISD::SELECT, MVT::i32, Custom);
500 setOperationAction(ISD::SELECT, MVT::i64, Custom);
501 setOperationAction(ISD::SELECT, MVT::f16, Custom);
502 setOperationAction(ISD::SELECT, MVT::bf16, Custom);
503 setOperationAction(ISD::SELECT, MVT::f32, Custom);
504 setOperationAction(ISD::SELECT, MVT::f64, Custom);
505 setOperationAction(ISD::SELECT_CC, MVT::i32, Custom);
506 setOperationAction(ISD::SELECT_CC, MVT::i64, Custom);
507 setOperationAction(ISD::SELECT_CC, MVT::f16, Custom);
508 setOperationAction(ISD::SELECT_CC, MVT::bf16, Custom);
509 setOperationAction(ISD::SELECT_CC, MVT::f32, Custom);
510 setOperationAction(ISD::SELECT_CC, MVT::f64, Custom);
511 setOperationAction(ISD::BR_JT, MVT::Other, Custom);
512 setOperationAction(ISD::JumpTable, MVT::i64, Custom);
513 setOperationAction(ISD::BRIND, MVT::Other, Custom);
514 setOperationAction(ISD::SETCCCARRY, MVT::i64, Custom);
515
516 setOperationAction(ISD::PtrAuthGlobalAddress, MVT::i64, Custom);
517
518 setOperationAction(ISD::SHL_PARTS, MVT::i64, Custom);
519 setOperationAction(ISD::SRA_PARTS, MVT::i64, Custom);
520 setOperationAction(ISD::SRL_PARTS, MVT::i64, Custom);
521
522 setOperationAction(ISD::FREM, MVT::f32, Expand);
523 setOperationAction(ISD::FREM, MVT::f64, Expand);
524 setOperationAction(ISD::FREM, MVT::f80, Expand);
525
526 setOperationAction(ISD::BUILD_PAIR, MVT::i64, Expand);
527
528 // Custom lowering hooks are needed for XOR
529 // to fold it into CSINC/CSINV.
530 setOperationAction(ISD::XOR, MVT::i32, Custom);
531 setOperationAction(ISD::XOR, MVT::i64, Custom);
532
533 // Virtually no operation on f128 is legal, but LLVM can't expand them when
534 // there's a valid register class, so we need custom operations in most cases.
535 setOperationAction(ISD::FABS, MVT::f128, Expand);
536 setOperationAction(ISD::FADD, MVT::f128, LibCall);
537 setOperationAction(ISD::FCOPYSIGN, MVT::f128, Expand);
538 setOperationAction(ISD::FCOS, MVT::f128, Expand);
539 setOperationAction(ISD::FDIV, MVT::f128, LibCall);
540 setOperationAction(ISD::FMA, MVT::f128, Expand);
541 setOperationAction(ISD::FMUL, MVT::f128, LibCall);
542 setOperationAction(ISD::FNEG, MVT::f128, Expand);
543 setOperationAction(ISD::FPOW, MVT::f128, Expand);
544 setOperationAction(ISD::FREM, MVT::f128, Expand);
545 setOperationAction(ISD::FRINT, MVT::f128, Expand);
546 setOperationAction(ISD::FSIN, MVT::f128, Expand);
547 setOperationAction(ISD::FSINCOS, MVT::f128, Expand);
548 setOperationAction(ISD::FSQRT, MVT::f128, Expand);
549 setOperationAction(ISD::FSUB, MVT::f128, LibCall);
550 setOperationAction(ISD::FTAN, MVT::f128, Expand);
551 setOperationAction(ISD::FTRUNC, MVT::f128, Expand);
552 setOperationAction(ISD::SETCC, MVT::f128, Custom);
553 setOperationAction(ISD::STRICT_FSETCC, MVT::f128, Custom);
554 setOperationAction(ISD::STRICT_FSETCCS, MVT::f128, Custom);
555 setOperationAction(ISD::BR_CC, MVT::f128, Custom);
556 setOperationAction(ISD::SELECT, MVT::f128, Custom);
557 setOperationAction(ISD::SELECT_CC, MVT::f128, Custom);
558 setOperationAction(ISD::FP_EXTEND, MVT::f128, Custom);
559 // FIXME: f128 FMINIMUM and FMAXIMUM (including STRICT versions) currently
560 // aren't handled.
561
562 // Lowering for many of the conversions is actually specified by the non-f128
563 // type. The LowerXXX function will be trivial when f128 isn't involved.
564 setOperationAction(ISD::FP_TO_SINT, MVT::i32, Custom);
565 setOperationAction(ISD::FP_TO_SINT, MVT::i64, Custom);
566 setOperationAction(ISD::FP_TO_SINT, MVT::i128, Custom);
567 setOperationAction(ISD::STRICT_FP_TO_SINT, MVT::i32, Custom);
568 setOperationAction(ISD::STRICT_FP_TO_SINT, MVT::i64, Custom);
569 setOperationAction(ISD::STRICT_FP_TO_SINT, MVT::i128, Custom);
570 setOperationAction(ISD::FP_TO_UINT, MVT::i32, Custom);
571 setOperationAction(ISD::FP_TO_UINT, MVT::i64, Custom);
572 setOperationAction(ISD::FP_TO_UINT, MVT::i128, Custom);
573 setOperationAction(ISD::STRICT_FP_TO_UINT, MVT::i32, Custom);
574 setOperationAction(ISD::STRICT_FP_TO_UINT, MVT::i64, Custom);
575 setOperationAction(ISD::STRICT_FP_TO_UINT, MVT::i128, Custom);
576 setOperationAction(ISD::SINT_TO_FP, MVT::i32, Custom);
577 setOperationAction(ISD::SINT_TO_FP, MVT::i64, Custom);
578 setOperationAction(ISD::SINT_TO_FP, MVT::i128, Custom);
579 setOperationAction(ISD::STRICT_SINT_TO_FP, MVT::i32, Custom);
580 setOperationAction(ISD::STRICT_SINT_TO_FP, MVT::i64, Custom);
581 setOperationAction(ISD::STRICT_SINT_TO_FP, MVT::i128, Custom);
582 setOperationAction(ISD::UINT_TO_FP, MVT::i32, Custom);
583 setOperationAction(ISD::UINT_TO_FP, MVT::i64, Custom);
584 setOperationAction(ISD::UINT_TO_FP, MVT::i128, Custom);
585 setOperationAction(ISD::STRICT_UINT_TO_FP, MVT::i32, Custom);
586 setOperationAction(ISD::STRICT_UINT_TO_FP, MVT::i64, Custom);
587 setOperationAction(ISD::STRICT_UINT_TO_FP, MVT::i128, Custom);
588 if (Subtarget->hasFPARMv8()) {
589 setOperationAction(ISD::FP_ROUND, MVT::f16, Custom);
590 setOperationAction(ISD::FP_ROUND, MVT::bf16, Custom);
591 }
592 setOperationAction(ISD::FP_ROUND, MVT::f32, Custom);
593 setOperationAction(ISD::FP_ROUND, MVT::f64, Custom);
594 if (Subtarget->hasFPARMv8()) {
595 setOperationAction(ISD::STRICT_FP_ROUND, MVT::f16, Custom);
596 setOperationAction(ISD::STRICT_FP_ROUND, MVT::bf16, Custom);
597 }
598 setOperationAction(ISD::STRICT_FP_ROUND, MVT::f32, Custom);
599 setOperationAction(ISD::STRICT_FP_ROUND, MVT::f64, Custom);
600
601 setOperationAction(ISD::FP_TO_UINT_SAT, MVT::i32, Custom);
602 setOperationAction(ISD::FP_TO_UINT_SAT, MVT::i64, Custom);
603 setOperationAction(ISD::FP_TO_SINT_SAT, MVT::i32, Custom);
604 setOperationAction(ISD::FP_TO_SINT_SAT, MVT::i64, Custom);
605
606 // Variable arguments.
607 setOperationAction(ISD::VASTART, MVT::Other, Custom);
608 setOperationAction(ISD::VAARG, MVT::Other, Custom);
609 setOperationAction(ISD::VACOPY, MVT::Other, Custom);
610 setOperationAction(ISD::VAEND, MVT::Other, Expand);
611
612 // Variable-sized objects.
613 setOperationAction(ISD::STACKSAVE, MVT::Other, Expand);
614 setOperationAction(ISD::STACKRESTORE, MVT::Other, Expand);
615
616 // Lowering Funnel Shifts to EXTR
617 setOperationAction(ISD::FSHR, MVT::i32, Custom);
618 setOperationAction(ISD::FSHR, MVT::i64, Custom);
619 setOperationAction(ISD::FSHL, MVT::i32, Custom);
620 setOperationAction(ISD::FSHL, MVT::i64, Custom);
621
622 setOperationAction(ISD::DYNAMIC_STACKALLOC, MVT::i64, Custom);
623
624 // Constant pool entries
625 setOperationAction(ISD::ConstantPool, MVT::i64, Custom);
626
627 // BlockAddress
628 setOperationAction(ISD::BlockAddress, MVT::i64, Custom);
629
630 // AArch64 lacks both left-rotate and popcount instructions.
631 setOperationAction(ISD::ROTL, MVT::i32, Expand);
632 setOperationAction(ISD::ROTL, MVT::i64, Expand);
633 for (MVT VT : MVT::fixedlen_vector_valuetypes()) {
634 setOperationAction(ISD::ROTL, VT, Expand);
635 setOperationAction(ISD::ROTR, VT, Expand);
636 }
637
638 // AArch64 doesn't have i32 MULH{S|U}.
639 setOperationAction(ISD::MULHU, MVT::i32, Expand);
640 setOperationAction(ISD::MULHS, MVT::i32, Expand);
641
642 // AArch64 doesn't have {U|S}MUL_LOHI.
643 setOperationAction(ISD::UMUL_LOHI, MVT::i32, Expand);
644 setOperationAction(ISD::SMUL_LOHI, MVT::i32, Expand);
645 setOperationAction(ISD::UMUL_LOHI, MVT::i64, Expand);
646 setOperationAction(ISD::SMUL_LOHI, MVT::i64, Expand);
647
648 if (Subtarget->hasCSSC()) {
649 setOperationAction(ISD::CTPOP, MVT::i32, Legal);
650 setOperationAction(ISD::CTPOP, MVT::i64, Legal);
651 setOperationAction(ISD::CTPOP, MVT::i128, Expand);
652
653 setOperationAction(ISD::PARITY, MVT::i128, Expand);
654
655 setOperationAction(ISD::CTTZ, MVT::i32, Legal);
656 setOperationAction(ISD::CTTZ, MVT::i64, Legal);
657 setOperationAction(ISD::CTTZ, MVT::i128, Expand);
658
659 setOperationAction(ISD::ABS, MVT::i32, Legal);
660 setOperationAction(ISD::ABS, MVT::i64, Legal);
661
662 setOperationAction(ISD::SMAX, MVT::i32, Legal);
663 setOperationAction(ISD::SMAX, MVT::i64, Legal);
664 setOperationAction(ISD::UMAX, MVT::i32, Legal);
665 setOperationAction(ISD::UMAX, MVT::i64, Legal);
666
667 setOperationAction(ISD::SMIN, MVT::i32, Legal);
668 setOperationAction(ISD::SMIN, MVT::i64, Legal);
669 setOperationAction(ISD::UMIN, MVT::i32, Legal);
670 setOperationAction(ISD::UMIN, MVT::i64, Legal);
671 } else {
672 setOperationAction(ISD::CTPOP, MVT::i32, Custom);
673 setOperationAction(ISD::CTPOP, MVT::i64, Custom);
674 setOperationAction(ISD::CTPOP, MVT::i128, Custom);
675
676 setOperationAction(ISD::PARITY, MVT::i64, Custom);
677 setOperationAction(ISD::PARITY, MVT::i128, Custom);
678
679 setOperationAction(ISD::ABS, MVT::i32, Custom);
680 setOperationAction(ISD::ABS, MVT::i64, Custom);
681 }
682
683 setOperationAction(ISD::SDIVREM, MVT::i32, Expand);
684 setOperationAction(ISD::SDIVREM, MVT::i64, Expand);
685 for (MVT VT : MVT::fixedlen_vector_valuetypes()) {
686 setOperationAction(ISD::SDIVREM, VT, Expand);
687 setOperationAction(ISD::UDIVREM, VT, Expand);
688 }
689 setOperationAction(ISD::SREM, MVT::i32, Expand);
690 setOperationAction(ISD::SREM, MVT::i64, Expand);
691 setOperationAction(ISD::UDIVREM, MVT::i32, Expand);
692 setOperationAction(ISD::UDIVREM, MVT::i64, Expand);
693 setOperationAction(ISD::UREM, MVT::i32, Expand);
694 setOperationAction(ISD::UREM, MVT::i64, Expand);
695
696 // Custom lower Add/Sub/Mul with overflow.
697 setOperationAction(ISD::SADDO, MVT::i32, Custom);
698 setOperationAction(ISD::SADDO, MVT::i64, Custom);
699 setOperationAction(ISD::UADDO, MVT::i32, Custom);
700 setOperationAction(ISD::UADDO, MVT::i64, Custom);
701 setOperationAction(ISD::SSUBO, MVT::i32, Custom);
702 setOperationAction(ISD::SSUBO, MVT::i64, Custom);
703 setOperationAction(ISD::USUBO, MVT::i32, Custom);
704 setOperationAction(ISD::USUBO, MVT::i64, Custom);
705 setOperationAction(ISD::SMULO, MVT::i32, Custom);
706 setOperationAction(ISD::SMULO, MVT::i64, Custom);
707 setOperationAction(ISD::UMULO, MVT::i32, Custom);
708 setOperationAction(ISD::UMULO, MVT::i64, Custom);
709
710 setOperationAction(ISD::UADDO_CARRY, MVT::i32, Custom);
711 setOperationAction(ISD::UADDO_CARRY, MVT::i64, Custom);
712 setOperationAction(ISD::USUBO_CARRY, MVT::i32, Custom);
713 setOperationAction(ISD::USUBO_CARRY, MVT::i64, Custom);
714 setOperationAction(ISD::SADDO_CARRY, MVT::i32, Custom);
715 setOperationAction(ISD::SADDO_CARRY, MVT::i64, Custom);
716 setOperationAction(ISD::SSUBO_CARRY, MVT::i32, Custom);
717 setOperationAction(ISD::SSUBO_CARRY, MVT::i64, Custom);
718
719 setOperationAction(ISD::FSIN, MVT::f32, Expand);
720 setOperationAction(ISD::FSIN, MVT::f64, Expand);
721 setOperationAction(ISD::FCOS, MVT::f32, Expand);
722 setOperationAction(ISD::FCOS, MVT::f64, Expand);
723 setOperationAction(ISD::FPOW, MVT::f32, Expand);
724 setOperationAction(ISD::FPOW, MVT::f64, Expand);
725 setOperationAction(ISD::FCOPYSIGN, MVT::f64, Custom);
726 setOperationAction(ISD::FCOPYSIGN, MVT::f32, Custom);
727 if (Subtarget->hasFullFP16()) {
728 setOperationAction(ISD::FCOPYSIGN, MVT::f16, Custom);
729 setOperationAction(ISD::FCOPYSIGN, MVT::bf16, Custom);
730 } else {
731 setOperationAction(ISD::FCOPYSIGN, MVT::f16, Promote);
732 setOperationAction(ISD::FCOPYSIGN, MVT::bf16, Promote);
733 }
734
735 for (auto Op : {ISD::FREM, ISD::FPOW, ISD::FPOWI,
736 ISD::FCOS, ISD::FSIN, ISD::FSINCOS,
737 ISD::FACOS, ISD::FASIN, ISD::FATAN,
738 ISD::FCOSH, ISD::FSINH, ISD::FTANH,
739 ISD::FTAN, ISD::FEXP, ISD::FEXP2,
740 ISD::FEXP10, ISD::FLOG, ISD::FLOG2,
741 ISD::FLOG10, ISD::STRICT_FREM, ISD::STRICT_FPOW,
742 ISD::STRICT_FPOWI, ISD::STRICT_FCOS, ISD::STRICT_FSIN,
743 ISD::STRICT_FACOS, ISD::STRICT_FASIN, ISD::STRICT_FATAN,
744 ISD::STRICT_FCOSH, ISD::STRICT_FSINH, ISD::STRICT_FTANH,
745 ISD::STRICT_FEXP, ISD::STRICT_FEXP2, ISD::STRICT_FLOG,
746 ISD::STRICT_FLOG2, ISD::STRICT_FLOG10, ISD::STRICT_FTAN}) {
747 setOperationAction(Op, MVT::f16, Promote);
748 setOperationAction(Op, MVT::v4f16, Expand);
749 setOperationAction(Op, MVT::v8f16, Expand);
750 setOperationAction(Op, MVT::bf16, Promote);
751 setOperationAction(Op, MVT::v4bf16, Expand);
752 setOperationAction(Op, MVT::v8bf16, Expand);
753 }
754
755 auto LegalizeNarrowFP = [this](MVT ScalarVT) {
756 for (auto Op : {
757 ISD::SETCC,
758 ISD::SELECT_CC,
759 ISD::BR_CC,
760 ISD::FADD,
761 ISD::FSUB,
762 ISD::FMUL,
763 ISD::FDIV,
764 ISD::FMA,
765 ISD::FCEIL,
766 ISD::FSQRT,
767 ISD::FFLOOR,
768 ISD::FNEARBYINT,
769 ISD::FRINT,
770 ISD::FROUND,
771 ISD::FROUNDEVEN,
772 ISD::FTRUNC,
773 ISD::FMINNUM,
774 ISD::FMAXNUM,
775 ISD::FMINIMUM,
776 ISD::FMAXIMUM,
777 ISD::STRICT_FADD,
778 ISD::STRICT_FSUB,
779 ISD::STRICT_FMUL,
780 ISD::STRICT_FDIV,
781 ISD::STRICT_FMA,
782 ISD::STRICT_FCEIL,
783 ISD::STRICT_FFLOOR,
784 ISD::STRICT_FSQRT,
785 ISD::STRICT_FRINT,
786 ISD::STRICT_FNEARBYINT,
787 ISD::STRICT_FROUND,
788 ISD::STRICT_FTRUNC,
789 ISD::STRICT_FROUNDEVEN,
790 ISD::STRICT_FMINNUM,
791 ISD::STRICT_FMAXNUM,
792 ISD::STRICT_FMINIMUM,
793 ISD::STRICT_FMAXIMUM,
794 })
795 setOperationAction(Op, ScalarVT, Promote);
796
797 for (auto Op : {ISD::FNEG, ISD::FABS})
798 setOperationAction(Op, ScalarVT, Legal);
799
800 // Round-to-integer need custom lowering for fp16, as Promote doesn't work
801 // because the result type is integer.
802 for (auto Op : {ISD::LROUND, ISD::LLROUND, ISD::LRINT, ISD::LLRINT,
803 ISD::STRICT_LROUND, ISD::STRICT_LLROUND, ISD::STRICT_LRINT,
804 ISD::STRICT_LLRINT})
805 setOperationAction(Op, ScalarVT, Custom);
806
807 // promote v4f16 to v4f32 when that is known to be safe.
808 auto V4Narrow = MVT::getVectorVT(ScalarVT, 4);
809 setOperationPromotedToType(ISD::FADD, V4Narrow, MVT::v4f32);
810 setOperationPromotedToType(ISD::FSUB, V4Narrow, MVT::v4f32);
811 setOperationPromotedToType(ISD::FMUL, V4Narrow, MVT::v4f32);
812 setOperationPromotedToType(ISD::FDIV, V4Narrow, MVT::v4f32);
813 setOperationPromotedToType(ISD::FCEIL, V4Narrow, MVT::v4f32);
814 setOperationPromotedToType(ISD::FFLOOR, V4Narrow, MVT::v4f32);
815 setOperationPromotedToType(ISD::FROUND, V4Narrow, MVT::v4f32);
816 setOperationPromotedToType(ISD::FTRUNC, V4Narrow, MVT::v4f32);
817 setOperationPromotedToType(ISD::FROUNDEVEN, V4Narrow, MVT::v4f32);
818 setOperationPromotedToType(ISD::FRINT, V4Narrow, MVT::v4f32);
819 setOperationPromotedToType(ISD::FNEARBYINT, V4Narrow, MVT::v4f32);
820
821 setOperationAction(ISD::FABS, V4Narrow, Legal);
822 setOperationAction(ISD::FNEG, V4Narrow, Legal);
823 setOperationAction(ISD::FMA, V4Narrow, Expand);
824 setOperationAction(ISD::SETCC, V4Narrow, Custom);
825 setOperationAction(ISD::BR_CC, V4Narrow, Expand);
826 setOperationAction(ISD::SELECT, V4Narrow, Expand);
827 setOperationAction(ISD::SELECT_CC, V4Narrow, Expand);
828 setOperationAction(ISD::FCOPYSIGN, V4Narrow, Custom);
829 setOperationAction(ISD::FSQRT, V4Narrow, Expand);
830
831 auto V8Narrow = MVT::getVectorVT(ScalarVT, 8);
832 setOperationAction(ISD::FABS, V8Narrow, Legal);
833 setOperationAction(ISD::FADD, V8Narrow, Legal);
834 setOperationAction(ISD::FCEIL, V8Narrow, Legal);
835 setOperationAction(ISD::FCOPYSIGN, V8Narrow, Custom);
836 setOperationAction(ISD::FDIV, V8Narrow, Legal);
837 setOperationAction(ISD::FFLOOR, V8Narrow, Legal);
838 setOperationAction(ISD::FMA, V8Narrow, Expand);
839 setOperationAction(ISD::FMUL, V8Narrow, Legal);
840 setOperationAction(ISD::FNEARBYINT, V8Narrow, Legal);
841 setOperationAction(ISD::FNEG, V8Narrow, Legal);
842 setOperationAction(ISD::FROUND, V8Narrow, Legal);
843 setOperationAction(ISD::FROUNDEVEN, V8Narrow, Legal);
844 setOperationAction(ISD::FRINT, V8Narrow, Legal);
845 setOperationAction(ISD::FSQRT, V8Narrow, Expand);
846 setOperationAction(ISD::FSUB, V8Narrow, Legal);
847 setOperationAction(ISD::FTRUNC, V8Narrow, Legal);
848 setOperationAction(ISD::SETCC, V8Narrow, Expand);
849 setOperationAction(ISD::BR_CC, V8Narrow, Expand);
850 setOperationAction(ISD::SELECT, V8Narrow, Expand);
851 setOperationAction(ISD::SELECT_CC, V8Narrow, Expand);
852 setOperationAction(ISD::FP_EXTEND, V8Narrow, Expand);
853 };
854
855 if (!Subtarget->hasFullFP16()) {
856 LegalizeNarrowFP(MVT::f16);
857 }
858 LegalizeNarrowFP(MVT::bf16);
859 setOperationAction(ISD::FP_ROUND, MVT::v4f32, Custom);
860 setOperationAction(ISD::FP_ROUND, MVT::v4bf16, Custom);
861
862 // AArch64 has implementations of a lot of rounding-like FP operations.
863 for (auto Op :
864 {ISD::FFLOOR, ISD::FNEARBYINT, ISD::FCEIL,
865 ISD::FRINT, ISD::FTRUNC, ISD::FROUND,
866 ISD::FROUNDEVEN, ISD::FMINNUM, ISD::FMAXNUM,
867 ISD::FMINIMUM, ISD::FMAXIMUM, ISD::LROUND,
868 ISD::LLROUND, ISD::LRINT, ISD::LLRINT,
869 ISD::STRICT_FFLOOR, ISD::STRICT_FCEIL, ISD::STRICT_FNEARBYINT,
870 ISD::STRICT_FRINT, ISD::STRICT_FTRUNC, ISD::STRICT_FROUNDEVEN,
871 ISD::STRICT_FROUND, ISD::STRICT_FMINNUM, ISD::STRICT_FMAXNUM,
872 ISD::STRICT_FMINIMUM, ISD::STRICT_FMAXIMUM, ISD::STRICT_LROUND,
873 ISD::STRICT_LLROUND, ISD::STRICT_LRINT, ISD::STRICT_LLRINT}) {
874 for (MVT Ty : {MVT::f32, MVT::f64})
875 setOperationAction(Op, Ty, Legal);
876 if (Subtarget->hasFullFP16())
877 setOperationAction(Op, MVT::f16, Legal);
878 }
879
880 // Basic strict FP operations are legal
881 for (auto Op : {ISD::STRICT_FADD, ISD::STRICT_FSUB, ISD::STRICT_FMUL,
882 ISD::STRICT_FDIV, ISD::STRICT_FMA, ISD::STRICT_FSQRT}) {
883 for (MVT Ty : {MVT::f32, MVT::f64})
884 setOperationAction(Op, Ty, Legal);
885 if (Subtarget->hasFullFP16())
886 setOperationAction(Op, MVT::f16, Legal);
887 }
888
889 // Strict conversion to a larger type is legal
890 for (auto VT : {MVT::f32, MVT::f64})
891 setOperationAction(ISD::STRICT_FP_EXTEND, VT, Legal);
892
893 setOperationAction(ISD::PREFETCH, MVT::Other, Custom);
894
895 setOperationAction(ISD::GET_ROUNDING, MVT::i32, Custom);
896 setOperationAction(ISD::SET_ROUNDING, MVT::Other, Custom);
897 setOperationAction(ISD::GET_FPMODE, MVT::i32, Custom);
898 setOperationAction(ISD::SET_FPMODE, MVT::i32, Custom);
899 setOperationAction(ISD::RESET_FPMODE, MVT::Other, Custom);
900
901 setOperationAction(ISD::ATOMIC_CMP_SWAP, MVT::i128, Custom);
902 if (!Subtarget->hasLSE() && !Subtarget->outlineAtomics()) {
903 setOperationAction(ISD::ATOMIC_LOAD_SUB, MVT::i32, LibCall);
904 setOperationAction(ISD::ATOMIC_LOAD_SUB, MVT::i64, LibCall);
905 } else {
906 setOperationAction(ISD::ATOMIC_LOAD_SUB, MVT::i32, Expand);
907 setOperationAction(ISD::ATOMIC_LOAD_SUB, MVT::i64, Expand);
908 }
909 setOperationAction(ISD::ATOMIC_LOAD_AND, MVT::i32, Custom);
910 setOperationAction(ISD::ATOMIC_LOAD_AND, MVT::i64, Custom);
911
912 // Generate outline atomics library calls only if LSE was not specified for
913 // subtarget
914 if (Subtarget->outlineAtomics() && !Subtarget->hasLSE()) {
915 setOperationAction(ISD::ATOMIC_CMP_SWAP, MVT::i8, LibCall);
916 setOperationAction(ISD::ATOMIC_CMP_SWAP, MVT::i16, LibCall);
917 setOperationAction(ISD::ATOMIC_CMP_SWAP, MVT::i32, LibCall);
918 setOperationAction(ISD::ATOMIC_CMP_SWAP, MVT::i64, LibCall);
919 setOperationAction(ISD::ATOMIC_CMP_SWAP, MVT::i128, LibCall);
920 setOperationAction(ISD::ATOMIC_SWAP, MVT::i8, LibCall);
921 setOperationAction(ISD::ATOMIC_SWAP, MVT::i16, LibCall);
922 setOperationAction(ISD::ATOMIC_SWAP, MVT::i32, LibCall);
923 setOperationAction(ISD::ATOMIC_SWAP, MVT::i64, LibCall);
924 setOperationAction(ISD::ATOMIC_LOAD_ADD, MVT::i8, LibCall);
925 setOperationAction(ISD::ATOMIC_LOAD_ADD, MVT::i16, LibCall);
926 setOperationAction(ISD::ATOMIC_LOAD_ADD, MVT::i32, LibCall);
927 setOperationAction(ISD::ATOMIC_LOAD_ADD, MVT::i64, LibCall);
928 setOperationAction(ISD::ATOMIC_LOAD_OR, MVT::i8, LibCall);
929 setOperationAction(ISD::ATOMIC_LOAD_OR, MVT::i16, LibCall);
930 setOperationAction(ISD::ATOMIC_LOAD_OR, MVT::i32, LibCall);
931 setOperationAction(ISD::ATOMIC_LOAD_OR, MVT::i64, LibCall);
932 setOperationAction(ISD::ATOMIC_LOAD_CLR, MVT::i8, LibCall);
933 setOperationAction(ISD::ATOMIC_LOAD_CLR, MVT::i16, LibCall);
934 setOperationAction(ISD::ATOMIC_LOAD_CLR, MVT::i32, LibCall);
935 setOperationAction(ISD::ATOMIC_LOAD_CLR, MVT::i64, LibCall);
936 setOperationAction(ISD::ATOMIC_LOAD_XOR, MVT::i8, LibCall);
937 setOperationAction(ISD::ATOMIC_LOAD_XOR, MVT::i16, LibCall);
938 setOperationAction(ISD::ATOMIC_LOAD_XOR, MVT::i32, LibCall);
939 setOperationAction(ISD::ATOMIC_LOAD_XOR, MVT::i64, LibCall);
940#define LCALLNAMES(A, B, N) \
941 setLibcallName(A##N##_RELAX, #B #N "_relax"); \
942 setLibcallName(A##N##_ACQ, #B #N "_acq"); \
943 setLibcallName(A##N##_REL, #B #N "_rel"); \
944 setLibcallName(A##N##_ACQ_REL, #B #N "_acq_rel");
945#define LCALLNAME4(A, B) \
946 LCALLNAMES(A, B, 1) \
947 LCALLNAMES(A, B, 2) LCALLNAMES(A, B, 4) LCALLNAMES(A, B, 8)
948#define LCALLNAME5(A, B) \
949 LCALLNAMES(A, B, 1) \
950 LCALLNAMES(A, B, 2) \
951 LCALLNAMES(A, B, 4) LCALLNAMES(A, B, 8) LCALLNAMES(A, B, 16)
952 LCALLNAME5(RTLIB::OUTLINE_ATOMIC_CAS, __aarch64_cas)
953 LCALLNAME4(RTLIB::OUTLINE_ATOMIC_SWP, __aarch64_swp)
954 LCALLNAME4(RTLIB::OUTLINE_ATOMIC_LDADD, __aarch64_ldadd)
955 LCALLNAME4(RTLIB::OUTLINE_ATOMIC_LDSET, __aarch64_ldset)
956 LCALLNAME4(RTLIB::OUTLINE_ATOMIC_LDCLR, __aarch64_ldclr)
957 LCALLNAME4(RTLIB::OUTLINE_ATOMIC_LDEOR, __aarch64_ldeor)
958#undef LCALLNAMES
959#undef LCALLNAME4
960#undef LCALLNAME5
961 }
962
963 if (Subtarget->hasLSE128()) {
964 // Custom lowering because i128 is not legal. Must be replaced by 2x64
965 // values. ATOMIC_LOAD_AND also needs op legalisation to emit LDCLRP.
966 setOperationAction(ISD::ATOMIC_LOAD_AND, MVT::i128, Custom);
967 setOperationAction(ISD::ATOMIC_LOAD_OR, MVT::i128, Custom);
968 setOperationAction(ISD::ATOMIC_SWAP, MVT::i128, Custom);
969 }
970
971 // 128-bit loads and stores can be done without expanding
972 setOperationAction(ISD::LOAD, MVT::i128, Custom);
973 setOperationAction(ISD::STORE, MVT::i128, Custom);
974
975 // Aligned 128-bit loads and stores are single-copy atomic according to the
976 // v8.4a spec. LRCPC3 introduces 128-bit STILP/LDIAPP but still requires LSE2.
977 if (Subtarget->hasLSE2()) {
978 setOperationAction(ISD::ATOMIC_LOAD, MVT::i128, Custom);
979 setOperationAction(ISD::ATOMIC_STORE, MVT::i128, Custom);
980 }
981
982 // 256 bit non-temporal stores can be lowered to STNP. Do this as part of the
983 // custom lowering, as there are no un-paired non-temporal stores and
984 // legalization will break up 256 bit inputs.
985 setOperationAction(ISD::STORE, MVT::v32i8, Custom);
986 setOperationAction(ISD::STORE, MVT::v16i16, Custom);
987 setOperationAction(ISD::STORE, MVT::v16f16, Custom);
988 setOperationAction(ISD::STORE, MVT::v16bf16, Custom);
989 setOperationAction(ISD::STORE, MVT::v8i32, Custom);
990 setOperationAction(ISD::STORE, MVT::v8f32, Custom);
991 setOperationAction(ISD::STORE, MVT::v4f64, Custom);
992 setOperationAction(ISD::STORE, MVT::v4i64, Custom);
993
994 // 256 bit non-temporal loads can be lowered to LDNP. This is done using
995 // custom lowering, as there are no un-paired non-temporal loads legalization
996 // will break up 256 bit inputs.
997 setOperationAction(ISD::LOAD, MVT::v32i8, Custom);
998 setOperationAction(ISD::LOAD, MVT::v16i16, Custom);
999 setOperationAction(ISD::LOAD, MVT::v16f16, Custom);
1000 setOperationAction(ISD::LOAD, MVT::v16bf16, Custom);
1001 setOperationAction(ISD::LOAD, MVT::v8i32, Custom);
1002 setOperationAction(ISD::LOAD, MVT::v8f32, Custom);
1003 setOperationAction(ISD::LOAD, MVT::v4f64, Custom);
1004 setOperationAction(ISD::LOAD, MVT::v4i64, Custom);
1005
1006 // Lower READCYCLECOUNTER using an mrs from CNTVCT_EL0.
1007 setOperationAction(ISD::READCYCLECOUNTER, MVT::i64, Legal);
1008
1009 if (getLibcallName(RTLIB::SINCOS_STRET_F32) != nullptr &&
1010 getLibcallName(RTLIB::SINCOS_STRET_F64) != nullptr) {
1011 // Issue __sincos_stret if available.
1012 setOperationAction(ISD::FSINCOS, MVT::f64, Custom);
1013 setOperationAction(ISD::FSINCOS, MVT::f32, Custom);
1014 } else {
1015 setOperationAction(ISD::FSINCOS, MVT::f64, Expand);
1016 setOperationAction(ISD::FSINCOS, MVT::f32, Expand);
1017 }
1018
1019 // Make floating-point constants legal for the large code model, so they don't
1020 // become loads from the constant pool.
1021 if (Subtarget->isTargetMachO() && TM.getCodeModel() == CodeModel::Large) {
1022 setOperationAction(ISD::ConstantFP, MVT::f32, Legal);
1023 setOperationAction(ISD::ConstantFP, MVT::f64, Legal);
1024 }
1025
1026 // AArch64 does not have floating-point extending loads, i1 sign-extending
1027 // load, floating-point truncating stores, or v2i32->v2i16 truncating store.
1028 for (MVT VT : MVT::fp_valuetypes()) {
1029 setLoadExtAction(ISD::EXTLOAD, VT, MVT::bf16, Expand);
1030 setLoadExtAction(ISD::EXTLOAD, VT, MVT::f16, Expand);
1031 setLoadExtAction(ISD::EXTLOAD, VT, MVT::f32, Expand);
1032 setLoadExtAction(ISD::EXTLOAD, VT, MVT::f64, Expand);
1033 setLoadExtAction(ISD::EXTLOAD, VT, MVT::f80, Expand);
1034 }
1035 for (MVT VT : MVT::integer_valuetypes())
1036 setLoadExtAction(ISD::SEXTLOAD, VT, MVT::i1, Expand);
1037
1038 for (MVT WideVT : MVT::fp_valuetypes()) {
1039 for (MVT NarrowVT : MVT::fp_valuetypes()) {
1040 if (WideVT.getScalarSizeInBits() > NarrowVT.getScalarSizeInBits()) {
1041 setTruncStoreAction(WideVT, NarrowVT, Expand);
1042 }
1043 }
1044 }
1045
1046 if (Subtarget->hasFPARMv8()) {
1047 setOperationAction(ISD::BITCAST, MVT::i16, Custom);
1048 setOperationAction(ISD::BITCAST, MVT::f16, Custom);
1049 setOperationAction(ISD::BITCAST, MVT::bf16, Custom);
1050 }
1051
1052 // Indexed loads and stores are supported.
1053 for (unsigned im = (unsigned)ISD::PRE_INC;
1054 im != (unsigned)ISD::LAST_INDEXED_MODE; ++im) {
1055 setIndexedLoadAction(im, MVT::i8, Legal);
1056 setIndexedLoadAction(im, MVT::i16, Legal);
1057 setIndexedLoadAction(im, MVT::i32, Legal);
1058 setIndexedLoadAction(im, MVT::i64, Legal);
1059 setIndexedLoadAction(im, MVT::f64, Legal);
1060 setIndexedLoadAction(im, MVT::f32, Legal);
1061 setIndexedLoadAction(im, MVT::f16, Legal);
1062 setIndexedLoadAction(im, MVT::bf16, Legal);
1063 setIndexedStoreAction(im, MVT::i8, Legal);
1064 setIndexedStoreAction(im, MVT::i16, Legal);
1065 setIndexedStoreAction(im, MVT::i32, Legal);
1066 setIndexedStoreAction(im, MVT::i64, Legal);
1067 setIndexedStoreAction(im, MVT::f64, Legal);
1068 setIndexedStoreAction(im, MVT::f32, Legal);
1069 setIndexedStoreAction(im, MVT::f16, Legal);
1070 setIndexedStoreAction(im, MVT::bf16, Legal);
1071 }
1072
1073 // Trap.
1074 setOperationAction(ISD::TRAP, MVT::Other, Legal);
1075 setOperationAction(ISD::DEBUGTRAP, MVT::Other, Legal);
1076 setOperationAction(ISD::UBSANTRAP, MVT::Other, Legal);
1077
1078 // We combine OR nodes for bitfield operations.
1079 setTargetDAGCombine(ISD::OR);
1080 // Try to create BICs for vector ANDs.
1081 setTargetDAGCombine(ISD::AND);
1082
1083 // llvm.init.trampoline and llvm.adjust.trampoline
1084 setOperationAction(ISD::INIT_TRAMPOLINE, MVT::Other, Custom);
1085 setOperationAction(ISD::ADJUST_TRAMPOLINE, MVT::Other, Custom);
1086
1087 // Vector add and sub nodes may conceal a high-half opportunity.
1088 // Also, try to fold ADD into CSINC/CSINV..
1089 setTargetDAGCombine({ISD::ADD, ISD::ABS, ISD::SUB, ISD::XOR, ISD::SINT_TO_FP,
1090 ISD::UINT_TO_FP});
1091
1092 setTargetDAGCombine({ISD::FP_TO_SINT, ISD::FP_TO_UINT, ISD::FP_TO_SINT_SAT,
1093 ISD::FP_TO_UINT_SAT, ISD::FADD});
1094
1095 // Try and combine setcc with csel
1096 setTargetDAGCombine(ISD::SETCC);
1097
1098 setTargetDAGCombine(ISD::INTRINSIC_WO_CHAIN);
1099
1100 setTargetDAGCombine({ISD::ANY_EXTEND, ISD::ZERO_EXTEND, ISD::SIGN_EXTEND,
1101 ISD::SIGN_EXTEND_INREG, ISD::CONCAT_VECTORS,
1102 ISD::EXTRACT_SUBVECTOR, ISD::INSERT_SUBVECTOR,
1103 ISD::STORE, ISD::BUILD_VECTOR});
1104 setTargetDAGCombine(ISD::TRUNCATE);
1105 setTargetDAGCombine(ISD::LOAD);
1106
1107 setTargetDAGCombine(ISD::MSTORE);
1108
1109 setTargetDAGCombine(ISD::MUL);
1110
1111 setTargetDAGCombine({ISD::SELECT, ISD::VSELECT});
1112
1113 setTargetDAGCombine({ISD::INTRINSIC_VOID, ISD::INTRINSIC_W_CHAIN,
1114 ISD::INSERT_VECTOR_ELT, ISD::EXTRACT_VECTOR_ELT,
1115 ISD::VECREDUCE_ADD, ISD::STEP_VECTOR});
1116
1117 setTargetDAGCombine({ISD::MGATHER, ISD::MSCATTER});
1118
1119 setTargetDAGCombine(ISD::FP_EXTEND);
1120
1121 setTargetDAGCombine(ISD::GlobalAddress);
1122
1123 setTargetDAGCombine(ISD::CTLZ);
1124
1125 setTargetDAGCombine(ISD::VECREDUCE_AND);
1126 setTargetDAGCombine(ISD::VECREDUCE_OR);
1127 setTargetDAGCombine(ISD::VECREDUCE_XOR);
1128
1129 setTargetDAGCombine(ISD::SCALAR_TO_VECTOR);
1130
1131 // In case of strict alignment, avoid an excessive number of byte wide stores.
1132 MaxStoresPerMemsetOptSize = 8;
1133 MaxStoresPerMemset =
1134 Subtarget->requiresStrictAlign() ? MaxStoresPerMemsetOptSize : 32;
1135
1136 MaxGluedStoresPerMemcpy = 4;
1137 MaxStoresPerMemcpyOptSize = 4;
1138 MaxStoresPerMemcpy =
1139 Subtarget->requiresStrictAlign() ? MaxStoresPerMemcpyOptSize : 16;
1140
1141 MaxStoresPerMemmoveOptSize = 4;
1142 MaxStoresPerMemmove = 4;
1143
1144 MaxLoadsPerMemcmpOptSize = 4;
1145 MaxLoadsPerMemcmp =
1146 Subtarget->requiresStrictAlign() ? MaxLoadsPerMemcmpOptSize : 8;
1147
1148 setStackPointerRegisterToSaveRestore(AArch64::SP);
1149
1150 setSchedulingPreference(Sched::Hybrid);
1151
1152 EnableExtLdPromotion = true;
1153
1154 // Set required alignment.
1155 setMinFunctionAlignment(Align(4));
1156 // Set preferred alignments.
1157
1158 // Don't align loops on Windows. The SEH unwind info generation needs to
1159 // know the exact length of functions before the alignments have been
1160 // expanded.
1161 if (!Subtarget->isTargetWindows())
1162 setPrefLoopAlignment(STI.getPrefLoopAlignment());
1163 setMaxBytesForAlignment(STI.getMaxBytesForLoopAlignment());
1164 setPrefFunctionAlignment(STI.getPrefFunctionAlignment());
1165
1166 // Only change the limit for entries in a jump table if specified by
1167 // the sub target, but not at the command line.
1168 unsigned MaxJT = STI.getMaximumJumpTableSize();
1169 if (MaxJT && getMaximumJumpTableSize() == UINT_MAX)
1170 setMaximumJumpTableSize(MaxJT);
1171
1172 setHasExtractBitsInsn(true);
1173
1174 setMaxDivRemBitWidthSupported(128);
1175
1176 setOperationAction(ISD::INTRINSIC_WO_CHAIN, MVT::Other, Custom);
1177
1178 if (Subtarget->isNeonAvailable()) {
1179 // FIXME: v1f64 shouldn't be legal if we can avoid it, because it leads to
1180 // silliness like this:
1181 // clang-format off
1182 for (auto Op :
1183 {ISD::SELECT, ISD::SELECT_CC,
1184 ISD::BR_CC, ISD::FADD, ISD::FSUB,
1185 ISD::FMUL, ISD::FDIV, ISD::FMA,
1186 ISD::FNEG, ISD::FABS, ISD::FCEIL,
1187 ISD::FSQRT, ISD::FFLOOR, ISD::FNEARBYINT,
1188 ISD::FSIN, ISD::FCOS, ISD::FTAN,
1189 ISD::FASIN, ISD::FACOS, ISD::FATAN,
1190 ISD::FSINH, ISD::FCOSH, ISD::FTANH,
1191 ISD::FPOW, ISD::FLOG, ISD::FLOG2,
1192 ISD::FLOG10, ISD::FEXP, ISD::FEXP2,
1193 ISD::FEXP10, ISD::FRINT, ISD::FROUND,
1194 ISD::FROUNDEVEN, ISD::FTRUNC, ISD::FMINNUM,
1195 ISD::FMAXNUM, ISD::FMINIMUM, ISD::FMAXIMUM,
1196 ISD::STRICT_FADD, ISD::STRICT_FSUB, ISD::STRICT_FMUL,
1197 ISD::STRICT_FDIV, ISD::STRICT_FMA, ISD::STRICT_FCEIL,
1198 ISD::STRICT_FFLOOR, ISD::STRICT_FSQRT, ISD::STRICT_FRINT,
1199 ISD::STRICT_FNEARBYINT, ISD::STRICT_FROUND, ISD::STRICT_FTRUNC,
1200 ISD::STRICT_FROUNDEVEN, ISD::STRICT_FMINNUM, ISD::STRICT_FMAXNUM,
1201 ISD::STRICT_FMINIMUM, ISD::STRICT_FMAXIMUM})
1202 setOperationAction(Op, MVT::v1f64, Expand);
1203 // clang-format on
1204 for (auto Op :
1205 {ISD::FP_TO_SINT, ISD::FP_TO_UINT, ISD::SINT_TO_FP, ISD::UINT_TO_FP,
1206 ISD::FP_ROUND, ISD::FP_TO_SINT_SAT, ISD::FP_TO_UINT_SAT, ISD::MUL,
1207 ISD::STRICT_FP_TO_SINT, ISD::STRICT_FP_TO_UINT,
1208 ISD::STRICT_SINT_TO_FP, ISD::STRICT_UINT_TO_FP, ISD::STRICT_FP_ROUND})
1209 setOperationAction(Op, MVT::v1i64, Expand);
1210
1211 // AArch64 doesn't have a direct vector ->f32 conversion instructions for
1212 // elements smaller than i32, so promote the input to i32 first.
1213 setOperationPromotedToType(ISD::UINT_TO_FP, MVT::v4i8, MVT::v4i32);
1214 setOperationPromotedToType(ISD::SINT_TO_FP, MVT::v4i8, MVT::v4i32);
1215
1216 // Similarly, there is no direct i32 -> f64 vector conversion instruction.
1217 // Or, direct i32 -> f16 vector conversion. Set it so custom, so the
1218 // conversion happens in two steps: v4i32 -> v4f32 -> v4f16
1219 for (auto Op : {ISD::SINT_TO_FP, ISD::UINT_TO_FP, ISD::STRICT_SINT_TO_FP,
1220 ISD::STRICT_UINT_TO_FP})
1221 for (auto VT : {MVT::v2i32, MVT::v2i64, MVT::v4i32})
1222 setOperationAction(Op, VT, Custom);
1223
1224 if (Subtarget->hasFullFP16()) {
1225 setOperationAction(ISD::ConstantFP, MVT::f16, Legal);
1226 setOperationAction(ISD::ConstantFP, MVT::bf16, Legal);
1227
1228 setOperationAction(ISD::SINT_TO_FP, MVT::v8i8, Custom);
1229 setOperationAction(ISD::UINT_TO_FP, MVT::v8i8, Custom);
1230 setOperationAction(ISD::SINT_TO_FP, MVT::v16i8, Custom);
1231 setOperationAction(ISD::UINT_TO_FP, MVT::v16i8, Custom);
1232 setOperationAction(ISD::SINT_TO_FP, MVT::v4i16, Custom);
1233 setOperationAction(ISD::UINT_TO_FP, MVT::v4i16, Custom);
1234 setOperationAction(ISD::SINT_TO_FP, MVT::v8i16, Custom);
1235 setOperationAction(ISD::UINT_TO_FP, MVT::v8i16, Custom);
1236 } else {
1237 // when AArch64 doesn't have fullfp16 support, promote the input
1238 // to i32 first.
1239 setOperationPromotedToType(ISD::SINT_TO_FP, MVT::v8i8, MVT::v8i32);
1240 setOperationPromotedToType(ISD::UINT_TO_FP, MVT::v8i8, MVT::v8i32);
1241 setOperationPromotedToType(ISD::UINT_TO_FP, MVT::v16i8, MVT::v16i32);
1242 setOperationPromotedToType(ISD::SINT_TO_FP, MVT::v16i8, MVT::v16i32);
1243 setOperationPromotedToType(ISD::UINT_TO_FP, MVT::v4i16, MVT::v4i32);
1244 setOperationPromotedToType(ISD::SINT_TO_FP, MVT::v4i16, MVT::v4i32);
1245 setOperationPromotedToType(ISD::SINT_TO_FP, MVT::v8i16, MVT::v8i32);
1246 setOperationPromotedToType(ISD::UINT_TO_FP, MVT::v8i16, MVT::v8i32);
1247 }
1248
1249 setOperationAction(ISD::CTLZ, MVT::v1i64, Expand);
1250 setOperationAction(ISD::CTLZ, MVT::v2i64, Expand);
1251 setOperationAction(ISD::BITREVERSE, MVT::v8i8, Legal);
1252 setOperationAction(ISD::BITREVERSE, MVT::v16i8, Legal);
1253 setOperationAction(ISD::BITREVERSE, MVT::v2i32, Custom);
1254 setOperationAction(ISD::BITREVERSE, MVT::v4i32, Custom);
1255 setOperationAction(ISD::BITREVERSE, MVT::v1i64, Custom);
1256 setOperationAction(ISD::BITREVERSE, MVT::v2i64, Custom);
1257 for (auto VT : {MVT::v1i64, MVT::v2i64}) {
1258 setOperationAction(ISD::UMAX, VT, Custom);
1259 setOperationAction(ISD::SMAX, VT, Custom);
1260 setOperationAction(ISD::UMIN, VT, Custom);
1261 setOperationAction(ISD::SMIN, VT, Custom);
1262 }
1263
1264 // Custom handling for some quad-vector types to detect MULL.
1265 setOperationAction(ISD::MUL, MVT::v8i16, Custom);
1266 setOperationAction(ISD::MUL, MVT::v4i32, Custom);
1267 setOperationAction(ISD::MUL, MVT::v2i64, Custom);
1268 setOperationAction(ISD::MUL, MVT::v4i16, Custom);
1269 setOperationAction(ISD::MUL, MVT::v2i32, Custom);
1270 setOperationAction(ISD::MUL, MVT::v1i64, Custom);
1271
1272 // Saturates
1273 for (MVT VT : { MVT::v8i8, MVT::v4i16, MVT::v2i32,
1274 MVT::v16i8, MVT::v8i16, MVT::v4i32, MVT::v2i64 }) {
1275 setOperationAction(ISD::SADDSAT, VT, Legal);
1276 setOperationAction(ISD::UADDSAT, VT, Legal);
1277 setOperationAction(ISD::SSUBSAT, VT, Legal);
1278 setOperationAction(ISD::USUBSAT, VT, Legal);
1279 }
1280
1281 for (MVT VT : {MVT::v8i8, MVT::v4i16, MVT::v2i32, MVT::v16i8, MVT::v8i16,
1282 MVT::v4i32}) {
1283 setOperationAction(ISD::AVGFLOORS, VT, Legal);
1284 setOperationAction(ISD::AVGFLOORU, VT, Legal);
1285 setOperationAction(ISD::AVGCEILS, VT, Legal);
1286 setOperationAction(ISD::AVGCEILU, VT, Legal);
1287 setOperationAction(ISD::ABDS, VT, Legal);
1288 setOperationAction(ISD::ABDU, VT, Legal);
1289 }
1290
1291 // Vector reductions
1292 for (MVT VT : { MVT::v4f16, MVT::v2f32,
1293 MVT::v8f16, MVT::v4f32, MVT::v2f64 }) {
1294 if (VT.getVectorElementType() != MVT::f16 || Subtarget->hasFullFP16()) {
1295 setOperationAction(ISD::VECREDUCE_FMAX, VT, Legal);
1296 setOperationAction(ISD::VECREDUCE_FMIN, VT, Legal);
1297 setOperationAction(ISD::VECREDUCE_FMAXIMUM, VT, Legal);
1298 setOperationAction(ISD::VECREDUCE_FMINIMUM, VT, Legal);
1299
1300 setOperationAction(ISD::VECREDUCE_FADD, VT, Legal);
1301 }
1302 }
1303 for (MVT VT : { MVT::v8i8, MVT::v4i16, MVT::v2i32,
1304 MVT::v16i8, MVT::v8i16, MVT::v4i32 }) {
1305 setOperationAction(ISD::VECREDUCE_ADD, VT, Custom);
1306 setOperationAction(ISD::VECREDUCE_SMAX, VT, Custom);
1307 setOperationAction(ISD::VECREDUCE_SMIN, VT, Custom);
1308 setOperationAction(ISD::VECREDUCE_UMAX, VT, Custom);
1309 setOperationAction(ISD::VECREDUCE_UMIN, VT, Custom);
1310 setOperationAction(ISD::VECREDUCE_AND, VT, Custom);
1311 setOperationAction(ISD::VECREDUCE_OR, VT, Custom);
1312 setOperationAction(ISD::VECREDUCE_XOR, VT, Custom);
1313 }
1314 setOperationAction(ISD::VECREDUCE_ADD, MVT::v2i64, Custom);
1315 setOperationAction(ISD::VECREDUCE_AND, MVT::v2i64, Custom);
1316 setOperationAction(ISD::VECREDUCE_OR, MVT::v2i64, Custom);
1317 setOperationAction(ISD::VECREDUCE_XOR, MVT::v2i64, Custom);
1318
1319 setOperationAction(ISD::ANY_EXTEND, MVT::v4i32, Legal);
1320 setTruncStoreAction(MVT::v2i32, MVT::v2i16, Expand);
1321 // Likewise, narrowing and extending vector loads/stores aren't handled
1322 // directly.
1323 for (MVT VT : MVT::fixedlen_vector_valuetypes()) {
1324 setOperationAction(ISD::SIGN_EXTEND_INREG, VT, Expand);
1325
1326 if (VT == MVT::v16i8 || VT == MVT::v8i16 || VT == MVT::v4i32) {
1327 setOperationAction(ISD::MULHS, VT, Legal);
1328 setOperationAction(ISD::MULHU, VT, Legal);
1329 } else {
1330 setOperationAction(ISD::MULHS, VT, Expand);
1331 setOperationAction(ISD::MULHU, VT, Expand);
1332 }
1333 setOperationAction(ISD::SMUL_LOHI, VT, Expand);
1334 setOperationAction(ISD::UMUL_LOHI, VT, Expand);
1335
1336 setOperationAction(ISD::BSWAP, VT, Expand);
1337 setOperationAction(ISD::CTTZ, VT, Expand);
1338
1339 for (MVT InnerVT : MVT::fixedlen_vector_valuetypes()) {
1340 setTruncStoreAction(VT, InnerVT, Expand);
1341 setLoadExtAction(ISD::SEXTLOAD, VT, InnerVT, Expand);
1342 setLoadExtAction(ISD::ZEXTLOAD, VT, InnerVT, Expand);
1343 setLoadExtAction(ISD::EXTLOAD, VT, InnerVT, Expand);
1344 }
1345 }
1346
1347 // AArch64 has implementations of a lot of rounding-like FP operations.
1348 for (auto Op :
1349 {ISD::FFLOOR, ISD::FNEARBYINT, ISD::FCEIL, ISD::FRINT, ISD::FTRUNC,
1350 ISD::FROUND, ISD::FROUNDEVEN, ISD::STRICT_FFLOOR,
1351 ISD::STRICT_FNEARBYINT, ISD::STRICT_FCEIL, ISD::STRICT_FRINT,
1352 ISD::STRICT_FTRUNC, ISD::STRICT_FROUND, ISD::STRICT_FROUNDEVEN}) {
1353 for (MVT Ty : {MVT::v2f32, MVT::v4f32, MVT::v2f64})
1354 setOperationAction(Op, Ty, Legal);
1355 if (Subtarget->hasFullFP16())
1356 for (MVT Ty : {MVT::v4f16, MVT::v8f16})
1357 setOperationAction(Op, Ty, Legal);
1358 }
1359
1360 // LRINT and LLRINT.
1361 for (auto Op : {ISD::LRINT, ISD::LLRINT}) {
1362 for (MVT Ty : {MVT::v2f32, MVT::v4f32, MVT::v2f64})
1363 setOperationAction(Op, Ty, Custom);
1364 if (Subtarget->hasFullFP16())
1365 for (MVT Ty : {MVT::v4f16, MVT::v8f16})
1366 setOperationAction(Op, Ty, Custom);
1367 }
1368
1369 setTruncStoreAction(MVT::v4i16, MVT::v4i8, Custom);
1370
1371 setOperationAction(ISD::BITCAST, MVT::i2, Custom);
1372 setOperationAction(ISD::BITCAST, MVT::i4, Custom);
1373 setOperationAction(ISD::BITCAST, MVT::i8, Custom);
1374 setOperationAction(ISD::BITCAST, MVT::i16, Custom);
1375
1376 setOperationAction(ISD::BITCAST, MVT::v2i8, Custom);
1377 setOperationAction(ISD::BITCAST, MVT::v2i16, Custom);
1378 setOperationAction(ISD::BITCAST, MVT::v4i8, Custom);
1379
1380 setLoadExtAction(ISD::EXTLOAD, MVT::v4i16, MVT::v4i8, Custom);
1381 setLoadExtAction(ISD::SEXTLOAD, MVT::v4i16, MVT::v4i8, Custom);
1382 setLoadExtAction(ISD::ZEXTLOAD, MVT::v4i16, MVT::v4i8, Custom);
1383 setLoadExtAction(ISD::EXTLOAD, MVT::v4i32, MVT::v4i8, Custom);
1384 setLoadExtAction(ISD::SEXTLOAD, MVT::v4i32, MVT::v4i8, Custom);
1385 setLoadExtAction(ISD::ZEXTLOAD, MVT::v4i32, MVT::v4i8, Custom);
1386
1387 // ADDP custom lowering
1388 for (MVT VT : { MVT::v32i8, MVT::v16i16, MVT::v8i32, MVT::v4i64 })
1389 setOperationAction(ISD::ADD, VT, Custom);
1390 // FADDP custom lowering
1391 for (MVT VT : { MVT::v16f16, MVT::v8f32, MVT::v4f64 })
1392 setOperationAction(ISD::FADD, VT, Custom);
1393 } else /* !isNeonAvailable */ {
1394 for (MVT VT : MVT::fixedlen_vector_valuetypes()) {
1395 for (unsigned Op = 0; Op < ISD::BUILTIN_OP_END; ++Op)
1396 setOperationAction(Op, VT, Expand);
1397
1398 if (VT.is128BitVector() || VT.is64BitVector()) {
1399 setOperationAction(ISD::LOAD, VT, Legal);
1400 setOperationAction(ISD::STORE, VT, Legal);
1401 setOperationAction(ISD::BITCAST, VT,
1402 Subtarget->isLittleEndian() ? Legal : Expand);
1403 }
1404 for (MVT InnerVT : MVT::fixedlen_vector_valuetypes()) {
1405 setTruncStoreAction(VT, InnerVT, Expand);
1406 setLoadExtAction(ISD::SEXTLOAD, VT, InnerVT, Expand);
1407 setLoadExtAction(ISD::ZEXTLOAD, VT, InnerVT, Expand);
1408 setLoadExtAction(ISD::EXTLOAD, VT, InnerVT, Expand);
1409 }
1410 }
1411 }
1412
1413 if (Subtarget->hasSME()) {
1414 setOperationAction(ISD::INTRINSIC_W_CHAIN, MVT::Other, Custom);
1415 }
1416
1417 // FIXME: Move lowering for more nodes here if those are common between
1418 // SVE and SME.
1419 if (Subtarget->isSVEorStreamingSVEAvailable()) {
1420 for (auto VT :
1421 {MVT::nxv16i1, MVT::nxv8i1, MVT::nxv4i1, MVT::nxv2i1, MVT::nxv1i1}) {
1422 setOperationAction(ISD::SPLAT_VECTOR, VT, Custom);
1423 setOperationAction(ISD::EXTRACT_VECTOR_ELT, VT, Custom);
1424 setOperationAction(ISD::VECTOR_DEINTERLEAVE, VT, Custom);
1425 setOperationAction(ISD::VECTOR_INTERLEAVE, VT, Custom);
1426 }
1427 }
1428
1429 if (Subtarget->isSVEorStreamingSVEAvailable()) {
1430 for (auto VT : {MVT::nxv16i8, MVT::nxv8i16, MVT::nxv4i32, MVT::nxv2i64}) {
1431 setOperationAction(ISD::BITREVERSE, VT, Custom);
1432 setOperationAction(ISD::BSWAP, VT, Custom);
1433 setOperationAction(ISD::CTLZ, VT, Custom);
1434 setOperationAction(ISD::CTPOP, VT, Custom);
1435 setOperationAction(ISD::CTTZ, VT, Custom);
1436 setOperationAction(ISD::INSERT_SUBVECTOR, VT, Custom);
1437 setOperationAction(ISD::UINT_TO_FP, VT, Custom);
1438 setOperationAction(ISD::SINT_TO_FP, VT, Custom);
1439 setOperationAction(ISD::FP_TO_UINT, VT, Custom);
1440 setOperationAction(ISD::FP_TO_SINT, VT, Custom);
1441 setOperationAction(ISD::MLOAD, VT, Custom);
1442 setOperationAction(ISD::MUL, VT, Custom);
1443 setOperationAction(ISD::MULHS, VT, Custom);
1444 setOperationAction(ISD::MULHU, VT, Custom);
1445 setOperationAction(ISD::SPLAT_VECTOR, VT, Legal);
1446 setOperationAction(ISD::VECTOR_SPLICE, VT, Custom);
1447 setOperationAction(ISD::SELECT, VT, Custom);
1448 setOperationAction(ISD::SETCC, VT, Custom);
1449 setOperationAction(ISD::SDIV, VT, Custom);
1450 setOperationAction(ISD::UDIV, VT, Custom);
1451 setOperationAction(ISD::SMIN, VT, Custom);
1452 setOperationAction(ISD::UMIN, VT, Custom);
1453 setOperationAction(ISD::SMAX, VT, Custom);
1454 setOperationAction(ISD::UMAX, VT, Custom);
1455 setOperationAction(ISD::SHL, VT, Custom);
1456 setOperationAction(ISD::SRL, VT, Custom);
1457 setOperationAction(ISD::SRA, VT, Custom);
1458 setOperationAction(ISD::ABS, VT, Custom);
1459 setOperationAction(ISD::ABDS, VT, Custom);
1460 setOperationAction(ISD::ABDU, VT, Custom);
1461 setOperationAction(ISD::VECREDUCE_ADD, VT, Custom);
1462 setOperationAction(ISD::VECREDUCE_AND, VT, Custom);
1463 setOperationAction(ISD::VECREDUCE_OR, VT, Custom);
1464 setOperationAction(ISD::VECREDUCE_XOR, VT, Custom);
1465 setOperationAction(ISD::VECREDUCE_UMIN, VT, Custom);
1466 setOperationAction(ISD::VECREDUCE_UMAX, VT, Custom);
1467 setOperationAction(ISD::VECREDUCE_SMIN, VT, Custom);
1468 setOperationAction(ISD::VECREDUCE_SMAX, VT, Custom);
1469 setOperationAction(ISD::VECTOR_DEINTERLEAVE, VT, Custom);
1470 setOperationAction(ISD::VECTOR_INTERLEAVE, VT, Custom);
1471
1472 setOperationAction(ISD::UMUL_LOHI, VT, Expand);
1473 setOperationAction(ISD::SMUL_LOHI, VT, Expand);
1474 setOperationAction(ISD::SELECT_CC, VT, Expand);
1475 setOperationAction(ISD::ROTL, VT, Expand);
1476 setOperationAction(ISD::ROTR, VT, Expand);
1477
1478 setOperationAction(ISD::SADDSAT, VT, Legal);
1479 setOperationAction(ISD::UADDSAT, VT, Legal);
1480 setOperationAction(ISD::SSUBSAT, VT, Legal);
1481 setOperationAction(ISD::USUBSAT, VT, Legal);
1482 setOperationAction(ISD::UREM, VT, Expand);
1483 setOperationAction(ISD::SREM, VT, Expand);
1484 setOperationAction(ISD::SDIVREM, VT, Expand);
1485 setOperationAction(ISD::UDIVREM, VT, Expand);
1486
1487 setOperationAction(ISD::AVGFLOORS, VT, Custom);
1488 setOperationAction(ISD::AVGFLOORU, VT, Custom);
1489 setOperationAction(ISD::AVGCEILS, VT, Custom);
1490 setOperationAction(ISD::AVGCEILU, VT, Custom);
1491
1492 if (!Subtarget->isLittleEndian())
1493 setOperationAction(ISD::BITCAST, VT, Expand);
1494
1495 if (Subtarget->hasSVE2() ||
1496 (Subtarget->hasSME() && Subtarget->isStreaming()))
1497 // For SLI/SRI.
1498 setOperationAction(ISD::OR, VT, Custom);
1499 }
1500
1501 // Illegal unpacked integer vector types.
1502 for (auto VT : {MVT::nxv8i8, MVT::nxv4i16, MVT::nxv2i32}) {
1503 setOperationAction(ISD::EXTRACT_SUBVECTOR, VT, Custom);
1504 setOperationAction(ISD::INSERT_SUBVECTOR, VT, Custom);
1505 }
1506
1507 // Legalize unpacked bitcasts to REINTERPRET_CAST.
1508 for (auto VT : {MVT::nxv2i16, MVT::nxv4i16, MVT::nxv2i32, MVT::nxv2bf16,
1509 MVT::nxv4bf16, MVT::nxv2f16, MVT::nxv4f16, MVT::nxv2f32})
1510 setOperationAction(ISD::BITCAST, VT, Custom);
1511
1512 for (auto VT :
1513 { MVT::nxv2i8, MVT::nxv2i16, MVT::nxv2i32, MVT::nxv2i64, MVT::nxv4i8,
1514 MVT::nxv4i16, MVT::nxv4i32, MVT::nxv8i8, MVT::nxv8i16 })
1515 setOperationAction(ISD::SIGN_EXTEND_INREG, VT, Legal);
1516
1517 for (auto VT :
1518 {MVT::nxv16i1, MVT::nxv8i1, MVT::nxv4i1, MVT::nxv2i1, MVT::nxv1i1}) {
1519 setOperationAction(ISD::CONCAT_VECTORS, VT, Custom);
1520 setOperationAction(ISD::SELECT, VT, Custom);
1521 setOperationAction(ISD::SETCC, VT, Custom);
1522 setOperationAction(ISD::TRUNCATE, VT, Custom);
1523 setOperationAction(ISD::VECREDUCE_AND, VT, Custom);
1524 setOperationAction(ISD::VECREDUCE_OR, VT, Custom);
1525 setOperationAction(ISD::VECREDUCE_XOR, VT, Custom);
1526 setOperationAction(ISD::OR, VT, Custom);
1527
1528 setOperationAction(ISD::SELECT_CC, VT, Expand);
1529 setOperationAction(ISD::INSERT_VECTOR_ELT, VT, Custom);
1530 setOperationAction(ISD::INSERT_SUBVECTOR, VT, Custom);
1531
1532 // There are no legal MVT::nxv16f## based types.
1533 if (VT != MVT::nxv16i1) {
1534 setOperationAction(ISD::SINT_TO_FP, VT, Custom);
1535 setOperationAction(ISD::UINT_TO_FP, VT, Custom);
1536 }
1537 }
1538
1539 // NEON doesn't support masked loads/stores, but SME and SVE do.
1540 for (auto VT :
1541 {MVT::v4f16, MVT::v8f16, MVT::v2f32, MVT::v4f32, MVT::v1f64,
1542 MVT::v2f64, MVT::v8i8, MVT::v16i8, MVT::v4i16, MVT::v8i16,
1543 MVT::v2i32, MVT::v4i32, MVT::v1i64, MVT::v2i64}) {
1544 setOperationAction(ISD::MLOAD, VT, Custom);
1545 setOperationAction(ISD::MSTORE, VT, Custom);
1546 }
1547
1548 // Firstly, exclude all scalable vector extending loads/truncating stores,
1549 // include both integer and floating scalable vector.
1550 for (MVT VT : MVT::scalable_vector_valuetypes()) {
1551 for (MVT InnerVT : MVT::scalable_vector_valuetypes()) {
1552 setTruncStoreAction(VT, InnerVT, Expand);
1553 setLoadExtAction(ISD::SEXTLOAD, VT, InnerVT, Expand);
1554 setLoadExtAction(ISD::ZEXTLOAD, VT, InnerVT, Expand);
1555 setLoadExtAction(ISD::EXTLOAD, VT, InnerVT, Expand);
1556 }
1557 }
1558
1559 // Then, selectively enable those which we directly support.
1560 setTruncStoreAction(MVT::nxv2i64, MVT::nxv2i8, Legal);
1561 setTruncStoreAction(MVT::nxv2i64, MVT::nxv2i16, Legal);
1562 setTruncStoreAction(MVT::nxv2i64, MVT::nxv2i32, Legal);
1563 setTruncStoreAction(MVT::nxv4i32, MVT::nxv4i8, Legal);
1564 setTruncStoreAction(MVT::nxv4i32, MVT::nxv4i16, Legal);
1565 setTruncStoreAction(MVT::nxv8i16, MVT::nxv8i8, Legal);
1566 for (auto Op : {ISD::ZEXTLOAD, ISD::SEXTLOAD, ISD::EXTLOAD}) {
1567 setLoadExtAction(Op, MVT::nxv2i64, MVT::nxv2i8, Legal);
1568 setLoadExtAction(Op, MVT::nxv2i64, MVT::nxv2i16, Legal);
1569 setLoadExtAction(Op, MVT::nxv2i64, MVT::nxv2i32, Legal);
1570 setLoadExtAction(Op, MVT::nxv4i32, MVT::nxv4i8, Legal);
1571 setLoadExtAction(Op, MVT::nxv4i32, MVT::nxv4i16, Legal);
1572 setLoadExtAction(Op, MVT::nxv8i16, MVT::nxv8i8, Legal);
1573 }
1574
1575 // SVE supports truncating stores of 64 and 128-bit vectors
1576 setTruncStoreAction(MVT::v2i64, MVT::v2i8, Custom);
1577 setTruncStoreAction(MVT::v2i64, MVT::v2i16, Custom);
1578 setTruncStoreAction(MVT::v2i64, MVT::v2i32, Custom);
1579 setTruncStoreAction(MVT::v2i32, MVT::v2i8, Custom);
1580 setTruncStoreAction(MVT::v2i32, MVT::v2i16, Custom);
1581
1582 for (auto VT : {MVT::nxv2f16, MVT::nxv4f16, MVT::nxv8f16, MVT::nxv2f32,
1583 MVT::nxv4f32, MVT::nxv2f64}) {
1584 setOperationAction(ISD::CONCAT_VECTORS, VT, Custom);
1585 setOperationAction(ISD::INSERT_SUBVECTOR, VT, Custom);
1586 setOperationAction(ISD::MLOAD, VT, Custom);
1587 setOperationAction(ISD::SPLAT_VECTOR, VT, Legal);
1588 setOperationAction(ISD::SELECT, VT, Custom);
1589 setOperationAction(ISD::SETCC, VT, Custom);
1590 setOperationAction(ISD::FADD, VT, Custom);
1591 setOperationAction(ISD::FCOPYSIGN, VT, Custom);
1592 setOperationAction(ISD::FDIV, VT, Custom);
1593 setOperationAction(ISD::FMA, VT, Custom);
1594 setOperationAction(ISD::FMAXIMUM, VT, Custom);
1595 setOperationAction(ISD::FMAXNUM, VT, Custom);
1596 setOperationAction(ISD::FMINIMUM, VT, Custom);
1597 setOperationAction(ISD::FMINNUM, VT, Custom);
1598 setOperationAction(ISD::FMUL, VT, Custom);
1599 setOperationAction(ISD::FNEG, VT, Custom);
1600 setOperationAction(ISD::FSUB, VT, Custom);
1601 setOperationAction(ISD::FCEIL, VT, Custom);
1602 setOperationAction(ISD::FFLOOR, VT, Custom);
1603 setOperationAction(ISD::FNEARBYINT, VT, Custom);
1604 setOperationAction(ISD::FRINT, VT, Custom);
1605 setOperationAction(ISD::LRINT, VT, Custom);
1606 setOperationAction(ISD::LLRINT, VT, Custom);
1607 setOperationAction(ISD::FROUND, VT, Custom);
1608 setOperationAction(ISD::FROUNDEVEN, VT, Custom);
1609 setOperationAction(ISD::FTRUNC, VT, Custom);
1610 setOperationAction(ISD::FSQRT, VT, Custom);
1611 setOperationAction(ISD::FABS, VT, Custom);
1612 setOperationAction(ISD::FP_EXTEND, VT, Custom);
1613 setOperationAction(ISD::FP_ROUND, VT, Custom);
1614 setOperationAction(ISD::VECREDUCE_FADD, VT, Custom);
1615 setOperationAction(ISD::VECREDUCE_FMAX, VT, Custom);
1616 setOperationAction(ISD::VECREDUCE_FMIN, VT, Custom);
1617 setOperationAction(ISD::VECREDUCE_FMAXIMUM, VT, Custom);
1618 setOperationAction(ISD::VECREDUCE_FMINIMUM, VT, Custom);
1619 setOperationAction(ISD::VECTOR_SPLICE, VT, Custom);
1620 setOperationAction(ISD::VECTOR_DEINTERLEAVE, VT, Custom);
1621 setOperationAction(ISD::VECTOR_INTERLEAVE, VT, Custom);
1622
1623 setOperationAction(ISD::SELECT_CC, VT, Expand);
1624 setOperationAction(ISD::FREM, VT, Expand);
1625 setOperationAction(ISD::FPOW, VT, Expand);
1626 setOperationAction(ISD::FPOWI, VT, Expand);
1627 setOperationAction(ISD::FCOS, VT, Expand);
1628 setOperationAction(ISD::FSIN, VT, Expand);
1629 setOperationAction(ISD::FSINCOS, VT, Expand);
1630 setOperationAction(ISD::FTAN, VT, Expand);
1631 setOperationAction(ISD::FACOS, VT, Expand);
1632 setOperationAction(ISD::FASIN, VT, Expand);
1633 setOperationAction(ISD::FATAN, VT, Expand);
1634 setOperationAction(ISD::FCOSH, VT, Expand);
1635 setOperationAction(ISD::FSINH, VT, Expand);
1636 setOperationAction(ISD::FTANH, VT, Expand);
1637 setOperationAction(ISD::FEXP, VT, Expand);
1638 setOperationAction(ISD::FEXP2, VT, Expand);
1639 setOperationAction(ISD::FEXP10, VT, Expand);
1640 setOperationAction(ISD::FLOG, VT, Expand);
1641 setOperationAction(ISD::FLOG2, VT, Expand);
1642 setOperationAction(ISD::FLOG10, VT, Expand);
1643
1644 setCondCodeAction(ISD::SETO, VT, Expand);
1645 setCondCodeAction(ISD::SETOLT, VT, Expand);
1646 setCondCodeAction(ISD::SETLT, VT, Expand);
1647 setCondCodeAction(ISD::SETOLE, VT, Expand);
1648 setCondCodeAction(ISD::SETLE, VT, Expand);
1649 setCondCodeAction(ISD::SETULT, VT, Expand);
1650 setCondCodeAction(ISD::SETULE, VT, Expand);
1651 setCondCodeAction(ISD::SETUGE, VT, Expand);
1652 setCondCodeAction(ISD::SETUGT, VT, Expand);
1653 setCondCodeAction(ISD::SETUEQ, VT, Expand);
1654 setCondCodeAction(ISD::SETONE, VT, Expand);
1655
1656 if (!Subtarget->isLittleEndian())
1657 setOperationAction(ISD::BITCAST, VT, Expand);
1658 }
1659
1660 for (auto VT : {MVT::nxv2bf16, MVT::nxv4bf16, MVT::nxv8bf16}) {
1661 setOperationAction(ISD::CONCAT_VECTORS, VT, Custom);
1662 setOperationAction(ISD::MLOAD, VT, Custom);
1663 setOperationAction(ISD::INSERT_SUBVECTOR, VT, Custom);
1664 setOperationAction(ISD::SPLAT_VECTOR, VT, Legal);
1665 setOperationAction(ISD::VECTOR_SPLICE, VT, Custom);
1666
1667 if (!Subtarget->isLittleEndian())
1668 setOperationAction(ISD::BITCAST, VT, Expand);
1669 }
1670
1671 setOperationAction(ISD::INTRINSIC_WO_CHAIN, MVT::i8, Custom);
1672 setOperationAction(ISD::INTRINSIC_WO_CHAIN, MVT::i16, Custom);
1673
1674 // NEON doesn't support integer divides, but SVE does
1675 for (auto VT : {MVT::v8i8, MVT::v16i8, MVT::v4i16, MVT::v8i16, MVT::v2i32,
1676 MVT::v4i32, MVT::v1i64, MVT::v2i64}) {
1677 setOperationAction(ISD::SDIV, VT, Custom);
1678 setOperationAction(ISD::UDIV, VT, Custom);
1679 }
1680
1681 // NEON doesn't support 64-bit vector integer muls, but SVE does.
1682 setOperationAction(ISD::MUL, MVT::v1i64, Custom);
1683 setOperationAction(ISD::MUL, MVT::v2i64, Custom);
1684
1685 // NOTE: Currently this has to happen after computeRegisterProperties rather
1686 // than the preferred option of combining it with the addRegisterClass call.
1687 if (Subtarget->useSVEForFixedLengthVectors()) {
1688 for (MVT VT : MVT::integer_fixedlen_vector_valuetypes()) {
1689 if (useSVEForFixedLengthVectorVT(
1690 VT, /*OverrideNEON=*/!Subtarget->isNeonAvailable()))
1691 addTypeForFixedLengthSVE(VT);
1692 }
1693 for (MVT VT : MVT::fp_fixedlen_vector_valuetypes()) {
1694 if (useSVEForFixedLengthVectorVT(
1695 VT, /*OverrideNEON=*/!Subtarget->isNeonAvailable()))
1696 addTypeForFixedLengthSVE(VT);
1697 }
1698
1699 // 64bit results can mean a bigger than NEON input.
1700 for (auto VT : {MVT::v8i8, MVT::v4i16})
1701 setOperationAction(ISD::TRUNCATE, VT, Custom);
1702 setOperationAction(ISD::FP_ROUND, MVT::v4f16, Custom);
1703
1704 // 128bit results imply a bigger than NEON input.
1705 for (auto VT : {MVT::v16i8, MVT::v8i16, MVT::v4i32})
1706 setOperationAction(ISD::TRUNCATE, VT, Custom);
1707 for (auto VT : {MVT::v8f16, MVT::v4f32})
1708 setOperationAction(ISD::FP_ROUND, VT, Custom);
1709
1710 // These operations are not supported on NEON but SVE can do them.
1711 setOperationAction(ISD::BITREVERSE, MVT::v1i64, Custom);
1712 setOperationAction(ISD::CTLZ, MVT::v1i64, Custom);
1713 setOperationAction(ISD::CTLZ, MVT::v2i64, Custom);
1714 setOperationAction(ISD::CTTZ, MVT::v1i64, Custom);
1715 setOperationAction(ISD::MULHS, MVT::v1i64, Custom);
1716 setOperationAction(ISD::MULHS, MVT::v2i64, Custom);
1717 setOperationAction(ISD::MULHU, MVT::v1i64, Custom);
1718 setOperationAction(ISD::MULHU, MVT::v2i64, Custom);
1719 setOperationAction(ISD::SMAX, MVT::v1i64, Custom);
1720 setOperationAction(ISD::SMAX, MVT::v2i64, Custom);
1721 setOperationAction(ISD::SMIN, MVT::v1i64, Custom);
1722 setOperationAction(ISD::SMIN, MVT::v2i64, Custom);
1723 setOperationAction(ISD::UMAX, MVT::v1i64, Custom);
1724 setOperationAction(ISD::UMAX, MVT::v2i64, Custom);
1725 setOperationAction(ISD::UMIN, MVT::v1i64, Custom);
1726 setOperationAction(ISD::UMIN, MVT::v2i64, Custom);
1727 setOperationAction(ISD::VECREDUCE_SMAX, MVT::v2i64, Custom);
1728 setOperationAction(ISD::VECREDUCE_SMIN, MVT::v2i64, Custom);
1729 setOperationAction(ISD::VECREDUCE_UMAX, MVT::v2i64, Custom);
1730 setOperationAction(ISD::VECREDUCE_UMIN, MVT::v2i64, Custom);
1731
1732 // Int operations with no NEON support.
1733 for (auto VT : {MVT::v8i8, MVT::v16i8, MVT::v4i16, MVT::v8i16,
1734 MVT::v2i32, MVT::v4i32, MVT::v2i64}) {
1735 setOperationAction(ISD::BITREVERSE, VT, Custom);
1736 setOperationAction(ISD::CTTZ, VT, Custom);
1737 setOperationAction(ISD::VECREDUCE_AND, VT, Custom);
1738 setOperationAction(ISD::VECREDUCE_OR, VT, Custom);
1739 setOperationAction(ISD::VECREDUCE_XOR, VT, Custom);
1740 setOperationAction(ISD::MULHS, VT, Custom);
1741 setOperationAction(ISD::MULHU, VT, Custom);
1742 }
1743
1744 // Use SVE for vectors with more than 2 elements.
1745 for (auto VT : {MVT::v4f16, MVT::v8f16, MVT::v4f32})
1746 setOperationAction(ISD::VECREDUCE_FADD, VT, Custom);
1747 }
1748
1749 setOperationPromotedToType(ISD::VECTOR_SPLICE, MVT::nxv2i1, MVT::nxv2i64);
1750 setOperationPromotedToType(ISD::VECTOR_SPLICE, MVT::nxv4i1, MVT::nxv4i32);
1751 setOperationPromotedToType(ISD::VECTOR_SPLICE, MVT::nxv8i1, MVT::nxv8i16);
1752 setOperationPromotedToType(ISD::VECTOR_SPLICE, MVT::nxv16i1, MVT::nxv16i8);
1753
1754 setOperationAction(ISD::VSCALE, MVT::i32, Custom);
1755
1756 for (auto VT : {MVT::v16i1, MVT::v8i1, MVT::v4i1, MVT::v2i1})
1757 setOperationAction(ISD::INTRINSIC_WO_CHAIN, VT, Custom);
1758 }
1759
1760 // Handle operations that are only available in non-streaming SVE mode.
1761 if (Subtarget->isSVEAvailable()) {
1762 for (auto VT : {MVT::nxv16i8, MVT::nxv8i16, MVT::nxv4i32, MVT::nxv2i64,
1763 MVT::nxv2f16, MVT::nxv4f16, MVT::nxv8f16, MVT::nxv2f32,
1764 MVT::nxv4f32, MVT::nxv2f64, MVT::nxv2bf16, MVT::nxv4bf16,
1765 MVT::nxv8bf16, MVT::v4f16, MVT::v8f16, MVT::v2f32,
1766 MVT::v4f32, MVT::v1f64, MVT::v2f64, MVT::v8i8,
1767 MVT::v16i8, MVT::v4i16, MVT::v8i16, MVT::v2i32,
1768 MVT::v4i32, MVT::v1i64, MVT::v2i64}) {
1769 setOperationAction(ISD::MGATHER, VT, Custom);
1770 setOperationAction(ISD::MSCATTER, VT, Custom);
1771 }
1772
1773 for (auto VT : {MVT::nxv2f16, MVT::nxv4f16, MVT::nxv8f16, MVT::nxv2f32,
1774 MVT::nxv4f32, MVT::nxv2f64, MVT::v4f16, MVT::v8f16,
1775 MVT::v2f32, MVT::v4f32, MVT::v2f64})
1776 setOperationAction(ISD::VECREDUCE_SEQ_FADD, VT, Custom);
1777
1778 // Histcnt is SVE2 only
1779 if (Subtarget->hasSVE2())
1780 setOperationAction(ISD::EXPERIMENTAL_VECTOR_HISTOGRAM, MVT::Other,
1781 Custom);
1782 }
1783
1784
1785 if (Subtarget->hasMOPS() && Subtarget->hasMTE()) {
1786 // Only required for llvm.aarch64.mops.memset.tag
1787 setOperationAction(ISD::INTRINSIC_W_CHAIN, MVT::i8, Custom);
1788 }
1789
1790 setOperationAction(ISD::INTRINSIC_VOID, MVT::Other, Custom);
1791
1792 if (Subtarget->hasSVE()) {
1793 setOperationAction(ISD::FLDEXP, MVT::f64, Custom);
1794 setOperationAction(ISD::FLDEXP, MVT::f32, Custom);
1795 setOperationAction(ISD::FLDEXP, MVT::f16, Custom);
1796 setOperationAction(ISD::FLDEXP, MVT::bf16, Custom);
1797 }
1798
1799 PredictableSelectIsExpensive = Subtarget->predictableSelectIsExpensive();
1800
1801 IsStrictFPEnabled = true;
1802 setMaxAtomicSizeInBitsSupported(128);
1803
1804 // On MSVC, both 32-bit and 64-bit, ldexpf(f32) is not defined. MinGW has
1805 // it, but it's just a wrapper around ldexp.
1806 if (Subtarget->isTargetWindows()) {
1807 for (ISD::NodeType Op : {ISD::FLDEXP, ISD::STRICT_FLDEXP, ISD::FFREXP})
1808 if (isOperationExpand(Op, MVT::f32))
1809 setOperationAction(Op, MVT::f32, Promote);
1810 }
1811
1812 // LegalizeDAG currently can't expand fp16 LDEXP/FREXP on targets where i16
1813 // isn't legal.
1814 for (ISD::NodeType Op : {ISD::FLDEXP, ISD::STRICT_FLDEXP, ISD::FFREXP})
1815 if (isOperationExpand(Op, MVT::f16))
1816 setOperationAction(Op, MVT::f16, Promote);
1817
1818 if (Subtarget->isWindowsArm64EC()) {
1819 // FIXME: are there intrinsics we need to exclude from this?
1820 for (int i = 0; i < RTLIB::UNKNOWN_LIBCALL; ++i) {
1821 auto code = static_cast<RTLIB::Libcall>(i);
1822 auto libcallName = getLibcallName(code);
1823 if ((libcallName != nullptr) && (libcallName[0] != '#')) {
1824 setLibcallName(code, Saver.save(Twine("#") + libcallName).data());
1825 }
1826 }
1827 }
1828}
1829
1830void AArch64TargetLowering::addTypeForNEON(MVT VT) {
1831 assert(VT.isVector() && "VT should be a vector type");
1832
1833 if (VT.isFloatingPoint()) {
1834 MVT PromoteTo = EVT(VT).changeVectorElementTypeToInteger().getSimpleVT();
1835 setOperationPromotedToType(ISD::LOAD, VT, PromoteTo);
1836 setOperationPromotedToType(ISD::STORE, VT, PromoteTo);
1837 }
1838
1839 // Mark vector float intrinsics as expand.
1840 if (VT == MVT::v2f32 || VT == MVT::v4f32 || VT == MVT::v2f64) {
1841 setOperationAction(ISD::FSIN, VT, Expand);
1842 setOperationAction(ISD::FCOS, VT, Expand);
1843 setOperationAction(ISD::FTAN, VT, Expand);
1844 setOperationAction(ISD::FASIN, VT, Expand);
1845 setOperationAction(ISD::FACOS, VT, Expand);
1846 setOperationAction(ISD::FATAN, VT, Expand);
1847 setOperationAction(ISD::FSINH, VT, Expand);
1848 setOperationAction(ISD::FCOSH, VT, Expand);
1849 setOperationAction(ISD::FTANH, VT, Expand);
1850 setOperationAction(ISD::FPOW, VT, Expand);
1851 setOperationAction(ISD::FLOG, VT, Expand);
1852 setOperationAction(ISD::FLOG2, VT, Expand);
1853 setOperationAction(ISD::FLOG10, VT, Expand);
1854 setOperationAction(ISD::FEXP, VT, Expand);
1855 setOperationAction(ISD::FEXP2, VT, Expand);
1856 setOperationAction(ISD::FEXP10, VT, Expand);
1857 }
1858
1859 // But we do support custom-lowering for FCOPYSIGN.
1860 if (VT == MVT::v2f32 || VT == MVT::v4f32 || VT == MVT::v2f64 ||
1861 ((VT == MVT::v4bf16 || VT == MVT::v8bf16 || VT == MVT::v4f16 ||
1862 VT == MVT::v8f16) &&
1863 Subtarget->hasFullFP16()))
1864 setOperationAction(ISD::FCOPYSIGN, VT, Custom);
1865
1866 setOperationAction(ISD::EXTRACT_VECTOR_ELT, VT, Custom);
1867 setOperationAction(ISD::INSERT_VECTOR_ELT, VT, Custom);
1868 setOperationAction(ISD::BUILD_VECTOR, VT, Custom);
1869 setOperationAction(ISD::ZERO_EXTEND_VECTOR_INREG, VT, Custom);
1870 setOperationAction(ISD::VECTOR_SHUFFLE, VT, Custom);
1871 setOperationAction(ISD::EXTRACT_SUBVECTOR, VT, Custom);
1872 setOperationAction(ISD::SRA, VT, Custom);
1873 setOperationAction(ISD::SRL, VT, Custom);
1874 setOperationAction(ISD::SHL, VT, Custom);
1875 setOperationAction(ISD::OR, VT, Custom);
1876 setOperationAction(ISD::SETCC, VT, Custom);
1877 setOperationAction(ISD::CONCAT_VECTORS, VT, Legal);
1878
1879 setOperationAction(ISD::SELECT, VT, Expand);
1880 setOperationAction(ISD::SELECT_CC, VT, Expand);
1881 setOperationAction(ISD::VSELECT, VT, Expand);
1882 for (MVT InnerVT : MVT::all_valuetypes())
1883 setLoadExtAction(ISD::EXTLOAD, InnerVT, VT, Expand);
1884
1885 // CNT supports only B element sizes, then use UADDLP to widen.
1886 if (VT != MVT::v8i8 && VT != MVT::v16i8)
1887 setOperationAction(ISD::CTPOP, VT, Custom);
1888
1889 setOperationAction(ISD::UDIV, VT, Expand);
1890 setOperationAction(ISD::SDIV, VT, Expand);
1891 setOperationAction(ISD::UREM, VT, Expand);
1892 setOperationAction(ISD::SREM, VT, Expand);
1893 setOperationAction(ISD::FREM, VT, Expand);
1894
1895 for (unsigned Opcode :
1896 {ISD::FP_TO_SINT, ISD::FP_TO_UINT, ISD::FP_TO_SINT_SAT,
1897 ISD::FP_TO_UINT_SAT, ISD::STRICT_FP_TO_SINT, ISD::STRICT_FP_TO_UINT})
1898 setOperationAction(Opcode, VT, Custom);
1899
1900 if (!VT.isFloatingPoint())
1901 setOperationAction(ISD::ABS, VT, Legal);
1902
1903 // [SU][MIN|MAX] are available for all NEON types apart from i64.
1904 if (!VT.isFloatingPoint() && VT != MVT::v2i64 && VT != MVT::v1i64)
1905 for (unsigned Opcode : {ISD::SMIN, ISD::SMAX, ISD::UMIN, ISD::UMAX})
1906 setOperationAction(Opcode, VT, Legal);
1907
1908 // F[MIN|MAX][NUM|NAN] and simple strict operations are available for all FP
1909 // NEON types.
1910 if (VT.isFloatingPoint() &&
1911 VT.getVectorElementType() != MVT::bf16 &&
1912 (VT.getVectorElementType() != MVT::f16 || Subtarget->hasFullFP16()))
1913 for (unsigned Opcode :
1914 {ISD::FMINIMUM, ISD::FMAXIMUM, ISD::FMINNUM, ISD::FMAXNUM,
1915 ISD::STRICT_FMINIMUM, ISD::STRICT_FMAXIMUM, ISD::STRICT_FMINNUM,
1916 ISD::STRICT_FMAXNUM, ISD::STRICT_FADD, ISD::STRICT_FSUB,
1917 ISD::STRICT_FMUL, ISD::STRICT_FDIV, ISD::STRICT_FMA,
1918 ISD::STRICT_FSQRT})
1919 setOperationAction(Opcode, VT, Legal);
1920
1921 // Strict fp extend and trunc are legal
1922 if (VT.isFloatingPoint() && VT.getScalarSizeInBits() != 16)
1923 setOperationAction(ISD::STRICT_FP_EXTEND, VT, Legal);
1924 if (VT.isFloatingPoint() && VT.getScalarSizeInBits() != 64)
1925 setOperationAction(ISD::STRICT_FP_ROUND, VT, Legal);
1926
1927 // FIXME: We could potentially make use of the vector comparison instructions
1928 // for STRICT_FSETCC and STRICT_FSETCSS, but there's a number of
1929 // complications:
1930 // * FCMPEQ/NE are quiet comparisons, the rest are signalling comparisons,
1931 // so we would need to expand when the condition code doesn't match the
1932 // kind of comparison.
1933 // * Some kinds of comparison require more than one FCMXY instruction so
1934 // would need to be expanded instead.
1935 // * The lowering of the non-strict versions involves target-specific ISD
1936 // nodes so we would likely need to add strict versions of all of them and
1937 // handle them appropriately.
1938 setOperationAction(ISD::STRICT_FSETCC, VT, Expand);
1939 setOperationAction(ISD::STRICT_FSETCCS, VT, Expand);
1940
1941 if (Subtarget->isLittleEndian()) {
1942 for (unsigned im = (unsigned)ISD::PRE_INC;
1943 im != (unsigned)ISD::LAST_INDEXED_MODE; ++im) {
1944 setIndexedLoadAction(im, VT, Legal);
1945 setIndexedStoreAction(im, VT, Legal);
1946 }
1947 }
1948
1949 if (Subtarget->hasD128()) {
1950 setOperationAction(ISD::READ_REGISTER, MVT::i128, Custom);
1951 setOperationAction(ISD::WRITE_REGISTER, MVT::i128, Custom);
1952 }
1953}
1954
1955bool AArch64TargetLowering::shouldExpandGetActiveLaneMask(EVT ResVT,
1956 EVT OpVT) const {
1957 // Only SVE has a 1:1 mapping from intrinsic -> instruction (whilelo).
1958 if (!Subtarget->hasSVE())
1959 return true;
1960
1961 // We can only support legal predicate result types. We can use the SVE
1962 // whilelo instruction for generating fixed-width predicates too.
1963 if (ResVT != MVT::nxv2i1 && ResVT != MVT::nxv4i1 && ResVT != MVT::nxv8i1 &&
1964 ResVT != MVT::nxv16i1 && ResVT != MVT::v2i1 && ResVT != MVT::v4i1 &&
1965 ResVT != MVT::v8i1 && ResVT != MVT::v16i1)
1966 return true;
1967
1968 // The whilelo instruction only works with i32 or i64 scalar inputs.
1969 if (OpVT != MVT::i32 && OpVT != MVT::i64)
1970 return true;
1971
1972 return false;
1973}
1974
1975bool AArch64TargetLowering::shouldExpandCttzElements(EVT VT) const {
1976 if (!Subtarget->isSVEorStreamingSVEAvailable())
1977 return true;
1978
1979 // We can only use the BRKB + CNTP sequence with legal predicate types. We can
1980 // also support fixed-width predicates.
1981 return VT != MVT::nxv16i1 && VT != MVT::nxv8i1 && VT != MVT::nxv4i1 &&
1982 VT != MVT::nxv2i1 && VT != MVT::v16i1 && VT != MVT::v8i1 &&
1983 VT != MVT::v4i1 && VT != MVT::v2i1;
1984}
1985
1986void AArch64TargetLowering::addTypeForFixedLengthSVE(MVT VT) {
1987 assert(VT.isFixedLengthVector() && "Expected fixed length vector type!");
1988
1989 // By default everything must be expanded.
1990 for (unsigned Op = 0; Op < ISD::BUILTIN_OP_END; ++Op)
1991 setOperationAction(Op, VT, Expand);
1992
1993 if (VT.isFloatingPoint()) {
1994 setCondCodeAction(ISD::SETO, VT, Expand);
1995 setCondCodeAction(ISD::SETOLT, VT, Expand);
1996 setCondCodeAction(ISD::SETOLE, VT, Expand);
1997 setCondCodeAction(ISD::SETULT, VT, Expand);
1998 setCondCodeAction(ISD::SETULE, VT, Expand);
1999 setCondCodeAction(ISD::SETUGE, VT, Expand);
2000 setCondCodeAction(ISD::SETUGT, VT, Expand);
2001 setCondCodeAction(ISD::SETUEQ, VT, Expand);
2002 setCondCodeAction(ISD::SETONE, VT, Expand);
2003 }
2004
2005 TargetLoweringBase::LegalizeAction Default =
2006 VT == MVT::v1f64 ? Expand : Custom;
2007
2008 // Mark integer truncating stores/extending loads as having custom lowering
2009 if (VT.isInteger()) {
2010 MVT InnerVT = VT.changeVectorElementType(MVT::i8);
2011 while (InnerVT != VT) {
2012 setTruncStoreAction(VT, InnerVT, Default);
2013 setLoadExtAction(ISD::ZEXTLOAD, VT, InnerVT, Default);
2014 setLoadExtAction(ISD::SEXTLOAD, VT, InnerVT, Default);
2015 setLoadExtAction(ISD::EXTLOAD, VT, InnerVT, Default);
2016 InnerVT = InnerVT.changeVectorElementType(
2017 MVT::getIntegerVT(2 * InnerVT.getScalarSizeInBits()));
2018 }
2019 }
2020
2021 // Mark floating-point truncating stores/extending loads as having custom
2022 // lowering
2023 if (VT.isFloatingPoint()) {
2024 MVT InnerVT = VT.changeVectorElementType(MVT::f16);
2025 while (InnerVT != VT) {
2026 setTruncStoreAction(VT, InnerVT, Custom);
2027 setLoadExtAction(ISD::EXTLOAD, VT, InnerVT, Default);
2028 InnerVT = InnerVT.changeVectorElementType(
2029 MVT::getFloatingPointVT(2 * InnerVT.getScalarSizeInBits()));
2030 }
2031 }
2032
2033 bool PreferNEON = VT.is64BitVector() || VT.is128BitVector();
2034 bool PreferSVE = !PreferNEON && Subtarget->isSVEAvailable();
2035
2036 // Lower fixed length vector operations to scalable equivalents.
2037 setOperationAction(ISD::ABS, VT, Default);
2038 setOperationAction(ISD::ADD, VT, Default);
2039 setOperationAction(ISD::AND, VT, Default);
2040 setOperationAction(ISD::ANY_EXTEND, VT, Default);
2041 setOperationAction(ISD::BITCAST, VT, PreferNEON ? Legal : Default);
2042 setOperationAction(ISD::BITREVERSE, VT, Default);
2043 setOperationAction(ISD::BSWAP, VT, Default);
2044 setOperationAction(ISD::BUILD_VECTOR, VT, Default);
2045 setOperationAction(ISD::CONCAT_VECTORS, VT, Default);
2046 setOperationAction(ISD::CTLZ, VT, Default);
2047 setOperationAction(ISD::CTPOP, VT, Default);
2048 setOperationAction(ISD::CTTZ, VT, Default);
2049 setOperationAction(ISD::EXTRACT_SUBVECTOR, VT, Default);
2050 setOperationAction(ISD::EXTRACT_VECTOR_ELT, VT, Default);
2051 setOperationAction(ISD::FABS, VT, Default);
2052 setOperationAction(ISD::FADD, VT, Default);
2053 setOperationAction(ISD::FCEIL, VT, Default);
2054 setOperationAction(ISD::FCOPYSIGN, VT, Default);
2055 setOperationAction(ISD::FDIV, VT, Default);
2056 setOperationAction(ISD::FFLOOR, VT, Default);
2057 setOperationAction(ISD::FMA, VT, Default);
2058 setOperationAction(ISD::FMAXIMUM, VT, Default);
2059 setOperationAction(ISD::FMAXNUM, VT, Default);
2060 setOperationAction(ISD::FMINIMUM, VT, Default);
2061 setOperationAction(ISD::FMINNUM, VT, Default);
2062 setOperationAction(ISD::FMUL, VT, Default);
2063 setOperationAction(ISD::FNEARBYINT, VT, Default);
2064 setOperationAction(ISD::FNEG, VT, Default);
2065 setOperationAction(ISD::FP_EXTEND, VT, Default);
2066 setOperationAction(ISD::FP_ROUND, VT, Default);
2067 setOperationAction(ISD::FP_TO_SINT, VT, Default);
2068 setOperationAction(ISD::FP_TO_UINT, VT, Default);
2069 setOperationAction(ISD::FRINT, VT, Default);
2070 setOperationAction(ISD::LRINT, VT, Default);
2071 setOperationAction(ISD::LLRINT, VT, Default);
2072 setOperationAction(ISD::FROUND, VT, Default);
2073 setOperationAction(ISD::FROUNDEVEN, VT, Default);
2074 setOperationAction(ISD::FSQRT, VT, Default);
2075 setOperationAction(ISD::FSUB, VT, Default);
2076 setOperationAction(ISD::FTRUNC, VT, Default);
2077 setOperationAction(ISD::INSERT_VECTOR_ELT, VT, Default);
2078 setOperationAction(ISD::LOAD, VT, PreferNEON ? Legal : Default);
2079 setOperationAction(ISD::MGATHER, VT, PreferSVE ? Default : Expand);
2080 setOperationAction(ISD::MLOAD, VT, Default);
2081 setOperationAction(ISD::MSCATTER, VT, PreferSVE ? Default : Expand);
2082 setOperationAction(ISD::MSTORE, VT, Default);
2083 setOperationAction(ISD::MUL, VT, Default);
2084 setOperationAction(ISD::MULHS, VT, Default);
2085 setOperationAction(ISD::MULHU, VT, Default);
2086 setOperationAction(ISD::OR, VT, Default);
2087 setOperationAction(ISD::SCALAR_TO_VECTOR, VT, PreferNEON ? Legal : Expand);
2088 setOperationAction(ISD::SDIV, VT, Default);
2089 setOperationAction(ISD::SELECT, VT, Default);
2090 setOperationAction(ISD::SETCC, VT, Default);
2091 setOperationAction(ISD::SHL, VT, Default);
2092 setOperationAction(ISD::SIGN_EXTEND, VT, Default);
2093 setOperationAction(ISD::SIGN_EXTEND_INREG, VT, Default);
2094 setOperationAction(ISD::SINT_TO_FP, VT, Default);
2095 setOperationAction(ISD::SMAX, VT, Default);
2096 setOperationAction(ISD::SMIN, VT, Default);
2097 setOperationAction(ISD::SPLAT_VECTOR, VT, Default);
2098 setOperationAction(ISD::SRA, VT, Default);
2099 setOperationAction(ISD::SRL, VT, Default);
2100 setOperationAction(ISD::STORE, VT, PreferNEON ? Legal : Default);
2101 setOperationAction(ISD::SUB, VT, Default);
2102 setOperationAction(ISD::TRUNCATE, VT, Default);
2103 setOperationAction(ISD::UDIV, VT, Default);
2104 setOperationAction(ISD::UINT_TO_FP, VT, Default);
2105 setOperationAction(ISD::UMAX, VT, Default);
2106 setOperationAction(ISD::UMIN, VT, Default);
2107 setOperationAction(ISD::VECREDUCE_ADD, VT, Default);
2108 setOperationAction(ISD::VECREDUCE_AND, VT, Default);
2109 setOperationAction(ISD::VECREDUCE_FADD, VT, Default);
2110 setOperationAction(ISD::VECREDUCE_FMAX, VT, Default);
2111 setOperationAction(ISD::VECREDUCE_FMIN, VT, Default);
2112 setOperationAction(ISD::VECREDUCE_FMAXIMUM, VT, Default);
2113 setOperationAction(ISD::VECREDUCE_FMINIMUM, VT, Default);
2114 setOperationAction(ISD::VECREDUCE_OR, VT, Default);
2115 setOperationAction(ISD::VECREDUCE_SEQ_FADD, VT, PreferSVE ? Default : Expand);
2116 setOperationAction(ISD::VECREDUCE_SMAX, VT, Default);
2117 setOperationAction(ISD::VECREDUCE_SMIN, VT, Default);
2118 setOperationAction(ISD::VECREDUCE_UMAX, VT, Default);
2119 setOperationAction(ISD::VECREDUCE_UMIN, VT, Default);
2120 setOperationAction(ISD::VECREDUCE_XOR, VT, Default);
2121 setOperationAction(ISD::VECTOR_SHUFFLE, VT, Default);
2122 setOperationAction(ISD::VECTOR_SPLICE, VT, Default);
2123 setOperationAction(ISD::VSELECT, VT, Default);
2124 setOperationAction(ISD::XOR, VT, Default);
2125 setOperationAction(ISD::ZERO_EXTEND, VT, Default);
2126}
2127
2128void AArch64TargetLowering::addDRType(MVT VT) {
2129 addRegisterClass(VT, &AArch64::FPR64RegClass);
2130 if (Subtarget->isNeonAvailable())
2131 addTypeForNEON(VT);
2132}
2133
2134void AArch64TargetLowering::addQRType(MVT VT) {
2135 addRegisterClass(VT, &AArch64::FPR128RegClass);
2136 if (Subtarget->isNeonAvailable())
2137 addTypeForNEON(VT);
2138}
2139
2140EVT AArch64TargetLowering::getSetCCResultType(const DataLayout &,
2141 LLVMContext &C, EVT VT) const {
2142 if (!VT.isVector())
2143 return MVT::i32;
2144 if (VT.isScalableVector())
2145 return EVT::getVectorVT(C, MVT::i1, VT.getVectorElementCount());
2146 return VT.changeVectorElementTypeToInteger();
2147}
2148
2149// isIntImmediate - This method tests to see if the node is a constant
2150// operand. If so Imm will receive the value.
2151static bool isIntImmediate(const SDNode *N, uint64_t &Imm) {
2152 if (const ConstantSDNode *C = dyn_cast<const ConstantSDNode>(N)) {
2153 Imm = C->getZExtValue();
2154 return true;
2155 }
2156 return false;
2157}
2158
2159// isOpcWithIntImmediate - This method tests to see if the node is a specific
2160// opcode and that it has a immediate integer right operand.
2161// If so Imm will receive the value.
2162static bool isOpcWithIntImmediate(const SDNode *N, unsigned Opc,
2163 uint64_t &Imm) {
2164 return N->getOpcode() == Opc &&
2165 isIntImmediate(N->getOperand(1).getNode(), Imm);
2166}
2167
2168static bool optimizeLogicalImm(SDValue Op, unsigned Size, uint64_t Imm,
2169 const APInt &Demanded,
2170 TargetLowering::TargetLoweringOpt &TLO,
2171 unsigned NewOpc) {
2172 uint64_t OldImm = Imm, NewImm, Enc;
2173 uint64_t Mask = ((uint64_t)(-1LL) >> (64 - Size)), OrigMask = Mask;
2174
2175 // Return if the immediate is already all zeros, all ones, a bimm32 or a
2176 // bimm64.
2177 if (Imm == 0 || Imm == Mask ||
2178 AArch64_AM::isLogicalImmediate(Imm & Mask, Size))
2179 return false;
2180
2181 unsigned EltSize = Size;
2182 uint64_t DemandedBits = Demanded.getZExtValue();
2183
2184 // Clear bits that are not demanded.
2185 Imm &= DemandedBits;
2186
2187 while (true) {
2188 // The goal here is to set the non-demanded bits in a way that minimizes
2189 // the number of switching between 0 and 1. In order to achieve this goal,
2190 // we set the non-demanded bits to the value of the preceding demanded bits.
2191 // For example, if we have an immediate 0bx10xx0x1 ('x' indicates a
2192 // non-demanded bit), we copy bit0 (1) to the least significant 'x',
2193 // bit2 (0) to 'xx', and bit6 (1) to the most significant 'x'.
2194 // The final result is 0b11000011.
2195 uint64_t NonDemandedBits = ~DemandedBits;
2196 uint64_t InvertedImm = ~Imm & DemandedBits;
2197 uint64_t RotatedImm =
2198 ((InvertedImm << 1) | (InvertedImm >> (EltSize - 1) & 1)) &
2199 NonDemandedBits;
2200 uint64_t Sum = RotatedImm + NonDemandedBits;
2201 bool Carry = NonDemandedBits & ~Sum & (1ULL << (EltSize - 1));
2202 uint64_t Ones = (Sum + Carry) & NonDemandedBits;
2203 NewImm = (Imm | Ones) & Mask;
2204
2205 // If NewImm or its bitwise NOT is a shifted mask, it is a bitmask immediate
2206 // or all-ones or all-zeros, in which case we can stop searching. Otherwise,
2207 // we halve the element size and continue the search.
2208 if (isShiftedMask_64(NewImm) || isShiftedMask_64(~(NewImm | ~Mask)))
2209 break;
2210
2211 // We cannot shrink the element size any further if it is 2-bits.
2212 if (EltSize == 2)
2213 return false;
2214
2215 EltSize /= 2;
2216 Mask >>= EltSize;
2217 uint64_t Hi = Imm >> EltSize, DemandedBitsHi = DemandedBits >> EltSize;
2218
2219 // Return if there is mismatch in any of the demanded bits of Imm and Hi.
2220 if (((Imm ^ Hi) & (DemandedBits & DemandedBitsHi) & Mask) != 0)
2221 return false;
2222
2223 // Merge the upper and lower halves of Imm and DemandedBits.
2224 Imm |= Hi;
2225 DemandedBits |= DemandedBitsHi;
2226 }
2227
2228 ++NumOptimizedImms;
2229
2230 // Replicate the element across the register width.
2231 while (EltSize < Size) {
2232 NewImm |= NewImm << EltSize;
2233 EltSize *= 2;
2234 }
2235
2236 (void)OldImm;
2237 assert(((OldImm ^ NewImm) & Demanded.getZExtValue()) == 0 &&
2238 "demanded bits should never be altered");
2239 assert(OldImm != NewImm && "the new imm shouldn't be equal to the old imm");
2240
2241 // Create the new constant immediate node.
2242 EVT VT = Op.getValueType();
2243 SDLoc DL(Op);
2244 SDValue New;
2245
2246 // If the new constant immediate is all-zeros or all-ones, let the target
2247 // independent DAG combine optimize this node.
2248 if (NewImm == 0 || NewImm == OrigMask) {
2249 New = TLO.DAG.getNode(Op.getOpcode(), DL, VT, Op.getOperand(0),
2250 TLO.DAG.getConstant(NewImm, DL, VT));
2251 // Otherwise, create a machine node so that target independent DAG combine
2252 // doesn't undo this optimization.
2253 } else {
2254 Enc = AArch64_AM::encodeLogicalImmediate(NewImm, Size);
2255 SDValue EncConst = TLO.DAG.getTargetConstant(Enc, DL, VT);
2256 New = SDValue(
2257 TLO.DAG.getMachineNode(NewOpc, DL, VT, Op.getOperand(0), EncConst), 0);
2258 }
2259
2260 return TLO.CombineTo(Op, New);
2261}
2262
2263bool AArch64TargetLowering::targetShrinkDemandedConstant(
2264 SDValue Op, const APInt &DemandedBits, const APInt &DemandedElts,
2265 TargetLoweringOpt &TLO) const {
2266 // Delay this optimization to as late as possible.
2267 if (!TLO.LegalOps)
2268 return false;
2269
2270 if (!EnableOptimizeLogicalImm)
2271 return false;
2272
2273 EVT VT = Op.getValueType();
2274 if (VT.isVector())
2275 return false;
2276
2277 unsigned Size = VT.getSizeInBits();
2278 assert((Size == 32 || Size == 64) &&
2279 "i32 or i64 is expected after legalization.");
2280
2281 // Exit early if we demand all bits.
2282 if (DemandedBits.popcount() == Size)
2283 return false;
2284
2285 unsigned NewOpc;
2286 switch (Op.getOpcode()) {
2287 default:
2288 return false;
2289 case ISD::AND:
2290 NewOpc = Size == 32 ? AArch64::ANDWri : AArch64::ANDXri;
2291 break;
2292 case ISD::OR:
2293 NewOpc = Size == 32 ? AArch64::ORRWri : AArch64::ORRXri;
2294 break;
2295 case ISD::XOR:
2296 NewOpc = Size == 32 ? AArch64::EORWri : AArch64::EORXri;
2297 break;
2298 }
2299 ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op.getOperand(1));
2300 if (!C)
2301 return false;
2302 uint64_t Imm = C->getZExtValue();
2303 return optimizeLogicalImm(Op, Size, Imm, DemandedBits, TLO, NewOpc);
2304}
2305
2306/// computeKnownBitsForTargetNode - Determine which of the bits specified in
2307/// Mask are known to be either zero or one and return them Known.
2308void AArch64TargetLowering::computeKnownBitsForTargetNode(
2309 const SDValue Op, KnownBits &Known, const APInt &DemandedElts,
2310 const SelectionDAG &DAG, unsigned Depth) const {
2311 switch (Op.getOpcode()) {
2312 default:
2313 break;
2314 case AArch64ISD::DUP: {
2315 SDValue SrcOp = Op.getOperand(0);
2316 Known = DAG.computeKnownBits(SrcOp, Depth + 1);
2317 if (SrcOp.getValueSizeInBits() != Op.getScalarValueSizeInBits()) {
2318 assert(SrcOp.getValueSizeInBits() > Op.getScalarValueSizeInBits() &&
2319 "Expected DUP implicit truncation");
2320 Known = Known.trunc(Op.getScalarValueSizeInBits());
2321 }
2322 break;
2323 }
2324 case AArch64ISD::CSEL: {
2325 KnownBits Known2;
2326 Known = DAG.computeKnownBits(Op->getOperand(0), Depth + 1);
2327 Known2 = DAG.computeKnownBits(Op->getOperand(1), Depth + 1);
2328 Known = Known.intersectWith(Known2);
2329 break;
2330 }
2331 case AArch64ISD::BICi: {
2332 // Compute the bit cleared value.
2333 uint64_t Mask =
2334 ~(Op->getConstantOperandVal(1) << Op->getConstantOperandVal(2));
2335 Known = DAG.computeKnownBits(Op->getOperand(0), Depth + 1);
2336 Known &= KnownBits::makeConstant(APInt(Known.getBitWidth(), Mask));
2337 break;
2338 }
2339 case AArch64ISD::VLSHR: {
2340 KnownBits Known2;
2341 Known = DAG.computeKnownBits(Op->getOperand(0), Depth + 1);
2342 Known2 = DAG.computeKnownBits(Op->getOperand(1), Depth + 1);
2343 Known = KnownBits::lshr(Known, Known2);
2344 break;
2345 }
2346 case AArch64ISD::VASHR: {
2347 KnownBits Known2;
2348 Known = DAG.computeKnownBits(Op->getOperand(0), Depth + 1);
2349 Known2 = DAG.computeKnownBits(Op->getOperand(1), Depth + 1);
2350 Known = KnownBits::ashr(Known, Known2);
2351 break;
2352 }
2353 case AArch64ISD::VSHL: {
2354 KnownBits Known2;
2355 Known = DAG.computeKnownBits(Op->getOperand(0), Depth + 1);
2356 Known2 = DAG.computeKnownBits(Op->getOperand(1), Depth + 1);
2357 Known = KnownBits::shl(Known, Known2);
2358 break;
2359 }
2360 case AArch64ISD::MOVI: {
2361 Known = KnownBits::makeConstant(
2362 APInt(Known.getBitWidth(), Op->getConstantOperandVal(0)));
2363 break;
2364 }
2365 case AArch64ISD::LOADgot:
2366 case AArch64ISD::ADDlow: {
2367 if (!Subtarget->isTargetILP32())
2368 break;
2369 // In ILP32 mode all valid pointers are in the low 4GB of the address-space.
2370 Known.Zero = APInt::getHighBitsSet(64, 32);
2371 break;
2372 }
2373 case AArch64ISD::ASSERT_ZEXT_BOOL: {
2374 Known = DAG.computeKnownBits(Op->getOperand(0), Depth + 1);
2375 Known.Zero |= APInt(Known.getBitWidth(), 0xFE);
2376 break;
2377 }
2378 case ISD::INTRINSIC_W_CHAIN: {
2379 Intrinsic::ID IntID =
2380 static_cast<Intrinsic::ID>(Op->getConstantOperandVal(1));
2381 switch (IntID) {
2382 default: return;
2383 case Intrinsic::aarch64_ldaxr:
2384 case Intrinsic::aarch64_ldxr: {
2385 unsigned BitWidth = Known.getBitWidth();
2386 EVT VT = cast<MemIntrinsicSDNode>(Op)->getMemoryVT();
2387 unsigned MemBits = VT.getScalarSizeInBits();
2388 Known.Zero |= APInt::getHighBitsSet(BitWidth, BitWidth - MemBits);
2389 return;
2390 }
2391 }
2392 break;
2393 }
2394 case ISD::INTRINSIC_WO_CHAIN:
2395 case ISD::INTRINSIC_VOID: {
2396 unsigned IntNo = Op.getConstantOperandVal(0);
2397 switch (IntNo) {
2398 default:
2399 break;
2400 case Intrinsic::aarch64_neon_uaddlv: {
2401 MVT VT = Op.getOperand(1).getValueType().getSimpleVT();
2402 unsigned BitWidth = Known.getBitWidth();
2403 if (VT == MVT::v8i8 || VT == MVT::v16i8) {
2404 unsigned Bound = (VT == MVT::v8i8) ? 11 : 12;
2405 assert(BitWidth >= Bound && "Unexpected width!");
2406 APInt Mask = APInt::getHighBitsSet(BitWidth, BitWidth - Bound);
2407 Known.Zero |= Mask;
2408 }
2409 break;
2410 }
2411 case Intrinsic::aarch64_neon_umaxv:
2412 case Intrinsic::aarch64_neon_uminv: {
2413 // Figure out the datatype of the vector operand. The UMINV instruction
2414 // will zero extend the result, so we can mark as known zero all the
2415 // bits larger than the element datatype. 32-bit or larget doesn't need
2416 // this as those are legal types and will be handled by isel directly.
2417 MVT VT = Op.getOperand(1).getValueType().getSimpleVT();
2418 unsigned BitWidth = Known.getBitWidth();
2419 if (VT == MVT::v8i8 || VT == MVT::v16i8) {
2420 assert(BitWidth >= 8 && "Unexpected width!");
2421 APInt Mask = APInt::getHighBitsSet(BitWidth, BitWidth - 8);
2422 Known.Zero |= Mask;
2423 } else if (VT == MVT::v4i16 || VT == MVT::v8i16) {
2424 assert(BitWidth >= 16 && "Unexpected width!");
2425 APInt Mask = APInt::getHighBitsSet(BitWidth, BitWidth - 16);
2426 Known.Zero |= Mask;
2427 }
2428 break;
2429 } break;
2430 }
2431 }
2432 }
2433}
2434
2435unsigned AArch64TargetLowering::ComputeNumSignBitsForTargetNode(
2436 SDValue Op, const APInt &DemandedElts, const SelectionDAG &DAG,
2437 unsigned Depth) const {
2438 EVT VT = Op.getValueType();
2439 unsigned VTBits = VT.getScalarSizeInBits();
2440 unsigned Opcode = Op.getOpcode();
2441 switch (Opcode) {
2442 case AArch64ISD::CMEQ:
2443 case AArch64ISD::CMGE:
2444 case AArch64ISD::CMGT:
2445 case AArch64ISD::CMHI:
2446 case AArch64ISD::CMHS:
2447 case AArch64ISD::FCMEQ:
2448 case AArch64ISD::FCMGE:
2449 case AArch64ISD::FCMGT:
2450 case AArch64ISD::CMEQz:
2451 case AArch64ISD::CMGEz:
2452 case AArch64ISD::CMGTz:
2453 case AArch64ISD::CMLEz:
2454 case AArch64ISD::CMLTz:
2455 case AArch64ISD::FCMEQz:
2456 case AArch64ISD::FCMGEz:
2457 case AArch64ISD::FCMGTz:
2458 case AArch64ISD::FCMLEz:
2459 case AArch64ISD::FCMLTz:
2460 // Compares return either 0 or all-ones
2461 return VTBits;
2462 }
2463
2464 return 1;
2465}
2466
2467MVT AArch64TargetLowering::getScalarShiftAmountTy(const DataLayout &DL,
2468 EVT) const {
2469 return MVT::i64;
2470}
2471
2472bool AArch64TargetLowering::allowsMisalignedMemoryAccesses(
2473 EVT VT, unsigned AddrSpace, Align Alignment, MachineMemOperand::Flags Flags,
2474 unsigned *Fast) const {
2475 if (Subtarget->requiresStrictAlign())
2476 return false;
2477
2478 if (Fast) {
2479 // Some CPUs are fine with unaligned stores except for 128-bit ones.
2480 *Fast = !Subtarget->isMisaligned128StoreSlow() || VT.getStoreSize() != 16 ||
2481 // See comments in performSTORECombine() for more details about
2482 // these conditions.
2483
2484 // Code that uses clang vector extensions can mark that it
2485 // wants unaligned accesses to be treated as fast by
2486 // underspecifying alignment to be 1 or 2.
2487 Alignment <= 2 ||
2488
2489 // Disregard v2i64. Memcpy lowering produces those and splitting
2490 // them regresses performance on micro-benchmarks and olden/bh.
2491 VT == MVT::v2i64;
2492 }
2493 return true;
2494}
2495
2496// Same as above but handling LLTs instead.
2497bool AArch64TargetLowering::allowsMisalignedMemoryAccesses(
2498 LLT Ty, unsigned AddrSpace, Align Alignment, MachineMemOperand::Flags Flags,
2499 unsigned *Fast) const {
2500 if (Subtarget->requiresStrictAlign())
2501 return false;
2502
2503 if (Fast) {
2504 // Some CPUs are fine with unaligned stores except for 128-bit ones.
2505 *Fast = !Subtarget->isMisaligned128StoreSlow() ||
2506 Ty.getSizeInBytes() != 16 ||
2507 // See comments in performSTORECombine() for more details about
2508 // these conditions.
2509
2510 // Code that uses clang vector extensions can mark that it
2511 // wants unaligned accesses to be treated as fast by
2512 // underspecifying alignment to be 1 or 2.
2513 Alignment <= 2 ||
2514
2515 // Disregard v2i64. Memcpy lowering produces those and splitting
2516 // them regresses performance on micro-benchmarks and olden/bh.
2517 Ty == LLT::fixed_vector(2, 64);
2518 }
2519 return true;
2520}
2521
2522FastISel *
2523AArch64TargetLowering::createFastISel(FunctionLoweringInfo &funcInfo,
2524 const TargetLibraryInfo *libInfo) const {
2525 return AArch64::createFastISel(funcInfo, libInfo);
2526}
2527
2528const char *AArch64TargetLowering::getTargetNodeName(unsigned Opcode) const {
2529#define MAKE_CASE(V) \
2530 case V: \
2531 return #V;
2532 switch ((AArch64ISD::NodeType)Opcode) {
2533 case AArch64ISD::FIRST_NUMBER:
2534 break;
2535 MAKE_CASE(AArch64ISD::ALLOCATE_ZA_BUFFER)
2536 MAKE_CASE(AArch64ISD::INIT_TPIDR2OBJ)
2537 MAKE_CASE(AArch64ISD::COALESCER_BARRIER)
2538 MAKE_CASE(AArch64ISD::VG_SAVE)
2539 MAKE_CASE(AArch64ISD::VG_RESTORE)
2540 MAKE_CASE(AArch64ISD::SMSTART)
2541 MAKE_CASE(AArch64ISD::SMSTOP)
2542 MAKE_CASE(AArch64ISD::RESTORE_ZA)
2543 MAKE_CASE(AArch64ISD::RESTORE_ZT)
2544 MAKE_CASE(AArch64ISD::SAVE_ZT)
2545 MAKE_CASE(AArch64ISD::CALL)
2546 MAKE_CASE(AArch64ISD::ADRP)
2547 MAKE_CASE(AArch64ISD::ADR)
2548 MAKE_CASE(AArch64ISD::ADDlow)
2549 MAKE_CASE(AArch64ISD::AUTH_CALL)
2550 MAKE_CASE(AArch64ISD::AUTH_TC_RETURN)
2551 MAKE_CASE(AArch64ISD::AUTH_CALL_RVMARKER)
2552 MAKE_CASE(AArch64ISD::LOADgot)
2553 MAKE_CASE(AArch64ISD::RET_GLUE)
2554 MAKE_CASE(AArch64ISD::BRCOND)
2555 MAKE_CASE(AArch64ISD::CSEL)
2556 MAKE_CASE(AArch64ISD::CSINV)
2557 MAKE_CASE(AArch64ISD::CSNEG)
2558 MAKE_CASE(AArch64ISD::CSINC)
2559 MAKE_CASE(AArch64ISD::THREAD_POINTER)
2560 MAKE_CASE(AArch64ISD::TLSDESC_CALLSEQ)
2561 MAKE_CASE(AArch64ISD::PROBED_ALLOCA)
2562 MAKE_CASE(AArch64ISD::ABDS_PRED)
2563 MAKE_CASE(AArch64ISD::ABDU_PRED)
2564 MAKE_CASE(AArch64ISD::HADDS_PRED)
2565 MAKE_CASE(AArch64ISD::HADDU_PRED)
2566 MAKE_CASE(AArch64ISD::MUL_PRED)
2567 MAKE_CASE(AArch64ISD::MULHS_PRED)
2568 MAKE_CASE(AArch64ISD::MULHU_PRED)
2569 MAKE_CASE(AArch64ISD::RHADDS_PRED)
2570 MAKE_CASE(AArch64ISD::RHADDU_PRED)
2571 MAKE_CASE(AArch64ISD::SDIV_PRED)
2572 MAKE_CASE(AArch64ISD::SHL_PRED)
2573 MAKE_CASE(AArch64ISD::SMAX_PRED)
2574 MAKE_CASE(AArch64ISD::SMIN_PRED)
2575 MAKE_CASE(AArch64ISD::SRA_PRED)
2576 MAKE_CASE(AArch64ISD::SRL_PRED)
2577 MAKE_CASE(AArch64ISD::UDIV_PRED)
2578 MAKE_CASE(AArch64ISD::UMAX_PRED)
2579 MAKE_CASE(AArch64ISD::UMIN_PRED)
2580 MAKE_CASE(AArch64ISD::SRAD_MERGE_OP1)
2581 MAKE_CASE(AArch64ISD::FNEG_MERGE_PASSTHRU)
2582 MAKE_CASE(AArch64ISD::SIGN_EXTEND_INREG_MERGE_PASSTHRU)
2583 MAKE_CASE(AArch64ISD::ZERO_EXTEND_INREG_MERGE_PASSTHRU)
2584 MAKE_CASE(AArch64ISD::FCEIL_MERGE_PASSTHRU)
2585 MAKE_CASE(AArch64ISD::FFLOOR_MERGE_PASSTHRU)
2586 MAKE_CASE(AArch64ISD::FNEARBYINT_MERGE_PASSTHRU)
2587 MAKE_CASE(AArch64ISD::FRINT_MERGE_PASSTHRU)
2588 MAKE_CASE(AArch64ISD::FROUND_MERGE_PASSTHRU)
2589 MAKE_CASE(AArch64ISD::FROUNDEVEN_MERGE_PASSTHRU)
2590 MAKE_CASE(AArch64ISD::FTRUNC_MERGE_PASSTHRU)
2591 MAKE_CASE(AArch64ISD::FP_ROUND_MERGE_PASSTHRU)
2592 MAKE_CASE(AArch64ISD::FP_EXTEND_MERGE_PASSTHRU)
2593 MAKE_CASE(AArch64ISD::SINT_TO_FP_MERGE_PASSTHRU)
2594 MAKE_CASE(AArch64ISD::UINT_TO_FP_MERGE_PASSTHRU)
2595 MAKE_CASE(AArch64ISD::FCVTZU_MERGE_PASSTHRU)
2596 MAKE_CASE(AArch64ISD::FCVTZS_MERGE_PASSTHRU)
2597 MAKE_CASE(AArch64ISD::FSQRT_MERGE_PASSTHRU)
2598 MAKE_CASE(AArch64ISD::FRECPX_MERGE_PASSTHRU)
2599 MAKE_CASE(AArch64ISD::FABS_MERGE_PASSTHRU)
2600 MAKE_CASE(AArch64ISD::ABS_MERGE_PASSTHRU)
2601 MAKE_CASE(AArch64ISD::NEG_MERGE_PASSTHRU)
2602 MAKE_CASE(AArch64ISD::SETCC_MERGE_ZERO)
2603 MAKE_CASE(AArch64ISD::ADC)
2604 MAKE_CASE(AArch64ISD::SBC)
2605 MAKE_CASE(AArch64ISD::ADDS)
2606 MAKE_CASE(AArch64ISD::SUBS)
2607 MAKE_CASE(AArch64ISD::ADCS)
2608 MAKE_CASE(AArch64ISD::SBCS)
2609 MAKE_CASE(AArch64ISD::ANDS)
2610 MAKE_CASE(AArch64ISD::CCMP)
2611 MAKE_CASE(AArch64ISD::CCMN)
2612 MAKE_CASE(AArch64ISD::FCCMP)
2613 MAKE_CASE(AArch64ISD::FCMP)
2614 MAKE_CASE(AArch64ISD::STRICT_FCMP)
2615 MAKE_CASE(AArch64ISD::STRICT_FCMPE)
2616 MAKE_CASE(AArch64ISD::FCVTXN)
2617 MAKE_CASE(AArch64ISD::SME_ZA_LDR)
2618 MAKE_CASE(AArch64ISD::SME_ZA_STR)
2619 MAKE_CASE(AArch64ISD::DUP)
2620 MAKE_CASE(AArch64ISD::DUPLANE8)
2621 MAKE_CASE(AArch64ISD::DUPLANE16)
2622 MAKE_CASE(AArch64ISD::DUPLANE32)
2623 MAKE_CASE(AArch64ISD::DUPLANE64)
2624 MAKE_CASE(AArch64ISD::DUPLANE128)
2625 MAKE_CASE(AArch64ISD::MOVI)
2626 MAKE_CASE(AArch64ISD::MOVIshift)
2627 MAKE_CASE(AArch64ISD::MOVIedit)
2628 MAKE_CASE(AArch64ISD::MOVImsl)
2629 MAKE_CASE(AArch64ISD::FMOV)
2630 MAKE_CASE(AArch64ISD::MVNIshift)
2631 MAKE_CASE(AArch64ISD::MVNImsl)
2632 MAKE_CASE(AArch64ISD::BICi)
2633 MAKE_CASE(AArch64ISD::ORRi)
2634 MAKE_CASE(AArch64ISD::BSP)
2635 MAKE_CASE(AArch64ISD::ZIP1)
2636 MAKE_CASE(AArch64ISD::ZIP2)
2637 MAKE_CASE(AArch64ISD::UZP1)
2638 MAKE_CASE(AArch64ISD::UZP2)
2639 MAKE_CASE(AArch64ISD::TRN1)
2640 MAKE_CASE(AArch64ISD::TRN2)
2641 MAKE_CASE(AArch64ISD::REV16)
2642 MAKE_CASE(AArch64ISD::REV32)
2643 MAKE_CASE(AArch64ISD::REV64)
2644 MAKE_CASE(AArch64ISD::EXT)
2645 MAKE_CASE(AArch64ISD::SPLICE)
2646 MAKE_CASE(AArch64ISD::VSHL)
2647 MAKE_CASE(AArch64ISD::VLSHR)
2648 MAKE_CASE(AArch64ISD::VASHR)
2649 MAKE_CASE(AArch64ISD::VSLI)
2650 MAKE_CASE(AArch64ISD::VSRI)
2651 MAKE_CASE(AArch64ISD::CMEQ)
2652 MAKE_CASE(AArch64ISD::CMGE)
2653 MAKE_CASE(AArch64ISD::CMGT)
2654 MAKE_CASE(AArch64ISD::CMHI)
2655 MAKE_CASE(AArch64ISD::CMHS)
2656 MAKE_CASE(AArch64ISD::FCMEQ)
2657 MAKE_CASE(AArch64ISD::FCMGE)
2658 MAKE_CASE(AArch64ISD::FCMGT)
2659 MAKE_CASE(AArch64ISD::CMEQz)
2660 MAKE_CASE(AArch64ISD::CMGEz)
2661 MAKE_CASE(AArch64ISD::CMGTz)
2662 MAKE_CASE(AArch64ISD::CMLEz)
2663 MAKE_CASE(AArch64ISD::CMLTz)
2664 MAKE_CASE(AArch64ISD::FCMEQz)
2665 MAKE_CASE(AArch64ISD::FCMGEz)
2666 MAKE_CASE(AArch64ISD::FCMGTz)
2667 MAKE_CASE(AArch64ISD::FCMLEz)
2668 MAKE_CASE(AArch64ISD::FCMLTz)
2669 MAKE_CASE(AArch64ISD::SADDV)
2670 MAKE_CASE(AArch64ISD::UADDV)
2671 MAKE_CASE(AArch64ISD::UADDLV)
2672 MAKE_CASE(AArch64ISD::SADDLV)
2673 MAKE_CASE(AArch64ISD::SDOT)
2674 MAKE_CASE(AArch64ISD::UDOT)
2675 MAKE_CASE(AArch64ISD::SMINV)
2676 MAKE_CASE(AArch64ISD::UMINV)
2677 MAKE_CASE(AArch64ISD::SMAXV)
2678 MAKE_CASE(AArch64ISD::UMAXV)
2679 MAKE_CASE(AArch64ISD::SADDV_PRED)
2680 MAKE_CASE(AArch64ISD::UADDV_PRED)
2681 MAKE_CASE(AArch64ISD::SMAXV_PRED)
2682 MAKE_CASE(AArch64ISD::UMAXV_PRED)
2683 MAKE_CASE(AArch64ISD::SMINV_PRED)
2684 MAKE_CASE(AArch64ISD::UMINV_PRED)
2685 MAKE_CASE(AArch64ISD::ORV_PRED)
2686 MAKE_CASE(AArch64ISD::EORV_PRED)
2687 MAKE_CASE(AArch64ISD::ANDV_PRED)
2688 MAKE_CASE(AArch64ISD::CLASTA_N)
2689 MAKE_CASE(AArch64ISD::CLASTB_N)
2690 MAKE_CASE(AArch64ISD::LASTA)
2691 MAKE_CASE(AArch64ISD::LASTB)
2692 MAKE_CASE(AArch64ISD::REINTERPRET_CAST)
2693 MAKE_CASE(AArch64ISD::LS64_BUILD)
2694 MAKE_CASE(AArch64ISD::LS64_EXTRACT)
2695 MAKE_CASE(AArch64ISD::TBL)
2696 MAKE_CASE(AArch64ISD::FADD_PRED)
2697 MAKE_CASE(AArch64ISD::FADDA_PRED)
2698 MAKE_CASE(AArch64ISD::FADDV_PRED)
2699 MAKE_CASE(AArch64ISD::FDIV_PRED)
2700 MAKE_CASE(AArch64ISD::FMA_PRED)
2701 MAKE_CASE(AArch64ISD::FMAX_PRED)
2702 MAKE_CASE(AArch64ISD::FMAXV_PRED)
2703 MAKE_CASE(AArch64ISD::FMAXNM_PRED)
2704 MAKE_CASE(AArch64ISD::FMAXNMV_PRED)
2705 MAKE_CASE(AArch64ISD::FMIN_PRED)
2706 MAKE_CASE(AArch64ISD::FMINV_PRED)
2707 MAKE_CASE(AArch64ISD::FMINNM_PRED)
2708 MAKE_CASE(AArch64ISD::FMINNMV_PRED)
2709 MAKE_CASE(AArch64ISD::FMUL_PRED)
2710 MAKE_CASE(AArch64ISD::FSUB_PRED)
2711 MAKE_CASE(AArch64ISD::RDSVL)
2712 MAKE_CASE(AArch64ISD::BIC)
2713 MAKE_CASE(AArch64ISD::CBZ)
2714 MAKE_CASE(AArch64ISD::CBNZ)
2715 MAKE_CASE(AArch64ISD::TBZ)
2716 MAKE_CASE(AArch64ISD::TBNZ)
2717 MAKE_CASE(AArch64ISD::TC_RETURN)
2718 MAKE_CASE(AArch64ISD::PREFETCH)
2719 MAKE_CASE(AArch64ISD::SITOF)
2720 MAKE_CASE(AArch64ISD::UITOF)
2721 MAKE_CASE(AArch64ISD::NVCAST)
2722 MAKE_CASE(AArch64ISD::MRS)
2723 MAKE_CASE(AArch64ISD::SQSHL_I)
2724 MAKE_CASE(AArch64ISD::UQSHL_I)
2725 MAKE_CASE(AArch64ISD::SRSHR_I)
2726 MAKE_CASE(AArch64ISD::URSHR_I)
2727 MAKE_CASE(AArch64ISD::SQSHLU_I)
2728 MAKE_CASE(AArch64ISD::WrapperLarge)
2729 MAKE_CASE(AArch64ISD::LD2post)
2730 MAKE_CASE(AArch64ISD::LD3post)
2731 MAKE_CASE(AArch64ISD::LD4post)
2732 MAKE_CASE(AArch64ISD::ST2post)
2733 MAKE_CASE(AArch64ISD::ST3post)
2734 MAKE_CASE(AArch64ISD::ST4post)
2735 MAKE_CASE(AArch64ISD::LD1x2post)
2736 MAKE_CASE(AArch64ISD::LD1x3post)
2737 MAKE_CASE(AArch64ISD::LD1x4post)
2738 MAKE_CASE(AArch64ISD::ST1x2post)
2739 MAKE_CASE(AArch64ISD::ST1x3post)
2740 MAKE_CASE(AArch64ISD::ST1x4post)
2741 MAKE_CASE(AArch64ISD::LD1DUPpost)
2742 MAKE_CASE(AArch64ISD::LD2DUPpost)
2743 MAKE_CASE(AArch64ISD::LD3DUPpost)
2744 MAKE_CASE(AArch64ISD::LD4DUPpost)
2745 MAKE_CASE(AArch64ISD::LD1LANEpost)
2746 MAKE_CASE(AArch64ISD::LD2LANEpost)
2747 MAKE_CASE(AArch64ISD::LD3LANEpost)
2748 MAKE_CASE(AArch64ISD::LD4LANEpost)
2749 MAKE_CASE(AArch64ISD::ST2LANEpost)
2750 MAKE_CASE(AArch64ISD::ST3LANEpost)
2751 MAKE_CASE(AArch64ISD::ST4LANEpost)
2752 MAKE_CASE(AArch64ISD::SMULL)
2753 MAKE_CASE(AArch64ISD::UMULL)
2754 MAKE_CASE(AArch64ISD::PMULL)
2755 MAKE_CASE(AArch64ISD::FRECPE)
2756 MAKE_CASE(AArch64ISD::FRECPS)
2757 MAKE_CASE(AArch64ISD::FRSQRTE)
2758 MAKE_CASE(AArch64ISD::FRSQRTS)
2759 MAKE_CASE(AArch64ISD::STG)
2760 MAKE_CASE(AArch64ISD::STZG)
2761 MAKE_CASE(AArch64ISD::ST2G)
2762 MAKE_CASE(AArch64ISD::STZ2G)
2763 MAKE_CASE(AArch64ISD::SUNPKHI)
2764 MAKE_CASE(AArch64ISD::SUNPKLO)
2765 MAKE_CASE(AArch64ISD::UUNPKHI)
2766 MAKE_CASE(AArch64ISD::UUNPKLO)
2767 MAKE_CASE(AArch64ISD::INSR)
2768 MAKE_CASE(AArch64ISD::PTEST)
2769 MAKE_CASE(AArch64ISD::PTEST_ANY)
2770 MAKE_CASE(AArch64ISD::PTRUE)
2771 MAKE_CASE(AArch64ISD::LD1_MERGE_ZERO)
2772 MAKE_CASE(AArch64ISD::LD1S_MERGE_ZERO)
2773 MAKE_CASE(AArch64ISD::LDNF1_MERGE_ZERO)
2774 MAKE_CASE(AArch64ISD::LDNF1S_MERGE_ZERO)
2775 MAKE_CASE(AArch64ISD::LDFF1_MERGE_ZERO)
2776 MAKE_CASE(AArch64ISD::LDFF1S_MERGE_ZERO)
2777 MAKE_CASE(AArch64ISD::LD1RQ_MERGE_ZERO)
2778 MAKE_CASE(AArch64ISD::LD1RO_MERGE_ZERO)
2779 MAKE_CASE(AArch64ISD::SVE_LD2_MERGE_ZERO)
2780 MAKE_CASE(AArch64ISD::SVE_LD3_MERGE_ZERO)
2781 MAKE_CASE(AArch64ISD::SVE_LD4_MERGE_ZERO)
2782 MAKE_CASE(AArch64ISD::GLD1_MERGE_ZERO)
2783 MAKE_CASE(AArch64ISD::GLD1_SCALED_MERGE_ZERO)
2784 MAKE_CASE(AArch64ISD::GLD1_SXTW_MERGE_ZERO)
2785 MAKE_CASE(AArch64ISD::GLD1_UXTW_MERGE_ZERO)
2786 MAKE_CASE(AArch64ISD::GLD1_SXTW_SCALED_MERGE_ZERO)
2787 MAKE_CASE(AArch64ISD::GLD1_UXTW_SCALED_MERGE_ZERO)
2788 MAKE_CASE(AArch64ISD::GLD1_IMM_MERGE_ZERO)
2789 MAKE_CASE(AArch64ISD::GLD1Q_MERGE_ZERO)
2790 MAKE_CASE(AArch64ISD::GLD1Q_INDEX_MERGE_ZERO)
2791 MAKE_CASE(AArch64ISD::GLD1S_MERGE_ZERO)
2792 MAKE_CASE(AArch64ISD::GLD1S_SCALED_MERGE_ZERO)
2793 MAKE_CASE(AArch64ISD::GLD1S_SXTW_MERGE_ZERO)
2794 MAKE_CASE(AArch64ISD::GLD1S_UXTW_MERGE_ZERO)
2795 MAKE_CASE(AArch64ISD::GLD1S_SXTW_SCALED_MERGE_ZERO)
2796 MAKE_CASE(AArch64ISD::GLD1S_UXTW_SCALED_MERGE_ZERO)
2797 MAKE_CASE(AArch64ISD::GLD1S_IMM_MERGE_ZERO)
2798 MAKE_CASE(AArch64ISD::GLDFF1_MERGE_ZERO)
2799 MAKE_CASE(AArch64ISD::GLDFF1_SCALED_MERGE_ZERO)
2800 MAKE_CASE(AArch64ISD::GLDFF1_SXTW_MERGE_ZERO)
2801 MAKE_CASE(AArch64ISD::GLDFF1_UXTW_MERGE_ZERO)
2802 MAKE_CASE(AArch64ISD::GLDFF1_SXTW_SCALED_MERGE_ZERO)
2803 MAKE_CASE(AArch64ISD::GLDFF1_UXTW_SCALED_MERGE_ZERO)
2804 MAKE_CASE(AArch64ISD::GLDFF1_IMM_MERGE_ZERO)
2805 MAKE_CASE(AArch64ISD::GLDFF1S_MERGE_ZERO)
2806 MAKE_CASE(AArch64ISD::GLDFF1S_SCALED_MERGE_ZERO)
2807 MAKE_CASE(AArch64ISD::GLDFF1S_SXTW_MERGE_ZERO)
2808 MAKE_CASE(AArch64ISD::GLDFF1S_UXTW_MERGE_ZERO)
2809 MAKE_CASE(AArch64ISD::GLDFF1S_SXTW_SCALED_MERGE_ZERO)
2810 MAKE_CASE(AArch64ISD::GLDFF1S_UXTW_SCALED_MERGE_ZERO)
2811 MAKE_CASE(AArch64ISD::GLDFF1S_IMM_MERGE_ZERO)
2812 MAKE_CASE(AArch64ISD::GLDNT1_MERGE_ZERO)
2813 MAKE_CASE(AArch64ISD::GLDNT1_INDEX_MERGE_ZERO)
2814 MAKE_CASE(AArch64ISD::GLDNT1S_MERGE_ZERO)
2815 MAKE_CASE(AArch64ISD::SST1Q_PRED)
2816 MAKE_CASE(AArch64ISD::SST1Q_INDEX_PRED)
2817 MAKE_CASE(AArch64ISD::ST1_PRED)
2818 MAKE_CASE(AArch64ISD::SST1_PRED)
2819 MAKE_CASE(AArch64ISD::SST1_SCALED_PRED)
2820 MAKE_CASE(AArch64ISD::SST1_SXTW_PRED)
2821 MAKE_CASE(AArch64ISD::SST1_UXTW_PRED)
2822 MAKE_CASE(AArch64ISD::SST1_SXTW_SCALED_PRED)
2823 MAKE_CASE(AArch64ISD::SST1_UXTW_SCALED_PRED)
2824 MAKE_CASE(AArch64ISD::SST1_IMM_PRED)
2825 MAKE_CASE(AArch64ISD::SSTNT1_PRED)
2826 MAKE_CASE(AArch64ISD::SSTNT1_INDEX_PRED)
2827 MAKE_CASE(AArch64ISD::LDP)
2828 MAKE_CASE(AArch64ISD::LDIAPP)
2829 MAKE_CASE(AArch64ISD::LDNP)
2830 MAKE_CASE(AArch64ISD::STP)
2831 MAKE_CASE(AArch64ISD::STILP)
2832 MAKE_CASE(AArch64ISD::STNP)
2833 MAKE_CASE(AArch64ISD::BITREVERSE_MERGE_PASSTHRU)
2834 MAKE_CASE(AArch64ISD::BSWAP_MERGE_PASSTHRU)
2835 MAKE_CASE(AArch64ISD::REVH_MERGE_PASSTHRU)
2836 MAKE_CASE(AArch64ISD::REVW_MERGE_PASSTHRU)
2837 MAKE_CASE(AArch64ISD::REVD_MERGE_PASSTHRU)
2838 MAKE_CASE(AArch64ISD::CTLZ_MERGE_PASSTHRU)
2839 MAKE_CASE(AArch64ISD::CTPOP_MERGE_PASSTHRU)
2840 MAKE_CASE(AArch64ISD::DUP_MERGE_PASSTHRU)
2841 MAKE_CASE(AArch64ISD::INDEX_VECTOR)
2842 MAKE_CASE(AArch64ISD::ADDP)
2843 MAKE_CASE(AArch64ISD::SADDLP)
2844 MAKE_CASE(AArch64ISD::UADDLP)
2845 MAKE_CASE(AArch64ISD::CALL_RVMARKER)
2846 MAKE_CASE(AArch64ISD::ASSERT_ZEXT_BOOL)
2847 MAKE_CASE(AArch64ISD::MOPS_MEMSET)
2848 MAKE_CASE(AArch64ISD::MOPS_MEMSET_TAGGING)
2849 MAKE_CASE(AArch64ISD::MOPS_MEMCOPY)
2850 MAKE_CASE(AArch64ISD::MOPS_MEMMOVE)
2851 MAKE_CASE(AArch64ISD::CALL_BTI)
2852 MAKE_CASE(AArch64ISD::MRRS)
2853 MAKE_CASE(AArch64ISD::MSRR)
2854 MAKE_CASE(AArch64ISD::RSHRNB_I)
2855 MAKE_CASE(AArch64ISD::CTTZ_ELTS)
2856 MAKE_CASE(AArch64ISD::CALL_ARM64EC_TO_X64)
2857 MAKE_CASE(AArch64ISD::URSHR_I_PRED)
2858 }
2859#undef MAKE_CASE
2860 return nullptr;
2861}
2862
2863MachineBasicBlock *
2864AArch64TargetLowering::EmitF128CSEL(MachineInstr &MI,
2865 MachineBasicBlock *MBB) const {
2866 // We materialise the F128CSEL pseudo-instruction as some control flow and a
2867 // phi node:
2868
2869 // OrigBB:
2870 // [... previous instrs leading to comparison ...]
2871 // b.ne TrueBB
2872 // b EndBB
2873 // TrueBB:
2874 // ; Fallthrough
2875 // EndBB:
2876 // Dest = PHI [IfTrue, TrueBB], [IfFalse, OrigBB]
2877
2878 MachineFunction *MF = MBB->getParent();
2879 const TargetInstrInfo *TII = Subtarget->getInstrInfo();
2880 const BasicBlock *LLVM_BB = MBB->getBasicBlock();
2881 DebugLoc DL = MI.getDebugLoc();
2882 MachineFunction::iterator It = ++MBB->getIterator();
2883
2884 Register DestReg = MI.getOperand(0).getReg();
2885 Register IfTrueReg = MI.getOperand(1).getReg();
2886 Register IfFalseReg = MI.getOperand(2).getReg();
2887 unsigned CondCode = MI.getOperand(3).getImm();
2888 bool NZCVKilled = MI.getOperand(4).isKill();
2889
2890 MachineBasicBlock *TrueBB = MF->CreateMachineBasicBlock(LLVM_BB);
2891 MachineBasicBlock *EndBB = MF->CreateMachineBasicBlock(LLVM_BB);
2892 MF->insert(It, TrueBB);
2893 MF->insert(It, EndBB);
2894
2895 // Transfer rest of current basic-block to EndBB
2896 EndBB->splice(EndBB->begin(), MBB, std::next(MachineBasicBlock::iterator(MI)),
2897 MBB->end());
2898 EndBB->transferSuccessorsAndUpdatePHIs(MBB);
2899
2900 BuildMI(MBB, DL, TII->get(AArch64::Bcc)).addImm(CondCode).addMBB(TrueBB);
2901 BuildMI(MBB, DL, TII->get(AArch64::B)).addMBB(EndBB);
2902 MBB->addSuccessor(TrueBB);
2903 MBB->addSuccessor(EndBB);
2904
2905 // TrueBB falls through to the end.
2906 TrueBB->addSuccessor(EndBB);
2907
2908 if (!NZCVKilled) {
2909 TrueBB->addLiveIn(AArch64::NZCV);
2910 EndBB->addLiveIn(AArch64::NZCV);
2911 }
2912
2913 BuildMI(*EndBB, EndBB->begin(), DL, TII->get(AArch64::PHI), DestReg)
2914 .addReg(IfTrueReg)
2915 .addMBB(TrueBB)
2916 .addReg(IfFalseReg)
2917 .addMBB(MBB);
2918
2919 MI.eraseFromParent();
2920 return EndBB;
2921}
2922
2923MachineBasicBlock *AArch64TargetLowering::EmitLoweredCatchRet(
2924 MachineInstr &MI, MachineBasicBlock *BB) const {
2925 assert(!isAsynchronousEHPersonality(classifyEHPersonality(
2926 BB->getParent()->getFunction().getPersonalityFn())) &&
2927 "SEH does not use catchret!");
2928 return BB;
2929}
2930
2931MachineBasicBlock *
2932AArch64TargetLowering::EmitDynamicProbedAlloc(MachineInstr &MI,
2933 MachineBasicBlock *MBB) const {
2934 MachineFunction &MF = *MBB->getParent();
2935 MachineBasicBlock::iterator MBBI = MI.getIterator();
2936 DebugLoc DL = MBB->findDebugLoc(MBBI);
2937 const AArch64InstrInfo &TII =
2938 *MF.getSubtarget<AArch64Subtarget>().getInstrInfo();
2939 Register TargetReg = MI.getOperand(0).getReg();
2940 MachineBasicBlock::iterator NextInst =
2941 TII.probedStackAlloc(MBBI, TargetReg, false);
2942
2943 MI.eraseFromParent();
2944 return NextInst->getParent();
2945}
2946
2947MachineBasicBlock *
2948AArch64TargetLowering::EmitTileLoad(unsigned Opc, unsigned BaseReg,
2949 MachineInstr &MI,
2950 MachineBasicBlock *BB) const {
2951 const TargetInstrInfo *TII = Subtarget->getInstrInfo();
2952 MachineInstrBuilder MIB = BuildMI(*BB, MI, MI.getDebugLoc(), TII->get(Opc));
2953
2954 MIB.addReg(BaseReg + MI.getOperand(0).getImm(), RegState::Define);
2955 MIB.add(MI.getOperand(1)); // slice index register
2956 MIB.add(MI.getOperand(2)); // slice index offset
2957 MIB.add(MI.getOperand(3)); // pg
2958 MIB.add(MI.getOperand(4)); // base
2959 MIB.add(MI.getOperand(5)); // offset
2960
2961 MI.eraseFromParent(); // The pseudo is gone now.
2962 return BB;
2963}
2964
2965MachineBasicBlock *
2966AArch64TargetLowering::EmitFill(MachineInstr &MI, MachineBasicBlock *BB) const {
2967 const TargetInstrInfo *TII = Subtarget->getInstrInfo();
2968 MachineInstrBuilder MIB =
2969 BuildMI(*BB, MI, MI.getDebugLoc(), TII->get(AArch64::LDR_ZA));
2970
2971 MIB.addReg(AArch64::ZA, RegState::Define);
2972 MIB.add(MI.getOperand(0)); // Vector select register
2973 MIB.add(MI.getOperand(1)); // Vector select offset
2974 MIB.add(MI.getOperand(2)); // Base
2975 MIB.add(MI.getOperand(1)); // Offset, same as vector select offset
2976
2977 MI.eraseFromParent(); // The pseudo is gone now.
2978 return BB;
2979}
2980
2981MachineBasicBlock *AArch64TargetLowering::EmitZTInstr(MachineInstr &MI,
2982 MachineBasicBlock *BB,
2983 unsigned Opcode,
2984 bool Op0IsDef) const {
2985 const TargetInstrInfo *TII = Subtarget->getInstrInfo();
2986 MachineInstrBuilder MIB;
2987
2988 MIB = BuildMI(*BB, MI, MI.getDebugLoc(), TII->get(Opcode))
2989 .addReg(MI.getOperand(0).getReg(), Op0IsDef ? RegState::Define : 0);
2990 for (unsigned I = 1; I < MI.getNumOperands(); ++I)
2991 MIB.add(MI.getOperand(I));
2992
2993 MI.eraseFromParent(); // The pseudo is gone now.
2994 return BB;
2995}
2996
2997MachineBasicBlock *
2998AArch64TargetLowering::EmitZAInstr(unsigned Opc, unsigned BaseReg,
2999 MachineInstr &MI,
3000 MachineBasicBlock *BB) const {
3001 const TargetInstrInfo *TII = Subtarget->getInstrInfo();
3002 MachineInstrBuilder MIB = BuildMI(*BB, MI, MI.getDebugLoc(), TII->get(Opc));
3003 unsigned StartIdx = 0;
3004
3005 bool HasTile = BaseReg != AArch64::ZA;
3006 bool HasZPROut = HasTile && MI.getOperand(0).isReg();
3007 if (HasZPROut) {
3008 MIB.add(MI.getOperand(StartIdx)); // Output ZPR
3009 ++StartIdx;
3010 }
3011 if (HasTile) {
3012 MIB.addReg(BaseReg + MI.getOperand(StartIdx).getImm(),
3013 RegState::Define); // Output ZA Tile
3014 MIB.addReg(BaseReg + MI.getOperand(StartIdx).getImm()); // Input Za Tile
3015 StartIdx++;
3016 } else {
3017 // Avoids all instructions with mnemonic za.<sz>[Reg, Imm,
3018 if (MI.getOperand(0).isReg() && !MI.getOperand(1).isImm()) {
3019 MIB.add(MI.getOperand(StartIdx)); // Output ZPR
3020 ++StartIdx;
3021 }
3022 MIB.addReg(BaseReg, RegState::Define).addReg(BaseReg);
3023 }
3024 for (unsigned I = StartIdx; I < MI.getNumOperands(); ++I)
3025 MIB.add(MI.getOperand(I));
3026
3027 MI.eraseFromParent(); // The pseudo is gone now.
3028 return BB;
3029}
3030
3031MachineBasicBlock *
3032AArch64TargetLowering::EmitZero(MachineInstr &MI, MachineBasicBlock *BB) const {
3033 const TargetInstrInfo *TII = Subtarget->getInstrInfo();
3034 MachineInstrBuilder MIB =
3035 BuildMI(*BB, MI, MI.getDebugLoc(), TII->get(AArch64::ZERO_M));
3036 MIB.add(MI.getOperand(0)); // Mask
3037
3038 unsigned Mask = MI.getOperand(0).getImm();
3039 for (unsigned I = 0; I < 8; I++) {
3040 if (Mask & (1 << I))
3041 MIB.addDef(AArch64::ZAD0 + I, RegState::ImplicitDefine);
3042 }
3043
3044 MI.eraseFromParent(); // The pseudo is gone now.
3045 return BB;
3046}
3047
3048MachineBasicBlock *
3049AArch64TargetLowering::EmitInitTPIDR2Object(MachineInstr &MI,
3050 MachineBasicBlock *BB) const {
3051 MachineFunction *MF = BB->getParent();
3052 MachineFrameInfo &MFI = MF->getFrameInfo();
3053 AArch64FunctionInfo *FuncInfo = MF->getInfo<AArch64FunctionInfo>();
3054 TPIDR2Object &TPIDR2 = FuncInfo->getTPIDR2Obj();
3055 if (TPIDR2.Uses > 0) {
3056 const TargetInstrInfo *TII = Subtarget->getInstrInfo();
3057 // Store the buffer pointer to the TPIDR2 stack object.
3058 BuildMI(*BB, MI, MI.getDebugLoc(), TII->get(AArch64::STRXui))
3059 .addReg(MI.getOperand(0).getReg())
3060 .addFrameIndex(TPIDR2.FrameIndex)
3061 .addImm(0);
3062 // Set the reserved bytes (10-15) to zero
3063 BuildMI(*BB, MI, MI.getDebugLoc(), TII->get(AArch64::STRHHui))
3064 .addReg(AArch64::WZR)
3065 .addFrameIndex(TPIDR2.FrameIndex)
3066 .addImm(5);
3067 BuildMI(*BB, MI, MI.getDebugLoc(), TII->get(AArch64::STRWui))
3068 .addReg(AArch64::WZR)
3069 .addFrameIndex(TPIDR2.FrameIndex)
3070 .addImm(3);
3071 } else
3072 MFI.RemoveStackObject(TPIDR2.FrameIndex);
3073
3074 BB->remove_instr(&MI);
3075 return BB;
3076}
3077
3078MachineBasicBlock *
3079AArch64TargetLowering::EmitAllocateZABuffer(MachineInstr &MI,
3080 MachineBasicBlock *BB) const {
3081 MachineFunction *MF = BB->getParent();
3082 MachineFrameInfo &MFI = MF->getFrameInfo();
3083 AArch64FunctionInfo *FuncInfo = MF->getInfo<AArch64FunctionInfo>();
3084 // TODO This function grows the stack with a subtraction, which doesn't work
3085 // on Windows. Some refactoring to share the functionality in
3086 // LowerWindowsDYNAMIC_STACKALLOC will be required once the Windows ABI
3087 // supports SME
3088 assert(!MF->getSubtarget<AArch64Subtarget>().isTargetWindows() &&
3089 "Lazy ZA save is not yet supported on Windows");
3090
3091 TPIDR2Object &TPIDR2 = FuncInfo->getTPIDR2Obj();
3092
3093 if (TPIDR2.Uses > 0) {
3094 const TargetInstrInfo *TII = Subtarget->getInstrInfo();
3095 MachineRegisterInfo &MRI = MF->getRegInfo();
3096
3097 // The SUBXrs below won't always be emitted in a form that accepts SP
3098 // directly
3099 Register SP = MRI.createVirtualRegister(&AArch64::GPR64RegClass);
3100 BuildMI(*BB, MI, MI.getDebugLoc(), TII->get(TargetOpcode::COPY), SP)
3101 .addReg(AArch64::SP);
3102
3103 // Allocate a lazy-save buffer object of the size given, normally SVL * SVL
3104 auto Size = MI.getOperand(1).getReg();
3105 auto Dest = MI.getOperand(0).getReg();
3106 BuildMI(*BB, MI, MI.getDebugLoc(), TII->get(AArch64::MSUBXrrr), Dest)
3107 .addReg(Size)
3108 .addReg(Size)
3109 .addReg(SP);
3110 BuildMI(*BB, MI, MI.getDebugLoc(), TII->get(TargetOpcode::COPY),
3111 AArch64::SP)
3112 .addReg(Dest);
3113
3114 // We have just allocated a variable sized object, tell this to PEI.
3115 MFI.CreateVariableSizedObject(Align(16), nullptr);
3116 }
3117
3118 BB->remove_instr(&MI);
3119 return BB;
3120}
3121
3122MachineBasicBlock *AArch64TargetLowering::EmitInstrWithCustomInserter(
3123 MachineInstr &MI, MachineBasicBlock *BB) const {
3124
3125 int SMEOrigInstr = AArch64::getSMEPseudoMap(MI.getOpcode());
3126 if (SMEOrigInstr != -1) {
3127 const TargetInstrInfo *TII = Subtarget->getInstrInfo();
3128 uint64_t SMEMatrixType =
3129 TII->get(MI.getOpcode()).TSFlags & AArch64::SMEMatrixTypeMask;
3130 switch (SMEMatrixType) {
3131 case (AArch64::SMEMatrixArray):
3132 return EmitZAInstr(SMEOrigInstr, AArch64::ZA, MI, BB);
3133 case (AArch64::SMEMatrixTileB):
3134 return EmitZAInstr(SMEOrigInstr, AArch64::ZAB0, MI, BB);
3135 case (AArch64::SMEMatrixTileH):
3136 return EmitZAInstr(SMEOrigInstr, AArch64::ZAH0, MI, BB);
3137 case (AArch64::SMEMatrixTileS):
3138 return EmitZAInstr(SMEOrigInstr, AArch64::ZAS0, MI, BB);
3139 case (AArch64::SMEMatrixTileD):
3140 return EmitZAInstr(SMEOrigInstr, AArch64::ZAD0, MI, BB);
3141 case (AArch64::SMEMatrixTileQ):
3142 return EmitZAInstr(SMEOrigInstr, AArch64::ZAQ0, MI, BB);
3143 }
3144 }
3145
3146 switch (MI.getOpcode()) {
3147 default:
3148#ifndef NDEBUG
3149 MI.dump();
3150#endif
3151 llvm_unreachable("Unexpected instruction for custom inserter!");
3152 case AArch64::InitTPIDR2Obj:
3153 return EmitInitTPIDR2Object(MI, BB);
3154 case AArch64::AllocateZABuffer:
3155 return EmitAllocateZABuffer(MI, BB);
3156 case AArch64::F128CSEL:
3157 return EmitF128CSEL(MI, BB);
3158 case TargetOpcode::STATEPOINT:
3159 // STATEPOINT is a pseudo instruction which has no implicit defs/uses
3160 // while bl call instruction (where statepoint will be lowered at the end)
3161 // has implicit def. This def is early-clobber as it will be set at
3162 // the moment of the call and earlier than any use is read.
3163 // Add this implicit dead def here as a workaround.
3164 MI.addOperand(*MI.getMF(),
3165 MachineOperand::CreateReg(
3166 AArch64::LR, /*isDef*/ true,
3167 /*isImp*/ true, /*isKill*/ false, /*isDead*/ true,
3168 /*isUndef*/ false, /*isEarlyClobber*/ true));
3169 [[fallthrough]];
3170 case TargetOpcode::STACKMAP:
3171 case TargetOpcode::PATCHPOINT:
3172 return emitPatchPoint(MI, BB);
3173
3174 case TargetOpcode::PATCHABLE_EVENT_CALL:
3175 case TargetOpcode::PATCHABLE_TYPED_EVENT_CALL:
3176 return BB;
3177
3178 case AArch64::CATCHRET:
3179 return EmitLoweredCatchRet(MI, BB);
3180
3181 case AArch64::PROBED_STACKALLOC_DYN:
3182 return EmitDynamicProbedAlloc(MI, BB);
3183
3184 case AArch64::LD1_MXIPXX_H_PSEUDO_B:
3185 return EmitTileLoad(AArch64::LD1_MXIPXX_H_B, AArch64::ZAB0, MI, BB);
3186 case AArch64::LD1_MXIPXX_H_PSEUDO_H:
3187 return EmitTileLoad(AArch64::LD1_MXIPXX_H_H, AArch64::ZAH0, MI, BB);
3188 case AArch64::LD1_MXIPXX_H_PSEUDO_S:
3189 return EmitTileLoad(AArch64::LD1_MXIPXX_H_S, AArch64::ZAS0, MI, BB);
3190 case AArch64::LD1_MXIPXX_H_PSEUDO_D:
3191 return EmitTileLoad(AArch64::LD1_MXIPXX_H_D, AArch64::ZAD0, MI, BB);
3192 case AArch64::LD1_MXIPXX_H_PSEUDO_Q:
3193 return EmitTileLoad(AArch64::LD1_MXIPXX_H_Q, AArch64::ZAQ0, MI, BB);
3194 case AArch64::LD1_MXIPXX_V_PSEUDO_B:
3195 return EmitTileLoad(AArch64::LD1_MXIPXX_V_B, AArch64::ZAB0, MI, BB);
3196 case AArch64::LD1_MXIPXX_V_PSEUDO_H:
3197 return EmitTileLoad(AArch64::LD1_MXIPXX_V_H, AArch64::ZAH0, MI, BB);
3198 case AArch64::LD1_MXIPXX_V_PSEUDO_S:
3199 return EmitTileLoad(AArch64::LD1_MXIPXX_V_S, AArch64::ZAS0, MI, BB);
3200 case AArch64::LD1_MXIPXX_V_PSEUDO_D:
3201 return EmitTileLoad(AArch64::LD1_MXIPXX_V_D, AArch64::ZAD0, MI, BB);
3202 case AArch64::LD1_MXIPXX_V_PSEUDO_Q:
3203 return EmitTileLoad(AArch64::LD1_MXIPXX_V_Q, AArch64::ZAQ0, MI, BB);
3204 case AArch64::LDR_ZA_PSEUDO:
3205 return EmitFill(MI, BB);
3206 case AArch64::LDR_TX_PSEUDO:
3207 return EmitZTInstr(MI, BB, AArch64::LDR_TX, /*Op0IsDef=*/true);
3208 case AArch64::STR_TX_PSEUDO:
3209 return EmitZTInstr(MI, BB, AArch64::STR_TX, /*Op0IsDef=*/false);
3210 case AArch64::ZERO_M_PSEUDO:
3211 return EmitZero(MI, BB);
3212 case AArch64::ZERO_T_PSEUDO:
3213 return EmitZTInstr(MI, BB, AArch64::ZERO_T, /*Op0IsDef=*/true);
3214 }
3215}
3216
3217//===----------------------------------------------------------------------===//
3218// AArch64 Lowering private implementation.
3219//===----------------------------------------------------------------------===//
3220
3221//===----------------------------------------------------------------------===//
3222// Lowering Code
3223//===----------------------------------------------------------------------===//
3224
3225// Forward declarations of SVE fixed length lowering helpers
3226static EVT getContainerForFixedLengthVector(SelectionDAG &DAG, EVT VT);
3227static SDValue convertToScalableVector(SelectionDAG &DAG, EVT VT, SDValue V);
3228static SDValue convertFromScalableVector(SelectionDAG &DAG, EVT VT, SDValue V);
3229static SDValue convertFixedMaskToScalableVector(SDValue Mask,
3230 SelectionDAG &DAG);
3231static SDValue getPredicateForVector(SelectionDAG &DAG, SDLoc &DL, EVT VT);
3232static SDValue getPredicateForScalableVector(SelectionDAG &DAG, SDLoc &DL,
3233 EVT VT);
3234
3235/// isZerosVector - Check whether SDNode N is a zero-filled vector.
3236static bool isZerosVector(const SDNode *N) {
3237 // Look through a bit convert.
3238 while (N->getOpcode() == ISD::BITCAST)
3239 N = N->getOperand(0).getNode();
3240
3241 if (ISD::isConstantSplatVectorAllZeros(N))
3242 return true;
3243
3244 if (N->getOpcode() != AArch64ISD::DUP)
3245 return false;
3246
3247 auto Opnd0 = N->getOperand(0);
3248 return isNullConstant(Opnd0) || isNullFPConstant(Opnd0);
3249}
3250
3251/// changeIntCCToAArch64CC - Convert a DAG integer condition code to an AArch64
3252/// CC
3253static AArch64CC::CondCode changeIntCCToAArch64CC(ISD::CondCode CC) {
3254 switch (CC) {
3255 default:
3256 llvm_unreachable("Unknown condition code!");
3257 case ISD::SETNE:
3258 return AArch64CC::NE;
3259 case ISD::SETEQ:
3260 return AArch64CC::EQ;
3261 case ISD::SETGT:
3262 return AArch64CC::GT;
3263 case ISD::SETGE:
3264 return AArch64CC::GE;
3265 case ISD::SETLT:
3266 return AArch64CC::LT;
3267 case ISD::SETLE:
3268 return AArch64CC::LE;
3269 case ISD::SETUGT:
3270 return AArch64CC::HI;
3271 case ISD::SETUGE:
3272 return AArch64CC::HS;
3273 case ISD::SETULT:
3274 return AArch64CC::LO;
3275 case ISD::SETULE:
3276 return AArch64CC::LS;
3277 }
3278}
3279
3280/// changeFPCCToAArch64CC - Convert a DAG fp condition code to an AArch64 CC.
3281static void changeFPCCToAArch64CC(ISD::CondCode CC,
3282 AArch64CC::CondCode &CondCode,
3283 AArch64CC::CondCode &CondCode2) {
3284 CondCode2 = AArch64CC::AL;
3285 switch (CC) {
3286 default:
3287 llvm_unreachable("Unknown FP condition!");
3288 case ISD::SETEQ:
3289 case ISD::SETOEQ:
3290 CondCode = AArch64CC::EQ;
3291 break;
3292 case ISD::SETGT:
3293 case ISD::SETOGT:
3294 CondCode = AArch64CC::GT;
3295 break;
3296 c