LLVM  13.0.0git
AArch64ISelLowering.cpp
Go to the documentation of this file.
1 //===-- AArch64ISelLowering.cpp - AArch64 DAG Lowering Implementation ----===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file implements the AArch64TargetLowering class.
10 //
11 //===----------------------------------------------------------------------===//
12 
13 #include "AArch64ISelLowering.h"
14 #include "AArch64CallingConvention.h"
15 #include "AArch64ExpandImm.h"
16 #include "AArch64MachineFunctionInfo.h"
17 #include "AArch64PerfectShuffle.h"
18 #include "AArch64RegisterInfo.h"
19 #include "AArch64Subtarget.h"
20 #include "MCTargetDesc/AArch64AddressingModes.h"
21 #include "Utils/AArch64BaseInfo.h"
22 #include "llvm/ADT/APFloat.h"
23 #include "llvm/ADT/APInt.h"
24 #include "llvm/ADT/ArrayRef.h"
25 #include "llvm/ADT/STLExtras.h"
26 #include "llvm/ADT/SmallSet.h"
27 #include "llvm/ADT/SmallVector.h"
28 #include "llvm/ADT/Statistic.h"
29 #include "llvm/ADT/StringRef.h"
30 #include "llvm/ADT/Triple.h"
31 #include "llvm/ADT/Twine.h"
32 #include "llvm/Analysis/ObjCARCUtil.h"
33 #include "llvm/Analysis/VectorUtils.h"
34 #include "llvm/CodeGen/CallingConvLower.h"
35 #include "llvm/CodeGen/MachineBasicBlock.h"
36 #include "llvm/CodeGen/MachineFrameInfo.h"
37 #include "llvm/CodeGen/MachineFunction.h"
38 #include "llvm/CodeGen/MachineInstr.h"
39 #include "llvm/CodeGen/MachineInstrBuilder.h"
40 #include "llvm/CodeGen/MachineMemOperand.h"
41 #include "llvm/CodeGen/MachineRegisterInfo.h"
42 #include "llvm/CodeGen/RuntimeLibcalls.h"
43 #include "llvm/CodeGen/SelectionDAG.h"
44 #include "llvm/CodeGen/SelectionDAGNodes.h"
45 #include "llvm/CodeGen/TargetCallingConv.h"
46 #include "llvm/CodeGen/TargetInstrInfo.h"
47 #include "llvm/CodeGen/ValueTypes.h"
48 #include "llvm/IR/Attributes.h"
49 #include "llvm/IR/Constants.h"
50 #include "llvm/IR/DataLayout.h"
51 #include "llvm/IR/DebugLoc.h"
52 #include "llvm/IR/DerivedTypes.h"
53 #include "llvm/IR/Function.h"
54 #include "llvm/IR/GetElementPtrTypeIterator.h"
55 #include "llvm/IR/GlobalValue.h"
56 #include "llvm/IR/IRBuilder.h"
57 #include "llvm/IR/Instruction.h"
58 #include "llvm/IR/Instructions.h"
59 #include "llvm/IR/IntrinsicInst.h"
60 #include "llvm/IR/Intrinsics.h"
61 #include "llvm/IR/IntrinsicsAArch64.h"
62 #include "llvm/IR/Module.h"
63 #include "llvm/IR/OperandTraits.h"
64 #include "llvm/IR/PatternMatch.h"
65 #include "llvm/IR/Type.h"
66 #include "llvm/IR/Use.h"
67 #include "llvm/IR/Value.h"
68 #include "llvm/MC/MCRegisterInfo.h"
69 #include "llvm/Support/Casting.h"
70 #include "llvm/Support/CodeGen.h"
71 #include "llvm/Support/CommandLine.h"
72 #include "llvm/Support/Compiler.h"
73 #include "llvm/Support/Debug.h"
74 #include "llvm/Support/ErrorHandling.h"
75 #include "llvm/Support/KnownBits.h"
76 #include "llvm/Support/MachineValueType.h"
77 #include "llvm/Support/MathExtras.h"
78 #include "llvm/Support/raw_ostream.h"
79 #include "llvm/Target/TargetMachine.h"
80 #include "llvm/Target/TargetOptions.h"
81 #include <algorithm>
82 #include <bitset>
83 #include <cassert>
84 #include <cctype>
85 #include <cstdint>
86 #include <cstdlib>
87 #include <iterator>
88 #include <limits>
89 #include <tuple>
90 #include <utility>
91 #include <vector>
92 
93 using namespace llvm;
94 using namespace llvm::PatternMatch;
95 
96 #define DEBUG_TYPE "aarch64-lower"
97 
98 STATISTIC(NumTailCalls, "Number of tail calls");
99 STATISTIC(NumShiftInserts, "Number of vector shift inserts");
100 STATISTIC(NumOptimizedImms, "Number of times immediates were optimized");
101 
102 // FIXME: The necessary dtprel relocations don't seem to be supported
103 // well in the GNU bfd and gold linkers at the moment. Therefore, by
104 // default, for now, fall back to GeneralDynamic code generation.
105 cl::opt<bool> EnableAArch64ELFLocalDynamicTLSGeneration(
106  "aarch64-elf-ldtls-generation", cl::Hidden,
107  cl::desc("Allow AArch64 Local Dynamic TLS code generation"),
108  cl::init(false));
109 
110 static cl::opt<bool>
111 EnableOptimizeLogicalImm("aarch64-enable-logical-imm", cl::Hidden,
112  cl::desc("Enable AArch64 logical imm instruction "
113  "optimization"),
114  cl::init(true));
115 
116 // Temporary option added for the purpose of testing functionality added
117 // to DAGCombiner.cpp in D92230. It is expected that this can be removed
118 // in future when both implementations will be based off MGATHER rather
119 // than the GLD1 nodes added for the SVE gather load intrinsics.
120 static cl::opt<bool>
121 EnableCombineMGatherIntrinsics("aarch64-enable-mgather-combine", cl::Hidden,
122  cl::desc("Combine extends of AArch64 masked "
123  "gather intrinsics"),
124  cl::init(true));
125 
126 /// Value type used for condition codes.
127 static const MVT MVT_CC = MVT::i32;
128 
129 static inline EVT getPackedSVEVectorVT(EVT VT) {
130  switch (VT.getSimpleVT().SimpleTy) {
131  default:
132  llvm_unreachable("unexpected element type for vector");
133  case MVT::i8:
134  return MVT::nxv16i8;
135  case MVT::i16:
136  return MVT::nxv8i16;
137  case MVT::i32:
138  return MVT::nxv4i32;
139  case MVT::i64:
140  return MVT::nxv2i64;
141  case MVT::f16:
142  return MVT::nxv8f16;
143  case MVT::f32:
144  return MVT::nxv4f32;
145  case MVT::f64:
146  return MVT::nxv2f64;
147  case MVT::bf16:
148  return MVT::nxv8bf16;
149  }
150 }
151 
152 // NOTE: Currently there's only a need to return integer vector types. If this
153 // changes then just add an extra "type" parameter.
154 static inline EVT getPackedSVEVectorVT(ElementCount EC) {
155  switch (EC.getKnownMinValue()) {
156  default:
157  llvm_unreachable("unexpected element count for vector");
158  case 16:
159  return MVT::nxv16i8;
160  case 8:
161  return MVT::nxv8i16;
162  case 4:
163  return MVT::nxv4i32;
164  case 2:
165  return MVT::nxv2i64;
166  }
167 }
168 
169 static inline EVT getPromotedVTForPredicate(EVT VT) {
170  assert(VT.isScalableVector() && (VT.getVectorElementType() == MVT::i1) &&
171  "Expected scalable predicate vector type!");
172  switch (VT.getVectorMinNumElements()) {
173  default:
174  llvm_unreachable("unexpected element count for vector");
175  case 2:
176  return MVT::nxv2i64;
177  case 4:
178  return MVT::nxv4i32;
179  case 8:
180  return MVT::nxv8i16;
181  case 16:
182  return MVT::nxv16i8;
183  }
184 }
185 
186 /// Returns true if VT's elements occupy the lowest bit positions of its
187 /// associated register class without any intervening space.
188 ///
189 /// For example, nxv2f16, nxv4f16 and nxv8f16 are legal types that belong to the
190 /// same register class, but only nxv8f16 can be treated as a packed vector.
191 static inline bool isPackedVectorType(EVT VT, SelectionDAG &DAG) {
192  assert(VT.isVector() && DAG.getTargetLoweringInfo().isTypeLegal(VT) &&
193  "Expected legal vector type!");
194  return VT.isFixedLengthVector() ||
195  VT.getSizeInBits().getKnownMinSize() == AArch64::SVEBitsPerBlock;
196 }
197 
198 // Returns true for ####_MERGE_PASSTHRU opcodes, whose operands have a leading
199 // predicate and end with a passthru value matching the result type.
200 static bool isMergePassthruOpcode(unsigned Opc) {
201  switch (Opc) {
202  default:
203  return false;
204  case AArch64ISD::BITREVERSE_MERGE_PASSTHRU:
205  case AArch64ISD::BSWAP_MERGE_PASSTHRU:
206  case AArch64ISD::CTLZ_MERGE_PASSTHRU:
207  case AArch64ISD::CTPOP_MERGE_PASSTHRU:
208  case AArch64ISD::DUP_MERGE_PASSTHRU:
209  case AArch64ISD::ABS_MERGE_PASSTHRU:
210  case AArch64ISD::NEG_MERGE_PASSTHRU:
211  case AArch64ISD::FNEG_MERGE_PASSTHRU:
212  case AArch64ISD::SIGN_EXTEND_INREG_MERGE_PASSTHRU:
213  case AArch64ISD::ZERO_EXTEND_INREG_MERGE_PASSTHRU:
214  case AArch64ISD::FCEIL_MERGE_PASSTHRU:
215  case AArch64ISD::FFLOOR_MERGE_PASSTHRU:
216  case AArch64ISD::FNEARBYINT_MERGE_PASSTHRU:
217  case AArch64ISD::FRINT_MERGE_PASSTHRU:
218  case AArch64ISD::FROUND_MERGE_PASSTHRU:
219  case AArch64ISD::FROUNDEVEN_MERGE_PASSTHRU:
220  case AArch64ISD::FTRUNC_MERGE_PASSTHRU:
221  case AArch64ISD::FP_ROUND_MERGE_PASSTHRU:
222  case AArch64ISD::FP_EXTEND_MERGE_PASSTHRU:
223  case AArch64ISD::SINT_TO_FP_MERGE_PASSTHRU:
224  case AArch64ISD::UINT_TO_FP_MERGE_PASSTHRU:
225  case AArch64ISD::FCVTZU_MERGE_PASSTHRU:
226  case AArch64ISD::FCVTZS_MERGE_PASSTHRU:
227  case AArch64ISD::FSQRT_MERGE_PASSTHRU:
228  case AArch64ISD::FRECPX_MERGE_PASSTHRU:
229  case AArch64ISD::FABS_MERGE_PASSTHRU:
230  return true;
231  }
232 }
233 
234 AArch64TargetLowering::AArch64TargetLowering(const TargetMachine &TM,
235  const AArch64Subtarget &STI)
236  : TargetLowering(TM), Subtarget(&STI) {
237  // AArch64 doesn't have comparisons which set GPRs or setcc instructions, so
238  // we have to make something up. Arbitrarily, choose ZeroOrOne.
239  setBooleanContents(ZeroOrOneBooleanContent);
240  // When comparing vectors the result sets the different elements in the
241  // vector to all-one or all-zero.
242  setBooleanVectorContents(ZeroOrNegativeOneBooleanContent);
243 
244  // Set up the register classes.
245  addRegisterClass(MVT::i32, &AArch64::GPR32allRegClass);
246  addRegisterClass(MVT::i64, &AArch64::GPR64allRegClass);
247 
248  if (Subtarget->hasFPARMv8()) {
249  addRegisterClass(MVT::f16, &AArch64::FPR16RegClass);
250  addRegisterClass(MVT::bf16, &AArch64::FPR16RegClass);
251  addRegisterClass(MVT::f32, &AArch64::FPR32RegClass);
252  addRegisterClass(MVT::f64, &AArch64::FPR64RegClass);
253  addRegisterClass(MVT::f128, &AArch64::FPR128RegClass);
254  }
255 
256  if (Subtarget->hasNEON()) {
257  addRegisterClass(MVT::v16i8, &AArch64::FPR8RegClass);
258  addRegisterClass(MVT::v8i16, &AArch64::FPR16RegClass);
259  // Someone set us up the NEON.
260  addDRTypeForNEON(MVT::v2f32);
261  addDRTypeForNEON(MVT::v8i8);
262  addDRTypeForNEON(MVT::v4i16);
263  addDRTypeForNEON(MVT::v2i32);
264  addDRTypeForNEON(MVT::v1i64);
265  addDRTypeForNEON(MVT::v1f64);
266  addDRTypeForNEON(MVT::v4f16);
267  if (Subtarget->hasBF16())
268  addDRTypeForNEON(MVT::v4bf16);
269 
270  addQRTypeForNEON(MVT::v4f32);
271  addQRTypeForNEON(MVT::v2f64);
272  addQRTypeForNEON(MVT::v16i8);
273  addQRTypeForNEON(MVT::v8i16);
274  addQRTypeForNEON(MVT::v4i32);
275  addQRTypeForNEON(MVT::v2i64);
276  addQRTypeForNEON(MVT::v8f16);
277  if (Subtarget->hasBF16())
278  addQRTypeForNEON(MVT::v8bf16);
279  }
280 
281  if (Subtarget->hasSVE()) {
282  // Add legal sve predicate types
283  addRegisterClass(MVT::nxv2i1, &AArch64::PPRRegClass);
284  addRegisterClass(MVT::nxv4i1, &AArch64::PPRRegClass);
285  addRegisterClass(MVT::nxv8i1, &AArch64::PPRRegClass);
286  addRegisterClass(MVT::nxv16i1, &AArch64::PPRRegClass);
287 
288  // Add legal sve data types
289  addRegisterClass(MVT::nxv16i8, &AArch64::ZPRRegClass);
290  addRegisterClass(MVT::nxv8i16, &AArch64::ZPRRegClass);
291  addRegisterClass(MVT::nxv4i32, &AArch64::ZPRRegClass);
292  addRegisterClass(MVT::nxv2i64, &AArch64::ZPRRegClass);
293 
294  addRegisterClass(MVT::nxv2f16, &AArch64::ZPRRegClass);
295  addRegisterClass(MVT::nxv4f16, &AArch64::ZPRRegClass);
296  addRegisterClass(MVT::nxv8f16, &AArch64::ZPRRegClass);
297  addRegisterClass(MVT::nxv2f32, &AArch64::ZPRRegClass);
298  addRegisterClass(MVT::nxv4f32, &AArch64::ZPRRegClass);
299  addRegisterClass(MVT::nxv2f64, &AArch64::ZPRRegClass);
300 
301  if (Subtarget->hasBF16()) {
302  addRegisterClass(MVT::nxv2bf16, &AArch64::ZPRRegClass);
303  addRegisterClass(MVT::nxv4bf16, &AArch64::ZPRRegClass);
304  addRegisterClass(MVT::nxv8bf16, &AArch64::ZPRRegClass);
305  }
306 
307  if (Subtarget->useSVEForFixedLengthVectors()) {
308  for (MVT VT : MVT::integer_fixedlen_vector_valuetypes())
309  if (useSVEForFixedLengthVectorVT(VT))
310  addRegisterClass(VT, &AArch64::ZPRRegClass);
311 
312  for (MVT VT : MVT::fp_fixedlen_vector_valuetypes())
313  if (useSVEForFixedLengthVectorVT(VT))
314  addRegisterClass(VT, &AArch64::ZPRRegClass);
315  }
316 
317  for (auto VT : { MVT::nxv16i8, MVT::nxv8i16, MVT::nxv4i32, MVT::nxv2i64 }) {
318  setOperationAction(ISD::SADDSAT, VT, Legal);
319  setOperationAction(ISD::UADDSAT, VT, Legal);
320  setOperationAction(ISD::SSUBSAT, VT, Legal);
321  setOperationAction(ISD::USUBSAT, VT, Legal);
322  setOperationAction(ISD::UREM, VT, Expand);
323  setOperationAction(ISD::SREM, VT, Expand);
324  setOperationAction(ISD::SDIVREM, VT, Expand);
325  setOperationAction(ISD::UDIVREM, VT, Expand);
326  }
327 
328  for (auto VT :
329  { MVT::nxv2i8, MVT::nxv2i16, MVT::nxv2i32, MVT::nxv2i64, MVT::nxv4i8,
330  MVT::nxv4i16, MVT::nxv4i32, MVT::nxv8i8, MVT::nxv8i16 })
331  setOperationAction(ISD::SIGN_EXTEND_INREG, VT, Legal);
332 
333  for (auto VT :
334  { MVT::nxv2f16, MVT::nxv4f16, MVT::nxv8f16, MVT::nxv2f32, MVT::nxv4f32,
335  MVT::nxv2f64 }) {
336  setCondCodeAction(ISD::SETO, VT, Expand);
337  setCondCodeAction(ISD::SETOLT, VT, Expand);
338  setCondCodeAction(ISD::SETLT, VT, Expand);
339  setCondCodeAction(ISD::SETOLE, VT, Expand);
340  setCondCodeAction(ISD::SETLE, VT, Expand);
341  setCondCodeAction(ISD::SETULT, VT, Expand);
342  setCondCodeAction(ISD::SETULE, VT, Expand);
343  setCondCodeAction(ISD::SETUGE, VT, Expand);
344  setCondCodeAction(ISD::SETUGT, VT, Expand);
345  setCondCodeAction(ISD::SETUEQ, VT, Expand);
346  setCondCodeAction(ISD::SETUNE, VT, Expand);
347 
348  setOperationAction(ISD::FREM, VT, Expand);
349  setOperationAction(ISD::FPOW, VT, Expand);
350  setOperationAction(ISD::FPOWI, VT, Expand);
351  setOperationAction(ISD::FCOS, VT, Expand);
352  setOperationAction(ISD::FSIN, VT, Expand);
353  setOperationAction(ISD::FSINCOS, VT, Expand);
354  setOperationAction(ISD::FEXP, VT, Expand);
355  setOperationAction(ISD::FEXP2, VT, Expand);
356  setOperationAction(ISD::FLOG, VT, Expand);
357  setOperationAction(ISD::FLOG2, VT, Expand);
358  setOperationAction(ISD::FLOG10, VT, Expand);
359  }
360  }
361 
362  // Compute derived properties from the register classes
363  computeRegisterProperties(Subtarget->getRegisterInfo());
364 
365  // Provide all sorts of operation actions
366  setOperationAction(ISD::GlobalAddress, MVT::i64, Custom);
367  setOperationAction(ISD::GlobalTLSAddress, MVT::i64, Custom);
368  setOperationAction(ISD::SETCC, MVT::i32, Custom);
369  setOperationAction(ISD::SETCC, MVT::i64, Custom);
370  setOperationAction(ISD::SETCC, MVT::f16, Custom);
371  setOperationAction(ISD::SETCC, MVT::f32, Custom);
372  setOperationAction(ISD::SETCC, MVT::f64, Custom);
373  setOperationAction(ISD::STRICT_FSETCC, MVT::f16, Custom);
374  setOperationAction(ISD::STRICT_FSETCC, MVT::f32, Custom);
375  setOperationAction(ISD::STRICT_FSETCC, MVT::f64, Custom);
376  setOperationAction(ISD::STRICT_FSETCCS, MVT::f16, Custom);
377  setOperationAction(ISD::STRICT_FSETCCS, MVT::f32, Custom);
378  setOperationAction(ISD::STRICT_FSETCCS, MVT::f64, Custom);
379  setOperationAction(ISD::BITREVERSE, MVT::i32, Legal);
380  setOperationAction(ISD::BITREVERSE, MVT::i64, Legal);
381  setOperationAction(ISD::BRCOND, MVT::Other, Expand);
382  setOperationAction(ISD::BR_CC, MVT::i32, Custom);
383  setOperationAction(ISD::BR_CC, MVT::i64, Custom);
384  setOperationAction(ISD::BR_CC, MVT::f16, Custom);
385  setOperationAction(ISD::BR_CC, MVT::f32, Custom);
386  setOperationAction(ISD::BR_CC, MVT::f64, Custom);
387  setOperationAction(ISD::SELECT, MVT::i32, Custom);
388  setOperationAction(ISD::SELECT, MVT::i64, Custom);
389  setOperationAction(ISD::SELECT, MVT::f16, Custom);
390  setOperationAction(ISD::SELECT, MVT::f32, Custom);
391  setOperationAction(ISD::SELECT, MVT::f64, Custom);
392  setOperationAction(ISD::SELECT_CC, MVT::i32, Custom);
393  setOperationAction(ISD::SELECT_CC, MVT::i64, Custom);
394  setOperationAction(ISD::SELECT_CC, MVT::f16, Custom);
395  setOperationAction(ISD::SELECT_CC, MVT::f32, Custom);
396  setOperationAction(ISD::SELECT_CC, MVT::f64, Custom);
397  setOperationAction(ISD::BR_JT, MVT::Other, Custom);
398  setOperationAction(ISD::JumpTable, MVT::i64, Custom);
399 
400  setOperationAction(ISD::SHL_PARTS, MVT::i64, Custom);
401  setOperationAction(ISD::SRA_PARTS, MVT::i64, Custom);
402  setOperationAction(ISD::SRL_PARTS, MVT::i64, Custom);
403 
404  setOperationAction(ISD::FREM, MVT::f32, Expand);
405  setOperationAction(ISD::FREM, MVT::f64, Expand);
406  setOperationAction(ISD::FREM, MVT::f80, Expand);
407 
408  setOperationAction(ISD::BUILD_PAIR, MVT::i64, Expand);
409 
410  // Custom lowering hooks are needed for XOR
411  // to fold it into CSINC/CSINV.
412  setOperationAction(ISD::XOR, MVT::i32, Custom);
413  setOperationAction(ISD::XOR, MVT::i64, Custom);
414 
415  // Virtually no operation on f128 is legal, but LLVM can't expand them when
416  // there's a valid register class, so we need custom operations in most cases.
417  setOperationAction(ISD::FABS, MVT::f128, Expand);
418  setOperationAction(ISD::FADD, MVT::f128, LibCall);
419  setOperationAction(ISD::FCOPYSIGN, MVT::f128, Expand);
420  setOperationAction(ISD::FCOS, MVT::f128, Expand);
421  setOperationAction(ISD::FDIV, MVT::f128, LibCall);
422  setOperationAction(ISD::FMA, MVT::f128, Expand);
423  setOperationAction(ISD::FMUL, MVT::f128, LibCall);
424  setOperationAction(ISD::FNEG, MVT::f128, Expand);
425  setOperationAction(ISD::FPOW, MVT::f128, Expand);
426  setOperationAction(ISD::FREM, MVT::f128, Expand);
427  setOperationAction(ISD::FRINT, MVT::f128, Expand);
428  setOperationAction(ISD::FSIN, MVT::f128, Expand);
429  setOperationAction(ISD::FSINCOS, MVT::f128, Expand);
430  setOperationAction(ISD::FSQRT, MVT::f128, Expand);
431  setOperationAction(ISD::FSUB, MVT::f128, LibCall);
432  setOperationAction(ISD::FTRUNC, MVT::f128, Expand);
433  setOperationAction(ISD::SETCC, MVT::f128, Custom);
434  setOperationAction(ISD::STRICT_FSETCC, MVT::f128, Custom);
435  setOperationAction(ISD::STRICT_FSETCCS, MVT::f128, Custom);
436  setOperationAction(ISD::BR_CC, MVT::f128, Custom);
437  setOperationAction(ISD::SELECT, MVT::f128, Custom);
438  setOperationAction(ISD::SELECT_CC, MVT::f128, Custom);
439  setOperationAction(ISD::FP_EXTEND, MVT::f128, Custom);
440 
441  // Lowering for many of the conversions is actually specified by the non-f128
442  // type. The LowerXXX function will be trivial when f128 isn't involved.
443  setOperationAction(ISD::FP_TO_SINT, MVT::i32, Custom);
444  setOperationAction(ISD::FP_TO_SINT, MVT::i64, Custom);
445  setOperationAction(ISD::FP_TO_SINT, MVT::i128, Custom);
446  setOperationAction(ISD::STRICT_FP_TO_SINT, MVT::i32, Custom);
447  setOperationAction(ISD::STRICT_FP_TO_SINT, MVT::i64, Custom);
448  setOperationAction(ISD::STRICT_FP_TO_SINT, MVT::i128, Custom);
449  setOperationAction(ISD::FP_TO_UINT, MVT::i32, Custom);
450  setOperationAction(ISD::FP_TO_UINT, MVT::i64, Custom);
451  setOperationAction(ISD::FP_TO_UINT, MVT::i128, Custom);
452  setOperationAction(ISD::STRICT_FP_TO_UINT, MVT::i32, Custom);
453  setOperationAction(ISD::STRICT_FP_TO_UINT, MVT::i64, Custom);
454  setOperationAction(ISD::STRICT_FP_TO_UINT, MVT::i128, Custom);
455  setOperationAction(ISD::SINT_TO_FP, MVT::i32, Custom);
456  setOperationAction(ISD::SINT_TO_FP, MVT::i64, Custom);
457  setOperationAction(ISD::SINT_TO_FP, MVT::i128, Custom);
458  setOperationAction(ISD::STRICT_SINT_TO_FP, MVT::i32, Custom);
459  setOperationAction(ISD::STRICT_SINT_TO_FP, MVT::i64, Custom);
460  setOperationAction(ISD::STRICT_SINT_TO_FP, MVT::i128, Custom);
461  setOperationAction(ISD::UINT_TO_FP, MVT::i32, Custom);
462  setOperationAction(ISD::UINT_TO_FP, MVT::i64, Custom);
463  setOperationAction(ISD::UINT_TO_FP, MVT::i128, Custom);
464  setOperationAction(ISD::STRICT_UINT_TO_FP, MVT::i32, Custom);
465  setOperationAction(ISD::STRICT_UINT_TO_FP, MVT::i64, Custom);
466  setOperationAction(ISD::STRICT_UINT_TO_FP, MVT::i128, Custom);
467  setOperationAction(ISD::FP_ROUND, MVT::f16, Custom);
468  setOperationAction(ISD::FP_ROUND, MVT::f32, Custom);
469  setOperationAction(ISD::FP_ROUND, MVT::f64, Custom);
470  setOperationAction(ISD::STRICT_FP_ROUND, MVT::f16, Custom);
471  setOperationAction(ISD::STRICT_FP_ROUND, MVT::f32, Custom);
472  setOperationAction(ISD::STRICT_FP_ROUND, MVT::f64, Custom);
473 
474  // Variable arguments.
475  setOperationAction(ISD::VASTART, MVT::Other, Custom);
476  setOperationAction(ISD::VAARG, MVT::Other, Custom);
477  setOperationAction(ISD::VACOPY, MVT::Other, Custom);
478  setOperationAction(ISD::VAEND, MVT::Other, Expand);
479 
480  // Variable-sized objects.
481  setOperationAction(ISD::STACKSAVE, MVT::Other, Expand);
482  setOperationAction(ISD::STACKRESTORE, MVT::Other, Expand);
483 
484  if (Subtarget->isTargetWindows())
485  setOperationAction(ISD::DYNAMIC_STACKALLOC, MVT::i64, Custom);
486  else
487  setOperationAction(ISD::DYNAMIC_STACKALLOC, MVT::i64, Expand);
488 
489  // Constant pool entries
490  setOperationAction(ISD::ConstantPool, MVT::i64, Custom);
491 
492  // BlockAddress
493  setOperationAction(ISD::BlockAddress, MVT::i64, Custom);
494 
495  // Add/Sub overflow ops with MVT::Glues are lowered to NZCV dependences.
496  setOperationAction(ISD::ADDC, MVT::i32, Custom);
497  setOperationAction(ISD::ADDE, MVT::i32, Custom);
498  setOperationAction(ISD::SUBC, MVT::i32, Custom);
499  setOperationAction(ISD::SUBE, MVT::i32, Custom);
500  setOperationAction(ISD::ADDC, MVT::i64, Custom);
501  setOperationAction(ISD::ADDE, MVT::i64, Custom);
502  setOperationAction(ISD::SUBC, MVT::i64, Custom);
503  setOperationAction(ISD::SUBE, MVT::i64, Custom);
504 
505  // AArch64 lacks both left-rotate and popcount instructions.
506  setOperationAction(ISD::ROTL, MVT::i32, Expand);
507  setOperationAction(ISD::ROTL, MVT::i64, Expand);
508  for (MVT VT : MVT::fixedlen_vector_valuetypes()) {
509  setOperationAction(ISD::ROTL, VT, Expand);
510  setOperationAction(ISD::ROTR, VT, Expand);
511  }
512 
513  // AArch64 doesn't have i32 MULH{S|U}.
514  setOperationAction(ISD::MULHU, MVT::i32, Expand);
515  setOperationAction(ISD::MULHS, MVT::i32, Expand);
516 
517  // AArch64 doesn't have {U|S}MUL_LOHI.
518  setOperationAction(ISD::UMUL_LOHI, MVT::i64, Expand);
519  setOperationAction(ISD::SMUL_LOHI, MVT::i64, Expand);
520 
521  setOperationAction(ISD::CTPOP, MVT::i32, Custom);
522  setOperationAction(ISD::CTPOP, MVT::i64, Custom);
523  setOperationAction(ISD::CTPOP, MVT::i128, Custom);
524 
525  setOperationAction(ISD::ABS, MVT::i32, Custom);
526  setOperationAction(ISD::ABS, MVT::i64, Custom);
527 
528  setOperationAction(ISD::SDIVREM, MVT::i32, Expand);
529  setOperationAction(ISD::SDIVREM, MVT::i64, Expand);
530  for (MVT VT : MVT::fixedlen_vector_valuetypes()) {
531  setOperationAction(ISD::SDIVREM, VT, Expand);
532  setOperationAction(ISD::UDIVREM, VT, Expand);
533  }
534  setOperationAction(ISD::SREM, MVT::i32, Expand);
535  setOperationAction(ISD::SREM, MVT::i64, Expand);
536  setOperationAction(ISD::UDIVREM, MVT::i32, Expand);
537  setOperationAction(ISD::UDIVREM, MVT::i64, Expand);
538  setOperationAction(ISD::UREM, MVT::i32, Expand);
539  setOperationAction(ISD::UREM, MVT::i64, Expand);
540 
541  // Custom lower Add/Sub/Mul with overflow.
542  setOperationAction(ISD::SADDO, MVT::i32, Custom);
543  setOperationAction(ISD::SADDO, MVT::i64, Custom);
544  setOperationAction(ISD::UADDO, MVT::i32, Custom);
545  setOperationAction(ISD::UADDO, MVT::i64, Custom);
546  setOperationAction(ISD::SSUBO, MVT::i32, Custom);
547  setOperationAction(ISD::SSUBO, MVT::i64, Custom);
548  setOperationAction(ISD::USUBO, MVT::i32, Custom);
549  setOperationAction(ISD::USUBO, MVT::i64, Custom);
550  setOperationAction(ISD::SMULO, MVT::i32, Custom);
551  setOperationAction(ISD::SMULO, MVT::i64, Custom);
552  setOperationAction(ISD::UMULO, MVT::i32, Custom);
553  setOperationAction(ISD::UMULO, MVT::i64, Custom);
554 
555  setOperationAction(ISD::FSIN, MVT::f32, Expand);
556  setOperationAction(ISD::FSIN, MVT::f64, Expand);
557  setOperationAction(ISD::FCOS, MVT::f32, Expand);
558  setOperationAction(ISD::FCOS, MVT::f64, Expand);
559  setOperationAction(ISD::FPOW, MVT::f32, Expand);
560  setOperationAction(ISD::FPOW, MVT::f64, Expand);
561  setOperationAction(ISD::FCOPYSIGN, MVT::f64, Custom);
562  setOperationAction(ISD::FCOPYSIGN, MVT::f32, Custom);
563  if (Subtarget->hasFullFP16())
564  setOperationAction(ISD::FCOPYSIGN, MVT::f16, Custom);
565  else
566  setOperationAction(ISD::FCOPYSIGN, MVT::f16, Promote);
567 
568  setOperationAction(ISD::FREM, MVT::f16, Promote);
569  setOperationAction(ISD::FREM, MVT::v4f16, Expand);
570  setOperationAction(ISD::FREM, MVT::v8f16, Expand);
571  setOperationAction(ISD::FPOW, MVT::f16, Promote);
572  setOperationAction(ISD::FPOW, MVT::v4f16, Expand);
573  setOperationAction(ISD::FPOW, MVT::v8f16, Expand);
574  setOperationAction(ISD::FPOWI, MVT::f16, Promote);
575  setOperationAction(ISD::FPOWI, MVT::v4f16, Expand);
576  setOperationAction(ISD::FPOWI, MVT::v8f16, Expand);
577  setOperationAction(ISD::FCOS, MVT::f16, Promote);
578  setOperationAction(ISD::FCOS, MVT::v4f16, Expand);
579  setOperationAction(ISD::FCOS, MVT::v8f16, Expand);
580  setOperationAction(ISD::FSIN, MVT::f16, Promote);
581  setOperationAction(ISD::FSIN, MVT::v4f16, Expand);
582  setOperationAction(ISD::FSIN, MVT::v8f16, Expand);
583  setOperationAction(ISD::FSINCOS, MVT::f16, Promote);
584  setOperationAction(ISD::FSINCOS, MVT::v4f16, Expand);
585  setOperationAction(ISD::FSINCOS, MVT::v8f16, Expand);
586  setOperationAction(ISD::FEXP, MVT::f16, Promote);
587  setOperationAction(ISD::FEXP, MVT::v4f16, Expand);
588  setOperationAction(ISD::FEXP, MVT::v8f16, Expand);
589  setOperationAction(ISD::FEXP2, MVT::f16, Promote);
590  setOperationAction(ISD::FEXP2, MVT::v4f16, Expand);
591  setOperationAction(ISD::FEXP2, MVT::v8f16, Expand);
592  setOperationAction(ISD::FLOG, MVT::f16, Promote);
593  setOperationAction(ISD::FLOG, MVT::v4f16, Expand);
594  setOperationAction(ISD::FLOG, MVT::v8f16, Expand);
595  setOperationAction(ISD::FLOG2, MVT::f16, Promote);
596  setOperationAction(ISD::FLOG2, MVT::v4f16, Expand);
597  setOperationAction(ISD::FLOG2, MVT::v8f16, Expand);
598  setOperationAction(ISD::FLOG10, MVT::f16, Promote);
599  setOperationAction(ISD::FLOG10, MVT::v4f16, Expand);
600  setOperationAction(ISD::FLOG10, MVT::v8f16, Expand);
601 
602  if (!Subtarget->hasFullFP16()) {
603  setOperationAction(ISD::SELECT, MVT::f16, Promote);
604  setOperationAction(ISD::SELECT_CC, MVT::f16, Promote);
605  setOperationAction(ISD::SETCC, MVT::f16, Promote);
606  setOperationAction(ISD::BR_CC, MVT::f16, Promote);
607  setOperationAction(ISD::FADD, MVT::f16, Promote);
608  setOperationAction(ISD::FSUB, MVT::f16, Promote);
609  setOperationAction(ISD::FMUL, MVT::f16, Promote);
610  setOperationAction(ISD::FDIV, MVT::f16, Promote);
611  setOperationAction(ISD::FMA, MVT::f16, Promote);
612  setOperationAction(ISD::FNEG, MVT::f16, Promote);
613  setOperationAction(ISD::FABS, MVT::f16, Promote);
614  setOperationAction(ISD::FCEIL, MVT::f16, Promote);
615  setOperationAction(ISD::FSQRT, MVT::f16, Promote);
616  setOperationAction(ISD::FFLOOR, MVT::f16, Promote);
617  setOperationAction(ISD::FNEARBYINT, MVT::f16, Promote);
618  setOperationAction(ISD::FRINT, MVT::f16, Promote);
619  setOperationAction(ISD::FROUND, MVT::f16, Promote);
620  setOperationAction(ISD::FROUNDEVEN, MVT::f16, Promote);
621  setOperationAction(ISD::FTRUNC, MVT::f16, Promote);
622  setOperationAction(ISD::FMINNUM, MVT::f16, Promote);
623  setOperationAction(ISD::FMAXNUM, MVT::f16, Promote);
624  setOperationAction(ISD::FMINIMUM, MVT::f16, Promote);
625  setOperationAction(ISD::FMAXIMUM, MVT::f16, Promote);
626 
627  // promote v4f16 to v4f32 when that is known to be safe.
628  setOperationAction(ISD::FADD, MVT::v4f16, Promote);
629  setOperationAction(ISD::FSUB, MVT::v4f16, Promote);
630  setOperationAction(ISD::FMUL, MVT::v4f16, Promote);
631  setOperationAction(ISD::FDIV, MVT::v4f16, Promote);
632  AddPromotedToType(ISD::FADD, MVT::v4f16, MVT::v4f32);
633  AddPromotedToType(ISD::FSUB, MVT::v4f16, MVT::v4f32);
634  AddPromotedToType(ISD::FMUL, MVT::v4f16, MVT::v4f32);
635  AddPromotedToType(ISD::FDIV, MVT::v4f16, MVT::v4f32);
636 
637  setOperationAction(ISD::FABS, MVT::v4f16, Expand);
638  setOperationAction(ISD::FNEG, MVT::v4f16, Expand);
639  setOperationAction(ISD::FROUND, MVT::v4f16, Expand);
640  setOperationAction(ISD::FROUNDEVEN, MVT::v4f16, Expand);
641  setOperationAction(ISD::FMA, MVT::v4f16, Expand);
642  setOperationAction(ISD::SETCC, MVT::v4f16, Expand);
643  setOperationAction(ISD::BR_CC, MVT::v4f16, Expand);
644  setOperationAction(ISD::SELECT, MVT::v4f16, Expand);
645  setOperationAction(ISD::SELECT_CC, MVT::v4f16, Expand);
646  setOperationAction(ISD::FTRUNC, MVT::v4f16, Expand);
647  setOperationAction(ISD::FCOPYSIGN, MVT::v4f16, Expand);
648  setOperationAction(ISD::FFLOOR, MVT::v4f16, Expand);
649  setOperationAction(ISD::FCEIL, MVT::v4f16, Expand);
650  setOperationAction(ISD::FRINT, MVT::v4f16, Expand);
651  setOperationAction(ISD::FNEARBYINT, MVT::v4f16, Expand);
652  setOperationAction(ISD::FSQRT, MVT::v4f16, Expand);
653 
654  setOperationAction(ISD::FABS, MVT::v8f16, Expand);
655  setOperationAction(ISD::FADD, MVT::v8f16, Expand);
656  setOperationAction(ISD::FCEIL, MVT::v8f16, Expand);
657  setOperationAction(ISD::FCOPYSIGN, MVT::v8f16, Expand);
658  setOperationAction(ISD::FDIV, MVT::v8f16, Expand);
659  setOperationAction(ISD::FFLOOR, MVT::v8f16, Expand);
660  setOperationAction(ISD::FMA, MVT::v8f16, Expand);
661  setOperationAction(ISD::FMUL, MVT::v8f16, Expand);
662  setOperationAction(ISD::FNEARBYINT, MVT::v8f16, Expand);
663  setOperationAction(ISD::FNEG, MVT::v8f16, Expand);
664  setOperationAction(ISD::FROUND, MVT::v8f16, Expand);
665  setOperationAction(ISD::FROUNDEVEN, MVT::v8f16, Expand);
666  setOperationAction(ISD::FRINT, MVT::v8f16, Expand);
667  setOperationAction(ISD::FSQRT, MVT::v8f16, Expand);
668  setOperationAction(ISD::FSUB, MVT::v8f16, Expand);
669  setOperationAction(ISD::FTRUNC, MVT::v8f16, Expand);
670  setOperationAction(ISD::SETCC, MVT::v8f16, Expand);
671  setOperationAction(ISD::BR_CC, MVT::v8f16, Expand);
672  setOperationAction(ISD::SELECT, MVT::v8f16, Expand);
673  setOperationAction(ISD::SELECT_CC, MVT::v8f16, Expand);
674  setOperationAction(ISD::FP_EXTEND, MVT::v8f16, Expand);
675  }
676 
677  // AArch64 has implementations of a lot of rounding-like FP operations.
678  for (MVT Ty : {MVT::f32, MVT::f64}) {
679  setOperationAction(ISD::FFLOOR, Ty, Legal);
680  setOperationAction(ISD::FNEARBYINT, Ty, Legal);
681  setOperationAction(ISD::FCEIL, Ty, Legal);
682  setOperationAction(ISD::FRINT, Ty, Legal);
683  setOperationAction(ISD::FTRUNC, Ty, Legal);
684  setOperationAction(ISD::FROUND, Ty, Legal);
685  setOperationAction(ISD::FROUNDEVEN, Ty, Legal);
686  setOperationAction(ISD::FMINNUM, Ty, Legal);
687  setOperationAction(ISD::FMAXNUM, Ty, Legal);
688  setOperationAction(ISD::FMINIMUM, Ty, Legal);
689  setOperationAction(ISD::FMAXIMUM, Ty, Legal);
690  setOperationAction(ISD::LROUND, Ty, Legal);
691  setOperationAction(ISD::LLROUND, Ty, Legal);
692  setOperationAction(ISD::LRINT, Ty, Legal);
693  setOperationAction(ISD::LLRINT, Ty, Legal);
694  }
695 
696  if (Subtarget->hasFullFP16()) {
697  setOperationAction(ISD::FNEARBYINT, MVT::f16, Legal);
698  setOperationAction(ISD::FFLOOR, MVT::f16, Legal);
699  setOperationAction(ISD::FCEIL, MVT::f16, Legal);
700  setOperationAction(ISD::FRINT, MVT::f16, Legal);
701  setOperationAction(ISD::FTRUNC, MVT::f16, Legal);
702  setOperationAction(ISD::FROUND, MVT::f16, Legal);
703  setOperationAction(ISD::FROUNDEVEN, MVT::f16, Legal);
704  setOperationAction(ISD::FMINNUM, MVT::f16, Legal);
705  setOperationAction(ISD::FMAXNUM, MVT::f16, Legal);
706  setOperationAction(ISD::FMINIMUM, MVT::f16, Legal);
707  setOperationAction(ISD::FMAXIMUM, MVT::f16, Legal);
708  }
709 
710  setOperationAction(ISD::PREFETCH, MVT::Other, Custom);
711 
712  setOperationAction(ISD::FLT_ROUNDS_, MVT::i32, Custom);
713  setOperationAction(ISD::SET_ROUNDING, MVT::Other, Custom);
714 
715  setOperationAction(ISD::ATOMIC_CMP_SWAP, MVT::i128, Custom);
716  setOperationAction(ISD::ATOMIC_LOAD_SUB, MVT::i32, Custom);
717  setOperationAction(ISD::ATOMIC_LOAD_SUB, MVT::i64, Custom);
718  setOperationAction(ISD::ATOMIC_LOAD_AND, MVT::i32, Custom);
719  setOperationAction(ISD::ATOMIC_LOAD_AND, MVT::i64, Custom);
720 
721  // Generate outline atomics library calls only if LSE was not specified for
722  // subtarget
723  if (Subtarget->outlineAtomics() && !Subtarget->hasLSE()) {
724  setOperationAction(ISD::ATOMIC_CMP_SWAP, MVT::i8, LibCall);
725  setOperationAction(ISD::ATOMIC_CMP_SWAP, MVT::i16, LibCall);
726  setOperationAction(ISD::ATOMIC_CMP_SWAP, MVT::i32, LibCall);
727  setOperationAction(ISD::ATOMIC_CMP_SWAP, MVT::i64, LibCall);
728  setOperationAction(ISD::ATOMIC_CMP_SWAP, MVT::i128, LibCall);
729  setOperationAction(ISD::ATOMIC_SWAP, MVT::i8, LibCall);
730  setOperationAction(ISD::ATOMIC_SWAP, MVT::i16, LibCall);
731  setOperationAction(ISD::ATOMIC_SWAP, MVT::i32, LibCall);
732  setOperationAction(ISD::ATOMIC_SWAP, MVT::i64, LibCall);
733  setOperationAction(ISD::ATOMIC_LOAD_ADD, MVT::i8, LibCall);
734  setOperationAction(ISD::ATOMIC_LOAD_ADD, MVT::i16, LibCall);
735  setOperationAction(ISD::ATOMIC_LOAD_ADD, MVT::i32, LibCall);
736  setOperationAction(ISD::ATOMIC_LOAD_ADD, MVT::i64, LibCall);
737  setOperationAction(ISD::ATOMIC_LOAD_OR, MVT::i8, LibCall);
738  setOperationAction(ISD::ATOMIC_LOAD_OR, MVT::i16, LibCall);
739  setOperationAction(ISD::ATOMIC_LOAD_OR, MVT::i32, LibCall);
740  setOperationAction(ISD::ATOMIC_LOAD_OR, MVT::i64, LibCall);
741  setOperationAction(ISD::ATOMIC_LOAD_CLR, MVT::i8, LibCall);
742  setOperationAction(ISD::ATOMIC_LOAD_CLR, MVT::i16, LibCall);
743  setOperationAction(ISD::ATOMIC_LOAD_CLR, MVT::i32, LibCall);
744  setOperationAction(ISD::ATOMIC_LOAD_CLR, MVT::i64, LibCall);
745  setOperationAction(ISD::ATOMIC_LOAD_XOR, MVT::i8, LibCall);
746  setOperationAction(ISD::ATOMIC_LOAD_XOR, MVT::i16, LibCall);
747  setOperationAction(ISD::ATOMIC_LOAD_XOR, MVT::i32, LibCall);
748  setOperationAction(ISD::ATOMIC_LOAD_XOR, MVT::i64, LibCall);
749 #define LCALLNAMES(A, B, N) \
750  setLibcallName(A##N##_RELAX, #B #N "_relax"); \
751  setLibcallName(A##N##_ACQ, #B #N "_acq"); \
752  setLibcallName(A##N##_REL, #B #N "_rel"); \
753  setLibcallName(A##N##_ACQ_REL, #B #N "_acq_rel");
754 #define LCALLNAME4(A, B) \
755  LCALLNAMES(A, B, 1) \
756  LCALLNAMES(A, B, 2) LCALLNAMES(A, B, 4) LCALLNAMES(A, B, 8)
757 #define LCALLNAME5(A, B) \
758  LCALLNAMES(A, B, 1) \
759  LCALLNAMES(A, B, 2) \
760  LCALLNAMES(A, B, 4) LCALLNAMES(A, B, 8) LCALLNAMES(A, B, 16)
761  LCALLNAME5(RTLIB::OUTLINE_ATOMIC_CAS, __aarch64_cas)
762  LCALLNAME4(RTLIB::OUTLINE_ATOMIC_SWP, __aarch64_swp)
763  LCALLNAME4(RTLIB::OUTLINE_ATOMIC_LDADD, __aarch64_ldadd)
764  LCALLNAME4(RTLIB::OUTLINE_ATOMIC_LDSET, __aarch64_ldset)
765  LCALLNAME4(RTLIB::OUTLINE_ATOMIC_LDCLR, __aarch64_ldclr)
766  LCALLNAME4(RTLIB::OUTLINE_ATOMIC_LDEOR, __aarch64_ldeor)
767 #undef LCALLNAMES
768 #undef LCALLNAME4
769 #undef LCALLNAME5
770  }
771 
772  // 128-bit loads and stores can be done without expanding
773  setOperationAction(ISD::LOAD, MVT::i128, Custom);
774  setOperationAction(ISD::STORE, MVT::i128, Custom);
775 
776  // 256 bit non-temporal stores can be lowered to STNP. Do this as part of the
777  // custom lowering, as there are no un-paired non-temporal stores and
778  // legalization will break up 256 bit inputs.
779  setOperationAction(ISD::STORE, MVT::v32i8, Custom);
780  setOperationAction(ISD::STORE, MVT::v16i16, Custom);
781  setOperationAction(ISD::STORE, MVT::v16f16, Custom);
782  setOperationAction(ISD::STORE, MVT::v8i32, Custom);
783  setOperationAction(ISD::STORE, MVT::v8f32, Custom);
784  setOperationAction(ISD::STORE, MVT::v4f64, Custom);
785  setOperationAction(ISD::STORE, MVT::v4i64, Custom);
786 
787  // Lower READCYCLECOUNTER using an mrs from PMCCNTR_EL0.
788  // This requires the Performance Monitors extension.
789  if (Subtarget->hasPerfMon())
790  setOperationAction(ISD::READCYCLECOUNTER, MVT::i64, Legal);
791 
792  if (getLibcallName(RTLIB::SINCOS_STRET_F32) != nullptr &&
793  getLibcallName(RTLIB::SINCOS_STRET_F64) != nullptr) {
794  // Issue __sincos_stret if available.
795  setOperationAction(ISD::FSINCOS, MVT::f64, Custom);
796  setOperationAction(ISD::FSINCOS, MVT::f32, Custom);
797  } else {
798  setOperationAction(ISD::FSINCOS, MVT::f64, Expand);
799  setOperationAction(ISD::FSINCOS, MVT::f32, Expand);
800  }
801 
802  if (Subtarget->getTargetTriple().isOSMSVCRT()) {
803  // MSVCRT doesn't have powi; fall back to pow
804  setLibcallName(RTLIB::POWI_F32, nullptr);
805  setLibcallName(RTLIB::POWI_F64, nullptr);
806  }
807 
808  // Make floating-point constants legal for the large code model, so they don't
809  // become loads from the constant pool.
810  if (Subtarget->isTargetMachO() && TM.getCodeModel() == CodeModel::Large) {
811  setOperationAction(ISD::ConstantFP, MVT::f32, Legal);
812  setOperationAction(ISD::ConstantFP, MVT::f64, Legal);
813  }
814 
815  // AArch64 does not have floating-point extending loads, i1 sign-extending
816  // load, floating-point truncating stores, or v2i32->v2i16 truncating store.
817  for (MVT VT : MVT::fp_valuetypes()) {
818  setLoadExtAction(ISD::EXTLOAD, VT, MVT::f16, Expand);
819  setLoadExtAction(ISD::EXTLOAD, VT, MVT::f32, Expand);
820  setLoadExtAction(ISD::EXTLOAD, VT, MVT::f64, Expand);
821  setLoadExtAction(ISD::EXTLOAD, VT, MVT::f80, Expand);
822  }
823  for (MVT VT : MVT::integer_valuetypes())
824  setLoadExtAction(ISD::SEXTLOAD, VT, MVT::i1, Expand);
825 
826  setTruncStoreAction(MVT::f32, MVT::f16, Expand);
827  setTruncStoreAction(MVT::f64, MVT::f32, Expand);
828  setTruncStoreAction(MVT::f64, MVT::f16, Expand);
829  setTruncStoreAction(MVT::f128, MVT::f80, Expand);
830  setTruncStoreAction(MVT::f128, MVT::f64, Expand);
831  setTruncStoreAction(MVT::f128, MVT::f32, Expand);
832  setTruncStoreAction(MVT::f128, MVT::f16, Expand);
833 
834  setOperationAction(ISD::BITCAST, MVT::i16, Custom);
835  setOperationAction(ISD::BITCAST, MVT::f16, Custom);
836  setOperationAction(ISD::BITCAST, MVT::bf16, Custom);
837 
838  // Indexed loads and stores are supported.
839  for (unsigned im = (unsigned)ISD::PRE_INC;
840  im != (unsigned)ISD::LAST_INDEXED_MODE; ++im) {
841  setIndexedLoadAction(im, MVT::i8, Legal);
842  setIndexedLoadAction(im, MVT::i16, Legal);
843  setIndexedLoadAction(im, MVT::i32, Legal);
844  setIndexedLoadAction(im, MVT::i64, Legal);
845  setIndexedLoadAction(im, MVT::f64, Legal);
846  setIndexedLoadAction(im, MVT::f32, Legal);
847  setIndexedLoadAction(im, MVT::f16, Legal);
848  setIndexedLoadAction(im, MVT::bf16, Legal);
849  setIndexedStoreAction(im, MVT::i8, Legal);
850  setIndexedStoreAction(im, MVT::i16, Legal);
851  setIndexedStoreAction(im, MVT::i32, Legal);
852  setIndexedStoreAction(im, MVT::i64, Legal);
853  setIndexedStoreAction(im, MVT::f64, Legal);
854  setIndexedStoreAction(im, MVT::f32, Legal);
855  setIndexedStoreAction(im, MVT::f16, Legal);
856  setIndexedStoreAction(im, MVT::bf16, Legal);
857  }
858 
859  // Trap.
860  setOperationAction(ISD::TRAP, MVT::Other, Legal);
861  setOperationAction(ISD::DEBUGTRAP, MVT::Other, Legal);
862  setOperationAction(ISD::UBSANTRAP, MVT::Other, Legal);
863 
864  // We combine OR nodes for bitfield operations.
865  setTargetDAGCombine(ISD::OR);
866  // Try to create BICs for vector ANDs.
867  setTargetDAGCombine(ISD::AND);
868 
869  // Vector add and sub nodes may conceal a high-half opportunity.
870  // Also, try to fold ADD into CSINC/CSINV..
871  setTargetDAGCombine(ISD::ADD);
872  setTargetDAGCombine(ISD::ABS);
873  setTargetDAGCombine(ISD::SUB);
874  setTargetDAGCombine(ISD::SRL);
875  setTargetDAGCombine(ISD::XOR);
876  setTargetDAGCombine(ISD::SINT_TO_FP);
877  setTargetDAGCombine(ISD::UINT_TO_FP);
878 
879  setTargetDAGCombine(ISD::FP_TO_SINT);
880  setTargetDAGCombine(ISD::FP_TO_UINT);
881  setTargetDAGCombine(ISD::FDIV);
882 
883  setTargetDAGCombine(ISD::INTRINSIC_WO_CHAIN);
884 
885  setTargetDAGCombine(ISD::ANY_EXTEND);
886  setTargetDAGCombine(ISD::ZERO_EXTEND);
887  setTargetDAGCombine(ISD::SIGN_EXTEND);
888  setTargetDAGCombine(ISD::SIGN_EXTEND_INREG);
889  setTargetDAGCombine(ISD::TRUNCATE);
890  setTargetDAGCombine(ISD::CONCAT_VECTORS);
891  setTargetDAGCombine(ISD::STORE);
892  if (Subtarget->supportsAddressTopByteIgnored())
893  setTargetDAGCombine(ISD::LOAD);
894 
895  setTargetDAGCombine(ISD::MUL);
896 
897  setTargetDAGCombine(ISD::SELECT);
898  setTargetDAGCombine(ISD::VSELECT);
899 
900  setTargetDAGCombine(ISD::INTRINSIC_VOID);
901  setTargetDAGCombine(ISD::INTRINSIC_W_CHAIN);
902  setTargetDAGCombine(ISD::INSERT_VECTOR_ELT);
903  setTargetDAGCombine(ISD::EXTRACT_VECTOR_ELT);
904  setTargetDAGCombine(ISD::VECREDUCE_ADD);
905 
906  setTargetDAGCombine(ISD::GlobalAddress);
907 
908  // In case of strict alignment, avoid an excessive number of byte wide stores.
909  MaxStoresPerMemsetOptSize = 8;
910  MaxStoresPerMemset = Subtarget->requiresStrictAlign()
911  ? MaxStoresPerMemsetOptSize : 32;
912 
913  MaxGluedStoresPerMemcpy = 4;
914  MaxStoresPerMemcpyOptSize = 4;
915  MaxStoresPerMemcpy = Subtarget->requiresStrictAlign()
916  ? MaxStoresPerMemcpyOptSize : 16;
917 
918  MaxStoresPerMemmoveOptSize = MaxStoresPerMemmove = 4;
919 
920  MaxLoadsPerMemcmpOptSize = 4;
921  MaxLoadsPerMemcmp = Subtarget->requiresStrictAlign()
922  ? MaxLoadsPerMemcmpOptSize : 8;
923 
924  setStackPointerRegisterToSaveRestore(AArch64::SP);
925 
926  setSchedulingPreference(Sched::Hybrid);
927 
928  EnableExtLdPromotion = true;
929 
930  // Set required alignment.
931  setMinFunctionAlignment(Align(4));
932  // Set preferred alignments.
933  setPrefLoopAlignment(Align(1ULL << STI.getPrefLoopLogAlignment()));
934  setPrefFunctionAlignment(Align(1ULL << STI.getPrefFunctionLogAlignment()));
935 
936  // Only change the limit for entries in a jump table if specified by
937  // the sub target, but not at the command line.
938  unsigned MaxJT = STI.getMaximumJumpTableSize();
939  if (MaxJT && getMaximumJumpTableSize() == UINT_MAX)
940  setMaximumJumpTableSize(MaxJT);
941 
942  setHasExtractBitsInsn(true);
943 
944  setOperationAction(ISD::INTRINSIC_WO_CHAIN, MVT::Other, Custom);
945 
946  if (Subtarget->hasNEON()) {
947  // FIXME: v1f64 shouldn't be legal if we can avoid it, because it leads to
948  // silliness like this:
949  setOperationAction(ISD::FABS, MVT::v1f64, Expand);
950  setOperationAction(ISD::FADD, MVT::v1f64, Expand);
951  setOperationAction(ISD::FCEIL, MVT::v1f64, Expand);
952  setOperationAction(ISD::FCOPYSIGN, MVT::v1f64, Expand);
953  setOperationAction(ISD::FCOS, MVT::v1f64, Expand);
954  setOperationAction(ISD::FDIV, MVT::v1f64, Expand);
955  setOperationAction(ISD::FFLOOR, MVT::v1f64, Expand);
956  setOperationAction(ISD::FMA, MVT::v1f64, Expand);
957  setOperationAction(ISD::FMUL, MVT::v1f64, Expand);
958  setOperationAction(ISD::FNEARBYINT, MVT::v1f64, Expand);
959  setOperationAction(ISD::FNEG, MVT::v1f64, Expand);
960  setOperationAction(ISD::FPOW, MVT::v1f64, Expand);
961  setOperationAction(ISD::FREM, MVT::v1f64, Expand);
962  setOperationAction(ISD::FROUND, MVT::v1f64, Expand);
963  setOperationAction(ISD::FROUNDEVEN, MVT::v1f64, Expand);
964  setOperationAction(ISD::FRINT, MVT::v1f64, Expand);
965  setOperationAction(ISD::FSIN, MVT::v1f64, Expand);
966  setOperationAction(ISD::FSINCOS, MVT::v1f64, Expand);
967  setOperationAction(ISD::FSQRT, MVT::v1f64, Expand);
968  setOperationAction(ISD::FSUB, MVT::v1f64, Expand);
969  setOperationAction(ISD::FTRUNC, MVT::v1f64, Expand);
970  setOperationAction(ISD::SETCC, MVT::v1f64, Expand);
971  setOperationAction(ISD::BR_CC, MVT::v1f64, Expand);
972  setOperationAction(ISD::SELECT, MVT::v1f64, Expand);
973  setOperationAction(ISD::SELECT_CC, MVT::v1f64, Expand);
974  setOperationAction(ISD::FP_EXTEND, MVT::v1f64, Expand);
975 
976  setOperationAction(ISD::FP_TO_SINT, MVT::v1i64, Expand);
977  setOperationAction(ISD::FP_TO_UINT, MVT::v1i64, Expand);
978  setOperationAction(ISD::SINT_TO_FP, MVT::v1i64, Expand);
979  setOperationAction(ISD::UINT_TO_FP, MVT::v1i64, Expand);
980  setOperationAction(ISD::FP_ROUND, MVT::v1f64, Expand);
981 
982  setOperationAction(ISD::MUL, MVT::v1i64, Expand);
983 
984  // AArch64 doesn't have a direct vector ->f32 conversion instructions for
985  // elements smaller than i32, so promote the input to i32 first.
986  setOperationPromotedToType(ISD::UINT_TO_FP, MVT::v4i8, MVT::v4i32);
987  setOperationPromotedToType(ISD::SINT_TO_FP, MVT::v4i8, MVT::v4i32);
988  // i8 vector elements also need promotion to i32 for v8i8
989  setOperationPromotedToType(ISD::SINT_TO_FP, MVT::v8i8, MVT::v8i32);
990  setOperationPromotedToType(ISD::UINT_TO_FP, MVT::v8i8, MVT::v8i32);
991  // Similarly, there is no direct i32 -> f64 vector conversion instruction.
992  setOperationAction(ISD::SINT_TO_FP, MVT::v2i32, Custom);
993  setOperationAction(ISD::UINT_TO_FP, MVT::v2i32, Custom);
994  setOperationAction(ISD::SINT_TO_FP, MVT::v2i64, Custom);
995  setOperationAction(ISD::UINT_TO_FP, MVT::v2i64, Custom);
996  // Or, direct i32 -> f16 vector conversion. Set it so custom, so the
997  // conversion happens in two steps: v4i32 -> v4f32 -> v4f16
998  setOperationAction(ISD::SINT_TO_FP, MVT::v4i32, Custom);
999  setOperationAction(ISD::UINT_TO_FP, MVT::v4i32, Custom);
1000 
1001  if (Subtarget->hasFullFP16()) {
1002  setOperationAction(ISD::SINT_TO_FP, MVT::v4i16, Custom);
1003  setOperationAction(ISD::UINT_TO_FP, MVT::v4i16, Custom);
1004  setOperationAction(ISD::SINT_TO_FP, MVT::v8i16, Custom);
1005  setOperationAction(ISD::UINT_TO_FP, MVT::v8i16, Custom);
1006  } else {
1007  // when AArch64 doesn't have fullfp16 support, promote the input
1008  // to i32 first.
1009  setOperationPromotedToType(ISD::UINT_TO_FP, MVT::v4i16, MVT::v4i32);
1010  setOperationPromotedToType(ISD::SINT_TO_FP, MVT::v4i16, MVT::v4i32);
1011  setOperationPromotedToType(ISD::SINT_TO_FP, MVT::v8i16, MVT::v8i32);
1012  setOperationPromotedToType(ISD::UINT_TO_FP, MVT::v8i16, MVT::v8i32);
1013  }
1014 
1015  setOperationAction(ISD::CTLZ, MVT::v1i64, Expand);
1016  setOperationAction(ISD::CTLZ, MVT::v2i64, Expand);
1017 
1018  // AArch64 doesn't have MUL.2d:
1019  setOperationAction(ISD::MUL, MVT::v2i64, Expand);
1020  // Custom handling for some quad-vector types to detect MULL.
1021  setOperationAction(ISD::MUL, MVT::v8i16, Custom);
1022  setOperationAction(ISD::MUL, MVT::v4i32, Custom);
1023  setOperationAction(ISD::MUL, MVT::v2i64, Custom);
1024 
1025  // Saturates
1026  for (MVT VT : { MVT::v8i8, MVT::v4i16, MVT::v2i32,
1027  MVT::v16i8, MVT::v8i16, MVT::v4i32, MVT::v2i64 }) {
1028  setOperationAction(ISD::SADDSAT, VT, Legal);
1029  setOperationAction(ISD::UADDSAT, VT, Legal);
1030  setOperationAction(ISD::SSUBSAT, VT, Legal);
1031  setOperationAction(ISD::USUBSAT, VT, Legal);
1032  }
1033 
1034  // Vector reductions
1035  for (MVT VT : { MVT::v4f16, MVT::v2f32,
1036  MVT::v8f16, MVT::v4f32, MVT::v2f64 }) {
1037  if (VT.getVectorElementType() != MVT::f16 || Subtarget->hasFullFP16()) {
1038  setOperationAction(ISD::VECREDUCE_FMAX, VT, Custom);
1039  setOperationAction(ISD::VECREDUCE_FMIN, VT, Custom);
1040 
1041  setOperationAction(ISD::VECREDUCE_FADD, VT, Legal);
1042  }
1043  }
1044  for (MVT VT : { MVT::v8i8, MVT::v4i16, MVT::v2i32,
1045  MVT::v16i8, MVT::v8i16, MVT::v4i32 }) {
1046  setOperationAction(ISD::VECREDUCE_ADD, VT, Custom);
1047  setOperationAction(ISD::VECREDUCE_SMAX, VT, Custom);
1048  setOperationAction(ISD::VECREDUCE_SMIN, VT, Custom);
1049  setOperationAction(ISD::VECREDUCE_UMAX, VT, Custom);
1050  setOperationAction(ISD::VECREDUCE_UMIN, VT, Custom);
1051  }
1052  setOperationAction(ISD::VECREDUCE_ADD, MVT::v2i64, Custom);
1053 
1054  setOperationAction(ISD::ANY_EXTEND, MVT::v4i32, Legal);
1055  setTruncStoreAction(MVT::v2i32, MVT::v2i16, Expand);
1056  // Likewise, narrowing and extending vector loads/stores aren't handled
1057  // directly.
1058  for (MVT VT : MVT::fixedlen_vector_valuetypes()) {
1059  setOperationAction(ISD::SIGN_EXTEND_INREG, VT, Expand);
1060 
1061  if (VT == MVT::v16i8 || VT == MVT::v8i16 || VT == MVT::v4i32) {
1062  setOperationAction(ISD::MULHS, VT, Legal);
1063  setOperationAction(ISD::MULHU, VT, Legal);
1064  } else {
1065  setOperationAction(ISD::MULHS, VT, Expand);
1066  setOperationAction(ISD::MULHU, VT, Expand);
1067  }
1068  setOperationAction(ISD::SMUL_LOHI, VT, Expand);
1069  setOperationAction(ISD::UMUL_LOHI, VT, Expand);
1070 
1071  setOperationAction(ISD::BSWAP, VT, Expand);
1072  setOperationAction(ISD::CTTZ, VT, Expand);
1073 
1074  for (MVT InnerVT : MVT::fixedlen_vector_valuetypes()) {
1075  setTruncStoreAction(VT, InnerVT, Expand);
1076  setLoadExtAction(ISD::SEXTLOAD, VT, InnerVT, Expand);
1077  setLoadExtAction(ISD::ZEXTLOAD, VT, InnerVT, Expand);
1078  setLoadExtAction(ISD::EXTLOAD, VT, InnerVT, Expand);
1079  }
1080  }
1081 
1082  // AArch64 has implementations of a lot of rounding-like FP operations.
1083  for (MVT Ty : {MVT::v2f32, MVT::v4f32, MVT::v2f64}) {
1084  setOperationAction(ISD::FFLOOR, Ty, Legal);
1085  setOperationAction(ISD::FNEARBYINT, Ty, Legal);
1086  setOperationAction(ISD::FCEIL, Ty, Legal);
1087  setOperationAction(ISD::FRINT, Ty, Legal);
1088  setOperationAction(ISD::FTRUNC, Ty, Legal);
1089  setOperationAction(ISD::FROUND, Ty, Legal);
1090  setOperationAction(ISD::FROUNDEVEN, Ty, Legal);
1091  }
1092 
1093  if (Subtarget->hasFullFP16()) {
1094  for (MVT Ty : {MVT::v4f16, MVT::v8f16}) {
1095  setOperationAction(ISD::FFLOOR, Ty, Legal);
1096  setOperationAction(ISD::FNEARBYINT, Ty, Legal);
1097  setOperationAction(ISD::FCEIL, Ty, Legal);
1098  setOperationAction(ISD::FRINT, Ty, Legal);
1099  setOperationAction(ISD::FTRUNC, Ty, Legal);
1100  setOperationAction(ISD::FROUND, Ty, Legal);
1101  setOperationAction(ISD::FROUNDEVEN, Ty, Legal);
1102  }
1103  }
1104 
1105  if (Subtarget->hasSVE())
1106  setOperationAction(ISD::VSCALE, MVT::i32, Custom);
1107 
1108  setTruncStoreAction(MVT::v4i16, MVT::v4i8, Custom);
1109  }
1110 
1111  if (Subtarget->hasSVE()) {
1112  // FIXME: Add custom lowering of MLOAD to handle different passthrus (not a
1113  // splat of 0 or undef) once vector selects supported in SVE codegen. See
1114  // D68877 for more details.
1115  for (auto VT : {MVT::nxv16i8, MVT::nxv8i16, MVT::nxv4i32, MVT::nxv2i64}) {
1116  setOperationAction(ISD::BITREVERSE, VT, Custom);
1117  setOperationAction(ISD::BSWAP, VT, Custom);
1118  setOperationAction(ISD::CTLZ, VT, Custom);
1119  setOperationAction(ISD::CTPOP, VT, Custom);
1120  setOperationAction(ISD::CTTZ, VT, Custom);
1121  setOperationAction(ISD::INSERT_SUBVECTOR, VT, Custom);
1122  setOperationAction(ISD::UINT_TO_FP, VT, Custom);
1123  setOperationAction(ISD::SINT_TO_FP, VT, Custom);
1124  setOperationAction(ISD::FP_TO_UINT, VT, Custom);
1125  setOperationAction(ISD::FP_TO_SINT, VT, Custom);
1126  setOperationAction(ISD::MGATHER, VT, Custom);
1127  setOperationAction(ISD::MSCATTER, VT, Custom);
1128  setOperationAction(ISD::MUL, VT, Custom);
1129  setOperationAction(ISD::MULHS, VT, Custom);
1130  setOperationAction(ISD::MULHU, VT, Custom);
1131  setOperationAction(ISD::SPLAT_VECTOR, VT, Custom);
1132  setOperationAction(ISD::SELECT, VT, Custom);
1133  setOperationAction(ISD::SETCC, VT, Custom);
1134  setOperationAction(ISD::SDIV, VT, Custom);
1135  setOperationAction(ISD::UDIV, VT, Custom);
1136  setOperationAction(ISD::SMIN, VT, Custom);
1137  setOperationAction(ISD::UMIN, VT, Custom);
1138  setOperationAction(ISD::SMAX, VT, Custom);
1139  setOperationAction(ISD::UMAX, VT, Custom);
1140  setOperationAction(ISD::SHL, VT, Custom);
1141  setOperationAction(ISD::SRL, VT, Custom);
1142  setOperationAction(ISD::SRA, VT, Custom);
1143  setOperationAction(ISD::ABS, VT, Custom);
1144  setOperationAction(ISD::VECREDUCE_ADD, VT, Custom);
1145  setOperationAction(ISD::VECREDUCE_AND, VT, Custom);
1146  setOperationAction(ISD::VECREDUCE_OR, VT, Custom);
1147  setOperationAction(ISD::VECREDUCE_XOR, VT, Custom);
1148  setOperationAction(ISD::VECREDUCE_UMIN, VT, Custom);
1149  setOperationAction(ISD::VECREDUCE_UMAX, VT, Custom);
1150  setOperationAction(ISD::VECREDUCE_SMIN, VT, Custom);
1151  setOperationAction(ISD::VECREDUCE_SMAX, VT, Custom);
1152  setOperationAction(ISD::STEP_VECTOR, VT, Custom);
1153 
1154  setOperationAction(ISD::UMUL_LOHI, VT, Expand);
1155  setOperationAction(ISD::SMUL_LOHI, VT, Expand);
1156  }
1157 
1158  // Illegal unpacked integer vector types.
1159  for (auto VT : {MVT::nxv8i8, MVT::nxv4i16, MVT::nxv2i32}) {
1160  setOperationAction(ISD::EXTRACT_SUBVECTOR, VT, Custom);
1161  setOperationAction(ISD::INSERT_SUBVECTOR, VT, Custom);
1162  }
1163 
1164  for (auto VT : {MVT::nxv16i1, MVT::nxv8i1, MVT::nxv4i1, MVT::nxv2i1}) {
1165  setOperationAction(ISD::CONCAT_VECTORS, VT, Custom);
1166  setOperationAction(ISD::SELECT, VT, Custom);
1167  setOperationAction(ISD::SETCC, VT, Custom);
1168  setOperationAction(ISD::SPLAT_VECTOR, VT, Custom);
1169  setOperationAction(ISD::TRUNCATE, VT, Custom);
1170  setOperationAction(ISD::VECREDUCE_AND, VT, Custom);
1171  setOperationAction(ISD::VECREDUCE_OR, VT, Custom);
1172  setOperationAction(ISD::VECREDUCE_XOR, VT, Custom);
1173 
1174  // There are no legal MVT::nxv16f## based types.
1175  if (VT != MVT::nxv16i1) {
1176  setOperationAction(ISD::SINT_TO_FP, VT, Custom);
1177  setOperationAction(ISD::UINT_TO_FP, VT, Custom);
1178  }
1179  }
1180 
1181  for (auto VT : {MVT::nxv2f16, MVT::nxv4f16, MVT::nxv8f16, MVT::nxv2f32,
1182  MVT::nxv4f32, MVT::nxv2f64}) {
1183  for (auto InnerVT : {MVT::nxv2f16, MVT::nxv4f16, MVT::nxv8f16,
1184  MVT::nxv2f32, MVT::nxv4f32, MVT::nxv2f64}) {
1185  // Avoid marking truncating FP stores as legal to prevent the
1186  // DAGCombiner from creating unsupported truncating stores.
1187  setTruncStoreAction(VT, InnerVT, Expand);
1188  }
1189 
1190  setOperationAction(ISD::CONCAT_VECTORS, VT, Custom);
1191  setOperationAction(ISD::INSERT_SUBVECTOR, VT, Custom);
1192  setOperationAction(ISD::MGATHER, VT, Custom);
1193  setOperationAction(ISD::MSCATTER, VT, Custom);
1194  setOperationAction(ISD::SPLAT_VECTOR, VT, Custom);
1195  setOperationAction(ISD::SELECT, VT, Custom);
1196  setOperationAction(ISD::FADD, VT, Custom);
1197  setOperationAction(ISD::FDIV, VT, Custom);
1198  setOperationAction(ISD::FMA, VT, Custom);
1199  setOperationAction(ISD::FMAXIMUM, VT, Custom);
1200  setOperationAction(ISD::FMAXNUM, VT, Custom);
1201  setOperationAction(ISD::FMINIMUM, VT, Custom);
1202  setOperationAction(ISD::FMINNUM, VT, Custom);
1203  setOperationAction(ISD::FMUL, VT, Custom);
1204  setOperationAction(ISD::FNEG, VT, Custom);
1205  setOperationAction(ISD::FSUB, VT, Custom);
1206  setOperationAction(ISD::FCEIL, VT, Custom);
1207  setOperationAction(ISD::FFLOOR, VT, Custom);
1208  setOperationAction(ISD::FNEARBYINT, VT, Custom);
1209  setOperationAction(ISD::FRINT, VT, Custom);
1210  setOperationAction(ISD::FROUND, VT, Custom);
1211  setOperationAction(ISD::FROUNDEVEN, VT, Custom);
1212  setOperationAction(ISD::FTRUNC, VT, Custom);
1213  setOperationAction(ISD::FSQRT, VT, Custom);
1214  setOperationAction(ISD::FABS, VT, Custom);
1215  setOperationAction(ISD::FP_EXTEND, VT, Custom);
1216  setOperationAction(ISD::FP_ROUND, VT, Custom);
1217  setOperationAction(ISD::VECREDUCE_FADD, VT, Custom);
1218  setOperationAction(ISD::VECREDUCE_FMAX, VT, Custom);
1219  setOperationAction(ISD::VECREDUCE_FMIN, VT, Custom);
1220  setOperationAction(ISD::VECREDUCE_SEQ_FADD, VT, Custom);
1221  }
1222 
1223  for (auto VT : {MVT::nxv2bf16, MVT::nxv4bf16, MVT::nxv8bf16}) {
1224  setOperationAction(ISD::CONCAT_VECTORS, VT, Custom);
1225  setOperationAction(ISD::MGATHER, VT, Custom);
1226  setOperationAction(ISD::MSCATTER, VT, Custom);
1227  }
1228 
1229  setOperationAction(ISD::SPLAT_VECTOR, MVT::nxv8bf16, Custom);
1230 
1231  setOperationAction(ISD::INTRINSIC_WO_CHAIN, MVT::i8, Custom);
1232  setOperationAction(ISD::INTRINSIC_WO_CHAIN, MVT::i16, Custom);
1233 
1234  // NOTE: Currently this has to happen after computeRegisterProperties rather
1235  // than the preferred option of combining it with the addRegisterClass call.
1236  if (Subtarget->useSVEForFixedLengthVectors()) {
1237  for (MVT VT : MVT::integer_fixedlen_vector_valuetypes())
1238  if (useSVEForFixedLengthVectorVT(VT))
1239  addTypeForFixedLengthSVE(VT);
1240  for (MVT VT : MVT::fp_fixedlen_vector_valuetypes())
1241  if (useSVEForFixedLengthVectorVT(VT))
1242  addTypeForFixedLengthSVE(VT);
1243 
1244  // 64bit results can mean a bigger than NEON input.
1245  for (auto VT : {MVT::v8i8, MVT::v4i16})
1246  setOperationAction(ISD::TRUNCATE, VT, Custom);
1247  setOperationAction(ISD::FP_ROUND, MVT::v4f16, Custom);
1248 
1249  // 128bit results imply a bigger than NEON input.
1250  for (auto VT : {MVT::v16i8, MVT::v8i16, MVT::v4i32})
1251  setOperationAction(ISD::TRUNCATE, VT, Custom);
1252  for (auto VT : {MVT::v8f16, MVT::v4f32})
1253  setOperationAction(ISD::FP_ROUND, VT, Expand);
1254 
1255  // These operations are not supported on NEON but SVE can do them.
1256  setOperationAction(ISD::BITREVERSE, MVT::v1i64, Custom);
1257  setOperationAction(ISD::CTLZ, MVT::v1i64, Custom);
1258  setOperationAction(ISD::CTLZ, MVT::v2i64, Custom);
1259  setOperationAction(ISD::CTTZ, MVT::v1i64, Custom);
1260  setOperationAction(ISD::MUL, MVT::v1i64, Custom);
1261  setOperationAction(ISD::MUL, MVT::v2i64, Custom);
1262  setOperationAction(ISD::MULHS, MVT::v1i64, Custom);
1263  setOperationAction(ISD::MULHS, MVT::v2i64, Custom);
1264  setOperationAction(ISD::MULHU, MVT::v1i64, Custom);
1265  setOperationAction(ISD::MULHU, MVT::v2i64, Custom);
1266  setOperationAction(ISD::SDIV, MVT::v8i8, Custom);
1267  setOperationAction(ISD::SDIV, MVT::v16i8, Custom);
1268  setOperationAction(ISD::SDIV, MVT::v4i16, Custom);
1269  setOperationAction(ISD::SDIV, MVT::v8i16, Custom);
1270  setOperationAction(ISD::SDIV, MVT::v2i32, Custom);
1271  setOperationAction(ISD::SDIV, MVT::v4i32, Custom);
1272  setOperationAction(ISD::SDIV, MVT::v1i64, Custom);
1273  setOperationAction(ISD::SDIV, MVT::v2i64, Custom);
1274  setOperationAction(ISD::SMAX, MVT::v1i64, Custom);
1275  setOperationAction(ISD::SMAX, MVT::v2i64, Custom);
1276  setOperationAction(ISD::SMIN, MVT::v1i64, Custom);
1277  setOperationAction(ISD::SMIN, MVT::v2i64, Custom);
1278  setOperationAction(ISD::UDIV, MVT::v8i8, Custom);
1279  setOperationAction(ISD::UDIV, MVT::v16i8, Custom);
1280  setOperationAction(ISD::UDIV, MVT::v4i16, Custom);
1281  setOperationAction(ISD::UDIV, MVT::v8i16, Custom);
1282  setOperationAction(ISD::UDIV, MVT::v2i32, Custom);
1283  setOperationAction(ISD::UDIV, MVT::v4i32, Custom);
1284  setOperationAction(ISD::UDIV, MVT::v1i64, Custom);
1285  setOperationAction(ISD::UDIV, MVT::v2i64, Custom);
1286  setOperationAction(ISD::UMAX, MVT::v1i64, Custom);
1287  setOperationAction(ISD::UMAX, MVT::v2i64, Custom);
1288  setOperationAction(ISD::UMIN, MVT::v1i64, Custom);
1289  setOperationAction(ISD::UMIN, MVT::v2i64, Custom);
1290  setOperationAction(ISD::VECREDUCE_SMAX, MVT::v2i64, Custom);
1291  setOperationAction(ISD::VECREDUCE_SMIN, MVT::v2i64, Custom);
1292  setOperationAction(ISD::VECREDUCE_UMAX, MVT::v2i64, Custom);
1293  setOperationAction(ISD::VECREDUCE_UMIN, MVT::v2i64, Custom);
1294 
1295  // Int operations with no NEON support.
1296  for (auto VT : {MVT::v8i8, MVT::v16i8, MVT::v4i16, MVT::v8i16,
1297  MVT::v2i32, MVT::v4i32, MVT::v2i64}) {
1298  setOperationAction(ISD::BITREVERSE, VT, Custom);
1299  setOperationAction(ISD::CTTZ, VT, Custom);
1300  setOperationAction(ISD::VECREDUCE_AND, VT, Custom);
1301  setOperationAction(ISD::VECREDUCE_OR, VT, Custom);
1302  setOperationAction(ISD::VECREDUCE_XOR, VT, Custom);
1303  }
1304 
1305  // FP operations with no NEON support.
1306  for (auto VT : {MVT::v4f16, MVT::v8f16, MVT::v2f32, MVT::v4f32,
1307  MVT::v1f64, MVT::v2f64})
1308  setOperationAction(ISD::VECREDUCE_SEQ_FADD, VT, Custom);
1309 
1310  // Use SVE for vectors with more than 2 elements.
1311  for (auto VT : {MVT::v4f16, MVT::v8f16, MVT::v4f32})
1312  setOperationAction(ISD::VECREDUCE_FADD, VT, Custom);
1313  }
1314 
1315  setOperationPromotedToType(ISD::VECTOR_SPLICE, MVT::nxv2i1, MVT::nxv2i64);
1316  setOperationPromotedToType(ISD::VECTOR_SPLICE, MVT::nxv4i1, MVT::nxv4i32);
1317  setOperationPromotedToType(ISD::VECTOR_SPLICE, MVT::nxv8i1, MVT::nxv8i16);
1318  setOperationPromotedToType(ISD::VECTOR_SPLICE, MVT::nxv16i1, MVT::nxv16i8);
1319  }
1320 
1321  PredictableSelectIsExpensive = Subtarget->predictableSelectIsExpensive();
1322 }
1323 
1324 void AArch64TargetLowering::addTypeForNEON(MVT VT, MVT PromotedBitwiseVT) {
1325  assert(VT.isVector() && "VT should be a vector type");
1326 
1327  if (VT.isFloatingPoint()) {
1328  MVT PromoteTo = EVT(VT).changeVectorElementTypeToInteger().getSimpleVT();
1329  setOperationPromotedToType(ISD::LOAD, VT, PromoteTo);
1330  setOperationPromotedToType(ISD::STORE, VT, PromoteTo);
1331  }
1332 
1333  // Mark vector float intrinsics as expand.
1334  if (VT == MVT::v2f32 || VT == MVT::v4f32 || VT == MVT::v2f64) {
1335  setOperationAction(ISD::FSIN, VT, Expand);
1336  setOperationAction(ISD::FCOS, VT, Expand);
1337  setOperationAction(ISD::FPOW, VT, Expand);
1338  setOperationAction(ISD::FLOG, VT, Expand);
1339  setOperationAction(ISD::FLOG2, VT, Expand);
1340  setOperationAction(ISD::FLOG10, VT, Expand);
1341  setOperationAction(ISD::FEXP, VT, Expand);
1342  setOperationAction(ISD::FEXP2, VT, Expand);
1343 
1344  // But we do support custom-lowering for FCOPYSIGN.
1345  setOperationAction(ISD::FCOPYSIGN, VT, Custom);
1346  }
1347 
1348  setOperationAction(ISD::EXTRACT_VECTOR_ELT, VT, Custom);
1349  setOperationAction(ISD::INSERT_VECTOR_ELT, VT, Custom);
1350  setOperationAction(ISD::BUILD_VECTOR, VT, Custom);
1351  setOperationAction(ISD::VECTOR_SHUFFLE, VT, Custom);
1352  setOperationAction(ISD::EXTRACT_SUBVECTOR, VT, Custom);
1353  setOperationAction(ISD::SRA, VT, Custom);
1354  setOperationAction(ISD::SRL, VT, Custom);
1355  setOperationAction(ISD::SHL, VT, Custom);
1356  setOperationAction(ISD::OR, VT, Custom);
1357  setOperationAction(ISD::SETCC, VT, Custom);
1358  setOperationAction(ISD::CONCAT_VECTORS, VT, Legal);
1359 
1360  setOperationAction(ISD::SELECT, VT, Expand);
1361  setOperationAction(ISD::SELECT_CC, VT, Expand);
1362  setOperationAction(ISD::VSELECT, VT, Expand);
1363  for (MVT InnerVT : MVT::all_valuetypes())
1364  setLoadExtAction(ISD::EXTLOAD, InnerVT, VT, Expand);
1365 
1366  // CNT supports only B element sizes, then use UADDLP to widen.
1367  if (VT != MVT::v8i8 && VT != MVT::v16i8)
1368  setOperationAction(ISD::CTPOP, VT, Custom);
1369 
1370  setOperationAction(ISD::UDIV, VT, Expand);
1371  setOperationAction(ISD::SDIV, VT, Expand);
1372  setOperationAction(ISD::UREM, VT, Expand);
1373  setOperationAction(ISD::SREM, VT, Expand);
1374  setOperationAction(ISD::FREM, VT, Expand);
1375 
1376  setOperationAction(ISD::FP_TO_SINT, VT, Custom);
1377  setOperationAction(ISD::FP_TO_UINT, VT, Custom);
1378 
1379  if (!VT.isFloatingPoint())
1380  setOperationAction(ISD::ABS, VT, Legal);
1381 
1382  // [SU][MIN|MAX] are available for all NEON types apart from i64.
1383  if (!VT.isFloatingPoint() && VT != MVT::v2i64 && VT != MVT::v1i64)
1384  for (unsigned Opcode : {ISD::SMIN, ISD::SMAX, ISD::UMIN, ISD::UMAX})
1385  setOperationAction(Opcode, VT, Legal);
1386 
1387  // F[MIN|MAX][NUM|NAN] are available for all FP NEON types.
1388  if (VT.isFloatingPoint() &&
1389  VT.getVectorElementType() != MVT::bf16 &&
1390  (VT.getVectorElementType() != MVT::f16 || Subtarget->hasFullFP16()))
1391  for (unsigned Opcode :
1392  {ISD::FMINIMUM, ISD::FMAXIMUM, ISD::FMINNUM, ISD::FMAXNUM})
1393  setOperationAction(Opcode, VT, Legal);
1394 
1395  if (Subtarget->isLittleEndian()) {
1396  for (unsigned im = (unsigned)ISD::PRE_INC;
1397  im != (unsigned)ISD::LAST_INDEXED_MODE; ++im) {
1398  setIndexedLoadAction(im, VT, Legal);
1399  setIndexedStoreAction(im, VT, Legal);
1400  }
1401  }
1402 }
1403 
1404 void AArch64TargetLowering::addTypeForFixedLengthSVE(MVT VT) {
1405  assert(VT.isFixedLengthVector() && "Expected fixed length vector type!");
1406 
1407  // By default everything must be expanded.
1408  for (unsigned Op = 0; Op < ISD::BUILTIN_OP_END; ++Op)
1409  setOperationAction(Op, VT, Expand);
1410 
1411  // We use EXTRACT_SUBVECTOR to "cast" a scalable vector to a fixed length one.
1412  setOperationAction(ISD::EXTRACT_SUBVECTOR, VT, Custom);
1413 
1414  if (VT.isFloatingPoint()) {
1415  setCondCodeAction(ISD::SETO, VT, Expand);
1416  setCondCodeAction(ISD::SETOLT, VT, Expand);
1417  setCondCodeAction(ISD::SETLT, VT, Expand);
1418  setCondCodeAction(ISD::SETOLE, VT, Expand);
1419  setCondCodeAction(ISD::SETLE, VT, Expand);
1420  setCondCodeAction(ISD::SETULT, VT, Expand);
1421  setCondCodeAction(ISD::SETULE, VT, Expand);
1422  setCondCodeAction(ISD::SETUGE, VT, Expand);
1423  setCondCodeAction(ISD::SETUGT, VT, Expand);
1424  setCondCodeAction(ISD::SETUEQ, VT, Expand);
1425  setCondCodeAction(ISD::SETUNE, VT, Expand);
1426  }
1427 
1428  // Lower fixed length vector operations to scalable equivalents.
1429  setOperationAction(ISD::ABS, VT, Custom);
1430  setOperationAction(ISD::ADD, VT, Custom);
1431  setOperationAction(ISD::AND, VT, Custom);
1432  setOperationAction(ISD::ANY_EXTEND, VT, Custom);
1433  setOperationAction(ISD::BITREVERSE, VT, Custom);
1434  setOperationAction(ISD::BSWAP, VT, Custom);
1435  setOperationAction(ISD::CTLZ, VT, Custom);
1436  setOperationAction(ISD::CTPOP, VT, Custom);
1437  setOperationAction(ISD::CTTZ, VT, Custom);
1438  setOperationAction(ISD::FABS, VT, Custom);
1439  setOperationAction(ISD::FADD, VT, Custom);
1440  setOperationAction(ISD::EXTRACT_VECTOR_ELT, VT, Custom);
1441  setOperationAction(ISD::FCEIL, VT, Custom);
1442  setOperationAction(ISD::FDIV, VT, Custom);
1443  setOperationAction(ISD::FFLOOR, VT, Custom);
1444  setOperationAction(ISD::FMA, VT, Custom);
1445  setOperationAction(ISD::FMAXIMUM, VT, Custom);
1446  setOperationAction(ISD::FMAXNUM, VT, Custom);
1447  setOperationAction(ISD::FMINIMUM, VT, Custom);
1448  setOperationAction(ISD::FMINNUM, VT, Custom);
1449  setOperationAction(ISD::FMUL, VT, Custom);
1450  setOperationAction(ISD::FNEARBYINT, VT, Custom);
1451  setOperationAction(ISD::FNEG, VT, Custom);
1452  setOperationAction(ISD::FRINT, VT, Custom);
1453  setOperationAction(ISD::FROUND, VT, Custom);
1454  setOperationAction(ISD::FROUNDEVEN, VT, Custom);
1455  setOperationAction(ISD::FSQRT, VT, Custom);
1456  setOperationAction(ISD::FSUB, VT, Custom);
1457  setOperationAction(ISD::FTRUNC, VT, Custom);
1458  setOperationAction(ISD::LOAD, VT, Custom);
1459  setOperationAction(ISD::MUL, VT, Custom);
1460  setOperationAction(ISD::MULHS, VT, Custom);
1461  setOperationAction(ISD::MULHU, VT, Custom);
1462  setOperationAction(ISD::OR, VT, Custom);
1463  setOperationAction(ISD::SDIV, VT, Custom);
1464  setOperationAction(ISD::SELECT, VT, Custom);
1465  setOperationAction(ISD::SETCC, VT, Custom);
1466  setOperationAction(ISD::SHL, VT, Custom);
1467  setOperationAction(ISD::SIGN_EXTEND, VT, Custom);
1468  setOperationAction(ISD::SIGN_EXTEND_INREG, VT, Custom);
1469  setOperationAction(ISD::SMAX, VT, Custom);
1470  setOperationAction(ISD::SMIN, VT, Custom);
1471  setOperationAction(ISD::SPLAT_VECTOR, VT, Custom);
1472  setOperationAction(ISD::SRA, VT, Custom);
1473  setOperationAction(ISD::SRL, VT, Custom);
1474  setOperationAction(ISD::STORE, VT, Custom);
1475  setOperationAction(ISD::SUB, VT, Custom);
1476  setOperationAction(ISD::TRUNCATE, VT, Custom);
1477  setOperationAction(ISD::UDIV, VT, Custom);
1478  setOperationAction(ISD::UMAX, VT, Custom);
1479  setOperationAction(ISD::UMIN, VT, Custom);
1480  setOperationAction(ISD::VECREDUCE_ADD, VT, Custom);
1481  setOperationAction(ISD::VECREDUCE_AND, VT, Custom);
1482  setOperationAction(ISD::VECREDUCE_FADD, VT, Custom);
1483  setOperationAction(ISD::VECREDUCE_SEQ_FADD, VT, Custom);
1484  setOperationAction(ISD::VECREDUCE_FMAX, VT, Custom);
1485  setOperationAction(ISD::VECREDUCE_FMIN, VT, Custom);
1486  setOperationAction(ISD::VECREDUCE_OR, VT, Custom);
1487  setOperationAction(ISD::INSERT_VECTOR_ELT, VT, Custom);
1488  setOperationAction(ISD::VECREDUCE_SMAX, VT, Custom);
1489  setOperationAction(ISD::VECREDUCE_SMIN, VT, Custom);
1490  setOperationAction(ISD::VECREDUCE_UMAX, VT, Custom);
1491  setOperationAction(ISD::VECREDUCE_UMIN, VT, Custom);
1492  setOperationAction(ISD::VECREDUCE_XOR, VT, Custom);
1493  setOperationAction(ISD::VSELECT, VT, Custom);
1494  setOperationAction(ISD::XOR, VT, Custom);
1495  setOperationAction(ISD::ZERO_EXTEND, VT, Custom);
1496 }
1497 
1498 void AArch64TargetLowering::addDRTypeForNEON(MVT VT) {
1499  addRegisterClass(VT, &AArch64::FPR64RegClass);
1500  addTypeForNEON(VT, MVT::v2i32);
1501 }
1502 
1503 void AArch64TargetLowering::addQRTypeForNEON(MVT VT) {
1504  addRegisterClass(VT, &AArch64::FPR128RegClass);
1505  addTypeForNEON(VT, MVT::v4i32);
1506 }
1507 
1508 EVT AArch64TargetLowering::getSetCCResultType(const DataLayout &,
1509  LLVMContext &C, EVT VT) const {
1510  if (!VT.isVector())
1511  return MVT::i32;
1512  if (VT.isScalableVector())
1513  return EVT::getVectorVT(C, MVT::i1, VT.getVectorElementCount());
1514  return VT.changeVectorElementTypeToInteger();
1515 }
1516 
1517 static bool optimizeLogicalImm(SDValue Op, unsigned Size, uint64_t Imm,
1518  const APInt &Demanded,
1519  TargetLowering::TargetLoweringOpt &TLO,
1520  unsigned NewOpc) {
1521  uint64_t OldImm = Imm, NewImm, Enc;
1522  uint64_t Mask = ((uint64_t)(-1LL) >> (64 - Size)), OrigMask = Mask;
1523 
1524  // Return if the immediate is already all zeros, all ones, a bimm32 or a
1525  // bimm64.
1526  if (Imm == 0 || Imm == Mask ||
1527  AArch64_AM::isLogicalImmediate(Imm & Mask, Size))
1528  return false;
1529 
1530  unsigned EltSize = Size;
1531  uint64_t DemandedBits = Demanded.getZExtValue();
1532 
1533  // Clear bits that are not demanded.
1534  Imm &= DemandedBits;
1535 
1536  while (true) {
1537  // The goal here is to set the non-demanded bits in a way that minimizes
1538  // the number of switching between 0 and 1. In order to achieve this goal,
1539  // we set the non-demanded bits to the value of the preceding demanded bits.
1540  // For example, if we have an immediate 0bx10xx0x1 ('x' indicates a
1541  // non-demanded bit), we copy bit0 (1) to the least significant 'x',
1542  // bit2 (0) to 'xx', and bit6 (1) to the most significant 'x'.
1543  // The final result is 0b11000011.
1544  uint64_t NonDemandedBits = ~DemandedBits;
1545  uint64_t InvertedImm = ~Imm & DemandedBits;
1546  uint64_t RotatedImm =
1547  ((InvertedImm << 1) | (InvertedImm >> (EltSize - 1) & 1)) &
1548  NonDemandedBits;
1549  uint64_t Sum = RotatedImm + NonDemandedBits;
1550  bool Carry = NonDemandedBits & ~Sum & (1ULL << (EltSize - 1));
1551  uint64_t Ones = (Sum + Carry) & NonDemandedBits;
1552  NewImm = (Imm | Ones) & Mask;
1553 
1554  // If NewImm or its bitwise NOT is a shifted mask, it is a bitmask immediate
1555  // or all-ones or all-zeros, in which case we can stop searching. Otherwise,
1556  // we halve the element size and continue the search.
1557  if (isShiftedMask_64(NewImm) || isShiftedMask_64(~(NewImm | ~Mask)))
1558  break;
1559 
1560  // We cannot shrink the element size any further if it is 2-bits.
1561  if (EltSize == 2)
1562  return false;
1563 
1564  EltSize /= 2;
1565  Mask >>= EltSize;
1566  uint64_t Hi = Imm >> EltSize, DemandedBitsHi = DemandedBits >> EltSize;
1567 
1568  // Return if there is mismatch in any of the demanded bits of Imm and Hi.
1569  if (((Imm ^ Hi) & (DemandedBits & DemandedBitsHi) & Mask) != 0)
1570  return false;
1571 
1572  // Merge the upper and lower halves of Imm and DemandedBits.
1573  Imm |= Hi;
1574  DemandedBits |= DemandedBitsHi;
1575  }
1576 
1577  ++NumOptimizedImms;
1578 
1579  // Replicate the element across the register width.
1580  while (EltSize < Size) {
1581  NewImm |= NewImm << EltSize;
1582  EltSize *= 2;
1583  }
1584 
1585  (void)OldImm;
1586  assert(((OldImm ^ NewImm) & Demanded.getZExtValue()) == 0 &&
1587  "demanded bits should never be altered");
1588  assert(OldImm != NewImm && "the new imm shouldn't be equal to the old imm");
1589 
1590  // Create the new constant immediate node.
1591  EVT VT = Op.getValueType();
1592  SDLoc DL(Op);
1593  SDValue New;
1594 
1595  // If the new constant immediate is all-zeros or all-ones, let the target
1596  // independent DAG combine optimize this node.
1597  if (NewImm == 0 || NewImm == OrigMask) {
1598  New = TLO.DAG.getNode(Op.getOpcode(), DL, VT, Op.getOperand(0),
1599  TLO.DAG.getConstant(NewImm, DL, VT));
1600  // Otherwise, create a machine node so that target independent DAG combine
1601  // doesn't undo this optimization.
1602  } else {
1603  Enc = AArch64_AM::encodeLogicalImmediate(NewImm, Size);
1604  SDValue EncConst = TLO.DAG.getTargetConstant(Enc, DL, VT);
1605  New = SDValue(
1606  TLO.DAG.getMachineNode(NewOpc, DL, VT, Op.getOperand(0), EncConst), 0);
1607  }
1608 
1609  return TLO.CombineTo(Op, New);
1610 }
1611 
1612 bool AArch64TargetLowering::targetShrinkDemandedConstant(
1613  SDValue Op, const APInt &DemandedBits, const APInt &DemandedElts,
1614  TargetLoweringOpt &TLO) const {
1615  // Delay this optimization to as late as possible.
1616  if (!TLO.LegalOps)
1617  return false;
1618 
1619  if (!EnableOptimizeLogicalImm)
1620  return false;
1621 
1622  EVT VT = Op.getValueType();
1623  if (VT.isVector())
1624  return false;
1625 
1626  unsigned Size = VT.getSizeInBits();
1627  assert((Size == 32 || Size == 64) &&
1628  "i32 or i64 is expected after legalization.");
1629 
1630  // Exit early if we demand all bits.
1631  if (DemandedBits.countPopulation() == Size)
1632  return false;
1633 
1634  unsigned NewOpc;
1635  switch (Op.getOpcode()) {
1636  default:
1637  return false;
1638  case ISD::AND:
1639  NewOpc = Size == 32 ? AArch64::ANDWri : AArch64::ANDXri;
1640  break;
1641  case ISD::OR:
1642  NewOpc = Size == 32 ? AArch64::ORRWri : AArch64::ORRXri;
1643  break;
1644  case ISD::XOR:
1645  NewOpc = Size == 32 ? AArch64::EORWri : AArch64::EORXri;
1646  break;
1647  }
1648  ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op.getOperand(1));
1649  if (!C)
1650  return false;
1651  uint64_t Imm = C->getZExtValue();
1652  return optimizeLogicalImm(Op, Size, Imm, DemandedBits, TLO, NewOpc);
1653 }
1654 
1655 /// computeKnownBitsForTargetNode - Determine which of the bits specified in
1656 /// Mask are known to be either zero or one and return them Known.
1657 void AArch64TargetLowering::computeKnownBitsForTargetNode(
1658  const SDValue Op, KnownBits &Known,
1659  const APInt &DemandedElts, const SelectionDAG &DAG, unsigned Depth) const {
1660  switch (Op.getOpcode()) {
1661  default:
1662  break;
1663  case AArch64ISD::CSEL: {
1664  KnownBits Known2;
1665  Known = DAG.computeKnownBits(Op->getOperand(0), Depth + 1);
1666  Known2 = DAG.computeKnownBits(Op->getOperand(1), Depth + 1);
1667  Known = KnownBits::commonBits(Known, Known2);
1668  break;
1669  }
1670  case AArch64ISD::LOADgot:
1671  case AArch64ISD::ADDlow: {
1672  if (!Subtarget->isTargetILP32())
1673  break;
1674  // In ILP32 mode all valid pointers are in the low 4GB of the address-space.
1675  Known.Zero = APInt::getHighBitsSet(64, 32);
1676  break;
1677  }
1678  case ISD::INTRINSIC_W_CHAIN: {
1679  ConstantSDNode *CN = cast<ConstantSDNode>(Op->getOperand(1));
1680  Intrinsic::ID IntID = static_cast<Intrinsic::ID>(CN->getZExtValue());
1681  switch (IntID) {
1682  default: return;
1683  case Intrinsic::aarch64_ldaxr:
1684  case Intrinsic::aarch64_ldxr: {
1685  unsigned BitWidth = Known.getBitWidth();
1686  EVT VT = cast<MemIntrinsicSDNode>(Op)->getMemoryVT();
1687  unsigned MemBits = VT.getScalarSizeInBits();
1688  Known.Zero |= APInt::getHighBitsSet(BitWidth, BitWidth - MemBits);
1689  return;
1690  }
1691  }
1692  break;
1693  }
1694  case ISD::INTRINSIC_WO_CHAIN:
1695  case ISD::INTRINSIC_VOID: {
1696  unsigned IntNo = cast<ConstantSDNode>(Op.getOperand(0))->getZExtValue();
1697  switch (IntNo) {
1698  default:
1699  break;
1700  case Intrinsic::aarch64_neon_umaxv:
1701  case Intrinsic::aarch64_neon_uminv: {
1702  // Figure out the datatype of the vector operand. The UMINV instruction
1703  // will zero extend the result, so we can mark as known zero all the
1704  // bits larger than the element datatype. 32-bit or larget doesn't need
1705  // this as those are legal types and will be handled by isel directly.
1706  MVT VT = Op.getOperand(1).getValueType().getSimpleVT();
1707  unsigned BitWidth = Known.getBitWidth();
1708  if (VT == MVT::v8i8 || VT == MVT::v16i8) {
1709  assert(BitWidth >= 8 && "Unexpected width!");
1710  APInt Mask = APInt::getHighBitsSet(BitWidth, BitWidth - 8);
1711  Known.Zero |= Mask;
1712  } else if (VT == MVT::v4i16 || VT == MVT::v8i16) {
1713  assert(BitWidth >= 16 && "Unexpected width!");
1714  APInt Mask = APInt::getHighBitsSet(BitWidth, BitWidth - 16);
1715  Known.Zero |= Mask;
1716  }
1717  break;
1718  } break;
1719  }
1720  }
1721  }
1722 }
1723 
1724 MVT AArch64TargetLowering::getScalarShiftAmountTy(const DataLayout &DL,
1725  EVT) const {
1726  return MVT::i64;
1727 }
1728 
1729 bool AArch64TargetLowering::allowsMisalignedMemoryAccesses(
1730  EVT VT, unsigned AddrSpace, Align Alignment, MachineMemOperand::Flags Flags,
1731  bool *Fast) const {
1732  if (Subtarget->requiresStrictAlign())
1733  return false;
1734 
1735  if (Fast) {
1736  // Some CPUs are fine with unaligned stores except for 128-bit ones.
1737  *Fast = !Subtarget->isMisaligned128StoreSlow() || VT.getStoreSize() != 16 ||
1738  // See comments in performSTORECombine() for more details about
1739  // these conditions.
1740 
1741  // Code that uses clang vector extensions can mark that it
1742  // wants unaligned accesses to be treated as fast by
1743  // underspecifying alignment to be 1 or 2.
1744  Alignment <= 2 ||
1745 
1746  // Disregard v2i64. Memcpy lowering produces those and splitting
1747  // them regresses performance on micro-benchmarks and olden/bh.
1748  VT == MVT::v2i64;
1749  }
1750  return true;
1751 }
1752 
1753 // Same as above but handling LLTs instead.
1754 bool AArch64TargetLowering::allowsMisalignedMemoryAccesses(
1755  LLT Ty, unsigned AddrSpace, Align Alignment, MachineMemOperand::Flags Flags,
1756  bool *Fast) const {
1757  if (Subtarget->requiresStrictAlign())
1758  return false;
1759 
1760  if (Fast) {
1761  // Some CPUs are fine with unaligned stores except for 128-bit ones.
1762  *Fast = !Subtarget->isMisaligned128StoreSlow() ||
1763  Ty.getSizeInBytes() != 16 ||
1764  // See comments in performSTORECombine() for more details about
1765  // these conditions.
1766 
1767  // Code that uses clang vector extensions can mark that it
1768  // wants unaligned accesses to be treated as fast by
1769  // underspecifying alignment to be 1 or 2.
1770  Alignment <= 2 ||
1771 
1772  // Disregard v2i64. Memcpy lowering produces those and splitting
1773  // them regresses performance on micro-benchmarks and olden/bh.
1774  Ty == LLT::vector(2, 64);
1775  }
1776  return true;
1777 }
1778 
1779 FastISel *
1780 AArch64TargetLowering::createFastISel(FunctionLoweringInfo &funcInfo,
1781  const TargetLibraryInfo *libInfo) const {
1782  return AArch64::createFastISel(funcInfo, libInfo);
1783 }
1784 
1785 const char *AArch64TargetLowering::getTargetNodeName(unsigned Opcode) const {
1786 #define MAKE_CASE(V) \
1787  case V: \
1788  return #V;
1789  switch ((AArch64ISD::NodeType)Opcode) {
1790  case AArch64ISD::FIRST_NUMBER:
1791  break;
1792  MAKE_CASE(AArch64ISD::CALL)
1793  MAKE_CASE(AArch64ISD::ADRP)
1794  MAKE_CASE(AArch64ISD::ADR)
1795  MAKE_CASE(AArch64ISD::ADDlow)
1796  MAKE_CASE(AArch64ISD::LOADgot)
1797  MAKE_CASE(AArch64ISD::RET_FLAG)
1798  MAKE_CASE(AArch64ISD::BRCOND)
1799  MAKE_CASE(AArch64ISD::CSEL)
1800  MAKE_CASE(AArch64ISD::FCSEL)
1801  MAKE_CASE(AArch64ISD::CSINV)
1802  MAKE_CASE(AArch64ISD::CSNEG)
1803  MAKE_CASE(AArch64ISD::CSINC)
1804  MAKE_CASE(AArch64ISD::THREAD_POINTER)
1805  MAKE_CASE(AArch64ISD::TLSDESC_CALLSEQ)
1806  MAKE_CASE(AArch64ISD::ADD_PRED)
1807  MAKE_CASE(AArch64ISD::MUL_PRED)
1808  MAKE_CASE(AArch64ISD::MULHS_PRED)
1809  MAKE_CASE(AArch64ISD::MULHU_PRED)
1810  MAKE_CASE(AArch64ISD::SDIV_PRED)
1811  MAKE_CASE(AArch64ISD::SHL_PRED)
1812  MAKE_CASE(AArch64ISD::SMAX_PRED)
1813  MAKE_CASE(AArch64ISD::SMIN_PRED)
1814  MAKE_CASE(AArch64ISD::SRA_PRED)
1815  MAKE_CASE(AArch64ISD::SRL_PRED)
1816  MAKE_CASE(AArch64ISD::SUB_PRED)
1817  MAKE_CASE(AArch64ISD::UDIV_PRED)
1818  MAKE_CASE(AArch64ISD::UMAX_PRED)
1819  MAKE_CASE(AArch64ISD::UMIN_PRED)
1820  MAKE_CASE(AArch64ISD::FNEG_MERGE_PASSTHRU)
1821  MAKE_CASE(AArch64ISD::SIGN_EXTEND_INREG_MERGE_PASSTHRU)
1822  MAKE_CASE(AArch64ISD::ZERO_EXTEND_INREG_MERGE_PASSTHRU)
1823  MAKE_CASE(AArch64ISD::FCEIL_MERGE_PASSTHRU)
1824  MAKE_CASE(AArch64ISD::FFLOOR_MERGE_PASSTHRU)
1825  MAKE_CASE(AArch64ISD::FNEARBYINT_MERGE_PASSTHRU)
1826  MAKE_CASE(AArch64ISD::FRINT_MERGE_PASSTHRU)
1827  MAKE_CASE(AArch64ISD::FROUND_MERGE_PASSTHRU)
1828  MAKE_CASE(AArch64ISD::FROUNDEVEN_MERGE_PASSTHRU)
1829  MAKE_CASE(AArch64ISD::FTRUNC_MERGE_PASSTHRU)
1830  MAKE_CASE(AArch64ISD::FP_ROUND_MERGE_PASSTHRU)
1831  MAKE_CASE(AArch64ISD::FP_EXTEND_MERGE_PASSTHRU)
1832  MAKE_CASE(AArch64ISD::SINT_TO_FP_MERGE_PASSTHRU)
1833  MAKE_CASE(AArch64ISD::UINT_TO_FP_MERGE_PASSTHRU)
1834  MAKE_CASE(AArch64ISD::FCVTZU_MERGE_PASSTHRU)
1835  MAKE_CASE(AArch64ISD::FCVTZS_MERGE_PASSTHRU)
1836  MAKE_CASE(AArch64ISD::FSQRT_MERGE_PASSTHRU)
1837  MAKE_CASE(AArch64ISD::FRECPX_MERGE_PASSTHRU)
1838  MAKE_CASE(AArch64ISD::FABS_MERGE_PASSTHRU)
1839  MAKE_CASE(AArch64ISD::ABS_MERGE_PASSTHRU)
1840  MAKE_CASE(AArch64ISD::NEG_MERGE_PASSTHRU)
1841  MAKE_CASE(AArch64ISD::SETCC_MERGE_ZERO)
1842  MAKE_CASE(AArch64ISD::ADC)
1843  MAKE_CASE(AArch64ISD::SBC)
1844  MAKE_CASE(AArch64ISD::ADDS)
1845  MAKE_CASE(AArch64ISD::SUBS)
1846  MAKE_CASE(AArch64ISD::ADCS)
1847  MAKE_CASE(AArch64ISD::SBCS)
1848  MAKE_CASE(AArch64ISD::ANDS)
1849  MAKE_CASE(AArch64ISD::CCMP)
1850  MAKE_CASE(AArch64ISD::CCMN)
1851  MAKE_CASE(AArch64ISD::FCCMP)
1852  MAKE_CASE(AArch64ISD::FCMP)
1853  MAKE_CASE(AArch64ISD::STRICT_FCMP)
1854  MAKE_CASE(AArch64ISD::STRICT_FCMPE)
1855  MAKE_CASE(AArch64ISD::DUP)
1856  MAKE_CASE(AArch64ISD::DUPLANE8)
1857  MAKE_CASE(AArch64ISD::DUPLANE16)
1858  MAKE_CASE(AArch64ISD::DUPLANE32)
1859  MAKE_CASE(AArch64ISD::DUPLANE64)
1860  MAKE_CASE(AArch64ISD::MOVI)
1861  MAKE_CASE(AArch64ISD::MOVIshift)
1862  MAKE_CASE(AArch64ISD::MOVIedit)
1863  MAKE_CASE(AArch64ISD::MOVImsl)
1864  MAKE_CASE(AArch64ISD::FMOV)
1865  MAKE_CASE(AArch64ISD::MVNIshift)
1866  MAKE_CASE(AArch64ISD::MVNImsl)
1867  MAKE_CASE(AArch64ISD::BICi)
1868  MAKE_CASE(AArch64ISD::ORRi)
1869  MAKE_CASE(AArch64ISD::BSP)
1870  MAKE_CASE(AArch64ISD::NEG)
1871  MAKE_CASE(AArch64ISD::EXTR)
1872  MAKE_CASE(AArch64ISD::ZIP1)
1873  MAKE_CASE(AArch64ISD::ZIP2)
1874  MAKE_CASE(AArch64ISD::UZP1)
1875  MAKE_CASE(AArch64ISD::UZP2)
1876  MAKE_CASE(AArch64ISD::TRN1)
1877  MAKE_CASE(AArch64ISD::TRN2)
1878  MAKE_CASE(AArch64ISD::REV16)
1879  MAKE_CASE(AArch64ISD::REV32)
1880  MAKE_CASE(AArch64ISD::REV64)
1881  MAKE_CASE(AArch64ISD::EXT)
1882  MAKE_CASE(AArch64ISD::VSHL)
1883  MAKE_CASE(AArch64ISD::VLSHR)
1884  MAKE_CASE(AArch64ISD::VASHR)
1885  MAKE_CASE(AArch64ISD::VSLI)
1886  MAKE_CASE(AArch64ISD::VSRI)
1887  MAKE_CASE(AArch64ISD::CMEQ)
1888  MAKE_CASE(AArch64ISD::CMGE)
1889  MAKE_CASE(AArch64ISD::CMGT)
1890  MAKE_CASE(AArch64ISD::CMHI)
1891  MAKE_CASE(AArch64ISD::CMHS)
1892  MAKE_CASE(AArch64ISD::FCMEQ)
1893  MAKE_CASE(AArch64ISD::FCMGE)
1894  MAKE_CASE(AArch64ISD::FCMGT)
1895  MAKE_CASE(AArch64ISD::CMEQz)
1896  MAKE_CASE(AArch64ISD::CMGEz)
1897  MAKE_CASE(AArch64ISD::CMGTz)
1898  MAKE_CASE(AArch64ISD::CMLEz)
1899  MAKE_CASE(AArch64ISD::CMLTz)
1900  MAKE_CASE(AArch64ISD::FCMEQz)
1901  MAKE_CASE(AArch64ISD::FCMGEz)
1902  MAKE_CASE(AArch64ISD::FCMGTz)
1903  MAKE_CASE(AArch64ISD::FCMLEz)
1904  MAKE_CASE(AArch64ISD::FCMLTz)
1905  MAKE_CASE(AArch64ISD::SADDV)
1906  MAKE_CASE(AArch64ISD::UADDV)
1907  MAKE_CASE(AArch64ISD::SRHADD)
1908  MAKE_CASE(AArch64ISD::URHADD)
1909  MAKE_CASE(AArch64ISD::SHADD)
1910  MAKE_CASE(AArch64ISD::UHADD)
1911  MAKE_CASE(AArch64ISD::SDOT)
1912  MAKE_CASE(AArch64ISD::UDOT)
1913  MAKE_CASE(AArch64ISD::SMINV)
1914  MAKE_CASE(AArch64ISD::UMINV)
1915  MAKE_CASE(AArch64ISD::SMAXV)
1916  MAKE_CASE(AArch64ISD::UMAXV)
1917  MAKE_CASE(AArch64ISD::SADDV_PRED)
1918  MAKE_CASE(AArch64ISD::UADDV_PRED)
1919  MAKE_CASE(AArch64ISD::SMAXV_PRED)
1920  MAKE_CASE(AArch64ISD::UMAXV_PRED)
1921  MAKE_CASE(AArch64ISD::SMINV_PRED)
1922  MAKE_CASE(AArch64ISD::UMINV_PRED)
1923  MAKE_CASE(AArch64ISD::ORV_PRED)
1924  MAKE_CASE(AArch64ISD::EORV_PRED)
1925  MAKE_CASE(AArch64ISD::ANDV_PRED)
1926  MAKE_CASE(AArch64ISD::CLASTA_N)
1927  MAKE_CASE(AArch64ISD::CLASTB_N)
1928  MAKE_CASE(AArch64ISD::LASTA)
1929  MAKE_CASE(AArch64ISD::LASTB)
1930  MAKE_CASE(AArch64ISD::REINTERPRET_CAST)
1931  MAKE_CASE(AArch64ISD::TBL)
1932  MAKE_CASE(AArch64ISD::FADD_PRED)
1933  MAKE_CASE(AArch64ISD::FADDA_PRED)
1934  MAKE_CASE(AArch64ISD::FADDV_PRED)
1935  MAKE_CASE(AArch64ISD::FDIV_PRED)
1936  MAKE_CASE(AArch64ISD::FMA_PRED)
1937  MAKE_CASE(AArch64ISD::FMAX_PRED)
1938  MAKE_CASE(AArch64ISD::FMAXV_PRED)
1939  MAKE_CASE(AArch64ISD::FMAXNM_PRED)
1940  MAKE_CASE(AArch64ISD::FMAXNMV_PRED)
1941  MAKE_CASE(AArch64ISD::FMIN_PRED)
1942  MAKE_CASE(AArch64ISD::FMINV_PRED)
1943  MAKE_CASE(AArch64ISD::FMINNM_PRED)
1944  MAKE_CASE(AArch64ISD::FMINNMV_PRED)
1945  MAKE_CASE(AArch64ISD::FMUL_PRED)
1946  MAKE_CASE(AArch64ISD::FSUB_PRED)
1947  MAKE_CASE(AArch64ISD::BIT)
1948  MAKE_CASE(AArch64ISD::CBZ)
1949  MAKE_CASE(AArch64ISD::CBNZ)
1950  MAKE_CASE(AArch64ISD::TBZ)
1951  MAKE_CASE(AArch64ISD::TBNZ)
1952  MAKE_CASE(AArch64ISD::TC_RETURN)
1953  MAKE_CASE(AArch64ISD::PREFETCH)
1954  MAKE_CASE(AArch64ISD::SITOF)
1955  MAKE_CASE(AArch64ISD::UITOF)
1956  MAKE_CASE(AArch64ISD::NVCAST)
1957  MAKE_CASE(AArch64ISD::MRS)
1958  MAKE_CASE(AArch64ISD::SQSHL_I)
1959  MAKE_CASE(AArch64ISD::UQSHL_I)
1960  MAKE_CASE(AArch64ISD::SRSHR_I)
1961  MAKE_CASE(AArch64ISD::URSHR_I)
1962  MAKE_CASE(AArch64ISD::SQSHLU_I)
1963  MAKE_CASE(AArch64ISD::WrapperLarge)
1964  MAKE_CASE(AArch64ISD::LD2post)
1965  MAKE_CASE(AArch64ISD::LD3post)
1966  MAKE_CASE(AArch64ISD::LD4post)
1967  MAKE_CASE(AArch64ISD::ST2post)
1968  MAKE_CASE(AArch64ISD::ST3post)
1969  MAKE_CASE(AArch64ISD::ST4post)
1970  MAKE_CASE(AArch64ISD::LD1x2post)
1971  MAKE_CASE(AArch64ISD::LD1x3post)
1972  MAKE_CASE(AArch64ISD::LD1x4post)
1973  MAKE_CASE(AArch64ISD::ST1x2post)
1974  MAKE_CASE(AArch64ISD::ST1x3post)
1975  MAKE_CASE(AArch64ISD::ST1x4post)
1976  MAKE_CASE(AArch64ISD::LD1DUPpost)
1977  MAKE_CASE(AArch64ISD::LD2DUPpost)
1978  MAKE_CASE(AArch64ISD::LD3DUPpost)
1979  MAKE_CASE(AArch64ISD::LD4DUPpost)
1980  MAKE_CASE(AArch64ISD::LD1LANEpost)
1981  MAKE_CASE(AArch64ISD::LD2LANEpost)
1982  MAKE_CASE(AArch64ISD::LD3LANEpost)
1983  MAKE_CASE(AArch64ISD::LD4LANEpost)
1984  MAKE_CASE(AArch64ISD::ST2LANEpost)
1985  MAKE_CASE(AArch64ISD::ST3LANEpost)
1986  MAKE_CASE(AArch64ISD::ST4LANEpost)
1987  MAKE_CASE(AArch64ISD::SMULL)
1988  MAKE_CASE(AArch64ISD::UMULL)
1989  MAKE_CASE(AArch64ISD::FRECPE)
1990  MAKE_CASE(AArch64ISD::FRECPS)
1991  MAKE_CASE(AArch64ISD::FRSQRTE)
1992  MAKE_CASE(AArch64ISD::FRSQRTS)
1993  MAKE_CASE(AArch64ISD::STG)
1994  MAKE_CASE(AArch64ISD::STZG)
1995  MAKE_CASE(AArch64ISD::ST2G)
1996  MAKE_CASE(AArch64ISD::STZ2G)
1997  MAKE_CASE(AArch64ISD::SUNPKHI)
1998  MAKE_CASE(AArch64ISD::SUNPKLO)
1999  MAKE_CASE(AArch64ISD::UUNPKHI)
2000  MAKE_CASE(AArch64ISD::UUNPKLO)
2001  MAKE_CASE(AArch64ISD::INSR)
2002  MAKE_CASE(AArch64ISD::PTEST)
2003  MAKE_CASE(AArch64ISD::PTRUE)
2004  MAKE_CASE(AArch64ISD::LD1_MERGE_ZERO)
2005  MAKE_CASE(AArch64ISD::LD1S_MERGE_ZERO)
2006  MAKE_CASE(AArch64ISD::LDNF1_MERGE_ZERO)
2007  MAKE_CASE(AArch64ISD::LDNF1S_MERGE_ZERO)
2008  MAKE_CASE(AArch64ISD::LDFF1_MERGE_ZERO)
2009  MAKE_CASE(AArch64ISD::LDFF1S_MERGE_ZERO)
2010  MAKE_CASE(AArch64ISD::LD1RQ_MERGE_ZERO)
2011  MAKE_CASE(AArch64ISD::LD1RO_MERGE_ZERO)
2012  MAKE_CASE(AArch64ISD::SVE_LD2_MERGE_ZERO)
2013  MAKE_CASE(AArch64ISD::SVE_LD3_MERGE_ZERO)
2014  MAKE_CASE(AArch64ISD::SVE_LD4_MERGE_ZERO)
2015  MAKE_CASE(AArch64ISD::GLD1_MERGE_ZERO)
2016  MAKE_CASE(AArch64ISD::GLD1_SCALED_MERGE_ZERO)
2017  MAKE_CASE(AArch64ISD::GLD1_SXTW_MERGE_ZERO)
2018  MAKE_CASE(AArch64ISD::GLD1_UXTW_MERGE_ZERO)
2019  MAKE_CASE(AArch64ISD::GLD1_SXTW_SCALED_MERGE_ZERO)
2020  MAKE_CASE(AArch64ISD::GLD1_UXTW_SCALED_MERGE_ZERO)
2021  MAKE_CASE(AArch64ISD::GLD1_IMM_MERGE_ZERO)
2022  MAKE_CASE(AArch64ISD::GLD1S_MERGE_ZERO)
2023  MAKE_CASE(AArch64ISD::GLD1S_SCALED_MERGE_ZERO)
2024  MAKE_CASE(AArch64ISD::GLD1S_SXTW_MERGE_ZERO)
2025  MAKE_CASE(AArch64ISD::GLD1S_UXTW_MERGE_ZERO)
2026  MAKE_CASE(AArch64ISD::GLD1S_SXTW_SCALED_MERGE_ZERO)
2027  MAKE_CASE(AArch64ISD::GLD1S_UXTW_SCALED_MERGE_ZERO)
2028  MAKE_CASE(AArch64ISD::GLD1S_IMM_MERGE_ZERO)
2029  MAKE_CASE(AArch64ISD::GLDFF1_MERGE_ZERO)
2030  MAKE_CASE(AArch64ISD::GLDFF1_SCALED_MERGE_ZERO)
2031  MAKE_CASE(AArch64ISD::GLDFF1_SXTW_MERGE_ZERO)
2032  MAKE_CASE(AArch64ISD::GLDFF1_UXTW_MERGE_ZERO)
2033  MAKE_CASE(AArch64ISD::GLDFF1_SXTW_SCALED_MERGE_ZERO)
2034  MAKE_CASE(AArch64ISD::GLDFF1_UXTW_SCALED_MERGE_ZERO)
2035  MAKE_CASE(AArch64ISD::GLDFF1_IMM_MERGE_ZERO)
2036  MAKE_CASE(AArch64ISD::GLDFF1S_MERGE_ZERO)
2037  MAKE_CASE(AArch64ISD::GLDFF1S_SCALED_MERGE_ZERO)
2038  MAKE_CASE(AArch64ISD::GLDFF1S_SXTW_MERGE_ZERO)
2039  MAKE_CASE(AArch64ISD::GLDFF1S_UXTW_MERGE_ZERO)
2040  MAKE_CASE(AArch64ISD::GLDFF1S_SXTW_SCALED_MERGE_ZERO)
2041  MAKE_CASE(AArch64ISD::GLDFF1S_UXTW_SCALED_MERGE_ZERO)
2042  MAKE_CASE(AArch64ISD::GLDFF1S_IMM_MERGE_ZERO)
2043  MAKE_CASE(AArch64ISD::GLDNT1_MERGE_ZERO)
2044  MAKE_CASE(AArch64ISD::GLDNT1_INDEX_MERGE_ZERO)
2045  MAKE_CASE(AArch64ISD::GLDNT1S_MERGE_ZERO)
2046  MAKE_CASE(AArch64ISD::ST1_PRED)
2047  MAKE_CASE(AArch64ISD::SST1_PRED)
2048  MAKE_CASE(AArch64ISD::SST1_SCALED_PRED)
2049  MAKE_CASE(AArch64ISD::SST1_SXTW_PRED)
2050  MAKE_CASE(AArch64ISD::SST1_UXTW_PRED)
2051  MAKE_CASE(AArch64ISD::SST1_SXTW_SCALED_PRED)
2052  MAKE_CASE(AArch64ISD::SST1_UXTW_SCALED_PRED)
2053  MAKE_CASE(AArch64ISD::SST1_IMM_PRED)
2054  MAKE_CASE(AArch64ISD::SSTNT1_PRED)
2055  MAKE_CASE(AArch64ISD::SSTNT1_INDEX_PRED)
2056  MAKE_CASE(AArch64ISD::LDP)
2057  MAKE_CASE(AArch64ISD::STP)
2058  MAKE_CASE(AArch64ISD::STNP)
2059  MAKE_CASE(AArch64ISD::BITREVERSE_MERGE_PASSTHRU)
2060  MAKE_CASE(AArch64ISD::BSWAP_MERGE_PASSTHRU)
2061  MAKE_CASE(AArch64ISD::CTLZ_MERGE_PASSTHRU)
2062  MAKE_CASE(AArch64ISD::CTPOP_MERGE_PASSTHRU)
2063  MAKE_CASE(AArch64ISD::DUP_MERGE_PASSTHRU)
2064  MAKE_CASE(AArch64ISD::INDEX_VECTOR)
2065  MAKE_CASE(AArch64ISD::UABD)
2066  MAKE_CASE(AArch64ISD::SABD)
2067  MAKE_CASE(AArch64ISD::CALL_RVMARKER)
2068  }
2069 #undef MAKE_CASE
2070  return nullptr;
2071 }
2072 
2073 MachineBasicBlock *
2074 AArch64TargetLowering::EmitF128CSEL(MachineInstr &MI,
2075  MachineBasicBlock *MBB) const {
2076  // We materialise the F128CSEL pseudo-instruction as some control flow and a
2077  // phi node:
2078 
2079  // OrigBB:
2080  // [... previous instrs leading to comparison ...]
2081  // b.ne TrueBB
2082  // b EndBB
2083  // TrueBB:
2084  // ; Fallthrough
2085  // EndBB:
2086  // Dest = PHI [IfTrue, TrueBB], [IfFalse, OrigBB]
2087 
2088  MachineFunction *MF = MBB->getParent();
2089  const TargetInstrInfo *TII = Subtarget->getInstrInfo();
2090  const BasicBlock *LLVM_BB = MBB->getBasicBlock();
2091  DebugLoc DL = MI.getDebugLoc();
2092  MachineFunction::iterator It = ++MBB->getIterator();
2093 
2094  Register DestReg = MI.getOperand(0).getReg();
2095  Register IfTrueReg = MI.getOperand(1).getReg();
2096  Register IfFalseReg = MI.getOperand(2).getReg();
2097  unsigned CondCode = MI.getOperand(3).getImm();
2098  bool NZCVKilled = MI.getOperand(4).isKill();
2099 
2100  MachineBasicBlock *TrueBB = MF->CreateMachineBasicBlock(LLVM_BB);
2101  MachineBasicBlock *EndBB = MF->CreateMachineBasicBlock(LLVM_BB);
2102  MF->insert(It, TrueBB);
2103  MF->insert(It, EndBB);
2104 
2105  // Transfer rest of current basic-block to EndBB
2106  EndBB->splice(EndBB->begin(), MBB, std::next(MachineBasicBlock::iterator(MI)),
2107  MBB->end());
2108  EndBB->transferSuccessorsAndUpdatePHIs(MBB);
2109 
2110  BuildMI(MBB, DL, TII->get(AArch64::Bcc)).addImm(CondCode).addMBB(TrueBB);
2111  BuildMI(MBB, DL, TII->get(AArch64::B)).addMBB(EndBB);
2112  MBB->addSuccessor(TrueBB);
2113  MBB->addSuccessor(EndBB);
2114 
2115  // TrueBB falls through to the end.
2116  TrueBB->addSuccessor(EndBB);
2117 
2118  if (!NZCVKilled) {
2119  TrueBB->addLiveIn(AArch64::NZCV);
2120  EndBB->addLiveIn(AArch64::NZCV);
2121  }
2122 
2123  BuildMI(*EndBB, EndBB->begin(), DL, TII->get(AArch64::PHI), DestReg)
2124  .addReg(IfTrueReg)
2125  .addMBB(TrueBB)
2126  .addReg(IfFalseReg)
2127  .addMBB(MBB);
2128 
2129  MI.eraseFromParent();
2130  return EndBB;
2131 }
2132 
2133 MachineBasicBlock *AArch64TargetLowering::EmitLoweredCatchRet(
2134  MachineInstr &MI, MachineBasicBlock *BB) const {
2135  assert(!isAsynchronousEHPersonality(classifyEHPersonality(
2136  BB->getParent()->getFunction().getPersonalityFn())) &&
2137  "SEH does not use catchret!");
2138  return BB;
2139 }
2140 
2141 MachineBasicBlock *AArch64TargetLowering::EmitInstrWithCustomInserter(
2142  MachineInstr &MI, MachineBasicBlock *BB) const {
2143  switch (MI.getOpcode()) {
2144  default:
2145 #ifndef NDEBUG
2146  MI.dump();
2147 #endif
2148  llvm_unreachable("Unexpected instruction for custom inserter!");
2149 
2150  case AArch64::F128CSEL:
2151  return EmitF128CSEL(MI, BB);
2152 
2153  case TargetOpcode::STACKMAP:
2154  case TargetOpcode::PATCHPOINT:
2155  case TargetOpcode::STATEPOINT:
2156  return emitPatchPoint(MI, BB);
2157 
2158  case AArch64::CATCHRET:
2159  return EmitLoweredCatchRet(MI, BB);
2160  }
2161 }
2162 
2163 //===----------------------------------------------------------------------===//
2164 // AArch64 Lowering private implementation.
2165 //===----------------------------------------------------------------------===//
2166 
2167 //===----------------------------------------------------------------------===//
2168 // Lowering Code
2169 //===----------------------------------------------------------------------===//
2170 
2171 /// isZerosVector - Check whether SDNode N is a zero-filled vector.
2172 static bool isZerosVector(const SDNode *N) {
2173  // Look through a bit convert.
2174  while (N->getOpcode() == ISD::BITCAST)
2175  N = N->getOperand(0).getNode();
2176 
2177  if (ISD::isConstantSplatVectorAllZeros(N))
2178  return true;
2179 
2180  if (N->getOpcode() != AArch64ISD::DUP)
2181  return false;
2182 
2183  auto Opnd0 = N->getOperand(0);
2184  auto *CINT = dyn_cast<ConstantSDNode>(Opnd0);
2185  auto *CFP = dyn_cast<ConstantFPSDNode>(Opnd0);
2186  return (CINT && CINT->isNullValue()) || (CFP && CFP->isZero());
2187 }
2188 
2189 /// changeIntCCToAArch64CC - Convert a DAG integer condition code to an AArch64
2190 /// CC
2191 static AArch64CC::CondCode changeIntCCToAArch64CC(ISD::CondCode CC) {
2192  switch (CC) {
2193  default:
2194  llvm_unreachable("Unknown condition code!");
2195  case ISD::SETNE:
2196  return AArch64CC::NE;
2197  case ISD::SETEQ:
2198  return AArch64CC::EQ;
2199  case ISD::SETGT:
2200  return AArch64CC::GT;
2201  case ISD::SETGE:
2202  return AArch64CC::GE;
2203  case ISD::SETLT:
2204  return AArch64CC::LT;
2205  case ISD::SETLE:
2206  return AArch64CC::LE;
2207  case ISD::SETUGT:
2208  return AArch64CC::HI;
2209  case ISD::SETUGE:
2210  return AArch64CC::HS;
2211  case ISD::SETULT:
2212  return AArch64CC::LO;
2213  case ISD::SETULE:
2214  return AArch64CC::LS;
2215  }
2216 }
2217 
2218 /// changeFPCCToAArch64CC - Convert a DAG fp condition code to an AArch64 CC.
2219 static void changeFPCCToAArch64CC(ISD::CondCode CC,
2220  AArch64CC::CondCode &CondCode,
2221  AArch64CC::CondCode &CondCode2) {
2222  CondCode2 = AArch64CC::AL;
2223  switch (CC) {
2224  default:
2225  llvm_unreachable("Unknown FP condition!");
2226  case ISD::SETEQ:
2227  case ISD::SETOEQ:
2228  CondCode = AArch64CC::EQ;
2229  break;
2230  case ISD::SETGT:
2231  case ISD::SETOGT:
2232  CondCode = AArch64CC::GT;
2233  break;
2234  case ISD::SETGE:
2235  case ISD::SETOGE:
2236  CondCode = AArch64CC::GE;
2237  break;
2238  case ISD::SETOLT:
2239  CondCode = AArch64CC::MI;
2240  break;
2241  case ISD::SETOLE:
2242  CondCode = AArch64CC::LS;
2243  break;
2244  case ISD::SETONE:
2245  CondCode = AArch64CC::MI;
2246  CondCode2 = AArch64CC::GT;
2247  break;
2248  case ISD::SETO:
2249  CondCode = AArch64CC::VC;
2250  break;
2251  case ISD::SETUO:
2252  CondCode = AArch64CC::VS;
2253  break;
2254  case ISD::SETUEQ:
2255  CondCode = AArch64CC::EQ;
2256  CondCode2 = AArch64CC::VS;
2257  break;
2258  case ISD::SETUGT:
2259  CondCode = AArch64CC::HI;
2260  break;
2261  case ISD::SETUGE:
2262  CondCode = AArch64CC::PL;
2263  break;
2264  case ISD::SETLT:
2265  case ISD::SETULT:
2266  CondCode = AArch64CC::LT;
2267  break;
2268  case ISD::SETLE:
2269  case ISD::SETULE:
2270  CondCode = AArch64CC::LE;
2271  break;
2272  case ISD::SETNE:
2273  case ISD::SETUNE:
2274  CondCode = AArch64CC::NE;
2275  break;
2276  }
2277 }
2278 
2279 /// Convert a DAG fp condition code to an AArch64 CC.
2280 /// This differs from changeFPCCToAArch64CC in that it returns cond codes that
2281 /// should be AND'ed instead of OR'ed.
2282 static void changeFPCCToANDAArch64CC(ISD::CondCode CC,
2283  AArch64CC::CondCode &CondCode,
2284  AArch64CC::CondCode &CondCode2) {
2285  CondCode2 = AArch64CC::AL;
2286  switch (CC) {
2287  default:
2288  changeFPCCToAArch64CC(CC, CondCode, CondCode2);
2289  assert(CondCode2 == AArch64CC::AL);
2290  break;
2291  case ISD::SETONE:
2292  // (a one b)
2293  // == ((a olt b) || (a ogt b))
2294  // == ((a ord b) && (a une b))
2295  CondCode = AArch64CC::VC;
2296  CondCode2 = AArch64CC::NE;
2297  break;
2298  case ISD::SETUEQ:
2299  // (a ueq b)
2300  // == ((a uno b) || (a oeq b))
2301  // == ((a ule b) && (a uge b))
2302  CondCode = AArch64CC::PL;
2303  CondCode2 = AArch64CC::LE;
2304  break;
2305  }
2306 }
2307 
2308 /// changeVectorFPCCToAArch64CC - Convert a DAG fp condition code to an AArch64
2309 /// CC usable with the vector instructions. Fewer operations are available
2310 /// without a real NZCV register, so we have to use less efficient combinations
2311 /// to get the same effect.
2312 static void changeVectorFPCCToAArch64CC(ISD::CondCode CC,
2313  AArch64CC::CondCode &CondCode,
2314  AArch64CC::CondCode &CondCode2,
2315  bool &Invert) {
2316  Invert = false;
2317  switch (CC) {
2318  default:
2319  // Mostly the scalar mappings work fine.
2320  changeFPCCToAArch64CC(CC, CondCode, CondCode2);
2321  break;
2322  case ISD::SETUO:
2323  Invert = true;
2324  LLVM_FALLTHROUGH;
2325  case ISD::SETO:
2326  CondCode = AArch64CC::MI;
2327  CondCode2 = AArch64CC::GE;
2328  break;
2329  case ISD::SETUEQ:
2330  case ISD::SETULT:
2331  case ISD::SETULE:
2332  case ISD::SETUGT:
2333  case ISD::SETUGE:
2334  // All of the compare-mask comparisons are ordered, but we can switch
2335  // between the two by a double inversion. E.g. ULE == !OGT.
2336  Invert = true;
2337  changeFPCCToAArch64CC(getSetCCInverse(CC, /* FP inverse */ MVT::f32),
2338  CondCode, CondCode2);
2339  break;
2340  }
2341 }
2342 
2343 static bool isLegalArithImmed(uint64_t C) {
2344  // Matches AArch64DAGToDAGISel::SelectArithImmed().
2345  bool IsLegal = (C >> 12 == 0) || ((C & 0xFFFULL) == 0 && C >> 24 == 0);
2346  LLVM_DEBUG(dbgs() << "Is imm " << C
2347  << " legal: " << (IsLegal ? "yes\n" : "no\n"));
2348  return IsLegal;
2349 }
2350 
2351 // Can a (CMP op1, (sub 0, op2) be turned into a CMN instruction on
2352 // the grounds that "op1 - (-op2) == op1 + op2" ? Not always, the C and V flags
2353 // can be set differently by this operation. It comes down to whether
2354 // "SInt(~op2)+1 == SInt(~op2+1)" (and the same for UInt). If they are then
2355 // everything is fine. If not then the optimization is wrong. Thus general
2356 // comparisons are only valid if op2 != 0.
2357 //
2358 // So, finally, the only LLVM-native comparisons that don't mention C and V
2359 // are SETEQ and SETNE. They're the only ones we can safely use CMN for in
2360 // the absence of information about op2.
2361 static bool isCMN(SDValue Op, ISD::CondCode CC) {
2362  return Op.getOpcode() == ISD::SUB && isNullConstant(Op.getOperand(0)) &&
2363  (CC == ISD::SETEQ || CC == ISD::SETNE);
2364 }
2365 
2366 static SDValue emitStrictFPComparison(SDValue LHS, SDValue RHS, const SDLoc &dl,
2367  SelectionDAG &DAG, SDValue Chain,
2368  bool IsSignaling) {
2369  EVT VT = LHS.getValueType();
2370  assert(VT != MVT::f128);
2371  assert(VT != MVT::f16 && "Lowering of strict fp16 not yet implemented");
2372  unsigned Opcode =
2373  IsSignaling ? AArch64ISD::STRICT_FCMPE : AArch64ISD::STRICT_FCMP;
2374  return DAG.getNode(Opcode, dl, {VT, MVT::Other}, {Chain, LHS, RHS});
2375 }
2376 
2377 static SDValue emitComparison(SDValue LHS, SDValue RHS, ISD::CondCode CC,
2378  const SDLoc &dl, SelectionDAG &DAG) {
2379  EVT VT = LHS.getValueType();
2380  const bool FullFP16 =
2381  static_cast<const AArch64Subtarget &>(DAG.getSubtarget()).hasFullFP16();
2382 
2383  if (VT.isFloatingPoint()) {
2384  assert(VT != MVT::f128);
2385  if (VT == MVT::f16 && !FullFP16) {
2386  LHS = DAG.getNode(ISD::FP_EXTEND, dl, MVT::f32, LHS);
2387  RHS = DAG.getNode(ISD::FP_EXTEND, dl, MVT::f32, RHS);
2388  VT = MVT::f32;
2389  }
2390  return DAG.getNode(AArch64ISD::FCMP, dl, VT, LHS, RHS);
2391  }
2392 
2393  // The CMP instruction is just an alias for SUBS, and representing it as
2394  // SUBS means that it's possible to get CSE with subtract operations.
2395  // A later phase can perform the optimization of setting the destination
2396  // register to WZR/XZR if it ends up being unused.
2397  unsigned Opcode = AArch64ISD::SUBS;
2398 
2399  if (isCMN(RHS, CC)) {
2400  // Can we combine a (CMP op1, (sub 0, op2) into a CMN instruction ?
2401  Opcode = AArch64ISD::ADDS;
2402  RHS = RHS.getOperand(1);
2403  } else if (isCMN(LHS, CC)) {
2404  // As we are looking for EQ/NE compares, the operands can be commuted ; can
2405  // we combine a (CMP (sub 0, op1), op2) into a CMN instruction ?
2406  Opcode = AArch64ISD::ADDS;
2407  LHS = LHS.getOperand(1);
2408  } else if (isNullConstant(RHS) && !isUnsignedIntSetCC(CC)) {
2409  if (LHS.getOpcode() == ISD::AND) {
2410  // Similarly, (CMP (and X, Y), 0) can be implemented with a TST
2411  // (a.k.a. ANDS) except that the flags are only guaranteed to work for one
2412  // of the signed comparisons.
2413  const SDValue ANDSNode = DAG.getNode(AArch64ISD::ANDS, dl,
2414  DAG.getVTList(VT, MVT_CC),
2415  LHS.getOperand(0),
2416  LHS.getOperand(1));
2417  // Replace all users of (and X, Y) with newly generated (ands X, Y)
2418  DAG.ReplaceAllUsesWith(LHS, ANDSNode);
2419  return ANDSNode.getValue(1);
2420  } else if (LHS.getOpcode() == AArch64ISD::ANDS) {
2421  // Use result of ANDS
2422  return LHS.getValue(1);
2423  }
2424  }
2425 
2426  return DAG.getNode(Opcode, dl, DAG.getVTList(VT, MVT_CC), LHS, RHS)
2427  .getValue(1);
2428 }
2429 
2430 /// \defgroup AArch64CCMP CMP;CCMP matching
2431 ///
2432 /// These functions deal with the formation of CMP;CCMP;... sequences.
2433 /// The CCMP/CCMN/FCCMP/FCCMPE instructions allow the conditional execution of
2434 /// a comparison. They set the NZCV flags to a predefined value if their
2435 /// predicate is false. This allows to express arbitrary conjunctions, for
2436 /// example "cmp 0 (and (setCA (cmp A)) (setCB (cmp B)))"
2437 /// expressed as:
2438 /// cmp A
2439 /// ccmp B, inv(CB), CA
2440 /// check for CB flags
2441 ///
2442 /// This naturally lets us implement chains of AND operations with SETCC
2443 /// operands. And we can even implement some other situations by transforming
2444 /// them:
2445 /// - We can implement (NEG SETCC) i.e. negating a single comparison by
2446 /// negating the flags used in a CCMP/FCCMP operations.
2447 /// - We can negate the result of a whole chain of CMP/CCMP/FCCMP operations
2448 /// by negating the flags we test for afterwards. i.e.
2449 /// NEG (CMP CCMP CCCMP ...) can be implemented.
2450 /// - Note that we can only ever negate all previously processed results.
2451 /// What we can not implement by flipping the flags to test is a negation
2452 /// of two sub-trees (because the negation affects all sub-trees emitted so
2453 /// far, so the 2nd sub-tree we emit would also affect the first).
2454 /// With those tools we can implement some OR operations:
2455 /// - (OR (SETCC A) (SETCC B)) can be implemented via:
2456 /// NEG (AND (NEG (SETCC A)) (NEG (SETCC B)))
2457 /// - After transforming OR to NEG/AND combinations we may be able to use NEG
2458 /// elimination rules from earlier to implement the whole thing as a
2459 /// CCMP/FCCMP chain.
2460 ///
2461 /// As complete example:
2462 /// or (or (setCA (cmp A)) (setCB (cmp B)))
2463 /// (and (setCC (cmp C)) (setCD (cmp D)))"
2464 /// can be reassociated to:
2465 /// or (and (setCC (cmp C)) setCD (cmp D))
2466 // (or (setCA (cmp A)) (setCB (cmp B)))
2467 /// can be transformed to:
2468 /// not (and (not (and (setCC (cmp C)) (setCD (cmp D))))
2469 /// (and (not (setCA (cmp A)) (not (setCB (cmp B))))))"
2470 /// which can be implemented as:
2471 /// cmp C
2472 /// ccmp D, inv(CD), CC
2473 /// ccmp A, CA, inv(CD)
2474 /// ccmp B, CB, inv(CA)
2475 /// check for CB flags
2476 ///
2477 /// A counterexample is "or (and A B) (and C D)" which translates to
2478 /// not (and (not (and (not A) (not B))) (not (and (not C) (not D)))), we
2479 /// can only implement 1 of the inner (not) operations, but not both!
2480 /// @{
2481 
2482 /// Create a conditional comparison; Use CCMP, CCMN or FCCMP as appropriate.
2483 static SDValue emitConditionalComparison(SDValue LHS, SDValue RHS,
2484  ISD::CondCode CC, SDValue CCOp,
2485  AArch64CC::CondCode Predicate,
2486  AArch64CC::CondCode OutCC,
2487  const SDLoc &DL, SelectionDAG &DAG) {
2488  unsigned Opcode = 0;
2489  const bool FullFP16 =
2490  static_cast<const AArch64Subtarget &>(DAG.getSubtarget()).hasFullFP16();
2491 
2492  if (LHS.getValueType().isFloatingPoint()) {
2493  assert(LHS.getValueType() != MVT::f128);
2494  if (LHS.getValueType() == MVT::f16 && !FullFP16) {
2495  LHS = DAG.getNode(ISD::FP_EXTEND, DL, MVT::f32, LHS);
2496  RHS = DAG.getNode(ISD::FP_EXTEND, DL, MVT::f32, RHS);
2497  }
2498  Opcode = AArch64ISD::FCCMP;
2499  } else if (RHS.getOpcode() == ISD::SUB) {
2500  SDValue SubOp0 = RHS.getOperand(0);
2501  if (isNullConstant(SubOp0) && (CC == ISD::SETEQ || CC == ISD::SETNE)) {
2502  // See emitComparison() on why we can only do this for SETEQ and SETNE.
2503  Opcode = AArch64ISD::CCMN;
2504  RHS = RHS.getOperand(1);
2505  }
2506  }
2507  if (Opcode == 0)
2508  Opcode = AArch64ISD::CCMP;
2509 
2510  SDValue Condition = DAG.getConstant(Predicate, DL, MVT_CC);
2511  AArch64CC::CondCode InvOutCC = AArch64CC::getInvertedCondCode(OutCC);
2512  unsigned NZCV = AArch64CC::getNZCVToSatisfyCondCode(InvOutCC);
2513  SDValue NZCVOp = DAG.getConstant(NZCV, DL, MVT::i32);
2514  return DAG.getNode(Opcode, DL, MVT_CC, LHS, RHS, NZCVOp, Condition, CCOp);
2515 }
2516 
2517 /// Returns true if @p Val is a tree of AND/OR/SETCC operations that can be
2518 /// expressed as a conjunction. See \ref AArch64CCMP.
2519 /// \param CanNegate Set to true if we can negate the whole sub-tree just by
2520 /// changing the conditions on the SETCC tests.
2521 /// (this means we can call emitConjunctionRec() with
2522 /// Negate==true on this sub-tree)
2523 /// \param MustBeFirst Set to true if this subtree needs to be negated and we
2524 /// cannot do the negation naturally. We are required to
2525 /// emit the subtree first in this case.
2526 /// \param WillNegate Is true if are called when the result of this
2527 /// subexpression must be negated. This happens when the
2528 /// outer expression is an OR. We can use this fact to know
2529 /// that we have a double negation (or (or ...) ...) that
2530 /// can be implemented for free.
2531 static bool canEmitConjunction(const SDValue Val, bool &CanNegate,
2532  bool &MustBeFirst, bool WillNegate,
2533  unsigned Depth = 0) {
2534  if (!Val.hasOneUse())
2535  return false;
2536  unsigned Opcode = Val->getOpcode();
2537  if (Opcode == ISD::SETCC) {
2538  if (Val->getOperand(0).getValueType() == MVT::f128)
2539  return false;
2540  CanNegate = true;
2541  MustBeFirst = false;
2542  return true;
2543  }
2544  // Protect against exponential runtime and stack overflow.
2545  if (Depth > 6)
2546  return false;
2547  if (Opcode == ISD::AND || Opcode == ISD::OR) {
2548  bool IsOR = Opcode == ISD::OR;
2549  SDValue O0 = Val->getOperand(0);
2550  SDValue O1 = Val->getOperand(1);
2551  bool CanNegateL;
2552  bool MustBeFirstL;
2553  if (!canEmitConjunction(O0, CanNegateL, MustBeFirstL, IsOR, Depth+1))
2554  return false;
2555  bool CanNegateR;
2556  bool MustBeFirstR;
2557  if (!canEmitConjunction(O1, CanNegateR, MustBeFirstR, IsOR, Depth+1))
2558  return false;
2559 
2560  if (MustBeFirstL && MustBeFirstR)
2561  return false;
2562 
2563  if (IsOR) {
2564  // For an OR expression we need to be able to naturally negate at least
2565  // one side or we cannot do the transformation at all.
2566  if (!CanNegateL && !CanNegateR)
2567  return false;
2568  // If we the result of the OR will be negated and we can naturally negate
2569  // the leafs, then this sub-tree as a whole negates naturally.
2570  CanNegate = WillNegate && CanNegateL && CanNegateR;
2571  // If we cannot naturally negate the whole sub-tree, then this must be
2572  // emitted first.
2573  MustBeFirst = !CanNegate;
2574  } else {
2575  assert(Opcode == ISD::AND && "Must be OR or AND");
2576  // We cannot naturally negate an AND operation.
2577  CanNegate = false;
2578  MustBeFirst = MustBeFirstL || MustBeFirstR;
2579  }
2580  return true;
2581  }
2582  return false;
2583 }
2584 
2585 /// Emit conjunction or disjunction tree with the CMP/FCMP followed by a chain
2586 /// of CCMP/CFCMP ops. See @ref AArch64CCMP.
2587 /// Tries to transform the given i1 producing node @p Val to a series compare
2588 /// and conditional compare operations. @returns an NZCV flags producing node
2589 /// and sets @p OutCC to the flags that should be tested or returns SDValue() if
2590 /// transformation was not possible.
2591 /// \p Negate is true if we want this sub-tree being negated just by changing
2592 /// SETCC conditions.
2593 static SDValue emitConjunctionRec(SelectionDAG &DAG, SDValue Val,
2594  AArch64CC::CondCode &OutCC, bool Negate, SDValue CCOp,
2595  AArch64CC::CondCode Predicate) {
2596  // We're at a tree leaf, produce a conditional comparison operation.
2597  unsigned Opcode = Val->getOpcode();
2598  if (Opcode == ISD::SETCC) {
2599  SDValue LHS = Val->getOperand(0);
2600  SDValue RHS = Val->getOperand(1);
2601  ISD::CondCode CC = cast<CondCodeSDNode>(Val->getOperand(2))->get();
2602  bool isInteger = LHS.getValueType().isInteger();
2603  if (Negate)
2604  CC = getSetCCInverse(CC, LHS.getValueType());
2605  SDLoc DL(Val);
2606  // Determine OutCC and handle FP special case.
2607  if (isInteger) {
2608  OutCC = changeIntCCToAArch64CC(CC);
2609  } else {
2610  assert(LHS.getValueType().isFloatingPoint());
2611  AArch64CC::CondCode ExtraCC;
2612  changeFPCCToANDAArch64CC(CC, OutCC, ExtraCC);
2613  // Some floating point conditions can't be tested with a single condition
2614  // code. Construct an additional comparison in this case.
2615  if (ExtraCC != AArch64CC::AL) {
2616  SDValue ExtraCmp;
2617  if (!CCOp.getNode())
2618  ExtraCmp = emitComparison(LHS, RHS, CC, DL, DAG);
2619  else
2620  ExtraCmp = emitConditionalComparison(LHS, RHS, CC, CCOp, Predicate,
2621  ExtraCC, DL, DAG);
2622  CCOp = ExtraCmp;
2623  Predicate = ExtraCC;
2624  }
2625  }
2626 
2627  // Produce a normal comparison if we are first in the chain
2628  if (!CCOp)
2629  return emitComparison(LHS, RHS, CC, DL, DAG);
2630  // Otherwise produce a ccmp.
2631  return emitConditionalComparison(LHS, RHS, CC, CCOp, Predicate, OutCC, DL,
2632  DAG);
2633  }
2634  assert(Val->hasOneUse() && "Valid conjunction/disjunction tree");
2635 
2636  bool IsOR = Opcode == ISD::OR;
2637 
2638  SDValue LHS = Val->getOperand(0);
2639  bool CanNegateL;
2640  bool MustBeFirstL;
2641  bool ValidL = canEmitConjunction(LHS, CanNegateL, MustBeFirstL, IsOR);
2642  assert(ValidL && "Valid conjunction/disjunction tree");
2643  (void)ValidL;
2644 
2645  SDValue RHS = Val->getOperand(1);
2646  bool CanNegateR;
2647  bool MustBeFirstR;
2648  bool ValidR = canEmitConjunction(RHS, CanNegateR, MustBeFirstR, IsOR);
2649  assert(ValidR && "Valid conjunction/disjunction tree");
2650  (void)ValidR;
2651 
2652  // Swap sub-tree that must come first to the right side.
2653  if (MustBeFirstL) {
2654  assert(!MustBeFirstR && "Valid conjunction/disjunction tree");
2655  std::swap(LHS, RHS);
2656  std::swap(CanNegateL, CanNegateR);
2657  std::swap(MustBeFirstL, MustBeFirstR);
2658  }
2659 
2660  bool NegateR;
2661  bool NegateAfterR;
2662  bool NegateL;
2663  bool NegateAfterAll;
2664  if (Opcode == ISD::OR) {
2665  // Swap the sub-tree that we can negate naturally to the left.
2666  if (!CanNegateL) {
2667  assert(CanNegateR && "at least one side must be negatable");
2668  assert(!MustBeFirstR && "invalid conjunction/disjunction tree");
2669  assert(!Negate);
2670  std::swap(LHS, RHS);
2671  NegateR = false;
2672  NegateAfterR = true;
2673  } else {
2674  // Negate the left sub-tree if possible, otherwise negate the result.
2675  NegateR = CanNegateR;
2676  NegateAfterR = !CanNegateR;
2677  }
2678  NegateL = true;
2679  NegateAfterAll = !Negate;
2680  } else {
2681  assert(Opcode == ISD::AND && "Valid conjunction/disjunction tree");
2682  assert(!Negate && "Valid conjunction/disjunction tree");
2683 
2684  NegateL = false;
2685  NegateR = false;
2686  NegateAfterR = false;
2687  NegateAfterAll = false;
2688  }
2689 
2690  // Emit sub-trees.
2691  AArch64CC::CondCode RHSCC;
2692  SDValue CmpR = emitConjunctionRec(DAG, RHS, RHSCC, NegateR, CCOp, Predicate);
2693  if (NegateAfterR)
2694  RHSCC = AArch64CC::getInvertedCondCode(RHSCC);
2695  SDValue CmpL = emitConjunctionRec(DAG, LHS, OutCC, NegateL, CmpR, RHSCC);
2696  if (NegateAfterAll)
2697  OutCC = AArch64CC::getInvertedCondCode(OutCC);
2698  return CmpL;
2699 }
2700 
2701 /// Emit expression as a conjunction (a series of CCMP/CFCMP ops).
2702 /// In some cases this is even possible with OR operations in the expression.
2703 /// See \ref AArch64CCMP.
2704 /// \see emitConjunctionRec().
2705 static SDValue emitConjunction(SelectionDAG &DAG, SDValue Val,
2706  AArch64CC::CondCode &OutCC) {
2707  bool DummyCanNegate;
2708  bool DummyMustBeFirst;
2709  if (!canEmitConjunction(Val, DummyCanNegate, DummyMustBeFirst, false))
2710  return SDValue();
2711 
2712  return emitConjunctionRec(DAG, Val, OutCC, false, SDValue(), AArch64CC::AL);
2713 }
2714 
2715 /// @}
2716 
2717 /// Returns how profitable it is to fold a comparison's operand's shift and/or
2718 /// extension operations.
2719 static unsigned getCmpOperandFoldingProfit(SDValue Op) {
2720  auto isSupportedExtend = [&](SDValue V) {
2721  if (V.getOpcode() == ISD::SIGN_EXTEND_INREG)
2722  return true;
2723 
2724  if (V.getOpcode() == ISD::AND)
2725  if (ConstantSDNode *MaskCst = dyn_cast<ConstantSDNode>(V.getOperand(1))) {
2726  uint64_t Mask = MaskCst->getZExtValue();
2727  return (Mask == 0xFF || Mask == 0xFFFF || Mask == 0xFFFFFFFF);
2728  }
2729 
2730  return false;
2731  };
2732 
2733  if (!Op.hasOneUse())
2734  return 0;
2735 
2736  if (isSupportedExtend(Op))
2737  return 1;
2738 
2739  unsigned Opc = Op.getOpcode();
2740  if (Opc == ISD::SHL || Opc == ISD::SRL || Opc == ISD::SRA)
2741  if (ConstantSDNode *ShiftCst = dyn_cast<ConstantSDNode>(Op.getOperand(1))) {
2742  uint64_t Shift = ShiftCst->getZExtValue();
2743  if (isSupportedExtend(Op.getOperand(0)))
2744  return (Shift <= 4) ? 2 : 1;
2745  EVT VT = Op.getValueType();
2746  if ((VT == MVT::i32 && Shift <= 31) || (VT == MVT::i64 && Shift <= 63))
2747  return 1;
2748  }
2749 
2750  return 0;
2751 }
2752 
2753 static SDValue getAArch64Cmp(SDValue LHS, SDValue RHS, ISD::CondCode CC,
2754  SDValue &AArch64cc, SelectionDAG &DAG,
2755  const SDLoc &dl) {
2756  if (ConstantSDNode *RHSC = dyn_cast<ConstantSDNode>(RHS.getNode())) {
2757  EVT VT = RHS.getValueType();
2758  uint64_t C = RHSC->getZExtValue();
2759  if (!isLegalArithImmed(C)) {
2760  // Constant does not fit, try adjusting it by one?
2761  switch (CC) {
2762  default:
2763  break;
2764  case ISD::SETLT:
2765  case ISD::SETGE:
2766  if ((VT == MVT::i32 && C != 0x80000000 &&
2767  isLegalArithImmed((uint32_t)(C - 1))) ||
2768  (VT == MVT::i64 && C != 0x80000000ULL &&
2769  isLegalArithImmed(C - 1ULL))) {
2770  CC = (CC == ISD::SETLT) ? ISD::SETLE : ISD::SETGT;
2771  C = (VT == MVT::i32) ? (uint32_t)(C - 1) : C - 1;
2772  RHS = DAG.getConstant(C, dl, VT);
2773  }
2774  break;
2775  case ISD::SETULT:
2776  case ISD::SETUGE:
2777  if ((VT == MVT::i32 && C != 0 &&
2778  isLegalArithImmed((uint32_t)(C - 1))) ||
2779  (VT == MVT::i64 && C != 0ULL && isLegalArithImmed(C - 1ULL))) {
2780  CC = (CC == ISD::SETULT) ? ISD::SETULE : ISD::SETUGT;
2781  C = (VT == MVT::i32) ? (uint32_t)(C - 1) : C - 1;
2782  RHS = DAG.getConstant(C, dl, VT);
2783  }
2784  break;
2785  case ISD::SETLE:
2786  case ISD::SETGT:
2787  if ((VT == MVT::i32 && C != INT32_MAX &&
2788  isLegalArithImmed((uint32_t)(C + 1))) ||
2789  (VT == MVT::i64 && C != INT64_MAX &&
2790  isLegalArithImmed(C + 1ULL))) {
2791  CC = (CC == ISD::SETLE) ? ISD::SETLT : ISD::SETGE;
2792  C = (VT == MVT::i32) ? (uint32_t)(C + 1) : C + 1;
2793  RHS = DAG.getConstant(C, dl, VT);
2794  }
2795  break;
2796  case ISD::SETULE:
2797  case ISD::SETUGT:
2798  if ((VT == MVT::i32 && C != UINT32_MAX &&
2799  isLegalArithImmed((uint32_t)(C + 1))) ||
2800  (VT == MVT::i64 && C != UINT64_MAX &&
2801  isLegalArithImmed(C + 1ULL))) {
2802  CC = (CC == ISD::SETULE) ? ISD::SETULT : ISD::SETUGE;
2803  C = (VT == MVT::i32) ? (uint32_t)(C + 1) : C + 1;
2804  RHS = DAG.getConstant(C, dl, VT);
2805  }
2806  break;
2807  }
2808  }
2809  }
2810 
2811  // Comparisons are canonicalized so that the RHS operand is simpler than the
2812  // LHS one, the extreme case being when RHS is an immediate. However, AArch64
2813  // can fold some shift+extend operations on the RHS operand, so swap the
2814  // operands if that can be done.
2815  //
2816  // For example:
2817  // lsl w13, w11, #1
2818  // cmp w13, w12
2819  // can be turned into:
2820  // cmp w12, w11, lsl #1
2821  if (!isa<ConstantSDNode>(RHS) ||
2822  !isLegalArithImmed(cast<ConstantSDNode>(RHS)->getZExtValue())) {
2823  SDValue TheLHS = isCMN(LHS, CC) ? LHS.getOperand(1) : LHS;
2824 
2825  if (getCmpOperandFoldingProfit(TheLHS) > getCmpOperandFoldingProfit(RHS)) {
2826  std::swap(LHS, RHS);
2827  CC = ISD::getSetCCSwappedOperands(CC);
2828  }
2829  }
2830 
2831  SDValue Cmp;
2832  AArch64CC::CondCode AArch64CC;
2833  if ((CC == ISD::SETEQ || CC == ISD::SETNE) && isa<ConstantSDNode>(RHS)) {
2834  const ConstantSDNode *RHSC = cast<ConstantSDNode>(RHS);
2835 
2836  // The imm operand of ADDS is an unsigned immediate, in the range 0 to 4095.
2837  // For the i8 operand, the largest immediate is 255, so this can be easily
2838  // encoded in the compare instruction. For the i16 operand, however, the
2839  // largest immediate cannot be encoded in the compare.
2840  // Therefore, use a sign extending load and cmn to avoid materializing the
2841  // -1 constant. For example,
2842  // movz w1, #65535
2843  // ldrh w0, [x0, #0]
2844  // cmp w0, w1
2845  // >
2846  // ldrsh w0, [x0, #0]
2847  // cmn w0, #1
2848  // Fundamental, we're relying on the property that (zext LHS) == (zext RHS)
2849  // if and only if (sext LHS) == (sext RHS). The checks are in place to
2850  // ensure both the LHS and RHS are truly zero extended and to make sure the
2851  // transformation is profitable.
2852  if ((RHSC->getZExtValue() >> 16 == 0) && isa<LoadSDNode>(LHS) &&
2853  cast<LoadSDNode>(LHS)->getExtensionType() == ISD::ZEXTLOAD &&
2854  cast<LoadSDNode>(LHS)->getMemoryVT() == MVT::i16 &&
2855  LHS.getNode()->hasNUsesOfValue(1, 0)) {
2856  int16_t ValueofRHS = cast<ConstantSDNode>(RHS)->getZExtValue();
2857  if (ValueofRHS < 0 && isLegalArithImmed(-ValueofRHS)) {
2858  SDValue SExt =
2859  DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, LHS.getValueType(), LHS,
2860  DAG.getValueType(MVT::i16));
2861  Cmp = emitComparison(SExt, DAG.getConstant(ValueofRHS, dl,
2862  RHS.getValueType()),
2863  CC, dl, DAG);
2864  AArch64CC = changeIntCCToAArch64CC(CC);
2865  }
2866  }
2867 
2868  if (!Cmp && (RHSC->isNullValue() || RHSC->isOne())) {
2869  if ((Cmp = emitConjunction(DAG, LHS, AArch64CC))) {
2870  if ((CC == ISD::SETNE) ^ RHSC->isNullValue())
2871  AArch64CC = AArch64CC::getInvertedCondCode(AArch64CC);
2872  }
2873  }
2874  }
2875 
2876  if (!Cmp) {
2877  Cmp = emitComparison(LHS, RHS, CC, dl, DAG);
2878  AArch64CC = changeIntCCToAArch64CC(CC);
2879  }
2880  AArch64cc = DAG.getConstant(AArch64CC, dl, MVT_CC);
2881  return Cmp;
2882 }
2883 
2884 static std::pair<SDValue, SDValue>
2885 getAArch64XALUOOp(AArch64CC::CondCode &CC, SDValue Op, SelectionDAG &DAG) {
2886  assert((Op.getValueType() == MVT::i32 || Op.getValueType() == MVT::i64) &&
2887  "Unsupported value type");
2888  SDValue Value, Overflow;
2889  SDLoc DL(Op);
2890  SDValue LHS = Op.getOperand(0);
2891  SDValue RHS = Op.getOperand(1);
2892  unsigned Opc = 0;
2893  switch (Op.getOpcode()) {
2894  default:
2895  llvm_unreachable("Unknown overflow instruction!");
2896  case ISD::SADDO:
2897  Opc = AArch64ISD::ADDS;
2898  CC = AArch64CC::VS;
2899  break;
2900  case ISD::UADDO:
2901  Opc = AArch64ISD::ADDS;
2902  CC = AArch64CC::HS;
2903  break;
2904  case ISD::SSUBO:
2905  Opc = AArch64ISD::SUBS;
2906  CC = AArch64CC::VS;
2907  break;
2908  case ISD::USUBO:
2909  Opc = AArch64ISD::SUBS;
2910  CC = AArch64CC::LO;
2911  break;
2912  // Multiply needs a little bit extra work.
2913  case ISD::SMULO:
2914  case ISD::UMULO: {
2915  CC = AArch64CC::NE;
2916  bool IsSigned = Op.getOpcode() == ISD::SMULO;
2917  if (Op.getValueType() == MVT::i32) {
2918  unsigned ExtendOpc = IsSigned ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND;
2919  // For a 32 bit multiply with overflow check we want the instruction
2920  // selector to generate a widening multiply (SMADDL/UMADDL). For that we
2921  // need to generate the following pattern:
2922  // (i64 add 0, (i64 mul (i64 sext|zext i32 %a), (i64 sext|zext i32 %b))
2923  LHS = DAG.getNode(ExtendOpc, DL, MVT::i64, LHS);
2924  RHS = DAG.getNode(ExtendOpc, DL, MVT::i64, RHS);
2925  SDValue Mul = DAG.getNode(ISD::MUL, DL, MVT::i64, LHS, RHS);
2926  SDValue Add = DAG.getNode(ISD::ADD, DL, MVT::i64, Mul,
2927  DAG.getConstant(0, DL, MVT::i64));
2928  // On AArch64 the upper 32 bits are always zero extended for a 32 bit
2929  // operation. We need to clear out the upper 32 bits, because we used a
2930  // widening multiply that wrote all 64 bits. In the end this should be a
2931  // noop.
2932  Value = DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Add);
2933  if (IsSigned) {
2934  // The signed overflow check requires more than just a simple check for
2935  // any bit set in the upper 32 bits of the result. These bits could be
2936  // just the sign bits of a negative number. To perform the overflow
2937  // check we have to arithmetic shift right the 32nd bit of the result by
2938  // 31 bits. Then we compare the result to the upper 32 bits.
2939  SDValue UpperBits = DAG.getNode(ISD::SRL, DL, MVT::i64, Add,
2940  DAG.getConstant(32, DL, MVT::i64));
2941  UpperBits = DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, UpperBits);
2942  SDValue LowerBits = DAG.getNode(ISD::SRA, DL, MVT::i32, Value,
2943  DAG.getConstant(31, DL, MVT::i64));
2944  // It is important that LowerBits is last, otherwise the arithmetic
2945  // shift will not be folded into the compare (SUBS).
2946  SDVTList VTs = DAG.getVTList(MVT::i32, MVT::i32);
2947  Overflow = DAG.getNode(AArch64ISD::SUBS, DL, VTs, UpperBits, LowerBits)
2948  .getValue(1);
2949  } else {
2950  // The overflow check for unsigned multiply is easy. We only need to
2951  // check if any of the upper 32 bits are set. This can be done with a
2952  // CMP (shifted register). For that we need to generate the following
2953  // pattern:
2954  // (i64 AArch64ISD::SUBS i64 0, (i64 srl i64 %Mul, i64 32)
2955  SDValue UpperBits = DAG.getNode(ISD::SRL, DL, MVT::i64, Mul,
2956  DAG.getConstant(32, DL, MVT::i64));
2957  SDVTList VTs = DAG.getVTList(MVT::i64, MVT::i32);
2958  Overflow =
2959  DAG.getNode(AArch64ISD::SUBS, DL, VTs,
2960  DAG.getConstant(0, DL, MVT::i64),
2961  UpperBits).getValue(1);
2962  }
2963  break;
2964  }
2965  assert(Op.getValueType() == MVT::i64 && "Expected an i64 value type");
2966  // For the 64 bit multiply
2967  Value = DAG.getNode(ISD::MUL, DL, MVT::i64, LHS, RHS);
2968  if (IsSigned) {
2969  SDValue UpperBits = DAG.getNode(ISD::MULHS, DL, MVT::i64, LHS, RHS);
2970  SDValue LowerBits = DAG.getNode(ISD::SRA, DL, MVT::i64, Value,
2971  DAG.getConstant(63, DL, MVT::i64));
2972  // It is important that LowerBits is last, otherwise the arithmetic
2973  // shift will not be folded into the compare (SUBS).
2974  SDVTList VTs = DAG.getVTList(MVT::i64, MVT::i32);
2975  Overflow = DAG.getNode(AArch64ISD::SUBS, DL, VTs, UpperBits, LowerBits)
2976  .getValue(1);
2977  } else {
2978  SDValue UpperBits = DAG.getNode(ISD::MULHU, DL, MVT::i64, LHS, RHS);
2979  SDVTList VTs = DAG.getVTList(MVT::i64, MVT::i32);
2980  Overflow =
2981  DAG.getNode(AArch64ISD::SUBS, DL, VTs,
2982  DAG.getConstant(0, DL, MVT::i64),
2983  UpperBits).getValue(1);
2984  }
2985  break;
2986  }
2987  } // switch (...)
2988 
2989  if (Opc) {
2990  SDVTList VTs = DAG.getVTList(Op->getValueType(0), MVT::i32);
2991 
2992  // Emit the AArch64 operation with overflow check.
2993  Value = DAG.getNode(Opc, DL, VTs, LHS, RHS);
2994  Overflow = Value.getValue(1);
2995  }
2996  return std::make_pair(Value, Overflow);
2997 }
2998 
2999 SDValue AArch64TargetLowering::LowerXOR(SDValue Op, SelectionDAG &DAG) const {
3000  if (useSVEForFixedLengthVectorVT(Op.getValueType()))
3001  return LowerToScalableOp(Op, DAG);
3002 
3003  SDValue Sel = Op.getOperand(0);
3004  SDValue Other = Op.getOperand(1);
3005  SDLoc dl(Sel);
3006 
3007  // If the operand is an overflow checking operation, invert the condition
3008  // code and kill the Not operation. I.e., transform:
3009  // (xor (overflow_op_bool, 1))
3010  // -->
3011  // (csel 1, 0, invert(cc), overflow_op_bool)
3012  // ... which later gets transformed to just a cset instruction with an
3013  // inverted condition code, rather than a cset + eor sequence.
3014  if (isOneConstant(Other) && ISD::isOverflowIntrOpRes(Sel)) {
3015  // Only lower legal XALUO ops.
3016  if (!DAG.getTargetLoweringInfo().isTypeLegal(Sel->getValueType(0)))
3017  return SDValue();
3018 
3019  SDValue TVal = DAG.getConstant(1, dl, MVT::i32);
3020  SDValue FVal = DAG.getConstant(0, dl, MVT::i32);
3021  AArch64CC::CondCode CC;
3022  SDValue Value, Overflow;
3023  std::tie(Value, Overflow) = getAArch64XALUOOp(CC, Sel.getValue(0), DAG);
3024  SDValue CCVal = DAG.getConstant(getInvertedCondCode(CC), dl, MVT::i32);
3025  return DAG.getNode(AArch64ISD::CSEL, dl, Op.getValueType(), TVal, FVal,
3026  CCVal, Overflow);
3027  }
3028  // If neither operand is a SELECT_CC, give up.
3029  if (Sel.getOpcode() != ISD::SELECT_CC)
3030  std::swap(Sel, Other);
3031  if (Sel.getOpcode() != ISD::SELECT_CC)
3032  return Op;
3033 
3034  // The folding we want to perform is:
3035  // (xor x, (select_cc a, b, cc, 0, -1) )
3036  // -->
3037  // (csel x, (xor x, -1), cc ...)
3038  //
3039  // The latter will get matched to a CSINV instruction.
3040 
3041  ISD::CondCode CC = cast<CondCodeSDNode>(Sel.getOperand(4))->get();
3042  SDValue LHS = Sel.getOperand(0);
3043  SDValue RHS = Sel.getOperand(1);
3044  SDValue TVal = Sel.getOperand(2);
3045  SDValue FVal = Sel.getOperand(3);
3046 
3047  // FIXME: This could be generalized to non-integer comparisons.
3048  if (LHS.getValueType() != MVT::i32 && LHS.getValueType() != MVT::i64)
3049  return Op;
3050 
3051  ConstantSDNode *CFVal = dyn_cast<ConstantSDNode>(FVal);
3052  ConstantSDNode *CTVal = dyn_cast<ConstantSDNode>(TVal);
3053 
3054  // The values aren't constants, this isn't the pattern we're looking for.
3055  if (!CFVal || !CTVal)
3056  return Op;
3057 
3058  // We can commute the SELECT_CC by inverting the condition. This
3059  // might be needed to make this fit into a CSINV pattern.
3060  if (CTVal->isAllOnesValue() && CFVal->isNullValue()) {
3061  std::swap(TVal, FVal);
3062  std::swap(CTVal, CFVal);
3063  CC = ISD::getSetCCInverse(CC, LHS.getValueType());
3064  }
3065 
3066  // If the constants line up, perform the transform!
3067  if (CTVal->isNullValue() && CFVal->isAllOnesValue()) {
3068  SDValue CCVal;
3069  SDValue Cmp = getAArch64Cmp(LHS, RHS, CC, CCVal, DAG, dl);
3070 
3071  FVal = Other;
3072  TVal = DAG.getNode(ISD::XOR, dl, Other.getValueType(), Other,
3073  DAG.getConstant(-1ULL, dl, Other.getValueType()));
3074 
3075  return DAG.getNode(AArch64ISD::CSEL, dl, Sel.getValueType(), FVal, TVal,
3076  CCVal, Cmp);
3077  }
3078 
3079  return Op;
3080 }
3081 
3082 static SDValue LowerADDC_ADDE_SUBC_SUBE(SDValue Op, SelectionDAG &DAG) {
3083  EVT VT = Op.getValueType();
3084 
3085  // Let legalize expand this if it isn't a legal type yet.
3086  if (!DAG.getTargetLoweringInfo().isTypeLegal(VT))
3087  return SDValue();
3088 
3089  SDVTList VTs = DAG.getVTList(VT, MVT::i32);
3090 
3091  unsigned Opc;
3092  bool ExtraOp = false;
3093  switch (Op.getOpcode()) {
3094  default:
3095  llvm_unreachable("Invalid code");
3096  case ISD::ADDC:
3097  Opc = AArch64ISD::ADDS;
3098  break;
3099  case ISD::SUBC:
3100  Opc = AArch64ISD::SUBS;
3101  break;
3102  case ISD::ADDE:
3103  Opc = AArch64ISD::ADCS;
3104  ExtraOp = true;
3105  break;
3106  case ISD::SUBE:
3107  Opc = AArch64ISD::SBCS;
3108  ExtraOp = true;
3109  break;
3110  }
3111 
3112  if (!ExtraOp)
3113  return DAG.getNode(Opc, SDLoc(Op), VTs, Op.getOperand(0), Op.getOperand(1));
3114  return DAG.getNode(Opc, SDLoc(Op), VTs, Op.getOperand(0), Op.getOperand(1),
3115  Op.getOperand(2));
3116 }
3117 
3118 static SDValue LowerXALUO(SDValue Op, SelectionDAG &DAG) {
3119  // Let legalize expand this if it isn't a legal type yet.
3120  if (!DAG.getTargetLoweringInfo().isTypeLegal(Op.getValueType()))
3121  return SDValue();
3122 
3123  SDLoc dl(Op);
3124  AArch64CC::CondCode CC;
3125  // The actual operation that sets the overflow or carry flag.
3126  SDValue Value, Overflow;
3127  std::tie(Value, Overflow) = getAArch64XALUOOp(CC, Op, DAG);
3128 
3129  // We use 0 and 1 as false and true values.
3130  SDValue TVal = DAG.getConstant(1, dl, MVT::i32);
3131  SDValue FVal = DAG.getConstant(0, dl, MVT::i32);
3132 
3133  // We use an inverted condition, because the conditional select is inverted
3134  // too. This will allow it to be selected to a single instruction:
3135  // CSINC Wd, WZR, WZR, invert(cond).
3136  SDValue CCVal = DAG.getConstant(getInvertedCondCode(CC), dl, MVT::i32);
3137  Overflow = DAG.getNode(AArch64ISD::CSEL, dl, MVT::i32, FVal, TVal,
3138  CCVal, Overflow);
3139 
3140  SDVTList VTs = DAG.getVTList(Op.getValueType(), MVT::i32);
3141  return DAG.getNode(ISD::MERGE_VALUES, dl, VTs, Value, Overflow);
3142 }
3143 
3144 // Prefetch operands are:
3145 // 1: Address to prefetch
3146 // 2: bool isWrite
3147 // 3: int locality (0 = no locality ... 3 = extreme locality)
3148 // 4: bool isDataCache
3149 static SDValue LowerPREFETCH(SDValue Op, SelectionDAG &DAG) {
3150  SDLoc DL(Op);
3151  unsigned IsWrite = cast<ConstantSDNode>(Op.getOperand(2))->getZExtValue();
3152  unsigned Locality = cast<ConstantSDNode>(Op.getOperand(3))->getZExtValue();
3153  unsigned IsData = cast<ConstantSDNode>(Op.getOperand(4))->getZExtValue();
3154 
3155  bool IsStream = !Locality;
3156  // When the locality number is set
3157  if (Locality) {
3158  // The front-end should have filtered out the out-of-range values
3159  assert(Locality <= 3 && "Prefetch locality out-of-range");
3160  // The locality degree is the opposite of the cache speed.
3161  // Put the number the other way around.
3162  // The encoding starts at 0 for level 1
3163  Locality = 3 - Locality;
3164  }
3165 
3166  // built the mask value encoding the expected behavior.
3167  unsigned PrfOp = (IsWrite << 4) | // Load/Store bit
3168  (!IsData << 3) | // IsDataCache bit
3169  (Locality << 1) | // Cache level bits
3170  (unsigned)IsStream; // Stream bit
3171  return DAG.getNode(AArch64ISD::PREFETCH, DL, MVT::Other, Op.getOperand(0),
3172  DAG.getConstant(PrfOp, DL, MVT::i32), Op.getOperand(1));
3173 }
3174 
3175 SDValue AArch64TargetLowering::LowerFP_EXTEND(SDValue Op,
3176  SelectionDAG &DAG) const {
3177  if (Op.getValueType().isScalableVector())
3178  return LowerToPredicatedOp(Op, DAG, AArch64ISD::FP_EXTEND_MERGE_PASSTHRU);
3179 
3180  assert(Op.getValueType() == MVT::f128 && "Unexpected lowering");
3181  return SDValue();
3182 }
3183 
3184 SDValue AArch64TargetLowering::LowerFP_ROUND(SDValue Op,
3185  SelectionDAG &DAG) const {
3186  if (Op.getValueType().isScalableVector())
3187  return LowerToPredicatedOp(Op, DAG, AArch64ISD::FP_ROUND_MERGE_PASSTHRU);
3188 
3189  bool IsStrict = Op->isStrictFPOpcode();
3190  SDValue SrcVal = Op.getOperand(IsStrict ? 1 : 0);
3191  EVT SrcVT = SrcVal.getValueType();
3192 
3193  if (SrcVT != MVT::f128) {
3194  // Expand cases where the input is a vector bigger than NEON.
3195  if (useSVEForFixedLengthVectorVT(SrcVT))
3196  return SDValue();
3197 
3198  // It's legal except when f128 is involved
3199  return Op;
3200  }
3201 
3202  return SDValue();
3203 }
3204 
3205 SDValue AArch64TargetLowering::LowerVectorFP_TO_INT(SDValue Op,
3206  SelectionDAG &DAG) const {
3207  // Warning: We maintain cost tables in AArch64TargetTransformInfo.cpp.
3208  // Any additional optimization in this function should be recorded
3209  // in the cost tables.
3210  EVT InVT = Op.getOperand(0).getValueType();
3211  EVT VT = Op.getValueType();
3212 
3213  if (VT.isScalableVector()) {
3214  unsigned Opcode = Op.getOpcode() == ISD::FP_TO_UINT
3215  ? AArch64ISD::FCVTZU_MERGE_PASSTHRU
3216  : AArch64ISD::FCVTZS_MERGE_PASSTHRU;
3217  return LowerToPredicatedOp(Op, DAG, Opcode);
3218  }
3219 
3220  unsigned NumElts = InVT.getVectorNumElements();
3221 
3222  // f16 conversions are promoted to f32 when full fp16 is not supported.
3223  if (InVT.getVectorElementType() == MVT::f16 &&
3224  !Subtarget->hasFullFP16()) {
3225  MVT NewVT = MVT::getVectorVT(MVT::f32, NumElts);
3226  SDLoc dl(Op);
3227  return DAG.getNode(
3228  Op.getOpcode(), dl, Op.getValueType(),
3229  DAG.getNode(ISD::FP_EXTEND, dl, NewVT, Op.getOperand(0)));
3230  }
3231 
3232  uint64_t VTSize = VT.getFixedSizeInBits();
3233  uint64_t InVTSize = InVT.getFixedSizeInBits();
3234  if (VTSize < InVTSize) {
3235  SDLoc dl(Op);
3236  SDValue Cv =
3237  DAG.getNode(Op.getOpcode(), dl, InVT.changeVectorElementTypeToInteger(),
3238  Op.getOperand(0));
3239  return DAG.getNode(ISD::TRUNCATE, dl, VT, Cv);
3240  }
3241 
3242  if (VTSize > InVTSize) {
3243  SDLoc dl(Op);
3244  MVT ExtVT =
3245  MVT::getVectorVT(MVT::getFloatingPointVT(VT.getScalarSizeInBits()),
3246  VT.getVectorNumElements());
3247  SDValue Ext = DAG.getNode(ISD::FP_EXTEND, dl, ExtVT, Op.getOperand(0));
3248  return DAG.getNode(Op.getOpcode(), dl, VT, Ext);
3249  }
3250 
3251  // Type changing conversions are illegal.
3252  return Op;
3253 }
3254 
3255 SDValue AArch64TargetLowering::LowerFP_TO_INT(SDValue Op,
3256  SelectionDAG &DAG) const {
3257  bool IsStrict = Op->isStrictFPOpcode();
3258  SDValue SrcVal = Op.getOperand(IsStrict ? 1 : 0);
3259 
3260  if (SrcVal.getValueType().isVector())
3261  return LowerVectorFP_TO_INT(Op, DAG);
3262 
3263  // f16 conversions are promoted to f32 when full fp16 is not supported.
3264  if (SrcVal.getValueType() == MVT::f16 && !Subtarget->hasFullFP16()) {
3265  assert(!IsStrict && "Lowering of strict fp16 not yet implemented");
3266  SDLoc dl(Op);
3267  return DAG.getNode(
3268  Op.getOpcode(), dl, Op.getValueType(),
3269  DAG.getNode(ISD::FP_EXTEND, dl, MVT::f32, SrcVal));
3270  }
3271 
3272  if (SrcVal.getValueType() != MVT::f128) {
3273  // It's legal except when f128 is involved
3274  return Op;
3275  }
3276 
3277  return SDValue();
3278 }
3279 
3280 SDValue AArch64TargetLowering::LowerVectorINT_TO_FP(SDValue Op,
3281  SelectionDAG &DAG) const {
3282  // Warning: We maintain cost tables in AArch64TargetTransformInfo.cpp.
3283  // Any additional optimization in this function should be recorded
3284  // in the cost tables.
3285  EVT VT = Op.getValueType();
3286  SDLoc dl(Op);
3287  SDValue In = Op.getOperand(0);
3288  EVT InVT = In.getValueType();
3289  unsigned Opc = Op.getOpcode();
3290  bool IsSigned = Opc == ISD::SINT_TO_FP || Opc == ISD::STRICT_SINT_TO_FP;
3291 
3292  if (VT.isScalableVector()) {
3293  if (InVT.getVectorElementType() == MVT::i1) {
3294  // We can't directly extend an SVE predicate; extend it first.
3295  unsigned CastOpc = IsSigned ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND;
3296  EVT CastVT = getPromotedVTForPredicate(InVT);
3297  In = DAG.getNode(CastOpc, dl, CastVT, In);
3298  return DAG.getNode(Opc, dl, VT, In);
3299  }
3300 
3301  unsigned Opcode = IsSigned ? AArch64ISD::SINT_TO_FP_MERGE_PASSTHRU
3302  : AArch64ISD::UINT_TO_FP_MERGE_PASSTHRU;
3303  return LowerToPredicatedOp(Op, DAG, Opcode);
3304  }
3305 
3306  uint64_t VTSize = VT.getFixedSizeInBits();
3307  uint64_t InVTSize = InVT.getFixedSizeInBits();
3308  if (VTSize < InVTSize) {
3309  MVT CastVT =
3310  MVT::getVectorVT(MVT::getFloatingPointVT(InVT.getScalarSizeInBits()),
3311  InVT.getVectorNumElements());
3312  In = DAG.getNode(Opc, dl, CastVT, In);
3313  return DAG.getNode(ISD::FP_ROUND, dl, VT, In, DAG.getIntPtrConstant(0, dl));
3314  }
3315 
3316  if (VTSize > InVTSize) {
3317  unsigned CastOpc = IsSigned ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND;
3318  EVT CastVT = VT.changeVectorElementTypeToInteger();
3319  In = DAG.getNode(CastOpc, dl, CastVT, In);
3320  return DAG.getNode(Opc, dl, VT, In);
3321  }
3322 
3323  return Op;
3324 }
3325 
3326 SDValue AArch64TargetLowering::LowerINT_TO_FP(SDValue Op,
3327  SelectionDAG &DAG) const {
3328  if (Op.getValueType().isVector())
3329  return LowerVectorINT_TO_FP(Op, DAG);
3330 
3331  bool IsStrict = Op->isStrictFPOpcode();
3332  SDValue SrcVal = Op.getOperand(IsStrict ? 1 : 0);
3333 
3334  // f16 conversions are promoted to f32 when full fp16 is not supported.
3335  if (Op.getValueType() == MVT::f16 &&
3336  !Subtarget->hasFullFP16()) {
3337  assert(!IsStrict && "Lowering of strict fp16 not yet implemented");
3338  SDLoc dl(Op);
3339  return DAG.getNode(
3340  ISD::FP_ROUND, dl, MVT::f16,
3341  DAG.getNode(Op.getOpcode(), dl, MVT::f32, SrcVal),
3342  DAG.getIntPtrConstant(0, dl));
3343  }
3344 
3345  // i128 conversions are libcalls.
3346  if (SrcVal.getValueType() == MVT::i128)
3347  return SDValue();
3348 
3349  // Other conversions are legal, unless it's to the completely software-based
3350  // fp128.
3351  if (Op.getValueType() != MVT::f128)
3352  return Op;
3353  return SDValue();
3354 }
3355 
3356 SDValue AArch64TargetLowering::LowerFSINCOS(SDValue Op,
3357  SelectionDAG &DAG) const {
3358  // For iOS, we want to call an alternative entry point: __sincos_stret,
3359  // which returns the values in two S / D registers.
3360  SDLoc dl(Op);
3361  SDValue Arg = Op.getOperand(0);
3362  EVT ArgVT = Arg.getValueType();
3363  Type *ArgTy = ArgVT.getTypeForEVT(*DAG.getContext());
3364 
3365  ArgListTy Args;
3366  ArgListEntry Entry;
3367 
3368  Entry.Node = Arg;
3369  Entry.Ty = ArgTy;
3370  Entry.IsSExt = false;
3371  Entry.IsZExt = false;
3372  Args.push_back(Entry);
3373 
3374  RTLIB::Libcall LC = ArgVT == MVT::f64 ? RTLIB::SINCOS_STRET_F64
3375  : RTLIB::SINCOS_STRET_F32;
3376  const char *LibcallName = getLibcallName(LC);
3377  SDValue Callee =
3378  DAG.getExternalSymbol(LibcallName, getPointerTy(DAG.getDataLayout()));
3379 
3380  StructType *RetTy = StructType::get(ArgTy, ArgTy);
3381  TargetLowering::CallLoweringInfo CLI(DAG);
3382  CLI.setDebugLoc(dl)
3383  .setChain(DAG.getEntryNode())
3384  .setLibCallee(CallingConv::Fast, RetTy, Callee, std::move(Args));
3385 
3386  std::pair<SDValue, SDValue> CallResult = LowerCallTo(CLI);
3387  return CallResult.first;
3388 }
3389 
3390 static SDValue LowerBITCAST(SDValue Op, SelectionDAG &DAG) {
3391  EVT OpVT = Op.getValueType();
3392  if (OpVT != MVT::f16 && OpVT != MVT::bf16)
3393  return SDValue();
3394 
3395  assert(Op.getOperand(0).getValueType() == MVT::i16);
3396  SDLoc DL(Op);
3397 
3398  Op = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i32, Op.getOperand(0));
3399  Op = DAG.getNode(ISD::BITCAST, DL, MVT::f32, Op);
3400  return SDValue(
3401  DAG.getMachineNode(TargetOpcode::EXTRACT_SUBREG, DL, OpVT, Op,
3402  DAG.getTargetConstant(AArch64::hsub, DL, MVT::i32)),
3403  0);
3404 }
3405 
3406 static EVT getExtensionTo64Bits(const EVT &OrigVT) {
3407  if (OrigVT.getSizeInBits() >= 64)
3408  return OrigVT;
3409 
3410  assert(OrigVT.isSimple() && "Expecting a simple value type");
3411 
3412  MVT::SimpleValueType OrigSimpleTy = OrigVT.getSimpleVT().SimpleTy;
3413  switch (OrigSimpleTy) {
3414  default: llvm_unreachable("Unexpected Vector Type");
3415  case MVT::v2i8:
3416  case MVT::v2i16:
3417  return MVT::v2i32;
3418  case MVT::v4i8:
3419  return MVT::v4i16;
3420  }
3421&