LLVM  14.0.0git
AArch64LegalizerInfo.cpp
Go to the documentation of this file.
1 //===- AArch64LegalizerInfo.cpp ----------------------------------*- C++ -*-==//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 /// \file
9 /// This file implements the targeting of the Machinelegalizer class for
10 /// AArch64.
11 /// \todo This should be generated by TableGen.
12 //===----------------------------------------------------------------------===//
13 
14 #include "AArch64LegalizerInfo.h"
16 #include "AArch64Subtarget.h"
26 #include "llvm/IR/DerivedTypes.h"
27 #include "llvm/IR/Intrinsics.h"
28 #include "llvm/IR/IntrinsicsAArch64.h"
29 #include "llvm/IR/Type.h"
31 #include <initializer_list>
32 
33 #define DEBUG_TYPE "aarch64-legalinfo"
34 
35 using namespace llvm;
36 using namespace LegalizeActions;
37 using namespace LegalizeMutations;
38 using namespace LegalityPredicates;
39 using namespace MIPatternMatch;
40 
42  : ST(&ST) {
43  using namespace TargetOpcode;
44  const LLT p0 = LLT::pointer(0, 64);
45  const LLT s1 = LLT::scalar(1);
46  const LLT s8 = LLT::scalar(8);
47  const LLT s16 = LLT::scalar(16);
48  const LLT s32 = LLT::scalar(32);
49  const LLT s64 = LLT::scalar(64);
50  const LLT s128 = LLT::scalar(128);
51  const LLT v16s8 = LLT::fixed_vector(16, 8);
52  const LLT v8s8 = LLT::fixed_vector(8, 8);
53  const LLT v4s8 = LLT::fixed_vector(4, 8);
54  const LLT v8s16 = LLT::fixed_vector(8, 16);
55  const LLT v4s16 = LLT::fixed_vector(4, 16);
56  const LLT v2s16 = LLT::fixed_vector(2, 16);
57  const LLT v2s32 = LLT::fixed_vector(2, 32);
58  const LLT v4s32 = LLT::fixed_vector(4, 32);
59  const LLT v2s64 = LLT::fixed_vector(2, 64);
60  const LLT v2p0 = LLT::fixed_vector(2, p0);
61 
62  std::initializer_list<LLT> PackedVectorAllTypeList = {/* Begin 128bit types */
63  v16s8, v8s16, v4s32,
64  v2s64, v2p0,
65  /* End 128bit types */
66  /* Begin 64bit types */
67  v8s8, v4s16, v2s32};
68 
69  const TargetMachine &TM = ST.getTargetLowering()->getTargetMachine();
70 
71  // FIXME: support subtargets which have neon/fp-armv8 disabled.
72  if (!ST.hasNEON() || !ST.hasFPARMv8()) {
74  return;
75  }
76 
77  // Some instructions only support s16 if the subtarget has full 16-bit FP
78  // support.
79  const bool HasFP16 = ST.hasFullFP16();
80  const LLT &MinFPScalar = HasFP16 ? s16 : s32;
81 
82  getActionDefinitionsBuilder({G_IMPLICIT_DEF, G_FREEZE})
83  .legalFor({p0, s1, s8, s16, s32, s64})
84  .legalFor(PackedVectorAllTypeList)
86  .clampScalar(0, s8, s64)
88  [=](const LegalityQuery &Query) {
89  return Query.Types[0].isVector() &&
90  (Query.Types[0].getElementType() != s64 ||
91  Query.Types[0].getNumElements() != 2);
92  },
93  [=](const LegalityQuery &Query) {
94  LLT EltTy = Query.Types[0].getElementType();
95  if (EltTy == s64)
96  return std::make_pair(0, LLT::fixed_vector(2, 64));
97  return std::make_pair(0, EltTy);
98  });
99 
101  .legalFor({p0, s16, s32, s64})
102  .legalFor(PackedVectorAllTypeList)
104  .clampScalar(0, s16, s64)
105  // Maximum: sN * k = 128
106  .clampMaxNumElements(0, s8, 16)
107  .clampMaxNumElements(0, s16, 8)
108  .clampMaxNumElements(0, s32, 4)
109  .clampMaxNumElements(0, s64, 2)
110  .clampMaxNumElements(0, p0, 2);
111 
113  .legalFor({s32, s64, v4s32, v2s32, v2s64})
114  .widenScalarToNextPow2(0)
115  .clampScalar(0, s32, s64);
116 
117  getActionDefinitionsBuilder({G_ADD, G_SUB, G_MUL, G_AND, G_OR, G_XOR})
118  .legalFor({s32, s64, v2s32, v4s32, v4s16, v8s16, v16s8, v8s8})
119  .scalarizeIf(
120  [=](const LegalityQuery &Query) {
121  return Query.Opcode == G_MUL && Query.Types[0] == v2s64;
122  },
123  0)
124  .legalFor({v2s64})
125  .widenScalarToNextPow2(0)
126  .clampScalar(0, s32, s64)
127  .clampNumElements(0, v2s32, v4s32)
128  .clampNumElements(0, v2s64, v2s64)
130 
131  getActionDefinitionsBuilder({G_SHL, G_ASHR, G_LSHR})
132  .customIf([=](const LegalityQuery &Query) {
133  const auto &SrcTy = Query.Types[0];
134  const auto &AmtTy = Query.Types[1];
135  return !SrcTy.isVector() && SrcTy.getSizeInBits() == 32 &&
136  AmtTy.getSizeInBits() == 32;
137  })
138  .legalFor({
139  {s32, s32},
140  {s32, s64},
141  {s64, s64},
142  {v8s8, v8s8},
143  {v16s8, v16s8},
144  {v4s16, v4s16},
145  {v8s16, v8s16},
146  {v2s32, v2s32},
147  {v4s32, v4s32},
148  {v2s64, v2s64},
149  })
150  .widenScalarToNextPow2(0)
151  .clampScalar(1, s32, s64)
152  .clampScalar(0, s32, s64)
153  .clampNumElements(0, v2s32, v4s32)
154  .clampNumElements(0, v2s64, v2s64)
156  .minScalarSameAs(1, 0);
157 
158  getActionDefinitionsBuilder(G_PTR_ADD)
159  .legalFor({{p0, s64}, {v2p0, v2s64}})
160  .clampScalar(1, s64, s64);
161 
162  getActionDefinitionsBuilder(G_PTRMASK).legalFor({{p0, s64}});
163 
164  getActionDefinitionsBuilder({G_SDIV, G_UDIV})
165  .legalFor({s32, s64})
166  .libcallFor({s128})
167  .clampScalar(0, s32, s64)
169  .scalarize(0);
170 
171  getActionDefinitionsBuilder({G_SREM, G_UREM, G_SDIVREM, G_UDIVREM})
172  .lowerFor({s1, s8, s16, s32, s64});
173 
174  getActionDefinitionsBuilder({G_SMULO, G_UMULO})
175  .widenScalarToNextPow2(0, /*Min = */ 32)
176  .clampScalar(0, s32, s64)
177  .lowerIf(typeIs(1, s1));
178 
179  getActionDefinitionsBuilder({G_SMULH, G_UMULH}).legalFor({s32, s64});
180 
181  getActionDefinitionsBuilder({G_SMIN, G_SMAX, G_UMIN, G_UMAX})
182  .legalFor({v8s8, v16s8, v4s16, v8s16, v2s32, v4s32})
183  .clampNumElements(0, v8s8, v16s8)
184  .clampNumElements(0, v4s16, v8s16)
185  .clampNumElements(0, v2s32, v4s32)
186  // FIXME: This sholdn't be needed as v2s64 types are going to
187  // be expanded anyway, but G_ICMP doesn't support splitting vectors yet
188  .clampNumElements(0, v2s64, v2s64)
189  .lower();
190 
192  {G_SADDE, G_SSUBE, G_UADDE, G_USUBE, G_SADDO, G_SSUBO, G_UADDO, G_USUBO})
193  .legalFor({{s32, s1}, {s64, s1}})
194  .clampScalar(0, s32, s64)
196 
197  getActionDefinitionsBuilder({G_FADD, G_FSUB, G_FMUL, G_FDIV, G_FNEG})
198  .legalFor({MinFPScalar, s32, s64, v2s64, v4s32, v2s32})
199  .clampScalar(0, MinFPScalar, s64)
200  .clampNumElements(0, v2s32, v4s32)
201  .clampNumElements(0, v2s64, v2s64);
202 
203  getActionDefinitionsBuilder(G_FREM).libcallFor({s32, s64});
204 
205  getActionDefinitionsBuilder({G_FCEIL, G_FABS, G_FSQRT, G_FFLOOR, G_FRINT,
206  G_FMA, G_INTRINSIC_TRUNC, G_INTRINSIC_ROUND,
207  G_FNEARBYINT, G_INTRINSIC_LRINT})
208  // If we don't have full FP16 support, then scalarize the elements of
209  // vectors containing fp16 types.
210  .fewerElementsIf(
211  [=, &ST](const LegalityQuery &Query) {
212  const auto &Ty = Query.Types[0];
213  return Ty.isVector() && Ty.getElementType() == s16 &&
214  !ST.hasFullFP16();
215  },
216  [=](const LegalityQuery &Query) { return std::make_pair(0, s16); })
217  // If we don't have full FP16 support, then widen s16 to s32 if we
218  // encounter it.
219  .widenScalarIf(
220  [=, &ST](const LegalityQuery &Query) {
221  return Query.Types[0] == s16 && !ST.hasFullFP16();
222  },
223  [=](const LegalityQuery &Query) { return std::make_pair(0, s32); })
224  .legalFor({s16, s32, s64, v2s32, v4s32, v2s64, v2s16, v4s16, v8s16});
225 
227  {G_FCOS, G_FSIN, G_FLOG10, G_FLOG, G_FLOG2, G_FEXP, G_FEXP2, G_FPOW})
228  // We need a call for these, so we always need to scalarize.
229  .scalarize(0)
230  // Regardless of FP16 support, widen 16-bit elements to 32-bits.
231  .minScalar(0, s32)
232  .libcallFor({s32, s64, v2s32, v4s32, v2s64});
233 
235  .legalIf(all(typeInSet(0, {s32, s64, p0}),
236  typeInSet(1, {s1, s8, s16, s32}), smallerThan(1, 0)))
238  .clampScalar(0, s32, s64)
240  .minScalar(1, s8)
241  .maxScalarIf(typeInSet(0, {s32}), 1, s16)
242  .maxScalarIf(typeInSet(0, {s64, p0}), 1, s32);
243 
244  getActionDefinitionsBuilder(G_EXTRACT)
245  .legalIf(all(typeInSet(0, {s16, s32, s64, p0}),
246  typeInSet(1, {s32, s64, s128, p0}), smallerThan(0, 1)))
248  .clampScalar(1, s32, s128)
250  .minScalar(0, s16)
251  .maxScalarIf(typeInSet(1, {s32}), 0, s16)
252  .maxScalarIf(typeInSet(1, {s64, p0}), 0, s32)
253  .maxScalarIf(typeInSet(1, {s128}), 0, s64);
254 
255  getActionDefinitionsBuilder({G_SEXTLOAD, G_ZEXTLOAD})
257  .legalForTypesWithMemDesc({{s32, p0, s8, 8},
258  {s32, p0, s16, 8},
259  {s32, p0, s32, 8},
260  {s64, p0, s8, 2},
261  {s64, p0, s16, 2},
262  {s64, p0, s32, 4},
263  {s64, p0, s64, 8},
264  {p0, p0, s64, 8},
265  {v2s32, p0, s64, 8}})
266  .widenScalarToNextPow2(0)
267  .clampScalar(0, s32, s64)
268  // TODO: We could support sum-of-pow2's but the lowering code doesn't know
269  // how to do that yet.
271  // Lower anything left over into G_*EXT and G_LOAD
272  .lower();
273 
274  auto IsPtrVecPred = [=](const LegalityQuery &Query) {
275  const LLT &ValTy = Query.Types[0];
276  if (!ValTy.isVector())
277  return false;
278  const LLT EltTy = ValTy.getElementType();
279  return EltTy.isPointer() && EltTy.getAddressSpace() == 0;
280  };
281 
283  .customIf([=](const LegalityQuery &Query) {
284  return Query.Types[0] == s128 &&
285  Query.MMODescrs[0].Ordering != AtomicOrdering::NotAtomic;
286  })
287  .legalForTypesWithMemDesc({{s8, p0, s8, 8},
288  {s16, p0, s16, 8},
289  {s32, p0, s32, 8},
290  {s64, p0, s64, 8},
291  {p0, p0, s64, 8},
292  {s128, p0, s128, 8},
293  {v8s8, p0, s64, 8},
294  {v16s8, p0, s128, 8},
295  {v4s16, p0, s64, 8},
296  {v8s16, p0, s128, 8},
297  {v2s32, p0, s64, 8},
298  {v4s32, p0, s128, 8},
299  {v2s64, p0, s128, 8}})
300  // These extends are also legal
301  .legalForTypesWithMemDesc({{s32, p0, s8, 8}, {s32, p0, s16, 8}})
302  .widenScalarToNextPow2(0, /* MinSize = */8)
304  .clampScalar(0, s8, s64)
305  .narrowScalarIf([=](const LegalityQuery &Query) {
306  // Clamp extending load results to 32-bits.
307  return Query.Types[0].isScalar() &&
308  Query.Types[0] != Query.MMODescrs[0].MemoryTy &&
309  Query.Types[0].getSizeInBits() > 32;
310  },
311  changeTo(0, s32))
312  // Lower any any-extending loads left into G_ANYEXT and G_LOAD
313  .lowerIf([=](const LegalityQuery &Query) {
314  return Query.Types[0] != Query.MMODescrs[0].MemoryTy;
315  })
316  .clampMaxNumElements(0, s8, 16)
317  .clampMaxNumElements(0, s16, 8)
318  .clampMaxNumElements(0, s32, 4)
319  .clampMaxNumElements(0, s64, 2)
320  .clampMaxNumElements(0, p0, 2)
321  .customIf(IsPtrVecPred)
322  .scalarizeIf(typeIs(0, v2s16), 0);
323 
325  .customIf([=](const LegalityQuery &Query) {
326  return Query.Types[0] == s128 &&
327  Query.MMODescrs[0].Ordering != AtomicOrdering::NotAtomic;
328  })
329  .legalForTypesWithMemDesc({{s8, p0, s8, 8},
330  {s16, p0, s8, 8}, // truncstorei8 from s16
331  {s32, p0, s8, 8}, // truncstorei8 from s32
332  {s64, p0, s8, 8}, // truncstorei8 from s64
333  {s16, p0, s16, 8},
334  {s32, p0, s16, 8}, // truncstorei16 from s32
335  {s64, p0, s16, 8}, // truncstorei16 from s64
336  {s32, p0, s8, 8},
337  {s32, p0, s16, 8},
338  {s32, p0, s32, 8},
339  {s64, p0, s64, 8},
340  {s64, p0, s32, 8}, // truncstorei32 from s64
341  {p0, p0, s64, 8},
342  {s128, p0, s128, 8},
343  {v16s8, p0, s128, 8},
344  {v8s8, p0, s64, 8},
345  {v4s16, p0, s64, 8},
346  {v8s16, p0, s128, 8},
347  {v2s32, p0, s64, 8},
348  {v4s32, p0, s128, 8},
349  {v2s64, p0, s128, 8}})
350  .clampScalar(0, s8, s64)
351  .lowerIf([=](const LegalityQuery &Query) {
352  return Query.Types[0].isScalar() &&
353  Query.Types[0] != Query.MMODescrs[0].MemoryTy;
354  })
355  // Maximum: sN * k = 128
356  .clampMaxNumElements(0, s8, 16)
357  .clampMaxNumElements(0, s16, 8)
358  .clampMaxNumElements(0, s32, 4)
359  .clampMaxNumElements(0, s64, 2)
360  .clampMaxNumElements(0, p0, 2)
362  .customIf(IsPtrVecPred)
363  .scalarizeIf(typeIs(0, v2s16), 0);
364 
365  // Constants
366  getActionDefinitionsBuilder(G_CONSTANT)
367  .legalFor({p0, s8, s16, s32, s64})
368  .widenScalarToNextPow2(0)
369  .clampScalar(0, s8, s64);
370  getActionDefinitionsBuilder(G_FCONSTANT)
371  .legalIf([=](const LegalityQuery &Query) {
372  const auto &Ty = Query.Types[0];
373  if (HasFP16 && Ty == s16)
374  return true;
375  return Ty == s32 || Ty == s64 || Ty == s128;
376  })
377  .clampScalar(0, MinFPScalar, s128);
378 
379  getActionDefinitionsBuilder({G_ICMP, G_FCMP})
380  .legalFor({{s32, s32},
381  {s32, s64},
382  {s32, p0},
383  {v4s32, v4s32},
384  {v2s32, v2s32},
385  {v2s64, v2s64},
386  {v2s64, v2p0},
387  {v4s16, v4s16},
388  {v8s16, v8s16},
389  {v8s8, v8s8},
390  {v16s8, v16s8}})
392  .clampScalar(1, s32, s64)
393  .clampScalar(0, s32, s32)
394  .minScalarEltSameAsIf(
395  [=](const LegalityQuery &Query) {
396  const LLT &Ty = Query.Types[0];
397  const LLT &SrcTy = Query.Types[1];
398  return Ty.isVector() && !SrcTy.getElementType().isPointer() &&
399  Ty.getElementType() != SrcTy.getElementType();
400  },
401  0, 1)
402  .minScalarOrEltIf(
403  [=](const LegalityQuery &Query) { return Query.Types[1] == v2s16; },
404  1, s32)
405  .minScalarOrEltIf(
406  [=](const LegalityQuery &Query) { return Query.Types[1] == v2p0; }, 0,
407  s64)
408  .clampNumElements(0, v2s32, v4s32);
409 
410  // Extensions
411  auto ExtLegalFunc = [=](const LegalityQuery &Query) {
412  unsigned DstSize = Query.Types[0].getSizeInBits();
413 
414  if (DstSize == 128 && !Query.Types[0].isVector())
415  return false; // Extending to a scalar s128 needs narrowing.
416 
417  // Make sure that we have something that will fit in a register, and
418  // make sure it's a power of 2.
419  if (DstSize < 8 || DstSize > 128 || !isPowerOf2_32(DstSize))
420  return false;
421 
422  const LLT &SrcTy = Query.Types[1];
423 
424  // Special case for s1.
425  if (SrcTy == s1)
426  return true;
427 
428  // Make sure we fit in a register otherwise. Don't bother checking that
429  // the source type is below 128 bits. We shouldn't be allowing anything
430  // through which is wider than the destination in the first place.
431  unsigned SrcSize = SrcTy.getSizeInBits();
432  if (SrcSize < 8 || !isPowerOf2_32(SrcSize))
433  return false;
434 
435  return true;
436  };
437  getActionDefinitionsBuilder({G_ZEXT, G_SEXT, G_ANYEXT})
438  .legalIf(ExtLegalFunc)
439  .clampScalar(0, s64, s64); // Just for s128, others are handled above.
440 
443  [=](const LegalityQuery &Query) { return Query.Types[0].isVector(); },
444  0, s8)
445  .customIf([=](const LegalityQuery &Query) {
446  LLT DstTy = Query.Types[0];
447  LLT SrcTy = Query.Types[1];
448  return DstTy == v8s8 && SrcTy.getSizeInBits() > 128;
449  })
450  .alwaysLegal();
451 
452  getActionDefinitionsBuilder(G_SEXT_INREG).legalFor({s32, s64}).lower();
453 
454  // FP conversions
455  getActionDefinitionsBuilder(G_FPTRUNC)
456  .legalFor(
457  {{s16, s32}, {s16, s64}, {s32, s64}, {v4s16, v4s32}, {v2s32, v2s64}})
458  .clampMaxNumElements(0, s32, 2);
460  .legalFor(
461  {{s32, s16}, {s64, s16}, {s64, s32}, {v4s32, v4s16}, {v2s64, v2s32}})
462  .clampMaxNumElements(0, s64, 2);
463 
464  // Conversions
465  getActionDefinitionsBuilder({G_FPTOSI, G_FPTOUI})
466  .legalForCartesianProduct({s32, s64, v2s64, v4s32, v2s32})
467  .widenScalarToNextPow2(0)
468  .clampScalar(0, s32, s64)
470  .clampScalar(1, s32, s64);
471 
472  getActionDefinitionsBuilder({G_SITOFP, G_UITOFP})
473  .legalForCartesianProduct({s32, s64, v2s64, v4s32, v2s32})
474  .clampScalar(1, s32, s64)
475  .minScalarSameAs(1, 0)
476  .clampScalar(0, s32, s64)
478 
479  // Control-flow
480  getActionDefinitionsBuilder(G_BRCOND).legalFor({s1, s8, s16, s32});
481  getActionDefinitionsBuilder(G_BRINDIRECT).legalFor({p0});
482 
484  .legalFor({{s32, s1}, {s64, s1}, {p0, s1}})
485  .widenScalarToNextPow2(0)
486  .clampScalar(0, s32, s64)
487  .minScalarEltSameAsIf(all(isVector(0), isVector(1)), 1, 0)
488  .lowerIf(isVector(0));
489 
490  // Pointer-handling
491  getActionDefinitionsBuilder(G_FRAME_INDEX).legalFor({p0});
492 
493  if (TM.getCodeModel() == CodeModel::Small)
494  getActionDefinitionsBuilder(G_GLOBAL_VALUE).custom();
495  else
496  getActionDefinitionsBuilder(G_GLOBAL_VALUE).legalFor({p0});
497 
498  getActionDefinitionsBuilder(G_PTRTOINT)
499  .legalForCartesianProduct({s1, s8, s16, s32, s64}, {p0})
500  .legalFor({{v2s64, v2p0}})
501  .maxScalar(0, s64)
502  .widenScalarToNextPow2(0, /*Min*/ 8);
503 
504  getActionDefinitionsBuilder(G_INTTOPTR)
505  .unsupportedIf([&](const LegalityQuery &Query) {
506  return Query.Types[0].getSizeInBits() != Query.Types[1].getSizeInBits();
507  })
508  .legalFor({{p0, s64}, {v2p0, v2s64}});
509 
510  // Casts for 32 and 64-bit width type are just copies.
511  // Same for 128-bit width type, except they are on the FPR bank.
512  getActionDefinitionsBuilder(G_BITCAST)
513  // FIXME: This is wrong since G_BITCAST is not allowed to change the
514  // number of bits but it's what the previous code described and fixing
515  // it breaks tests.
516  .legalForCartesianProduct({s1, s8, s16, s32, s64, s128, v16s8, v8s8, v4s8,
517  v8s16, v4s16, v2s16, v4s32, v2s32, v2s64,
518  v2p0});
519 
520  getActionDefinitionsBuilder(G_VASTART).legalFor({p0});
521 
522  // va_list must be a pointer, but most sized types are pretty easy to handle
523  // as the destination.
525  .customForCartesianProduct({s8, s16, s32, s64, p0}, {p0})
526  .clampScalar(0, s8, s64)
527  .widenScalarToNextPow2(0, /*Min*/ 8);
528 
529  getActionDefinitionsBuilder(G_ATOMIC_CMPXCHG_WITH_SUCCESS)
530  .lowerIf(
531  all(typeInSet(0, {s8, s16, s32, s64, s128}), typeIs(1, s1), typeIs(2, p0)));
532 
533  getActionDefinitionsBuilder(G_ATOMIC_CMPXCHG)
534  .customIf([](const LegalityQuery &Query) {
535  return Query.Types[0].getSizeInBits() == 128;
536  })
537  .clampScalar(0, s32, s64)
538  .legalIf(all(typeInSet(0, {s32, s64}), typeIs(1, p0)));
539 
541  {G_ATOMICRMW_XCHG, G_ATOMICRMW_ADD, G_ATOMICRMW_SUB, G_ATOMICRMW_AND,
542  G_ATOMICRMW_OR, G_ATOMICRMW_XOR, G_ATOMICRMW_MIN, G_ATOMICRMW_MAX,
543  G_ATOMICRMW_UMIN, G_ATOMICRMW_UMAX})
544  .clampScalar(0, s32, s64)
545  .legalIf(all(typeInSet(0, {s32, s64}), typeIs(1, p0)));
546 
547  getActionDefinitionsBuilder(G_BLOCK_ADDR).legalFor({p0});
548 
549  // Merge/Unmerge
550  for (unsigned Op : {G_MERGE_VALUES, G_UNMERGE_VALUES}) {
551  unsigned BigTyIdx = Op == G_MERGE_VALUES ? 0 : 1;
552  unsigned LitTyIdx = Op == G_MERGE_VALUES ? 1 : 0;
554  .widenScalarToNextPow2(LitTyIdx, 8)
555  .widenScalarToNextPow2(BigTyIdx, 32)
556  .clampScalar(LitTyIdx, s8, s64)
557  .clampScalar(BigTyIdx, s32, s128)
558  .legalIf([=](const LegalityQuery &Q) {
559  switch (Q.Types[BigTyIdx].getSizeInBits()) {
560  case 32:
561  case 64:
562  case 128:
563  break;
564  default:
565  return false;
566  }
567  switch (Q.Types[LitTyIdx].getSizeInBits()) {
568  case 8:
569  case 16:
570  case 32:
571  case 64:
572  return true;
573  default:
574  return false;
575  }
576  });
577  }
578 
579  getActionDefinitionsBuilder(G_EXTRACT_VECTOR_ELT)
580  .unsupportedIf([=](const LegalityQuery &Query) {
581  const LLT &EltTy = Query.Types[1].getElementType();
582  return Query.Types[0] != EltTy;
583  })
584  .minScalar(2, s64)
585  .legalIf([=](const LegalityQuery &Query) {
586  const LLT &VecTy = Query.Types[1];
587  return VecTy == v2s16 || VecTy == v4s16 || VecTy == v8s16 ||
588  VecTy == v4s32 || VecTy == v2s64 || VecTy == v2s32 ||
589  VecTy == v8s8 || VecTy == v16s8 || VecTy == v2s32 ||
590  VecTy == v2p0;
591  })
592  .minScalarOrEltIf(
593  [=](const LegalityQuery &Query) {
594  // We want to promote to <M x s1> to <M x s64> if that wouldn't
595  // cause the total vec size to be > 128b.
596  return Query.Types[1].getNumElements() <= 2;
597  },
598  0, s64)
599  .minScalarOrEltIf(
600  [=](const LegalityQuery &Query) {
601  return Query.Types[1].getNumElements() <= 4;
602  },
603  0, s32)
604  .minScalarOrEltIf(
605  [=](const LegalityQuery &Query) {
606  return Query.Types[1].getNumElements() <= 8;
607  },
608  0, s16)
609  .minScalarOrEltIf(
610  [=](const LegalityQuery &Query) {
611  return Query.Types[1].getNumElements() <= 16;
612  },
613  0, s8)
614  .minScalarOrElt(0, s8) // Worst case, we need at least s8.
615  .clampMaxNumElements(1, s64, 2)
616  .clampMaxNumElements(1, s32, 4)
617  .clampMaxNumElements(1, s16, 8)
618  .clampMaxNumElements(1, p0, 2);
619 
620  getActionDefinitionsBuilder(G_INSERT_VECTOR_ELT)
621  .legalIf(typeInSet(0, {v8s16, v2s32, v4s32, v2s64}));
622 
623  getActionDefinitionsBuilder(G_BUILD_VECTOR)
624  .legalFor({{v8s8, s8},
625  {v16s8, s8},
626  {v2s16, s16},
627  {v4s16, s16},
628  {v8s16, s16},
629  {v2s32, s32},
630  {v4s32, s32},
631  {v2p0, p0},
632  {v2s64, s64}})
633  .clampNumElements(0, v4s32, v4s32)
634  .clampNumElements(0, v2s64, v2s64)
635  .minScalarSameAs(1, 0);
636 
637  getActionDefinitionsBuilder(G_BUILD_VECTOR_TRUNC).lower();
638 
641  {s32, s64, v8s8, v16s8, v4s16, v8s16, v2s32, v4s32})
642  .scalarize(1);
643  getActionDefinitionsBuilder(G_CTLZ_ZERO_UNDEF).lower();
644 
645  // TODO: Custom lowering for v2s32, v4s32, v2s64.
646  getActionDefinitionsBuilder(G_BITREVERSE)
647  .legalFor({s32, s64, v8s8, v16s8})
648  .widenScalarToNextPow2(0, /*Min = */ 32)
649  .clampScalar(0, s32, s64);
650 
651  getActionDefinitionsBuilder(G_CTTZ_ZERO_UNDEF).lower();
652 
653  // TODO: Handle vector types.
655  .clampScalar(0, s32, s64)
656  .scalarSameSizeAs(1, 0)
657  .customFor({s32, s64});
658 
659  getActionDefinitionsBuilder(G_SHUFFLE_VECTOR)
660  .legalIf([=](const LegalityQuery &Query) {
661  const LLT &DstTy = Query.Types[0];
662  const LLT &SrcTy = Query.Types[1];
663  // For now just support the TBL2 variant which needs the source vectors
664  // to be the same size as the dest.
665  if (DstTy != SrcTy)
666  return false;
667  for (auto &Ty : {v2s32, v4s32, v2s64, v2p0, v16s8, v8s16}) {
668  if (DstTy == Ty)
669  return true;
670  }
671  return false;
672  })
673  // G_SHUFFLE_VECTOR can have scalar sources (from 1 x s vectors), we
674  // just want those lowered into G_BUILD_VECTOR
675  .lowerIf([=](const LegalityQuery &Query) {
676  return !Query.Types[1].isVector();
677  })
679  .clampNumElements(0, v4s32, v4s32)
680  .clampNumElements(0, v2s64, v2s64);
681 
682  getActionDefinitionsBuilder(G_CONCAT_VECTORS)
683  .legalFor({{v4s32, v2s32}, {v8s16, v4s16}, {v16s8, v8s8}});
684 
685  getActionDefinitionsBuilder(G_JUMP_TABLE).legalFor({{p0}, {s64}});
686 
687  getActionDefinitionsBuilder(G_BRJT).legalIf([=](const LegalityQuery &Query) {
688  return Query.Types[0] == p0 && Query.Types[1] == s64;
689  });
690 
691  getActionDefinitionsBuilder(G_DYN_STACKALLOC).lower();
692 
693  getActionDefinitionsBuilder({G_BZERO, G_MEMCPY, G_MEMMOVE, G_MEMSET})
694  .libcall();
695 
696  // FIXME: Legal types are only legal with NEON.
698  .lowerIf(isScalar(0))
699  .legalFor(PackedVectorAllTypeList);
700 
701  getActionDefinitionsBuilder(G_VECREDUCE_FADD)
702  // We only have FADDP to do reduction-like operations. Lower the rest.
703  .legalFor({{s32, v2s32}, {s64, v2s64}})
704  .clampMaxNumElements(1, s64, 2)
705  .clampMaxNumElements(1, s32, 2)
706  .lower();
707 
708  getActionDefinitionsBuilder(G_VECREDUCE_ADD)
709  .legalFor(
710  {{s8, v16s8}, {s16, v8s16}, {s32, v4s32}, {s32, v2s32}, {s64, v2s64}})
711  .clampMaxNumElements(1, s64, 2)
712  .clampMaxNumElements(1, s32, 4)
713  .lower();
714 
715  getActionDefinitionsBuilder(G_VECREDUCE_OR)
716  // Try to break down into smaller vectors as long as they're at least 64
717  // bits. This lets us use vector operations for some parts of the
718  // reduction.
720  [=](const LegalityQuery &Q) {
721  LLT SrcTy = Q.Types[1];
722  if (SrcTy.isScalar())
723  return false;
724  if (!isPowerOf2_32(SrcTy.getNumElements()))
725  return false;
726  // We can usually perform 64b vector operations.
727  return SrcTy.getSizeInBits() > 64;
728  },
729  [=](const LegalityQuery &Q) {
730  LLT SrcTy = Q.Types[1];
731  return std::make_pair(1, SrcTy.divide(2));
732  })
733  .scalarize(1)
734  .lower();
735 
736  getActionDefinitionsBuilder({G_UADDSAT, G_USUBSAT})
737  .lowerIf([=](const LegalityQuery &Q) { return Q.Types[0].isScalar(); });
738 
739  getActionDefinitionsBuilder({G_FSHL, G_FSHR}).lower();
740 
742  .legalFor({{s32, s64}, {s64, s64}})
743  .customIf([=](const LegalityQuery &Q) {
744  return Q.Types[0].isScalar() && Q.Types[1].getScalarSizeInBits() < 64;
745  })
746  .lower();
748 
749  getActionDefinitionsBuilder({G_SBFX, G_UBFX})
750  .customFor({{s32, s32}, {s64, s64}});
751 
752  // TODO: Use generic lowering when custom lowering is not possible.
753  auto always = [=](const LegalityQuery &Q) { return true; };
755  .legalFor({{v8s8, v8s8}, {v16s8, v16s8}})
756  .clampScalar(0, s32, s128)
760  .customFor({{s32, s32},
761  {s64, s64},
762  {s128, s128},
763  {v2s64, v2s64},
764  {v2s32, v2s32},
765  {v4s32, v4s32},
766  {v4s16, v4s16},
767  {v8s16, v8s16}});
768 
769  // TODO: Vector types.
770  getActionDefinitionsBuilder({G_SADDSAT, G_SSUBSAT}).lowerIf(isScalar(0));
771 
772  // TODO: Vector types.
773  getActionDefinitionsBuilder({G_FMAXNUM, G_FMINNUM})
774  .legalFor({MinFPScalar, s32, s64})
775  .libcallFor({s128})
776  .minScalar(0, MinFPScalar);
777 
778  // TODO: Libcall support for s128.
779  // TODO: s16 should be legal with full FP16 support.
780  getActionDefinitionsBuilder({G_LROUND, G_LLROUND})
781  .legalFor({{s64, s32}, {s64, s64}});
782 
784  verify(*ST.getInstrInfo());
785 }
786 
788  MachineInstr &MI) const {
789  MachineIRBuilder &MIRBuilder = Helper.MIRBuilder;
790  MachineRegisterInfo &MRI = *MIRBuilder.getMRI();
791  GISelChangeObserver &Observer = Helper.Observer;
792  switch (MI.getOpcode()) {
793  default:
794  // No idea what to do.
795  return false;
796  case TargetOpcode::G_VAARG:
797  return legalizeVaArg(MI, MRI, MIRBuilder);
798  case TargetOpcode::G_LOAD:
799  case TargetOpcode::G_STORE:
800  return legalizeLoadStore(MI, MRI, MIRBuilder, Observer);
801  case TargetOpcode::G_SHL:
802  case TargetOpcode::G_ASHR:
803  case TargetOpcode::G_LSHR:
804  return legalizeShlAshrLshr(MI, MRI, MIRBuilder, Observer);
805  case TargetOpcode::G_GLOBAL_VALUE:
806  return legalizeSmallCMGlobalValue(MI, MRI, MIRBuilder, Observer);
807  case TargetOpcode::G_TRUNC:
808  return legalizeVectorTrunc(MI, Helper);
809  case TargetOpcode::G_SBFX:
810  case TargetOpcode::G_UBFX:
811  return legalizeBitfieldExtract(MI, MRI, Helper);
812  case TargetOpcode::G_ROTR:
813  return legalizeRotate(MI, MRI, Helper);
814  case TargetOpcode::G_CTPOP:
815  return legalizeCTPOP(MI, MRI, Helper);
816  case TargetOpcode::G_ATOMIC_CMPXCHG:
817  return legalizeAtomicCmpxchg128(MI, MRI, Helper);
818  case TargetOpcode::G_CTTZ:
819  return legalizeCTTZ(MI, Helper);
820  }
821 
822  llvm_unreachable("expected switch to return");
823 }
824 
825 bool AArch64LegalizerInfo::legalizeRotate(MachineInstr &MI,
827  LegalizerHelper &Helper) const {
828  // To allow for imported patterns to match, we ensure that the rotate amount
829  // is 64b with an extension.
830  Register AmtReg = MI.getOperand(2).getReg();
831  LLT AmtTy = MRI.getType(AmtReg);
832  (void)AmtTy;
833  assert(AmtTy.isScalar() && "Expected a scalar rotate");
834  assert(AmtTy.getSizeInBits() < 64 && "Expected this rotate to be legal");
835  auto NewAmt = Helper.MIRBuilder.buildSExt(LLT::scalar(64), AmtReg);
836  Helper.Observer.changingInstr(MI);
837  MI.getOperand(2).setReg(NewAmt.getReg(0));
838  Helper.Observer.changedInstr(MI);
839  return true;
840 }
841 
843  MachineIRBuilder &MIRBuilder, LLT Ty, int NumParts,
844  SmallVectorImpl<Register> &VRegs) {
845  for (int I = 0; I < NumParts; ++I)
846  VRegs.push_back(MRI.createGenericVirtualRegister(Ty));
847  MIRBuilder.buildUnmerge(VRegs, Reg);
848 }
849 
850 bool AArch64LegalizerInfo::legalizeVectorTrunc(
851  MachineInstr &MI, LegalizerHelper &Helper) const {
852  MachineIRBuilder &MIRBuilder = Helper.MIRBuilder;
853  MachineRegisterInfo &MRI = *MIRBuilder.getMRI();
854  // Similar to how operand splitting is done in SelectiondDAG, we can handle
855  // %res(v8s8) = G_TRUNC %in(v8s32) by generating:
856  // %inlo(<4x s32>), %inhi(<4 x s32>) = G_UNMERGE %in(<8 x s32>)
857  // %lo16(<4 x s16>) = G_TRUNC %inlo
858  // %hi16(<4 x s16>) = G_TRUNC %inhi
859  // %in16(<8 x s16>) = G_CONCAT_VECTORS %lo16, %hi16
860  // %res(<8 x s8>) = G_TRUNC %in16
861 
862  Register DstReg = MI.getOperand(0).getReg();
863  Register SrcReg = MI.getOperand(1).getReg();
864  LLT DstTy = MRI.getType(DstReg);
865  LLT SrcTy = MRI.getType(SrcReg);
867  isPowerOf2_32(SrcTy.getSizeInBits()));
868 
869  // Split input type.
870  LLT SplitSrcTy =
872  // First, split the source into two smaller vectors.
873  SmallVector<Register, 2> SplitSrcs;
874  extractParts(SrcReg, MRI, MIRBuilder, SplitSrcTy, 2, SplitSrcs);
875 
876  // Truncate the splits into intermediate narrower elements.
877  LLT InterTy = SplitSrcTy.changeElementSize(DstTy.getScalarSizeInBits() * 2);
878  for (unsigned I = 0; I < SplitSrcs.size(); ++I)
879  SplitSrcs[I] = MIRBuilder.buildTrunc(InterTy, SplitSrcs[I]).getReg(0);
880 
881  auto Concat = MIRBuilder.buildConcatVectors(
882  DstTy.changeElementSize(DstTy.getScalarSizeInBits() * 2), SplitSrcs);
883 
884  Helper.Observer.changingInstr(MI);
885  MI.getOperand(1).setReg(Concat.getReg(0));
886  Helper.Observer.changedInstr(MI);
887  return true;
888 }
889 
890 bool AArch64LegalizerInfo::legalizeSmallCMGlobalValue(
892  GISelChangeObserver &Observer) const {
893  assert(MI.getOpcode() == TargetOpcode::G_GLOBAL_VALUE);
894  // We do this custom legalization to convert G_GLOBAL_VALUE into target ADRP +
895  // G_ADD_LOW instructions.
896  // By splitting this here, we can optimize accesses in the small code model by
897  // folding in the G_ADD_LOW into the load/store offset.
898  auto &GlobalOp = MI.getOperand(1);
899  const auto* GV = GlobalOp.getGlobal();
900  if (GV->isThreadLocal())
901  return true; // Don't want to modify TLS vars.
902 
903  auto &TM = ST->getTargetLowering()->getTargetMachine();
904  unsigned OpFlags = ST->ClassifyGlobalReference(GV, TM);
905 
906  if (OpFlags & AArch64II::MO_GOT)
907  return true;
908 
909  auto Offset = GlobalOp.getOffset();
910  Register DstReg = MI.getOperand(0).getReg();
911  auto ADRP = MIRBuilder.buildInstr(AArch64::ADRP, {LLT::pointer(0, 64)}, {})
912  .addGlobalAddress(GV, Offset, OpFlags | AArch64II::MO_PAGE);
913  // Set the regclass on the dest reg too.
914  MRI.setRegClass(ADRP.getReg(0), &AArch64::GPR64RegClass);
915 
916  // MO_TAGGED on the page indicates a tagged address. Set the tag now. We do so
917  // by creating a MOVK that sets bits 48-63 of the register to (global address
918  // + 0x100000000 - PC) >> 48. The additional 0x100000000 offset here is to
919  // prevent an incorrect tag being generated during relocation when the the
920  // global appears before the code section. Without the offset, a global at
921  // `0x0f00'0000'0000'1000` (i.e. at `0x1000` with tag `0xf`) that's referenced
922  // by code at `0x2000` would result in `0x0f00'0000'0000'1000 - 0x2000 =
923  // 0x0eff'ffff'ffff'f000`, meaning the tag would be incorrectly set to `0xe`
924  // instead of `0xf`.
925  // This assumes that we're in the small code model so we can assume a binary
926  // size of <= 4GB, which makes the untagged PC relative offset positive. The
927  // binary must also be loaded into address range [0, 2^48). Both of these
928  // properties need to be ensured at runtime when using tagged addresses.
929  if (OpFlags & AArch64II::MO_TAGGED) {
930  assert(!Offset &&
931  "Should not have folded in an offset for a tagged global!");
932  ADRP = MIRBuilder.buildInstr(AArch64::MOVKXi, {LLT::pointer(0, 64)}, {ADRP})
933  .addGlobalAddress(GV, 0x100000000,
935  .addImm(48);
936  MRI.setRegClass(ADRP.getReg(0), &AArch64::GPR64RegClass);
937  }
938 
939  MIRBuilder.buildInstr(AArch64::G_ADD_LOW, {DstReg}, {ADRP})
940  .addGlobalAddress(GV, Offset,
942  MI.eraseFromParent();
943  return true;
944 }
945 
947  MachineInstr &MI) const {
948  switch (MI.getIntrinsicID()) {
949  case Intrinsic::vacopy: {
950  unsigned PtrSize = ST->isTargetILP32() ? 4 : 8;
951  unsigned VaListSize =
952  (ST->isTargetDarwin() || ST->isTargetWindows())
953  ? PtrSize
954  : ST->isTargetILP32() ? 20 : 32;
955 
956  MachineFunction &MF = *MI.getMF();
958  LLT::scalar(VaListSize * 8));
959  MachineIRBuilder MIB(MI);
960  MIB.buildLoad(Val, MI.getOperand(2),
963  VaListSize, Align(PtrSize)));
964  MIB.buildStore(Val, MI.getOperand(1),
967  VaListSize, Align(PtrSize)));
968  MI.eraseFromParent();
969  return true;
970  }
971  case Intrinsic::get_dynamic_area_offset: {
972  MachineIRBuilder &MIB = Helper.MIRBuilder;
973  MIB.buildConstant(MI.getOperand(0).getReg(), 0);
974  MI.eraseFromParent();
975  return true;
976  }
977  }
978 
979  return true;
980 }
981 
982 bool AArch64LegalizerInfo::legalizeShlAshrLshr(
984  GISelChangeObserver &Observer) const {
985  assert(MI.getOpcode() == TargetOpcode::G_ASHR ||
986  MI.getOpcode() == TargetOpcode::G_LSHR ||
987  MI.getOpcode() == TargetOpcode::G_SHL);
988  // If the shift amount is a G_CONSTANT, promote it to a 64 bit type so the
989  // imported patterns can select it later. Either way, it will be legal.
990  Register AmtReg = MI.getOperand(2).getReg();
991  auto VRegAndVal = getIConstantVRegValWithLookThrough(AmtReg, MRI);
992  if (!VRegAndVal)
993  return true;
994  // Check the shift amount is in range for an immediate form.
995  int64_t Amount = VRegAndVal->Value.getSExtValue();
996  if (Amount > 31)
997  return true; // This will have to remain a register variant.
998  auto ExtCst = MIRBuilder.buildConstant(LLT::scalar(64), Amount);
999  Observer.changingInstr(MI);
1000  MI.getOperand(2).setReg(ExtCst.getReg(0));
1001  Observer.changedInstr(MI);
1002  return true;
1003 }
1004 
1007  Base = Root;
1008  Offset = 0;
1009 
1010  Register NewBase;
1011  int64_t NewOffset;
1012  if (mi_match(Root, MRI, m_GPtrAdd(m_Reg(NewBase), m_ICst(NewOffset))) &&
1013  isShiftedInt<7, 3>(NewOffset)) {
1014  Base = NewBase;
1015  Offset = NewOffset;
1016  }
1017 }
1018 
1019 // FIXME: This should be removed and replaced with the generic bitcast legalize
1020 // action.
1021 bool AArch64LegalizerInfo::legalizeLoadStore(
1023  GISelChangeObserver &Observer) const {
1024  assert(MI.getOpcode() == TargetOpcode::G_STORE ||
1025  MI.getOpcode() == TargetOpcode::G_LOAD);
1026  // Here we just try to handle vector loads/stores where our value type might
1027  // have pointer elements, which the SelectionDAG importer can't handle. To
1028  // allow the existing patterns for s64 to fire for p0, we just try to bitcast
1029  // the value to use s64 types.
1030 
1031  // Custom legalization requires the instruction, if not deleted, must be fully
1032  // legalized. In order to allow further legalization of the inst, we create
1033  // a new instruction and erase the existing one.
1034 
1035  Register ValReg = MI.getOperand(0).getReg();
1036  const LLT ValTy = MRI.getType(ValReg);
1037 
1038  if (ValTy == LLT::scalar(128)) {
1039  assert((*MI.memoperands_begin())->getSuccessOrdering() ==
1041  (*MI.memoperands_begin())->getSuccessOrdering() ==
1043  assert(ST->hasLSE2() && "ldp/stp not single copy atomic without +lse2");
1044  LLT s64 = LLT::scalar(64);
1045  MachineInstrBuilder NewI;
1046  if (MI.getOpcode() == TargetOpcode::G_LOAD) {
1047  NewI = MIRBuilder.buildInstr(AArch64::LDPXi, {s64, s64}, {});
1048  MIRBuilder.buildMerge(ValReg, {NewI->getOperand(0), NewI->getOperand(1)});
1049  } else {
1050  auto Split = MIRBuilder.buildUnmerge(s64, MI.getOperand(0));
1051  NewI = MIRBuilder.buildInstr(
1052  AArch64::STPXi, {}, {Split->getOperand(0), Split->getOperand(1)});
1053  }
1054  Register Base;
1055  int Offset;
1056  matchLDPSTPAddrMode(MI.getOperand(1).getReg(), Base, Offset, MRI);
1057  NewI.addUse(Base);
1058  NewI.addImm(Offset / 8);
1059 
1060  NewI.cloneMemRefs(MI);
1063  *ST->getRegBankInfo());
1064  MI.eraseFromParent();
1065  return true;
1066  }
1067 
1068  if (!ValTy.isVector() || !ValTy.getElementType().isPointer() ||
1069  ValTy.getElementType().getAddressSpace() != 0) {
1070  LLVM_DEBUG(dbgs() << "Tried to do custom legalization on wrong load/store");
1071  return false;
1072  }
1073 
1074  unsigned PtrSize = ValTy.getElementType().getSizeInBits();
1075  const LLT NewTy = LLT::vector(ValTy.getElementCount(), PtrSize);
1076  auto &MMO = **MI.memoperands_begin();
1077  MMO.setType(NewTy);
1078 
1079  if (MI.getOpcode() == TargetOpcode::G_STORE) {
1080  auto Bitcast = MIRBuilder.buildBitcast(NewTy, ValReg);
1081  MIRBuilder.buildStore(Bitcast.getReg(0), MI.getOperand(1), MMO);
1082  } else {
1083  auto NewLoad = MIRBuilder.buildLoad(NewTy, MI.getOperand(1), MMO);
1084  MIRBuilder.buildBitcast(ValReg, NewLoad);
1085  }
1086  MI.eraseFromParent();
1087  return true;
1088 }
1089 
1090 bool AArch64LegalizerInfo::legalizeVaArg(MachineInstr &MI,
1092  MachineIRBuilder &MIRBuilder) const {
1093  MachineFunction &MF = MIRBuilder.getMF();
1094  Align Alignment(MI.getOperand(2).getImm());
1095  Register Dst = MI.getOperand(0).getReg();
1096  Register ListPtr = MI.getOperand(1).getReg();
1097 
1098  LLT PtrTy = MRI.getType(ListPtr);
1099  LLT IntPtrTy = LLT::scalar(PtrTy.getSizeInBits());
1100 
1101  const unsigned PtrSize = PtrTy.getSizeInBits() / 8;
1102  const Align PtrAlign = Align(PtrSize);
1103  auto List = MIRBuilder.buildLoad(
1104  PtrTy, ListPtr,
1106  PtrTy, PtrAlign));
1107 
1108  MachineInstrBuilder DstPtr;
1109  if (Alignment > PtrAlign) {
1110  // Realign the list to the actual required alignment.
1111  auto AlignMinus1 =
1112  MIRBuilder.buildConstant(IntPtrTy, Alignment.value() - 1);
1113  auto ListTmp = MIRBuilder.buildPtrAdd(PtrTy, List, AlignMinus1.getReg(0));
1114  DstPtr = MIRBuilder.buildMaskLowPtrBits(PtrTy, ListTmp, Log2(Alignment));
1115  } else
1116  DstPtr = List;
1117 
1118  LLT ValTy = MRI.getType(Dst);
1119  uint64_t ValSize = ValTy.getSizeInBits() / 8;
1120  MIRBuilder.buildLoad(
1121  Dst, DstPtr,
1123  ValTy, std::max(Alignment, PtrAlign)));
1124 
1125  auto Size = MIRBuilder.buildConstant(IntPtrTy, alignTo(ValSize, PtrAlign));
1126 
1127  auto NewList = MIRBuilder.buildPtrAdd(PtrTy, DstPtr, Size.getReg(0));
1128 
1129  MIRBuilder.buildStore(NewList, ListPtr,
1132  PtrTy, PtrAlign));
1133 
1134  MI.eraseFromParent();
1135  return true;
1136 }
1137 
1138 bool AArch64LegalizerInfo::legalizeBitfieldExtract(
1140  // Only legal if we can select immediate forms.
1141  // TODO: Lower this otherwise.
1142  return getIConstantVRegValWithLookThrough(MI.getOperand(2).getReg(), MRI) &&
1143  getIConstantVRegValWithLookThrough(MI.getOperand(3).getReg(), MRI);
1144 }
1145 
1146 bool AArch64LegalizerInfo::legalizeCTPOP(MachineInstr &MI,
1148  LegalizerHelper &Helper) const {
1149  // While there is no integer popcount instruction, it can
1150  // be more efficiently lowered to the following sequence that uses
1151  // AdvSIMD registers/instructions as long as the copies to/from
1152  // the AdvSIMD registers are cheap.
1153  // FMOV D0, X0 // copy 64-bit int to vector, high bits zero'd
1154  // CNT V0.8B, V0.8B // 8xbyte pop-counts
1155  // ADDV B0, V0.8B // sum 8xbyte pop-counts
1156  // UMOV X0, V0.B[0] // copy byte result back to integer reg
1157  //
1158  // For 128 bit vector popcounts, we lower to the following sequence:
1159  // cnt.16b v0, v0 // v8s16, v4s32, v2s64
1160  // uaddlp.8h v0, v0 // v8s16, v4s32, v2s64
1161  // uaddlp.4s v0, v0 // v4s32, v2s64
1162  // uaddlp.2d v0, v0 // v2s64
1163  //
1164  // For 64 bit vector popcounts, we lower to the following sequence:
1165  // cnt.8b v0, v0 // v4s16, v2s32
1166  // uaddlp.4h v0, v0 // v4s16, v2s32
1167  // uaddlp.2s v0, v0 // v2s32
1168 
1169  if (!ST->hasNEON() ||
1170  MI.getMF()->getFunction().hasFnAttribute(Attribute::NoImplicitFloat))
1171  return false;
1172  MachineIRBuilder &MIRBuilder = Helper.MIRBuilder;
1173  Register Dst = MI.getOperand(0).getReg();
1174  Register Val = MI.getOperand(1).getReg();
1175  LLT Ty = MRI.getType(Val);
1176 
1177  assert(Ty == MRI.getType(Dst) &&
1178  "Expected src and dst to have the same type!");
1179  unsigned Size = Ty.getSizeInBits();
1180 
1181  // Pre-conditioning: widen Val up to the nearest vector type.
1182  // s32,s64,v4s16,v2s32 -> v8i8
1183  // v8s16,v4s32,v2s64 -> v16i8
1184  LLT VTy = Size == 128 ? LLT::fixed_vector(16, 8) : LLT::fixed_vector(8, 8);
1185  if (Ty.isScalar()) {
1186  assert((Size == 32 || Size == 64 || Size == 128) && "Expected only 32, 64, or 128 bit scalars!");
1187  if (Size == 32) {
1188  Val = MIRBuilder.buildZExt(LLT::scalar(64), Val).getReg(0);
1189  }
1190  }
1191  Val = MIRBuilder.buildBitcast(VTy, Val).getReg(0);
1192 
1193  // Count bits in each byte-sized lane.
1194  auto CTPOP = MIRBuilder.buildCTPOP(VTy, Val);
1195 
1196  // Sum across lanes.
1197  Register HSum = CTPOP.getReg(0);
1198  unsigned Opc;
1199  SmallVector<LLT> HAddTys;
1200  if (Ty.isScalar()) {
1201  Opc = Intrinsic::aarch64_neon_uaddlv;
1202  HAddTys.push_back(LLT::scalar(32));
1203  } else if (Ty == LLT::fixed_vector(8, 16)) {
1204  Opc = Intrinsic::aarch64_neon_uaddlp;
1205  HAddTys.push_back(LLT::fixed_vector(8, 16));
1206  } else if (Ty == LLT::fixed_vector(4, 32)) {
1207  Opc = Intrinsic::aarch64_neon_uaddlp;
1208  HAddTys.push_back(LLT::fixed_vector(8, 16));
1209  HAddTys.push_back(LLT::fixed_vector(4, 32));
1210  } else if (Ty == LLT::fixed_vector(2, 64)) {
1211  Opc = Intrinsic::aarch64_neon_uaddlp;
1212  HAddTys.push_back(LLT::fixed_vector(8, 16));
1213  HAddTys.push_back(LLT::fixed_vector(4, 32));
1214  HAddTys.push_back(LLT::fixed_vector(2, 64));
1215  } else if (Ty == LLT::fixed_vector(4, 16)) {
1216  Opc = Intrinsic::aarch64_neon_uaddlp;
1217  HAddTys.push_back(LLT::fixed_vector(4, 16));
1218  } else if (Ty == LLT::fixed_vector(2, 32)) {
1219  Opc = Intrinsic::aarch64_neon_uaddlp;
1220  HAddTys.push_back(LLT::fixed_vector(4, 16));
1221  HAddTys.push_back(LLT::fixed_vector(2, 32));
1222  } else
1223  llvm_unreachable("unexpected vector shape");
1224  MachineInstrBuilder UADD;
1225  for (LLT HTy : HAddTys) {
1226  UADD = MIRBuilder.buildIntrinsic(Opc, {HTy}, /*HasSideEffects =*/false)
1227  .addUse(HSum);
1228  HSum = UADD.getReg(0);
1229  }
1230 
1231  // Post-conditioning.
1232  if (Ty.isScalar() && (Size == 64 || Size == 128))
1233  MIRBuilder.buildZExt(Dst, UADD);
1234  else
1235  UADD->getOperand(0).setReg(Dst);
1236  MI.eraseFromParent();
1237  return true;
1238 }
1239 
1240 bool AArch64LegalizerInfo::legalizeAtomicCmpxchg128(
1242  MachineIRBuilder &MIRBuilder = Helper.MIRBuilder;
1243  LLT s64 = LLT::scalar(64);
1244  auto Addr = MI.getOperand(1).getReg();
1245  auto DesiredI = MIRBuilder.buildUnmerge({s64, s64}, MI.getOperand(2));
1246  auto NewI = MIRBuilder.buildUnmerge({s64, s64}, MI.getOperand(3));
1247  auto DstLo = MRI.createGenericVirtualRegister(s64);
1248  auto DstHi = MRI.createGenericVirtualRegister(s64);
1249 
1250  MachineInstrBuilder CAS;
1251  if (ST->hasLSE()) {
1252  // We have 128-bit CASP instructions taking XSeqPair registers, which are
1253  // s128. We need the merge/unmerge to bracket the expansion and pair up with
1254  // the rest of the MIR so we must reassemble the extracted registers into a
1255  // 128-bit known-regclass one with code like this:
1256  //
1257  // %in1 = REG_SEQUENCE Lo, Hi ; One for each input
1258  // %out = CASP %in1, ...
1259  // %OldLo = G_EXTRACT %out, 0
1260  // %OldHi = G_EXTRACT %out, 64
1261  auto Ordering = (*MI.memoperands_begin())->getMergedOrdering();
1262  unsigned Opcode;
1263  switch (Ordering) {
1265  Opcode = AArch64::CASPAX;
1266  break;
1268  Opcode = AArch64::CASPLX;
1269  break;
1272  Opcode = AArch64::CASPALX;
1273  break;
1274  default:
1275  Opcode = AArch64::CASPX;
1276  break;
1277  }
1278 
1279  LLT s128 = LLT::scalar(128);
1280  auto CASDst = MRI.createGenericVirtualRegister(s128);
1281  auto CASDesired = MRI.createGenericVirtualRegister(s128);
1282  auto CASNew = MRI.createGenericVirtualRegister(s128);
1283  MIRBuilder.buildInstr(TargetOpcode::REG_SEQUENCE, {CASDesired}, {})
1284  .addUse(DesiredI->getOperand(0).getReg())
1285  .addImm(AArch64::sube64)
1286  .addUse(DesiredI->getOperand(1).getReg())
1287  .addImm(AArch64::subo64);
1288  MIRBuilder.buildInstr(TargetOpcode::REG_SEQUENCE, {CASNew}, {})
1289  .addUse(NewI->getOperand(0).getReg())
1290  .addImm(AArch64::sube64)
1291  .addUse(NewI->getOperand(1).getReg())
1292  .addImm(AArch64::subo64);
1293 
1294  CAS = MIRBuilder.buildInstr(Opcode, {CASDst}, {CASDesired, CASNew, Addr});
1295 
1296  MIRBuilder.buildExtract({DstLo}, {CASDst}, 0);
1297  MIRBuilder.buildExtract({DstHi}, {CASDst}, 64);
1298  } else {
1299  // The -O0 CMP_SWAP_128 is friendlier to generate code for because LDXP/STXP
1300  // can take arbitrary registers so it just has the normal GPR64 operands the
1301  // rest of AArch64 is expecting.
1302  auto Ordering = (*MI.memoperands_begin())->getMergedOrdering();
1303  unsigned Opcode;
1304  switch (Ordering) {
1306  Opcode = AArch64::CMP_SWAP_128_ACQUIRE;
1307  break;
1309  Opcode = AArch64::CMP_SWAP_128_RELEASE;
1310  break;
1313  Opcode = AArch64::CMP_SWAP_128;
1314  break;
1315  default:
1316  Opcode = AArch64::CMP_SWAP_128_MONOTONIC;
1317  break;
1318  }
1319 
1320  auto Scratch = MRI.createVirtualRegister(&AArch64::GPR64RegClass);
1321  CAS = MIRBuilder.buildInstr(Opcode, {DstLo, DstHi, Scratch},
1322  {Addr, DesiredI->getOperand(0),
1323  DesiredI->getOperand(1), NewI->getOperand(0),
1324  NewI->getOperand(1)});
1325  }
1326 
1327  CAS.cloneMemRefs(MI);
1330  *ST->getRegBankInfo());
1331 
1332  MIRBuilder.buildMerge(MI.getOperand(0), {DstLo, DstHi});
1333  MI.eraseFromParent();
1334  return true;
1335 }
1336 
1337 bool AArch64LegalizerInfo::legalizeCTTZ(MachineInstr &MI,
1338  LegalizerHelper &Helper) const {
1339  MachineIRBuilder &MIRBuilder = Helper.MIRBuilder;
1340  MachineRegisterInfo &MRI = *MIRBuilder.getMRI();
1341  LLT Ty = MRI.getType(MI.getOperand(1).getReg());
1342  auto BitReverse = MIRBuilder.buildBitReverse(Ty, MI.getOperand(1));
1343  MIRBuilder.buildCTLZ(MI.getOperand(0).getReg(), BitReverse);
1344  MI.eraseFromParent();
1345  return true;
1346 }
AArch64LegalizerInfo.h
llvm::Check::Size
@ Size
Definition: FileCheck.h:73
MIPatternMatch.h
llvm::alignTo
uint64_t alignTo(uint64_t Size, Align A)
Returns a multiple of A needed to store Size bytes.
Definition: Alignment.h:148
llvm::AArch64II::MO_G3
@ MO_G3
MO_G3 - A symbol operand with this flag (granule 3) represents the high 16-bits of a 64-bit address,...
Definition: AArch64BaseInfo.h:686
ValueTypes.h
llvm::AArch64Subtarget::isTargetWindows
bool isTargetWindows() const
Definition: AArch64Subtarget.h:506
llvm::LegalizeRuleSet::widenScalarToNextPow2
LegalizeRuleSet & widenScalarToNextPow2(unsigned TypeIdx, unsigned MinSize=0)
Widen the scalar to the next power of two that is at least MinSize.
Definition: LegalizerInfo.h:840
llvm::AtomicOrdering::AcquireRelease
@ AcquireRelease
matchLDPSTPAddrMode
static void matchLDPSTPAddrMode(Register Root, Register &Base, int &Offset, MachineRegisterInfo &MRI)
Definition: AArch64LegalizerInfo.cpp:1005
MI
IRTranslator LLVM IR MI
Definition: IRTranslator.cpp:103
MachineInstr.h
MathExtras.h
llvm::MachineInstrBuilder::addImm
const MachineInstrBuilder & addImm(int64_t Val) const
Add a new immediate operand.
Definition: MachineInstrBuilder.h:131
llvm
---------------------— PointerInfo ------------------------------------—
Definition: AllocatorList.h:23
Reg
unsigned Reg
Definition: MachineSink.cpp:1566
llvm::LLT::getScalarSizeInBits
unsigned getScalarSizeInBits() const
Definition: LowLevelTypeImpl.h:213
llvm::MachineRegisterInfo::createVirtualRegister
Register createVirtualRegister(const TargetRegisterClass *RegClass, StringRef Name="")
createVirtualRegister - Create and return a new virtual register in the function with the specified r...
Definition: MachineRegisterInfo.cpp:158
AArch64RegisterBankInfo.h
llvm::LegalizeRuleSet::unsupportedIfMemSizeNotPow2
LegalizeRuleSet & unsupportedIfMemSizeNotPow2()
Definition: LegalizerInfo.h:799
llvm::MIPatternMatch::m_Reg
operand_type_match m_Reg()
Definition: MIPatternMatch.h:137
llvm::LegalizerInfo::getActionDefinitionsBuilder
LegalizeRuleSet & getActionDefinitionsBuilder(unsigned Opcode)
Get the action definition builder for the given opcode.
Definition: LegalizerInfo.cpp:290
llvm::MachineRegisterInfo
MachineRegisterInfo - Keep track of information for virtual and physical registers,...
Definition: MachineRegisterInfo.h:52
llvm::LegalizeRuleSet::minScalarEltSameAsIf
LegalizeRuleSet & minScalarEltSameAsIf(LegalityPredicate Predicate, unsigned TypeIdx, unsigned LargeTypeIdx)
Conditionally widen the scalar or elt to match the size of another.
Definition: LegalizerInfo.h:995
llvm::LegalizeRuleSet::maxScalarIf
LegalizeRuleSet & maxScalarIf(LegalityPredicate Predicate, unsigned TypeIdx, const LLT Ty)
Conditionally limit the maximum size of the scalar.
Definition: LegalizerInfo.h:937
llvm::SmallVector
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
Definition: SmallVector.h:1168
llvm::MachineIRBuilder::getMRI
MachineRegisterInfo * getMRI()
Getter for MRI.
Definition: MachineIRBuilder.h:280
llvm::LegalizeRuleSet::customFor
LegalizeRuleSet & customFor(std::initializer_list< LLT > Types)
Definition: LegalizerInfo.h:814
llvm::MachineFunction::getMachineMemOperand
MachineMemOperand * getMachineMemOperand(MachinePointerInfo PtrInfo, MachineMemOperand::Flags f, uint64_t s, Align base_alignment, const AAMDNodes &AAInfo=AAMDNodes(), const MDNode *Ranges=nullptr, SyncScope::ID SSID=SyncScope::System, AtomicOrdering Ordering=AtomicOrdering::NotAtomic, AtomicOrdering FailureOrdering=AtomicOrdering::NotAtomic)
getMachineMemOperand - Allocate a new MachineMemOperand.
Definition: MachineFunction.cpp:430
llvm::LegacyLegalizeActions::Bitcast
@ Bitcast
Perform the operation on a different, but equivalently sized type.
Definition: LegacyLegalizerInfo.h:54
llvm::LegalizeRuleSet::clampNumElements
LegalizeRuleSet & clampNumElements(unsigned TypeIdx, const LLT MinTy, const LLT MaxTy)
Limit the number of elements for the given vectors to at least MinTy's number of elements and at most...
Definition: LegalizerInfo.h:1080
llvm::MachineRegisterInfo::getTargetRegisterInfo
const TargetRegisterInfo * getTargetRegisterInfo() const
Definition: MachineRegisterInfo.h:153
llvm::LegacyLegalizerInfo::computeTables
void computeTables()
Compute any ancillary tables needed to quickly decide how an operation should be handled.
Definition: LegacyLegalizerInfo.cpp:102
llvm::AArch64Subtarget::isTargetDarwin
bool isTargetDarwin() const
Definition: AArch64Subtarget.h:503
llvm::AtomicOrdering::SequentiallyConsistent
@ SequentiallyConsistent
llvm::LegalizeRuleSet::minScalarOrEltIf
LegalizeRuleSet & minScalarOrEltIf(LegalityPredicate Predicate, unsigned TypeIdx, const LLT Ty)
Ensure the scalar or element is at least as wide as Ty.
Definition: LegalizerInfo.h:897
llvm::AArch64II::MO_PREL
@ MO_PREL
MO_PREL - Indicates that the bits of the symbol operand represented by MO_G0 etc are PC relative.
Definition: AArch64BaseInfo.h:737
llvm::AArch64Subtarget::getInstrInfo
const AArch64InstrInfo * getInstrInfo() const override
Definition: AArch64Subtarget.h:319
llvm::LLT::changeElementCount
LLT changeElementCount(ElementCount EC) const
Return a vector or scalar with the same element type and the new element count.
Definition: LowLevelTypeImpl.h:190
Offset
uint64_t Offset
Definition: ELFObjHandler.cpp:81
llvm::LLT::vector
static LLT vector(ElementCount EC, unsigned ScalarSizeInBits)
Get a low-level vector of some number of elements and element width.
Definition: LowLevelTypeImpl.h:57
llvm::LegalizeRuleSet::scalarizeIf
LegalizeRuleSet & scalarizeIf(LegalityPredicate Predicate, unsigned TypeIdx)
Definition: LegalizerInfo.h:880
MachineIRBuilder.h
llvm::LegalizeRuleSet::scalarize
LegalizeRuleSet & scalarize(unsigned TypeIdx)
Definition: LegalizerInfo.h:874
llvm::isPowerOf2_32
constexpr bool isPowerOf2_32(uint32_t Value)
Return true if the argument is a power of two > 0.
Definition: MathExtras.h:491
llvm::LegalizeRuleSet::lower
LegalizeRuleSet & lower()
The instruction is lowered.
Definition: LegalizerInfo.h:645
llvm::LegalizerHelper
Definition: LegalizerHelper.h:39
llvm::LegalizeMutations::changeTo
LegalizeMutation changeTo(unsigned TypeIdx, LLT Ty)
Select this specific type for the given type index.
Definition: LegalizeMutations.cpp:17
LegalizerInfo.h
llvm::LegalityPredicates::atomicOrderingAtLeastOrStrongerThan
LegalityPredicate atomicOrderingAtLeastOrStrongerThan(unsigned MMOIdx, AtomicOrdering Ordering)
True iff the specified MMO index has at an atomic ordering of at Ordering or stronger.
Definition: LegalityPredicates.cpp:198
LLVM_DEBUG
#define LLVM_DEBUG(X)
Definition: Debug.h:101
llvm::AArch64Subtarget::getTargetLowering
const AArch64TargetLowering * getTargetLowering() const override
Definition: AArch64Subtarget.h:316
llvm::constrainSelectedInstRegOperands
bool constrainSelectedInstRegOperands(MachineInstr &I, const TargetInstrInfo &TII, const TargetRegisterInfo &TRI, const RegisterBankInfo &RBI)
Mutate the newly-selected instruction I to constrain its (possibly generic) virtual register operands...
Definition: Utils.cpp:134
llvm::MachineIRBuilder::buildConstant
virtual MachineInstrBuilder buildConstant(const DstOp &Res, const ConstantInt &Val)
Build and insert Res = G_CONSTANT Val.
Definition: MachineIRBuilder.cpp:255
llvm::LegalizeRuleSet::legalIf
LegalizeRuleSet & legalIf(LegalityPredicate Predicate)
The instruction is legal if predicate is true.
Definition: LegalizerInfo.h:572
MachineRegisterInfo.h
llvm::AtomicOrdering::Monotonic
@ Monotonic
llvm::LegalizeRuleSet::minScalar
LegalizeRuleSet & minScalar(unsigned TypeIdx, const LLT Ty)
Ensure the scalar is at least as wide as Ty.
Definition: LegalizerInfo.h:908
always
bar al al movzbl eax ret Missed when stored in a memory are stored as single byte objects the value of which is always(false) or 1(true). We are not using this fact
Definition: README.txt:1412
llvm::dbgs
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
Definition: Debug.cpp:163
llvm::LLT::fixed_vector
static LLT fixed_vector(unsigned NumElements, unsigned ScalarSizeInBits)
Get a low-level fixed-width vector of some number of elements and element width.
Definition: LowLevelTypeImpl.h:75
llvm::MachineIRBuilder::buildBitReverse
MachineInstrBuilder buildBitReverse(const DstOp &Dst, const SrcOp &Src)
Build and insert Dst = G_BITREVERSE Src.
Definition: MachineIRBuilder.h:1872
llvm::MachineFunction::getRegInfo
MachineRegisterInfo & getRegInfo()
getRegInfo - Return information about the registers currently in use.
Definition: MachineFunction.h:636
llvm::LegalizeRuleSet::customIf
LegalizeRuleSet & customIf(LegalityPredicate Predicate)
Definition: LegalizerInfo.h:808
llvm::LegalityPredicates::typeIs
LegalityPredicate typeIs(unsigned TypeIdx, LLT TypesInit)
True iff the given type index is the specified type.
Definition: LegalityPredicates.cpp:28
llvm::MachineIRBuilder::buildUnmerge
MachineInstrBuilder buildUnmerge(ArrayRef< LLT > Res, const SrcOp &Op)
Build and insert Res0, ...
Definition: MachineIRBuilder.cpp:603
llvm::MachineIRBuilder::buildZExt
MachineInstrBuilder buildZExt(const DstOp &Res, const SrcOp &Op)
Build and insert Res = G_ZEXT Op.
Definition: MachineIRBuilder.cpp:424
llvm::MachineIRBuilder::buildLoad
MachineInstrBuilder buildLoad(const DstOp &Res, const SrcOp &Addr, MachineMemOperand &MMO)
Build and insert Res = G_LOAD Addr, MMO.
Definition: MachineIRBuilder.h:832
llvm::LegalityPredicates::smallerThan
LegalityPredicate smallerThan(unsigned TypeIdx0, unsigned TypeIdx1)
True iff the first type index has a smaller total bit size than second type index.
Definition: LegalityPredicates.cpp:117
llvm::MIPatternMatch::m_GPtrAdd
BinaryOp_match< LHS, RHS, TargetOpcode::G_PTR_ADD, true > m_GPtrAdd(const LHS &L, const RHS &R)
Definition: MIPatternMatch.h:313
llvm::AArch64Subtarget::isTargetILP32
bool isTargetILP32() const
Definition: AArch64Subtarget.h:514
Intrinsics.h
llvm::MachineInstr::getOperand
const MachineOperand & getOperand(unsigned i) const
Definition: MachineInstr.h:499
llvm::Log2
unsigned Log2(Align A)
Returns the log2 of the alignment.
Definition: Alignment.h:207
llvm::LLT::getSizeInBits
TypeSize getSizeInBits() const
Returns the total size of the type. Must only be called on sized types.
Definition: LowLevelTypeImpl.h:153
llvm::AArch64LegalizerInfo::legalizeIntrinsic
bool legalizeIntrinsic(LegalizerHelper &Helper, MachineInstr &MI) const override
Definition: AArch64LegalizerInfo.cpp:946
llvm::LegalizeRuleSet::lowerIfMemSizeNotPow2
LegalizeRuleSet & lowerIfMemSizeNotPow2()
Definition: LegalizerInfo.h:803
Utils.h
llvm::MachineIRBuilder::buildCTLZ
MachineInstrBuilder buildCTLZ(const DstOp &Dst, const SrcOp &Src0)
Build and insert Res = G_CTLZ Op0, Src0.
Definition: MachineIRBuilder.h:1554
llvm::LegalizeRuleSet::fewerElementsIf
LegalizeRuleSet & fewerElementsIf(LegalityPredicate Predicate, LegalizeMutation Mutation)
Remove elements to reach the type selected by the mutation if the predicate is true.
Definition: LegalizerInfo.h:778
TargetOpcodes.h
llvm::MachineIRBuilder::buildConcatVectors
MachineInstrBuilder buildConcatVectors(const DstOp &Res, ArrayRef< Register > Ops)
Build and insert Res = G_CONCAT_VECTORS Op0, ...
Definition: MachineIRBuilder.cpp:689
llvm::CodeModel::Small
@ Small
Definition: CodeGen.h:28
Concat
static constexpr int Concat[]
Definition: X86InterleavedAccess.cpp:239
llvm::MachineIRBuilder::getMF
MachineFunction & getMF()
Getter for the function we currently build.
Definition: MachineIRBuilder.h:262
llvm::AtomicOrdering::Acquire
@ Acquire
llvm::AArch64LegalizerInfo::AArch64LegalizerInfo
AArch64LegalizerInfo(const AArch64Subtarget &ST)
Definition: AArch64LegalizerInfo.cpp:41
llvm::GISelChangeObserver::changingInstr
virtual void changingInstr(MachineInstr &MI)=0
This instruction is about to be mutated in some way.
llvm::LegalizeRuleSet::clampMaxNumElements
LegalizeRuleSet & clampMaxNumElements(unsigned TypeIdx, const LLT EltTy, unsigned MaxElements)
Limit the number of elements in EltTy vectors to at most MaxElements.
Definition: LegalizerInfo.h:1056
Align
uint64_t Align
Definition: ELFObjHandler.cpp:83
llvm::Align
This struct is a compact representation of a valid (non-zero power of two) alignment.
Definition: Alignment.h:39
llvm::LegalizeRuleSet::maxScalarEltSameAsIf
LegalizeRuleSet & maxScalarEltSameAsIf(LegalityPredicate Predicate, unsigned TypeIdx, unsigned SmallTypeIdx)
Conditionally narrow the scalar or elt to match the size of another.
Definition: LegalizerInfo.h:1011
Type.h
llvm::MachineInstrBuilder::getReg
Register getReg(unsigned Idx) const
Get the register for the operand index.
Definition: MachineInstrBuilder.h:94
llvm::MachineInstrBuilder::cloneMemRefs
const MachineInstrBuilder & cloneMemRefs(const MachineInstr &OtherMI) const
Definition: MachineInstrBuilder.h:213
llvm::AArch64LegalizerInfo::legalizeCustom
bool legalizeCustom(LegalizerHelper &Helper, MachineInstr &MI) const override
Called for instructions with the Custom LegalizationAction.
Definition: AArch64LegalizerInfo.cpp:787
llvm::MachineIRBuilder::buildCTPOP
MachineInstrBuilder buildCTPOP(const DstOp &Dst, const SrcOp &Src0)
Build and insert Res = G_CTPOP Op0, Src0.
Definition: MachineIRBuilder.h:1549
llvm::LLT::pointer
static LLT pointer(unsigned AddressSpace, unsigned SizeInBits)
Get a low-level pointer in the given address space.
Definition: LowLevelTypeImpl.h:50
llvm::LLT::getAddressSpace
unsigned getAddressSpace() const
Definition: LowLevelTypeImpl.h:227
llvm::GISelChangeObserver::changedInstr
virtual void changedInstr(MachineInstr &MI)=0
This instruction was mutated in some way.
llvm::MachineIRBuilder::buildMaskLowPtrBits
MachineInstrBuilder buildMaskLowPtrBits(const DstOp &Res, const SrcOp &Op0, uint32_t NumBits)
Build and insert Res = G_PTRMASK Op0, G_CONSTANT (1 << NumBits) - 1.
Definition: MachineIRBuilder.cpp:208
llvm::LegalizeMutations::scalarize
LegalizeMutation scalarize(unsigned TypeIdx)
Break up the vector type for the given type index into the element type.
Definition: LegalizeMutations.cpp:87
llvm::LegalizeRuleSet::lowerIf
LegalizeRuleSet & lowerIf(LegalityPredicate Predicate)
The instruction is lowered if predicate is true.
Definition: LegalizerInfo.h:654
llvm::LLT::divide
LLT divide(int Factor) const
Return a type that is Factor times smaller.
Definition: LowLevelTypeImpl.h:197
llvm::MachineIRBuilder
Helper class to build MachineInstr.
Definition: MachineIRBuilder.h:212
llvm::LegalizeRuleSet::legalFor
LegalizeRuleSet & legalFor(std::initializer_list< LLT > Types)
The instruction is legal when type index 0 is any type in the given list.
Definition: LegalizerInfo.h:579
llvm::MachineInstr
Representation of each machine instruction.
Definition: MachineInstr.h:64
llvm::MachineInstrBuilder
Definition: MachineInstrBuilder.h:69
uint64_t
llvm::ARM_MB::ST
@ ST
Definition: ARMBaseInfo.h:73
Addr
uint64_t Addr
Definition: ELFObjHandler.cpp:80
llvm::MachineIRBuilder::buildBitcast
MachineInstrBuilder buildBitcast(const DstOp &Dst, const SrcOp &Src)
Build and insert Dst = G_BITCAST Src.
Definition: MachineIRBuilder.h:633
llvm::LegalityPredicates::all
Predicate all(Predicate P0, Predicate P1)
True iff P0 and P1 are true.
Definition: LegalizerInfo.h:228
llvm::AArch64II::MO_NC
@ MO_NC
MO_NC - Indicates whether the linker is expected to check the symbol reference for overflow.
Definition: AArch64BaseInfo.h:718
llvm::MachinePointerInfo
This class contains a discriminated union of information about pointers in memory operands,...
Definition: MachineMemOperand.h:38
llvm::AArch64II::MO_PAGEOFF
@ MO_PAGEOFF
MO_PAGEOFF - A symbol operand with this flag represents the offset of that symbol within a 4K page.
Definition: AArch64BaseInfo.h:682
llvm::MachineIRBuilder::buildPtrAdd
MachineInstrBuilder buildPtrAdd(const DstOp &Res, const SrcOp &Op0, const SrcOp &Op1)
Build and insert Res = G_PTR_ADD Op0, Op1.
Definition: MachineIRBuilder.cpp:182
llvm::AtomicOrdering::Unordered
@ Unordered
I
#define I(x, y, z)
Definition: MD5.cpp:59
llvm::LLT::isVector
bool isVector() const
Definition: LowLevelTypeImpl.h:123
llvm::LegalizeRuleSet::clampScalar
LegalizeRuleSet & clampScalar(unsigned TypeIdx, const LLT MinTy, const LLT MaxTy)
Limit the range of scalar sizes to MinTy and MaxTy.
Definition: LegalizerInfo.h:953
llvm::LLT::getNumElements
uint16_t getNumElements() const
Returns the number of elements in a vector LLT.
Definition: LowLevelTypeImpl.h:127
llvm::LegalizeRuleSet::legalForCartesianProduct
LegalizeRuleSet & legalForCartesianProduct(std::initializer_list< LLT > Types)
The instruction is legal when type indexes 0 and 1 are both in the given list.
Definition: LegalizerInfo.h:611
assert
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
llvm::TargetMachine
Primary interface to the complete machine description for the target machine.
Definition: TargetMachine.h:79
llvm::LLT::isPointer
bool isPointer() const
Definition: LowLevelTypeImpl.h:121
llvm::LegalityPredicates::typeInSet
LegalityPredicate typeInSet(unsigned TypeIdx, std::initializer_list< LLT > TypesInit)
True iff the given type index is one of the specified types.
Definition: LegalityPredicates.cpp:34
llvm::MachineRegisterInfo::createGenericVirtualRegister
Register createGenericVirtualRegister(LLT Ty, StringRef Name="")
Create and return a new generic virtual register with low-level type Ty.
Definition: MachineRegisterInfo.cpp:188
llvm::MachineInstrBuilder::addUse
const MachineInstrBuilder & addUse(Register RegNo, unsigned Flags=0, unsigned SubReg=0) const
Add a virtual register use operand.
Definition: MachineInstrBuilder.h:123
llvm::MachineOperand::getReg
Register getReg() const
getReg - Returns the register number.
Definition: MachineOperand.h:360
llvm::AArch64Subtarget::hasNEON
bool hasNEON() const
Definition: AArch64Subtarget.h:374
llvm::LLT::isScalar
bool isScalar() const
Definition: LowLevelTypeImpl.h:119
llvm::MachineFunction
Definition: MachineFunction.h:230
llvm::LegalityQuery::Opcode
unsigned Opcode
Definition: LegalizerInfo.h:109
llvm::MachineIRBuilder::buildInstr
MachineInstrBuilder buildInstr(unsigned Opcode)
Build and insert <empty> = Opcode <empty>.
Definition: MachineIRBuilder.h:367
llvm::LegalityQuery
The LegalityQuery object bundles together all the information that's needed to decide whether a given...
Definition: LegalizerInfo.h:108
llvm::LegalizeRuleSet::customForCartesianProduct
LegalizeRuleSet & customForCartesianProduct(std::initializer_list< LLT > Types)
Definition: LegalizerInfo.h:824
llvm::getIConstantVRegValWithLookThrough
Optional< ValueAndVReg > getIConstantVRegValWithLookThrough(Register VReg, const MachineRegisterInfo &MRI, bool LookThroughInstrs=true)
If VReg is defined by a statically evaluable chain of instructions rooted on a G_CONSTANT returns its...
Definition: Utils.cpp:391
llvm_unreachable
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
Definition: ErrorHandling.h:136
if
if(llvm_vc STREQUAL "") set(fake_version_inc "$
Definition: CMakeLists.txt:14
llvm::LegalizerHelper::Observer
GISelChangeObserver & Observer
To keep track of changes made by the LegalizerHelper.
Definition: LegalizerHelper.h:46
llvm::MachineIRBuilder::buildExtract
MachineInstrBuilder buildExtract(const DstOp &Res, const SrcOp &Src, uint64_t Index)
Build and insert Res0, ...
Definition: MachineIRBuilder.cpp:516
llvm::GISelChangeObserver
Abstract class that contains various methods for clients to notify about changes.
Definition: GISelChangeObserver.h:29
llvm::LegalizeMutations::widenScalarOrEltToNextPow2
LegalizeMutation widenScalarOrEltToNextPow2(unsigned TypeIdx, unsigned Min=0)
Widen the scalar type or vector element type for the given type index to the next power of 2.
Definition: LegalizeMutations.cpp:56
llvm::MachineMemOperand::MOLoad
@ MOLoad
The memory access reads data.
Definition: MachineMemOperand.h:135
MRI
unsigned const MachineRegisterInfo * MRI
Definition: AArch64AdvSIMDScalarPass.cpp:105
llvm::Register
Wrapper class representing virtual and physical registers.
Definition: Register.h:19
llvm::LegalityPredicates::isScalar
LegalityPredicate isScalar(unsigned TypeIdx)
True iff the specified type index is a scalar.
Definition: LegalityPredicates.cpp:67
llvm::AtomicOrdering::Release
@ Release
llvm::LegalizerInfo::getLegacyLegalizerInfo
const LegacyLegalizerInfo & getLegacyLegalizerInfo() const
Definition: LegalizerInfo.h:1114
llvm::TargetLoweringBase::getTargetMachine
const TargetMachine & getTargetMachine() const
Definition: TargetLowering.h:339
llvm::LegalizeRuleSet::narrowScalarIf
LegalizeRuleSet & narrowScalarIf(LegalityPredicate Predicate, LegalizeMutation Mutation)
Narrow the scalar to the one selected by the mutation if the predicate is true.
Definition: LegalizerInfo.h:752
llvm::MachineIRBuilder::buildTrunc
MachineInstrBuilder buildTrunc(const DstOp &Res, const SrcOp &Op)
Build and insert Res = G_TRUNC Op.
Definition: MachineIRBuilder.cpp:737
llvm::AArch64Subtarget::getRegBankInfo
const RegisterBankInfo * getRegBankInfo() const override
Definition: AArch64Subtarget.cpp:246
llvm::AArch64Subtarget::ClassifyGlobalReference
unsigned ClassifyGlobalReference(const GlobalValue *GV, const TargetMachine &TM) const
ClassifyGlobalReference - Find the target operand flags that describe how a global value should be re...
Definition: AArch64Subtarget.cpp:253
llvm::AMDGPU::SendMsg::Op
Op
Definition: SIDefines.h:321
llvm::AArch64Subtarget::hasLSE2
bool hasLSE2() const
Definition: AArch64Subtarget.h:379
llvm::LegalityQuery::MMODescrs
ArrayRef< MemDesc > MMODescrs
Operations which require memory can use this to place requirements on the memory type for each MMO.
Definition: LegalizerInfo.h:128
llvm::LegalizeRuleSet::unsupportedIf
LegalizeRuleSet & unsupportedIf(LegalityPredicate Predicate)
Definition: LegalizerInfo.h:791
LegalizerHelper.h
llvm::AArch64ISD::ADRP
@ ADRP
Definition: AArch64ISelLowering.h:61
llvm::LegalizeRuleSet::minScalarSameAs
LegalizeRuleSet & minScalarSameAs(unsigned TypeIdx, unsigned LargeTypeIdx)
Widen the scalar to match the size of another.
Definition: LegalizerInfo.h:966
llvm::AArch64Subtarget::hasLSE
bool hasLSE() const
Definition: AArch64Subtarget.h:378
llvm::MachineMemOperand::MOStore
@ MOStore
The memory access writes data.
Definition: MachineMemOperand.h:137
llvm::MIPatternMatch::m_ICst
ConstantMatch m_ICst(int64_t &Cst)
Definition: MIPatternMatch.h:74
llvm::LegalizerInfo::verify
void verify(const MCInstrInfo &MII) const
Perform simple self-diagnostic and assert if there is anything obviously wrong with the actions set u...
Definition: LegalizerInfo.cpp:378
llvm::LegalityPredicates::isVector
LegalityPredicate isVector(unsigned TypeIdx)
True iff the specified type index is a vector.
Definition: LegalityPredicates.cpp:73
AArch64Subtarget.h
llvm::MachineIRBuilder::buildIntrinsic
MachineInstrBuilder buildIntrinsic(Intrinsic::ID ID, ArrayRef< Register > Res, bool HasSideEffects)
Build and insert either a G_INTRINSIC (if HasSideEffects is false) or G_INTRINSIC_W_SIDE_EFFECTS inst...
Definition: MachineIRBuilder.cpp:713
s1
int s1
Definition: README.txt:182
llvm::MachineRegisterInfo::getType
LLT getType(Register Reg) const
Get the low-level type of Reg or LLT{} if Reg is not a generic (target independent) virtual register.
Definition: MachineRegisterInfo.h:732
llvm::LegalityQuery::Types
ArrayRef< LLT > Types
Definition: LegalizerInfo.h:110
llvm::LegalizeRuleSet::moreElementsToNextPow2
LegalizeRuleSet & moreElementsToNextPow2(unsigned TypeIdx)
Add more elements to the vector to reach the next power of two.
Definition: LegalizerInfo.h:1030
llvm::MachineIRBuilder::buildSExt
MachineInstrBuilder buildSExt(const DstOp &Res, const SrcOp &Op)
Build and insert Res = G_SEXT Op.
Definition: MachineIRBuilder.cpp:419
List
const NodeList & List
Definition: RDFGraph.cpp:201
llvm::LLT::getElementCount
ElementCount getElementCount() const
Definition: LowLevelTypeImpl.h:144
llvm::MachineOperand::setReg
void setReg(Register Reg)
Change the register this operand corresponds to.
Definition: MachineOperand.cpp:55
llvm::LegalizerHelper::MIRBuilder
MachineIRBuilder & MIRBuilder
Expose MIRBuilder so clients can set their own RecordInsertInstruction functions.
Definition: LegalizerHelper.h:43
llvm::max
Align max(MaybeAlign Lhs, Align Rhs)
Definition: Alignment.h:340
llvm::MachineIRBuilder::buildMerge
MachineInstrBuilder buildMerge(const DstOp &Res, ArrayRef< Register > Ops)
Build and insert Res = G_MERGE_VALUES Op0, ...
Definition: MachineIRBuilder.cpp:586
llvm::LegalizeRuleSet::custom
LegalizeRuleSet & custom()
Unconditionally custom lower.
Definition: LegalizerInfo.h:834
llvm::SmallVectorImpl
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
Definition: APFloat.h:43
llvm::LegalizeMutations::moreElementsToNextPow2
LegalizeMutation moreElementsToNextPow2(unsigned TypeIdx, unsigned Min=0)
Add more elements to the type for the given type index to the next power of.
Definition: LegalizeMutations.cpp:76
extractParts
static void extractParts(Register Reg, MachineRegisterInfo &MRI, MachineIRBuilder &MIRBuilder, LLT Ty, int NumParts, SmallVectorImpl< Register > &VRegs)
Definition: AArch64LegalizerInfo.cpp:842
DerivedTypes.h
TM
const char LLVMTargetMachineRef TM
Definition: PassBuilderBindings.cpp:47
llvm::LegalizeRuleSet::scalarSameSizeAs
LegalizeRuleSet & scalarSameSizeAs(unsigned TypeIdx, unsigned SameSizeIdx)
Change the type TypeIdx to have the same scalar size as type SameSizeIdx.
Definition: LegalizerInfo.h:989
llvm::MIPatternMatch::mi_match
bool mi_match(Reg R, const MachineRegisterInfo &MRI, Pattern &&P)
Definition: MIPatternMatch.h:24
llvm::AArch64II::MO_TAGGED
@ MO_TAGGED
MO_TAGGED - With MO_PAGE, indicates that the page includes a memory tag in bits 56-63.
Definition: AArch64BaseInfo.h:745
llvm::MachineIRBuilder::buildStore
MachineInstrBuilder buildStore(const SrcOp &Val, const SrcOp &Addr, MachineMemOperand &MMO)
Build and insert G_STORE Val, Addr, MMO.
Definition: MachineIRBuilder.cpp:387
llvm::LLT::getElementType
LLT getElementType() const
Returns the vector's element type. Only valid for vector types.
Definition: LowLevelTypeImpl.h:237
llvm::AArch64Subtarget
Definition: AArch64Subtarget.h:38
llvm::LegalizeRuleSet::libcallFor
LegalizeRuleSet & libcallFor(std::initializer_list< LLT > Types)
Definition: LegalizerInfo.h:724
llvm::LinearPolySize::divideCoefficientBy
LeafTy divideCoefficientBy(ScalarTy RHS) const
We do not provide the '/' operator here because division for polynomial types does not work in the sa...
Definition: TypeSize.h:361
llvm::LLT::scalar
static LLT scalar(unsigned SizeInBits)
Get a low-level scalar or aggregate "bag of bits".
Definition: LowLevelTypeImpl.h:43
llvm::LLT::changeElementSize
LLT changeElementSize(unsigned NewEltSize) const
If this type is a vector, return a vector with the same number of elements but the new element size.
Definition: LowLevelTypeImpl.h:181
llvm::AArch64II::MO_GOT
@ MO_GOT
MO_GOT - This flag indicates that a symbol operand represents the address of the GOT entry for the sy...
Definition: AArch64BaseInfo.h:713
libcall
Common register allocation spilling lr str ldr sxth r3 ldr mla r4 can lr mov lr str ldr sxth r3 mla r4 and then merge mul and lr str ldr sxth r3 mla r4 It also increase the likelihood the store may become dead bb27 Successors according to LLVM ID Predecessors according to mbb< bb27, 0x8b0a7c0 > Note ADDri is not a two address instruction its result reg1037 is an operand of the PHI node in bb76 and its operand reg1039 is the result of the PHI node We should treat it as a two address code and make sure the ADDri is scheduled after any node that reads reg1039 Use info(i.e. register scavenger) to assign it a free register to allow reuse the collector could move the objects and invalidate the derived pointer This is bad enough in the first but safe points can crop up unpredictably **array_addr i32 n y store obj obj **nth_el If the i64 division is lowered to a libcall
Definition: README.txt:127
llvm::LegalizeRuleSet::legalForTypesWithMemDesc
LegalizeRuleSet & legalForTypesWithMemDesc(std::initializer_list< LegalityPredicates::TypePairAndMemDesc > TypesAndMemDesc)
The instruction is legal when type indexes 0 and 1 along with the memory size and minimum alignment i...
Definition: LegalizerInfo.h:602
llvm::AtomicOrdering::NotAtomic
@ NotAtomic
llvm::ISD::CTPOP
@ CTPOP
Definition: ISDOpcodes.h:669
llvm::MachineRegisterInfo::setRegClass
void setRegClass(Register Reg, const TargetRegisterClass *RC)
setRegClass - Set the register class of the specified virtual register.
Definition: MachineRegisterInfo.cpp:58
llvm::sampleprof::Base
@ Base
Definition: Discriminator.h:58
llvm::AArch64II::MO_PAGE
@ MO_PAGE
MO_PAGE - A symbol operand with this flag represents the pc-relative offset of the 4K page containing...
Definition: AArch64BaseInfo.h:677
llvm::LLT
Definition: LowLevelTypeImpl.h:40