LLVM  13.0.0git
AArch64LegalizerInfo.cpp
Go to the documentation of this file.
1 //===- AArch64LegalizerInfo.cpp ----------------------------------*- C++ -*-==//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 /// \file
9 /// This file implements the targeting of the Machinelegalizer class for
10 /// AArch64.
11 /// \todo This should be generated by TableGen.
12 //===----------------------------------------------------------------------===//
13 
14 #include "AArch64LegalizerInfo.h"
15 #include "AArch64Subtarget.h"
24 #include "llvm/IR/DerivedTypes.h"
25 #include "llvm/IR/Type.h"
26 #include <initializer_list>
28 
29 #define DEBUG_TYPE "aarch64-legalinfo"
30 
31 using namespace llvm;
32 using namespace LegalizeActions;
33 using namespace LegalizeMutations;
34 using namespace LegalityPredicates;
35 
37  : ST(&ST) {
38  using namespace TargetOpcode;
39  const LLT p0 = LLT::pointer(0, 64);
40  const LLT s1 = LLT::scalar(1);
41  const LLT s8 = LLT::scalar(8);
42  const LLT s16 = LLT::scalar(16);
43  const LLT s32 = LLT::scalar(32);
44  const LLT s64 = LLT::scalar(64);
45  const LLT s128 = LLT::scalar(128);
46  const LLT s256 = LLT::scalar(256);
47  const LLT s512 = LLT::scalar(512);
48  const LLT v16s8 = LLT::vector(16, 8);
49  const LLT v8s8 = LLT::vector(8, 8);
50  const LLT v4s8 = LLT::vector(4, 8);
51  const LLT v8s16 = LLT::vector(8, 16);
52  const LLT v4s16 = LLT::vector(4, 16);
53  const LLT v2s16 = LLT::vector(2, 16);
54  const LLT v2s32 = LLT::vector(2, 32);
55  const LLT v4s32 = LLT::vector(4, 32);
56  const LLT v2s64 = LLT::vector(2, 64);
57  const LLT v2p0 = LLT::vector(2, p0);
58 
59  std::initializer_list<LLT> PackedVectorAllTypeList = {/* Begin 128bit types */
60  v16s8, v8s16, v4s32,
61  v2s64, v2p0,
62  /* End 128bit types */
63  /* Begin 64bit types */
64  v8s8, v4s16, v2s32};
65 
66  const TargetMachine &TM = ST.getTargetLowering()->getTargetMachine();
67 
68  // FIXME: support subtargets which have neon/fp-armv8 disabled.
69  if (!ST.hasNEON() || !ST.hasFPARMv8()) {
70  computeTables();
71  return;
72  }
73 
74  // Some instructions only support s16 if the subtarget has full 16-bit FP
75  // support.
76  const bool HasFP16 = ST.hasFullFP16();
77  const LLT &MinFPScalar = HasFP16 ? s16 : s32;
78 
79  getActionDefinitionsBuilder({G_IMPLICIT_DEF, G_FREEZE})
80  .legalFor({p0, s1, s8, s16, s32, s64})
81  .legalFor(PackedVectorAllTypeList)
82  .clampScalar(0, s1, s64)
85  [=](const LegalityQuery &Query) {
86  return Query.Types[0].isVector() &&
87  (Query.Types[0].getElementType() != s64 ||
88  Query.Types[0].getNumElements() != 2);
89  },
90  [=](const LegalityQuery &Query) {
91  LLT EltTy = Query.Types[0].getElementType();
92  if (EltTy == s64)
93  return std::make_pair(0, LLT::vector(2, 64));
94  return std::make_pair(0, EltTy);
95  });
96 
97  getActionDefinitionsBuilder(G_PHI).legalFor({p0, s16, s32, s64})
98  .legalFor(PackedVectorAllTypeList)
99  .clampScalar(0, s16, s64)
101 
103  .legalFor({s32, s64, v4s32, v2s32, v2s64})
104  .clampScalar(0, s32, s64)
106 
107  getActionDefinitionsBuilder({G_ADD, G_SUB, G_MUL, G_AND, G_OR, G_XOR})
108  .legalFor({s32, s64, v2s32, v4s32, v4s16, v8s16, v16s8, v8s8})
109  .scalarizeIf(
110  [=](const LegalityQuery &Query) {
111  return Query.Opcode == G_MUL && Query.Types[0] == v2s64;
112  },
113  0)
114  .legalFor({v2s64})
115  .clampScalar(0, s32, s64)
117  .clampNumElements(0, v2s32, v4s32)
118  .clampNumElements(0, v2s64, v2s64)
120 
121  getActionDefinitionsBuilder({G_SHL, G_ASHR, G_LSHR})
122  .customIf([=](const LegalityQuery &Query) {
123  const auto &SrcTy = Query.Types[0];
124  const auto &AmtTy = Query.Types[1];
125  return !SrcTy.isVector() && SrcTy.getSizeInBits() == 32 &&
126  AmtTy.getSizeInBits() == 32;
127  })
128  .legalFor({
129  {s32, s32},
130  {s32, s64},
131  {s64, s64},
132  {v8s8, v8s8},
133  {v16s8, v16s8},
134  {v4s16, v4s16},
135  {v8s16, v8s16},
136  {v2s32, v2s32},
137  {v4s32, v4s32},
138  {v2s64, v2s64},
139  })
140  .clampScalar(1, s32, s64)
141  .clampScalar(0, s32, s64)
143  .clampNumElements(0, v2s32, v4s32)
144  .clampNumElements(0, v2s64, v2s64)
146  .minScalarSameAs(1, 0);
147 
148  getActionDefinitionsBuilder(G_PTR_ADD)
149  .legalFor({{p0, s64}, {v2p0, v2s64}})
150  .clampScalar(1, s64, s64);
151 
152  getActionDefinitionsBuilder(G_PTRMASK).legalFor({{p0, s64}});
153 
154  getActionDefinitionsBuilder({G_SDIV, G_UDIV})
155  .legalFor({s32, s64})
156  .libcallFor({s128})
157  .clampScalar(0, s32, s64)
159  .scalarize(0);
160 
161  getActionDefinitionsBuilder({G_SREM, G_UREM})
162  .lowerFor({s1, s8, s16, s32, s64});
163 
164  getActionDefinitionsBuilder({G_SMULO, G_UMULO}).lowerFor({{s64, s1}});
165 
166  getActionDefinitionsBuilder({G_SMULH, G_UMULH}).legalFor({s32, s64});
167 
168  getActionDefinitionsBuilder({G_SMIN, G_SMAX, G_UMIN, G_UMAX})
169  .lowerIf([=](const LegalityQuery &Q) { return Q.Types[0].isScalar(); });
170 
172  {G_SADDE, G_SSUBE, G_UADDE, G_USUBE, G_SADDO, G_SSUBO, G_UADDO, G_USUBO})
173  .legalFor({{s32, s1}, {s64, s1}})
174  .clampScalar(0, s32, s64)
176 
177  getActionDefinitionsBuilder({G_FADD, G_FSUB, G_FMUL, G_FDIV, G_FNEG})
178  .legalFor({s32, s64, v2s64, v4s32, v2s32})
179  .clampNumElements(0, v2s32, v4s32)
180  .clampNumElements(0, v2s64, v2s64);
181 
182  getActionDefinitionsBuilder(G_FREM).libcallFor({s32, s64});
183 
184  getActionDefinitionsBuilder({G_FCEIL, G_FABS, G_FSQRT, G_FFLOOR, G_FRINT,
185  G_FMA, G_INTRINSIC_TRUNC, G_INTRINSIC_ROUND,
186  G_FNEARBYINT, G_INTRINSIC_LRINT})
187  // If we don't have full FP16 support, then scalarize the elements of
188  // vectors containing fp16 types.
189  .fewerElementsIf(
190  [=, &ST](const LegalityQuery &Query) {
191  const auto &Ty = Query.Types[0];
192  return Ty.isVector() && Ty.getElementType() == s16 &&
193  !ST.hasFullFP16();
194  },
195  [=](const LegalityQuery &Query) { return std::make_pair(0, s16); })
196  // If we don't have full FP16 support, then widen s16 to s32 if we
197  // encounter it.
198  .widenScalarIf(
199  [=, &ST](const LegalityQuery &Query) {
200  return Query.Types[0] == s16 && !ST.hasFullFP16();
201  },
202  [=](const LegalityQuery &Query) { return std::make_pair(0, s32); })
203  .legalFor({s16, s32, s64, v2s32, v4s32, v2s64, v2s16, v4s16, v8s16});
204 
206  {G_FCOS, G_FSIN, G_FLOG10, G_FLOG, G_FLOG2, G_FEXP, G_FEXP2, G_FPOW})
207  // We need a call for these, so we always need to scalarize.
208  .scalarize(0)
209  // Regardless of FP16 support, widen 16-bit elements to 32-bits.
210  .minScalar(0, s32)
211  .libcallFor({s32, s64, v2s32, v4s32, v2s64});
212 
214  .unsupportedIf([=](const LegalityQuery &Query) {
215  return Query.Types[0].getSizeInBits() <= Query.Types[1].getSizeInBits();
216  })
217  .legalIf([=](const LegalityQuery &Query) {
218  const LLT &Ty0 = Query.Types[0];
219  const LLT &Ty1 = Query.Types[1];
220  if (Ty0 != s32 && Ty0 != s64 && Ty0 != p0)
221  return false;
222  return isPowerOf2_32(Ty1.getSizeInBits()) &&
223  (Ty1.getSizeInBits() == 1 || Ty1.getSizeInBits() >= 8);
224  })
225  .clampScalar(0, s32, s64)
227  .maxScalarIf(typeInSet(0, {s32}), 1, s16)
228  .maxScalarIf(typeInSet(0, {s64}), 1, s32)
230 
231  getActionDefinitionsBuilder(G_EXTRACT)
232  .unsupportedIf([=](const LegalityQuery &Query) {
233  return Query.Types[0].getSizeInBits() >= Query.Types[1].getSizeInBits();
234  })
235  .legalIf([=](const LegalityQuery &Query) {
236  const LLT &Ty0 = Query.Types[0];
237  const LLT &Ty1 = Query.Types[1];
238  if (Ty1 != s32 && Ty1 != s64 && Ty1 != s128)
239  return false;
240  if (Ty1 == p0)
241  return true;
242  return isPowerOf2_32(Ty0.getSizeInBits()) &&
243  (Ty0.getSizeInBits() == 1 || Ty0.getSizeInBits() >= 8);
244  })
245  .clampScalar(1, s32, s128)
247  .maxScalarIf(typeInSet(1, {s32}), 0, s16)
248  .maxScalarIf(typeInSet(1, {s64}), 0, s32)
250 
251  getActionDefinitionsBuilder({G_SEXTLOAD, G_ZEXTLOAD})
252  .legalForTypesWithMemDesc({{s32, p0, 8, 8},
253  {s32, p0, 16, 8},
254  {s32, p0, 32, 8},
255  {s64, p0, 8, 2},
256  {s64, p0, 16, 2},
257  {s64, p0, 32, 4},
258  {s64, p0, 64, 8},
259  {p0, p0, 64, 8},
260  {v2s32, p0, 64, 8}})
261  .clampScalar(0, s32, s64)
263  // TODO: We could support sum-of-pow2's but the lowering code doesn't know
264  // how to do that yet.
266  // Lower anything left over into G_*EXT and G_LOAD
267  .lower();
268 
269  auto IsPtrVecPred = [=](const LegalityQuery &Query) {
270  const LLT &ValTy = Query.Types[0];
271  if (!ValTy.isVector())
272  return false;
273  const LLT EltTy = ValTy.getElementType();
274  return EltTy.isPointer() && EltTy.getAddressSpace() == 0;
275  };
276 
278  .legalForTypesWithMemDesc({{s8, p0, 8, 8},
279  {s16, p0, 16, 8},
280  {s32, p0, 32, 8},
281  {s64, p0, 64, 8},
282  {p0, p0, 64, 8},
283  {s128, p0, 128, 8},
284  {v8s8, p0, 64, 8},
285  {v16s8, p0, 128, 8},
286  {v4s16, p0, 64, 8},
287  {v8s16, p0, 128, 8},
288  {v2s32, p0, 64, 8},
289  {v4s32, p0, 128, 8},
290  {v2s64, p0, 128, 8}})
291  // These extends are also legal
292  .legalForTypesWithMemDesc({{s32, p0, 8, 8}, {s32, p0, 16, 8}})
293  .clampScalar(0, s8, s64)
295  // Lower any any-extending loads left into G_ANYEXT and G_LOAD
296  .lowerIf([=](const LegalityQuery &Query) {
297  return Query.Types[0].getSizeInBits() != Query.MMODescrs[0].SizeInBits;
298  })
299  .widenScalarToNextPow2(0)
300  .clampMaxNumElements(0, s8, 16)
301  .clampMaxNumElements(0, s16, 8)
302  .clampMaxNumElements(0, s32, 4)
303  .clampMaxNumElements(0, s64, 2)
304  .customIf(IsPtrVecPred);
305 
307  .legalForTypesWithMemDesc({{s8, p0, 8, 8},
308  {s16, p0, 16, 8},
309  {s32, p0, 8, 8},
310  {s32, p0, 16, 8},
311  {s32, p0, 32, 8},
312  {s64, p0, 64, 8},
313  {p0, p0, 64, 8},
314  {s128, p0, 128, 8},
315  {v16s8, p0, 128, 8},
316  {v8s8, p0, 64, 8},
317  {v4s16, p0, 64, 8},
318  {v8s16, p0, 128, 8},
319  {v2s32, p0, 64, 8},
320  {v4s32, p0, 128, 8},
321  {v2s64, p0, 128, 8}})
322  .clampScalar(0, s8, s64)
324  .lowerIf([=](const LegalityQuery &Query) {
325  return Query.Types[0].isScalar() &&
326  Query.Types[0].getSizeInBits() != Query.MMODescrs[0].SizeInBits;
327  })
328  // Maximum: sN * k = 128
329  .clampMaxNumElements(0, s8, 16)
330  .clampMaxNumElements(0, s16, 8)
331  .clampMaxNumElements(0, s32, 4)
332  .clampMaxNumElements(0, s64, 2)
333  .customIf(IsPtrVecPred);
334 
335  // Constants
336  getActionDefinitionsBuilder(G_CONSTANT)
337  .legalFor({p0, s8, s16, s32, s64})
338  .clampScalar(0, s8, s64)
340  getActionDefinitionsBuilder(G_FCONSTANT)
341  .legalIf([=](const LegalityQuery &Query) {
342  const auto &Ty = Query.Types[0];
343  if (HasFP16 && Ty == s16)
344  return true;
345  return Ty == s32 || Ty == s64 || Ty == s128;
346  })
347  .clampScalar(0, MinFPScalar, s128);
348 
349  getActionDefinitionsBuilder({G_ICMP, G_FCMP})
350  .legalFor({{s32, s32},
351  {s32, s64},
352  {s32, p0},
353  {v4s32, v4s32},
354  {v2s32, v2s32},
355  {v2s64, v2s64},
356  {v2s64, v2p0},
357  {v4s16, v4s16},
358  {v8s16, v8s16},
359  {v8s8, v8s8},
360  {v16s8, v16s8}})
361  .clampScalar(1, s32, s64)
362  .clampScalar(0, s32, s32)
364  [=](const LegalityQuery &Query) {
365  const LLT &Ty = Query.Types[0];
366  const LLT &SrcTy = Query.Types[1];
367  return Ty.isVector() && !SrcTy.getElementType().isPointer() &&
368  Ty.getElementType() != SrcTy.getElementType();
369  },
370  0, 1)
371  .minScalarOrEltIf(
372  [=](const LegalityQuery &Query) { return Query.Types[1] == v2s16; },
373  1, s32)
374  .minScalarOrEltIf(
375  [=](const LegalityQuery &Query) { return Query.Types[1] == v2p0; }, 0,
376  s64)
378  .clampNumElements(0, v2s32, v4s32);
379 
380  // Extensions
381  auto ExtLegalFunc = [=](const LegalityQuery &Query) {
382  unsigned DstSize = Query.Types[0].getSizeInBits();
383 
384  if (DstSize == 128 && !Query.Types[0].isVector())
385  return false; // Extending to a scalar s128 needs narrowing.
386 
387  // Make sure that we have something that will fit in a register, and
388  // make sure it's a power of 2.
389  if (DstSize < 8 || DstSize > 128 || !isPowerOf2_32(DstSize))
390  return false;
391 
392  const LLT &SrcTy = Query.Types[1];
393 
394  // Special case for s1.
395  if (SrcTy == s1)
396  return true;
397 
398  // Make sure we fit in a register otherwise. Don't bother checking that
399  // the source type is below 128 bits. We shouldn't be allowing anything
400  // through which is wider than the destination in the first place.
401  unsigned SrcSize = SrcTy.getSizeInBits();
402  if (SrcSize < 8 || !isPowerOf2_32(SrcSize))
403  return false;
404 
405  return true;
406  };
407  getActionDefinitionsBuilder({G_ZEXT, G_SEXT, G_ANYEXT})
408  .legalIf(ExtLegalFunc)
409  .clampScalar(0, s64, s64); // Just for s128, others are handled above.
410 
413  [=](const LegalityQuery &Query) { return Query.Types[0].isVector(); },
414  0, s8)
415  .customIf([=](const LegalityQuery &Query) {
416  LLT DstTy = Query.Types[0];
417  LLT SrcTy = Query.Types[1];
418  return DstTy == v8s8 && SrcTy.getSizeInBits() > 128;
419  })
420  .alwaysLegal();
421 
422  getActionDefinitionsBuilder(G_SEXT_INREG).legalFor({s32, s64}).lower();
423 
424  // FP conversions
425  getActionDefinitionsBuilder(G_FPTRUNC)
426  .legalFor(
427  {{s16, s32}, {s16, s64}, {s32, s64}, {v4s16, v4s32}, {v2s32, v2s64}})
428  .clampMaxNumElements(0, s32, 2);
430  .legalFor(
431  {{s32, s16}, {s64, s16}, {s64, s32}, {v4s32, v4s16}, {v2s64, v2s32}})
432  .clampMaxNumElements(0, s64, 2);
433 
434  // Conversions
435  getActionDefinitionsBuilder({G_FPTOSI, G_FPTOUI})
436  .legalForCartesianProduct({s32, s64, v2s64, v4s32, v2s32})
437  .clampScalar(0, s32, s64)
439  .clampScalar(1, s32, s64)
441 
442  getActionDefinitionsBuilder({G_SITOFP, G_UITOFP})
443  .legalForCartesianProduct({s32, s64, v2s64, v4s32, v2s32})
444  .clampScalar(1, s32, s64)
445  .minScalarSameAs(1, 0)
446  .clampScalar(0, s32, s64)
448 
449  // Control-flow
450  getActionDefinitionsBuilder(G_BRCOND).legalFor({s1, s8, s16, s32});
451  getActionDefinitionsBuilder(G_BRINDIRECT).legalFor({p0});
452 
454  .legalFor({{s32, s1}, {s64, s1}, {p0, s1}})
455  .clampScalar(0, s32, s64)
457  .minScalarEltSameAsIf(all(isVector(0), isVector(1)), 1, 0)
458  .lowerIf(isVector(0));
459 
460  // Pointer-handling
461  getActionDefinitionsBuilder(G_FRAME_INDEX).legalFor({p0});
462 
463  if (TM.getCodeModel() == CodeModel::Small)
464  getActionDefinitionsBuilder(G_GLOBAL_VALUE).custom();
465  else
466  getActionDefinitionsBuilder(G_GLOBAL_VALUE).legalFor({p0});
467 
468  getActionDefinitionsBuilder(G_PTRTOINT)
469  .legalForCartesianProduct({s1, s8, s16, s32, s64}, {p0})
470  .maxScalar(0, s64)
471  .widenScalarToNextPow2(0, /*Min*/ 8);
472 
473  getActionDefinitionsBuilder(G_INTTOPTR)
474  .unsupportedIf([&](const LegalityQuery &Query) {
475  return Query.Types[0].getSizeInBits() != Query.Types[1].getSizeInBits();
476  })
477  .legalFor({{p0, s64}});
478 
479  // Casts for 32 and 64-bit width type are just copies.
480  // Same for 128-bit width type, except they are on the FPR bank.
481  getActionDefinitionsBuilder(G_BITCAST)
482  // FIXME: This is wrong since G_BITCAST is not allowed to change the
483  // number of bits but it's what the previous code described and fixing
484  // it breaks tests.
485  .legalForCartesianProduct({s1, s8, s16, s32, s64, s128, v16s8, v8s8, v4s8,
486  v8s16, v4s16, v2s16, v4s32, v2s32, v2s64,
487  v2p0});
488 
489  getActionDefinitionsBuilder(G_VASTART).legalFor({p0});
490 
491  // va_list must be a pointer, but most sized types are pretty easy to handle
492  // as the destination.
494  .customForCartesianProduct({s8, s16, s32, s64, p0}, {p0})
495  .clampScalar(0, s8, s64)
496  .widenScalarToNextPow2(0, /*Min*/ 8);
497 
498  if (ST.hasLSE()) {
499  getActionDefinitionsBuilder(G_ATOMIC_CMPXCHG_WITH_SUCCESS)
500  .lowerIf(all(
501  typeInSet(0, {s8, s16, s32, s64}), typeIs(1, s1), typeIs(2, p0),
503 
505  {G_ATOMICRMW_XCHG, G_ATOMICRMW_ADD, G_ATOMICRMW_SUB, G_ATOMICRMW_AND,
506  G_ATOMICRMW_OR, G_ATOMICRMW_XOR, G_ATOMICRMW_MIN, G_ATOMICRMW_MAX,
507  G_ATOMICRMW_UMIN, G_ATOMICRMW_UMAX, G_ATOMIC_CMPXCHG})
508  .legalIf(all(
509  typeInSet(0, {s8, s16, s32, s64}), typeIs(1, p0),
511  }
512 
513  getActionDefinitionsBuilder(G_BLOCK_ADDR).legalFor({p0});
514 
515  // Merge/Unmerge
516  for (unsigned Op : {G_MERGE_VALUES, G_UNMERGE_VALUES}) {
517  unsigned BigTyIdx = Op == G_MERGE_VALUES ? 0 : 1;
518  unsigned LitTyIdx = Op == G_MERGE_VALUES ? 1 : 0;
519 
520  auto notValidElt = [](const LegalityQuery &Query, unsigned TypeIdx) {
521  const LLT &Ty = Query.Types[TypeIdx];
522  if (Ty.isVector()) {
523  const LLT &EltTy = Ty.getElementType();
524  if (EltTy.getSizeInBits() < 8 || EltTy.getSizeInBits() > 64)
525  return true;
526  if (!isPowerOf2_32(EltTy.getSizeInBits()))
527  return true;
528  }
529  return false;
530  };
531 
532  // FIXME: This rule is horrible, but specifies the same as what we had
533  // before with the particularly strange definitions removed (e.g.
534  // s8 = G_MERGE_VALUES s32, s32).
535  // Part of the complexity comes from these ops being extremely flexible. For
536  // example, you can build/decompose vectors with it, concatenate vectors,
537  // etc. and in addition to this you can also bitcast with it at the same
538  // time. We've been considering breaking it up into multiple ops to make it
539  // more manageable throughout the backend.
541  // Break up vectors with weird elements into scalars
543  [=](const LegalityQuery &Query) { return notValidElt(Query, 0); },
544  scalarize(0))
545  .fewerElementsIf(
546  [=](const LegalityQuery &Query) { return notValidElt(Query, 1); },
547  scalarize(1))
548  // Clamp the big scalar to s8-s512 and make it either a power of 2, 192,
549  // or 384.
550  .clampScalar(BigTyIdx, s8, s512)
551  .widenScalarIf(
552  [=](const LegalityQuery &Query) {
553  const LLT &Ty = Query.Types[BigTyIdx];
554  return !isPowerOf2_32(Ty.getSizeInBits()) &&
555  Ty.getSizeInBits() % 64 != 0;
556  },
557  [=](const LegalityQuery &Query) {
558  // Pick the next power of 2, or a multiple of 64 over 128.
559  // Whichever is smaller.
560  const LLT &Ty = Query.Types[BigTyIdx];
561  unsigned NewSizeInBits = 1
562  << Log2_32_Ceil(Ty.getSizeInBits() + 1);
563  if (NewSizeInBits >= 256) {
564  unsigned RoundedTo = alignTo<64>(Ty.getSizeInBits() + 1);
565  if (RoundedTo < NewSizeInBits)
566  NewSizeInBits = RoundedTo;
567  }
568  return std::make_pair(BigTyIdx, LLT::scalar(NewSizeInBits));
569  })
570  // Clamp the little scalar to s8-s256 and make it a power of 2. It's not
571  // worth considering the multiples of 64 since 2*192 and 2*384 are not
572  // valid.
573  .clampScalar(LitTyIdx, s8, s256)
574  .widenScalarToNextPow2(LitTyIdx, /*Min*/ 8)
575  // So at this point, we have s8, s16, s32, s64, s128, s192, s256, s384,
576  // s512, <X x s8>, <X x s16>, <X x s32>, or <X x s64>.
577  // At this point it's simple enough to accept the legal types.
578  .legalIf([=](const LegalityQuery &Query) {
579  const LLT &BigTy = Query.Types[BigTyIdx];
580  const LLT &LitTy = Query.Types[LitTyIdx];
581  if (BigTy.isVector() && BigTy.getSizeInBits() < 32)
582  return false;
583  if (LitTy.isVector() && LitTy.getSizeInBits() < 32)
584  return false;
585  return BigTy.getSizeInBits() % LitTy.getSizeInBits() == 0;
586  })
587  // Any vectors left are the wrong size. Scalarize them.
588  .scalarize(0)
589  .scalarize(1);
590  }
591 
592  getActionDefinitionsBuilder(G_EXTRACT_VECTOR_ELT)
593  .unsupportedIf([=](const LegalityQuery &Query) {
594  const LLT &EltTy = Query.Types[1].getElementType();
595  return Query.Types[0] != EltTy;
596  })
597  .minScalar(2, s64)
598  .legalIf([=](const LegalityQuery &Query) {
599  const LLT &VecTy = Query.Types[1];
600  return VecTy == v2s16 || VecTy == v4s16 || VecTy == v8s16 ||
601  VecTy == v4s32 || VecTy == v2s64 || VecTy == v2s32 ||
602  VecTy == v16s8 || VecTy == v2s32 || VecTy == v2p0;
603  })
604  .minScalarOrEltIf(
605  [=](const LegalityQuery &Query) {
606  // We want to promote to <M x s1> to <M x s64> if that wouldn't
607  // cause the total vec size to be > 128b.
608  return Query.Types[1].getNumElements() <= 2;
609  },
610  0, s64)
611  .minScalarOrEltIf(
612  [=](const LegalityQuery &Query) {
613  return Query.Types[1].getNumElements() <= 4;
614  },
615  0, s32)
616  .minScalarOrEltIf(
617  [=](const LegalityQuery &Query) {
618  return Query.Types[1].getNumElements() <= 8;
619  },
620  0, s16)
621  .minScalarOrEltIf(
622  [=](const LegalityQuery &Query) {
623  return Query.Types[1].getNumElements() <= 16;
624  },
625  0, s8)
626  .minScalarOrElt(0, s8); // Worst case, we need at least s8.
627 
628  getActionDefinitionsBuilder(G_INSERT_VECTOR_ELT)
629  .legalIf(typeInSet(0, {v8s16, v2s32, v4s32, v2s64}));
630 
631  getActionDefinitionsBuilder(G_BUILD_VECTOR)
632  .legalFor({{v8s8, s8},
633  {v16s8, s8},
634  {v4s16, s16},
635  {v8s16, s16},
636  {v2s32, s32},
637  {v4s32, s32},
638  {v2p0, p0},
639  {v2s64, s64}})
640  .clampNumElements(0, v4s32, v4s32)
641  .clampNumElements(0, v2s64, v2s64)
642  .minScalarSameAs(1, 0);
643 
644  getActionDefinitionsBuilder(G_BUILD_VECTOR_TRUNC).lower();
645 
648  {s32, s64, v8s8, v16s8, v4s16, v8s16, v2s32, v4s32})
649  .scalarize(1);
650  getActionDefinitionsBuilder(G_CTLZ_ZERO_UNDEF).lower();
651 
652  getActionDefinitionsBuilder(G_SHUFFLE_VECTOR)
653  .legalIf([=](const LegalityQuery &Query) {
654  const LLT &DstTy = Query.Types[0];
655  const LLT &SrcTy = Query.Types[1];
656  // For now just support the TBL2 variant which needs the source vectors
657  // to be the same size as the dest.
658  if (DstTy != SrcTy)
659  return false;
660  for (auto &Ty : {v2s32, v4s32, v2s64, v2p0, v16s8, v8s16}) {
661  if (DstTy == Ty)
662  return true;
663  }
664  return false;
665  })
666  // G_SHUFFLE_VECTOR can have scalar sources (from 1 x s vectors), we
667  // just want those lowered into G_BUILD_VECTOR
668  .lowerIf([=](const LegalityQuery &Query) {
669  return !Query.Types[1].isVector();
670  })
671  .clampNumElements(0, v4s32, v4s32)
672  .clampNumElements(0, v2s64, v2s64);
673 
674  getActionDefinitionsBuilder(G_CONCAT_VECTORS)
675  .legalFor({{v4s32, v2s32}, {v8s16, v4s16}});
676 
677  getActionDefinitionsBuilder(G_JUMP_TABLE).legalFor({{p0}, {s64}});
678 
679  getActionDefinitionsBuilder(G_BRJT).legalIf([=](const LegalityQuery &Query) {
680  return Query.Types[0] == p0 && Query.Types[1] == s64;
681  });
682 
683  getActionDefinitionsBuilder(G_DYN_STACKALLOC).lower();
684 
685  getActionDefinitionsBuilder({G_BZERO, G_MEMCPY, G_MEMMOVE, G_MEMSET})
686  .libcall();
687 
689  [=](const LegalityQuery &Query) { return Query.Types[0].isScalar(); });
690 
691  getActionDefinitionsBuilder(G_VECREDUCE_FADD)
692  // We only have FADDP to do reduction-like operations. Lower the rest.
693  .legalFor({{s32, v2s32}, {s64, v2s64}})
694  .clampMaxNumElements(1, s64, 2)
695  .clampMaxNumElements(1, s32, 2)
696  .lower();
697 
698  getActionDefinitionsBuilder(G_VECREDUCE_ADD)
699  .legalFor(
700  {{s8, v16s8}, {s16, v8s16}, {s32, v4s32}, {s32, v2s32}, {s64, v2s64}})
701  .clampMaxNumElements(1, s64, 2)
702  .clampMaxNumElements(1, s32, 4)
703  .lower();
704 
705  getActionDefinitionsBuilder({G_UADDSAT, G_USUBSAT})
706  .lowerIf([=](const LegalityQuery &Q) { return Q.Types[0].isScalar(); });
707 
708  getActionDefinitionsBuilder({G_FSHL, G_FSHR}).lower();
709 
711  .legalFor({{s32, s64}, {s64, s64}})
712  .customIf([=](const LegalityQuery &Q) {
713  return Q.Types[0].isScalar() && Q.Types[1].getScalarSizeInBits() < 64;
714  })
715  .lower();
717 
718  getActionDefinitionsBuilder({G_SBFX, G_UBFX})
719  .customFor({{s32, s32}, {s64, s64}});
720 
721  getActionDefinitionsBuilder(G_CTPOP).legalFor({{v8s8, v8s8}, {v16s8, v16s8}});
722 
723  computeTables();
724  verify(*ST.getInstrInfo());
725 }
726 
728  MachineInstr &MI) const {
729  MachineIRBuilder &MIRBuilder = Helper.MIRBuilder;
730  MachineRegisterInfo &MRI = *MIRBuilder.getMRI();
731  GISelChangeObserver &Observer = Helper.Observer;
732  switch (MI.getOpcode()) {
733  default:
734  // No idea what to do.
735  return false;
736  case TargetOpcode::G_VAARG:
737  return legalizeVaArg(MI, MRI, MIRBuilder);
738  case TargetOpcode::G_LOAD:
739  case TargetOpcode::G_STORE:
740  return legalizeLoadStore(MI, MRI, MIRBuilder, Observer);
741  case TargetOpcode::G_SHL:
742  case TargetOpcode::G_ASHR:
743  case TargetOpcode::G_LSHR:
744  return legalizeShlAshrLshr(MI, MRI, MIRBuilder, Observer);
745  case TargetOpcode::G_GLOBAL_VALUE:
746  return legalizeSmallCMGlobalValue(MI, MRI, MIRBuilder, Observer);
747  case TargetOpcode::G_TRUNC:
748  return legalizeVectorTrunc(MI, Helper);
749  case TargetOpcode::G_SBFX:
750  case TargetOpcode::G_UBFX:
751  return legalizeBitfieldExtract(MI, MRI, Helper);
752  case TargetOpcode::G_ROTR:
753  return legalizeRotate(MI, MRI, Helper);
754  }
755 
756  llvm_unreachable("expected switch to return");
757 }
758 
759 bool AArch64LegalizerInfo::legalizeRotate(MachineInstr &MI,
761  LegalizerHelper &Helper) const {
762  // To allow for imported patterns to match, we ensure that the rotate amount
763  // is 64b with an extension.
764  Register AmtReg = MI.getOperand(2).getReg();
765  LLT AmtTy = MRI.getType(AmtReg);
766  (void)AmtTy;
767  assert(AmtTy.isScalar() && "Expected a scalar rotate");
768  assert(AmtTy.getSizeInBits() < 64 && "Expected this rotate to be legal");
769  auto NewAmt = Helper.MIRBuilder.buildSExt(LLT::scalar(64), AmtReg);
770  Helper.Observer.changingInstr(MI);
771  MI.getOperand(2).setReg(NewAmt.getReg(0));
772  Helper.Observer.changedInstr(MI);
773  return true;
774 }
775 
777  MachineIRBuilder &MIRBuilder, LLT Ty, int NumParts,
778  SmallVectorImpl<Register> &VRegs) {
779  for (int I = 0; I < NumParts; ++I)
780  VRegs.push_back(MRI.createGenericVirtualRegister(Ty));
781  MIRBuilder.buildUnmerge(VRegs, Reg);
782 }
783 
784 bool AArch64LegalizerInfo::legalizeVectorTrunc(
785  MachineInstr &MI, LegalizerHelper &Helper) const {
786  MachineIRBuilder &MIRBuilder = Helper.MIRBuilder;
787  MachineRegisterInfo &MRI = *MIRBuilder.getMRI();
788  // Similar to how operand splitting is done in SelectiondDAG, we can handle
789  // %res(v8s8) = G_TRUNC %in(v8s32) by generating:
790  // %inlo(<4x s32>), %inhi(<4 x s32>) = G_UNMERGE %in(<8 x s32>)
791  // %lo16(<4 x s16>) = G_TRUNC %inlo
792  // %hi16(<4 x s16>) = G_TRUNC %inhi
793  // %in16(<8 x s16>) = G_CONCAT_VECTORS %lo16, %hi16
794  // %res(<8 x s8>) = G_TRUNC %in16
795 
796  Register DstReg = MI.getOperand(0).getReg();
797  Register SrcReg = MI.getOperand(1).getReg();
798  LLT DstTy = MRI.getType(DstReg);
799  LLT SrcTy = MRI.getType(SrcReg);
801  isPowerOf2_32(SrcTy.getSizeInBits()));
802 
803  // Split input type.
804  LLT SplitSrcTy = SrcTy.changeNumElements(SrcTy.getNumElements() / 2);
805  // First, split the source into two smaller vectors.
806  SmallVector<Register, 2> SplitSrcs;
807  extractParts(SrcReg, MRI, MIRBuilder, SplitSrcTy, 2, SplitSrcs);
808 
809  // Truncate the splits into intermediate narrower elements.
810  LLT InterTy = SplitSrcTy.changeElementSize(DstTy.getScalarSizeInBits() * 2);
811  for (unsigned I = 0; I < SplitSrcs.size(); ++I)
812  SplitSrcs[I] = MIRBuilder.buildTrunc(InterTy, SplitSrcs[I]).getReg(0);
813 
814  auto Concat = MIRBuilder.buildConcatVectors(
815  DstTy.changeElementSize(DstTy.getScalarSizeInBits() * 2), SplitSrcs);
816 
817  Helper.Observer.changingInstr(MI);
818  MI.getOperand(1).setReg(Concat.getReg(0));
819  Helper.Observer.changedInstr(MI);
820  return true;
821 }
822 
823 bool AArch64LegalizerInfo::legalizeSmallCMGlobalValue(
825  GISelChangeObserver &Observer) const {
826  assert(MI.getOpcode() == TargetOpcode::G_GLOBAL_VALUE);
827  // We do this custom legalization to convert G_GLOBAL_VALUE into target ADRP +
828  // G_ADD_LOW instructions.
829  // By splitting this here, we can optimize accesses in the small code model by
830  // folding in the G_ADD_LOW into the load/store offset.
831  auto &GlobalOp = MI.getOperand(1);
832  const auto* GV = GlobalOp.getGlobal();
833  if (GV->isThreadLocal())
834  return true; // Don't want to modify TLS vars.
835 
836  auto &TM = ST->getTargetLowering()->getTargetMachine();
837  unsigned OpFlags = ST->ClassifyGlobalReference(GV, TM);
838 
839  if (OpFlags & AArch64II::MO_GOT)
840  return true;
841 
842  auto Offset = GlobalOp.getOffset();
843  Register DstReg = MI.getOperand(0).getReg();
844  auto ADRP = MIRBuilder.buildInstr(AArch64::ADRP, {LLT::pointer(0, 64)}, {})
845  .addGlobalAddress(GV, Offset, OpFlags | AArch64II::MO_PAGE);
846  // Set the regclass on the dest reg too.
847  MRI.setRegClass(ADRP.getReg(0), &AArch64::GPR64RegClass);
848 
849  // MO_TAGGED on the page indicates a tagged address. Set the tag now. We do so
850  // by creating a MOVK that sets bits 48-63 of the register to (global address
851  // + 0x100000000 - PC) >> 48. The additional 0x100000000 offset here is to
852  // prevent an incorrect tag being generated during relocation when the the
853  // global appears before the code section. Without the offset, a global at
854  // `0x0f00'0000'0000'1000` (i.e. at `0x1000` with tag `0xf`) that's referenced
855  // by code at `0x2000` would result in `0x0f00'0000'0000'1000 - 0x2000 =
856  // 0x0eff'ffff'ffff'f000`, meaning the tag would be incorrectly set to `0xe`
857  // instead of `0xf`.
858  // This assumes that we're in the small code model so we can assume a binary
859  // size of <= 4GB, which makes the untagged PC relative offset positive. The
860  // binary must also be loaded into address range [0, 2^48). Both of these
861  // properties need to be ensured at runtime when using tagged addresses.
862  if (OpFlags & AArch64II::MO_TAGGED) {
863  assert(!Offset &&
864  "Should not have folded in an offset for a tagged global!");
865  ADRP = MIRBuilder.buildInstr(AArch64::MOVKXi, {LLT::pointer(0, 64)}, {ADRP})
866  .addGlobalAddress(GV, 0x100000000,
868  .addImm(48);
869  MRI.setRegClass(ADRP.getReg(0), &AArch64::GPR64RegClass);
870  }
871 
872  MIRBuilder.buildInstr(AArch64::G_ADD_LOW, {DstReg}, {ADRP})
873  .addGlobalAddress(GV, Offset,
875  MI.eraseFromParent();
876  return true;
877 }
878 
880  MachineInstr &MI) const {
881  return true;
882 }
883 
884 bool AArch64LegalizerInfo::legalizeShlAshrLshr(
886  GISelChangeObserver &Observer) const {
887  assert(MI.getOpcode() == TargetOpcode::G_ASHR ||
888  MI.getOpcode() == TargetOpcode::G_LSHR ||
889  MI.getOpcode() == TargetOpcode::G_SHL);
890  // If the shift amount is a G_CONSTANT, promote it to a 64 bit type so the
891  // imported patterns can select it later. Either way, it will be legal.
892  Register AmtReg = MI.getOperand(2).getReg();
893  auto VRegAndVal = getConstantVRegValWithLookThrough(AmtReg, MRI);
894  if (!VRegAndVal)
895  return true;
896  // Check the shift amount is in range for an immediate form.
897  int64_t Amount = VRegAndVal->Value.getSExtValue();
898  if (Amount > 31)
899  return true; // This will have to remain a register variant.
900  auto ExtCst = MIRBuilder.buildConstant(LLT::scalar(64), Amount);
901  Observer.changingInstr(MI);
902  MI.getOperand(2).setReg(ExtCst.getReg(0));
903  Observer.changedInstr(MI);
904  return true;
905 }
906 
907 bool AArch64LegalizerInfo::legalizeLoadStore(
909  GISelChangeObserver &Observer) const {
910  assert(MI.getOpcode() == TargetOpcode::G_STORE ||
911  MI.getOpcode() == TargetOpcode::G_LOAD);
912  // Here we just try to handle vector loads/stores where our value type might
913  // have pointer elements, which the SelectionDAG importer can't handle. To
914  // allow the existing patterns for s64 to fire for p0, we just try to bitcast
915  // the value to use s64 types.
916 
917  // Custom legalization requires the instruction, if not deleted, must be fully
918  // legalized. In order to allow further legalization of the inst, we create
919  // a new instruction and erase the existing one.
920 
921  Register ValReg = MI.getOperand(0).getReg();
922  const LLT ValTy = MRI.getType(ValReg);
923 
924  if (!ValTy.isVector() || !ValTy.getElementType().isPointer() ||
925  ValTy.getElementType().getAddressSpace() != 0) {
926  LLVM_DEBUG(dbgs() << "Tried to do custom legalization on wrong load/store");
927  return false;
928  }
929 
930  unsigned PtrSize = ValTy.getElementType().getSizeInBits();
931  const LLT NewTy = LLT::vector(ValTy.getNumElements(), PtrSize);
932  auto &MMO = **MI.memoperands_begin();
933  if (MI.getOpcode() == TargetOpcode::G_STORE) {
934  auto Bitcast = MIRBuilder.buildBitcast(NewTy, ValReg);
935  MIRBuilder.buildStore(Bitcast.getReg(0), MI.getOperand(1), MMO);
936  } else {
937  auto NewLoad = MIRBuilder.buildLoad(NewTy, MI.getOperand(1), MMO);
938  MIRBuilder.buildBitcast(ValReg, NewLoad);
939  }
940  MI.eraseFromParent();
941  return true;
942 }
943 
944 bool AArch64LegalizerInfo::legalizeVaArg(MachineInstr &MI,
946  MachineIRBuilder &MIRBuilder) const {
947  MachineFunction &MF = MIRBuilder.getMF();
948  Align Alignment(MI.getOperand(2).getImm());
949  Register Dst = MI.getOperand(0).getReg();
950  Register ListPtr = MI.getOperand(1).getReg();
951 
952  LLT PtrTy = MRI.getType(ListPtr);
953  LLT IntPtrTy = LLT::scalar(PtrTy.getSizeInBits());
954 
955  const unsigned PtrSize = PtrTy.getSizeInBits() / 8;
956  const Align PtrAlign = Align(PtrSize);
957  auto List = MIRBuilder.buildLoad(
958  PtrTy, ListPtr,
960  PtrSize, PtrAlign));
961 
962  MachineInstrBuilder DstPtr;
963  if (Alignment > PtrAlign) {
964  // Realign the list to the actual required alignment.
965  auto AlignMinus1 =
966  MIRBuilder.buildConstant(IntPtrTy, Alignment.value() - 1);
967  auto ListTmp = MIRBuilder.buildPtrAdd(PtrTy, List, AlignMinus1.getReg(0));
968  DstPtr = MIRBuilder.buildMaskLowPtrBits(PtrTy, ListTmp, Log2(Alignment));
969  } else
970  DstPtr = List;
971 
972  uint64_t ValSize = MRI.getType(Dst).getSizeInBits() / 8;
973  MIRBuilder.buildLoad(
974  Dst, DstPtr,
976  ValSize, std::max(Alignment, PtrAlign)));
977 
978  auto Size = MIRBuilder.buildConstant(IntPtrTy, alignTo(ValSize, PtrAlign));
979 
980  auto NewList = MIRBuilder.buildPtrAdd(PtrTy, DstPtr, Size.getReg(0));
981 
982  MIRBuilder.buildStore(NewList, ListPtr,
985  PtrSize, PtrAlign));
986 
987  MI.eraseFromParent();
988  return true;
989 }
990 
991 bool AArch64LegalizerInfo::legalizeBitfieldExtract(
993  // Only legal if we can select immediate forms.
994  // TODO: Lower this otherwise.
995  return getConstantVRegValWithLookThrough(MI.getOperand(2).getReg(), MRI) &&
996  getConstantVRegValWithLookThrough(MI.getOperand(3).getReg(), MRI);
997 }
AArch64LegalizerInfo.h
llvm::Check::Size
@ Size
Definition: FileCheck.h:73
llvm::alignTo
uint64_t alignTo(uint64_t Size, Align A)
Returns a multiple of A needed to store Size bytes.
Definition: Alignment.h:158
llvm::AArch64II::MO_G3
@ MO_G3
MO_G3 - A symbol operand with this flag (granule 3) represents the high 16-bits of a 64-bit address,...
Definition: AArch64BaseInfo.h:605
ValueTypes.h
llvm::LegalizeRuleSet::widenScalarToNextPow2
LegalizeRuleSet & widenScalarToNextPow2(unsigned TypeIdx, unsigned MinSize=0)
Widen the scalar to the next power of two that is at least MinSize.
Definition: LegalizerInfo.h:799
MI
IRTranslator LLVM IR MI
Definition: IRTranslator.cpp:100
MachineInstr.h
MathExtras.h
llvm::MachineInstrBuilder::addImm
const MachineInstrBuilder & addImm(int64_t Val) const
Add a new immediate operand.
Definition: MachineInstrBuilder.h:132
llvm
Definition: AllocatorList.h:23
Reg
unsigned Reg
Definition: MachineSink.cpp:1566
llvm::SystemZISD::TM
@ TM
Definition: SystemZISelLowering.h:65
llvm::LLT::getScalarSizeInBits
unsigned getScalarSizeInBits() const
Definition: LowLevelTypeImpl.h:163
llvm::LegalizeRuleSet::unsupportedIfMemSizeNotPow2
LegalizeRuleSet & unsupportedIfMemSizeNotPow2()
Definition: LegalizerInfo.h:758
llvm::LegalizerInfo::getActionDefinitionsBuilder
LegalizeRuleSet & getActionDefinitionsBuilder(unsigned Opcode)
Get the action definition builder for the given opcode.
Definition: LegalizerInfo.cpp:435
llvm::MachineRegisterInfo
MachineRegisterInfo - Keep track of information for virtual and physical registers,...
Definition: MachineRegisterInfo.h:52
llvm::LegalizeRuleSet::minScalarEltSameAsIf
LegalizeRuleSet & minScalarEltSameAsIf(LegalityPredicate Predicate, unsigned TypeIdx, unsigned LargeTypeIdx)
Conditionally widen the scalar or elt to match the size of another.
Definition: LegalizerInfo.h:944
llvm::LegalizeRuleSet::maxScalarIf
LegalizeRuleSet & maxScalarIf(LegalityPredicate Predicate, unsigned TypeIdx, const LLT Ty)
Conditionally limit the maximum size of the scalar.
Definition: LegalizerInfo.h:886
llvm::SmallVector
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
Definition: SmallVector.h:1168
llvm::MachineIRBuilder::getMRI
MachineRegisterInfo * getMRI()
Getter for MRI.
Definition: MachineIRBuilder.h:288
llvm::MachineFunction::getMachineMemOperand
MachineMemOperand * getMachineMemOperand(MachinePointerInfo PtrInfo, MachineMemOperand::Flags f, uint64_t s, Align base_alignment, const AAMDNodes &AAInfo=AAMDNodes(), const MDNode *Ranges=nullptr, SyncScope::ID SSID=SyncScope::System, AtomicOrdering Ordering=AtomicOrdering::NotAtomic, AtomicOrdering FailureOrdering=AtomicOrdering::NotAtomic)
getMachineMemOperand - Allocate a new MachineMemOperand.
Definition: MachineFunction.cpp:430
llvm::LegalizeRuleSet::clampNumElements
LegalizeRuleSet & clampNumElements(unsigned TypeIdx, const LLT MinTy, const LLT MaxTy)
Limit the number of elements for the given vectors to at least MinTy's number of elements and at most...
Definition: LegalizerInfo.h:1011
llvm::LegalizeRuleSet::minScalarOrEltIf
LegalizeRuleSet & minScalarOrEltIf(LegalityPredicate Predicate, unsigned TypeIdx, const LLT Ty)
Ensure the scalar or element is at least as wide as Ty.
Definition: LegalizerInfo.h:846
llvm::AArch64II::MO_PREL
@ MO_PREL
MO_PREL - Indicates that the bits of the symbol operand represented by MO_G0 etc are PC relative.
Definition: AArch64BaseInfo.h:656
Offset
uint64_t Offset
Definition: ELFObjHandler.cpp:81
llvm::LegalizeRuleSet::widenScalarIf
LegalizeRuleSet & widenScalarIf(LegalityPredicate Predicate, LegalizeMutation Mutation)
Widen the scalar to the one selected by the mutation if the predicate is true.
Definition: LegalizerInfo.h:702
MachineIRBuilder.h
llvm::LegalizeRuleSet::scalarize
LegalizeRuleSet & scalarize(unsigned TypeIdx)
Definition: LegalizerInfo.h:823
llvm::isPowerOf2_32
constexpr bool isPowerOf2_32(uint32_t Value)
Return true if the argument is a power of two > 0.
Definition: MathExtras.h:491
llvm::LegalizeRuleSet::lower
LegalizeRuleSet & lower()
The instruction is lowered.
Definition: LegalizerInfo.h:604
llvm::LegalizerHelper
Definition: LegalizerHelper.h:37
LegalizerInfo.h
llvm::LegalityPredicates::atomicOrderingAtLeastOrStrongerThan
LegalityPredicate atomicOrderingAtLeastOrStrongerThan(unsigned MMOIdx, AtomicOrdering Ordering)
True iff the specified MMO index has at an atomic ordering of at Ordering or stronger.
Definition: LegalityPredicates.cpp:190
LLVM_DEBUG
#define LLVM_DEBUG(X)
Definition: Debug.h:122
llvm::AArch64Subtarget::getTargetLowering
const AArch64TargetLowering * getTargetLowering() const override
Definition: AArch64Subtarget.h:302
llvm::MachineIRBuilder::buildConstant
virtual MachineInstrBuilder buildConstant(const DstOp &Res, const ConstantInt &Val)
Build and insert Res = G_CONSTANT Val.
Definition: MachineIRBuilder.cpp:255
llvm::LegalizeRuleSet::legalIf
LegalizeRuleSet & legalIf(LegalityPredicate Predicate)
The instruction is legal if predicate is true.
Definition: LegalizerInfo.h:531
MachineRegisterInfo.h
llvm::AtomicOrdering::Monotonic
@ Monotonic
llvm::dbgs
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
Definition: Debug.cpp:132
llvm::LegalizeRuleSet::customIf
LegalizeRuleSet & customIf(LegalityPredicate Predicate)
Definition: LegalizerInfo.h:767
llvm::LegalityPredicates::typeIs
LegalityPredicate typeIs(unsigned TypeIdx, LLT TypesInit)
True iff the given type index is the specified type.
Definition: LegalityPredicates.cpp:28
llvm::MachineIRBuilder::buildUnmerge
MachineInstrBuilder buildUnmerge(ArrayRef< LLT > Res, const SrcOp &Op)
Build and insert Res0, ...
Definition: MachineIRBuilder.cpp:605
llvm::MachineIRBuilder::buildLoad
MachineInstrBuilder buildLoad(const DstOp &Res, const SrcOp &Addr, MachineMemOperand &MMO)
Build and insert Res = G_LOAD Addr, MMO.
Definition: MachineIRBuilder.h:840
llvm::LLT::changeNumElements
LLT changeNumElements(unsigned NewNumElts) const
Return a vector or scalar with the same element type and the new number of elements.
Definition: LowLevelTypeImpl.h:143
llvm::LLT::getSizeInBits
unsigned getSizeInBits() const
Returns the total size of the type. Must only be called on sized types.
Definition: LowLevelTypeImpl.h:109
llvm::Log2
unsigned Log2(Align A)
Returns the log2 of the alignment.
Definition: Alignment.h:217
llvm::AArch64LegalizerInfo::legalizeIntrinsic
bool legalizeIntrinsic(LegalizerHelper &Helper, MachineInstr &MI) const override
Definition: AArch64LegalizerInfo.cpp:879
llvm::LegalizeRuleSet::lowerIfMemSizeNotPow2
LegalizeRuleSet & lowerIfMemSizeNotPow2()
Definition: LegalizerInfo.h:762
Utils.h
llvm::LegalizeRuleSet::fewerElementsIf
LegalizeRuleSet & fewerElementsIf(LegalityPredicate Predicate, LegalizeMutation Mutation)
Remove elements to reach the type selected by the mutation if the predicate is true.
Definition: LegalizerInfo.h:737
TargetOpcodes.h
llvm::MachineIRBuilder::buildConcatVectors
MachineInstrBuilder buildConcatVectors(const DstOp &Res, ArrayRef< Register > Ops)
Build and insert Res = G_CONCAT_VECTORS Op0, ...
Definition: MachineIRBuilder.cpp:689
llvm::CodeModel::Small
@ Small
Definition: CodeGen.h:28
Concat
static constexpr int Concat[]
Definition: X86InterleavedAccess.cpp:239
llvm::MachineIRBuilder::getMF
MachineFunction & getMF()
Getter for the function we currently build.
Definition: MachineIRBuilder.h:270
llvm::AArch64LegalizerInfo::AArch64LegalizerInfo
AArch64LegalizerInfo(const AArch64Subtarget &ST)
Definition: AArch64LegalizerInfo.cpp:36
llvm::LLT::vector
static LLT vector(uint16_t NumElements, unsigned ScalarSizeInBits)
Get a low-level vector of some number of elements and element width.
Definition: LowLevelTypeImpl.h:58
llvm::GISelChangeObserver::changingInstr
virtual void changingInstr(MachineInstr &MI)=0
This instruction is about to be mutated in some way.
llvm::LegalizeRuleSet::clampMaxNumElements
LegalizeRuleSet & clampMaxNumElements(unsigned TypeIdx, const LLT EltTy, unsigned MaxElements)
Limit the number of elements in EltTy vectors to at most MaxElements.
Definition: LegalizerInfo.h:988
Align
uint64_t Align
Definition: ELFObjHandler.cpp:83
llvm::Align
This struct is a compact representation of a valid (non-zero power of two) alignment.
Definition: Alignment.h:39
Type.h
llvm::MachineInstrBuilder::getReg
Register getReg(unsigned Idx) const
Get the register for the operand index.
Definition: MachineInstrBuilder.h:95
llvm::Log2_32_Ceil
unsigned Log2_32_Ceil(uint32_t Value)
Return the ceil log base 2 of the specified value, 32 if the value is zero.
Definition: MathExtras.h:609
llvm::AArch64LegalizerInfo::legalizeCustom
bool legalizeCustom(LegalizerHelper &Helper, MachineInstr &MI) const override
Called for instructions with the Custom LegalizationAction.
Definition: AArch64LegalizerInfo.cpp:727
llvm::LLT::pointer
static LLT pointer(unsigned AddressSpace, unsigned SizeInBits)
Get a low-level pointer in the given address space.
Definition: LowLevelTypeImpl.h:50
llvm::LLT::getAddressSpace
unsigned getAddressSpace() const
Definition: LowLevelTypeImpl.h:178
llvm::GISelChangeObserver::changedInstr
virtual void changedInstr(MachineInstr &MI)=0
This instruction was mutated in some way.
llvm::MachineIRBuilder::buildMaskLowPtrBits
MachineInstrBuilder buildMaskLowPtrBits(const DstOp &Res, const SrcOp &Op0, uint32_t NumBits)
Build and insert Res = G_PTRMASK Op0, G_CONSTANT (1 << NumBits) - 1.
Definition: MachineIRBuilder.cpp:208
llvm::LegalizeMutations::scalarize
LegalizeMutation scalarize(unsigned TypeIdx)
Break up the vector type for the given type index into the element type.
Definition: LegalizeMutations.cpp:77
llvm::LegalizeRuleSet::lowerIf
LegalizeRuleSet & lowerIf(LegalityPredicate Predicate)
The instruction is lowered if predicate is true.
Definition: LegalizerInfo.h:613
llvm::MachineIRBuilder
Helper class to build MachineInstr.
Definition: MachineIRBuilder.h:220
llvm::LegalizeRuleSet::legalFor
LegalizeRuleSet & legalFor(std::initializer_list< LLT > Types)
The instruction is legal when type index 0 is any type in the given list.
Definition: LegalizerInfo.h:538
llvm::MachineInstr
Representation of each machine instruction.
Definition: MachineInstr.h:64
llvm::MachineInstrBuilder
Definition: MachineInstrBuilder.h:70
llvm::ARM_MB::ST
@ ST
Definition: ARMBaseInfo.h:73
llvm::MachineIRBuilder::buildBitcast
MachineInstrBuilder buildBitcast(const DstOp &Dst, const SrcOp &Src)
Build and insert Dst = G_BITCAST Src.
Definition: MachineIRBuilder.h:641
llvm::LegalityPredicates::all
Predicate all(Predicate P0, Predicate P1)
True iff P0 and P1 are true.
Definition: LegalizerInfo.h:196
llvm::LegalizeActions::Bitcast
@ Bitcast
Perform the operation on a different, but equivalently sized type.
Definition: LegalizerInfo.h:72
llvm::AArch64II::MO_NC
@ MO_NC
MO_NC - Indicates whether the linker is expected to check the symbol reference for overflow.
Definition: AArch64BaseInfo.h:637
llvm::MachinePointerInfo
This class contains a discriminated union of information about pointers in memory operands,...
Definition: MachineMemOperand.h:37
llvm::AArch64II::MO_PAGEOFF
@ MO_PAGEOFF
MO_PAGEOFF - A symbol operand with this flag represents the offset of that symbol within a 4K page.
Definition: AArch64BaseInfo.h:601
llvm::MachineIRBuilder::buildPtrAdd
MachineInstrBuilder buildPtrAdd(const DstOp &Res, const SrcOp &Op0, const SrcOp &Op1)
Build and insert Res = G_PTR_ADD Op0, Op1.
Definition: MachineIRBuilder.cpp:182
I
#define I(x, y, z)
Definition: MD5.cpp:59
llvm::LLT::isVector
bool isVector() const
Definition: LowLevelTypeImpl.h:96
llvm::LegalizeRuleSet::clampScalar
LegalizeRuleSet & clampScalar(unsigned TypeIdx, const LLT MinTy, const LLT MaxTy)
Limit the range of scalar sizes to MinTy and MaxTy.
Definition: LegalizerInfo.h:902
llvm::LLT::getNumElements
uint16_t getNumElements() const
Returns the number of elements in a vector LLT.
Definition: LowLevelTypeImpl.h:100
llvm::LegalizeRuleSet::legalForCartesianProduct
LegalizeRuleSet & legalForCartesianProduct(std::initializer_list< LLT > Types)
The instruction is legal when type indexes 0 and 1 are both in the given list.
Definition: LegalizerInfo.h:570
assert
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
llvm::TargetMachine
Primary interface to the complete machine description for the target machine.
Definition: TargetMachine.h:77
llvm::LLT::isPointer
bool isPointer() const
Definition: LowLevelTypeImpl.h:94
llvm::LegalityPredicates::typeInSet
LegalityPredicate typeInSet(unsigned TypeIdx, std::initializer_list< LLT > TypesInit)
True iff the given type index is one of the specified types.
Definition: LegalityPredicates.cpp:34
llvm::MachineRegisterInfo::createGenericVirtualRegister
Register createGenericVirtualRegister(LLT Ty, StringRef Name="")
Create and return a new generic virtual register with low-level type Ty.
Definition: MachineRegisterInfo.cpp:188
llvm::LLT::isScalar
bool isScalar() const
Definition: LowLevelTypeImpl.h:92
llvm::MachineFunction
Definition: MachineFunction.h:227
llvm::LegalityQuery::Opcode
unsigned Opcode
Definition: LegalizerInfo.h:125
llvm::MachineIRBuilder::buildInstr
MachineInstrBuilder buildInstr(unsigned Opcode)
Build and insert <empty> = Opcode <empty>.
Definition: MachineIRBuilder.h:375
llvm::LegalityQuery
The LegalityQuery object bundles together all the information that's needed to decide whether a given...
Definition: LegalizerInfo.h:124
llvm::LegalizeRuleSet::customForCartesianProduct
LegalizeRuleSet & customForCartesianProduct(std::initializer_list< LLT > Types)
Definition: LegalizerInfo.h:783
llvm_unreachable
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
Definition: ErrorHandling.h:136
llvm::LegalizerHelper::Observer
GISelChangeObserver & Observer
To keep track of changes made by the LegalizerHelper.
Definition: LegalizerHelper.h:44
llvm::GISelChangeObserver
Abstract class that contains various methods for clients to notify about changes.
Definition: GISelChangeObserver.h:29
llvm::LegalizeMutations::widenScalarOrEltToNextPow2
LegalizeMutation widenScalarOrEltToNextPow2(unsigned TypeIdx, unsigned Min=0)
Widen the scalar type or vector element type for the given type index to the next power of 2.
Definition: LegalizeMutations.cpp:56
llvm::MachineMemOperand::MOLoad
@ MOLoad
The memory access reads data.
Definition: MachineMemOperand.h:134
MRI
unsigned const MachineRegisterInfo * MRI
Definition: AArch64AdvSIMDScalarPass.cpp:105
llvm::Register
Wrapper class representing virtual and physical registers.
Definition: Register.h:19
llvm::TargetLoweringBase::getTargetMachine
const TargetMachine & getTargetMachine() const
Definition: TargetLowering.h:335
llvm::MachineIRBuilder::buildTrunc
MachineInstrBuilder buildTrunc(const DstOp &Res, const SrcOp &Op)
Build and insert Res = G_TRUNC Op.
Definition: MachineIRBuilder.cpp:737
llvm::AArch64Subtarget::ClassifyGlobalReference
unsigned ClassifyGlobalReference(const GlobalValue *GV, const TargetMachine &TM) const
ClassifyGlobalReference - Find the target operand flags that describe how a global value should be re...
Definition: AArch64Subtarget.cpp:260
llvm::AMDGPU::SendMsg::Op
Op
Definition: SIDefines.h:314
llvm::LegalityQuery::MMODescrs
ArrayRef< MemDesc > MMODescrs
Operations which require memory can use this to place requirements on the memory type for each MMO.
Definition: LegalizerInfo.h:136
llvm::LegalizeRuleSet::unsupportedIf
LegalizeRuleSet & unsupportedIf(LegalityPredicate Predicate)
Definition: LegalizerInfo.h:750
LegalizerHelper.h
llvm::AArch64ISD::ADRP
@ ADRP
Definition: AArch64ISelLowering.h:61
llvm::LegalizeRuleSet::minScalarSameAs
LegalizeRuleSet & minScalarSameAs(unsigned TypeIdx, unsigned LargeTypeIdx)
Widen the scalar to match the size of another.
Definition: LegalizerInfo.h:915
llvm::MachineMemOperand::MOStore
@ MOStore
The memory access writes data.
Definition: MachineMemOperand.h:136
llvm::LegalizerInfo::verify
void verify(const MCInstrInfo &MII) const
Perform simple self-diagnostic and assert if there is anything obviously wrong with the actions set u...
Definition: LegalizerInfo.cpp:691
llvm::LegalityPredicates::isVector
LegalityPredicate isVector(unsigned TypeIdx)
True iff the specified type index is a vector.
Definition: LegalityPredicates.cpp:73
AArch64Subtarget.h
s1
int s1
Definition: README.txt:182
llvm::MachineRegisterInfo::getType
LLT getType(Register Reg) const
Get the low-level type of Reg or LLT{} if Reg is not a generic (target independent) virtual register.
Definition: MachineRegisterInfo.h:732
llvm::LegalityQuery::Types
ArrayRef< LLT > Types
Definition: LegalizerInfo.h:126
llvm::LegalizeRuleSet::moreElementsToNextPow2
LegalizeRuleSet & moreElementsToNextPow2(unsigned TypeIdx)
Add more elements to the vector to reach the next power of two.
Definition: LegalizerInfo.h:962
llvm::MachineIRBuilder::buildSExt
MachineInstrBuilder buildSExt(const DstOp &Res, const SrcOp &Op)
Build and insert Res = G_SEXT Op.
Definition: MachineIRBuilder.cpp:421
List
const NodeList & List
Definition: RDFGraph.cpp:201
llvm::LegalizerHelper::MIRBuilder
MachineIRBuilder & MIRBuilder
Expose MIRBuilder so clients can set their own RecordInsertInstruction functions.
Definition: LegalizerHelper.h:41
llvm::max
Align max(MaybeAlign Lhs, Align Rhs)
Definition: Alignment.h:350
llvm::LegalizeRuleSet::custom
LegalizeRuleSet & custom()
Unconditionally custom lower.
Definition: LegalizerInfo.h:793
llvm::SmallVectorImpl
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
Definition: APFloat.h:43
extractParts
static void extractParts(Register Reg, MachineRegisterInfo &MRI, MachineIRBuilder &MIRBuilder, LLT Ty, int NumParts, SmallVectorImpl< Register > &VRegs)
Definition: AArch64LegalizerInfo.cpp:776
DerivedTypes.h
llvm::LegalizerInfo::computeTables
void computeTables()
Compute any ancillary tables needed to quickly decide how an operation should be handled.
Definition: LegalizerInfo.cpp:294
llvm::AArch64II::MO_TAGGED
@ MO_TAGGED
MO_TAGGED - With MO_PAGE, indicates that the page includes a memory tag in bits 56-63.
Definition: AArch64BaseInfo.h:664
llvm::MachineIRBuilder::buildStore
MachineInstrBuilder buildStore(const SrcOp &Val, const SrcOp &Addr, MachineMemOperand &MMO)
Build and insert G_STORE Val, Addr, MMO.
Definition: MachineIRBuilder.cpp:388
llvm::LLT::getElementType
LLT getElementType() const
Returns the vector's element type. Only valid for vector types.
Definition: LowLevelTypeImpl.h:188
llvm::AArch64Subtarget
Definition: AArch64Subtarget.h:38
llvm::LegalizeRuleSet::libcallFor
LegalizeRuleSet & libcallFor(std::initializer_list< LLT > Types)
Definition: LegalizerInfo.h:683
llvm::LLT::scalar
static LLT scalar(unsigned SizeInBits)
Get a low-level scalar or aggregate "bag of bits".
Definition: LowLevelTypeImpl.h:43
llvm::LLT::changeElementSize
LLT changeElementSize(unsigned NewEltSize) const
If this type is a vector, return a vector with the same number of elements but the new element size.
Definition: LowLevelTypeImpl.h:134
llvm::AArch64II::MO_GOT
@ MO_GOT
MO_GOT - This flag indicates that a symbol operand represents the address of the GOT entry for the sy...
Definition: AArch64BaseInfo.h:632
libcall
Common register allocation spilling lr str ldr sxth r3 ldr mla r4 can lr mov lr str ldr sxth r3 mla r4 and then merge mul and lr str ldr sxth r3 mla r4 It also increase the likelihood the store may become dead bb27 Successors according to LLVM ID Predecessors according to mbb< bb27, 0x8b0a7c0 > Note ADDri is not a two address instruction its result reg1037 is an operand of the PHI node in bb76 and its operand reg1039 is the result of the PHI node We should treat it as a two address code and make sure the ADDri is scheduled after any node that reads reg1039 Use info(i.e. register scavenger) to assign it a free register to allow reuse the collector could move the objects and invalidate the derived pointer This is bad enough in the first but safe points can crop up unpredictably **array_addr i32 n y store obj obj **nth_el If the i64 division is lowered to a libcall
Definition: README.txt:127
llvm::LegalizeRuleSet::legalForTypesWithMemDesc
LegalizeRuleSet & legalForTypesWithMemDesc(std::initializer_list< LegalityPredicates::TypePairAndMemDesc > TypesAndMemDesc)
The instruction is legal when type indexes 0 and 1 along with the memory size and minimum alignment i...
Definition: LegalizerInfo.h:561
llvm::MachineRegisterInfo::setRegClass
void setRegClass(Register Reg, const TargetRegisterClass *RC)
setRegClass - Set the register class of the specified virtual register.
Definition: MachineRegisterInfo.cpp:58
llvm::getConstantVRegValWithLookThrough
Optional< ValueAndVReg > getConstantVRegValWithLookThrough(Register VReg, const MachineRegisterInfo &MRI, bool LookThroughInstrs=true, bool HandleFConstants=true, bool LookThroughAnyExt=false)
If VReg is defined by a statically evaluable chain of instructions rooted on a G_F/CONSTANT (LookThro...
Definition: Utils.cpp:289
llvm::AArch64II::MO_PAGE
@ MO_PAGE
MO_PAGE - A symbol operand with this flag represents the pc-relative offset of the 4K page containing...
Definition: AArch64BaseInfo.h:596
llvm::LLT
Definition: LowLevelTypeImpl.h:40