LLVM  13.0.0git
AArch64PostLegalizerLowering.cpp
Go to the documentation of this file.
1 //=== AArch64PostLegalizerLowering.cpp --------------------------*- C++ -*-===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 ///
9 /// \file
10 /// Post-legalization lowering for instructions.
11 ///
12 /// This is used to offload pattern matching from the selector.
13 ///
14 /// For example, this combiner will notice that a G_SHUFFLE_VECTOR is actually
15 /// a G_ZIP, G_UZP, etc.
16 ///
17 /// General optimization combines should be handled by either the
18 /// AArch64PostLegalizerCombiner or the AArch64PreLegalizerCombiner.
19 ///
20 //===----------------------------------------------------------------------===//
21 
22 #include "AArch64TargetMachine.h"
23 #include "AArch64GlobalISelUtils.h"
36 #include "llvm/InitializePasses.h"
37 #include "llvm/Support/Debug.h"
38 
39 #define DEBUG_TYPE "aarch64-postlegalizer-lowering"
40 
41 using namespace llvm;
42 using namespace MIPatternMatch;
43 using namespace AArch64GISelUtils;
44 
45 /// Represents a pseudo instruction which replaces a G_SHUFFLE_VECTOR.
46 ///
47 /// Used for matching target-supported shuffles before codegen.
49  unsigned Opc; ///< Opcode for the instruction. (E.g. G_ZIP1)
50  Register Dst; ///< Destination register.
51  SmallVector<SrcOp, 2> SrcOps; ///< Source registers.
52  ShuffleVectorPseudo(unsigned Opc, Register Dst,
53  std::initializer_list<SrcOp> SrcOps)
54  : Opc(Opc), Dst(Dst), SrcOps(SrcOps){};
56 };
57 
58 /// Check if a vector shuffle corresponds to a REV instruction with the
59 /// specified blocksize.
60 static bool isREVMask(ArrayRef<int> M, unsigned EltSize, unsigned NumElts,
61  unsigned BlockSize) {
62  assert((BlockSize == 16 || BlockSize == 32 || BlockSize == 64) &&
63  "Only possible block sizes for REV are: 16, 32, 64");
64  assert(EltSize != 64 && "EltSize cannot be 64 for REV mask.");
65 
66  unsigned BlockElts = M[0] + 1;
67 
68  // If the first shuffle index is UNDEF, be optimistic.
69  if (M[0] < 0)
70  BlockElts = BlockSize / EltSize;
71 
72  if (BlockSize <= EltSize || BlockSize != BlockElts * EltSize)
73  return false;
74 
75  for (unsigned i = 0; i < NumElts; ++i) {
76  // Ignore undef indices.
77  if (M[i] < 0)
78  continue;
79  if (static_cast<unsigned>(M[i]) !=
80  (i - i % BlockElts) + (BlockElts - 1 - i % BlockElts))
81  return false;
82  }
83 
84  return true;
85 }
86 
87 /// Determines if \p M is a shuffle vector mask for a TRN of \p NumElts.
88 /// Whether or not G_TRN1 or G_TRN2 should be used is stored in \p WhichResult.
89 static bool isTRNMask(ArrayRef<int> M, unsigned NumElts,
90  unsigned &WhichResult) {
91  if (NumElts % 2 != 0)
92  return false;
93  WhichResult = (M[0] == 0 ? 0 : 1);
94  for (unsigned i = 0; i < NumElts; i += 2) {
95  if ((M[i] >= 0 && static_cast<unsigned>(M[i]) != i + WhichResult) ||
96  (M[i + 1] >= 0 &&
97  static_cast<unsigned>(M[i + 1]) != i + NumElts + WhichResult))
98  return false;
99  }
100  return true;
101 }
102 
103 /// Check if a G_EXT instruction can handle a shuffle mask \p M when the vector
104 /// sources of the shuffle are different.
106  unsigned NumElts) {
107  // Look for the first non-undef element.
108  auto FirstRealElt = find_if(M, [](int Elt) { return Elt >= 0; });
109  if (FirstRealElt == M.end())
110  return None;
111 
112  // Use APInt to handle overflow when calculating expected element.
113  unsigned MaskBits = APInt(32, NumElts * 2).logBase2();
114  APInt ExpectedElt = APInt(MaskBits, *FirstRealElt + 1);
115 
116  // The following shuffle indices must be the successive elements after the
117  // first real element.
118  if (any_of(
119  make_range(std::next(FirstRealElt), M.end()),
120  [&ExpectedElt](int Elt) { return Elt != ExpectedElt++ && Elt >= 0; }))
121  return None;
122 
123  // The index of an EXT is the first element if it is not UNDEF.
124  // Watch out for the beginning UNDEFs. The EXT index should be the expected
125  // value of the first element. E.g.
126  // <-1, -1, 3, ...> is treated as <1, 2, 3, ...>.
127  // <-1, -1, 0, 1, ...> is treated as <2*NumElts-2, 2*NumElts-1, 0, 1, ...>.
128  // ExpectedElt is the last mask index plus 1.
129  uint64_t Imm = ExpectedElt.getZExtValue();
130  bool ReverseExt = false;
131 
132  // There are two difference cases requiring to reverse input vectors.
133  // For example, for vector <4 x i32> we have the following cases,
134  // Case 1: shufflevector(<4 x i32>,<4 x i32>,<-1, -1, -1, 0>)
135  // Case 2: shufflevector(<4 x i32>,<4 x i32>,<-1, -1, 7, 0>)
136  // For both cases, we finally use mask <5, 6, 7, 0>, which requires
137  // to reverse two input vectors.
138  if (Imm < NumElts)
139  ReverseExt = true;
140  else
141  Imm -= NumElts;
142  return std::make_pair(ReverseExt, Imm);
143 }
144 
145 /// Determines if \p M is a shuffle vector mask for a UZP of \p NumElts.
146 /// Whether or not G_UZP1 or G_UZP2 should be used is stored in \p WhichResult.
147 static bool isUZPMask(ArrayRef<int> M, unsigned NumElts,
148  unsigned &WhichResult) {
149  WhichResult = (M[0] == 0 ? 0 : 1);
150  for (unsigned i = 0; i != NumElts; ++i) {
151  // Skip undef indices.
152  if (M[i] < 0)
153  continue;
154  if (static_cast<unsigned>(M[i]) != 2 * i + WhichResult)
155  return false;
156  }
157  return true;
158 }
159 
160 /// \return true if \p M is a zip mask for a shuffle vector of \p NumElts.
161 /// Whether or not G_ZIP1 or G_ZIP2 should be used is stored in \p WhichResult.
162 static bool isZipMask(ArrayRef<int> M, unsigned NumElts,
163  unsigned &WhichResult) {
164  if (NumElts % 2 != 0)
165  return false;
166 
167  // 0 means use ZIP1, 1 means use ZIP2.
168  WhichResult = (M[0] == 0 ? 0 : 1);
169  unsigned Idx = WhichResult * NumElts / 2;
170  for (unsigned i = 0; i != NumElts; i += 2) {
171  if ((M[i] >= 0 && static_cast<unsigned>(M[i]) != Idx) ||
172  (M[i + 1] >= 0 && static_cast<unsigned>(M[i + 1]) != Idx + NumElts))
173  return false;
174  Idx += 1;
175  }
176  return true;
177 }
178 
179 /// Helper function for matchINS.
180 ///
181 /// \returns a value when \p M is an ins mask for \p NumInputElements.
182 ///
183 /// First element of the returned pair is true when the produced
184 /// G_INSERT_VECTOR_ELT destination should be the LHS of the G_SHUFFLE_VECTOR.
185 ///
186 /// Second element is the destination lane for the G_INSERT_VECTOR_ELT.
188  int NumInputElements) {
189  if (M.size() != static_cast<size_t>(NumInputElements))
190  return None;
191  int NumLHSMatch = 0, NumRHSMatch = 0;
192  int LastLHSMismatch = -1, LastRHSMismatch = -1;
193  for (int Idx = 0; Idx < NumInputElements; ++Idx) {
194  if (M[Idx] == -1) {
195  ++NumLHSMatch;
196  ++NumRHSMatch;
197  continue;
198  }
199  M[Idx] == Idx ? ++NumLHSMatch : LastLHSMismatch = Idx;
200  M[Idx] == Idx + NumInputElements ? ++NumRHSMatch : LastRHSMismatch = Idx;
201  }
202  const int NumNeededToMatch = NumInputElements - 1;
203  if (NumLHSMatch == NumNeededToMatch)
204  return std::make_pair(true, LastLHSMismatch);
205  if (NumRHSMatch == NumNeededToMatch)
206  return std::make_pair(false, LastRHSMismatch);
207  return None;
208 }
209 
210 /// \return true if a G_SHUFFLE_VECTOR instruction \p MI can be replaced with a
211 /// G_REV instruction. Returns the appropriate G_REV opcode in \p Opc.
213  ShuffleVectorPseudo &MatchInfo) {
214  assert(MI.getOpcode() == TargetOpcode::G_SHUFFLE_VECTOR);
215  ArrayRef<int> ShuffleMask = MI.getOperand(3).getShuffleMask();
216  Register Dst = MI.getOperand(0).getReg();
217  Register Src = MI.getOperand(1).getReg();
218  LLT Ty = MRI.getType(Dst);
219  unsigned EltSize = Ty.getScalarSizeInBits();
220 
221  // Element size for a rev cannot be 64.
222  if (EltSize == 64)
223  return false;
224 
225  unsigned NumElts = Ty.getNumElements();
226 
227  // Try to produce G_REV64
228  if (isREVMask(ShuffleMask, EltSize, NumElts, 64)) {
229  MatchInfo = ShuffleVectorPseudo(AArch64::G_REV64, Dst, {Src});
230  return true;
231  }
232 
233  // TODO: Produce G_REV32 and G_REV16 once we have proper legalization support.
234  // This should be identical to above, but with a constant 32 and constant
235  // 16.
236  return false;
237 }
238 
239 /// \return true if a G_SHUFFLE_VECTOR instruction \p MI can be replaced with
240 /// a G_TRN1 or G_TRN2 instruction.
242  ShuffleVectorPseudo &MatchInfo) {
243  assert(MI.getOpcode() == TargetOpcode::G_SHUFFLE_VECTOR);
244  unsigned WhichResult;
245  ArrayRef<int> ShuffleMask = MI.getOperand(3).getShuffleMask();
246  Register Dst = MI.getOperand(0).getReg();
247  unsigned NumElts = MRI.getType(Dst).getNumElements();
248  if (!isTRNMask(ShuffleMask, NumElts, WhichResult))
249  return false;
250  unsigned Opc = (WhichResult == 0) ? AArch64::G_TRN1 : AArch64::G_TRN2;
251  Register V1 = MI.getOperand(1).getReg();
252  Register V2 = MI.getOperand(2).getReg();
253  MatchInfo = ShuffleVectorPseudo(Opc, Dst, {V1, V2});
254  return true;
255 }
256 
257 /// \return true if a G_SHUFFLE_VECTOR instruction \p MI can be replaced with
258 /// a G_UZP1 or G_UZP2 instruction.
259 ///
260 /// \param [in] MI - The shuffle vector instruction.
261 /// \param [out] MatchInfo - Either G_UZP1 or G_UZP2 on success.
263  ShuffleVectorPseudo &MatchInfo) {
264  assert(MI.getOpcode() == TargetOpcode::G_SHUFFLE_VECTOR);
265  unsigned WhichResult;
266  ArrayRef<int> ShuffleMask = MI.getOperand(3).getShuffleMask();
267  Register Dst = MI.getOperand(0).getReg();
268  unsigned NumElts = MRI.getType(Dst).getNumElements();
269  if (!isUZPMask(ShuffleMask, NumElts, WhichResult))
270  return false;
271  unsigned Opc = (WhichResult == 0) ? AArch64::G_UZP1 : AArch64::G_UZP2;
272  Register V1 = MI.getOperand(1).getReg();
273  Register V2 = MI.getOperand(2).getReg();
274  MatchInfo = ShuffleVectorPseudo(Opc, Dst, {V1, V2});
275  return true;
276 }
277 
279  ShuffleVectorPseudo &MatchInfo) {
280  assert(MI.getOpcode() == TargetOpcode::G_SHUFFLE_VECTOR);
281  unsigned WhichResult;
282  ArrayRef<int> ShuffleMask = MI.getOperand(3).getShuffleMask();
283  Register Dst = MI.getOperand(0).getReg();
284  unsigned NumElts = MRI.getType(Dst).getNumElements();
285  if (!isZipMask(ShuffleMask, NumElts, WhichResult))
286  return false;
287  unsigned Opc = (WhichResult == 0) ? AArch64::G_ZIP1 : AArch64::G_ZIP2;
288  Register V1 = MI.getOperand(1).getReg();
289  Register V2 = MI.getOperand(2).getReg();
290  MatchInfo = ShuffleVectorPseudo(Opc, Dst, {V1, V2});
291  return true;
292 }
293 
294 /// Helper function for matchDup.
297  ShuffleVectorPseudo &MatchInfo) {
298  if (Lane != 0)
299  return false;
300 
301  // Try to match a vector splat operation into a dup instruction.
302  // We're looking for this pattern:
303  //
304  // %scalar:gpr(s64) = COPY $x0
305  // %undef:fpr(<2 x s64>) = G_IMPLICIT_DEF
306  // %cst0:gpr(s32) = G_CONSTANT i32 0
307  // %zerovec:fpr(<2 x s32>) = G_BUILD_VECTOR %cst0(s32), %cst0(s32)
308  // %ins:fpr(<2 x s64>) = G_INSERT_VECTOR_ELT %undef, %scalar(s64), %cst0(s32)
309  // %splat:fpr(<2 x s64>) = G_SHUFFLE_VECTOR %ins(<2 x s64>), %undef, %zerovec(<2 x s32>)
310  //
311  // ...into:
312  // %splat = G_DUP %scalar
313 
314  // Begin matching the insert.
315  auto *InsMI = getOpcodeDef(TargetOpcode::G_INSERT_VECTOR_ELT,
316  MI.getOperand(1).getReg(), MRI);
317  if (!InsMI)
318  return false;
319  // Match the undef vector operand.
320  if (!getOpcodeDef(TargetOpcode::G_IMPLICIT_DEF, InsMI->getOperand(1).getReg(),
321  MRI))
322  return false;
323 
324  // Match the index constant 0.
325  if (!mi_match(InsMI->getOperand(3).getReg(), MRI, m_ZeroInt()))
326  return false;
327 
328  MatchInfo = ShuffleVectorPseudo(AArch64::G_DUP, MI.getOperand(0).getReg(),
329  {InsMI->getOperand(2).getReg()});
330  return true;
331 }
332 
333 /// Helper function for matchDup.
334 static bool matchDupFromBuildVector(int Lane, MachineInstr &MI,
336  ShuffleVectorPseudo &MatchInfo) {
337  assert(Lane >= 0 && "Expected positive lane?");
338  // Test if the LHS is a BUILD_VECTOR. If it is, then we can just reference the
339  // lane's definition directly.
340  auto *BuildVecMI = getOpcodeDef(TargetOpcode::G_BUILD_VECTOR,
341  MI.getOperand(1).getReg(), MRI);
342  if (!BuildVecMI)
343  return false;
344  Register Reg = BuildVecMI->getOperand(Lane + 1).getReg();
345  MatchInfo =
346  ShuffleVectorPseudo(AArch64::G_DUP, MI.getOperand(0).getReg(), {Reg});
347  return true;
348 }
349 
351  ShuffleVectorPseudo &MatchInfo) {
352  assert(MI.getOpcode() == TargetOpcode::G_SHUFFLE_VECTOR);
353  auto MaybeLane = getSplatIndex(MI);
354  if (!MaybeLane)
355  return false;
356  int Lane = *MaybeLane;
357  // If this is undef splat, generate it via "just" vdup, if possible.
358  if (Lane < 0)
359  Lane = 0;
360  if (matchDupFromInsertVectorElt(Lane, MI, MRI, MatchInfo))
361  return true;
362  if (matchDupFromBuildVector(Lane, MI, MRI, MatchInfo))
363  return true;
364  return false;
365 }
366 
368  ShuffleVectorPseudo &MatchInfo) {
369  assert(MI.getOpcode() == TargetOpcode::G_SHUFFLE_VECTOR);
370  Register Dst = MI.getOperand(0).getReg();
371  auto ExtInfo = getExtMask(MI.getOperand(3).getShuffleMask(),
372  MRI.getType(Dst).getNumElements());
373  if (!ExtInfo)
374  return false;
375  bool ReverseExt;
376  uint64_t Imm;
377  std::tie(ReverseExt, Imm) = *ExtInfo;
378  Register V1 = MI.getOperand(1).getReg();
379  Register V2 = MI.getOperand(2).getReg();
380  if (ReverseExt)
381  std::swap(V1, V2);
382  uint64_t ExtFactor = MRI.getType(V1).getScalarSizeInBits() / 8;
383  Imm *= ExtFactor;
384  MatchInfo = ShuffleVectorPseudo(AArch64::G_EXT, Dst, {V1, V2, Imm});
385  return true;
386 }
387 
388 /// Replace a G_SHUFFLE_VECTOR instruction with a pseudo.
389 /// \p Opc is the opcode to use. \p MI is the G_SHUFFLE_VECTOR.
391  ShuffleVectorPseudo &MatchInfo) {
392  MachineIRBuilder MIRBuilder(MI);
393  MIRBuilder.buildInstr(MatchInfo.Opc, {MatchInfo.Dst}, MatchInfo.SrcOps);
394  MI.eraseFromParent();
395  return true;
396 }
397 
398 /// Replace a G_SHUFFLE_VECTOR instruction with G_EXT.
399 /// Special-cased because the constant operand must be emitted as a G_CONSTANT
400 /// for the imported tablegen patterns to work.
401 static bool applyEXT(MachineInstr &MI, ShuffleVectorPseudo &MatchInfo) {
402  MachineIRBuilder MIRBuilder(MI);
403  // Tablegen patterns expect an i32 G_CONSTANT as the final op.
404  auto Cst =
405  MIRBuilder.buildConstant(LLT::scalar(32), MatchInfo.SrcOps[2].getImm());
406  MIRBuilder.buildInstr(MatchInfo.Opc, {MatchInfo.Dst},
407  {MatchInfo.SrcOps[0], MatchInfo.SrcOps[1], Cst});
408  MI.eraseFromParent();
409  return true;
410 }
411 
412 /// Match a G_SHUFFLE_VECTOR with a mask which corresponds to a
413 /// G_INSERT_VECTOR_ELT and G_EXTRACT_VECTOR_ELT pair.
414 ///
415 /// e.g.
416 /// %shuf = G_SHUFFLE_VECTOR %left, %right, shufflemask(0, 0)
417 ///
418 /// Can be represented as
419 ///
420 /// %extract = G_EXTRACT_VECTOR_ELT %left, 0
421 /// %ins = G_INSERT_VECTOR_ELT %left, %extract, 1
422 ///
424  std::tuple<Register, int, Register, int> &MatchInfo) {
425  assert(MI.getOpcode() == TargetOpcode::G_SHUFFLE_VECTOR);
426  ArrayRef<int> ShuffleMask = MI.getOperand(3).getShuffleMask();
427  Register Dst = MI.getOperand(0).getReg();
428  int NumElts = MRI.getType(Dst).getNumElements();
429  auto DstIsLeftAndDstLane = isINSMask(ShuffleMask, NumElts);
430  if (!DstIsLeftAndDstLane)
431  return false;
432  bool DstIsLeft;
433  int DstLane;
434  std::tie(DstIsLeft, DstLane) = *DstIsLeftAndDstLane;
435  Register Left = MI.getOperand(1).getReg();
436  Register Right = MI.getOperand(2).getReg();
437  Register DstVec = DstIsLeft ? Left : Right;
438  Register SrcVec = Left;
439 
440  int SrcLane = ShuffleMask[DstLane];
441  if (SrcLane >= NumElts) {
442  SrcVec = Right;
443  SrcLane -= NumElts;
444  }
445 
446  MatchInfo = std::make_tuple(DstVec, DstLane, SrcVec, SrcLane);
447  return true;
448 }
449 
452  std::tuple<Register, int, Register, int> &MatchInfo) {
453  Builder.setInstrAndDebugLoc(MI);
454  Register Dst = MI.getOperand(0).getReg();
455  auto ScalarTy = MRI.getType(Dst).getElementType();
456  Register DstVec, SrcVec;
457  int DstLane, SrcLane;
458  std::tie(DstVec, DstLane, SrcVec, SrcLane) = MatchInfo;
459  auto SrcCst = Builder.buildConstant(LLT::scalar(64), SrcLane);
460  auto Extract = Builder.buildExtractVectorElement(ScalarTy, SrcVec, SrcCst);
461  auto DstCst = Builder.buildConstant(LLT::scalar(64), DstLane);
462  Builder.buildInsertVectorElement(Dst, DstVec, Extract, DstCst);
463  MI.eraseFromParent();
464  return true;
465 }
466 
467 /// isVShiftRImm - Check if this is a valid vector for the immediate
468 /// operand of a vector shift right operation. The value must be in the range:
469 /// 1 <= Value <= ElementBits for a right shift.
471  int64_t &Cnt) {
472  assert(Ty.isVector() && "vector shift count is not a vector type");
474  auto Cst = getAArch64VectorSplatScalar(*MI, MRI);
475  if (!Cst)
476  return false;
477  Cnt = *Cst;
478  int64_t ElementBits = Ty.getScalarSizeInBits();
479  return Cnt >= 1 && Cnt <= ElementBits;
480 }
481 
482 /// Match a vector G_ASHR or G_LSHR with a valid immediate shift.
484  int64_t &Imm) {
485  assert(MI.getOpcode() == TargetOpcode::G_ASHR ||
486  MI.getOpcode() == TargetOpcode::G_LSHR);
487  LLT Ty = MRI.getType(MI.getOperand(1).getReg());
488  if (!Ty.isVector())
489  return false;
490  return isVShiftRImm(MI.getOperand(2).getReg(), MRI, Ty, Imm);
491 }
492 
494  int64_t &Imm) {
495  unsigned Opc = MI.getOpcode();
496  assert(Opc == TargetOpcode::G_ASHR || Opc == TargetOpcode::G_LSHR);
497  unsigned NewOpc =
498  Opc == TargetOpcode::G_ASHR ? AArch64::G_VASHR : AArch64::G_VLSHR;
499  MachineIRBuilder MIB(MI);
500  auto ImmDef = MIB.buildConstant(LLT::scalar(32), Imm);
501  MIB.buildInstr(NewOpc, {MI.getOperand(0)}, {MI.getOperand(1), ImmDef});
502  MI.eraseFromParent();
503  return true;
504 }
505 
506 /// Determine if it is possible to modify the \p RHS and predicate \p P of a
507 /// G_ICMP instruction such that the right-hand side is an arithmetic immediate.
508 ///
509 /// \returns A pair containing the updated immediate and predicate which may
510 /// be used to optimize the instruction.
511 ///
512 /// \note This assumes that the comparison has been legalized.
515  const MachineRegisterInfo &MRI) {
516  const auto &Ty = MRI.getType(RHS);
517  if (Ty.isVector())
518  return None;
519  unsigned Size = Ty.getSizeInBits();
520  assert((Size == 32 || Size == 64) && "Expected 32 or 64 bit compare only?");
521 
522  // If the RHS is not a constant, or the RHS is already a valid arithmetic
523  // immediate, then there is nothing to change.
524  auto ValAndVReg = getConstantVRegValWithLookThrough(RHS, MRI);
525  if (!ValAndVReg)
526  return None;
527  uint64_t C = ValAndVReg->Value.getZExtValue();
528  if (isLegalArithImmed(C))
529  return None;
530 
531  // We have a non-arithmetic immediate. Check if adjusting the immediate and
532  // adjusting the predicate will result in a legal arithmetic immediate.
533  switch (P) {
534  default:
535  return None;
536  case CmpInst::ICMP_SLT:
537  case CmpInst::ICMP_SGE:
538  // Check for
539  //
540  // x slt c => x sle c - 1
541  // x sge c => x sgt c - 1
542  //
543  // When c is not the smallest possible negative number.
544  if ((Size == 64 && static_cast<int64_t>(C) == INT64_MIN) ||
545  (Size == 32 && static_cast<int32_t>(C) == INT32_MIN))
546  return None;
548  C -= 1;
549  break;
550  case CmpInst::ICMP_ULT:
551  case CmpInst::ICMP_UGE:
552  // Check for
553  //
554  // x ult c => x ule c - 1
555  // x uge c => x ugt c - 1
556  //
557  // When c is not zero.
558  if (C == 0)
559  return None;
561  C -= 1;
562  break;
563  case CmpInst::ICMP_SLE:
564  case CmpInst::ICMP_SGT:
565  // Check for
566  //
567  // x sle c => x slt c + 1
568  // x sgt c => s sge c + 1
569  //
570  // When c is not the largest possible signed integer.
571  if ((Size == 32 && static_cast<int32_t>(C) == INT32_MAX) ||
572  (Size == 64 && static_cast<int64_t>(C) == INT64_MAX))
573  return None;
575  C += 1;
576  break;
577  case CmpInst::ICMP_ULE:
578  case CmpInst::ICMP_UGT:
579  // Check for
580  //
581  // x ule c => x ult c + 1
582  // x ugt c => s uge c + 1
583  //
584  // When c is not the largest possible unsigned integer.
585  if ((Size == 32 && static_cast<uint32_t>(C) == UINT32_MAX) ||
586  (Size == 64 && C == UINT64_MAX))
587  return None;
589  C += 1;
590  break;
591  }
592 
593  // Check if the new constant is valid, and return the updated constant and
594  // predicate if it is.
595  if (Size == 32)
596  C = static_cast<uint32_t>(C);
597  if (!isLegalArithImmed(C))
598  return None;
599  return {{C, P}};
600 }
601 
602 /// Determine whether or not it is possible to update the RHS and predicate of
603 /// a G_ICMP instruction such that the RHS will be selected as an arithmetic
604 /// immediate.
605 ///
606 /// \p MI - The G_ICMP instruction
607 /// \p MatchInfo - The new RHS immediate and predicate on success
608 ///
609 /// See tryAdjustICmpImmAndPred for valid transformations.
612  std::pair<uint64_t, CmpInst::Predicate> &MatchInfo) {
613  assert(MI.getOpcode() == TargetOpcode::G_ICMP);
614  Register RHS = MI.getOperand(3).getReg();
615  auto Pred = static_cast<CmpInst::Predicate>(MI.getOperand(1).getPredicate());
616  if (auto MaybeNewImmAndPred = tryAdjustICmpImmAndPred(RHS, Pred, MRI)) {
617  MatchInfo = *MaybeNewImmAndPred;
618  return true;
619  }
620  return false;
621 }
622 
624  MachineInstr &MI, std::pair<uint64_t, CmpInst::Predicate> &MatchInfo,
625  MachineIRBuilder &MIB, GISelChangeObserver &Observer) {
626  MIB.setInstrAndDebugLoc(MI);
627  MachineOperand &RHS = MI.getOperand(3);
628  MachineRegisterInfo &MRI = *MIB.getMRI();
629  auto Cst = MIB.buildConstant(MRI.cloneVirtualRegister(RHS.getReg()),
630  MatchInfo.first);
631  Observer.changingInstr(MI);
632  RHS.setReg(Cst->getOperand(0).getReg());
633  MI.getOperand(1).setPredicate(MatchInfo.second);
634  Observer.changedInstr(MI);
635  return true;
636 }
637 
639  std::pair<unsigned, int> &MatchInfo) {
640  assert(MI.getOpcode() == TargetOpcode::G_SHUFFLE_VECTOR);
641  Register Src1Reg = MI.getOperand(1).getReg();
642  const LLT SrcTy = MRI.getType(Src1Reg);
643  const LLT DstTy = MRI.getType(MI.getOperand(0).getReg());
644 
645  auto LaneIdx = getSplatIndex(MI);
646  if (!LaneIdx)
647  return false;
648 
649  // The lane idx should be within the first source vector.
650  if (*LaneIdx >= SrcTy.getNumElements())
651  return false;
652 
653  if (DstTy != SrcTy)
654  return false;
655 
656  LLT ScalarTy = SrcTy.getElementType();
657  unsigned ScalarSize = ScalarTy.getSizeInBits();
658 
659  unsigned Opc = 0;
660  switch (SrcTy.getNumElements()) {
661  case 2:
662  if (ScalarSize == 64)
663  Opc = AArch64::G_DUPLANE64;
664  else if (ScalarSize == 32)
665  Opc = AArch64::G_DUPLANE32;
666  break;
667  case 4:
668  if (ScalarSize == 32)
669  Opc = AArch64::G_DUPLANE32;
670  break;
671  case 8:
672  if (ScalarSize == 16)
673  Opc = AArch64::G_DUPLANE16;
674  break;
675  case 16:
676  if (ScalarSize == 8)
677  Opc = AArch64::G_DUPLANE8;
678  break;
679  default:
680  break;
681  }
682  if (!Opc)
683  return false;
684 
685  MatchInfo.first = Opc;
686  MatchInfo.second = *LaneIdx;
687  return true;
688 }
689 
691  MachineIRBuilder &B, std::pair<unsigned, int> &MatchInfo) {
692  assert(MI.getOpcode() == TargetOpcode::G_SHUFFLE_VECTOR);
693  Register Src1Reg = MI.getOperand(1).getReg();
694  const LLT SrcTy = MRI.getType(Src1Reg);
695 
696  B.setInstrAndDebugLoc(MI);
697  auto Lane = B.buildConstant(LLT::scalar(64), MatchInfo.second);
698 
699  Register DupSrc = MI.getOperand(1).getReg();
700  // For types like <2 x s32>, we can use G_DUPLANE32, with a <4 x s32> source.
701  // To do this, we can use a G_CONCAT_VECTORS to do the widening.
702  if (SrcTy == LLT::vector(2, LLT::scalar(32))) {
703  assert(MRI.getType(MI.getOperand(0).getReg()).getNumElements() == 2 &&
704  "Unexpected dest elements");
705  auto Undef = B.buildUndef(SrcTy);
706  DupSrc = B.buildConcatVectors(SrcTy.changeNumElements(4),
707  {Src1Reg, Undef.getReg(0)})
708  .getReg(0);
709  }
710  B.buildInstr(MatchInfo.first, {MI.getOperand(0).getReg()}, {DupSrc, Lane});
711  MI.eraseFromParent();
712  return true;
713 }
714 
716  assert(MI.getOpcode() == TargetOpcode::G_BUILD_VECTOR);
717  auto Splat = getAArch64VectorSplat(MI, MRI);
718  if (!Splat)
719  return false;
720  if (Splat->isReg())
721  return true;
722  // Later, during selection, we'll try to match imported patterns using
723  // immAllOnesV and immAllZerosV. These require G_BUILD_VECTOR. Don't lower
724  // G_BUILD_VECTORs which could match those patterns.
725  int64_t Cst = Splat->getCst();
726  return (Cst != 0 && Cst != -1);
727 }
728 
730  MachineIRBuilder &B) {
731  B.setInstrAndDebugLoc(MI);
732  B.buildInstr(AArch64::G_DUP, {MI.getOperand(0).getReg()},
733  {MI.getOperand(1).getReg()});
734  MI.eraseFromParent();
735  return true;
736 }
737 
738 /// \returns how many instructions would be saved by folding a G_ICMP's shift
739 /// and/or extension operations.
740 static unsigned getCmpOperandFoldingProfit(Register CmpOp,
741  const MachineRegisterInfo &MRI) {
742  // No instructions to save if there's more than one use or no uses.
743  if (!MRI.hasOneNonDBGUse(CmpOp))
744  return 0;
745 
746  // FIXME: This is duplicated with the selector. (See: selectShiftedRegister)
747  auto IsSupportedExtend = [&](const MachineInstr &MI) {
748  if (MI.getOpcode() == TargetOpcode::G_SEXT_INREG)
749  return true;
750  if (MI.getOpcode() != TargetOpcode::G_AND)
751  return false;
752  auto ValAndVReg =
753  getConstantVRegValWithLookThrough(MI.getOperand(2).getReg(), MRI);
754  if (!ValAndVReg)
755  return false;
756  uint64_t Mask = ValAndVReg->Value.getZExtValue();
757  return (Mask == 0xFF || Mask == 0xFFFF || Mask == 0xFFFFFFFF);
758  };
759 
761  if (IsSupportedExtend(*Def))
762  return 1;
763 
764  unsigned Opc = Def->getOpcode();
765  if (Opc != TargetOpcode::G_SHL && Opc != TargetOpcode::G_ASHR &&
766  Opc != TargetOpcode::G_LSHR)
767  return 0;
768 
769  auto MaybeShiftAmt =
770  getConstantVRegValWithLookThrough(Def->getOperand(2).getReg(), MRI);
771  if (!MaybeShiftAmt)
772  return 0;
773  uint64_t ShiftAmt = MaybeShiftAmt->Value.getZExtValue();
774  MachineInstr *ShiftLHS =
775  getDefIgnoringCopies(Def->getOperand(1).getReg(), MRI);
776 
777  // Check if we can fold an extend and a shift.
778  // FIXME: This is duplicated with the selector. (See:
779  // selectArithExtendedRegister)
780  if (IsSupportedExtend(*ShiftLHS))
781  return (ShiftAmt <= 4) ? 2 : 1;
782 
783  LLT Ty = MRI.getType(Def->getOperand(0).getReg());
784  if (Ty.isVector())
785  return 0;
786  unsigned ShiftSize = Ty.getSizeInBits();
787  if ((ShiftSize == 32 && ShiftAmt <= 31) ||
788  (ShiftSize == 64 && ShiftAmt <= 63))
789  return 1;
790  return 0;
791 }
792 
793 /// \returns true if it would be profitable to swap the LHS and RHS of a G_ICMP
794 /// instruction \p MI.
796  const MachineRegisterInfo &MRI) {
797  assert(MI.getOpcode() == TargetOpcode::G_ICMP);
798  // Swap the operands if it would introduce a profitable folding opportunity.
799  // (e.g. a shift + extend).
800  //
801  // For example:
802  // lsl w13, w11, #1
803  // cmp w13, w12
804  // can be turned into:
805  // cmp w12, w11, lsl #1
806 
807  // Don't swap if there's a constant on the RHS, because we know we can fold
808  // that.
809  Register RHS = MI.getOperand(3).getReg();
810  auto RHSCst = getConstantVRegValWithLookThrough(RHS, MRI);
811  if (RHSCst && isLegalArithImmed(RHSCst->Value.getSExtValue()))
812  return false;
813 
814  Register LHS = MI.getOperand(2).getReg();
815  auto Pred = static_cast<CmpInst::Predicate>(MI.getOperand(1).getPredicate());
816  auto GetRegForProfit = [&](Register Reg) {
818  return isCMN(Def, Pred, MRI) ? Def->getOperand(2).getReg() : Reg;
819  };
820 
821  // Don't have a constant on the RHS. If we swap the LHS and RHS of the
822  // compare, would we be able to fold more instructions?
823  Register TheLHS = GetRegForProfit(LHS);
824  Register TheRHS = GetRegForProfit(RHS);
825 
826  // If the LHS is more likely to give us a folding opportunity, then swap the
827  // LHS and RHS.
828  return (getCmpOperandFoldingProfit(TheLHS, MRI) >
830 }
831 
833  GISelChangeObserver &Observer) {
834  auto Pred = static_cast<CmpInst::Predicate>(MI.getOperand(1).getPredicate());
835  Register LHS = MI.getOperand(2).getReg();
836  Register RHS = MI.getOperand(3).getReg();
837  Observer.changedInstr(MI);
838  MI.getOperand(1).setPredicate(CmpInst::getSwappedPredicate(Pred));
839  MI.getOperand(2).setReg(RHS);
840  MI.getOperand(3).setReg(LHS);
841  Observer.changedInstr(MI);
842  return true;
843 }
844 
845 #define AARCH64POSTLEGALIZERLOWERINGHELPER_GENCOMBINERHELPER_DEPS
846 #include "AArch64GenPostLegalizeGILowering.inc"
847 #undef AARCH64POSTLEGALIZERLOWERINGHELPER_GENCOMBINERHELPER_DEPS
848 
849 namespace {
850 #define AARCH64POSTLEGALIZERLOWERINGHELPER_GENCOMBINERHELPER_H
851 #include "AArch64GenPostLegalizeGILowering.inc"
852 #undef AARCH64POSTLEGALIZERLOWERINGHELPER_GENCOMBINERHELPER_H
853 
854 class AArch64PostLegalizerLoweringInfo : public CombinerInfo {
855 public:
856  AArch64GenPostLegalizerLoweringHelperRuleConfig GeneratedRuleCfg;
857 
858  AArch64PostLegalizerLoweringInfo(bool OptSize, bool MinSize)
859  : CombinerInfo(/*AllowIllegalOps*/ true, /*ShouldLegalizeIllegal*/ false,
860  /*LegalizerInfo*/ nullptr, /*OptEnabled = */ true, OptSize,
861  MinSize) {
862  if (!GeneratedRuleCfg.parseCommandLineOption())
863  report_fatal_error("Invalid rule identifier");
864  }
865 
866  virtual bool combine(GISelChangeObserver &Observer, MachineInstr &MI,
867  MachineIRBuilder &B) const override;
868 };
869 
871  MachineInstr &MI,
872  MachineIRBuilder &B) const {
873  CombinerHelper Helper(Observer, B);
874  AArch64GenPostLegalizerLoweringHelper Generated(GeneratedRuleCfg);
875  return Generated.tryCombineAll(Observer, MI, B, Helper);
876 }
877 
878 #define AARCH64POSTLEGALIZERLOWERINGHELPER_GENCOMBINERHELPER_CPP
879 #include "AArch64GenPostLegalizeGILowering.inc"
880 #undef AARCH64POSTLEGALIZERLOWERINGHELPER_GENCOMBINERHELPER_CPP
881 
882 class AArch64PostLegalizerLowering : public MachineFunctionPass {
883 public:
884  static char ID;
885 
886  AArch64PostLegalizerLowering();
887 
888  StringRef getPassName() const override {
889  return "AArch64PostLegalizerLowering";
890  }
891 
892  bool runOnMachineFunction(MachineFunction &MF) override;
893  void getAnalysisUsage(AnalysisUsage &AU) const override;
894 };
895 } // end anonymous namespace
896 
897 void AArch64PostLegalizerLowering::getAnalysisUsage(AnalysisUsage &AU) const {
899  AU.setPreservesCFG();
902 }
903 
904 AArch64PostLegalizerLowering::AArch64PostLegalizerLowering()
906  initializeAArch64PostLegalizerLoweringPass(*PassRegistry::getPassRegistry());
907 }
908 
909 bool AArch64PostLegalizerLowering::runOnMachineFunction(MachineFunction &MF) {
910  if (MF.getProperties().hasProperty(
911  MachineFunctionProperties::Property::FailedISel))
912  return false;
914  MachineFunctionProperties::Property::Legalized) &&
915  "Expected a legalized function?");
916  auto *TPC = &getAnalysis<TargetPassConfig>();
917  const Function &F = MF.getFunction();
918  AArch64PostLegalizerLoweringInfo PCInfo(F.hasOptSize(), F.hasMinSize());
919  Combiner C(PCInfo, TPC);
920  return C.combineMachineInstrs(MF, /*CSEInfo*/ nullptr);
921 }
922 
924 INITIALIZE_PASS_BEGIN(AArch64PostLegalizerLowering, DEBUG_TYPE,
925  "Lower AArch64 MachineInstrs after legalization", false,
926  false)
928 INITIALIZE_PASS_END(AArch64PostLegalizerLowering, DEBUG_TYPE,
929  "Lower AArch64 MachineInstrs after legalization", false,
930  false)
931 
932 namespace llvm {
934  return new AArch64PostLegalizerLowering();
935 }
936 } // end namespace llvm
llvm::Check::Size
@ Size
Definition: FileCheck.h:73
i
i
Definition: README.txt:29
MIPatternMatch.h
ShuffleVectorPseudo::ShuffleVectorPseudo
ShuffleVectorPseudo()
Definition: AArch64PostLegalizerLowering.cpp:55
CombinerInfo.h
applyShuffleVectorPseudo
static bool applyShuffleVectorPseudo(MachineInstr &MI, ShuffleVectorPseudo &MatchInfo)
Replace a G_SHUFFLE_VECTOR instruction with a pseudo.
Definition: AArch64PostLegalizerLowering.cpp:390
applyDupLane
bool applyDupLane(MachineInstr &MI, MachineRegisterInfo &MRI, MachineIRBuilder &B, std::pair< unsigned, int > &MatchInfo)
Definition: AArch64PostLegalizerLowering.cpp:690
llvm::MachineFunctionProperties::hasProperty
bool hasProperty(Property P) const
Definition: MachineFunction.h:162
llvm::getDefIgnoringCopies
MachineInstr * getDefIgnoringCopies(Register Reg, const MachineRegisterInfo &MRI)
Find the def instruction for Reg, folding away any trivial copies.
Definition: Utils.cpp:396
MI
IRTranslator LLVM IR MI
Definition: IRTranslator.cpp:100
llvm::CmpInst::getSwappedPredicate
Predicate getSwappedPredicate() const
For example, EQ->EQ, SLE->SGE, ULT->UGT, OEQ->OEQ, ULE->UGE, OLT->OGT, etc.
Definition: InstrTypes.h:839
llvm
Definition: AllocatorList.h:23
llvm::tgtok::Def
@ Def
Definition: TGLexer.h:50
Reg
unsigned Reg
Definition: MachineSink.cpp:1566
BlockSize
static const int BlockSize
Definition: TarWriter.cpp:33
M
We currently emits eax Perhaps this is what we really should generate is Is imull three or four cycles eax eax The current instruction priority is based on pattern complexity The former is more complex because it folds a load so the latter will not be emitted Perhaps we should use AddedComplexity to give LEA32r a higher priority We should always try to match LEA first since the LEA matching code does some estimate to determine whether the match is profitable if we care more about code then imull is better It s two bytes shorter than movl leal On a Pentium M
Definition: README.txt:252
llvm::make_range
iterator_range< T > make_range(T x, T y)
Convenience function for iterating over sub-ranges.
Definition: iterator_range.h:53
llvm::LLT::getScalarSizeInBits
unsigned getScalarSizeInBits() const
Definition: LowLevelTypeImpl.h:163
llvm::CmpInst::Predicate
Predicate
This enumeration lists the possible predicates for CmpInst subclasses.
Definition: InstrTypes.h:722
llvm::MachineRegisterInfo
MachineRegisterInfo - Keep track of information for virtual and physical registers,...
Definition: MachineRegisterInfo.h:52
matchDupFromInsertVectorElt
static bool matchDupFromInsertVectorElt(int Lane, MachineInstr &MI, MachineRegisterInfo &MRI, ShuffleVectorPseudo &MatchInfo)
Helper function for matchDup.
Definition: AArch64PostLegalizerLowering.cpp:295
llvm::Function
Definition: Function.h:61
llvm::AArch64GISelUtils::getAArch64VectorSplat
Optional< RegOrConstant > getAArch64VectorSplat(const MachineInstr &MI, const MachineRegisterInfo &MRI)
Definition: AArch64GlobalISelUtils.cpp:19
P
This currently compiles esp xmm0 movsd esp eax eax esp ret We should use not the dag combiner This is because dagcombine2 needs to be able to see through the X86ISD::Wrapper which DAGCombine can t really do The code for turning x load into a single vector load is target independent and should be moved to the dag combiner The code for turning x load into a vector load can only handle a direct load from a global or a direct load from the stack It should be generalized to handle any load from P
Definition: README-SSE.txt:411
llvm::getOpcodeDef
MachineInstr * getOpcodeDef(unsigned Opcode, Register Reg, const MachineRegisterInfo &MRI)
See if Reg is defined by an single def instruction that is Opcode.
Definition: Utils.cpp:410
llvm::SmallVector< SrcOp, 2 >
llvm::MachineIRBuilder::getMRI
MachineRegisterInfo * getMRI()
Getter for MRI.
Definition: MachineIRBuilder.h:288
llvm::MachineFunctionPass
MachineFunctionPass - This class adapts the FunctionPass interface to allow convenient creation of pa...
Definition: MachineFunctionPass.h:30
isUZPMask
static bool isUZPMask(ArrayRef< int > M, unsigned NumElts, unsigned &WhichResult)
Determines if M is a shuffle vector mask for a UZP of NumElts.
Definition: AArch64PostLegalizerLowering.cpp:147
Right
Vector Shift Left Right
Definition: README_P9.txt:118
applyAdjustICmpImmAndPred
bool applyAdjustICmpImmAndPred(MachineInstr &MI, std::pair< uint64_t, CmpInst::Predicate > &MatchInfo, MachineIRBuilder &MIB, GISelChangeObserver &Observer)
Definition: AArch64PostLegalizerLowering.cpp:623
llvm::CmpInst::ICMP_SGT
@ ICMP_SGT
signed greater than
Definition: InstrTypes.h:749
matchDupLane
bool matchDupLane(MachineInstr &MI, MachineRegisterInfo &MRI, std::pair< unsigned, int > &MatchInfo)
Definition: AArch64PostLegalizerLowering.cpp:638
INITIALIZE_PASS_BEGIN
INITIALIZE_PASS_BEGIN(AArch64PostLegalizerLowering, DEBUG_TYPE, "Lower AArch64 MachineInstrs after legalization", false, false) INITIALIZE_PASS_END(AArch64PostLegalizerLowering
matchZip
static bool matchZip(MachineInstr &MI, MachineRegisterInfo &MRI, ShuffleVectorPseudo &MatchInfo)
Definition: AArch64PostLegalizerLowering.cpp:278
llvm::Optional
Definition: APInt.h:33
llvm::getSelectionDAGFallbackAnalysisUsage
void getSelectionDAGFallbackAnalysisUsage(AnalysisUsage &AU)
Modify analysis usage so it preserves passes required for the SelectionDAG fallback.
Definition: Utils.cpp:741
llvm::CmpInst::ICMP_SLE
@ ICMP_SLE
signed less or equal
Definition: InstrTypes.h:752
matchREV
static bool matchREV(MachineInstr &MI, MachineRegisterInfo &MRI, ShuffleVectorPseudo &MatchInfo)
Definition: AArch64PostLegalizerLowering.cpp:212
MachineIRBuilder.h
llvm::CombinerInfo
Definition: CombinerInfo.h:27
llvm::BitmaskEnumDetail::Mask
std::underlying_type_t< E > Mask()
Get a bitmask with 1s in all places up to the high-order bit of E's largest value.
Definition: BitmaskEnum.h:80
getCmpOperandFoldingProfit
static unsigned getCmpOperandFoldingProfit(Register CmpOp, const MachineRegisterInfo &MRI)
Definition: AArch64PostLegalizerLowering.cpp:740
INITIALIZE_PASS_END
INITIALIZE_PASS_END(RegBankSelect, DEBUG_TYPE, "Assign register bank of generic virtual registers", false, false) RegBankSelect
Definition: RegBankSelect.cpp:69
llvm::MachineFunctionPass::getAnalysisUsage
void getAnalysisUsage(AnalysisUsage &AU) const override
getAnalysisUsage - Subclasses that override getAnalysisUsage must call this.
Definition: MachineFunctionPass.cpp:102
llvm::MachineIRBuilder::buildConstant
virtual MachineInstrBuilder buildConstant(const DstOp &Res, const ConstantInt &Val)
Build and insert Res = G_CONSTANT Val.
Definition: MachineIRBuilder.cpp:255
F
#define F(x, y, z)
Definition: MD5.cpp:56
MachineRegisterInfo.h
llvm::AlignStyle::Left
@ Left
llvm::createAArch64PostLegalizerLowering
FunctionPass * createAArch64PostLegalizerLowering()
Definition: AArch64PostLegalizerLowering.cpp:933
AArch64TargetMachine.h
llvm::LLT::changeNumElements
LLT changeNumElements(unsigned NewNumElts) const
Return a vector or scalar with the same element type and the new number of elements.
Definition: LowLevelTypeImpl.h:143
llvm::LLT::getSizeInBits
unsigned getSizeInBits() const
Returns the total size of the type. Must only be called on sized types.
Definition: LowLevelTypeImpl.h:109
C
(vector float) vec_cmpeq(*A, *B) C
Definition: README_ALTIVEC.txt:86
applyBuildVectorToDup
static bool applyBuildVectorToDup(MachineInstr &MI, MachineRegisterInfo &MRI, MachineIRBuilder &B)
Definition: AArch64PostLegalizerLowering.cpp:729
llvm::CmpInst::ICMP_ULE
@ ICMP_ULE
unsigned less or equal
Definition: InstrTypes.h:748
ShuffleVectorPseudo::ShuffleVectorPseudo
ShuffleVectorPseudo(unsigned Opc, Register Dst, std::initializer_list< SrcOp > SrcOps)
Definition: AArch64PostLegalizerLowering.cpp:52
llvm::AnalysisUsage
Represent the analysis usage information of a pass.
Definition: PassAnalysisSupport.h:47
Utils.h
llvm::MachineFunction::getProperties
const MachineFunctionProperties & getProperties() const
Get the function properties.
Definition: MachineFunction.h:646
UINT64_MAX
#define UINT64_MAX
Definition: DataTypes.h:77
false
Definition: StackSlotColoring.cpp:142
TargetOpcodes.h
B
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
llvm::MachineOperand
MachineOperand class - Representation of each machine instruction operand.
Definition: MachineOperand.h:49
llvm::MIPatternMatch::m_ZeroInt
SpecificConstantMatch m_ZeroInt()
{ Convenience matchers for specific integer values.
Definition: MIPatternMatch.h:88
llvm::report_fatal_error
LLVM_ATTRIBUTE_NORETURN void report_fatal_error(Error Err, bool gen_crash_diag=true)
Report a serious error, calling any installed error handler.
Definition: Error.cpp:140
llvm::APInt::getZExtValue
uint64_t getZExtValue() const
Get zero extended value.
Definition: APInt.h:1631
INT64_MAX
#define INT64_MAX
Definition: DataTypes.h:71
llvm::LLT::vector
static LLT vector(uint16_t NumElements, unsigned ScalarSizeInBits)
Get a low-level vector of some number of elements and element width.
Definition: LowLevelTypeImpl.h:58
llvm::GISelChangeObserver::changingInstr
virtual void changingInstr(MachineInstr &MI)=0
This instruction is about to be mutated in some way.
llvm::CombinerHelper
Definition: CombinerHelper.h:89
llvm::MachineRegisterInfo::getVRegDef
MachineInstr * getVRegDef(Register Reg) const
getVRegDef - Return the machine instr that defines the specified virtual register or null if none is ...
Definition: MachineRegisterInfo.cpp:400
llvm::None
const NoneType None
Definition: None.h:23
isREVMask
static bool isREVMask(ArrayRef< int > M, unsigned EltSize, unsigned NumElts, unsigned BlockSize)
Check if a vector shuffle corresponds to a REV instruction with the specified blocksize.
Definition: AArch64PostLegalizerLowering.cpp:60
ShuffleVectorPseudo::SrcOps
SmallVector< SrcOp, 2 > SrcOps
Source registers.
Definition: AArch64PostLegalizerLowering.cpp:51
isTRNMask
static bool isTRNMask(ArrayRef< int > M, unsigned NumElts, unsigned &WhichResult)
Determines if M is a shuffle vector mask for a TRN of NumElts.
Definition: AArch64PostLegalizerLowering.cpp:89
llvm::TargetPassConfig
Target-Independent Code Generator Pass Configuration Options.
Definition: TargetPassConfig.h:84
llvm::GISelChangeObserver::changedInstr
virtual void changedInstr(MachineInstr &MI)=0
This instruction was mutated in some way.
AArch64GlobalISelUtils.h
isVShiftRImm
static bool isVShiftRImm(Register Reg, MachineRegisterInfo &MRI, LLT Ty, int64_t &Cnt)
isVShiftRImm - Check if this is a valid vector for the immediate operand of a vector shift right oper...
Definition: AArch64PostLegalizerLowering.cpp:470
llvm::MachineIRBuilder
Helper class to build MachineInstr.
Definition: MachineIRBuilder.h:220
llvm::MachineInstr
Representation of each machine instruction.
Definition: MachineInstr.h:64
ShuffleVectorPseudo::Opc
unsigned Opc
Opcode for the instruction. (E.g. G_ZIP1)
Definition: AArch64PostLegalizerLowering.cpp:49
llvm::Combiner
Definition: Combiner.h:27
isZipMask
static bool isZipMask(ArrayRef< int > M, unsigned NumElts, unsigned &WhichResult)
Definition: AArch64PostLegalizerLowering.cpp:162
applySwapICmpOperands
static bool applySwapICmpOperands(MachineInstr &MI, GISelChangeObserver &Observer)
Definition: AArch64PostLegalizerLowering.cpp:832
isCMN
static bool isCMN(SDValue Op, ISD::CondCode CC)
Definition: AArch64ISelLowering.cpp:2353
INITIALIZE_PASS_DEPENDENCY
INITIALIZE_PASS_DEPENDENCY(DominatorTreeWrapperPass)
llvm::APInt::logBase2
unsigned logBase2() const
Definition: APInt.h:1816
llvm::initializeAArch64PostLegalizerLoweringPass
void initializeAArch64PostLegalizerLoweringPass(PassRegistry &)
matchDupFromBuildVector
static bool matchDupFromBuildVector(int Lane, MachineInstr &MI, MachineRegisterInfo &MRI, ShuffleVectorPseudo &MatchInfo)
Helper function for matchDup.
Definition: AArch64PostLegalizerLowering.cpp:334
llvm::MachineIRBuilder::setInstrAndDebugLoc
void setInstrAndDebugLoc(MachineInstr &MI)
Set the insertion point to before MI, and set the debug loc to MI's loc.
Definition: MachineIRBuilder.h:350
ShuffleVectorPseudo::Dst
Register Dst
Destination register.
Definition: AArch64PostLegalizerLowering.cpp:50
llvm::LLT::isVector
bool isVector() const
Definition: LowLevelTypeImpl.h:96
llvm::LLT::getNumElements
uint16_t getNumElements() const
Returns the number of elements in a vector LLT.
Definition: LowLevelTypeImpl.h:100
TargetPassConfig.h
MachineFunctionPass.h
ShuffleVectorPseudo
Represents a pseudo instruction which replaces a G_SHUFFLE_VECTOR.
Definition: AArch64PostLegalizerLowering.cpp:48
assert
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
std::swap
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
Definition: BitVector.h:958
llvm::CmpInst::ICMP_UGE
@ ICMP_UGE
unsigned greater or equal
Definition: InstrTypes.h:746
matchUZP
static bool matchUZP(MachineInstr &MI, MachineRegisterInfo &MRI, ShuffleVectorPseudo &MatchInfo)
Definition: AArch64PostLegalizerLowering.cpp:262
matchBuildVectorToDup
static bool matchBuildVectorToDup(MachineInstr &MI, MachineRegisterInfo &MRI)
Definition: AArch64PostLegalizerLowering.cpp:715
matchAdjustICmpImmAndPred
bool matchAdjustICmpImmAndPred(MachineInstr &MI, const MachineRegisterInfo &MRI, std::pair< uint64_t, CmpInst::Predicate > &MatchInfo)
Determine whether or not it is possible to update the RHS and predicate of a G_ICMP instruction such ...
Definition: AArch64PostLegalizerLowering.cpp:610
llvm::MachineOperand::getReg
Register getReg() const
getReg - Returns the register number.
Definition: MachineOperand.h:357
DEBUG_TYPE
#define DEBUG_TYPE
Definition: AArch64PostLegalizerLowering.cpp:39
Builder
assume Assume Builder
Definition: AssumeBundleBuilder.cpp:649
llvm::APInt
Class for arbitrary precision integers.
Definition: APInt.h:70
llvm::MachineFunction
Definition: MachineFunction.h:227
llvm::CmpInst::ICMP_SLT
@ ICMP_SLT
signed less than
Definition: InstrTypes.h:751
llvm::MachineIRBuilder::buildInstr
MachineInstrBuilder buildInstr(unsigned Opcode)
Build and insert <empty> = Opcode <empty>.
Definition: MachineIRBuilder.h:375
tryAdjustICmpImmAndPred
Optional< std::pair< uint64_t, CmpInst::Predicate > > tryAdjustICmpImmAndPred(Register RHS, CmpInst::Predicate P, const MachineRegisterInfo &MRI)
Determine if it is possible to modify the RHS and predicate P of a G_ICMP instruction such that the r...
Definition: AArch64PostLegalizerLowering.cpp:514
CombinerHelper.h
matchDup
static bool matchDup(MachineInstr &MI, MachineRegisterInfo &MRI, ShuffleVectorPseudo &MatchInfo)
Definition: AArch64PostLegalizerLowering.cpp:350
matchEXT
static bool matchEXT(MachineInstr &MI, MachineRegisterInfo &MRI, ShuffleVectorPseudo &MatchInfo)
Definition: AArch64PostLegalizerLowering.cpp:367
llvm::ArrayRef< int >
legalization
Lower AArch64 MachineInstrs after legalization
Definition: AArch64PostLegalizerLowering.cpp:929
llvm::any_of
bool any_of(R &&range, UnaryPredicate P)
Provide wrappers to std::any_of which take ranges instead of having to pass begin/end explicitly.
Definition: STLExtras.h:1512
llvm::AnalysisUsage::setPreservesCFG
void setPreservesCFG()
This function should be called by the pass, iff they do not:
Definition: Pass.cpp:253
llvm::StringRef
StringRef - Represent a constant reference to a string, i.e.
Definition: StringRef.h:57
llvm::MachineRegisterInfo::hasOneNonDBGUse
bool hasOneNonDBGUse(Register RegNo) const
hasOneNonDBGUse - Return true if there is exactly one non-Debug use of the specified register.
Definition: MachineRegisterInfo.cpp:419
llvm::CmpInst::ICMP_ULT
@ ICMP_ULT
unsigned less than
Definition: InstrTypes.h:747
Combiner.h
uint32_t
isINSMask
static Optional< std::pair< bool, int > > isINSMask(ArrayRef< int > M, int NumInputElements)
Helper function for matchINS.
Definition: AArch64PostLegalizerLowering.cpp:187
llvm::GISelChangeObserver
Abstract class that contains various methods for clients to notify about changes.
Definition: GISelChangeObserver.h:29
MRI
unsigned const MachineRegisterInfo * MRI
Definition: AArch64AdvSIMDScalarPass.cpp:105
llvm::Register
Wrapper class representing virtual and physical registers.
Definition: Register.h:19
applyVAshrLshrImm
static bool applyVAshrLshrImm(MachineInstr &MI, MachineRegisterInfo &MRI, int64_t &Imm)
Definition: AArch64PostLegalizerLowering.cpp:493
llvm::find_if
auto find_if(R &&Range, UnaryPredicate P)
Provide wrappers to std::find_if which take ranges instead of having to pass begin/end explicitly.
Definition: STLExtras.h:1532
llvm::NVPTX::PTXLdStInstCode::V2
@ V2
Definition: NVPTX.h:123
llvm::MachineFunction::getFunction
Function & getFunction()
Return the LLVM function that this machine code represents.
Definition: MachineFunction.h:521
llvm::RegState::Undef
@ Undef
Value of the register doesn't matter.
Definition: MachineInstrBuilder.h:53
applyINS
static bool applyINS(MachineInstr &MI, MachineRegisterInfo &MRI, MachineIRBuilder &Builder, std::tuple< Register, int, Register, int > &MatchInfo)
Definition: AArch64PostLegalizerLowering.cpp:450
matchVAshrLshrImm
static bool matchVAshrLshrImm(MachineInstr &MI, MachineRegisterInfo &MRI, int64_t &Imm)
Match a vector G_ASHR or G_LSHR with a valid immediate shift.
Definition: AArch64PostLegalizerLowering.cpp:483
matchINS
static bool matchINS(MachineInstr &MI, MachineRegisterInfo &MRI, std::tuple< Register, int, Register, int > &MatchInfo)
Match a G_SHUFFLE_VECTOR with a mask which corresponds to a G_INSERT_VECTOR_ELT and G_EXTRACT_VECTOR_...
Definition: AArch64PostLegalizerLowering.cpp:423
AArch64MCTargetDesc.h
trySwapICmpOperands
static bool trySwapICmpOperands(MachineInstr &MI, const MachineRegisterInfo &MRI)
Definition: AArch64PostLegalizerLowering.cpp:795
llvm::CmpInst::ICMP_SGE
@ ICMP_SGE
signed greater or equal
Definition: InstrTypes.h:750
applyEXT
static bool applyEXT(MachineInstr &MI, ShuffleVectorPseudo &MatchInfo)
Replace a G_SHUFFLE_VECTOR instruction with G_EXT.
Definition: AArch64PostLegalizerLowering.cpp:401
llvm::getSplatIndex
int getSplatIndex(ArrayRef< int > Mask)
If all non-negative Mask elements are the same value, return that value.
Definition: VectorUtils.cpp:328
llvm::MachineRegisterInfo::getType
LLT getType(Register Reg) const
Get the low-level type of Reg or LLT{} if Reg is not a generic (target independent) virtual register.
Definition: MachineRegisterInfo.h:732
MachineInstrBuilder.h
llvm::MachineRegisterInfo::cloneVirtualRegister
Register cloneVirtualRegister(Register VReg, StringRef Name="")
Create and return a new virtual register in the function with the same attributes as the given regist...
Definition: MachineRegisterInfo.cpp:172
llvm::CmpInst::ICMP_UGT
@ ICMP_UGT
unsigned greater than
Definition: InstrTypes.h:745
getExtMask
static Optional< std::pair< bool, uint64_t > > getExtMask(ArrayRef< int > M, unsigned NumElts)
Check if a G_EXT instruction can handle a shuffle mask M when the vector sources of the shuffle are d...
Definition: AArch64PostLegalizerLowering.cpp:105
llvm::MachineOperand::setReg
void setReg(Register Reg)
Change the register this operand corresponds to.
Definition: MachineOperand.cpp:55
llvm::FunctionPass
FunctionPass class - This class is used to implement most global optimizations.
Definition: Pass.h:298
matchTRN
static bool matchTRN(MachineInstr &MI, MachineRegisterInfo &MRI, ShuffleVectorPseudo &MatchInfo)
Definition: AArch64PostLegalizerLowering.cpp:241
llvm::MIPatternMatch::mi_match
bool mi_match(Reg R, const MachineRegisterInfo &MRI, Pattern &&P)
Definition: MIPatternMatch.h:24
llvm::AnalysisUsage::addRequired
AnalysisUsage & addRequired()
Definition: PassAnalysisSupport.h:75
llvm::LLT::getElementType
LLT getElementType() const
Returns the vector's element type. Only valid for vector types.
Definition: LowLevelTypeImpl.h:188
llvm::LegalizeActions::Lower
@ Lower
The operation itself must be expressed in terms of simpler actions on this target.
Definition: LegalizerInfo.h:76
combine
vector combine
Definition: VectorCombine.cpp:833
llvm::LLT::scalar
static LLT scalar(unsigned SizeInBits)
Get a low-level scalar or aggregate "bag of bits".
Definition: LowLevelTypeImpl.h:43
llvm::AArch64GISelUtils::getAArch64VectorSplatScalar
Optional< int64_t > getAArch64VectorSplatScalar(const MachineInstr &MI, const MachineRegisterInfo &MRI)
Definition: AArch64GlobalISelUtils.cpp:33
InitializePasses.h
Debug.h
llvm::getConstantVRegValWithLookThrough
Optional< ValueAndVReg > getConstantVRegValWithLookThrough(Register VReg, const MachineRegisterInfo &MRI, bool LookThroughInstrs=true, bool HandleFConstants=true, bool LookThroughAnyExt=false)
If VReg is defined by a statically evaluable chain of instructions rooted on a G_F/CONSTANT (LookThro...
Definition: Utils.cpp:289
getReg
static unsigned getReg(const void *D, unsigned RC, unsigned RegNo)
Definition: MipsDisassembler.cpp:580
isLegalArithImmed
static bool isLegalArithImmed(uint64_t C)
Definition: AArch64ISelLowering.cpp:2335
llvm::Intrinsic::ID
unsigned ID
Definition: TargetTransformInfo.h:38
llvm::LLT
Definition: LowLevelTypeImpl.h:40
INT64_MIN
#define INT64_MIN
Definition: DataTypes.h:74