LLVM  14.0.0git
LegalizeVectorOps.cpp
Go to the documentation of this file.
1 //===- LegalizeVectorOps.cpp - Implement SelectionDAG::LegalizeVectors ----===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file implements the SelectionDAG::LegalizeVectors method.
10 //
11 // The vector legalizer looks for vector operations which might need to be
12 // scalarized and legalizes them. This is a separate step from Legalize because
13 // scalarizing can introduce illegal types. For example, suppose we have an
14 // ISD::SDIV of type v2i64 on x86-32. The type is legal (for example, addition
15 // on a v2i64 is legal), but ISD::SDIV isn't legal, so we have to unroll the
16 // operation, which introduces nodes with the illegal type i64 which must be
17 // expanded. Similarly, suppose we have an ISD::SRA of type v16i8 on PowerPC;
18 // the operation must be unrolled, which introduces nodes with the illegal
19 // type i8 which must be promoted.
20 //
21 // This does not legalize vector manipulations like ISD::BUILD_VECTOR,
22 // or operations that happen to take a vector which are custom-lowered;
23 // the legalization for such operations never produces nodes
24 // with illegal types, so it's okay to put off legalizing them until
25 // SelectionDAG::Legalize runs.
26 //
27 //===----------------------------------------------------------------------===//
28 
29 #include "llvm/ADT/APInt.h"
30 #include "llvm/ADT/DenseMap.h"
31 #include "llvm/ADT/SmallVector.h"
38 #include "llvm/IR/DataLayout.h"
39 #include "llvm/Support/Casting.h"
40 #include "llvm/Support/Compiler.h"
41 #include "llvm/Support/Debug.h"
45 #include <cassert>
46 #include <cstdint>
47 #include <iterator>
48 #include <utility>
49 
50 using namespace llvm;
51 
52 #define DEBUG_TYPE "legalizevectorops"
53 
54 namespace {
55 
56 class VectorLegalizer {
57  SelectionDAG& DAG;
58  const TargetLowering &TLI;
59  bool Changed = false; // Keep track of whether anything changed
60 
61  /// For nodes that are of legal width, and that have more than one use, this
62  /// map indicates what regularized operand to use. This allows us to avoid
63  /// legalizing the same thing more than once.
65 
66  /// Adds a node to the translation cache.
67  void AddLegalizedOperand(SDValue From, SDValue To) {
68  LegalizedNodes.insert(std::make_pair(From, To));
69  // If someone requests legalization of the new node, return itself.
70  if (From != To)
71  LegalizedNodes.insert(std::make_pair(To, To));
72  }
73 
74  /// Legalizes the given node.
75  SDValue LegalizeOp(SDValue Op);
76 
77  /// Assuming the node is legal, "legalize" the results.
78  SDValue TranslateLegalizeResults(SDValue Op, SDNode *Result);
79 
80  /// Make sure Results are legal and update the translation cache.
81  SDValue RecursivelyLegalizeResults(SDValue Op,
83 
84  /// Wrapper to interface LowerOperation with a vector of Results.
85  /// Returns false if the target wants to use default expansion. Otherwise
86  /// returns true. If return is true and the Results are empty, then the
87  /// target wants to keep the input node as is.
88  bool LowerOperationWrapper(SDNode *N, SmallVectorImpl<SDValue> &Results);
89 
90  /// Implements unrolling a VSETCC.
91  SDValue UnrollVSETCC(SDNode *Node);
92 
93  /// Implement expand-based legalization of vector operations.
94  ///
95  /// This is just a high-level routine to dispatch to specific code paths for
96  /// operations to legalize them.
97  void Expand(SDNode *Node, SmallVectorImpl<SDValue> &Results);
98 
99  /// Implements expansion for FP_TO_UINT; falls back to UnrollVectorOp if
100  /// FP_TO_SINT isn't legal.
101  void ExpandFP_TO_UINT(SDNode *Node, SmallVectorImpl<SDValue> &Results);
102 
103  /// Implements expansion for UINT_TO_FLOAT; falls back to UnrollVectorOp if
104  /// SINT_TO_FLOAT and SHR on vectors isn't legal.
105  void ExpandUINT_TO_FLOAT(SDNode *Node, SmallVectorImpl<SDValue> &Results);
106 
107  /// Implement expansion for SIGN_EXTEND_INREG using SRL and SRA.
108  SDValue ExpandSEXTINREG(SDNode *Node);
109 
110  /// Implement expansion for ANY_EXTEND_VECTOR_INREG.
111  ///
112  /// Shuffles the low lanes of the operand into place and bitcasts to the proper
113  /// type. The contents of the bits in the extended part of each element are
114  /// undef.
115  SDValue ExpandANY_EXTEND_VECTOR_INREG(SDNode *Node);
116 
117  /// Implement expansion for SIGN_EXTEND_VECTOR_INREG.
118  ///
119  /// Shuffles the low lanes of the operand into place, bitcasts to the proper
120  /// type, then shifts left and arithmetic shifts right to introduce a sign
121  /// extension.
122  SDValue ExpandSIGN_EXTEND_VECTOR_INREG(SDNode *Node);
123 
124  /// Implement expansion for ZERO_EXTEND_VECTOR_INREG.
125  ///
126  /// Shuffles the low lanes of the operand into place and blends zeros into
127  /// the remaining lanes, finally bitcasting to the proper type.
128  SDValue ExpandZERO_EXTEND_VECTOR_INREG(SDNode *Node);
129 
130  /// Expand bswap of vectors into a shuffle if legal.
131  SDValue ExpandBSWAP(SDNode *Node);
132 
133  /// Implement vselect in terms of XOR, AND, OR when blend is not
134  /// supported by the target.
135  SDValue ExpandVSELECT(SDNode *Node);
136  SDValue ExpandSELECT(SDNode *Node);
137  std::pair<SDValue, SDValue> ExpandLoad(SDNode *N);
138  SDValue ExpandStore(SDNode *N);
139  SDValue ExpandFNEG(SDNode *Node);
140  void ExpandFSUB(SDNode *Node, SmallVectorImpl<SDValue> &Results);
141  void ExpandSETCC(SDNode *Node, SmallVectorImpl<SDValue> &Results);
142  void ExpandBITREVERSE(SDNode *Node, SmallVectorImpl<SDValue> &Results);
143  void ExpandUADDSUBO(SDNode *Node, SmallVectorImpl<SDValue> &Results);
144  void ExpandSADDSUBO(SDNode *Node, SmallVectorImpl<SDValue> &Results);
145  void ExpandMULO(SDNode *Node, SmallVectorImpl<SDValue> &Results);
146  void ExpandFixedPointDiv(SDNode *Node, SmallVectorImpl<SDValue> &Results);
147  void ExpandStrictFPOp(SDNode *Node, SmallVectorImpl<SDValue> &Results);
148  void ExpandREM(SDNode *Node, SmallVectorImpl<SDValue> &Results);
149 
150  void UnrollStrictFPOp(SDNode *Node, SmallVectorImpl<SDValue> &Results);
151 
152  /// Implements vector promotion.
153  ///
154  /// This is essentially just bitcasting the operands to a different type and
155  /// bitcasting the result back to the original type.
156  void Promote(SDNode *Node, SmallVectorImpl<SDValue> &Results);
157 
158  /// Implements [SU]INT_TO_FP vector promotion.
159  ///
160  /// This is a [zs]ext of the input operand to a larger integer type.
161  void PromoteINT_TO_FP(SDNode *Node, SmallVectorImpl<SDValue> &Results);
162 
163  /// Implements FP_TO_[SU]INT vector promotion of the result type.
164  ///
165  /// It is promoted to a larger integer type. The result is then
166  /// truncated back to the original type.
167  void PromoteFP_TO_INT(SDNode *Node, SmallVectorImpl<SDValue> &Results);
168 
169 public:
170  VectorLegalizer(SelectionDAG& dag) :
171  DAG(dag), TLI(dag.getTargetLoweringInfo()) {}
172 
173  /// Begin legalizer the vector operations in the DAG.
174  bool Run();
175 };
176 
177 } // end anonymous namespace
178 
179 bool VectorLegalizer::Run() {
180  // Before we start legalizing vector nodes, check if there are any vectors.
181  bool HasVectors = false;
182  for (SelectionDAG::allnodes_iterator I = DAG.allnodes_begin(),
183  E = std::prev(DAG.allnodes_end()); I != std::next(E); ++I) {
184  // Check if the values of the nodes contain vectors. We don't need to check
185  // the operands because we are going to check their values at some point.
186  HasVectors = llvm::any_of(I->values(), [](EVT T) { return T.isVector(); });
187 
188  // If we found a vector node we can start the legalization.
189  if (HasVectors)
190  break;
191  }
192 
193  // If this basic block has no vectors then no need to legalize vectors.
194  if (!HasVectors)
195  return false;
196 
197  // The legalize process is inherently a bottom-up recursive process (users
198  // legalize their uses before themselves). Given infinite stack space, we
199  // could just start legalizing on the root and traverse the whole graph. In
200  // practice however, this causes us to run out of stack space on large basic
201  // blocks. To avoid this problem, compute an ordering of the nodes where each
202  // node is only legalized after all of its operands are legalized.
203  DAG.AssignTopologicalOrder();
204  for (SelectionDAG::allnodes_iterator I = DAG.allnodes_begin(),
205  E = std::prev(DAG.allnodes_end()); I != std::next(E); ++I)
206  LegalizeOp(SDValue(&*I, 0));
207 
208  // Finally, it's possible the root changed. Get the new root.
209  SDValue OldRoot = DAG.getRoot();
210  assert(LegalizedNodes.count(OldRoot) && "Root didn't get legalized?");
211  DAG.setRoot(LegalizedNodes[OldRoot]);
212 
213  LegalizedNodes.clear();
214 
215  // Remove dead nodes now.
216  DAG.RemoveDeadNodes();
217 
218  return Changed;
219 }
220 
221 SDValue VectorLegalizer::TranslateLegalizeResults(SDValue Op, SDNode *Result) {
222  assert(Op->getNumValues() == Result->getNumValues() &&
223  "Unexpected number of results");
224  // Generic legalization: just pass the operand through.
225  for (unsigned i = 0, e = Op->getNumValues(); i != e; ++i)
226  AddLegalizedOperand(Op.getValue(i), SDValue(Result, i));
227  return SDValue(Result, Op.getResNo());
228 }
229 
230 SDValue
231 VectorLegalizer::RecursivelyLegalizeResults(SDValue Op,
233  assert(Results.size() == Op->getNumValues() &&
234  "Unexpected number of results");
235  // Make sure that the generated code is itself legal.
236  for (unsigned i = 0, e = Results.size(); i != e; ++i) {
237  Results[i] = LegalizeOp(Results[i]);
238  AddLegalizedOperand(Op.getValue(i), Results[i]);
239  }
240 
241  return Results[Op.getResNo()];
242 }
243 
244 SDValue VectorLegalizer::LegalizeOp(SDValue Op) {
245  // Note that LegalizeOp may be reentered even from single-use nodes, which
246  // means that we always must cache transformed nodes.
247  DenseMap<SDValue, SDValue>::iterator I = LegalizedNodes.find(Op);
248  if (I != LegalizedNodes.end()) return I->second;
249 
250  // Legalize the operands
252  for (const SDValue &Oper : Op->op_values())
253  Ops.push_back(LegalizeOp(Oper));
254 
255  SDNode *Node = DAG.UpdateNodeOperands(Op.getNode(), Ops);
256 
257  if (Op.getOpcode() == ISD::LOAD) {
258  LoadSDNode *LD = cast<LoadSDNode>(Node);
259  ISD::LoadExtType ExtType = LD->getExtensionType();
260  if (LD->getMemoryVT().isVector() && ExtType != ISD::NON_EXTLOAD) {
261  LLVM_DEBUG(dbgs() << "\nLegalizing extending vector load: ";
262  Node->dump(&DAG));
263  switch (TLI.getLoadExtAction(LD->getExtensionType(), LD->getValueType(0),
264  LD->getMemoryVT())) {
265  default: llvm_unreachable("This action is not supported yet!");
267  return TranslateLegalizeResults(Op, Node);
268  case TargetLowering::Custom: {
269  SmallVector<SDValue, 2> ResultVals;
270  if (LowerOperationWrapper(Node, ResultVals)) {
271  if (ResultVals.empty())
272  return TranslateLegalizeResults(Op, Node);
273 
274  Changed = true;
275  return RecursivelyLegalizeResults(Op, ResultVals);
276  }
278  }
279  case TargetLowering::Expand: {
280  Changed = true;
281  std::pair<SDValue, SDValue> Tmp = ExpandLoad(Node);
282  AddLegalizedOperand(Op.getValue(0), Tmp.first);
283  AddLegalizedOperand(Op.getValue(1), Tmp.second);
284  return Op.getResNo() ? Tmp.first : Tmp.second;
285  }
286  }
287  }
288  } else if (Op.getOpcode() == ISD::STORE) {
289  StoreSDNode *ST = cast<StoreSDNode>(Node);
290  EVT StVT = ST->getMemoryVT();
291  MVT ValVT = ST->getValue().getSimpleValueType();
292  if (StVT.isVector() && ST->isTruncatingStore()) {
293  LLVM_DEBUG(dbgs() << "\nLegalizing truncating vector store: ";
294  Node->dump(&DAG));
295  switch (TLI.getTruncStoreAction(ValVT, StVT)) {
296  default: llvm_unreachable("This action is not supported yet!");
298  return TranslateLegalizeResults(Op, Node);
299  case TargetLowering::Custom: {
300  SmallVector<SDValue, 1> ResultVals;
301  if (LowerOperationWrapper(Node, ResultVals)) {
302  if (ResultVals.empty())
303  return TranslateLegalizeResults(Op, Node);
304 
305  Changed = true;
306  return RecursivelyLegalizeResults(Op, ResultVals);
307  }
309  }
310  case TargetLowering::Expand: {
311  Changed = true;
312  SDValue Chain = ExpandStore(Node);
313  AddLegalizedOperand(Op, Chain);
314  return Chain;
315  }
316  }
317  }
318  }
319 
320  bool HasVectorValueOrOp =
321  llvm::any_of(Node->values(), [](EVT T) { return T.isVector(); }) ||
322  llvm::any_of(Node->op_values(),
323  [](SDValue O) { return O.getValueType().isVector(); });
324  if (!HasVectorValueOrOp)
325  return TranslateLegalizeResults(Op, Node);
326 
328  EVT ValVT;
329  switch (Op.getOpcode()) {
330  default:
331  return TranslateLegalizeResults(Op, Node);
332  case ISD::MERGE_VALUES:
333  Action = TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0));
334  // This operation lies about being legal: when it claims to be legal,
335  // it should actually be expanded.
336  if (Action == TargetLowering::Legal)
337  Action = TargetLowering::Expand;
338  break;
339 #define DAG_INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC, DAGN) \
340  case ISD::STRICT_##DAGN:
341 #include "llvm/IR/ConstrainedOps.def"
342  ValVT = Node->getValueType(0);
343  if (Op.getOpcode() == ISD::STRICT_SINT_TO_FP ||
344  Op.getOpcode() == ISD::STRICT_UINT_TO_FP)
345  ValVT = Node->getOperand(1).getValueType();
346  Action = TLI.getOperationAction(Node->getOpcode(), ValVT);
347  // If we're asked to expand a strict vector floating-point operation,
348  // by default we're going to simply unroll it. That is usually the
349  // best approach, except in the case where the resulting strict (scalar)
350  // operations would themselves use the fallback mutation to non-strict.
351  // In that specific case, just do the fallback on the vector op.
352  if (Action == TargetLowering::Expand && !TLI.isStrictFPEnabled() &&
353  TLI.getStrictFPOperationAction(Node->getOpcode(), ValVT) ==
355  EVT EltVT = ValVT.getVectorElementType();
356  if (TLI.getOperationAction(Node->getOpcode(), EltVT)
358  TLI.getStrictFPOperationAction(Node->getOpcode(), EltVT)
360  Action = TargetLowering::Legal;
361  }
362  break;
363  case ISD::ADD:
364  case ISD::SUB:
365  case ISD::MUL:
366  case ISD::MULHS:
367  case ISD::MULHU:
368  case ISD::SDIV:
369  case ISD::UDIV:
370  case ISD::SREM:
371  case ISD::UREM:
372  case ISD::SDIVREM:
373  case ISD::UDIVREM:
374  case ISD::FADD:
375  case ISD::FSUB:
376  case ISD::FMUL:
377  case ISD::FDIV:
378  case ISD::FREM:
379  case ISD::AND:
380  case ISD::OR:
381  case ISD::XOR:
382  case ISD::SHL:
383  case ISD::SRA:
384  case ISD::SRL:
385  case ISD::FSHL:
386  case ISD::FSHR:
387  case ISD::ROTL:
388  case ISD::ROTR:
389  case ISD::ABS:
390  case ISD::BSWAP:
391  case ISD::BITREVERSE:
392  case ISD::CTLZ:
393  case ISD::CTTZ:
396  case ISD::CTPOP:
397  case ISD::SELECT:
398  case ISD::VSELECT:
399  case ISD::SELECT_CC:
400  case ISD::ZERO_EXTEND:
401  case ISD::ANY_EXTEND:
402  case ISD::TRUNCATE:
403  case ISD::SIGN_EXTEND:
404  case ISD::FP_TO_SINT:
405  case ISD::FP_TO_UINT:
406  case ISD::FNEG:
407  case ISD::FABS:
408  case ISD::FMINNUM:
409  case ISD::FMAXNUM:
410  case ISD::FMINNUM_IEEE:
411  case ISD::FMAXNUM_IEEE:
412  case ISD::FMINIMUM:
413  case ISD::FMAXIMUM:
414  case ISD::FCOPYSIGN:
415  case ISD::FSQRT:
416  case ISD::FSIN:
417  case ISD::FCOS:
418  case ISD::FPOWI:
419  case ISD::FPOW:
420  case ISD::FLOG:
421  case ISD::FLOG2:
422  case ISD::FLOG10:
423  case ISD::FEXP:
424  case ISD::FEXP2:
425  case ISD::FCEIL:
426  case ISD::FTRUNC:
427  case ISD::FRINT:
428  case ISD::FNEARBYINT:
429  case ISD::FROUND:
430  case ISD::FROUNDEVEN:
431  case ISD::FFLOOR:
432  case ISD::FP_ROUND:
433  case ISD::FP_EXTEND:
434  case ISD::FMA:
439  case ISD::SMIN:
440  case ISD::SMAX:
441  case ISD::UMIN:
442  case ISD::UMAX:
443  case ISD::SMUL_LOHI:
444  case ISD::UMUL_LOHI:
445  case ISD::SADDO:
446  case ISD::UADDO:
447  case ISD::SSUBO:
448  case ISD::USUBO:
449  case ISD::SMULO:
450  case ISD::UMULO:
451  case ISD::FCANONICALIZE:
452  case ISD::SADDSAT:
453  case ISD::UADDSAT:
454  case ISD::SSUBSAT:
455  case ISD::USUBSAT:
456  case ISD::SSHLSAT:
457  case ISD::USHLSAT:
458  case ISD::FP_TO_SINT_SAT:
459  case ISD::FP_TO_UINT_SAT:
460  case ISD::MGATHER:
461  Action = TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0));
462  break;
463  case ISD::SMULFIX:
464  case ISD::SMULFIXSAT:
465  case ISD::UMULFIX:
466  case ISD::UMULFIXSAT:
467  case ISD::SDIVFIX:
468  case ISD::SDIVFIXSAT:
469  case ISD::UDIVFIX:
470  case ISD::UDIVFIXSAT: {
471  unsigned Scale = Node->getConstantOperandVal(2);
472  Action = TLI.getFixedPointOperationAction(Node->getOpcode(),
473  Node->getValueType(0), Scale);
474  break;
475  }
476  case ISD::SINT_TO_FP:
477  case ISD::UINT_TO_FP:
478  case ISD::VECREDUCE_ADD:
479  case ISD::VECREDUCE_MUL:
480  case ISD::VECREDUCE_AND:
481  case ISD::VECREDUCE_OR:
482  case ISD::VECREDUCE_XOR:
483  case ISD::VECREDUCE_SMAX:
484  case ISD::VECREDUCE_SMIN:
485  case ISD::VECREDUCE_UMAX:
486  case ISD::VECREDUCE_UMIN:
487  case ISD::VECREDUCE_FADD:
488  case ISD::VECREDUCE_FMUL:
489  case ISD::VECREDUCE_FMAX:
490  case ISD::VECREDUCE_FMIN:
491  Action = TLI.getOperationAction(Node->getOpcode(),
492  Node->getOperand(0).getValueType());
493  break;
496  Action = TLI.getOperationAction(Node->getOpcode(),
497  Node->getOperand(1).getValueType());
498  break;
499  case ISD::SETCC: {
500  MVT OpVT = Node->getOperand(0).getSimpleValueType();
501  ISD::CondCode CCCode = cast<CondCodeSDNode>(Node->getOperand(2))->get();
502  Action = TLI.getCondCodeAction(CCCode, OpVT);
503  if (Action == TargetLowering::Legal)
504  Action = TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0));
505  break;
506  }
507  }
508 
509  LLVM_DEBUG(dbgs() << "\nLegalizing vector op: "; Node->dump(&DAG));
510 
511  SmallVector<SDValue, 8> ResultVals;
512  switch (Action) {
513  default: llvm_unreachable("This action is not supported yet!");
515  LLVM_DEBUG(dbgs() << "Promoting\n");
516  Promote(Node, ResultVals);
517  assert(!ResultVals.empty() && "No results for promotion?");
518  break;
520  LLVM_DEBUG(dbgs() << "Legal node: nothing to do\n");
521  break;
523  LLVM_DEBUG(dbgs() << "Trying custom legalization\n");
524  if (LowerOperationWrapper(Node, ResultVals))
525  break;
526  LLVM_DEBUG(dbgs() << "Could not custom legalize node\n");
529  LLVM_DEBUG(dbgs() << "Expanding\n");
530  Expand(Node, ResultVals);
531  break;
532  }
533 
534  if (ResultVals.empty())
535  return TranslateLegalizeResults(Op, Node);
536 
537  Changed = true;
538  return RecursivelyLegalizeResults(Op, ResultVals);
539 }
540 
541 // FIXME: This is very similar to TargetLowering::LowerOperationWrapper. Can we
542 // merge them somehow?
543 bool VectorLegalizer::LowerOperationWrapper(SDNode *Node,
545  SDValue Res = TLI.LowerOperation(SDValue(Node, 0), DAG);
546 
547  if (!Res.getNode())
548  return false;
549 
550  if (Res == SDValue(Node, 0))
551  return true;
552 
553  // If the original node has one result, take the return value from
554  // LowerOperation as is. It might not be result number 0.
555  if (Node->getNumValues() == 1) {
556  Results.push_back(Res);
557  return true;
558  }
559 
560  // If the original node has multiple results, then the return node should
561  // have the same number of results.
562  assert((Node->getNumValues() == Res->getNumValues()) &&
563  "Lowering returned the wrong number of results!");
564 
565  // Places new result values base on N result number.
566  for (unsigned I = 0, E = Node->getNumValues(); I != E; ++I)
567  Results.push_back(Res.getValue(I));
568 
569  return true;
570 }
571 
572 void VectorLegalizer::Promote(SDNode *Node, SmallVectorImpl<SDValue> &Results) {
573  // For a few operations there is a specific concept for promotion based on
574  // the operand's type.
575  switch (Node->getOpcode()) {
576  case ISD::SINT_TO_FP:
577  case ISD::UINT_TO_FP:
580  // "Promote" the operation by extending the operand.
581  PromoteINT_TO_FP(Node, Results);
582  return;
583  case ISD::FP_TO_UINT:
584  case ISD::FP_TO_SINT:
587  // Promote the operation by extending the operand.
588  PromoteFP_TO_INT(Node, Results);
589  return;
590  case ISD::FP_ROUND:
591  case ISD::FP_EXTEND:
592  // These operations are used to do promotion so they can't be promoted
593  // themselves.
594  llvm_unreachable("Don't know how to promote this operation!");
595  }
596 
597  // There are currently two cases of vector promotion:
598  // 1) Bitcasting a vector of integers to a different type to a vector of the
599  // same overall length. For example, x86 promotes ISD::AND v2i32 to v1i64.
600  // 2) Extending a vector of floats to a vector of the same number of larger
601  // floats. For example, AArch64 promotes ISD::FADD on v4f16 to v4f32.
602  assert(Node->getNumValues() == 1 &&
603  "Can't promote a vector with multiple results!");
604  MVT VT = Node->getSimpleValueType(0);
605  MVT NVT = TLI.getTypeToPromoteTo(Node->getOpcode(), VT);
606  SDLoc dl(Node);
607  SmallVector<SDValue, 4> Operands(Node->getNumOperands());
608 
609  for (unsigned j = 0; j != Node->getNumOperands(); ++j) {
610  if (Node->getOperand(j).getValueType().isVector())
611  if (Node->getOperand(j)
612  .getValueType()
613  .getVectorElementType()
614  .isFloatingPoint() &&
616  Operands[j] = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(j));
617  else
618  Operands[j] = DAG.getNode(ISD::BITCAST, dl, NVT, Node->getOperand(j));
619  else
620  Operands[j] = Node->getOperand(j);
621  }
622 
623  SDValue Res =
624  DAG.getNode(Node->getOpcode(), dl, NVT, Operands, Node->getFlags());
625 
626  if ((VT.isFloatingPoint() && NVT.isFloatingPoint()) ||
629  Res = DAG.getNode(ISD::FP_ROUND, dl, VT, Res, DAG.getIntPtrConstant(0, dl));
630  else
631  Res = DAG.getNode(ISD::BITCAST, dl, VT, Res);
632 
633  Results.push_back(Res);
634 }
635 
636 void VectorLegalizer::PromoteINT_TO_FP(SDNode *Node,
638  // INT_TO_FP operations may require the input operand be promoted even
639  // when the type is otherwise legal.
640  bool IsStrict = Node->isStrictFPOpcode();
641  MVT VT = Node->getOperand(IsStrict ? 1 : 0).getSimpleValueType();
642  MVT NVT = TLI.getTypeToPromoteTo(Node->getOpcode(), VT);
644  "Vectors have different number of elements!");
645 
646  SDLoc dl(Node);
647  SmallVector<SDValue, 4> Operands(Node->getNumOperands());
648 
649  unsigned Opc = (Node->getOpcode() == ISD::UINT_TO_FP ||
650  Node->getOpcode() == ISD::STRICT_UINT_TO_FP)
653  for (unsigned j = 0; j != Node->getNumOperands(); ++j) {
654  if (Node->getOperand(j).getValueType().isVector())
655  Operands[j] = DAG.getNode(Opc, dl, NVT, Node->getOperand(j));
656  else
657  Operands[j] = Node->getOperand(j);
658  }
659 
660  if (IsStrict) {
661  SDValue Res = DAG.getNode(Node->getOpcode(), dl,
662  {Node->getValueType(0), MVT::Other}, Operands);
663  Results.push_back(Res);
664  Results.push_back(Res.getValue(1));
665  return;
666  }
667 
668  SDValue Res =
669  DAG.getNode(Node->getOpcode(), dl, Node->getValueType(0), Operands);
670  Results.push_back(Res);
671 }
672 
673 // For FP_TO_INT we promote the result type to a vector type with wider
674 // elements and then truncate the result. This is different from the default
675 // PromoteVector which uses bitcast to promote thus assumning that the
676 // promoted vector type has the same overall size.
677 void VectorLegalizer::PromoteFP_TO_INT(SDNode *Node,
679  MVT VT = Node->getSimpleValueType(0);
680  MVT NVT = TLI.getTypeToPromoteTo(Node->getOpcode(), VT);
681  bool IsStrict = Node->isStrictFPOpcode();
683  "Vectors have different number of elements!");
684 
685  unsigned NewOpc = Node->getOpcode();
686  // Change FP_TO_UINT to FP_TO_SINT if possible.
687  // TODO: Should we only do this if FP_TO_UINT itself isn't legal?
688  if (NewOpc == ISD::FP_TO_UINT &&
689  TLI.isOperationLegalOrCustom(ISD::FP_TO_SINT, NVT))
690  NewOpc = ISD::FP_TO_SINT;
691 
692  if (NewOpc == ISD::STRICT_FP_TO_UINT &&
693  TLI.isOperationLegalOrCustom(ISD::STRICT_FP_TO_SINT, NVT))
694  NewOpc = ISD::STRICT_FP_TO_SINT;
695 
696  SDLoc dl(Node);
697  SDValue Promoted, Chain;
698  if (IsStrict) {
699  Promoted = DAG.getNode(NewOpc, dl, {NVT, MVT::Other},
700  {Node->getOperand(0), Node->getOperand(1)});
701  Chain = Promoted.getValue(1);
702  } else
703  Promoted = DAG.getNode(NewOpc, dl, NVT, Node->getOperand(0));
704 
705  // Assert that the converted value fits in the original type. If it doesn't
706  // (eg: because the value being converted is too big), then the result of the
707  // original operation was undefined anyway, so the assert is still correct.
708  if (Node->getOpcode() == ISD::FP_TO_UINT ||
709  Node->getOpcode() == ISD::STRICT_FP_TO_UINT)
710  NewOpc = ISD::AssertZext;
711  else
712  NewOpc = ISD::AssertSext;
713 
714  Promoted = DAG.getNode(NewOpc, dl, NVT, Promoted,
715  DAG.getValueType(VT.getScalarType()));
716  Promoted = DAG.getNode(ISD::TRUNCATE, dl, VT, Promoted);
717  Results.push_back(Promoted);
718  if (IsStrict)
719  Results.push_back(Chain);
720 }
721 
722 std::pair<SDValue, SDValue> VectorLegalizer::ExpandLoad(SDNode *N) {
723  LoadSDNode *LD = cast<LoadSDNode>(N);
724  return TLI.scalarizeVectorLoad(LD, DAG);
725 }
726 
727 SDValue VectorLegalizer::ExpandStore(SDNode *N) {
728  StoreSDNode *ST = cast<StoreSDNode>(N);
729  SDValue TF = TLI.scalarizeVectorStore(ST, DAG);
730  return TF;
731 }
732 
733 void VectorLegalizer::Expand(SDNode *Node, SmallVectorImpl<SDValue> &Results) {
734  SDValue Tmp;
735  switch (Node->getOpcode()) {
736  case ISD::MERGE_VALUES:
737  for (unsigned i = 0, e = Node->getNumValues(); i != e; ++i)
738  Results.push_back(Node->getOperand(i));
739  return;
741  Results.push_back(ExpandSEXTINREG(Node));
742  return;
744  Results.push_back(ExpandANY_EXTEND_VECTOR_INREG(Node));
745  return;
747  Results.push_back(ExpandSIGN_EXTEND_VECTOR_INREG(Node));
748  return;
750  Results.push_back(ExpandZERO_EXTEND_VECTOR_INREG(Node));
751  return;
752  case ISD::BSWAP:
753  Results.push_back(ExpandBSWAP(Node));
754  return;
755  case ISD::VSELECT:
756  Results.push_back(ExpandVSELECT(Node));
757  return;
758  case ISD::SELECT:
759  Results.push_back(ExpandSELECT(Node));
760  return;
761  case ISD::FP_TO_UINT:
762  ExpandFP_TO_UINT(Node, Results);
763  return;
764  case ISD::UINT_TO_FP:
765  ExpandUINT_TO_FLOAT(Node, Results);
766  return;
767  case ISD::FNEG:
768  Results.push_back(ExpandFNEG(Node));
769  return;
770  case ISD::FSUB:
771  ExpandFSUB(Node, Results);
772  return;
773  case ISD::SETCC:
774  ExpandSETCC(Node, Results);
775  return;
776  case ISD::ABS:
777  if (TLI.expandABS(Node, Tmp, DAG)) {
778  Results.push_back(Tmp);
779  return;
780  }
781  break;
782  case ISD::BITREVERSE:
783  ExpandBITREVERSE(Node, Results);
784  return;
785  case ISD::CTPOP:
786  if (TLI.expandCTPOP(Node, Tmp, DAG)) {
787  Results.push_back(Tmp);
788  return;
789  }
790  break;
791  case ISD::CTLZ:
793  if (TLI.expandCTLZ(Node, Tmp, DAG)) {
794  Results.push_back(Tmp);
795  return;
796  }
797  break;
798  case ISD::CTTZ:
800  if (TLI.expandCTTZ(Node, Tmp, DAG)) {
801  Results.push_back(Tmp);
802  return;
803  }
804  break;
805  case ISD::FSHL:
806  case ISD::FSHR:
807  if (TLI.expandFunnelShift(Node, Tmp, DAG)) {
808  Results.push_back(Tmp);
809  return;
810  }
811  break;
812  case ISD::ROTL:
813  case ISD::ROTR:
814  if (TLI.expandROT(Node, false /*AllowVectorOps*/, Tmp, DAG)) {
815  Results.push_back(Tmp);
816  return;
817  }
818  break;
819  case ISD::FMINNUM:
820  case ISD::FMAXNUM:
821  if (SDValue Expanded = TLI.expandFMINNUM_FMAXNUM(Node, DAG)) {
822  Results.push_back(Expanded);
823  return;
824  }
825  break;
826  case ISD::SMIN:
827  case ISD::SMAX:
828  case ISD::UMIN:
829  case ISD::UMAX:
830  if (SDValue Expanded = TLI.expandIntMINMAX(Node, DAG)) {
831  Results.push_back(Expanded);
832  return;
833  }
834  break;
835  case ISD::UADDO:
836  case ISD::USUBO:
837  ExpandUADDSUBO(Node, Results);
838  return;
839  case ISD::SADDO:
840  case ISD::SSUBO:
841  ExpandSADDSUBO(Node, Results);
842  return;
843  case ISD::UMULO:
844  case ISD::SMULO:
845  ExpandMULO(Node, Results);
846  return;
847  case ISD::USUBSAT:
848  case ISD::SSUBSAT:
849  case ISD::UADDSAT:
850  case ISD::SADDSAT:
851  if (SDValue Expanded = TLI.expandAddSubSat(Node, DAG)) {
852  Results.push_back(Expanded);
853  return;
854  }
855  break;
856  case ISD::SMULFIX:
857  case ISD::UMULFIX:
858  if (SDValue Expanded = TLI.expandFixedPointMul(Node, DAG)) {
859  Results.push_back(Expanded);
860  return;
861  }
862  break;
863  case ISD::SMULFIXSAT:
864  case ISD::UMULFIXSAT:
865  // FIXME: We do not expand SMULFIXSAT/UMULFIXSAT here yet, not sure exactly
866  // why. Maybe it results in worse codegen compared to the unroll for some
867  // targets? This should probably be investigated. And if we still prefer to
868  // unroll an explanation could be helpful.
869  break;
870  case ISD::SDIVFIX:
871  case ISD::UDIVFIX:
872  ExpandFixedPointDiv(Node, Results);
873  return;
874  case ISD::SDIVFIXSAT:
875  case ISD::UDIVFIXSAT:
876  break;
877 #define DAG_INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC, DAGN) \
878  case ISD::STRICT_##DAGN:
879 #include "llvm/IR/ConstrainedOps.def"
880  ExpandStrictFPOp(Node, Results);
881  return;
882  case ISD::VECREDUCE_ADD:
883  case ISD::VECREDUCE_MUL:
884  case ISD::VECREDUCE_AND:
885  case ISD::VECREDUCE_OR:
886  case ISD::VECREDUCE_XOR:
887  case ISD::VECREDUCE_SMAX:
888  case ISD::VECREDUCE_SMIN:
889  case ISD::VECREDUCE_UMAX:
890  case ISD::VECREDUCE_UMIN:
891  case ISD::VECREDUCE_FADD:
892  case ISD::VECREDUCE_FMUL:
893  case ISD::VECREDUCE_FMAX:
894  case ISD::VECREDUCE_FMIN:
895  Results.push_back(TLI.expandVecReduce(Node, DAG));
896  return;
899  Results.push_back(TLI.expandVecReduceSeq(Node, DAG));
900  return;
901  case ISD::SREM:
902  case ISD::UREM:
903  ExpandREM(Node, Results);
904  return;
905  }
906 
907  Results.push_back(DAG.UnrollVectorOp(Node));
908 }
909 
910 SDValue VectorLegalizer::ExpandSELECT(SDNode *Node) {
911  // Lower a select instruction where the condition is a scalar and the
912  // operands are vectors. Lower this select to VSELECT and implement it
913  // using XOR AND OR. The selector bit is broadcasted.
914  EVT VT = Node->getValueType(0);
915  SDLoc DL(Node);
916 
917  SDValue Mask = Node->getOperand(0);
918  SDValue Op1 = Node->getOperand(1);
919  SDValue Op2 = Node->getOperand(2);
920 
921  assert(VT.isVector() && !Mask.getValueType().isVector()
922  && Op1.getValueType() == Op2.getValueType() && "Invalid type");
923 
924  // If we can't even use the basic vector operations of
925  // AND,OR,XOR, we will have to scalarize the op.
926  // Notice that the operation may be 'promoted' which means that it is
927  // 'bitcasted' to another type which is handled.
928  // Also, we need to be able to construct a splat vector using either
929  // BUILD_VECTOR or SPLAT_VECTOR.
930  // FIXME: Should we also permit fixed-length SPLAT_VECTOR as a fallback to
931  // BUILD_VECTOR?
932  if (TLI.getOperationAction(ISD::AND, VT) == TargetLowering::Expand ||
933  TLI.getOperationAction(ISD::XOR, VT) == TargetLowering::Expand ||
934  TLI.getOperationAction(ISD::OR, VT) == TargetLowering::Expand ||
935  TLI.getOperationAction(VT.isFixedLengthVector() ? ISD::BUILD_VECTOR
937  VT) == TargetLowering::Expand)
938  return DAG.UnrollVectorOp(Node);
939 
940  // Generate a mask operand.
942 
943  // What is the size of each element in the vector mask.
944  EVT BitTy = MaskTy.getScalarType();
945 
946  Mask = DAG.getSelect(DL, BitTy, Mask, DAG.getAllOnesConstant(DL, BitTy),
947  DAG.getConstant(0, DL, BitTy));
948 
949  // Broadcast the mask so that the entire vector is all one or all zero.
950  if (VT.isFixedLengthVector())
951  Mask = DAG.getSplatBuildVector(MaskTy, DL, Mask);
952  else
953  Mask = DAG.getSplatVector(MaskTy, DL, Mask);
954 
955  // Bitcast the operands to be the same type as the mask.
956  // This is needed when we select between FP types because
957  // the mask is a vector of integers.
958  Op1 = DAG.getNode(ISD::BITCAST, DL, MaskTy, Op1);
959  Op2 = DAG.getNode(ISD::BITCAST, DL, MaskTy, Op2);
960 
961  SDValue NotMask = DAG.getNOT(DL, Mask, MaskTy);
962 
963  Op1 = DAG.getNode(ISD::AND, DL, MaskTy, Op1, Mask);
964  Op2 = DAG.getNode(ISD::AND, DL, MaskTy, Op2, NotMask);
965  SDValue Val = DAG.getNode(ISD::OR, DL, MaskTy, Op1, Op2);
966  return DAG.getNode(ISD::BITCAST, DL, Node->getValueType(0), Val);
967 }
968 
969 SDValue VectorLegalizer::ExpandSEXTINREG(SDNode *Node) {
970  EVT VT = Node->getValueType(0);
971 
972  // Make sure that the SRA and SHL instructions are available.
973  if (TLI.getOperationAction(ISD::SRA, VT) == TargetLowering::Expand ||
974  TLI.getOperationAction(ISD::SHL, VT) == TargetLowering::Expand)
975  return DAG.UnrollVectorOp(Node);
976 
977  SDLoc DL(Node);
978  EVT OrigTy = cast<VTSDNode>(Node->getOperand(1))->getVT();
979 
980  unsigned BW = VT.getScalarSizeInBits();
981  unsigned OrigBW = OrigTy.getScalarSizeInBits();
982  SDValue ShiftSz = DAG.getConstant(BW - OrigBW, DL, VT);
983 
984  SDValue Op = DAG.getNode(ISD::SHL, DL, VT, Node->getOperand(0), ShiftSz);
985  return DAG.getNode(ISD::SRA, DL, VT, Op, ShiftSz);
986 }
987 
988 // Generically expand a vector anyext in register to a shuffle of the relevant
989 // lanes into the appropriate locations, with other lanes left undef.
990 SDValue VectorLegalizer::ExpandANY_EXTEND_VECTOR_INREG(SDNode *Node) {
991  SDLoc DL(Node);
992  EVT VT = Node->getValueType(0);
993  int NumElements = VT.getVectorNumElements();
994  SDValue Src = Node->getOperand(0);
995  EVT SrcVT = Src.getValueType();
996  int NumSrcElements = SrcVT.getVectorNumElements();
997 
998  // *_EXTEND_VECTOR_INREG SrcVT can be smaller than VT - so insert the vector
999  // into a larger vector type.
1000  if (SrcVT.bitsLE(VT)) {
1001  assert((VT.getSizeInBits() % SrcVT.getScalarSizeInBits()) == 0 &&
1002  "ANY_EXTEND_VECTOR_INREG vector size mismatch");
1003  NumSrcElements = VT.getSizeInBits() / SrcVT.getScalarSizeInBits();
1004  SrcVT = EVT::getVectorVT(*DAG.getContext(), SrcVT.getScalarType(),
1005  NumSrcElements);
1006  Src = DAG.getNode(ISD::INSERT_SUBVECTOR, DL, SrcVT, DAG.getUNDEF(SrcVT),
1007  Src, DAG.getVectorIdxConstant(0, DL));
1008  }
1009 
1010  // Build a base mask of undef shuffles.
1011  SmallVector<int, 16> ShuffleMask;
1012  ShuffleMask.resize(NumSrcElements, -1);
1013 
1014  // Place the extended lanes into the correct locations.
1015  int ExtLaneScale = NumSrcElements / NumElements;
1016  int EndianOffset = DAG.getDataLayout().isBigEndian() ? ExtLaneScale - 1 : 0;
1017  for (int i = 0; i < NumElements; ++i)
1018  ShuffleMask[i * ExtLaneScale + EndianOffset] = i;
1019 
1020  return DAG.getNode(
1021  ISD::BITCAST, DL, VT,
1022  DAG.getVectorShuffle(SrcVT, DL, Src, DAG.getUNDEF(SrcVT), ShuffleMask));
1023 }
1024 
1025 SDValue VectorLegalizer::ExpandSIGN_EXTEND_VECTOR_INREG(SDNode *Node) {
1026  SDLoc DL(Node);
1027  EVT VT = Node->getValueType(0);
1028  SDValue Src = Node->getOperand(0);
1029  EVT SrcVT = Src.getValueType();
1030 
1031  // First build an any-extend node which can be legalized above when we
1032  // recurse through it.
1033  SDValue Op = DAG.getNode(ISD::ANY_EXTEND_VECTOR_INREG, DL, VT, Src);
1034 
1035  // Now we need sign extend. Do this by shifting the elements. Even if these
1036  // aren't legal operations, they have a better chance of being legalized
1037  // without full scalarization than the sign extension does.
1038  unsigned EltWidth = VT.getScalarSizeInBits();
1039  unsigned SrcEltWidth = SrcVT.getScalarSizeInBits();
1040  SDValue ShiftAmount = DAG.getConstant(EltWidth - SrcEltWidth, DL, VT);
1041  return DAG.getNode(ISD::SRA, DL, VT,
1042  DAG.getNode(ISD::SHL, DL, VT, Op, ShiftAmount),
1043  ShiftAmount);
1044 }
1045 
1046 // Generically expand a vector zext in register to a shuffle of the relevant
1047 // lanes into the appropriate locations, a blend of zero into the high bits,
1048 // and a bitcast to the wider element type.
1049 SDValue VectorLegalizer::ExpandZERO_EXTEND_VECTOR_INREG(SDNode *Node) {
1050  SDLoc DL(Node);
1051  EVT VT = Node->getValueType(0);
1052  int NumElements = VT.getVectorNumElements();
1053  SDValue Src = Node->getOperand(0);
1054  EVT SrcVT = Src.getValueType();
1055  int NumSrcElements = SrcVT.getVectorNumElements();
1056 
1057  // *_EXTEND_VECTOR_INREG SrcVT can be smaller than VT - so insert the vector
1058  // into a larger vector type.
1059  if (SrcVT.bitsLE(VT)) {
1060  assert((VT.getSizeInBits() % SrcVT.getScalarSizeInBits()) == 0 &&
1061  "ZERO_EXTEND_VECTOR_INREG vector size mismatch");
1062  NumSrcElements = VT.getSizeInBits() / SrcVT.getScalarSizeInBits();
1063  SrcVT = EVT::getVectorVT(*DAG.getContext(), SrcVT.getScalarType(),
1064  NumSrcElements);
1065  Src = DAG.getNode(ISD::INSERT_SUBVECTOR, DL, SrcVT, DAG.getUNDEF(SrcVT),
1066  Src, DAG.getVectorIdxConstant(0, DL));
1067  }
1068 
1069  // Build up a zero vector to blend into this one.
1070  SDValue Zero = DAG.getConstant(0, DL, SrcVT);
1071 
1072  // Shuffle the incoming lanes into the correct position, and pull all other
1073  // lanes from the zero vector.
1074  SmallVector<int, 16> ShuffleMask;
1075  ShuffleMask.reserve(NumSrcElements);
1076  for (int i = 0; i < NumSrcElements; ++i)
1077  ShuffleMask.push_back(i);
1078 
1079  int ExtLaneScale = NumSrcElements / NumElements;
1080  int EndianOffset = DAG.getDataLayout().isBigEndian() ? ExtLaneScale - 1 : 0;
1081  for (int i = 0; i < NumElements; ++i)
1082  ShuffleMask[i * ExtLaneScale + EndianOffset] = NumSrcElements + i;
1083 
1084  return DAG.getNode(ISD::BITCAST, DL, VT,
1085  DAG.getVectorShuffle(SrcVT, DL, Zero, Src, ShuffleMask));
1086 }
1087 
1088 static void createBSWAPShuffleMask(EVT VT, SmallVectorImpl<int> &ShuffleMask) {
1089  int ScalarSizeInBytes = VT.getScalarSizeInBits() / 8;
1090  for (int I = 0, E = VT.getVectorNumElements(); I != E; ++I)
1091  for (int J = ScalarSizeInBytes - 1; J >= 0; --J)
1092  ShuffleMask.push_back((I * ScalarSizeInBytes) + J);
1093 }
1094 
1095 SDValue VectorLegalizer::ExpandBSWAP(SDNode *Node) {
1096  EVT VT = Node->getValueType(0);
1097 
1098  // Generate a byte wise shuffle mask for the BSWAP.
1099  SmallVector<int, 16> ShuffleMask;
1100  createBSWAPShuffleMask(VT, ShuffleMask);
1101  EVT ByteVT = EVT::getVectorVT(*DAG.getContext(), MVT::i8, ShuffleMask.size());
1102 
1103  // Only emit a shuffle if the mask is legal.
1104  if (!TLI.isShuffleMaskLegal(ShuffleMask, ByteVT))
1105  return DAG.UnrollVectorOp(Node);
1106 
1107  SDLoc DL(Node);
1108  SDValue Op = DAG.getNode(ISD::BITCAST, DL, ByteVT, Node->getOperand(0));
1109  Op = DAG.getVectorShuffle(ByteVT, DL, Op, DAG.getUNDEF(ByteVT), ShuffleMask);
1110  return DAG.getNode(ISD::BITCAST, DL, VT, Op);
1111 }
1112 
1113 void VectorLegalizer::ExpandBITREVERSE(SDNode *Node,
1115  EVT VT = Node->getValueType(0);
1116 
1117  // If we have the scalar operation, it's probably cheaper to unroll it.
1118  if (TLI.isOperationLegalOrCustom(ISD::BITREVERSE, VT.getScalarType())) {
1119  SDValue Tmp = DAG.UnrollVectorOp(Node);
1120  Results.push_back(Tmp);
1121  return;
1122  }
1123 
1124  // If the vector element width is a whole number of bytes, test if its legal
1125  // to BSWAP shuffle the bytes and then perform the BITREVERSE on the byte
1126  // vector. This greatly reduces the number of bit shifts necessary.
1127  unsigned ScalarSizeInBits = VT.getScalarSizeInBits();
1128  if (ScalarSizeInBits > 8 && (ScalarSizeInBits % 8) == 0) {
1129  SmallVector<int, 16> BSWAPMask;
1130  createBSWAPShuffleMask(VT, BSWAPMask);
1131 
1132  EVT ByteVT = EVT::getVectorVT(*DAG.getContext(), MVT::i8, BSWAPMask.size());
1133  if (TLI.isShuffleMaskLegal(BSWAPMask, ByteVT) &&
1134  (TLI.isOperationLegalOrCustom(ISD::BITREVERSE, ByteVT) ||
1135  (TLI.isOperationLegalOrCustom(ISD::SHL, ByteVT) &&
1136  TLI.isOperationLegalOrCustom(ISD::SRL, ByteVT) &&
1137  TLI.isOperationLegalOrCustomOrPromote(ISD::AND, ByteVT) &&
1138  TLI.isOperationLegalOrCustomOrPromote(ISD::OR, ByteVT)))) {
1139  SDLoc DL(Node);
1140  SDValue Op = DAG.getNode(ISD::BITCAST, DL, ByteVT, Node->getOperand(0));
1141  Op = DAG.getVectorShuffle(ByteVT, DL, Op, DAG.getUNDEF(ByteVT),
1142  BSWAPMask);
1143  Op = DAG.getNode(ISD::BITREVERSE, DL, ByteVT, Op);
1144  Op = DAG.getNode(ISD::BITCAST, DL, VT, Op);
1145  Results.push_back(Op);
1146  return;
1147  }
1148  }
1149 
1150  // If we have the appropriate vector bit operations, it is better to use them
1151  // than unrolling and expanding each component.
1152  if (TLI.isOperationLegalOrCustom(ISD::SHL, VT) &&
1153  TLI.isOperationLegalOrCustom(ISD::SRL, VT) &&
1154  TLI.isOperationLegalOrCustomOrPromote(ISD::AND, VT) &&
1155  TLI.isOperationLegalOrCustomOrPromote(ISD::OR, VT))
1156  // Let LegalizeDAG handle this later.
1157  return;
1158 
1159  // Otherwise unroll.
1160  SDValue Tmp = DAG.UnrollVectorOp(Node);
1161  Results.push_back(Tmp);
1162 }
1163 
1164 SDValue VectorLegalizer::ExpandVSELECT(SDNode *Node) {
1165  // Implement VSELECT in terms of XOR, AND, OR
1166  // on platforms which do not support blend natively.
1167  SDLoc DL(Node);
1168 
1169  SDValue Mask = Node->getOperand(0);
1170  SDValue Op1 = Node->getOperand(1);
1171  SDValue Op2 = Node->getOperand(2);
1172 
1173  EVT VT = Mask.getValueType();
1174 
1175  // If we can't even use the basic vector operations of
1176  // AND,OR,XOR, we will have to scalarize the op.
1177  // Notice that the operation may be 'promoted' which means that it is
1178  // 'bitcasted' to another type which is handled.
1179  if (TLI.getOperationAction(ISD::AND, VT) == TargetLowering::Expand ||
1180  TLI.getOperationAction(ISD::XOR, VT) == TargetLowering::Expand ||
1181  TLI.getOperationAction(ISD::OR, VT) == TargetLowering::Expand)
1182  return DAG.UnrollVectorOp(Node);
1183 
1184  // This operation also isn't safe with AND, OR, XOR when the boolean type is
1185  // 0/1 and the select operands aren't also booleans, as we need an all-ones
1186  // vector constant to mask with.
1187  // FIXME: Sign extend 1 to all ones if that's legal on the target.
1188  auto BoolContents = TLI.getBooleanContents(Op1.getValueType());
1190  !(BoolContents == TargetLowering::ZeroOrOneBooleanContent &&
1192  return DAG.UnrollVectorOp(Node);
1193 
1194  // If the mask and the type are different sizes, unroll the vector op. This
1195  // can occur when getSetCCResultType returns something that is different in
1196  // size from the operand types. For example, v4i8 = select v4i32, v4i8, v4i8.
1197  if (VT.getSizeInBits() != Op1.getValueSizeInBits())
1198  return DAG.UnrollVectorOp(Node);
1199 
1200  // Bitcast the operands to be the same type as the mask.
1201  // This is needed when we select between FP types because
1202  // the mask is a vector of integers.
1203  Op1 = DAG.getNode(ISD::BITCAST, DL, VT, Op1);
1204  Op2 = DAG.getNode(ISD::BITCAST, DL, VT, Op2);
1205 
1206  SDValue NotMask = DAG.getNOT(DL, Mask, VT);
1207 
1208  Op1 = DAG.getNode(ISD::AND, DL, VT, Op1, Mask);
1209  Op2 = DAG.getNode(ISD::AND, DL, VT, Op2, NotMask);
1210  SDValue Val = DAG.getNode(ISD::OR, DL, VT, Op1, Op2);
1211  return DAG.getNode(ISD::BITCAST, DL, Node->getValueType(0), Val);
1212 }
1213 
1214 void VectorLegalizer::ExpandFP_TO_UINT(SDNode *Node,
1216  // Attempt to expand using TargetLowering.
1217  SDValue Result, Chain;
1218  if (TLI.expandFP_TO_UINT(Node, Result, Chain, DAG)) {
1219  Results.push_back(Result);
1220  if (Node->isStrictFPOpcode())
1221  Results.push_back(Chain);
1222  return;
1223  }
1224 
1225  // Otherwise go ahead and unroll.
1226  if (Node->isStrictFPOpcode()) {
1227  UnrollStrictFPOp(Node, Results);
1228  return;
1229  }
1230 
1231  Results.push_back(DAG.UnrollVectorOp(Node));
1232 }
1233 
1234 void VectorLegalizer::ExpandUINT_TO_FLOAT(SDNode *Node,
1236  bool IsStrict = Node->isStrictFPOpcode();
1237  unsigned OpNo = IsStrict ? 1 : 0;
1238  SDValue Src = Node->getOperand(OpNo);
1239  EVT VT = Src.getValueType();
1240  SDLoc DL(Node);
1241 
1242  // Attempt to expand using TargetLowering.
1243  SDValue Result;
1244  SDValue Chain;
1245  if (TLI.expandUINT_TO_FP(Node, Result, Chain, DAG)) {
1246  Results.push_back(Result);
1247  if (IsStrict)
1248  Results.push_back(Chain);
1249  return;
1250  }
1251 
1252  // Make sure that the SINT_TO_FP and SRL instructions are available.
1253  if (((!IsStrict && TLI.getOperationAction(ISD::SINT_TO_FP, VT) ==
1255  (IsStrict && TLI.getOperationAction(ISD::STRICT_SINT_TO_FP, VT) ==
1257  TLI.getOperationAction(ISD::SRL, VT) == TargetLowering::Expand) {
1258  if (IsStrict) {
1259  UnrollStrictFPOp(Node, Results);
1260  return;
1261  }
1262 
1263  Results.push_back(DAG.UnrollVectorOp(Node));
1264  return;
1265  }
1266 
1267  unsigned BW = VT.getScalarSizeInBits();
1268  assert((BW == 64 || BW == 32) &&
1269  "Elements in vector-UINT_TO_FP must be 32 or 64 bits wide");
1270 
1271  SDValue HalfWord = DAG.getConstant(BW / 2, DL, VT);
1272 
1273  // Constants to clear the upper part of the word.
1274  // Notice that we can also use SHL+SHR, but using a constant is slightly
1275  // faster on x86.
1276  uint64_t HWMask = (BW == 64) ? 0x00000000FFFFFFFF : 0x0000FFFF;
1277  SDValue HalfWordMask = DAG.getConstant(HWMask, DL, VT);
1278 
1279  // Two to the power of half-word-size.
1280  SDValue TWOHW =
1281  DAG.getConstantFP(1ULL << (BW / 2), DL, Node->getValueType(0));
1282 
1283  // Clear upper part of LO, lower HI
1284  SDValue HI = DAG.getNode(ISD::SRL, DL, VT, Src, HalfWord);
1285  SDValue LO = DAG.getNode(ISD::AND, DL, VT, Src, HalfWordMask);
1286 
1287  if (IsStrict) {
1288  // Convert hi and lo to floats
1289  // Convert the hi part back to the upper values
1290  // TODO: Can any fast-math-flags be set on these nodes?
1292  {Node->getValueType(0), MVT::Other},
1293  {Node->getOperand(0), HI});
1294  fHI = DAG.getNode(ISD::STRICT_FMUL, DL, {Node->getValueType(0), MVT::Other},
1295  {fHI.getValue(1), fHI, TWOHW});
1297  {Node->getValueType(0), MVT::Other},
1298  {Node->getOperand(0), LO});
1299 
1301  fLO.getValue(1));
1302 
1303  // Add the two halves
1304  SDValue Result =
1305  DAG.getNode(ISD::STRICT_FADD, DL, {Node->getValueType(0), MVT::Other},
1306  {TF, fHI, fLO});
1307 
1308  Results.push_back(Result);
1309  Results.push_back(Result.getValue(1));
1310  return;
1311  }
1312 
1313  // Convert hi and lo to floats
1314  // Convert the hi part back to the upper values
1315  // TODO: Can any fast-math-flags be set on these nodes?
1316  SDValue fHI = DAG.getNode(ISD::SINT_TO_FP, DL, Node->getValueType(0), HI);
1317  fHI = DAG.getNode(ISD::FMUL, DL, Node->getValueType(0), fHI, TWOHW);
1318  SDValue fLO = DAG.getNode(ISD::SINT_TO_FP, DL, Node->getValueType(0), LO);
1319 
1320  // Add the two halves
1321  Results.push_back(
1322  DAG.getNode(ISD::FADD, DL, Node->getValueType(0), fHI, fLO));
1323 }
1324 
1325 SDValue VectorLegalizer::ExpandFNEG(SDNode *Node) {
1326  if (TLI.isOperationLegalOrCustom(ISD::FSUB, Node->getValueType(0))) {
1327  SDLoc DL(Node);
1328  SDValue Zero = DAG.getConstantFP(-0.0, DL, Node->getValueType(0));
1329  // TODO: If FNEG had fast-math-flags, they'd get propagated to this FSUB.
1330  return DAG.getNode(ISD::FSUB, DL, Node->getValueType(0), Zero,
1331  Node->getOperand(0));
1332  }
1333  return DAG.UnrollVectorOp(Node);
1334 }
1335 
1336 void VectorLegalizer::ExpandFSUB(SDNode *Node,
1338  // For floating-point values, (a-b) is the same as a+(-b). If FNEG is legal,
1339  // we can defer this to operation legalization where it will be lowered as
1340  // a+(-b).
1341  EVT VT = Node->getValueType(0);
1342  if (TLI.isOperationLegalOrCustom(ISD::FNEG, VT) &&
1343  TLI.isOperationLegalOrCustom(ISD::FADD, VT))
1344  return; // Defer to LegalizeDAG
1345 
1346  SDValue Tmp = DAG.UnrollVectorOp(Node);
1347  Results.push_back(Tmp);
1348 }
1349 
1350 void VectorLegalizer::ExpandSETCC(SDNode *Node,
1352  bool NeedInvert = false;
1353  SDLoc dl(Node);
1354  MVT OpVT = Node->getOperand(0).getSimpleValueType();
1355  ISD::CondCode CCCode = cast<CondCodeSDNode>(Node->getOperand(2))->get();
1356 
1357  if (TLI.getCondCodeAction(CCCode, OpVT) != TargetLowering::Expand) {
1358  Results.push_back(UnrollVSETCC(Node));
1359  return;
1360  }
1361 
1362  SDValue Chain;
1363  SDValue LHS = Node->getOperand(0);
1364  SDValue RHS = Node->getOperand(1);
1365  SDValue CC = Node->getOperand(2);
1366  bool Legalized = TLI.LegalizeSetCCCondCode(DAG, Node->getValueType(0), LHS,
1367  RHS, CC, NeedInvert, dl, Chain);
1368 
1369  if (Legalized) {
1370  // If we expanded the SETCC by swapping LHS and RHS, or by inverting the
1371  // condition code, create a new SETCC node.
1372  if (CC.getNode())
1373  LHS = DAG.getNode(ISD::SETCC, dl, Node->getValueType(0), LHS, RHS, CC,
1374  Node->getFlags());
1375 
1376  // If we expanded the SETCC by inverting the condition code, then wrap
1377  // the existing SETCC in a NOT to restore the intended condition.
1378  if (NeedInvert)
1379  LHS = DAG.getLogicalNOT(dl, LHS, LHS->getValueType(0));
1380  } else {
1381  // Otherwise, SETCC for the given comparison type must be completely
1382  // illegal; expand it into a SELECT_CC.
1383  EVT VT = Node->getValueType(0);
1384  LHS =
1385  DAG.getNode(ISD::SELECT_CC, dl, VT, LHS, RHS,
1386  DAG.getBoolConstant(true, dl, VT, LHS.getValueType()),
1387  DAG.getBoolConstant(false, dl, VT, LHS.getValueType()), CC);
1388  LHS->setFlags(Node->getFlags());
1389  }
1390 
1391  Results.push_back(LHS);
1392 }
1393 
1394 void VectorLegalizer::ExpandUADDSUBO(SDNode *Node,
1396  SDValue Result, Overflow;
1397  TLI.expandUADDSUBO(Node, Result, Overflow, DAG);
1398  Results.push_back(Result);
1399  Results.push_back(Overflow);
1400 }
1401 
1402 void VectorLegalizer::ExpandSADDSUBO(SDNode *Node,
1404  SDValue Result, Overflow;
1405  TLI.expandSADDSUBO(Node, Result, Overflow, DAG);
1406  Results.push_back(Result);
1407  Results.push_back(Overflow);
1408 }
1409 
1410 void VectorLegalizer::ExpandMULO(SDNode *Node,
1412  SDValue Result, Overflow;
1413  if (!TLI.expandMULO(Node, Result, Overflow, DAG))
1414  std::tie(Result, Overflow) = DAG.UnrollVectorOverflowOp(Node);
1415 
1416  Results.push_back(Result);
1417  Results.push_back(Overflow);
1418 }
1419 
1420 void VectorLegalizer::ExpandFixedPointDiv(SDNode *Node,
1422  SDNode *N = Node;
1423  if (SDValue Expanded = TLI.expandFixedPointDiv(N->getOpcode(), SDLoc(N),
1424  N->getOperand(0), N->getOperand(1), N->getConstantOperandVal(2), DAG))
1425  Results.push_back(Expanded);
1426 }
1427 
1428 void VectorLegalizer::ExpandStrictFPOp(SDNode *Node,
1430  if (Node->getOpcode() == ISD::STRICT_UINT_TO_FP) {
1431  ExpandUINT_TO_FLOAT(Node, Results);
1432  return;
1433  }
1434  if (Node->getOpcode() == ISD::STRICT_FP_TO_UINT) {
1435  ExpandFP_TO_UINT(Node, Results);
1436  return;
1437  }
1438 
1439  UnrollStrictFPOp(Node, Results);
1440 }
1441 
1442 void VectorLegalizer::ExpandREM(SDNode *Node,
1444  assert((Node->getOpcode() == ISD::SREM || Node->getOpcode() == ISD::UREM) &&
1445  "Expected REM node");
1446 
1447  SDValue Result;
1448  if (!TLI.expandREM(Node, Result, DAG))
1449  Result = DAG.UnrollVectorOp(Node);
1450  Results.push_back(Result);
1451 }
1452 
1453 void VectorLegalizer::UnrollStrictFPOp(SDNode *Node,
1455  EVT VT = Node->getValueType(0);
1456  EVT EltVT = VT.getVectorElementType();
1457  unsigned NumElems = VT.getVectorNumElements();
1458  unsigned NumOpers = Node->getNumOperands();
1459  const TargetLowering &TLI = DAG.getTargetLoweringInfo();
1460 
1461  EVT TmpEltVT = EltVT;
1462  if (Node->getOpcode() == ISD::STRICT_FSETCC ||
1463  Node->getOpcode() == ISD::STRICT_FSETCCS)
1464  TmpEltVT = TLI.getSetCCResultType(DAG.getDataLayout(),
1465  *DAG.getContext(), TmpEltVT);
1466 
1467  EVT ValueVTs[] = {TmpEltVT, MVT::Other};
1468  SDValue Chain = Node->getOperand(0);
1469  SDLoc dl(Node);
1470 
1471  SmallVector<SDValue, 32> OpValues;
1472  SmallVector<SDValue, 32> OpChains;
1473  for (unsigned i = 0; i < NumElems; ++i) {
1475  SDValue Idx = DAG.getVectorIdxConstant(i, dl);
1476 
1477  // The Chain is the first operand.
1478  Opers.push_back(Chain);
1479 
1480  // Now process the remaining operands.
1481  for (unsigned j = 1; j < NumOpers; ++j) {
1482  SDValue Oper = Node->getOperand(j);
1483  EVT OperVT = Oper.getValueType();
1484 
1485  if (OperVT.isVector())
1486  Oper = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl,
1487  OperVT.getVectorElementType(), Oper, Idx);
1488 
1489  Opers.push_back(Oper);
1490  }
1491 
1492  SDValue ScalarOp = DAG.getNode(Node->getOpcode(), dl, ValueVTs, Opers);
1493  SDValue ScalarResult = ScalarOp.getValue(0);
1494  SDValue ScalarChain = ScalarOp.getValue(1);
1495 
1496  if (Node->getOpcode() == ISD::STRICT_FSETCC ||
1497  Node->getOpcode() == ISD::STRICT_FSETCCS)
1498  ScalarResult = DAG.getSelect(dl, EltVT, ScalarResult,
1499  DAG.getAllOnesConstant(dl, EltVT),
1500  DAG.getConstant(0, dl, EltVT));
1501 
1502  OpValues.push_back(ScalarResult);
1503  OpChains.push_back(ScalarChain);
1504  }
1505 
1506  SDValue Result = DAG.getBuildVector(VT, dl, OpValues);
1507  SDValue NewChain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, OpChains);
1508 
1509  Results.push_back(Result);
1510  Results.push_back(NewChain);
1511 }
1512 
1513 SDValue VectorLegalizer::UnrollVSETCC(SDNode *Node) {
1514  EVT VT = Node->getValueType(0);
1515  unsigned NumElems = VT.getVectorNumElements();
1516  EVT EltVT = VT.getVectorElementType();
1517  SDValue LHS = Node->getOperand(0);
1518  SDValue RHS = Node->getOperand(1);
1519  SDValue CC = Node->getOperand(2);
1520  EVT TmpEltVT = LHS.getValueType().getVectorElementType();
1521  SDLoc dl(Node);
1522  SmallVector<SDValue, 8> Ops(NumElems);
1523  for (unsigned i = 0; i < NumElems; ++i) {
1524  SDValue LHSElem = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, TmpEltVT, LHS,
1525  DAG.getVectorIdxConstant(i, dl));
1526  SDValue RHSElem = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, TmpEltVT, RHS,
1527  DAG.getVectorIdxConstant(i, dl));
1528  Ops[i] = DAG.getNode(ISD::SETCC, dl,
1529  TLI.getSetCCResultType(DAG.getDataLayout(),
1530  *DAG.getContext(), TmpEltVT),
1531  LHSElem, RHSElem, CC);
1532  Ops[i] = DAG.getSelect(dl, EltVT, Ops[i], DAG.getAllOnesConstant(dl, EltVT),
1533  DAG.getConstant(0, dl, EltVT));
1534  }
1535  return DAG.getBuildVector(VT, dl, Ops);
1536 }
1537 
1539  return VectorLegalizer(*this).Run();
1540 }
llvm::ISD::SUB
@ SUB
Definition: ISDOpcodes.h:240
llvm::ISD::FPOWI
@ FPOWI
Definition: ISDOpcodes.h:872
llvm::ISD::FROUNDEVEN
@ FROUNDEVEN
Definition: ISDOpcodes.h:884
i
i
Definition: README.txt:29
ValueTypes.h
llvm::ISD::STRICT_FSETCC
@ STRICT_FSETCC
STRICT_FSETCC/STRICT_FSETCCS - Constrained versions of SETCC, used for floating-point operands only.
Definition: ISDOpcodes.h:462
llvm::AArch64CC::LO
@ LO
Definition: AArch64BaseInfo.h:258
llvm::MVT::getVectorElementType
MVT getVectorElementType() const
Definition: MachineValueType.h:519
llvm::AArch64CC::HI
@ HI
Definition: AArch64BaseInfo.h:263
llvm::ISD::UMULO
@ UMULO
Definition: ISDOpcodes.h:319
MathExtras.h
llvm
---------------------— PointerInfo ------------------------------------—
Definition: AllocatorList.h:23
llvm::SDNode::getValueType
EVT getValueType(unsigned ResNo) const
Return the type of a specified result.
Definition: SelectionDAGNodes.h:966
llvm::SDLoc
Wrapper class for IR location info (IR ordering and DebugLoc) to be passed into SDNode creation funct...
Definition: SelectionDAGNodes.h:1086
llvm::TargetLoweringBase::Legal
@ Legal
Definition: TargetLowering.h:197
llvm::ISD::OR
@ OR
Definition: ISDOpcodes.h:633
llvm::ISD::BITCAST
@ BITCAST
BITCAST - This operator converts between integer, vector and FP values, as if the value was stored to...
Definition: ISDOpcodes.h:848
llvm::ISD::NON_EXTLOAD
@ NON_EXTLOAD
Definition: ISDOpcodes.h:1335
llvm::ISD::FMINNUM
@ FMINNUM
FMINNUM/FMAXNUM - Perform floating-point minimum or maximum on two values.
Definition: ISDOpcodes.h:898
llvm::ISD::AssertSext
@ AssertSext
AssertSext, AssertZext - These nodes record if a register contains a value that has already been zero...
Definition: ISDOpcodes.h:61
llvm::ISD::UDIVFIXSAT
@ UDIVFIXSAT
Definition: ISDOpcodes.h:374
T
llvm::SDValue::getNode
SDNode * getNode() const
get the SDNode which holds the desired result
Definition: SelectionDAGNodes.h:152
llvm::ISD::BSWAP
@ BSWAP
Byte Swap and Counting operators.
Definition: ISDOpcodes.h:666
llvm::ISD::UDIV
@ UDIV
Definition: ISDOpcodes.h:243
llvm::ISD::STRICT_UINT_TO_FP
@ STRICT_UINT_TO_FP
Definition: ISDOpcodes.h:436
llvm::ISD::UADDO
@ UADDO
Definition: ISDOpcodes.h:311
llvm::ISD::CTTZ_ZERO_UNDEF
@ CTTZ_ZERO_UNDEF
Bit counting operators with an undefined result for zero inputs.
Definition: ISDOpcodes.h:674
llvm::ARM_MB::LD
@ LD
Definition: ARMBaseInfo.h:72
llvm::ISD::FSHL
@ FSHL
Definition: ISDOpcodes.h:662
llvm::SmallVector
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
Definition: SmallVector.h:1168
llvm::ISD::FP_TO_UINT_SAT
@ FP_TO_UINT_SAT
Definition: ISDOpcodes.h:801
llvm::MVT::isVector
bool isVector() const
Return true if this is a vector value type.
Definition: MachineValueType.h:366
ErrorHandling.h
llvm::ISD::SDIVFIX
@ SDIVFIX
RESULT = [US]DIVFIX(LHS, RHS, SCALE) - Perform fixed point division on 2 integers with the same width...
Definition: ISDOpcodes.h:367
llvm::ISD::MGATHER
@ MGATHER
Definition: ISDOpcodes.h:1172
llvm::SmallDenseMap
Definition: DenseMap.h:880
llvm::ISD::FLOG2
@ FLOG2
Definition: ISDOpcodes.h:875
llvm::ISD::ANY_EXTEND
@ ANY_EXTEND
ANY_EXTEND - Used for integer types. The high bits are undefined.
Definition: ISDOpcodes.h:732
llvm::ISD::USUBSAT
@ USUBSAT
Definition: ISDOpcodes.h:337
llvm::SDNode
Represents one node in the SelectionDAG.
Definition: SelectionDAGNodes.h:455
llvm::ISD::FMA
@ FMA
FMA - Perform a * b + c with no intermediate rounding step.
Definition: ISDOpcodes.h:466
llvm::ISD::FP_TO_SINT
@ FP_TO_SINT
FP_TO_[US]INT - Convert a floating point value to a signed or unsigned integer.
Definition: ISDOpcodes.h:785
llvm::LoadSDNode
This class is used to represent ISD::LOAD nodes.
Definition: SelectionDAGNodes.h:2281
APInt.h
llvm::ISD::USHLSAT
@ USHLSAT
Definition: ISDOpcodes.h:347
DenseMap.h
llvm::ISD::SETCC
@ SETCC
SetCC operator - This evaluates to a true value iff the condition is true.
Definition: ISDOpcodes.h:702
llvm::EVT::getVectorVT
static EVT getVectorVT(LLVMContext &Context, EVT VT, unsigned NumElements, bool IsScalable=false)
Returns the EVT that represents a vector NumElements in length, where each element is of type VT.
Definition: ValueTypes.h:74
llvm::ISD::VECREDUCE_FMAX
@ VECREDUCE_FMAX
FMIN/FMAX nodes can have flags, for NaN/NoNaN variants.
Definition: ISDOpcodes.h:1228
llvm::ISD::FMAXNUM_IEEE
@ FMAXNUM_IEEE
Definition: ISDOpcodes.h:906
Results
Function Alias Analysis Results
Definition: AliasAnalysis.cpp:847
llvm::ISD::MERGE_VALUES
@ MERGE_VALUES
MERGE_VALUES - This node takes multiple discrete operands and returns them all as its individual resu...
Definition: ISDOpcodes.h:236
llvm::ISD::SIGN_EXTEND_VECTOR_INREG
@ SIGN_EXTEND_VECTOR_INREG
SIGN_EXTEND_VECTOR_INREG(Vector) - This operator represents an in-register sign-extension of the low ...
Definition: ISDOpcodes.h:769
llvm::ISD::VECREDUCE_SEQ_FADD
@ VECREDUCE_SEQ_FADD
Generic reduction nodes.
Definition: ISDOpcodes.h:1212
llvm::TargetLoweringBase::LegalizeAction
LegalizeAction
This enum indicates whether operations are valid for a target, and if not, what action should be used...
Definition: TargetLowering.h:196
llvm::EVT::bitsLE
bool bitsLE(EVT VT) const
Return true if this has no more bits than VT.
Definition: ValueTypes.h:281
SelectionDAG.h
llvm::BitmaskEnumDetail::Mask
std::underlying_type_t< E > Mask()
Get a bitmask with 1s in all places up to the high-order bit of E's largest value.
Definition: BitmaskEnum.h:80
llvm::ISD::STRICT_FP_TO_UINT
@ STRICT_FP_TO_UINT
Definition: ISDOpcodes.h:429
llvm::ISD::SMAX
@ SMAX
Definition: ISDOpcodes.h:627
LLVM_DEBUG
#define LLVM_DEBUG(X)
Definition: Debug.h:101
llvm::ISD::FABS
@ FABS
Definition: ISDOpcodes.h:867
MachineValueType.h
llvm::ISD::ROTL
@ ROTL
Definition: ISDOpcodes.h:660
llvm::dbgs
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
Definition: Debug.cpp:163
llvm::ISD::VECREDUCE_UMAX
@ VECREDUCE_UMAX
Definition: ISDOpcodes.h:1240
llvm::ISD::FFLOOR
@ FFLOOR
Definition: ISDOpcodes.h:885
llvm::ISD::LoadExtType
LoadExtType
LoadExtType enum - This enum defines the three variants of LOADEXT (load with extension).
Definition: ISDOpcodes.h:1335
TargetLowering.h
llvm::ISD::FSHR
@ FSHR
Definition: ISDOpcodes.h:663
llvm::MVT::i1
@ i1
Definition: MachineValueType.h:43
llvm::ISD::STRICT_FP_TO_SINT
@ STRICT_FP_TO_SINT
STRICT_FP_TO_[US]INT - Convert a floating point value to a signed or unsigned integer.
Definition: ISDOpcodes.h:428
llvm::ISD::SELECT_CC
@ SELECT_CC
Select with condition operator - This selects between a true value and a false value (ops #2 and #3) ...
Definition: ISDOpcodes.h:694
llvm::SDValue::getValueType
EVT getValueType() const
Return the ValueType of the referenced return value.
Definition: SelectionDAGNodes.h:1121
llvm::ISD::CTLZ
@ CTLZ
Definition: ISDOpcodes.h:668
llvm::MutableArrayRef
MutableArrayRef - Represent a mutable reference to an array (0 or more elements consecutively in memo...
Definition: ArrayRef.h:307
llvm::SelectionDAG
This is used to represent a portion of an LLVM function in a low-level Data Dependence DAG representa...
Definition: SelectionDAG.h:216
SelectionDAGNodes.h
llvm::ISD::SELECT
@ SELECT
Select(COND, TRUEVAL, FALSEVAL).
Definition: ISDOpcodes.h:679
llvm::ISD::SMULFIXSAT
@ SMULFIXSAT
Same as the corresponding unsaturated fixed point instructions, but the result is clamped between the...
Definition: ISDOpcodes.h:360
llvm::ISD::ZERO_EXTEND
@ ZERO_EXTEND
ZERO_EXTEND - Used for integer types, zeroing the new bits.
Definition: ISDOpcodes.h:729
llvm::ISD::ABS
@ ABS
ABS - Determine the unsigned absolute value of a signed integer value of the same bitwidth.
Definition: ISDOpcodes.h:640
E
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
llvm::ISD::SMULFIX
@ SMULFIX
RESULT = [US]MULFIX(LHS, RHS, SCALE) - Perform fixed point multiplication on 2 integers with the same...
Definition: ISDOpcodes.h:354
llvm::ISD::SIGN_EXTEND_INREG
@ SIGN_EXTEND_INREG
SIGN_EXTEND_INREG - This operator atomically performs a SHL/SRA pair to sign extend a small value in ...
Definition: ISDOpcodes.h:747
llvm::SDNode::setFlags
void setFlags(SDNodeFlags NewFlags)
Definition: SelectionDAGNodes.h:956
llvm::SelectionDAG::getTargetLoweringInfo
const TargetLowering & getTargetLoweringInfo() const
Definition: SelectionDAG.h:443
llvm::EVT
Extended Value Type.
Definition: ValueTypes.h:35
llvm::ISD::ANY_EXTEND_VECTOR_INREG
@ ANY_EXTEND_VECTOR_INREG
ANY_EXTEND_VECTOR_INREG(Vector) - This operator represents an in-register any-extension of the low la...
Definition: ISDOpcodes.h:758
llvm::EVT::getVectorNumElements
unsigned getVectorNumElements() const
Given a vector type, return the number of elements it contains.
Definition: ValueTypes.h:309
llvm::TargetLowering
This class defines information used to lower LLVM code to legal SelectionDAG operators that the targe...
Definition: TargetLowering.h:3189
llvm::ISD::FROUND
@ FROUND
Definition: ISDOpcodes.h:883
llvm::ms_demangle::QualifierMangleMode::Result
@ Result
llvm::ISD::TRUNCATE
@ TRUNCATE
TRUNCATE - Completely drop the high bits.
Definition: ISDOpcodes.h:735
llvm::ISD::SRA
@ SRA
Definition: ISDOpcodes.h:658
llvm::TargetLoweringBase::ZeroOrNegativeOneBooleanContent
@ ZeroOrNegativeOneBooleanContent
Definition: TargetLowering.h:234
llvm::ISD::FMINNUM_IEEE
@ FMINNUM_IEEE
FMINNUM_IEEE/FMAXNUM_IEEE - Perform floating-point minimum or maximum on two values,...
Definition: ISDOpcodes.h:905
llvm::ISD::UDIVREM
@ UDIVREM
Definition: ISDOpcodes.h:256
llvm::ISD::SINT_TO_FP
@ SINT_TO_FP
[SU]INT_TO_FP - These operators convert integers (whose interpreted sign depends on the first letter)...
Definition: ISDOpcodes.h:739
llvm::ISD::FNEARBYINT
@ FNEARBYINT
Definition: ISDOpcodes.h:882
llvm::ISD::FRINT
@ FRINT
Definition: ISDOpcodes.h:881
llvm::SmallVectorImpl::resize
void resize(size_type N)
Definition: SmallVector.h:606
llvm::ISD::AND
@ AND
Bitwise operators - logical and, logical or, logical xor.
Definition: ISDOpcodes.h:632
llvm::ISD::SMULO
@ SMULO
Same for multiplication.
Definition: ISDOpcodes.h:318
llvm::ISD::SPLAT_VECTOR
@ SPLAT_VECTOR
SPLAT_VECTOR(VAL) - Returns a vector with the scalar value VAL duplicated in all lanes.
Definition: ISDOpcodes.h:590
llvm::EVT::changeVectorElementTypeToInteger
EVT changeVectorElementTypeToInteger() const
Return a vector with the same number of elements as this vector, but with the element type converted ...
Definition: ValueTypes.h:94
llvm::SDValue::getValueSizeInBits
TypeSize getValueSizeInBits() const
Returns the size of the value in bits.
Definition: SelectionDAGNodes.h:192
llvm::ISD::USUBO
@ USUBO
Definition: ISDOpcodes.h:315
llvm::MVT::getScalarType
MVT getScalarType() const
If this is a vector, return the element type, otherwise return this.
Definition: MachineValueType.h:515
llvm::ISD::VECREDUCE_FMUL
@ VECREDUCE_FMUL
Definition: ISDOpcodes.h:1226
Operands
mir Rename Register Operands
Definition: MIRNamerPass.cpp:78
llvm::MVT::isFloatingPoint
bool isFloatingPoint() const
Return true if this is a FP or a vector FP type.
Definition: MachineValueType.h:340
llvm::ISD::FPOW
@ FPOW
Definition: ISDOpcodes.h:873
llvm::ISD::FADD
@ FADD
Simple binary floating point operators.
Definition: ISDOpcodes.h:377
llvm::ISD::STRICT_FSETCCS
@ STRICT_FSETCCS
Definition: ISDOpcodes.h:463
llvm::ISD::SMIN
@ SMIN
[US]{MIN/MAX} - Binary minimum or maximum of signed or unsigned integers.
Definition: ISDOpcodes.h:626
llvm::ISD::FMINIMUM
@ FMINIMUM
FMINIMUM/FMAXIMUM - NaN-propagating minimum/maximum that also treat -0.0 as less than 0....
Definition: ISDOpcodes.h:911
llvm::RISCVFenceField::O
@ O
Definition: RISCVBaseInfo.h:197
llvm::ISD::SADDO
@ SADDO
RESULT, BOOL = [SU]ADDO(LHS, RHS) - Overflow-aware nodes for addition.
Definition: ISDOpcodes.h:310
llvm::ISD::FLOG10
@ FLOG10
Definition: ISDOpcodes.h:876
llvm::ISD::VECREDUCE_FMIN
@ VECREDUCE_FMIN
Definition: ISDOpcodes.h:1229
llvm::EVT::getSizeInBits
TypeSize getSizeInBits() const
Return the size of the specified value type in bits.
Definition: ValueTypes.h:341
uint64_t
llvm::ISD::FP_TO_UINT
@ FP_TO_UINT
Definition: ISDOpcodes.h:786
llvm::ISD::VECREDUCE_ADD
@ VECREDUCE_ADD
Integer reductions may have a result type larger than the vector element type.
Definition: ISDOpcodes.h:1233
llvm::ISD::LOAD
@ LOAD
LOAD and STORE have token chains as their first operand, then the same operands as an LLVM load/store...
Definition: ISDOpcodes.h:921
llvm::ISD::VECREDUCE_SMAX
@ VECREDUCE_SMAX
Definition: ISDOpcodes.h:1238
llvm::ARM_MB::ST
@ ST
Definition: ARMBaseInfo.h:73
llvm::ISD::AssertZext
@ AssertZext
Definition: ISDOpcodes.h:62
llvm::TargetLoweringBase::Promote
@ Promote
Definition: TargetLowering.h:198
llvm::numbers::e
constexpr double e
Definition: MathExtras.h:57
llvm::DenseMap
Definition: DenseMap.h:714
llvm::ISD::EXTRACT_VECTOR_ELT
@ EXTRACT_VECTOR_ELT
EXTRACT_VECTOR_ELT(VECTOR, IDX) - Returns a single element from VECTOR identified by the (potentially...
Definition: ISDOpcodes.h:511
llvm::ISD::VECREDUCE_FADD
@ VECREDUCE_FADD
These reductions have relaxed evaluation order semantics, and have a single vector operand.
Definition: ISDOpcodes.h:1225
llvm::ISD::CTLZ_ZERO_UNDEF
@ CTLZ_ZERO_UNDEF
Definition: ISDOpcodes.h:675
I
#define I(x, y, z)
Definition: MD5.cpp:59
llvm::ISD::VECREDUCE_SEQ_FMUL
@ VECREDUCE_SEQ_FMUL
Definition: ISDOpcodes.h:1213
llvm::ISD::UADDSAT
@ UADDSAT
Definition: ISDOpcodes.h:328
llvm::ISD::SSUBSAT
@ SSUBSAT
RESULT = [US]SUBSAT(LHS, RHS) - Perform saturation subtraction on 2 integers with the same bit width ...
Definition: ISDOpcodes.h:336
llvm::ISD::FCOPYSIGN
@ FCOPYSIGN
FCOPYSIGN(X, Y) - Return the value of X with the sign of Y.
Definition: ISDOpcodes.h:476
llvm::MVT::getVectorNumElements
unsigned getVectorNumElements() const
Definition: MachineValueType.h:850
llvm::StoreSDNode
This class is used to represent ISD::STORE nodes.
Definition: SelectionDAGNodes.h:2309
llvm::MVT::i8
@ i8
Definition: MachineValueType.h:44
llvm::SDValue::getValue
SDValue getValue(unsigned R) const
Definition: SelectionDAGNodes.h:172
llvm::TargetLoweringBase::getSetCCResultType
virtual EVT getSetCCResultType(const DataLayout &DL, LLVMContext &Context, EVT VT) const
Return the ValueType of the result of SETCC operations.
Definition: TargetLoweringBase.cpp:1512
assert
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
llvm::ISD::MULHS
@ MULHS
Definition: ISDOpcodes.h:615
llvm::MVT::Other
@ Other
Definition: MachineValueType.h:42
llvm::ISD::CondCode
CondCode
ISD::CondCode enum - These are ordered carefully to make the bitfields below work out,...
Definition: ISDOpcodes.h:1355
llvm::ISD::SDIVFIXSAT
@ SDIVFIXSAT
Same as the corresponding unsaturated fixed point instructions, but the result is clamped between the...
Definition: ISDOpcodes.h:373
llvm::ISD::VECREDUCE_AND
@ VECREDUCE_AND
Definition: ISDOpcodes.h:1235
llvm::MVT
Machine Value Type.
Definition: MachineValueType.h:31
llvm::ISD::FMAXIMUM
@ FMAXIMUM
Definition: ISDOpcodes.h:912
llvm::EVT::isVector
bool isVector() const
Return true if this is a vector value type.
Definition: ValueTypes.h:155
llvm::ISD::UMAX
@ UMAX
Definition: ISDOpcodes.h:629
llvm::any_of
bool any_of(R &&range, UnaryPredicate P)
Provide wrappers to std::any_of which take ranges instead of having to pass begin/end explicitly.
Definition: STLExtras.h:1558
DataLayout.h
llvm::EVT::getScalarSizeInBits
uint64_t getScalarSizeInBits() const
Definition: ValueTypes.h:353
llvm::ISD::STRICT_SINT_TO_FP
@ STRICT_SINT_TO_FP
STRICT_[US]INT_TO_FP - Convert a signed or unsigned integer to a floating point value.
Definition: ISDOpcodes.h:435
llvm_unreachable
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
Definition: ErrorHandling.h:136
llvm::ISD::SREM
@ SREM
Definition: ISDOpcodes.h:244
llvm::ISD::UMUL_LOHI
@ UMUL_LOHI
Definition: ISDOpcodes.h:251
Compiler.h
DL
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
Definition: AArch64SLSHardening.cpp:76
llvm::ISD::FEXP
@ FEXP
Definition: ISDOpcodes.h:877
llvm::ISD::SMUL_LOHI
@ SMUL_LOHI
SMUL_LOHI/UMUL_LOHI - Multiply two integers of type iN, producing a signed/unsigned value of type i[2...
Definition: ISDOpcodes.h:250
llvm::ISD::UMULFIX
@ UMULFIX
Definition: ISDOpcodes.h:355
llvm::ISD::FEXP2
@ FEXP2
Definition: ISDOpcodes.h:878
LLVM_FALLTHROUGH
#define LLVM_FALLTHROUGH
LLVM_FALLTHROUGH - Mark fallthrough cases in switch statements.
Definition: Compiler.h:273
llvm::ISD::VECREDUCE_XOR
@ VECREDUCE_XOR
Definition: ISDOpcodes.h:1237
llvm::ISD::FMUL
@ FMUL
Definition: ISDOpcodes.h:379
llvm::DenseMapBase< SmallDenseMap< KeyT, ValueT, 4, DenseMapInfo< KeyT >, llvm::detail::DenseMapPair< KeyT, ValueT > >, KeyT, ValueT, DenseMapInfo< KeyT >, llvm::detail::DenseMapPair< KeyT, ValueT > >::insert
std::pair< iterator, bool > insert(const std::pair< KeyT, ValueT > &KV)
Definition: DenseMap.h:207
llvm::ISD::SSHLSAT
@ SSHLSAT
RESULT = [US]SHLSAT(LHS, RHS) - Perform saturation left shift.
Definition: ISDOpcodes.h:346
createBSWAPShuffleMask
static void createBSWAPShuffleMask(EVT VT, SmallVectorImpl< int > &ShuffleMask)
Definition: LegalizeVectorOps.cpp:1088
llvm::ISD::XOR
@ XOR
Definition: ISDOpcodes.h:634
llvm::ISD::FSQRT
@ FSQRT
Definition: ISDOpcodes.h:868
llvm::ISD::INSERT_SUBVECTOR
@ INSERT_SUBVECTOR
INSERT_SUBVECTOR(VECTOR1, VECTOR2, IDX) - Returns a vector with VECTOR2 inserted into VECTOR1.
Definition: ISDOpcodes.h:535
j
return j(j<< 16)
llvm::ISD::STRICT_FMUL
@ STRICT_FMUL
Definition: ISDOpcodes.h:389
llvm::ISD::FMAXNUM
@ FMAXNUM
Definition: ISDOpcodes.h:899
llvm::ISD::FP_TO_SINT_SAT
@ FP_TO_SINT_SAT
FP_TO_[US]INT_SAT - Convert floating point value in operand 0 to a signed or unsigned scalar integer ...
Definition: ISDOpcodes.h:800
llvm::ISD::VECREDUCE_MUL
@ VECREDUCE_MUL
Definition: ISDOpcodes.h:1234
llvm::EVT::getScalarType
EVT getScalarType() const
If this is a vector type, return the element type, otherwise return this.
Definition: ValueTypes.h:296
llvm::ISD::UMULFIXSAT
@ UMULFIXSAT
Definition: ISDOpcodes.h:361
llvm::AMDGPU::SendMsg::Op
Op
Definition: SIDefines.h:321
llvm::ilist_iterator
Iterator for intrusive lists based on ilist_node.
Definition: ilist_iterator.h:57
llvm::ISD::FCOS
@ FCOS
Definition: ISDOpcodes.h:871
llvm::ISD::FCEIL
@ FCEIL
Definition: ISDOpcodes.h:879
llvm::ISD::FSIN
@ FSIN
Definition: ISDOpcodes.h:870
ISDOpcodes.h
llvm::ISD::BUILD_VECTOR
@ BUILD_VECTOR
BUILD_VECTOR(ELT0, ELT1, ELT2, ELT3,...) - Return a fixed-width vector with the specified,...
Definition: ISDOpcodes.h:491
Casting.h
llvm::TargetLoweringBase::Custom
@ Custom
Definition: TargetLowering.h:201
llvm::ISD::VECREDUCE_OR
@ VECREDUCE_OR
Definition: ISDOpcodes.h:1236
llvm::ISD::SDIV
@ SDIV
Definition: ISDOpcodes.h:242
llvm::SDValue
Unlike LLVM values, Selection DAG nodes may return multiple values as the result of a computation.
Definition: SelectionDAGNodes.h:138
llvm::SDNode::getNumValues
unsigned getNumValues() const
Return the number of values defined/returned by this operator.
Definition: SelectionDAGNodes.h:963
llvm::SelectionDAG::LegalizeVectors
bool LegalizeVectors()
This transforms the SelectionDAG into a SelectionDAG that only uses vector math operations supported ...
Definition: LegalizeVectorOps.cpp:1538
llvm::TargetLoweringBase::ZeroOrOneBooleanContent
@ ZeroOrOneBooleanContent
Definition: TargetLowering.h:233
llvm::ISD::STORE
@ STORE
Definition: ISDOpcodes.h:922
llvm::ISD::UINT_TO_FP
@ UINT_TO_FP
Definition: ISDOpcodes.h:740
llvm::ISD::ADD
@ ADD
Simple integer binary arithmetic operators.
Definition: ISDOpcodes.h:239
llvm::ISD::SSUBO
@ SSUBO
Same for subtraction.
Definition: ISDOpcodes.h:314
llvm::EVT::getVectorElementType
EVT getVectorElementType() const
Given a vector type, return the type of each element.
Definition: ValueTypes.h:301
llvm::ISD::FP_EXTEND
@ FP_EXTEND
X = FP_EXTEND(Y) - Extend a smaller FP type into a larger FP type.
Definition: ISDOpcodes.h:833
llvm::ISD::FSUB
@ FSUB
Definition: ISDOpcodes.h:378
llvm::ISD::SHL
@ SHL
Shift and rotation operations.
Definition: ISDOpcodes.h:657
SmallVector.h
llvm::ISD::FREM
@ FREM
Definition: ISDOpcodes.h:381
llvm::ISD::MUL
@ MUL
Definition: ISDOpcodes.h:241
llvm::ISD::UREM
@ UREM
Definition: ISDOpcodes.h:245
llvm::TargetLoweringBase::Expand
@ Expand
Definition: TargetLowering.h:199
llvm::ISD::ZERO_EXTEND_VECTOR_INREG
@ ZERO_EXTEND_VECTOR_INREG
ZERO_EXTEND_VECTOR_INREG(Vector) - This operator represents an in-register zero-extension of the low ...
Definition: ISDOpcodes.h:780
N
#define N
llvm::ISD::BITREVERSE
@ BITREVERSE
Definition: ISDOpcodes.h:670
llvm::ISD::SRL
@ SRL
Definition: ISDOpcodes.h:659
llvm::ISD::CTTZ
@ CTTZ
Definition: ISDOpcodes.h:667
llvm::ISD::STRICT_FADD
@ STRICT_FADD
Constrained versions of the binary floating point operators.
Definition: ISDOpcodes.h:387
llvm::ISD::UMIN
@ UMIN
Definition: ISDOpcodes.h:628
MachineMemOperand.h
llvm::SmallVectorImpl
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
Definition: APFloat.h:43
llvm::ISD::MULHU
@ MULHU
MULHU/MULHS - Multiply high - Multiply two integers of type iN, producing an unsigned/signed value of...
Definition: ISDOpcodes.h:614
llvm::ISD::FNEG
@ FNEG
Perform various unary floating-point operations inspired by libm.
Definition: ISDOpcodes.h:866
llvm::ISD::UDIVFIX
@ UDIVFIX
Definition: ISDOpcodes.h:368
llvm::ISD::SDIVREM
@ SDIVREM
SDIVREM/UDIVREM - Divide two integers and produce both a quotient and remainder result.
Definition: ISDOpcodes.h:255
From
BlockVerifier::State From
Definition: BlockVerifier.cpp:55
llvm::ISD::VECREDUCE_SMIN
@ VECREDUCE_SMIN
Definition: ISDOpcodes.h:1239
llvm::ISD::SIGN_EXTEND
@ SIGN_EXTEND
Conversion operators.
Definition: ISDOpcodes.h:726
llvm::ISD::VECREDUCE_UMIN
@ VECREDUCE_UMIN
Definition: ISDOpcodes.h:1241
llvm::ISD::FTRUNC
@ FTRUNC
Definition: ISDOpcodes.h:880
llvm::SmallVectorImpl::reserve
void reserve(size_type N)
Definition: SmallVector.h:624
llvm::ISD::FCANONICALIZE
@ FCANONICALIZE
Returns platform specific canonical encoding of a floating point number.
Definition: ISDOpcodes.h:483
llvm::ISD::FLOG
@ FLOG
Definition: ISDOpcodes.h:874
llvm::EVT::isFixedLengthVector
bool isFixedLengthVector() const
Definition: ValueTypes.h:165
llvm::ISD::FP_ROUND
@ FP_ROUND
X = FP_ROUND(Y, TRUNC) - Rounding 'Y' from a larger floating point type down to the precision of the ...
Definition: ISDOpcodes.h:814
llvm::ISD::ROTR
@ ROTR
Definition: ISDOpcodes.h:661
Debug.h
llvm::ISD::CTPOP
@ CTPOP
Definition: ISDOpcodes.h:669
llvm::ISD::SADDSAT
@ SADDSAT
RESULT = [US]ADDSAT(LHS, RHS) - Perform saturation addition on 2 integers with the same bit width (W)...
Definition: ISDOpcodes.h:327
llvm::ISD::TokenFactor
@ TokenFactor
TokenFactor - This node takes multiple tokens as input and produces a single token result.
Definition: ISDOpcodes.h:52
llvm::Function::size
size_t size() const
Definition: Function.h:738
llvm::ISD::VSELECT
@ VSELECT
Select with a vector condition (op #0) and two vector operands (ops #1 and #2), returning a vector re...
Definition: ISDOpcodes.h:688
llvm::ISD::FDIV
@ FDIV
Definition: ISDOpcodes.h:380