LLVM  9.0.0svn
LegalizeTypes.h
Go to the documentation of this file.
1 //===-- LegalizeTypes.h - DAG Type Legalizer class definition ---*- C++ -*-===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file defines the DAGTypeLegalizer class. This is a private interface
10 // shared between the code that implements the SelectionDAG::LegalizeTypes
11 // method.
12 //
13 //===----------------------------------------------------------------------===//
14 
15 #ifndef LLVM_LIB_CODEGEN_SELECTIONDAG_LEGALIZETYPES_H
16 #define LLVM_LIB_CODEGEN_SELECTIONDAG_LEGALIZETYPES_H
17 
18 #include "llvm/ADT/DenseMap.h"
21 #include "llvm/Support/Compiler.h"
22 #include "llvm/Support/Debug.h"
23 
24 namespace llvm {
25 
26 //===----------------------------------------------------------------------===//
27 /// This takes an arbitrary SelectionDAG as input and hacks on it until only
28 /// value types the target machine can handle are left. This involves promoting
29 /// small sizes to large sizes or splitting up large values into small values.
30 ///
32  const TargetLowering &TLI;
33  SelectionDAG &DAG;
34 public:
35  /// This pass uses the NodeId on the SDNodes to hold information about the
36  /// state of the node. The enum has all the values.
37  enum NodeIdFlags {
38  /// All operands have been processed, so this node is ready to be handled.
39  ReadyToProcess = 0,
40 
41  /// This is a new node, not before seen, that was created in the process of
42  /// legalizing some other node.
43  NewNode = -1,
44 
45  /// This node's ID needs to be set to the number of its unprocessed
46  /// operands.
47  Unanalyzed = -2,
48 
49  /// This is a node that has already been processed.
50  Processed = -3
51 
52  // 1+ - This is a node which has this many unprocessed operands.
53  };
54 private:
55 
56  /// This is a bitvector that contains two bits for each simple value type,
57  /// where the two bits correspond to the LegalizeAction enum from
58  /// TargetLowering. This can be queried with "getTypeAction(VT)".
59  TargetLowering::ValueTypeActionImpl ValueTypeActions;
60 
61  /// Return how we should legalize values of this type.
62  TargetLowering::LegalizeTypeAction getTypeAction(EVT VT) const {
63  return TLI.getTypeAction(*DAG.getContext(), VT);
64  }
65 
66  /// Return true if this type is legal on this target.
67  bool isTypeLegal(EVT VT) const {
68  return TLI.getTypeAction(*DAG.getContext(), VT) == TargetLowering::TypeLegal;
69  }
70 
71  /// Return true if this is a simple legal type.
72  bool isSimpleLegalType(EVT VT) const {
73  return VT.isSimple() && TLI.isTypeLegal(VT);
74  }
75 
76  /// Return true if this type can be passed in registers.
77  /// For example, x86_64's f128, should to be legally in registers
78  /// and only some operations converted to library calls or integer
79  /// bitwise operations.
80  bool isLegalInHWReg(EVT VT) const {
81  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
82  return VT == NVT && isSimpleLegalType(VT);
83  }
84 
85  EVT getSetCCResultType(EVT VT) const {
86  return TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT);
87  }
88 
89  /// Pretend all of this node's results are legal.
90  bool IgnoreNodeResults(SDNode *N) const {
91  return N->getOpcode() == ISD::TargetConstant ||
92  N->getOpcode() == ISD::Register;
93  }
94 
95  // Bijection from SDValue to unique id. As each created node gets a
96  // new id we do not need to worry about reuse expunging. Should we
97  // run out of ids, we can do a one time expensive compactifcation.
98  typedef unsigned TableId;
99 
100  TableId NextValueId = 1;
101 
104 
105  /// For integer nodes that are below legal width, this map indicates what
106  /// promoted value to use.
107  SmallDenseMap<TableId, TableId, 8> PromotedIntegers;
108 
109  /// For integer nodes that need to be expanded this map indicates which
110  /// operands are the expanded version of the input.
112 
113  /// For floating-point nodes converted to integers of the same size, this map
114  /// indicates the converted value to use.
115  SmallDenseMap<TableId, TableId, 8> SoftenedFloats;
116 
117  /// For floating-point nodes that have a smaller precision than the smallest
118  /// supported precision, this map indicates what promoted value to use.
119  SmallDenseMap<TableId, TableId, 8> PromotedFloats;
120 
121  /// For float nodes that need to be expanded this map indicates which operands
122  /// are the expanded version of the input.
124 
125  /// For nodes that are <1 x ty>, this map indicates the scalar value of type
126  /// 'ty' to use.
127  SmallDenseMap<TableId, TableId, 8> ScalarizedVectors;
128 
129  /// For nodes that need to be split this map indicates which operands are the
130  /// expanded version of the input.
132 
133  /// For vector nodes that need to be widened, indicates the widened value to
134  /// use.
135  SmallDenseMap<TableId, TableId, 8> WidenedVectors;
136 
137  /// For values that have been replaced with another, indicates the replacement
138  /// value to use.
139  SmallDenseMap<TableId, TableId, 8> ReplacedValues;
140 
141  /// This defines a worklist of nodes to process. In order to be pushed onto
142  /// this worklist, all operands of a node must have already been processed.
143  SmallVector<SDNode*, 128> Worklist;
144 
145  TableId getTableId(SDValue V) {
146  assert(V.getNode() && "Getting TableId on SDValue()");
147 
148  auto I = ValueToIdMap.find(V);
149  if (I != ValueToIdMap.end()) {
150  // replace if there's been a shift.
151  RemapId(I->second);
152  assert(I->second && "All Ids should be nonzero");
153  return I->second;
154  }
155  // Add if it's not there.
156  ValueToIdMap.insert(std::make_pair(V, NextValueId));
157  IdToValueMap.insert(std::make_pair(NextValueId, V));
158  ++NextValueId;
159  assert(NextValueId != 0 &&
160  "Ran out of Ids. Increase id type size or add compactification");
161  return NextValueId - 1;
162  }
163 
164  const SDValue &getSDValue(TableId &Id) {
165  RemapId(Id);
166  assert(Id && "TableId should be non-zero");
167  return IdToValueMap[Id];
168  }
169 
170 public:
172  : TLI(dag.getTargetLoweringInfo()), DAG(dag),
173  ValueTypeActions(TLI.getValueTypeActions()) {
175  "Too many value types for ValueTypeActions to hold!");
176  }
177 
178  /// This is the main entry point for the type legalizer. This does a
179  /// top-down traversal of the dag, legalizing types as it goes. Returns
180  /// "true" if it made any changes.
181  bool run();
182 
183  void NoteDeletion(SDNode *Old, SDNode *New) {
184  for (unsigned i = 0, e = Old->getNumValues(); i != e; ++i) {
185  TableId NewId = getTableId(SDValue(New, i));
186  TableId OldId = getTableId(SDValue(Old, i));
187 
188  if (OldId != NewId)
189  ReplacedValues[OldId] = NewId;
190 
191  // Delete Node from tables.
192  ValueToIdMap.erase(SDValue(Old, i));
193  IdToValueMap.erase(OldId);
194  PromotedIntegers.erase(OldId);
195  ExpandedIntegers.erase(OldId);
196  SoftenedFloats.erase(OldId);
197  PromotedFloats.erase(OldId);
198  ExpandedFloats.erase(OldId);
199  ScalarizedVectors.erase(OldId);
200  SplitVectors.erase(OldId);
201  WidenedVectors.erase(OldId);
202  }
203  }
204 
205  SelectionDAG &getDAG() const { return DAG; }
206 
207 private:
208  SDNode *AnalyzeNewNode(SDNode *N);
209  void AnalyzeNewValue(SDValue &Val);
210  void PerformExpensiveChecks();
211  void RemapId(TableId &Id);
212  void RemapValue(SDValue &V);
213 
214  // Common routines.
215  SDValue BitConvertToInteger(SDValue Op);
216  SDValue BitConvertVectorToIntegerVector(SDValue Op);
217  SDValue CreateStackStoreLoad(SDValue Op, EVT DestVT);
218  bool CustomLowerNode(SDNode *N, EVT VT, bool LegalizeResult);
219  bool CustomWidenLowerNode(SDNode *N, EVT VT);
220 
221  /// Replace each result of the given MERGE_VALUES node with the corresponding
222  /// input operand, except for the result 'ResNo', for which the corresponding
223  /// input operand is returned.
224  SDValue DisintegrateMERGE_VALUES(SDNode *N, unsigned ResNo);
225 
226  SDValue JoinIntegers(SDValue Lo, SDValue Hi);
227  SDValue LibCallify(RTLIB::Libcall LC, SDNode *N, bool isSigned);
228 
229  std::pair<SDValue, SDValue> ExpandChainLibCall(RTLIB::Libcall LC,
230  SDNode *Node, bool isSigned);
231  std::pair<SDValue, SDValue> ExpandAtomic(SDNode *Node);
232 
233  SDValue PromoteTargetBoolean(SDValue Bool, EVT ValVT);
234 
235  void ReplaceValueWith(SDValue From, SDValue To);
236  void SplitInteger(SDValue Op, SDValue &Lo, SDValue &Hi);
237  void SplitInteger(SDValue Op, EVT LoVT, EVT HiVT,
238  SDValue &Lo, SDValue &Hi);
239 
240  void AddToWorklist(SDNode *N) {
241  N->setNodeId(ReadyToProcess);
242  Worklist.push_back(N);
243  }
244 
245  //===--------------------------------------------------------------------===//
246  // Integer Promotion Support: LegalizeIntegerTypes.cpp
247  //===--------------------------------------------------------------------===//
248 
249  /// Given a processed operand Op which was promoted to a larger integer type,
250  /// this returns the promoted value. The low bits of the promoted value
251  /// corresponding to the original type are exactly equal to Op.
252  /// The extra bits contain rubbish, so the promoted value may need to be zero-
253  /// or sign-extended from the original type before it is usable (the helpers
254  /// SExtPromotedInteger and ZExtPromotedInteger can do this for you).
255  /// For example, if Op is an i16 and was promoted to an i32, then this method
256  /// returns an i32, the lower 16 bits of which coincide with Op, and the upper
257  /// 16 bits of which contain rubbish.
258  SDValue GetPromotedInteger(SDValue Op) {
259  TableId &PromotedId = PromotedIntegers[getTableId(Op)];
260  SDValue PromotedOp = getSDValue(PromotedId);
261  assert(PromotedOp.getNode() && "Operand wasn't promoted?");
262  return PromotedOp;
263  }
264  void SetPromotedInteger(SDValue Op, SDValue Result);
265 
266  /// Get a promoted operand and sign extend it to the final size.
267  SDValue SExtPromotedInteger(SDValue Op) {
268  EVT OldVT = Op.getValueType();
269  SDLoc dl(Op);
270  Op = GetPromotedInteger(Op);
271  return DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, Op.getValueType(), Op,
272  DAG.getValueType(OldVT));
273  }
274 
275  /// Get a promoted operand and zero extend it to the final size.
276  SDValue ZExtPromotedInteger(SDValue Op) {
277  EVT OldVT = Op.getValueType();
278  SDLoc dl(Op);
279  Op = GetPromotedInteger(Op);
280  return DAG.getZeroExtendInReg(Op, dl, OldVT.getScalarType());
281  }
282 
283  // Get a promoted operand and sign or zero extend it to the final size
284  // (depending on TargetLoweringInfo::isSExtCheaperThanZExt). For a given
285  // subtarget and type, the choice of sign or zero-extension will be
286  // consistent.
287  SDValue SExtOrZExtPromotedInteger(SDValue Op) {
288  EVT OldVT = Op.getValueType();
289  SDLoc DL(Op);
290  Op = GetPromotedInteger(Op);
291  if (TLI.isSExtCheaperThanZExt(OldVT, Op.getValueType()))
292  return DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, Op.getValueType(), Op,
293  DAG.getValueType(OldVT));
294  return DAG.getZeroExtendInReg(Op, DL, OldVT.getScalarType());
295  }
296 
297  // Integer Result Promotion.
298  void PromoteIntegerResult(SDNode *N, unsigned ResNo);
299  SDValue PromoteIntRes_MERGE_VALUES(SDNode *N, unsigned ResNo);
300  SDValue PromoteIntRes_AssertSext(SDNode *N);
301  SDValue PromoteIntRes_AssertZext(SDNode *N);
302  SDValue PromoteIntRes_Atomic0(AtomicSDNode *N);
303  SDValue PromoteIntRes_Atomic1(AtomicSDNode *N);
304  SDValue PromoteIntRes_AtomicCmpSwap(AtomicSDNode *N, unsigned ResNo);
305  SDValue PromoteIntRes_EXTRACT_SUBVECTOR(SDNode *N);
306  SDValue PromoteIntRes_VECTOR_SHUFFLE(SDNode *N);
307  SDValue PromoteIntRes_BUILD_VECTOR(SDNode *N);
308  SDValue PromoteIntRes_SCALAR_TO_VECTOR(SDNode *N);
309  SDValue PromoteIntRes_EXTEND_VECTOR_INREG(SDNode *N);
310  SDValue PromoteIntRes_INSERT_VECTOR_ELT(SDNode *N);
311  SDValue PromoteIntRes_CONCAT_VECTORS(SDNode *N);
312  SDValue PromoteIntRes_BITCAST(SDNode *N);
313  SDValue PromoteIntRes_BSWAP(SDNode *N);
314  SDValue PromoteIntRes_BITREVERSE(SDNode *N);
315  SDValue PromoteIntRes_BUILD_PAIR(SDNode *N);
316  SDValue PromoteIntRes_Constant(SDNode *N);
317  SDValue PromoteIntRes_CTLZ(SDNode *N);
318  SDValue PromoteIntRes_CTPOP(SDNode *N);
319  SDValue PromoteIntRes_CTTZ(SDNode *N);
320  SDValue PromoteIntRes_EXTRACT_VECTOR_ELT(SDNode *N);
321  SDValue PromoteIntRes_FP_TO_XINT(SDNode *N);
322  SDValue PromoteIntRes_FP_TO_FP16(SDNode *N);
323  SDValue PromoteIntRes_INT_EXTEND(SDNode *N);
324  SDValue PromoteIntRes_LOAD(LoadSDNode *N);
325  SDValue PromoteIntRes_MLOAD(MaskedLoadSDNode *N);
326  SDValue PromoteIntRes_MGATHER(MaskedGatherSDNode *N);
327  SDValue PromoteIntRes_Overflow(SDNode *N);
328  SDValue PromoteIntRes_SADDSUBO(SDNode *N, unsigned ResNo);
329  SDValue PromoteIntRes_SELECT(SDNode *N);
330  SDValue PromoteIntRes_VSELECT(SDNode *N);
331  SDValue PromoteIntRes_SELECT_CC(SDNode *N);
332  SDValue PromoteIntRes_SETCC(SDNode *N);
333  SDValue PromoteIntRes_SHL(SDNode *N);
334  SDValue PromoteIntRes_SimpleIntBinOp(SDNode *N);
335  SDValue PromoteIntRes_ZExtIntBinOp(SDNode *N);
336  SDValue PromoteIntRes_SExtIntBinOp(SDNode *N);
337  SDValue PromoteIntRes_SIGN_EXTEND_INREG(SDNode *N);
338  SDValue PromoteIntRes_SRA(SDNode *N);
339  SDValue PromoteIntRes_SRL(SDNode *N);
340  SDValue PromoteIntRes_TRUNCATE(SDNode *N);
341  SDValue PromoteIntRes_UADDSUBO(SDNode *N, unsigned ResNo);
342  SDValue PromoteIntRes_ADDSUBCARRY(SDNode *N, unsigned ResNo);
343  SDValue PromoteIntRes_UNDEF(SDNode *N);
344  SDValue PromoteIntRes_VAARG(SDNode *N);
345  SDValue PromoteIntRes_XMULO(SDNode *N, unsigned ResNo);
346  SDValue PromoteIntRes_ADDSUBSAT(SDNode *N);
347  SDValue PromoteIntRes_MULFIX(SDNode *N);
348  SDValue PromoteIntRes_FLT_ROUNDS(SDNode *N);
349  SDValue PromoteIntRes_VECREDUCE(SDNode *N);
350  SDValue PromoteIntRes_ABS(SDNode *N);
351 
352  // Integer Operand Promotion.
353  bool PromoteIntegerOperand(SDNode *N, unsigned OpNo);
354  SDValue PromoteIntOp_ANY_EXTEND(SDNode *N);
355  SDValue PromoteIntOp_ATOMIC_STORE(AtomicSDNode *N);
356  SDValue PromoteIntOp_BITCAST(SDNode *N);
357  SDValue PromoteIntOp_BUILD_PAIR(SDNode *N);
358  SDValue PromoteIntOp_BR_CC(SDNode *N, unsigned OpNo);
359  SDValue PromoteIntOp_BRCOND(SDNode *N, unsigned OpNo);
360  SDValue PromoteIntOp_BUILD_VECTOR(SDNode *N);
361  SDValue PromoteIntOp_INSERT_VECTOR_ELT(SDNode *N, unsigned OpNo);
362  SDValue PromoteIntOp_EXTRACT_VECTOR_ELT(SDNode *N);
363  SDValue PromoteIntOp_EXTRACT_SUBVECTOR(SDNode *N);
364  SDValue PromoteIntOp_CONCAT_VECTORS(SDNode *N);
365  SDValue PromoteIntOp_SCALAR_TO_VECTOR(SDNode *N);
366  SDValue PromoteIntOp_SELECT(SDNode *N, unsigned OpNo);
367  SDValue PromoteIntOp_SELECT_CC(SDNode *N, unsigned OpNo);
368  SDValue PromoteIntOp_SETCC(SDNode *N, unsigned OpNo);
369  SDValue PromoteIntOp_Shift(SDNode *N);
370  SDValue PromoteIntOp_SIGN_EXTEND(SDNode *N);
371  SDValue PromoteIntOp_SINT_TO_FP(SDNode *N);
372  SDValue PromoteIntOp_STORE(StoreSDNode *N, unsigned OpNo);
373  SDValue PromoteIntOp_TRUNCATE(SDNode *N);
374  SDValue PromoteIntOp_UINT_TO_FP(SDNode *N);
375  SDValue PromoteIntOp_ZERO_EXTEND(SDNode *N);
376  SDValue PromoteIntOp_MSTORE(MaskedStoreSDNode *N, unsigned OpNo);
377  SDValue PromoteIntOp_MLOAD(MaskedLoadSDNode *N, unsigned OpNo);
378  SDValue PromoteIntOp_MSCATTER(MaskedScatterSDNode *N, unsigned OpNo);
379  SDValue PromoteIntOp_MGATHER(MaskedGatherSDNode *N, unsigned OpNo);
380  SDValue PromoteIntOp_ADDSUBCARRY(SDNode *N, unsigned OpNo);
381  SDValue PromoteIntOp_FRAMERETURNADDR(SDNode *N);
382  SDValue PromoteIntOp_PREFETCH(SDNode *N, unsigned OpNo);
383  SDValue PromoteIntOp_MULFIX(SDNode *N);
384  SDValue PromoteIntOp_FPOWI(SDNode *N);
385  SDValue PromoteIntOp_VECREDUCE(SDNode *N);
386 
387  void PromoteSetCCOperands(SDValue &LHS,SDValue &RHS, ISD::CondCode Code);
388 
389  //===--------------------------------------------------------------------===//
390  // Integer Expansion Support: LegalizeIntegerTypes.cpp
391  //===--------------------------------------------------------------------===//
392 
393  /// Given a processed operand Op which was expanded into two integers of half
394  /// the size, this returns the two halves. The low bits of Op are exactly
395  /// equal to the bits of Lo; the high bits exactly equal Hi.
396  /// For example, if Op is an i64 which was expanded into two i32's, then this
397  /// method returns the two i32's, with Lo being equal to the lower 32 bits of
398  /// Op, and Hi being equal to the upper 32 bits.
399  void GetExpandedInteger(SDValue Op, SDValue &Lo, SDValue &Hi);
400  void SetExpandedInteger(SDValue Op, SDValue Lo, SDValue Hi);
401 
402  // Integer Result Expansion.
403  void ExpandIntegerResult(SDNode *N, unsigned ResNo);
404  void ExpandIntRes_ANY_EXTEND (SDNode *N, SDValue &Lo, SDValue &Hi);
405  void ExpandIntRes_AssertSext (SDNode *N, SDValue &Lo, SDValue &Hi);
406  void ExpandIntRes_AssertZext (SDNode *N, SDValue &Lo, SDValue &Hi);
407  void ExpandIntRes_Constant (SDNode *N, SDValue &Lo, SDValue &Hi);
408  void ExpandIntRes_ABS (SDNode *N, SDValue &Lo, SDValue &Hi);
409  void ExpandIntRes_CTLZ (SDNode *N, SDValue &Lo, SDValue &Hi);
410  void ExpandIntRes_CTPOP (SDNode *N, SDValue &Lo, SDValue &Hi);
411  void ExpandIntRes_CTTZ (SDNode *N, SDValue &Lo, SDValue &Hi);
412  void ExpandIntRes_LOAD (LoadSDNode *N, SDValue &Lo, SDValue &Hi);
413  void ExpandIntRes_READCYCLECOUNTER (SDNode *N, SDValue &Lo, SDValue &Hi);
414  void ExpandIntRes_SIGN_EXTEND (SDNode *N, SDValue &Lo, SDValue &Hi);
415  void ExpandIntRes_SIGN_EXTEND_INREG (SDNode *N, SDValue &Lo, SDValue &Hi);
416  void ExpandIntRes_TRUNCATE (SDNode *N, SDValue &Lo, SDValue &Hi);
417  void ExpandIntRes_ZERO_EXTEND (SDNode *N, SDValue &Lo, SDValue &Hi);
418  void ExpandIntRes_FLT_ROUNDS (SDNode *N, SDValue &Lo, SDValue &Hi);
419  void ExpandIntRes_FP_TO_SINT (SDNode *N, SDValue &Lo, SDValue &Hi);
420  void ExpandIntRes_FP_TO_UINT (SDNode *N, SDValue &Lo, SDValue &Hi);
421  void ExpandIntRes_LLROUND (SDNode *N, SDValue &Lo, SDValue &Hi);
422  void ExpandIntRes_LLRINT (SDNode *N, SDValue &Lo, SDValue &Hi);
423 
424  void ExpandIntRes_Logical (SDNode *N, SDValue &Lo, SDValue &Hi);
425  void ExpandIntRes_ADDSUB (SDNode *N, SDValue &Lo, SDValue &Hi);
426  void ExpandIntRes_ADDSUBC (SDNode *N, SDValue &Lo, SDValue &Hi);
427  void ExpandIntRes_ADDSUBE (SDNode *N, SDValue &Lo, SDValue &Hi);
428  void ExpandIntRes_ADDSUBCARRY (SDNode *N, SDValue &Lo, SDValue &Hi);
429  void ExpandIntRes_BITREVERSE (SDNode *N, SDValue &Lo, SDValue &Hi);
430  void ExpandIntRes_BSWAP (SDNode *N, SDValue &Lo, SDValue &Hi);
431  void ExpandIntRes_MUL (SDNode *N, SDValue &Lo, SDValue &Hi);
432  void ExpandIntRes_SDIV (SDNode *N, SDValue &Lo, SDValue &Hi);
433  void ExpandIntRes_SREM (SDNode *N, SDValue &Lo, SDValue &Hi);
434  void ExpandIntRes_UDIV (SDNode *N, SDValue &Lo, SDValue &Hi);
435  void ExpandIntRes_UREM (SDNode *N, SDValue &Lo, SDValue &Hi);
436  void ExpandIntRes_Shift (SDNode *N, SDValue &Lo, SDValue &Hi);
437 
438  void ExpandIntRes_MINMAX (SDNode *N, SDValue &Lo, SDValue &Hi);
439 
440  void ExpandIntRes_SADDSUBO (SDNode *N, SDValue &Lo, SDValue &Hi);
441  void ExpandIntRes_UADDSUBO (SDNode *N, SDValue &Lo, SDValue &Hi);
442  void ExpandIntRes_XMULO (SDNode *N, SDValue &Lo, SDValue &Hi);
443  void ExpandIntRes_ADDSUBSAT (SDNode *N, SDValue &Lo, SDValue &Hi);
444  void ExpandIntRes_MULFIX (SDNode *N, SDValue &Lo, SDValue &Hi);
445 
446  void ExpandIntRes_ATOMIC_LOAD (SDNode *N, SDValue &Lo, SDValue &Hi);
447  void ExpandIntRes_VECREDUCE (SDNode *N, SDValue &Lo, SDValue &Hi);
448 
449  void ExpandShiftByConstant(SDNode *N, const APInt &Amt,
450  SDValue &Lo, SDValue &Hi);
451  bool ExpandShiftWithKnownAmountBit(SDNode *N, SDValue &Lo, SDValue &Hi);
452  bool ExpandShiftWithUnknownAmountBit(SDNode *N, SDValue &Lo, SDValue &Hi);
453 
454  // Integer Operand Expansion.
455  bool ExpandIntegerOperand(SDNode *N, unsigned OpNo);
456  SDValue ExpandIntOp_BR_CC(SDNode *N);
457  SDValue ExpandIntOp_SELECT_CC(SDNode *N);
458  SDValue ExpandIntOp_SETCC(SDNode *N);
459  SDValue ExpandIntOp_SETCCCARRY(SDNode *N);
460  SDValue ExpandIntOp_Shift(SDNode *N);
461  SDValue ExpandIntOp_SINT_TO_FP(SDNode *N);
462  SDValue ExpandIntOp_STORE(StoreSDNode *N, unsigned OpNo);
463  SDValue ExpandIntOp_TRUNCATE(SDNode *N);
464  SDValue ExpandIntOp_UINT_TO_FP(SDNode *N);
465  SDValue ExpandIntOp_RETURNADDR(SDNode *N);
466  SDValue ExpandIntOp_ATOMIC_STORE(SDNode *N);
467 
468  void IntegerExpandSetCCOperands(SDValue &NewLHS, SDValue &NewRHS,
469  ISD::CondCode &CCCode, const SDLoc &dl);
470 
471  //===--------------------------------------------------------------------===//
472  // Float to Integer Conversion Support: LegalizeFloatTypes.cpp
473  //===--------------------------------------------------------------------===//
474 
475  /// Given an operand Op of Float type, returns the integer if the Op is not
476  /// supported in target HW and converted to the integer.
477  /// The integer contains exactly the same bits as Op - only the type changed.
478  /// For example, if Op is an f32 which was softened to an i32, then this
479  /// method returns an i32, the bits of which coincide with those of Op.
480  /// If the Op can be efficiently supported in target HW or the operand must
481  /// stay in a register, the Op is not converted to an integer.
482  /// In that case, the given op is returned.
483  SDValue GetSoftenedFloat(SDValue Op) {
484  TableId Id = getTableId(Op);
485  auto Iter = SoftenedFloats.find(Id);
486  if (Iter == SoftenedFloats.end()) {
487  assert(isSimpleLegalType(Op.getValueType()) &&
488  "Operand wasn't converted to integer?");
489  return Op;
490  }
491  SDValue SoftenedOp = getSDValue(Iter->second);
492  assert(SoftenedOp.getNode() && "Unconverted op in SoftenedFloats?");
493  return SoftenedOp;
494  }
495  void SetSoftenedFloat(SDValue Op, SDValue Result);
496 
497  // Convert Float Results to Integer for Non-HW-supported Operations.
498  bool SoftenFloatResult(SDNode *N, unsigned ResNo);
499  SDValue SoftenFloatRes_MERGE_VALUES(SDNode *N, unsigned ResNo);
500  SDValue SoftenFloatRes_BITCAST(SDNode *N, unsigned ResNo);
501  SDValue SoftenFloatRes_BUILD_PAIR(SDNode *N);
502  SDValue SoftenFloatRes_ConstantFP(SDNode *N, unsigned ResNo);
503  SDValue SoftenFloatRes_EXTRACT_VECTOR_ELT(SDNode *N, unsigned ResNo);
504  SDValue SoftenFloatRes_FABS(SDNode *N, unsigned ResNo);
505  SDValue SoftenFloatRes_FMINNUM(SDNode *N);
506  SDValue SoftenFloatRes_FMAXNUM(SDNode *N);
507  SDValue SoftenFloatRes_FADD(SDNode *N);
508  SDValue SoftenFloatRes_FCEIL(SDNode *N);
509  SDValue SoftenFloatRes_FCOPYSIGN(SDNode *N, unsigned ResNo);
510  SDValue SoftenFloatRes_FCOS(SDNode *N);
511  SDValue SoftenFloatRes_FDIV(SDNode *N);
512  SDValue SoftenFloatRes_FEXP(SDNode *N);
513  SDValue SoftenFloatRes_FEXP2(SDNode *N);
514  SDValue SoftenFloatRes_FFLOOR(SDNode *N);
515  SDValue SoftenFloatRes_FLOG(SDNode *N);
516  SDValue SoftenFloatRes_FLOG2(SDNode *N);
517  SDValue SoftenFloatRes_FLOG10(SDNode *N);
518  SDValue SoftenFloatRes_FMA(SDNode *N);
519  SDValue SoftenFloatRes_FMUL(SDNode *N);
520  SDValue SoftenFloatRes_FNEARBYINT(SDNode *N);
521  SDValue SoftenFloatRes_FNEG(SDNode *N, unsigned ResNo);
522  SDValue SoftenFloatRes_FP_EXTEND(SDNode *N);
523  SDValue SoftenFloatRes_FP16_TO_FP(SDNode *N);
524  SDValue SoftenFloatRes_FP_ROUND(SDNode *N);
525  SDValue SoftenFloatRes_FPOW(SDNode *N);
526  SDValue SoftenFloatRes_FPOWI(SDNode *N);
527  SDValue SoftenFloatRes_FREM(SDNode *N);
528  SDValue SoftenFloatRes_FRINT(SDNode *N);
529  SDValue SoftenFloatRes_FROUND(SDNode *N);
530  SDValue SoftenFloatRes_FSIN(SDNode *N);
531  SDValue SoftenFloatRes_FSQRT(SDNode *N);
532  SDValue SoftenFloatRes_FSUB(SDNode *N);
533  SDValue SoftenFloatRes_FTRUNC(SDNode *N);
534  SDValue SoftenFloatRes_LOAD(SDNode *N, unsigned ResNo);
535  SDValue SoftenFloatRes_SELECT(SDNode *N, unsigned ResNo);
536  SDValue SoftenFloatRes_SELECT_CC(SDNode *N, unsigned ResNo);
537  SDValue SoftenFloatRes_UNDEF(SDNode *N);
538  SDValue SoftenFloatRes_VAARG(SDNode *N);
539  SDValue SoftenFloatRes_XINT_TO_FP(SDNode *N);
540 
541  // Return true if we can skip softening the given operand or SDNode because
542  // either it was soften before by SoftenFloatResult and references to the
543  // operand were replaced by ReplaceValueWith or it's value type is legal in HW
544  // registers and the operand can be left unchanged.
545  bool CanSkipSoftenFloatOperand(SDNode *N, unsigned OpNo);
546 
547  // Convert Float Operand to Integer for Non-HW-supported Operations.
548  bool SoftenFloatOperand(SDNode *N, unsigned OpNo);
549  SDValue SoftenFloatOp_BITCAST(SDNode *N);
550  SDValue SoftenFloatOp_COPY_TO_REG(SDNode *N);
551  SDValue SoftenFloatOp_BR_CC(SDNode *N);
552  SDValue SoftenFloatOp_FABS(SDNode *N);
553  SDValue SoftenFloatOp_FCOPYSIGN(SDNode *N);
554  SDValue SoftenFloatOp_FNEG(SDNode *N);
555  SDValue SoftenFloatOp_FP_EXTEND(SDNode *N);
556  SDValue SoftenFloatOp_FP_ROUND(SDNode *N);
557  SDValue SoftenFloatOp_FP_TO_XINT(SDNode *N);
558  SDValue SoftenFloatOp_LROUND(SDNode *N);
559  SDValue SoftenFloatOp_LLROUND(SDNode *N);
560  SDValue SoftenFloatOp_LRINT(SDNode *N);
561  SDValue SoftenFloatOp_LLRINT(SDNode *N);
562  SDValue SoftenFloatOp_SELECT(SDNode *N);
563  SDValue SoftenFloatOp_SELECT_CC(SDNode *N);
564  SDValue SoftenFloatOp_SETCC(SDNode *N);
565  SDValue SoftenFloatOp_STORE(SDNode *N, unsigned OpNo);
566 
567  //===--------------------------------------------------------------------===//
568  // Float Expansion Support: LegalizeFloatTypes.cpp
569  //===--------------------------------------------------------------------===//
570 
571  /// Given a processed operand Op which was expanded into two floating-point
572  /// values of half the size, this returns the two halves.
573  /// The low bits of Op are exactly equal to the bits of Lo; the high bits
574  /// exactly equal Hi. For example, if Op is a ppcf128 which was expanded
575  /// into two f64's, then this method returns the two f64's, with Lo being
576  /// equal to the lower 64 bits of Op, and Hi to the upper 64 bits.
577  void GetExpandedFloat(SDValue Op, SDValue &Lo, SDValue &Hi);
578  void SetExpandedFloat(SDValue Op, SDValue Lo, SDValue Hi);
579 
580  // Float Result Expansion.
581  void ExpandFloatResult(SDNode *N, unsigned ResNo);
582  void ExpandFloatRes_ConstantFP(SDNode *N, SDValue &Lo, SDValue &Hi);
583  void ExpandFloatRes_FABS (SDNode *N, SDValue &Lo, SDValue &Hi);
584  void ExpandFloatRes_FMINNUM (SDNode *N, SDValue &Lo, SDValue &Hi);
585  void ExpandFloatRes_FMAXNUM (SDNode *N, SDValue &Lo, SDValue &Hi);
586  void ExpandFloatRes_FADD (SDNode *N, SDValue &Lo, SDValue &Hi);
587  void ExpandFloatRes_FCEIL (SDNode *N, SDValue &Lo, SDValue &Hi);
588  void ExpandFloatRes_FCOPYSIGN (SDNode *N, SDValue &Lo, SDValue &Hi);
589  void ExpandFloatRes_FCOS (SDNode *N, SDValue &Lo, SDValue &Hi);
590  void ExpandFloatRes_FDIV (SDNode *N, SDValue &Lo, SDValue &Hi);
591  void ExpandFloatRes_FEXP (SDNode *N, SDValue &Lo, SDValue &Hi);
592  void ExpandFloatRes_FEXP2 (SDNode *N, SDValue &Lo, SDValue &Hi);
593  void ExpandFloatRes_FFLOOR (SDNode *N, SDValue &Lo, SDValue &Hi);
594  void ExpandFloatRes_FLOG (SDNode *N, SDValue &Lo, SDValue &Hi);
595  void ExpandFloatRes_FLOG2 (SDNode *N, SDValue &Lo, SDValue &Hi);
596  void ExpandFloatRes_FLOG10 (SDNode *N, SDValue &Lo, SDValue &Hi);
597  void ExpandFloatRes_FMA (SDNode *N, SDValue &Lo, SDValue &Hi);
598  void ExpandFloatRes_FMUL (SDNode *N, SDValue &Lo, SDValue &Hi);
599  void ExpandFloatRes_FNEARBYINT(SDNode *N, SDValue &Lo, SDValue &Hi);
600  void ExpandFloatRes_FNEG (SDNode *N, SDValue &Lo, SDValue &Hi);
601  void ExpandFloatRes_FP_EXTEND (SDNode *N, SDValue &Lo, SDValue &Hi);
602  void ExpandFloatRes_FPOW (SDNode *N, SDValue &Lo, SDValue &Hi);
603  void ExpandFloatRes_FPOWI (SDNode *N, SDValue &Lo, SDValue &Hi);
604  void ExpandFloatRes_FREM (SDNode *N, SDValue &Lo, SDValue &Hi);
605  void ExpandFloatRes_FRINT (SDNode *N, SDValue &Lo, SDValue &Hi);
606  void ExpandFloatRes_FROUND (SDNode *N, SDValue &Lo, SDValue &Hi);
607  void ExpandFloatRes_FSIN (SDNode *N, SDValue &Lo, SDValue &Hi);
608  void ExpandFloatRes_FSQRT (SDNode *N, SDValue &Lo, SDValue &Hi);
609  void ExpandFloatRes_FSUB (SDNode *N, SDValue &Lo, SDValue &Hi);
610  void ExpandFloatRes_FTRUNC (SDNode *N, SDValue &Lo, SDValue &Hi);
611  void ExpandFloatRes_LOAD (SDNode *N, SDValue &Lo, SDValue &Hi);
612  void ExpandFloatRes_XINT_TO_FP(SDNode *N, SDValue &Lo, SDValue &Hi);
613 
614  // Float Operand Expansion.
615  bool ExpandFloatOperand(SDNode *N, unsigned OpNo);
616  SDValue ExpandFloatOp_BR_CC(SDNode *N);
617  SDValue ExpandFloatOp_FCOPYSIGN(SDNode *N);
618  SDValue ExpandFloatOp_FP_ROUND(SDNode *N);
619  SDValue ExpandFloatOp_FP_TO_SINT(SDNode *N);
620  SDValue ExpandFloatOp_FP_TO_UINT(SDNode *N);
621  SDValue ExpandFloatOp_LROUND(SDNode *N);
622  SDValue ExpandFloatOp_LLROUND(SDNode *N);
623  SDValue ExpandFloatOp_LRINT(SDNode *N);
624  SDValue ExpandFloatOp_LLRINT(SDNode *N);
625  SDValue ExpandFloatOp_SELECT_CC(SDNode *N);
626  SDValue ExpandFloatOp_SETCC(SDNode *N);
627  SDValue ExpandFloatOp_STORE(SDNode *N, unsigned OpNo);
628 
629  void FloatExpandSetCCOperands(SDValue &NewLHS, SDValue &NewRHS,
630  ISD::CondCode &CCCode, const SDLoc &dl);
631 
632  //===--------------------------------------------------------------------===//
633  // Float promotion support: LegalizeFloatTypes.cpp
634  //===--------------------------------------------------------------------===//
635 
636  SDValue GetPromotedFloat(SDValue Op) {
637  TableId &PromotedId = PromotedFloats[getTableId(Op)];
638  SDValue PromotedOp = getSDValue(PromotedId);
639  assert(PromotedOp.getNode() && "Operand wasn't promoted?");
640  return PromotedOp;
641  }
642  void SetPromotedFloat(SDValue Op, SDValue Result);
643 
644  void PromoteFloatResult(SDNode *N, unsigned ResNo);
645  SDValue PromoteFloatRes_BITCAST(SDNode *N);
646  SDValue PromoteFloatRes_BinOp(SDNode *N);
647  SDValue PromoteFloatRes_ConstantFP(SDNode *N);
648  SDValue PromoteFloatRes_EXTRACT_VECTOR_ELT(SDNode *N);
649  SDValue PromoteFloatRes_FCOPYSIGN(SDNode *N);
650  SDValue PromoteFloatRes_FMAD(SDNode *N);
651  SDValue PromoteFloatRes_FPOWI(SDNode *N);
652  SDValue PromoteFloatRes_FP_ROUND(SDNode *N);
653  SDValue PromoteFloatRes_LOAD(SDNode *N);
654  SDValue PromoteFloatRes_SELECT(SDNode *N);
655  SDValue PromoteFloatRes_SELECT_CC(SDNode *N);
656  SDValue PromoteFloatRes_UnaryOp(SDNode *N);
657  SDValue PromoteFloatRes_UNDEF(SDNode *N);
658  SDValue BitcastToInt_ATOMIC_SWAP(SDNode *N);
659  SDValue PromoteFloatRes_XINT_TO_FP(SDNode *N);
660 
661  bool PromoteFloatOperand(SDNode *N, unsigned OpNo);
662  SDValue PromoteFloatOp_BITCAST(SDNode *N, unsigned OpNo);
663  SDValue PromoteFloatOp_FCOPYSIGN(SDNode *N, unsigned OpNo);
664  SDValue PromoteFloatOp_FP_EXTEND(SDNode *N, unsigned OpNo);
665  SDValue PromoteFloatOp_FP_TO_XINT(SDNode *N, unsigned OpNo);
666  SDValue PromoteFloatOp_STORE(SDNode *N, unsigned OpNo);
667  SDValue PromoteFloatOp_SELECT_CC(SDNode *N, unsigned OpNo);
668  SDValue PromoteFloatOp_SETCC(SDNode *N, unsigned OpNo);
669 
670  //===--------------------------------------------------------------------===//
671  // Scalarization Support: LegalizeVectorTypes.cpp
672  //===--------------------------------------------------------------------===//
673 
674  /// Given a processed one-element vector Op which was scalarized to its
675  /// element type, this returns the element. For example, if Op is a v1i32,
676  /// Op = < i32 val >, this method returns val, an i32.
677  SDValue GetScalarizedVector(SDValue Op) {
678  TableId &ScalarizedId = ScalarizedVectors[getTableId(Op)];
679  SDValue ScalarizedOp = getSDValue(ScalarizedId);
680  assert(ScalarizedOp.getNode() && "Operand wasn't scalarized?");
681  return ScalarizedOp;
682  }
683  void SetScalarizedVector(SDValue Op, SDValue Result);
684 
685  // Vector Result Scalarization: <1 x ty> -> ty.
686  void ScalarizeVectorResult(SDNode *N, unsigned ResNo);
687  SDValue ScalarizeVecRes_MERGE_VALUES(SDNode *N, unsigned ResNo);
688  SDValue ScalarizeVecRes_BinOp(SDNode *N);
689  SDValue ScalarizeVecRes_TernaryOp(SDNode *N);
690  SDValue ScalarizeVecRes_UnaryOp(SDNode *N);
691  SDValue ScalarizeVecRes_StrictFPOp(SDNode *N);
692  SDValue ScalarizeVecRes_OverflowOp(SDNode *N, unsigned ResNo);
693  SDValue ScalarizeVecRes_InregOp(SDNode *N);
694  SDValue ScalarizeVecRes_VecInregOp(SDNode *N);
695 
696  SDValue ScalarizeVecRes_BITCAST(SDNode *N);
697  SDValue ScalarizeVecRes_BUILD_VECTOR(SDNode *N);
698  SDValue ScalarizeVecRes_EXTRACT_SUBVECTOR(SDNode *N);
699  SDValue ScalarizeVecRes_FP_ROUND(SDNode *N);
700  SDValue ScalarizeVecRes_STRICT_FP_ROUND(SDNode *N);
701  SDValue ScalarizeVecRes_FPOWI(SDNode *N);
702  SDValue ScalarizeVecRes_INSERT_VECTOR_ELT(SDNode *N);
703  SDValue ScalarizeVecRes_LOAD(LoadSDNode *N);
704  SDValue ScalarizeVecRes_SCALAR_TO_VECTOR(SDNode *N);
705  SDValue ScalarizeVecRes_VSELECT(SDNode *N);
706  SDValue ScalarizeVecRes_SELECT(SDNode *N);
707  SDValue ScalarizeVecRes_SELECT_CC(SDNode *N);
708  SDValue ScalarizeVecRes_SETCC(SDNode *N);
709  SDValue ScalarizeVecRes_UNDEF(SDNode *N);
710  SDValue ScalarizeVecRes_VECTOR_SHUFFLE(SDNode *N);
711 
712  SDValue ScalarizeVecRes_MULFIX(SDNode *N);
713 
714  // Vector Operand Scalarization: <1 x ty> -> ty.
715  bool ScalarizeVectorOperand(SDNode *N, unsigned OpNo);
716  SDValue ScalarizeVecOp_BITCAST(SDNode *N);
717  SDValue ScalarizeVecOp_UnaryOp(SDNode *N);
718  SDValue ScalarizeVecOp_CONCAT_VECTORS(SDNode *N);
719  SDValue ScalarizeVecOp_EXTRACT_VECTOR_ELT(SDNode *N);
720  SDValue ScalarizeVecOp_VSELECT(SDNode *N);
721  SDValue ScalarizeVecOp_VSETCC(SDNode *N);
722  SDValue ScalarizeVecOp_STORE(StoreSDNode *N, unsigned OpNo);
723  SDValue ScalarizeVecOp_FP_ROUND(SDNode *N, unsigned OpNo);
724  SDValue ScalarizeVecOp_STRICT_FP_ROUND(SDNode *N, unsigned OpNo);
725  SDValue ScalarizeVecOp_VECREDUCE(SDNode *N);
726 
727  //===--------------------------------------------------------------------===//
728  // Vector Splitting Support: LegalizeVectorTypes.cpp
729  //===--------------------------------------------------------------------===//
730 
731  /// Given a processed vector Op which was split into vectors of half the size,
732  /// this method returns the halves. The first elements of Op coincide with the
733  /// elements of Lo; the remaining elements of Op coincide with the elements of
734  /// Hi: Op is what you would get by concatenating Lo and Hi.
735  /// For example, if Op is a v8i32 that was split into two v4i32's, then this
736  /// method returns the two v4i32's, with Lo corresponding to the first 4
737  /// elements of Op, and Hi to the last 4 elements.
738  void GetSplitVector(SDValue Op, SDValue &Lo, SDValue &Hi);
739  void SetSplitVector(SDValue Op, SDValue Lo, SDValue Hi);
740 
741  // Vector Result Splitting: <128 x ty> -> 2 x <64 x ty>.
742  void SplitVectorResult(SDNode *N, unsigned ResNo);
743  void SplitVecRes_BinOp(SDNode *N, SDValue &Lo, SDValue &Hi);
744  void SplitVecRes_TernaryOp(SDNode *N, SDValue &Lo, SDValue &Hi);
745  void SplitVecRes_UnaryOp(SDNode *N, SDValue &Lo, SDValue &Hi);
746  void SplitVecRes_ExtendOp(SDNode *N, SDValue &Lo, SDValue &Hi);
747  void SplitVecRes_InregOp(SDNode *N, SDValue &Lo, SDValue &Hi);
748  void SplitVecRes_ExtVecInRegOp(SDNode *N, SDValue &Lo, SDValue &Hi);
749  void SplitVecRes_StrictFPOp(SDNode *N, SDValue &Lo, SDValue &Hi);
750  void SplitVecRes_OverflowOp(SDNode *N, unsigned ResNo,
751  SDValue &Lo, SDValue &Hi);
752 
753  void SplitVecRes_MULFIX(SDNode *N, SDValue &Lo, SDValue &Hi);
754 
755  void SplitVecRes_BITCAST(SDNode *N, SDValue &Lo, SDValue &Hi);
756  void SplitVecRes_BUILD_VECTOR(SDNode *N, SDValue &Lo, SDValue &Hi);
757  void SplitVecRes_CONCAT_VECTORS(SDNode *N, SDValue &Lo, SDValue &Hi);
758  void SplitVecRes_EXTRACT_SUBVECTOR(SDNode *N, SDValue &Lo, SDValue &Hi);
759  void SplitVecRes_INSERT_SUBVECTOR(SDNode *N, SDValue &Lo, SDValue &Hi);
760  void SplitVecRes_FPOWI(SDNode *N, SDValue &Lo, SDValue &Hi);
761  void SplitVecRes_FCOPYSIGN(SDNode *N, SDValue &Lo, SDValue &Hi);
762  void SplitVecRes_INSERT_VECTOR_ELT(SDNode *N, SDValue &Lo, SDValue &Hi);
763  void SplitVecRes_LOAD(LoadSDNode *LD, SDValue &Lo, SDValue &Hi);
764  void SplitVecRes_MLOAD(MaskedLoadSDNode *MLD, SDValue &Lo, SDValue &Hi);
765  void SplitVecRes_MGATHER(MaskedGatherSDNode *MGT, SDValue &Lo, SDValue &Hi);
766  void SplitVecRes_SCALAR_TO_VECTOR(SDNode *N, SDValue &Lo, SDValue &Hi);
767  void SplitVecRes_SETCC(SDNode *N, SDValue &Lo, SDValue &Hi);
768  void SplitVecRes_VECTOR_SHUFFLE(ShuffleVectorSDNode *N, SDValue &Lo,
769  SDValue &Hi);
770  void SplitVecRes_VAARG(SDNode *N, SDValue &Lo, SDValue &Hi);
771 
772  // Vector Operand Splitting: <128 x ty> -> 2 x <64 x ty>.
773  bool SplitVectorOperand(SDNode *N, unsigned OpNo);
774  SDValue SplitVecOp_VSELECT(SDNode *N, unsigned OpNo);
775  SDValue SplitVecOp_VECREDUCE(SDNode *N, unsigned OpNo);
776  SDValue SplitVecOp_UnaryOp(SDNode *N);
777  SDValue SplitVecOp_TruncateHelper(SDNode *N);
778 
779  SDValue SplitVecOp_BITCAST(SDNode *N);
780  SDValue SplitVecOp_EXTRACT_SUBVECTOR(SDNode *N);
781  SDValue SplitVecOp_EXTRACT_VECTOR_ELT(SDNode *N);
782  SDValue SplitVecOp_ExtVecInRegOp(SDNode *N);
783  SDValue SplitVecOp_STORE(StoreSDNode *N, unsigned OpNo);
784  SDValue SplitVecOp_MSTORE(MaskedStoreSDNode *N, unsigned OpNo);
785  SDValue SplitVecOp_MSCATTER(MaskedScatterSDNode *N, unsigned OpNo);
786  SDValue SplitVecOp_MGATHER(MaskedGatherSDNode *MGT, unsigned OpNo);
787  SDValue SplitVecOp_CONCAT_VECTORS(SDNode *N);
788  SDValue SplitVecOp_VSETCC(SDNode *N);
789  SDValue SplitVecOp_FP_ROUND(SDNode *N);
790  SDValue SplitVecOp_FCOPYSIGN(SDNode *N);
791 
792  //===--------------------------------------------------------------------===//
793  // Vector Widening Support: LegalizeVectorTypes.cpp
794  //===--------------------------------------------------------------------===//
795 
796  /// Given a processed vector Op which was widened into a larger vector, this
797  /// method returns the larger vector. The elements of the returned vector
798  /// consist of the elements of Op followed by elements containing rubbish.
799  /// For example, if Op is a v2i32 that was widened to a v4i32, then this
800  /// method returns a v4i32 for which the first two elements are the same as
801  /// those of Op, while the last two elements contain rubbish.
802  SDValue GetWidenedVector(SDValue Op) {
803  TableId &WidenedId = WidenedVectors[getTableId(Op)];
804  SDValue WidenedOp = getSDValue(WidenedId);
805  assert(WidenedOp.getNode() && "Operand wasn't widened?");
806  return WidenedOp;
807  }
808  void SetWidenedVector(SDValue Op, SDValue Result);
809 
810  // Widen Vector Result Promotion.
811  void WidenVectorResult(SDNode *N, unsigned ResNo);
812  SDValue WidenVecRes_MERGE_VALUES(SDNode* N, unsigned ResNo);
813  SDValue WidenVecRes_BITCAST(SDNode* N);
814  SDValue WidenVecRes_BUILD_VECTOR(SDNode* N);
815  SDValue WidenVecRes_CONCAT_VECTORS(SDNode* N);
816  SDValue WidenVecRes_EXTEND_VECTOR_INREG(SDNode* N);
817  SDValue WidenVecRes_EXTRACT_SUBVECTOR(SDNode* N);
818  SDValue WidenVecRes_INSERT_VECTOR_ELT(SDNode* N);
819  SDValue WidenVecRes_LOAD(SDNode* N);
820  SDValue WidenVecRes_MLOAD(MaskedLoadSDNode* N);
821  SDValue WidenVecRes_MGATHER(MaskedGatherSDNode* N);
822  SDValue WidenVecRes_SCALAR_TO_VECTOR(SDNode* N);
823  SDValue WidenVecRes_SELECT(SDNode* N);
824  SDValue WidenVSELECTAndMask(SDNode *N);
825  SDValue WidenVecRes_SELECT_CC(SDNode* N);
826  SDValue WidenVecRes_SETCC(SDNode* N);
827  SDValue WidenVecRes_UNDEF(SDNode *N);
828  SDValue WidenVecRes_VECTOR_SHUFFLE(ShuffleVectorSDNode *N);
829 
830  SDValue WidenVecRes_Ternary(SDNode *N);
831  SDValue WidenVecRes_Binary(SDNode *N);
832  SDValue WidenVecRes_BinaryCanTrap(SDNode *N);
833  SDValue WidenVecRes_StrictFP(SDNode *N);
834  SDValue WidenVecRes_OverflowOp(SDNode *N, unsigned ResNo);
835  SDValue WidenVecRes_Convert(SDNode *N);
836  SDValue WidenVecRes_Convert_StrictFP(SDNode *N);
837  SDValue WidenVecRes_FCOPYSIGN(SDNode *N);
838  SDValue WidenVecRes_POWI(SDNode *N);
839  SDValue WidenVecRes_Shift(SDNode *N);
840  SDValue WidenVecRes_Unary(SDNode *N);
841  SDValue WidenVecRes_InregOp(SDNode *N);
842 
843  // Widen Vector Operand.
844  bool WidenVectorOperand(SDNode *N, unsigned OpNo);
845  SDValue WidenVecOp_BITCAST(SDNode *N);
846  SDValue WidenVecOp_CONCAT_VECTORS(SDNode *N);
847  SDValue WidenVecOp_EXTEND(SDNode *N);
848  SDValue WidenVecOp_EXTRACT_VECTOR_ELT(SDNode *N);
849  SDValue WidenVecOp_EXTRACT_SUBVECTOR(SDNode *N);
850  SDValue WidenVecOp_STORE(SDNode* N);
851  SDValue WidenVecOp_MSTORE(SDNode* N, unsigned OpNo);
852  SDValue WidenVecOp_MGATHER(SDNode* N, unsigned OpNo);
853  SDValue WidenVecOp_MSCATTER(SDNode* N, unsigned OpNo);
854  SDValue WidenVecOp_SETCC(SDNode* N);
855  SDValue WidenVecOp_VSELECT(SDNode *N);
856 
857  SDValue WidenVecOp_Convert(SDNode *N);
858  SDValue WidenVecOp_FCOPYSIGN(SDNode *N);
859  SDValue WidenVecOp_VECREDUCE(SDNode *N);
860 
861  /// Helper function to generate a set of operations to perform
862  /// a vector operation for a wider type.
863  ///
864  SDValue UnrollVectorOp_StrictFP(SDNode *N, unsigned ResNE);
865 
866  //===--------------------------------------------------------------------===//
867  // Vector Widening Utilities Support: LegalizeVectorTypes.cpp
868  //===--------------------------------------------------------------------===//
869 
870  /// Helper function to generate a set of loads to load a vector with a
871  /// resulting wider type. It takes:
872  /// LdChain: list of chains for the load to be generated.
873  /// Ld: load to widen
874  SDValue GenWidenVectorLoads(SmallVectorImpl<SDValue> &LdChain,
875  LoadSDNode *LD);
876 
877  /// Helper function to generate a set of extension loads to load a vector with
878  /// a resulting wider type. It takes:
879  /// LdChain: list of chains for the load to be generated.
880  /// Ld: load to widen
881  /// ExtType: extension element type
882  SDValue GenWidenVectorExtLoads(SmallVectorImpl<SDValue> &LdChain,
884 
885  /// Helper function to generate a set of stores to store a widen vector into
886  /// non-widen memory.
887  /// StChain: list of chains for the stores we have generated
888  /// ST: store of a widen value
889  void GenWidenVectorStores(SmallVectorImpl<SDValue> &StChain, StoreSDNode *ST);
890 
891  /// Helper function to generate a set of stores to store a truncate widen
892  /// vector into non-widen memory.
893  /// StChain: list of chains for the stores we have generated
894  /// ST: store of a widen value
895  void GenWidenVectorTruncStores(SmallVectorImpl<SDValue> &StChain,
896  StoreSDNode *ST);
897 
898  /// Modifies a vector input (widen or narrows) to a vector of NVT. The
899  /// input vector must have the same element type as NVT.
900  /// When FillWithZeroes is "on" the vector will be widened with zeroes.
901  /// By default, the vector will be widened with undefined values.
902  SDValue ModifyToType(SDValue InOp, EVT NVT, bool FillWithZeroes = false);
903 
904  /// Return a mask of vector type MaskVT to replace InMask. Also adjust
905  /// MaskVT to ToMaskVT if needed with vector extension or truncation.
906  SDValue convertMask(SDValue InMask, EVT MaskVT, EVT ToMaskVT);
907 
908  //===--------------------------------------------------------------------===//
909  // Generic Splitting: LegalizeTypesGeneric.cpp
910  //===--------------------------------------------------------------------===//
911 
912  // Legalization methods which only use that the illegal type is split into two
913  // not necessarily identical types. As such they can be used for splitting
914  // vectors and expanding integers and floats.
915 
916  void GetSplitOp(SDValue Op, SDValue &Lo, SDValue &Hi) {
917  if (Op.getValueType().isVector())
918  GetSplitVector(Op, Lo, Hi);
919  else if (Op.getValueType().isInteger())
920  GetExpandedInteger(Op, Lo, Hi);
921  else
922  GetExpandedFloat(Op, Lo, Hi);
923  }
924 
925  /// Use ISD::EXTRACT_ELEMENT nodes to extract the low and high parts of the
926  /// given value.
927  void GetPairElements(SDValue Pair, SDValue &Lo, SDValue &Hi);
928 
929  // Generic Result Splitting.
930  void SplitRes_MERGE_VALUES(SDNode *N, unsigned ResNo,
931  SDValue &Lo, SDValue &Hi);
932  void SplitRes_SELECT (SDNode *N, SDValue &Lo, SDValue &Hi);
933  void SplitRes_SELECT_CC (SDNode *N, SDValue &Lo, SDValue &Hi);
934  void SplitRes_UNDEF (SDNode *N, SDValue &Lo, SDValue &Hi);
935 
936  //===--------------------------------------------------------------------===//
937  // Generic Expansion: LegalizeTypesGeneric.cpp
938  //===--------------------------------------------------------------------===//
939 
940  // Legalization methods which only use that the illegal type is split into two
941  // identical types of half the size, and that the Lo/Hi part is stored first
942  // in memory on little/big-endian machines, followed by the Hi/Lo part. As
943  // such they can be used for expanding integers and floats.
944 
945  void GetExpandedOp(SDValue Op, SDValue &Lo, SDValue &Hi) {
946  if (Op.getValueType().isInteger())
947  GetExpandedInteger(Op, Lo, Hi);
948  else
949  GetExpandedFloat(Op, Lo, Hi);
950  }
951 
952 
953  /// This function will split the integer \p Op into \p NumElements
954  /// operations of type \p EltVT and store them in \p Ops.
955  void IntegerToVector(SDValue Op, unsigned NumElements,
956  SmallVectorImpl<SDValue> &Ops, EVT EltVT);
957 
958  // Generic Result Expansion.
959  void ExpandRes_MERGE_VALUES (SDNode *N, unsigned ResNo,
960  SDValue &Lo, SDValue &Hi);
961  void ExpandRes_BITCAST (SDNode *N, SDValue &Lo, SDValue &Hi);
962  void ExpandRes_BUILD_PAIR (SDNode *N, SDValue &Lo, SDValue &Hi);
963  void ExpandRes_EXTRACT_ELEMENT (SDNode *N, SDValue &Lo, SDValue &Hi);
964  void ExpandRes_EXTRACT_VECTOR_ELT(SDNode *N, SDValue &Lo, SDValue &Hi);
965  void ExpandRes_NormalLoad (SDNode *N, SDValue &Lo, SDValue &Hi);
966  void ExpandRes_VAARG (SDNode *N, SDValue &Lo, SDValue &Hi);
967 
968  // Generic Operand Expansion.
969  SDValue ExpandOp_BITCAST (SDNode *N);
970  SDValue ExpandOp_BUILD_VECTOR (SDNode *N);
971  SDValue ExpandOp_EXTRACT_ELEMENT (SDNode *N);
972  SDValue ExpandOp_INSERT_VECTOR_ELT(SDNode *N);
973  SDValue ExpandOp_SCALAR_TO_VECTOR (SDNode *N);
974  SDValue ExpandOp_NormalStore (SDNode *N, unsigned OpNo);
975 };
976 
977 } // end namespace llvm.
978 
979 #endif
EVT getValueType() const
Return the ValueType of the referenced return value.
unsigned getOpcode() const
Return the SelectionDAG opcode value for this node.
This class represents lattice values for constants.
Definition: AllocatorList.h:23
EVT getScalarType() const
If this is a vector type, return the element type, otherwise return this.
Definition: ValueTypes.h:259
Libcall
RTLIB::Libcall enum - This enum defines all of the runtime library calls the backend can emit...
bool isInteger() const
Return true if this is an integer or a vector integer type.
Definition: ValueTypes.h:140
This takes an arbitrary SelectionDAG as input and hacks on it until only value types the target machi...
Definition: LegalizeTypes.h:31
void setNodeId(int Id)
Set unique node id.
SDNode * getNode() const
get the SDNode which holds the desired result
SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT, ArrayRef< SDUse > Ops)
Gets or creates the specified node.
This SDNode is used to implement the code generator support for the llvm IR shufflevector instruction...
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
Definition: APFloat.h:41
const DataLayout & getDataLayout() const
Definition: SelectionDAG.h:407
This is an SDNode representing atomic operations.
SelectionDAG & getDAG() const
This class defines information used to lower LLVM code to legal SelectionDAG operators that the targe...
This class is used to represent an MSTORE node.
CondCode
ISD::CondCode enum - These are ordered carefully to make the bitfields below work out...
Definition: ISDOpcodes.h:995
TargetConstant* - Like Constant*, but the DAG does not do any folding, simplification, or lowering of the constant.
Definition: ISDOpcodes.h:124
This class is used to represent ISD::STORE nodes.
unsigned getNumValues() const
Return the number of values defined/returned by this operator.
LoadExtType
LoadExtType enum - This enum defines the three variants of LOADEXT (load with extension).
Definition: ISDOpcodes.h:970
#define LLVM_LIBRARY_VISIBILITY
LLVM_LIBRARY_VISIBILITY - If a class marked with this attribute is linked into a shared library...
Definition: Compiler.h:107
Extended Value Type.
Definition: ValueTypes.h:33
NodeIdFlags
This pass uses the NodeId on the SDNodes to hold information about the state of the node...
Definition: LegalizeTypes.h:37
SDValue getZeroExtendInReg(SDValue Op, const SDLoc &DL, EVT VT)
Return the expression required to zero extend the Op value assuming it was the smaller SrcTy value...
void NoteDeletion(SDNode *Old, SDNode *New)
BlockVerifier::State From
This is used to represent a portion of an LLVM function in a low-level Data Dependence DAG representa...
Definition: SelectionDAG.h:221
This is a &#39;vector&#39; (really, a variable-sized array), optimized for the case when the array is small...
Definition: SmallVector.h:837
Wrapper class for IR location info (IR ordering and DebugLoc) to be passed into SDNode creation funct...
Represents one node in the SelectionDAG.
Class for arbitrary precision integers.
Definition: APInt.h:69
bool isVector() const
Return true if this is a vector value type.
Definition: ValueTypes.h:150
SIGN_EXTEND_INREG - This operator atomically performs a SHL/SRA pair to sign extend a small value in ...
Definition: ISDOpcodes.h:510
#define I(x, y, z)
Definition: MD5.cpp:58
#define N
This class is used to represent an MSCATTER node.
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
LegalizeTypeAction
This enum indicates whether a types are legal for a target, and if not, what action should be used to...
This class is used to represent an MLOAD node.
This class is used to represent an MGATHER node.
SDValue getValueType(EVT)
bool isSimple() const
Test if the given EVT is simple (as opposed to being extended).
Definition: ValueTypes.h:125
Unlike LLVM values, Selection DAG nodes may return multiple values as the result of a computation...
LLVMContext * getContext() const
Definition: SelectionDAG.h:414
This file describes how to lower LLVM code to machine code.
DAGTypeLegalizer(SelectionDAG &dag)
This class is used to represent ISD::LOAD nodes.