LLVM  10.0.0svn
LegalizeIntegerTypes.cpp
Go to the documentation of this file.
1 //===----- LegalizeIntegerTypes.cpp - Legalization of integer types -------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file implements integer type expansion and promotion for LegalizeTypes.
10 // Promotion is the act of changing a computation in an illegal type into a
11 // computation in a larger type. For example, implementing i8 arithmetic in an
12 // i32 register (often needed on powerpc).
13 // Expansion is the act of changing a computation in an illegal type into a
14 // computation in two identical registers of a smaller type. For example,
15 // implementing i64 arithmetic in two i32 registers (often needed on 32-bit
16 // targets).
17 //
18 //===----------------------------------------------------------------------===//
19 
20 #include "LegalizeTypes.h"
21 #include "llvm/IR/DerivedTypes.h"
23 #include "llvm/Support/KnownBits.h"
25 using namespace llvm;
26 
27 #define DEBUG_TYPE "legalize-types"
28 
29 //===----------------------------------------------------------------------===//
30 // Integer Result Promotion
31 //===----------------------------------------------------------------------===//
32 
33 /// PromoteIntegerResult - This method is called when a result of a node is
34 /// found to be in need of promotion to a larger type. At this point, the node
35 /// may also have invalid operands or may have other results that need
36 /// expansion, we just know that (at least) one result needs promotion.
37 void DAGTypeLegalizer::PromoteIntegerResult(SDNode *N, unsigned ResNo) {
38  LLVM_DEBUG(dbgs() << "Promote integer result: "; N->dump(&DAG);
39  dbgs() << "\n");
40  SDValue Res = SDValue();
41 
42  // See if the target wants to custom expand this node.
43  if (CustomLowerNode(N, N->getValueType(ResNo), true)) {
44  LLVM_DEBUG(dbgs() << "Node has been custom expanded, done\n");
45  return;
46  }
47 
48  switch (N->getOpcode()) {
49  default:
50 #ifndef NDEBUG
51  dbgs() << "PromoteIntegerResult #" << ResNo << ": ";
52  N->dump(&DAG); dbgs() << "\n";
53 #endif
54  llvm_unreachable("Do not know how to promote this operator!");
55  case ISD::MERGE_VALUES:Res = PromoteIntRes_MERGE_VALUES(N, ResNo); break;
56  case ISD::AssertSext: Res = PromoteIntRes_AssertSext(N); break;
57  case ISD::AssertZext: Res = PromoteIntRes_AssertZext(N); break;
58  case ISD::BITCAST: Res = PromoteIntRes_BITCAST(N); break;
59  case ISD::BITREVERSE: Res = PromoteIntRes_BITREVERSE(N); break;
60  case ISD::BSWAP: Res = PromoteIntRes_BSWAP(N); break;
61  case ISD::BUILD_PAIR: Res = PromoteIntRes_BUILD_PAIR(N); break;
62  case ISD::Constant: Res = PromoteIntRes_Constant(N); break;
64  case ISD::CTLZ: Res = PromoteIntRes_CTLZ(N); break;
65  case ISD::CTPOP: Res = PromoteIntRes_CTPOP(N); break;
67  case ISD::CTTZ: Res = PromoteIntRes_CTTZ(N); break;
69  Res = PromoteIntRes_EXTRACT_VECTOR_ELT(N); break;
70  case ISD::LOAD: Res = PromoteIntRes_LOAD(cast<LoadSDNode>(N)); break;
71  case ISD::MLOAD: Res = PromoteIntRes_MLOAD(cast<MaskedLoadSDNode>(N));
72  break;
73  case ISD::MGATHER: Res = PromoteIntRes_MGATHER(cast<MaskedGatherSDNode>(N));
74  break;
75  case ISD::SELECT: Res = PromoteIntRes_SELECT(N); break;
76  case ISD::VSELECT: Res = PromoteIntRes_VSELECT(N); break;
77  case ISD::SELECT_CC: Res = PromoteIntRes_SELECT_CC(N); break;
78  case ISD::SETCC: Res = PromoteIntRes_SETCC(N); break;
79  case ISD::SMIN:
80  case ISD::SMAX: Res = PromoteIntRes_SExtIntBinOp(N); break;
81  case ISD::UMIN:
82  case ISD::UMAX: Res = PromoteIntRes_ZExtIntBinOp(N); break;
83 
84  case ISD::SHL: Res = PromoteIntRes_SHL(N); break;
86  Res = PromoteIntRes_SIGN_EXTEND_INREG(N); break;
87  case ISD::SRA: Res = PromoteIntRes_SRA(N); break;
88  case ISD::SRL: Res = PromoteIntRes_SRL(N); break;
89  case ISD::TRUNCATE: Res = PromoteIntRes_TRUNCATE(N); break;
90  case ISD::UNDEF: Res = PromoteIntRes_UNDEF(N); break;
91  case ISD::VAARG: Res = PromoteIntRes_VAARG(N); break;
92 
94  Res = PromoteIntRes_EXTRACT_SUBVECTOR(N); break;
96  Res = PromoteIntRes_VECTOR_SHUFFLE(N); break;
98  Res = PromoteIntRes_INSERT_VECTOR_ELT(N); break;
99  case ISD::BUILD_VECTOR:
100  Res = PromoteIntRes_BUILD_VECTOR(N); break;
102  Res = PromoteIntRes_SCALAR_TO_VECTOR(N); break;
103  case ISD::SPLAT_VECTOR:
104  Res = PromoteIntRes_SPLAT_VECTOR(N); break;
105  case ISD::CONCAT_VECTORS:
106  Res = PromoteIntRes_CONCAT_VECTORS(N); break;
107 
111  Res = PromoteIntRes_EXTEND_VECTOR_INREG(N); break;
112 
113  case ISD::SIGN_EXTEND:
114  case ISD::ZERO_EXTEND:
115  case ISD::ANY_EXTEND: Res = PromoteIntRes_INT_EXTEND(N); break;
116 
119  case ISD::FP_TO_SINT:
120  case ISD::FP_TO_UINT: Res = PromoteIntRes_FP_TO_XINT(N); break;
121 
122  case ISD::FP_TO_FP16: Res = PromoteIntRes_FP_TO_FP16(N); break;
123 
124  case ISD::FLT_ROUNDS_: Res = PromoteIntRes_FLT_ROUNDS(N); break;
125 
126  case ISD::AND:
127  case ISD::OR:
128  case ISD::XOR:
129  case ISD::ADD:
130  case ISD::SUB:
131  case ISD::MUL: Res = PromoteIntRes_SimpleIntBinOp(N); break;
132 
133  case ISD::SDIV:
134  case ISD::SREM: Res = PromoteIntRes_SExtIntBinOp(N); break;
135 
136  case ISD::UDIV:
137  case ISD::UREM: Res = PromoteIntRes_ZExtIntBinOp(N); break;
138 
139  case ISD::SADDO:
140  case ISD::SSUBO: Res = PromoteIntRes_SADDSUBO(N, ResNo); break;
141  case ISD::UADDO:
142  case ISD::USUBO: Res = PromoteIntRes_UADDSUBO(N, ResNo); break;
143  case ISD::SMULO:
144  case ISD::UMULO: Res = PromoteIntRes_XMULO(N, ResNo); break;
145 
146  case ISD::ADDE:
147  case ISD::SUBE:
148  case ISD::ADDCARRY:
149  case ISD::SUBCARRY: Res = PromoteIntRes_ADDSUBCARRY(N, ResNo); break;
150 
151  case ISD::SADDSAT:
152  case ISD::UADDSAT:
153  case ISD::SSUBSAT:
154  case ISD::USUBSAT: Res = PromoteIntRes_ADDSUBSAT(N); break;
155 
156  case ISD::SMULFIX:
157  case ISD::SMULFIXSAT:
158  case ISD::UMULFIX:
159  case ISD::UMULFIXSAT: Res = PromoteIntRes_MULFIX(N); break;
160 
161  case ISD::ABS: Res = PromoteIntRes_ABS(N); break;
162 
163  case ISD::ATOMIC_LOAD:
164  Res = PromoteIntRes_Atomic0(cast<AtomicSDNode>(N)); break;
165 
170  case ISD::ATOMIC_LOAD_OR:
177  case ISD::ATOMIC_SWAP:
178  Res = PromoteIntRes_Atomic1(cast<AtomicSDNode>(N)); break;
179 
182  Res = PromoteIntRes_AtomicCmpSwap(cast<AtomicSDNode>(N), ResNo);
183  break;
184 
185  case ISD::VECREDUCE_ADD:
186  case ISD::VECREDUCE_MUL:
187  case ISD::VECREDUCE_AND:
188  case ISD::VECREDUCE_OR:
189  case ISD::VECREDUCE_XOR:
190  case ISD::VECREDUCE_SMAX:
191  case ISD::VECREDUCE_SMIN:
192  case ISD::VECREDUCE_UMAX:
193  case ISD::VECREDUCE_UMIN:
194  Res = PromoteIntRes_VECREDUCE(N);
195  break;
196  }
197 
198  // If the result is null then the sub-method took care of registering it.
199  if (Res.getNode())
200  SetPromotedInteger(SDValue(N, ResNo), Res);
201 }
202 
203 SDValue DAGTypeLegalizer::PromoteIntRes_MERGE_VALUES(SDNode *N,
204  unsigned ResNo) {
205  SDValue Op = DisintegrateMERGE_VALUES(N, ResNo);
206  return GetPromotedInteger(Op);
207 }
208 
209 SDValue DAGTypeLegalizer::PromoteIntRes_AssertSext(SDNode *N) {
210  // Sign-extend the new bits, and continue the assertion.
211  SDValue Op = SExtPromotedInteger(N->getOperand(0));
212  return DAG.getNode(ISD::AssertSext, SDLoc(N),
213  Op.getValueType(), Op, N->getOperand(1));
214 }
215 
216 SDValue DAGTypeLegalizer::PromoteIntRes_AssertZext(SDNode *N) {
217  // Zero the new bits, and continue the assertion.
218  SDValue Op = ZExtPromotedInteger(N->getOperand(0));
219  return DAG.getNode(ISD::AssertZext, SDLoc(N),
220  Op.getValueType(), Op, N->getOperand(1));
221 }
222 
223 SDValue DAGTypeLegalizer::PromoteIntRes_Atomic0(AtomicSDNode *N) {
224  EVT ResVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
225  SDValue Res = DAG.getAtomic(N->getOpcode(), SDLoc(N),
226  N->getMemoryVT(), ResVT,
227  N->getChain(), N->getBasePtr(),
228  N->getMemOperand());
229  // Legalize the chain result - switch anything that used the old chain to
230  // use the new one.
231  ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
232  return Res;
233 }
234 
235 SDValue DAGTypeLegalizer::PromoteIntRes_Atomic1(AtomicSDNode *N) {
236  SDValue Op2 = GetPromotedInteger(N->getOperand(2));
237  SDValue Res = DAG.getAtomic(N->getOpcode(), SDLoc(N),
238  N->getMemoryVT(),
239  N->getChain(), N->getBasePtr(),
240  Op2, N->getMemOperand());
241  // Legalize the chain result - switch anything that used the old chain to
242  // use the new one.
243  ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
244  return Res;
245 }
246 
247 SDValue DAGTypeLegalizer::PromoteIntRes_AtomicCmpSwap(AtomicSDNode *N,
248  unsigned ResNo) {
249  if (ResNo == 1) {
251  EVT SVT = getSetCCResultType(N->getOperand(2).getValueType());
252  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(1));
253 
254  // Only use the result of getSetCCResultType if it is legal,
255  // otherwise just use the promoted result type (NVT).
256  if (!TLI.isTypeLegal(SVT))
257  SVT = NVT;
258 
259  SDVTList VTs = DAG.getVTList(N->getValueType(0), SVT, MVT::Other);
260  SDValue Res = DAG.getAtomicCmpSwap(
262  N->getChain(), N->getBasePtr(), N->getOperand(2), N->getOperand(3),
263  N->getMemOperand());
264  ReplaceValueWith(SDValue(N, 0), Res.getValue(0));
265  ReplaceValueWith(SDValue(N, 2), Res.getValue(2));
266  return Res.getValue(1);
267  }
268 
269  SDValue Op2 = GetPromotedInteger(N->getOperand(2));
270  SDValue Op3 = GetPromotedInteger(N->getOperand(3));
271  SDVTList VTs =
272  DAG.getVTList(Op2.getValueType(), N->getValueType(1), MVT::Other);
273  SDValue Res = DAG.getAtomicCmpSwap(
274  N->getOpcode(), SDLoc(N), N->getMemoryVT(), VTs, N->getChain(),
275  N->getBasePtr(), Op2, Op3, N->getMemOperand());
276  // Update the use to N with the newly created Res.
277  for (unsigned i = 1, NumResults = N->getNumValues(); i < NumResults; ++i)
278  ReplaceValueWith(SDValue(N, i), Res.getValue(i));
279  return Res;
280 }
281 
282 SDValue DAGTypeLegalizer::PromoteIntRes_BITCAST(SDNode *N) {
283  SDValue InOp = N->getOperand(0);
284  EVT InVT = InOp.getValueType();
285  EVT NInVT = TLI.getTypeToTransformTo(*DAG.getContext(), InVT);
286  EVT OutVT = N->getValueType(0);
287  EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT);
288  SDLoc dl(N);
289 
290  switch (getTypeAction(InVT)) {
292  break;
294  if (NOutVT.bitsEq(NInVT) && !NOutVT.isVector() && !NInVT.isVector())
295  // The input promotes to the same size. Convert the promoted value.
296  return DAG.getNode(ISD::BITCAST, dl, NOutVT, GetPromotedInteger(InOp));
297  break;
299  // Promote the integer operand by hand.
300  return DAG.getNode(ISD::ANY_EXTEND, dl, NOutVT, GetSoftenedFloat(InOp));
302  // Convert the promoted float by hand.
303  if (!NOutVT.isVector())
304  return DAG.getNode(ISD::FP_TO_FP16, dl, NOutVT, GetPromotedFloat(InOp));
305  break;
306  }
309  break;
311  // Convert the element to an integer and promote it by hand.
312  if (!NOutVT.isVector())
313  return DAG.getNode(ISD::ANY_EXTEND, dl, NOutVT,
314  BitConvertToInteger(GetScalarizedVector(InOp)));
315  break;
317  if (!NOutVT.isVector()) {
318  // For example, i32 = BITCAST v2i16 on alpha. Convert the split
319  // pieces of the input into integers and reassemble in the final type.
320  SDValue Lo, Hi;
321  GetSplitVector(N->getOperand(0), Lo, Hi);
322  Lo = BitConvertToInteger(Lo);
323  Hi = BitConvertToInteger(Hi);
324 
325  if (DAG.getDataLayout().isBigEndian())
326  std::swap(Lo, Hi);
327 
328  InOp = DAG.getNode(ISD::ANY_EXTEND, dl,
330  NOutVT.getSizeInBits()),
331  JoinIntegers(Lo, Hi));
332  return DAG.getNode(ISD::BITCAST, dl, NOutVT, InOp);
333  }
334  break;
335  }
337  // The input is widened to the same size. Convert to the widened value.
338  // Make sure that the outgoing value is not a vector, because this would
339  // make us bitcast between two vectors which are legalized in different ways.
340  if (NOutVT.bitsEq(NInVT) && !NOutVT.isVector())
341  return DAG.getNode(ISD::BITCAST, dl, NOutVT, GetWidenedVector(InOp));
342  // If the output type is also a vector and widening it to the same size
343  // as the widened input type would be a legal type, we can widen the bitcast
344  // and handle the promotion after.
345  if (NOutVT.isVector()) {
346  unsigned WidenInSize = NInVT.getSizeInBits();
347  unsigned OutSize = OutVT.getSizeInBits();
348  if (WidenInSize % OutSize == 0) {
349  unsigned Scale = WidenInSize / OutSize;
350  EVT WideOutVT = EVT::getVectorVT(*DAG.getContext(),
351  OutVT.getVectorElementType(),
352  OutVT.getVectorNumElements() * Scale);
353  if (isTypeLegal(WideOutVT)) {
354  InOp = DAG.getBitcast(WideOutVT, GetWidenedVector(InOp));
355  MVT IdxTy = TLI.getVectorIdxTy(DAG.getDataLayout());
356  InOp = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, OutVT, InOp,
357  DAG.getConstant(0, dl, IdxTy));
358  return DAG.getNode(ISD::ANY_EXTEND, dl, NOutVT, InOp);
359  }
360  }
361  }
362  }
363 
364  return DAG.getNode(ISD::ANY_EXTEND, dl, NOutVT,
365  CreateStackStoreLoad(InOp, OutVT));
366 }
367 
368 // Helper for BSWAP/BITREVERSE promotion to ensure we can fit the shift amount
369 // in the VT returned by getShiftAmountTy and to return a safe VT if we can't.
370 static EVT getShiftAmountTyForConstant(unsigned Val, EVT VT,
371  const TargetLowering &TLI,
372  SelectionDAG &DAG) {
373  EVT ShiftVT = TLI.getShiftAmountTy(VT, DAG.getDataLayout());
374  // If the value won't fit in the prefered type, just use something safe. It
375  // will be legalized when the shift is expanded.
376  if ((Log2_32(Val) + 1) > ShiftVT.getScalarSizeInBits())
377  ShiftVT = MVT::i32;
378  return ShiftVT;
379 }
380 
381 SDValue DAGTypeLegalizer::PromoteIntRes_BSWAP(SDNode *N) {
382  SDValue Op = GetPromotedInteger(N->getOperand(0));
383  EVT OVT = N->getValueType(0);
384  EVT NVT = Op.getValueType();
385  SDLoc dl(N);
386 
387  unsigned DiffBits = NVT.getScalarSizeInBits() - OVT.getScalarSizeInBits();
388  EVT ShiftVT = getShiftAmountTyForConstant(DiffBits, NVT, TLI, DAG);
389  return DAG.getNode(ISD::SRL, dl, NVT, DAG.getNode(ISD::BSWAP, dl, NVT, Op),
390  DAG.getConstant(DiffBits, dl, ShiftVT));
391 }
392 
393 SDValue DAGTypeLegalizer::PromoteIntRes_BITREVERSE(SDNode *N) {
394  SDValue Op = GetPromotedInteger(N->getOperand(0));
395  EVT OVT = N->getValueType(0);
396  EVT NVT = Op.getValueType();
397  SDLoc dl(N);
398 
399  unsigned DiffBits = NVT.getScalarSizeInBits() - OVT.getScalarSizeInBits();
400  EVT ShiftVT = getShiftAmountTyForConstant(DiffBits, NVT, TLI, DAG);
401  return DAG.getNode(ISD::SRL, dl, NVT,
402  DAG.getNode(ISD::BITREVERSE, dl, NVT, Op),
403  DAG.getConstant(DiffBits, dl, ShiftVT));
404 }
405 
406 SDValue DAGTypeLegalizer::PromoteIntRes_BUILD_PAIR(SDNode *N) {
407  // The pair element type may be legal, or may not promote to the same type as
408  // the result, for example i14 = BUILD_PAIR (i7, i7). Handle all cases.
409  return DAG.getNode(ISD::ANY_EXTEND, SDLoc(N),
410  TLI.getTypeToTransformTo(*DAG.getContext(),
411  N->getValueType(0)), JoinIntegers(N->getOperand(0),
412  N->getOperand(1)));
413 }
414 
415 SDValue DAGTypeLegalizer::PromoteIntRes_Constant(SDNode *N) {
416  EVT VT = N->getValueType(0);
417  // FIXME there is no actual debug info here
418  SDLoc dl(N);
419  // Zero extend things like i1, sign extend everything else. It shouldn't
420  // matter in theory which one we pick, but this tends to give better code?
421  unsigned Opc = VT.isByteSized() ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND;
422  SDValue Result = DAG.getNode(Opc, dl,
423  TLI.getTypeToTransformTo(*DAG.getContext(), VT),
424  SDValue(N, 0));
425  assert(isa<ConstantSDNode>(Result) && "Didn't constant fold ext?");
426  return Result;
427 }
428 
429 SDValue DAGTypeLegalizer::PromoteIntRes_CTLZ(SDNode *N) {
430  // Zero extend to the promoted type and do the count there.
431  SDValue Op = ZExtPromotedInteger(N->getOperand(0));
432  SDLoc dl(N);
433  EVT OVT = N->getValueType(0);
434  EVT NVT = Op.getValueType();
435  Op = DAG.getNode(N->getOpcode(), dl, NVT, Op);
436  // Subtract off the extra leading bits in the bigger type.
437  return DAG.getNode(
438  ISD::SUB, dl, NVT, Op,
439  DAG.getConstant(NVT.getScalarSizeInBits() - OVT.getScalarSizeInBits(), dl,
440  NVT));
441 }
442 
443 SDValue DAGTypeLegalizer::PromoteIntRes_CTPOP(SDNode *N) {
444  // Zero extend to the promoted type and do the count there.
445  SDValue Op = ZExtPromotedInteger(N->getOperand(0));
446  return DAG.getNode(ISD::CTPOP, SDLoc(N), Op.getValueType(), Op);
447 }
448 
449 SDValue DAGTypeLegalizer::PromoteIntRes_CTTZ(SDNode *N) {
450  SDValue Op = GetPromotedInteger(N->getOperand(0));
451  EVT OVT = N->getValueType(0);
452  EVT NVT = Op.getValueType();
453  SDLoc dl(N);
454  if (N->getOpcode() == ISD::CTTZ) {
455  // The count is the same in the promoted type except if the original
456  // value was zero. This can be handled by setting the bit just off
457  // the top of the original type.
458  auto TopBit = APInt::getOneBitSet(NVT.getScalarSizeInBits(),
459  OVT.getScalarSizeInBits());
460  Op = DAG.getNode(ISD::OR, dl, NVT, Op, DAG.getConstant(TopBit, dl, NVT));
461  }
462  return DAG.getNode(N->getOpcode(), dl, NVT, Op);
463 }
464 
465 SDValue DAGTypeLegalizer::PromoteIntRes_EXTRACT_VECTOR_ELT(SDNode *N) {
466  SDLoc dl(N);
467  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
468 
469  SDValue Op0 = N->getOperand(0);
470  SDValue Op1 = N->getOperand(1);
471 
472  // If the input also needs to be promoted, do that first so we can get a
473  // get a good idea for the output type.
474  if (TLI.getTypeAction(*DAG.getContext(), Op0.getValueType())
476  SDValue In = GetPromotedInteger(Op0);
477 
478  // If the new type is larger than NVT, use it. We probably won't need to
479  // promote it again.
480  EVT SVT = In.getValueType().getScalarType();
481  if (SVT.bitsGE(NVT)) {
482  SDValue Ext = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, SVT, In, Op1);
483  return DAG.getAnyExtOrTrunc(Ext, dl, NVT);
484  }
485  }
486 
487  return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, NVT, Op0, Op1);
488 }
489 
490 SDValue DAGTypeLegalizer::PromoteIntRes_FP_TO_XINT(SDNode *N) {
491  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
492  unsigned NewOpc = N->getOpcode();
493  SDLoc dl(N);
494 
495  // If we're promoting a UINT to a larger size and the larger FP_TO_UINT is
496  // not Legal, check to see if we can use FP_TO_SINT instead. (If both UINT
497  // and SINT conversions are Custom, there is no way to tell which is
498  // preferable. We choose SINT because that's the right thing on PPC.)
499  if (N->getOpcode() == ISD::FP_TO_UINT &&
500  !TLI.isOperationLegal(ISD::FP_TO_UINT, NVT) &&
502  NewOpc = ISD::FP_TO_SINT;
503 
504  if (N->getOpcode() == ISD::STRICT_FP_TO_UINT &&
507  NewOpc = ISD::STRICT_FP_TO_SINT;
508 
509  SDValue Res;
510  if (N->isStrictFPOpcode()) {
511  Res = DAG.getNode(NewOpc, dl, { NVT, MVT::Other },
512  { N->getOperand(0), N->getOperand(1) });
513  // Legalize the chain result - switch anything that used the old chain to
514  // use the new one.
515  ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
516  } else
517  Res = DAG.getNode(NewOpc, dl, NVT, N->getOperand(0));
518 
519  // Assert that the converted value fits in the original type. If it doesn't
520  // (eg: because the value being converted is too big), then the result of the
521  // original operation was undefined anyway, so the assert is still correct.
522  //
523  // NOTE: fp-to-uint to fp-to-sint promotion guarantees zero extend. For example:
524  // before legalization: fp-to-uint16, 65534. -> 0xfffe
525  // after legalization: fp-to-sint32, 65534. -> 0x0000fffe
526  return DAG.getNode((N->getOpcode() == ISD::FP_TO_UINT ||
528  ISD::AssertZext : ISD::AssertSext, dl, NVT, Res,
530 }
531 
532 SDValue DAGTypeLegalizer::PromoteIntRes_FP_TO_FP16(SDNode *N) {
533  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
534  SDLoc dl(N);
535 
536  return DAG.getNode(N->getOpcode(), dl, NVT, N->getOperand(0));
537 }
538 
539 SDValue DAGTypeLegalizer::PromoteIntRes_FLT_ROUNDS(SDNode *N) {
540  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
541  SDLoc dl(N);
542 
543  return DAG.getNode(N->getOpcode(), dl, NVT);
544 }
545 
546 SDValue DAGTypeLegalizer::PromoteIntRes_INT_EXTEND(SDNode *N) {
547  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
548  SDLoc dl(N);
549 
550  if (getTypeAction(N->getOperand(0).getValueType())
552  SDValue Res = GetPromotedInteger(N->getOperand(0));
553  assert(Res.getValueType().bitsLE(NVT) && "Extension doesn't make sense!");
554 
555  // If the result and operand types are the same after promotion, simplify
556  // to an in-register extension.
557  if (NVT == Res.getValueType()) {
558  // The high bits are not guaranteed to be anything. Insert an extend.
559  if (N->getOpcode() == ISD::SIGN_EXTEND)
560  return DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, NVT, Res,
561  DAG.getValueType(N->getOperand(0).getValueType()));
562  if (N->getOpcode() == ISD::ZERO_EXTEND)
563  return DAG.getZeroExtendInReg(Res, dl,
565  assert(N->getOpcode() == ISD::ANY_EXTEND && "Unknown integer extension!");
566  return Res;
567  }
568  }
569 
570  // Otherwise, just extend the original operand all the way to the larger type.
571  return DAG.getNode(N->getOpcode(), dl, NVT, N->getOperand(0));
572 }
573 
574 SDValue DAGTypeLegalizer::PromoteIntRes_LOAD(LoadSDNode *N) {
575  assert(ISD::isUNINDEXEDLoad(N) && "Indexed load during type legalization!");
576  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
579  SDLoc dl(N);
580  SDValue Res = DAG.getExtLoad(ExtType, dl, NVT, N->getChain(), N->getBasePtr(),
581  N->getMemoryVT(), N->getMemOperand());
582 
583  // Legalize the chain result - switch anything that used the old chain to
584  // use the new one.
585  ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
586  return Res;
587 }
588 
589 SDValue DAGTypeLegalizer::PromoteIntRes_MLOAD(MaskedLoadSDNode *N) {
590  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
591  SDValue ExtPassThru = GetPromotedInteger(N->getPassThru());
592 
593  SDLoc dl(N);
594  SDValue Res = DAG.getMaskedLoad(NVT, dl, N->getChain(), N->getBasePtr(),
595  N->getMask(), ExtPassThru, N->getMemoryVT(),
597  // Legalize the chain result - switch anything that used the old chain to
598  // use the new one.
599  ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
600  return Res;
601 }
602 
603 SDValue DAGTypeLegalizer::PromoteIntRes_MGATHER(MaskedGatherSDNode *N) {
604  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
605  SDValue ExtPassThru = GetPromotedInteger(N->getPassThru());
606  assert(NVT == ExtPassThru.getValueType() &&
607  "Gather result type and the passThru agrument type should be the same");
608 
609  SDLoc dl(N);
610  SDValue Ops[] = {N->getChain(), ExtPassThru, N->getMask(), N->getBasePtr(),
611  N->getIndex(), N->getScale() };
612  SDValue Res = DAG.getMaskedGather(DAG.getVTList(NVT, MVT::Other),
613  N->getMemoryVT(), dl, Ops,
614  N->getMemOperand(), N->getIndexType());
615  // Legalize the chain result - switch anything that used the old chain to
616  // use the new one.
617  ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
618  return Res;
619 }
620 
621 /// Promote the overflow flag of an overflowing arithmetic node.
622 SDValue DAGTypeLegalizer::PromoteIntRes_Overflow(SDNode *N) {
623  // Change the return type of the boolean result while obeying
624  // getSetCCResultType.
625  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(1));
626  EVT VT = N->getValueType(0);
627  EVT SVT = getSetCCResultType(VT);
628  SDValue Ops[3] = { N->getOperand(0), N->getOperand(1) };
629  unsigned NumOps = N->getNumOperands();
630  assert(NumOps <= 3 && "Too many operands");
631  if (NumOps == 3)
632  Ops[2] = N->getOperand(2);
633 
634  SDLoc dl(N);
635  SDValue Res = DAG.getNode(N->getOpcode(), dl, DAG.getVTList(VT, SVT),
636  makeArrayRef(Ops, NumOps));
637 
638  // Modified the sum result - switch anything that used the old sum to use
639  // the new one.
640  ReplaceValueWith(SDValue(N, 0), Res);
641 
642  // Convert to the expected type.
643  return DAG.getBoolExtOrTrunc(Res.getValue(1), dl, NVT, VT);
644 }
645 
646 SDValue DAGTypeLegalizer::PromoteIntRes_ADDSUBSAT(SDNode *N) {
647  // If the promoted type is legal, we can convert this to:
648  // 1. ANY_EXTEND iN to iM
649  // 2. SHL by M-N
650  // 3. [US][ADD|SUB]SAT
651  // 4. L/ASHR by M-N
652  // Else it is more efficient to convert this to a min and a max
653  // operation in the higher precision arithmetic.
654  SDLoc dl(N);
655  SDValue Op1 = N->getOperand(0);
656  SDValue Op2 = N->getOperand(1);
657  unsigned OldBits = Op1.getScalarValueSizeInBits();
658 
659  unsigned Opcode = N->getOpcode();
660 
661  SDValue Op1Promoted, Op2Promoted;
662  if (Opcode == ISD::UADDSAT || Opcode == ISD::USUBSAT) {
663  Op1Promoted = ZExtPromotedInteger(Op1);
664  Op2Promoted = ZExtPromotedInteger(Op2);
665  } else {
666  Op1Promoted = SExtPromotedInteger(Op1);
667  Op2Promoted = SExtPromotedInteger(Op2);
668  }
669  EVT PromotedType = Op1Promoted.getValueType();
670  unsigned NewBits = PromotedType.getScalarSizeInBits();
671 
672  if (TLI.isOperationLegalOrCustom(Opcode, PromotedType)) {
673  unsigned ShiftOp;
674  switch (Opcode) {
675  case ISD::SADDSAT:
676  case ISD::SSUBSAT:
677  ShiftOp = ISD::SRA;
678  break;
679  case ISD::UADDSAT:
680  case ISD::USUBSAT:
681  ShiftOp = ISD::SRL;
682  break;
683  default:
684  llvm_unreachable("Expected opcode to be signed or unsigned saturation "
685  "addition or subtraction");
686  }
687 
688  unsigned SHLAmount = NewBits - OldBits;
689  EVT SHVT = TLI.getShiftAmountTy(PromotedType, DAG.getDataLayout());
690  SDValue ShiftAmount = DAG.getConstant(SHLAmount, dl, SHVT);
691  Op1Promoted =
692  DAG.getNode(ISD::SHL, dl, PromotedType, Op1Promoted, ShiftAmount);
693  Op2Promoted =
694  DAG.getNode(ISD::SHL, dl, PromotedType, Op2Promoted, ShiftAmount);
695 
696  SDValue Result =
697  DAG.getNode(Opcode, dl, PromotedType, Op1Promoted, Op2Promoted);
698  return DAG.getNode(ShiftOp, dl, PromotedType, Result, ShiftAmount);
699  } else {
700  if (Opcode == ISD::USUBSAT) {
701  SDValue Max =
702  DAG.getNode(ISD::UMAX, dl, PromotedType, Op1Promoted, Op2Promoted);
703  return DAG.getNode(ISD::SUB, dl, PromotedType, Max, Op2Promoted);
704  }
705 
706  if (Opcode == ISD::UADDSAT) {
707  APInt MaxVal = APInt::getAllOnesValue(OldBits).zext(NewBits);
708  SDValue SatMax = DAG.getConstant(MaxVal, dl, PromotedType);
709  SDValue Add =
710  DAG.getNode(ISD::ADD, dl, PromotedType, Op1Promoted, Op2Promoted);
711  return DAG.getNode(ISD::UMIN, dl, PromotedType, Add, SatMax);
712  }
713 
714  unsigned AddOp = Opcode == ISD::SADDSAT ? ISD::ADD : ISD::SUB;
715  APInt MinVal = APInt::getSignedMinValue(OldBits).sext(NewBits);
716  APInt MaxVal = APInt::getSignedMaxValue(OldBits).sext(NewBits);
717  SDValue SatMin = DAG.getConstant(MinVal, dl, PromotedType);
718  SDValue SatMax = DAG.getConstant(MaxVal, dl, PromotedType);
719  SDValue Result =
720  DAG.getNode(AddOp, dl, PromotedType, Op1Promoted, Op2Promoted);
721  Result = DAG.getNode(ISD::SMIN, dl, PromotedType, Result, SatMax);
722  Result = DAG.getNode(ISD::SMAX, dl, PromotedType, Result, SatMin);
723  return Result;
724  }
725 }
726 
727 SDValue DAGTypeLegalizer::PromoteIntRes_MULFIX(SDNode *N) {
728  // Can just promote the operands then continue with operation.
729  SDLoc dl(N);
730  SDValue Op1Promoted, Op2Promoted;
731  bool Signed =
732  N->getOpcode() == ISD::SMULFIX || N->getOpcode() == ISD::SMULFIXSAT;
733  bool Saturating =
735  if (Signed) {
736  Op1Promoted = SExtPromotedInteger(N->getOperand(0));
737  Op2Promoted = SExtPromotedInteger(N->getOperand(1));
738  } else {
739  Op1Promoted = ZExtPromotedInteger(N->getOperand(0));
740  Op2Promoted = ZExtPromotedInteger(N->getOperand(1));
741  }
742  EVT OldType = N->getOperand(0).getValueType();
743  EVT PromotedType = Op1Promoted.getValueType();
744  unsigned DiffSize =
745  PromotedType.getScalarSizeInBits() - OldType.getScalarSizeInBits();
746 
747  if (Saturating) {
748  // Promoting the operand and result values changes the saturation width,
749  // which is extends the values that we clamp to on saturation. This could be
750  // resolved by shifting one of the operands the same amount, which would
751  // also shift the result we compare against, then shifting back.
752  EVT ShiftTy = TLI.getShiftAmountTy(PromotedType, DAG.getDataLayout());
753  Op1Promoted = DAG.getNode(ISD::SHL, dl, PromotedType, Op1Promoted,
754  DAG.getConstant(DiffSize, dl, ShiftTy));
755  SDValue Result = DAG.getNode(N->getOpcode(), dl, PromotedType, Op1Promoted,
756  Op2Promoted, N->getOperand(2));
757  unsigned ShiftOp = Signed ? ISD::SRA : ISD::SRL;
758  return DAG.getNode(ShiftOp, dl, PromotedType, Result,
759  DAG.getConstant(DiffSize, dl, ShiftTy));
760  }
761  return DAG.getNode(N->getOpcode(), dl, PromotedType, Op1Promoted, Op2Promoted,
762  N->getOperand(2));
763 }
764 
765 SDValue DAGTypeLegalizer::PromoteIntRes_SADDSUBO(SDNode *N, unsigned ResNo) {
766  if (ResNo == 1)
767  return PromoteIntRes_Overflow(N);
768 
769  // The operation overflowed iff the result in the larger type is not the
770  // sign extension of its truncation to the original type.
771  SDValue LHS = SExtPromotedInteger(N->getOperand(0));
772  SDValue RHS = SExtPromotedInteger(N->getOperand(1));
773  EVT OVT = N->getOperand(0).getValueType();
774  EVT NVT = LHS.getValueType();
775  SDLoc dl(N);
776 
777  // Do the arithmetic in the larger type.
778  unsigned Opcode = N->getOpcode() == ISD::SADDO ? ISD::ADD : ISD::SUB;
779  SDValue Res = DAG.getNode(Opcode, dl, NVT, LHS, RHS);
780 
781  // Calculate the overflow flag: sign extend the arithmetic result from
782  // the original type.
783  SDValue Ofl = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, NVT, Res,
784  DAG.getValueType(OVT));
785  // Overflowed if and only if this is not equal to Res.
786  Ofl = DAG.getSetCC(dl, N->getValueType(1), Ofl, Res, ISD::SETNE);
787 
788  // Use the calculated overflow everywhere.
789  ReplaceValueWith(SDValue(N, 1), Ofl);
790 
791  return Res;
792 }
793 
794 SDValue DAGTypeLegalizer::PromoteIntRes_SELECT(SDNode *N) {
795  SDValue LHS = GetPromotedInteger(N->getOperand(1));
796  SDValue RHS = GetPromotedInteger(N->getOperand(2));
797  return DAG.getSelect(SDLoc(N),
798  LHS.getValueType(), N->getOperand(0), LHS, RHS);
799 }
800 
801 SDValue DAGTypeLegalizer::PromoteIntRes_VSELECT(SDNode *N) {
802  SDValue Mask = N->getOperand(0);
803 
804  SDValue LHS = GetPromotedInteger(N->getOperand(1));
805  SDValue RHS = GetPromotedInteger(N->getOperand(2));
806  return DAG.getNode(ISD::VSELECT, SDLoc(N),
807  LHS.getValueType(), Mask, LHS, RHS);
808 }
809 
810 SDValue DAGTypeLegalizer::PromoteIntRes_SELECT_CC(SDNode *N) {
811  SDValue LHS = GetPromotedInteger(N->getOperand(2));
812  SDValue RHS = GetPromotedInteger(N->getOperand(3));
813  return DAG.getNode(ISD::SELECT_CC, SDLoc(N),
814  LHS.getValueType(), N->getOperand(0),
815  N->getOperand(1), LHS, RHS, N->getOperand(4));
816 }
817 
818 SDValue DAGTypeLegalizer::PromoteIntRes_SETCC(SDNode *N) {
819  EVT InVT = N->getOperand(0).getValueType();
820  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
821 
822  EVT SVT = getSetCCResultType(InVT);
823 
824  // If we got back a type that needs to be promoted, this likely means the
825  // the input type also needs to be promoted. So get the promoted type for
826  // the input and try the query again.
827  if (getTypeAction(SVT) == TargetLowering::TypePromoteInteger) {
828  if (getTypeAction(InVT) == TargetLowering::TypePromoteInteger) {
829  InVT = TLI.getTypeToTransformTo(*DAG.getContext(), InVT);
830  SVT = getSetCCResultType(InVT);
831  } else {
832  // Input type isn't promoted, just use the default promoted type.
833  SVT = NVT;
834  }
835  }
836 
837  SDLoc dl(N);
838  assert(SVT.isVector() == N->getOperand(0).getValueType().isVector() &&
839  "Vector compare must return a vector result!");
840 
841  // Get the SETCC result using the canonical SETCC type.
842  SDValue SetCC = DAG.getNode(N->getOpcode(), dl, SVT, N->getOperand(0),
843  N->getOperand(1), N->getOperand(2));
844 
845  // Convert to the expected type.
846  return DAG.getSExtOrTrunc(SetCC, dl, NVT);
847 }
848 
849 SDValue DAGTypeLegalizer::PromoteIntRes_SHL(SDNode *N) {
850  SDValue LHS = GetPromotedInteger(N->getOperand(0));
851  SDValue RHS = N->getOperand(1);
852  if (getTypeAction(RHS.getValueType()) == TargetLowering::TypePromoteInteger)
853  RHS = ZExtPromotedInteger(RHS);
854  return DAG.getNode(ISD::SHL, SDLoc(N), LHS.getValueType(), LHS, RHS);
855 }
856 
857 SDValue DAGTypeLegalizer::PromoteIntRes_SIGN_EXTEND_INREG(SDNode *N) {
858  SDValue Op = GetPromotedInteger(N->getOperand(0));
859  return DAG.getNode(ISD::SIGN_EXTEND_INREG, SDLoc(N),
860  Op.getValueType(), Op, N->getOperand(1));
861 }
862 
863 SDValue DAGTypeLegalizer::PromoteIntRes_SimpleIntBinOp(SDNode *N) {
864  // The input may have strange things in the top bits of the registers, but
865  // these operations don't care. They may have weird bits going out, but
866  // that too is okay if they are integer operations.
867  SDValue LHS = GetPromotedInteger(N->getOperand(0));
868  SDValue RHS = GetPromotedInteger(N->getOperand(1));
869  return DAG.getNode(N->getOpcode(), SDLoc(N),
870  LHS.getValueType(), LHS, RHS);
871 }
872 
873 SDValue DAGTypeLegalizer::PromoteIntRes_SExtIntBinOp(SDNode *N) {
874  // Sign extend the input.
875  SDValue LHS = SExtPromotedInteger(N->getOperand(0));
876  SDValue RHS = SExtPromotedInteger(N->getOperand(1));
877  return DAG.getNode(N->getOpcode(), SDLoc(N),
878  LHS.getValueType(), LHS, RHS);
879 }
880 
881 SDValue DAGTypeLegalizer::PromoteIntRes_ZExtIntBinOp(SDNode *N) {
882  // Zero extend the input.
883  SDValue LHS = ZExtPromotedInteger(N->getOperand(0));
884  SDValue RHS = ZExtPromotedInteger(N->getOperand(1));
885  return DAG.getNode(N->getOpcode(), SDLoc(N),
886  LHS.getValueType(), LHS, RHS);
887 }
888 
889 SDValue DAGTypeLegalizer::PromoteIntRes_SRA(SDNode *N) {
890  // The input value must be properly sign extended.
891  SDValue LHS = SExtPromotedInteger(N->getOperand(0));
892  SDValue RHS = N->getOperand(1);
893  if (getTypeAction(RHS.getValueType()) == TargetLowering::TypePromoteInteger)
894  RHS = ZExtPromotedInteger(RHS);
895  return DAG.getNode(ISD::SRA, SDLoc(N), LHS.getValueType(), LHS, RHS);
896 }
897 
898 SDValue DAGTypeLegalizer::PromoteIntRes_SRL(SDNode *N) {
899  // The input value must be properly zero extended.
900  SDValue LHS = ZExtPromotedInteger(N->getOperand(0));
901  SDValue RHS = N->getOperand(1);
902  if (getTypeAction(RHS.getValueType()) == TargetLowering::TypePromoteInteger)
903  RHS = ZExtPromotedInteger(RHS);
904  return DAG.getNode(ISD::SRL, SDLoc(N), LHS.getValueType(), LHS, RHS);
905 }
906 
907 SDValue DAGTypeLegalizer::PromoteIntRes_TRUNCATE(SDNode *N) {
908  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
909  SDValue Res;
910  SDValue InOp = N->getOperand(0);
911  SDLoc dl(N);
912 
913  switch (getTypeAction(InOp.getValueType())) {
914  default: llvm_unreachable("Unknown type action!");
917  Res = InOp;
918  break;
920  Res = GetPromotedInteger(InOp);
921  break;
923  EVT InVT = InOp.getValueType();
924  assert(InVT.isVector() && "Cannot split scalar types");
925  unsigned NumElts = InVT.getVectorNumElements();
926  assert(NumElts == NVT.getVectorNumElements() &&
927  "Dst and Src must have the same number of elements");
928  assert(isPowerOf2_32(NumElts) &&
929  "Promoted vector type must be a power of two");
930 
931  SDValue EOp1, EOp2;
932  GetSplitVector(InOp, EOp1, EOp2);
933 
934  EVT HalfNVT = EVT::getVectorVT(*DAG.getContext(), NVT.getScalarType(),
935  NumElts/2);
936  EOp1 = DAG.getNode(ISD::TRUNCATE, dl, HalfNVT, EOp1);
937  EOp2 = DAG.getNode(ISD::TRUNCATE, dl, HalfNVT, EOp2);
938 
939  return DAG.getNode(ISD::CONCAT_VECTORS, dl, NVT, EOp1, EOp2);
940  }
942  SDValue WideInOp = GetWidenedVector(InOp);
943 
944  // Truncate widened InOp.
945  unsigned NumElem = WideInOp.getValueType().getVectorNumElements();
946  EVT TruncVT = EVT::getVectorVT(*DAG.getContext(),
947  N->getValueType(0).getScalarType(), NumElem);
948  SDValue WideTrunc = DAG.getNode(ISD::TRUNCATE, dl, TruncVT, WideInOp);
949 
950  // Zero extend so that the elements are of same type as those of NVT
951  EVT ExtVT = EVT::getVectorVT(*DAG.getContext(), NVT.getVectorElementType(),
952  NumElem);
953  SDValue WideExt = DAG.getNode(ISD::ZERO_EXTEND, dl, ExtVT, WideTrunc);
954 
955  // Extract the low NVT subvector.
956  MVT IdxTy = TLI.getVectorIdxTy(DAG.getDataLayout());
957  SDValue ZeroIdx = DAG.getConstant(0, dl, IdxTy);
958  return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, NVT, WideExt, ZeroIdx);
959  }
960  }
961 
962  // Truncate to NVT instead of VT
963  return DAG.getNode(ISD::TRUNCATE, dl, NVT, Res);
964 }
965 
966 SDValue DAGTypeLegalizer::PromoteIntRes_UADDSUBO(SDNode *N, unsigned ResNo) {
967  if (ResNo == 1)
968  return PromoteIntRes_Overflow(N);
969 
970  // The operation overflowed iff the result in the larger type is not the
971  // zero extension of its truncation to the original type.
972  SDValue LHS = ZExtPromotedInteger(N->getOperand(0));
973  SDValue RHS = ZExtPromotedInteger(N->getOperand(1));
974  EVT OVT = N->getOperand(0).getValueType();
975  EVT NVT = LHS.getValueType();
976  SDLoc dl(N);
977 
978  // Do the arithmetic in the larger type.
979  unsigned Opcode = N->getOpcode() == ISD::UADDO ? ISD::ADD : ISD::SUB;
980  SDValue Res = DAG.getNode(Opcode, dl, NVT, LHS, RHS);
981 
982  // Calculate the overflow flag: zero extend the arithmetic result from
983  // the original type.
984  SDValue Ofl = DAG.getZeroExtendInReg(Res, dl, OVT.getScalarType());
985  // Overflowed if and only if this is not equal to Res.
986  Ofl = DAG.getSetCC(dl, N->getValueType(1), Ofl, Res, ISD::SETNE);
987 
988  // Use the calculated overflow everywhere.
989  ReplaceValueWith(SDValue(N, 1), Ofl);
990 
991  return Res;
992 }
993 
994 // Handle promotion for the ADDE/SUBE/ADDCARRY/SUBCARRY nodes. Notice that
995 // the third operand of ADDE/SUBE nodes is carry flag, which differs from
996 // the ADDCARRY/SUBCARRY nodes in that the third operand is carry Boolean.
997 SDValue DAGTypeLegalizer::PromoteIntRes_ADDSUBCARRY(SDNode *N, unsigned ResNo) {
998  if (ResNo == 1)
999  return PromoteIntRes_Overflow(N);
1000 
1001  // We need to sign-extend the operands so the carry value computed by the
1002  // wide operation will be equivalent to the carry value computed by the
1003  // narrow operation.
1004  // An ADDCARRY can generate carry only if any of the operands has its
1005  // most significant bit set. Sign extension propagates the most significant
1006  // bit into the higher bits which means the extra bit that the narrow
1007  // addition would need (i.e. the carry) will be propagated through the higher
1008  // bits of the wide addition.
1009  // A SUBCARRY can generate borrow only if LHS < RHS and this property will be
1010  // preserved by sign extension.
1011  SDValue LHS = SExtPromotedInteger(N->getOperand(0));
1012  SDValue RHS = SExtPromotedInteger(N->getOperand(1));
1013 
1014  EVT ValueVTs[] = {LHS.getValueType(), N->getValueType(1)};
1015 
1016  // Do the arithmetic in the wide type.
1017  SDValue Res = DAG.getNode(N->getOpcode(), SDLoc(N), DAG.getVTList(ValueVTs),
1018  LHS, RHS, N->getOperand(2));
1019 
1020  // Update the users of the original carry/borrow value.
1021  ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
1022 
1023  return SDValue(Res.getNode(), 0);
1024 }
1025 
1026 SDValue DAGTypeLegalizer::PromoteIntRes_ABS(SDNode *N) {
1027  SDValue Op0 = SExtPromotedInteger(N->getOperand(0));
1028  return DAG.getNode(ISD::ABS, SDLoc(N), Op0.getValueType(), Op0);
1029 }
1030 
1031 SDValue DAGTypeLegalizer::PromoteIntRes_XMULO(SDNode *N, unsigned ResNo) {
1032  // Promote the overflow bit trivially.
1033  if (ResNo == 1)
1034  return PromoteIntRes_Overflow(N);
1035 
1036  SDValue LHS = N->getOperand(0), RHS = N->getOperand(1);
1037  SDLoc DL(N);
1038  EVT SmallVT = LHS.getValueType();
1039 
1040  // To determine if the result overflowed in a larger type, we extend the
1041  // input to the larger type, do the multiply (checking if it overflows),
1042  // then also check the high bits of the result to see if overflow happened
1043  // there.
1044  if (N->getOpcode() == ISD::SMULO) {
1045  LHS = SExtPromotedInteger(LHS);
1046  RHS = SExtPromotedInteger(RHS);
1047  } else {
1048  LHS = ZExtPromotedInteger(LHS);
1049  RHS = ZExtPromotedInteger(RHS);
1050  }
1051  SDVTList VTs = DAG.getVTList(LHS.getValueType(), N->getValueType(1));
1052  SDValue Mul = DAG.getNode(N->getOpcode(), DL, VTs, LHS, RHS);
1053 
1054  // Overflow occurred if it occurred in the larger type, or if the high part
1055  // of the result does not zero/sign-extend the low part. Check this second
1056  // possibility first.
1057  SDValue Overflow;
1058  if (N->getOpcode() == ISD::UMULO) {
1059  // Unsigned overflow occurred if the high part is non-zero.
1060  unsigned Shift = SmallVT.getScalarSizeInBits();
1061  EVT ShiftTy = getShiftAmountTyForConstant(Shift, Mul.getValueType(),
1062  TLI, DAG);
1063  SDValue Hi = DAG.getNode(ISD::SRL, DL, Mul.getValueType(), Mul,
1064  DAG.getConstant(Shift, DL, ShiftTy));
1065  Overflow = DAG.getSetCC(DL, N->getValueType(1), Hi,
1066  DAG.getConstant(0, DL, Hi.getValueType()),
1067  ISD::SETNE);
1068  } else {
1069  // Signed overflow occurred if the high part does not sign extend the low.
1070  SDValue SExt = DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, Mul.getValueType(),
1071  Mul, DAG.getValueType(SmallVT));
1072  Overflow = DAG.getSetCC(DL, N->getValueType(1), SExt, Mul, ISD::SETNE);
1073  }
1074 
1075  // The only other way for overflow to occur is if the multiplication in the
1076  // larger type itself overflowed.
1077  Overflow = DAG.getNode(ISD::OR, DL, N->getValueType(1), Overflow,
1078  SDValue(Mul.getNode(), 1));
1079 
1080  // Use the calculated overflow everywhere.
1081  ReplaceValueWith(SDValue(N, 1), Overflow);
1082  return Mul;
1083 }
1084 
1085 SDValue DAGTypeLegalizer::PromoteIntRes_UNDEF(SDNode *N) {
1086  return DAG.getUNDEF(TLI.getTypeToTransformTo(*DAG.getContext(),
1087  N->getValueType(0)));
1088 }
1089 
1090 SDValue DAGTypeLegalizer::PromoteIntRes_VAARG(SDNode *N) {
1091  SDValue Chain = N->getOperand(0); // Get the chain.
1092  SDValue Ptr = N->getOperand(1); // Get the pointer.
1093  EVT VT = N->getValueType(0);
1094  SDLoc dl(N);
1095 
1096  MVT RegVT = TLI.getRegisterType(*DAG.getContext(), VT);
1097  unsigned NumRegs = TLI.getNumRegisters(*DAG.getContext(), VT);
1098  // The argument is passed as NumRegs registers of type RegVT.
1099 
1100  SmallVector<SDValue, 8> Parts(NumRegs);
1101  for (unsigned i = 0; i < NumRegs; ++i) {
1102  Parts[i] = DAG.getVAArg(RegVT, dl, Chain, Ptr, N->getOperand(2),
1103  N->getConstantOperandVal(3));
1104  Chain = Parts[i].getValue(1);
1105  }
1106 
1107  // Handle endianness of the load.
1108  if (DAG.getDataLayout().isBigEndian())
1109  std::reverse(Parts.begin(), Parts.end());
1110 
1111  // Assemble the parts in the promoted type.
1112  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
1113  SDValue Res = DAG.getNode(ISD::ZERO_EXTEND, dl, NVT, Parts[0]);
1114  for (unsigned i = 1; i < NumRegs; ++i) {
1115  SDValue Part = DAG.getNode(ISD::ZERO_EXTEND, dl, NVT, Parts[i]);
1116  // Shift it to the right position and "or" it in.
1117  Part = DAG.getNode(ISD::SHL, dl, NVT, Part,
1118  DAG.getConstant(i * RegVT.getSizeInBits(), dl,
1119  TLI.getPointerTy(DAG.getDataLayout())));
1120  Res = DAG.getNode(ISD::OR, dl, NVT, Res, Part);
1121  }
1122 
1123  // Modified the chain result - switch anything that used the old chain to
1124  // use the new one.
1125  ReplaceValueWith(SDValue(N, 1), Chain);
1126 
1127  return Res;
1128 }
1129 
1130 //===----------------------------------------------------------------------===//
1131 // Integer Operand Promotion
1132 //===----------------------------------------------------------------------===//
1133 
1134 /// PromoteIntegerOperand - This method is called when the specified operand of
1135 /// the specified node is found to need promotion. At this point, all of the
1136 /// result types of the node are known to be legal, but other operands of the
1137 /// node may need promotion or expansion as well as the specified one.
1138 bool DAGTypeLegalizer::PromoteIntegerOperand(SDNode *N, unsigned OpNo) {
1139  LLVM_DEBUG(dbgs() << "Promote integer operand: "; N->dump(&DAG);
1140  dbgs() << "\n");
1141  SDValue Res = SDValue();
1142 
1143  if (CustomLowerNode(N, N->getOperand(OpNo).getValueType(), false)) {
1144  LLVM_DEBUG(dbgs() << "Node has been custom lowered, done\n");
1145  return false;
1146  }
1147 
1148  switch (N->getOpcode()) {
1149  default:
1150  #ifndef NDEBUG
1151  dbgs() << "PromoteIntegerOperand Op #" << OpNo << ": ";
1152  N->dump(&DAG); dbgs() << "\n";
1153  #endif
1154  llvm_unreachable("Do not know how to promote this operator's operand!");
1155 
1156  case ISD::ANY_EXTEND: Res = PromoteIntOp_ANY_EXTEND(N); break;
1157  case ISD::ATOMIC_STORE:
1158  Res = PromoteIntOp_ATOMIC_STORE(cast<AtomicSDNode>(N));
1159  break;
1160  case ISD::BITCAST: Res = PromoteIntOp_BITCAST(N); break;
1161  case ISD::BR_CC: Res = PromoteIntOp_BR_CC(N, OpNo); break;
1162  case ISD::BRCOND: Res = PromoteIntOp_BRCOND(N, OpNo); break;
1163  case ISD::BUILD_PAIR: Res = PromoteIntOp_BUILD_PAIR(N); break;
1164  case ISD::BUILD_VECTOR: Res = PromoteIntOp_BUILD_VECTOR(N); break;
1165  case ISD::CONCAT_VECTORS: Res = PromoteIntOp_CONCAT_VECTORS(N); break;
1166  case ISD::EXTRACT_VECTOR_ELT: Res = PromoteIntOp_EXTRACT_VECTOR_ELT(N); break;
1168  Res = PromoteIntOp_INSERT_VECTOR_ELT(N, OpNo);break;
1169  case ISD::SCALAR_TO_VECTOR:
1170  Res = PromoteIntOp_SCALAR_TO_VECTOR(N); break;
1171  case ISD::SPLAT_VECTOR:
1172  Res = PromoteIntOp_SPLAT_VECTOR(N); break;
1173  case ISD::VSELECT:
1174  case ISD::SELECT: Res = PromoteIntOp_SELECT(N, OpNo); break;
1175  case ISD::SELECT_CC: Res = PromoteIntOp_SELECT_CC(N, OpNo); break;
1176  case ISD::SETCC: Res = PromoteIntOp_SETCC(N, OpNo); break;
1177  case ISD::SIGN_EXTEND: Res = PromoteIntOp_SIGN_EXTEND(N); break;
1178  case ISD::SINT_TO_FP: Res = PromoteIntOp_SINT_TO_FP(N); break;
1179  case ISD::STORE: Res = PromoteIntOp_STORE(cast<StoreSDNode>(N),
1180  OpNo); break;
1181  case ISD::MSTORE: Res = PromoteIntOp_MSTORE(cast<MaskedStoreSDNode>(N),
1182  OpNo); break;
1183  case ISD::MLOAD: Res = PromoteIntOp_MLOAD(cast<MaskedLoadSDNode>(N),
1184  OpNo); break;
1185  case ISD::MGATHER: Res = PromoteIntOp_MGATHER(cast<MaskedGatherSDNode>(N),
1186  OpNo); break;
1187  case ISD::MSCATTER: Res = PromoteIntOp_MSCATTER(cast<MaskedScatterSDNode>(N),
1188  OpNo); break;
1189  case ISD::TRUNCATE: Res = PromoteIntOp_TRUNCATE(N); break;
1190  case ISD::FP16_TO_FP:
1191  case ISD::UINT_TO_FP: Res = PromoteIntOp_UINT_TO_FP(N); break;
1192  case ISD::ZERO_EXTEND: Res = PromoteIntOp_ZERO_EXTEND(N); break;
1193  case ISD::EXTRACT_SUBVECTOR: Res = PromoteIntOp_EXTRACT_SUBVECTOR(N); break;
1194 
1195  case ISD::SHL:
1196  case ISD::SRA:
1197  case ISD::SRL:
1198  case ISD::ROTL:
1199  case ISD::ROTR: Res = PromoteIntOp_Shift(N); break;
1200 
1201  case ISD::ADDCARRY:
1202  case ISD::SUBCARRY: Res = PromoteIntOp_ADDSUBCARRY(N, OpNo); break;
1203 
1204  case ISD::FRAMEADDR:
1205  case ISD::RETURNADDR: Res = PromoteIntOp_FRAMERETURNADDR(N); break;
1206 
1207  case ISD::PREFETCH: Res = PromoteIntOp_PREFETCH(N, OpNo); break;
1208 
1209  case ISD::SMULFIX:
1210  case ISD::SMULFIXSAT:
1211  case ISD::UMULFIX:
1212  case ISD::UMULFIXSAT: Res = PromoteIntOp_MULFIX(N); break;
1213 
1214  case ISD::FPOWI: Res = PromoteIntOp_FPOWI(N); break;
1215 
1216  case ISD::VECREDUCE_ADD:
1217  case ISD::VECREDUCE_MUL:
1218  case ISD::VECREDUCE_AND:
1219  case ISD::VECREDUCE_OR:
1220  case ISD::VECREDUCE_XOR:
1221  case ISD::VECREDUCE_SMAX:
1222  case ISD::VECREDUCE_SMIN:
1223  case ISD::VECREDUCE_UMAX:
1224  case ISD::VECREDUCE_UMIN: Res = PromoteIntOp_VECREDUCE(N); break;
1225  }
1226 
1227  // If the result is null, the sub-method took care of registering results etc.
1228  if (!Res.getNode()) return false;
1229 
1230  // If the result is N, the sub-method updated N in place. Tell the legalizer
1231  // core about this.
1232  if (Res.getNode() == N)
1233  return true;
1234 
1235  assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 &&
1236  "Invalid operand expansion");
1237 
1238  ReplaceValueWith(SDValue(N, 0), Res);
1239  return false;
1240 }
1241 
1242 /// PromoteSetCCOperands - Promote the operands of a comparison. This code is
1243 /// shared among BR_CC, SELECT_CC, and SETCC handlers.
1244 void DAGTypeLegalizer::PromoteSetCCOperands(SDValue &NewLHS,SDValue &NewRHS,
1245  ISD::CondCode CCCode) {
1246  // We have to insert explicit sign or zero extends. Note that we could
1247  // insert sign extends for ALL conditions. For those operations where either
1248  // zero or sign extension would be valid, use SExtOrZExtPromotedInteger
1249  // which will choose the cheapest for the target.
1250  switch (CCCode) {
1251  default: llvm_unreachable("Unknown integer comparison!");
1252  case ISD::SETEQ:
1253  case ISD::SETNE: {
1254  SDValue OpL = GetPromotedInteger(NewLHS);
1255  SDValue OpR = GetPromotedInteger(NewRHS);
1256 
1257  // We would prefer to promote the comparison operand with sign extension.
1258  // If the width of OpL/OpR excluding the duplicated sign bits is no greater
1259  // than the width of NewLHS/NewRH, we can avoid inserting real truncate
1260  // instruction, which is redundant eventually.
1261  unsigned OpLEffectiveBits =
1262  OpL.getScalarValueSizeInBits() - DAG.ComputeNumSignBits(OpL) + 1;
1263  unsigned OpREffectiveBits =
1264  OpR.getScalarValueSizeInBits() - DAG.ComputeNumSignBits(OpR) + 1;
1265  if (OpLEffectiveBits <= NewLHS.getScalarValueSizeInBits() &&
1266  OpREffectiveBits <= NewRHS.getScalarValueSizeInBits()) {
1267  NewLHS = OpL;
1268  NewRHS = OpR;
1269  } else {
1270  NewLHS = SExtOrZExtPromotedInteger(NewLHS);
1271  NewRHS = SExtOrZExtPromotedInteger(NewRHS);
1272  }
1273  break;
1274  }
1275  case ISD::SETUGE:
1276  case ISD::SETUGT:
1277  case ISD::SETULE:
1278  case ISD::SETULT:
1279  NewLHS = SExtOrZExtPromotedInteger(NewLHS);
1280  NewRHS = SExtOrZExtPromotedInteger(NewRHS);
1281  break;
1282  case ISD::SETGE:
1283  case ISD::SETGT:
1284  case ISD::SETLT:
1285  case ISD::SETLE:
1286  NewLHS = SExtPromotedInteger(NewLHS);
1287  NewRHS = SExtPromotedInteger(NewRHS);
1288  break;
1289  }
1290 }
1291 
1292 SDValue DAGTypeLegalizer::PromoteIntOp_ANY_EXTEND(SDNode *N) {
1293  SDValue Op = GetPromotedInteger(N->getOperand(0));
1294  return DAG.getNode(ISD::ANY_EXTEND, SDLoc(N), N->getValueType(0), Op);
1295 }
1296 
1297 SDValue DAGTypeLegalizer::PromoteIntOp_ATOMIC_STORE(AtomicSDNode *N) {
1298  SDValue Op2 = GetPromotedInteger(N->getOperand(2));
1299  return DAG.getAtomic(N->getOpcode(), SDLoc(N), N->getMemoryVT(),
1300  N->getChain(), N->getBasePtr(), Op2, N->getMemOperand());
1301 }
1302 
1303 SDValue DAGTypeLegalizer::PromoteIntOp_BITCAST(SDNode *N) {
1304  // This should only occur in unusual situations like bitcasting to an
1305  // x86_fp80, so just turn it into a store+load
1306  return CreateStackStoreLoad(N->getOperand(0), N->getValueType(0));
1307 }
1308 
1309 SDValue DAGTypeLegalizer::PromoteIntOp_BR_CC(SDNode *N, unsigned OpNo) {
1310  assert(OpNo == 2 && "Don't know how to promote this operand!");
1311 
1312  SDValue LHS = N->getOperand(2);
1313  SDValue RHS = N->getOperand(3);
1314  PromoteSetCCOperands(LHS, RHS, cast<CondCodeSDNode>(N->getOperand(1))->get());
1315 
1316  // The chain (Op#0), CC (#1) and basic block destination (Op#4) are always
1317  // legal types.
1318  return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0),
1319  N->getOperand(1), LHS, RHS, N->getOperand(4)),
1320  0);
1321 }
1322 
1323 SDValue DAGTypeLegalizer::PromoteIntOp_BRCOND(SDNode *N, unsigned OpNo) {
1324  assert(OpNo == 1 && "only know how to promote condition");
1325 
1326  // Promote all the way up to the canonical SetCC type.
1327  SDValue Cond = PromoteTargetBoolean(N->getOperand(1), MVT::Other);
1328 
1329  // The chain (Op#0) and basic block destination (Op#2) are always legal types.
1330  return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0), Cond,
1331  N->getOperand(2)), 0);
1332 }
1333 
1334 SDValue DAGTypeLegalizer::PromoteIntOp_BUILD_PAIR(SDNode *N) {
1335  // Since the result type is legal, the operands must promote to it.
1336  EVT OVT = N->getOperand(0).getValueType();
1337  SDValue Lo = ZExtPromotedInteger(N->getOperand(0));
1338  SDValue Hi = GetPromotedInteger(N->getOperand(1));
1339  assert(Lo.getValueType() == N->getValueType(0) && "Operand over promoted?");
1340  SDLoc dl(N);
1341 
1342  Hi = DAG.getNode(ISD::SHL, dl, N->getValueType(0), Hi,
1343  DAG.getConstant(OVT.getSizeInBits(), dl,
1344  TLI.getPointerTy(DAG.getDataLayout())));
1345  return DAG.getNode(ISD::OR, dl, N->getValueType(0), Lo, Hi);
1346 }
1347 
1348 SDValue DAGTypeLegalizer::PromoteIntOp_BUILD_VECTOR(SDNode *N) {
1349  // The vector type is legal but the element type is not. This implies
1350  // that the vector is a power-of-two in length and that the element
1351  // type does not have a strange size (eg: it is not i1).
1352  EVT VecVT = N->getValueType(0);
1353  unsigned NumElts = VecVT.getVectorNumElements();
1354  assert(!((NumElts & 1) && (!TLI.isTypeLegal(VecVT))) &&
1355  "Legal vector of one illegal element?");
1356 
1357  // Promote the inserted value. The type does not need to match the
1358  // vector element type. Check that any extra bits introduced will be
1359  // truncated away.
1362  "Type of inserted value narrower than vector element type!");
1363 
1364  SmallVector<SDValue, 16> NewOps;
1365  for (unsigned i = 0; i < NumElts; ++i)
1366  NewOps.push_back(GetPromotedInteger(N->getOperand(i)));
1367 
1368  return SDValue(DAG.UpdateNodeOperands(N, NewOps), 0);
1369 }
1370 
1371 SDValue DAGTypeLegalizer::PromoteIntOp_INSERT_VECTOR_ELT(SDNode *N,
1372  unsigned OpNo) {
1373  if (OpNo == 1) {
1374  // Promote the inserted value. This is valid because the type does not
1375  // have to match the vector element type.
1376 
1377  // Check that any extra bits introduced will be truncated away.
1380  "Type of inserted value narrower than vector element type!");
1381  return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0),
1382  GetPromotedInteger(N->getOperand(1)),
1383  N->getOperand(2)),
1384  0);
1385  }
1386 
1387  assert(OpNo == 2 && "Different operand and result vector types?");
1388 
1389  // Promote the index.
1390  SDValue Idx = DAG.getZExtOrTrunc(N->getOperand(2), SDLoc(N),
1391  TLI.getVectorIdxTy(DAG.getDataLayout()));
1392  return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0),
1393  N->getOperand(1), Idx), 0);
1394 }
1395 
1396 SDValue DAGTypeLegalizer::PromoteIntOp_SCALAR_TO_VECTOR(SDNode *N) {
1397  // Integer SCALAR_TO_VECTOR operands are implicitly truncated, so just promote
1398  // the operand in place.
1399  return SDValue(DAG.UpdateNodeOperands(N,
1400  GetPromotedInteger(N->getOperand(0))), 0);
1401 }
1402 
1403 SDValue DAGTypeLegalizer::PromoteIntOp_SPLAT_VECTOR(SDNode *N) {
1404  // Integer SPLAT_VECTOR operands are implicitly truncated, so just promote the
1405  // operand in place.
1406  return SDValue(
1407  DAG.UpdateNodeOperands(N, GetPromotedInteger(N->getOperand(0))), 0);
1408 }
1409 
1410 SDValue DAGTypeLegalizer::PromoteIntOp_SELECT(SDNode *N, unsigned OpNo) {
1411  assert(OpNo == 0 && "Only know how to promote the condition!");
1412  SDValue Cond = N->getOperand(0);
1413  EVT OpTy = N->getOperand(1).getValueType();
1414 
1415  if (N->getOpcode() == ISD::VSELECT)
1416  if (SDValue Res = WidenVSELECTAndMask(N))
1417  return Res;
1418 
1419  // Promote all the way up to the canonical SetCC type.
1420  EVT OpVT = N->getOpcode() == ISD::SELECT ? OpTy.getScalarType() : OpTy;
1421  Cond = PromoteTargetBoolean(Cond, OpVT);
1422 
1423  return SDValue(DAG.UpdateNodeOperands(N, Cond, N->getOperand(1),
1424  N->getOperand(2)), 0);
1425 }
1426 
1427 SDValue DAGTypeLegalizer::PromoteIntOp_SELECT_CC(SDNode *N, unsigned OpNo) {
1428  assert(OpNo == 0 && "Don't know how to promote this operand!");
1429 
1430  SDValue LHS = N->getOperand(0);
1431  SDValue RHS = N->getOperand(1);
1432  PromoteSetCCOperands(LHS, RHS, cast<CondCodeSDNode>(N->getOperand(4))->get());
1433 
1434  // The CC (#4) and the possible return values (#2 and #3) have legal types.
1435  return SDValue(DAG.UpdateNodeOperands(N, LHS, RHS, N->getOperand(2),
1436  N->getOperand(3), N->getOperand(4)), 0);
1437 }
1438 
1439 SDValue DAGTypeLegalizer::PromoteIntOp_SETCC(SDNode *N, unsigned OpNo) {
1440  assert(OpNo == 0 && "Don't know how to promote this operand!");
1441 
1442  SDValue LHS = N->getOperand(0);
1443  SDValue RHS = N->getOperand(1);
1444  PromoteSetCCOperands(LHS, RHS, cast<CondCodeSDNode>(N->getOperand(2))->get());
1445 
1446  // The CC (#2) is always legal.
1447  return SDValue(DAG.UpdateNodeOperands(N, LHS, RHS, N->getOperand(2)), 0);
1448 }
1449 
1450 SDValue DAGTypeLegalizer::PromoteIntOp_Shift(SDNode *N) {
1451  return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0),
1452  ZExtPromotedInteger(N->getOperand(1))), 0);
1453 }
1454 
1455 SDValue DAGTypeLegalizer::PromoteIntOp_SIGN_EXTEND(SDNode *N) {
1456  SDValue Op = GetPromotedInteger(N->getOperand(0));
1457  SDLoc dl(N);
1458  Op = DAG.getNode(ISD::ANY_EXTEND, dl, N->getValueType(0), Op);
1459  return DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, Op.getValueType(),
1460  Op, DAG.getValueType(N->getOperand(0).getValueType()));
1461 }
1462 
1463 SDValue DAGTypeLegalizer::PromoteIntOp_SINT_TO_FP(SDNode *N) {
1464  return SDValue(DAG.UpdateNodeOperands(N,
1465  SExtPromotedInteger(N->getOperand(0))), 0);
1466 }
1467 
1468 SDValue DAGTypeLegalizer::PromoteIntOp_STORE(StoreSDNode *N, unsigned OpNo){
1469  assert(ISD::isUNINDEXEDStore(N) && "Indexed store during type legalization!");
1470  SDValue Ch = N->getChain(), Ptr = N->getBasePtr();
1471  SDLoc dl(N);
1472 
1473  SDValue Val = GetPromotedInteger(N->getValue()); // Get promoted value.
1474 
1475  // Truncate the value and store the result.
1476  return DAG.getTruncStore(Ch, dl, Val, Ptr,
1477  N->getMemoryVT(), N->getMemOperand());
1478 }
1479 
1480 SDValue DAGTypeLegalizer::PromoteIntOp_MSTORE(MaskedStoreSDNode *N,
1481  unsigned OpNo) {
1482 
1483  SDValue DataOp = N->getValue();
1484  EVT DataVT = DataOp.getValueType();
1485  SDValue Mask = N->getMask();
1486  SDLoc dl(N);
1487 
1488  bool TruncateStore = false;
1489  if (OpNo == 3) {
1490  Mask = PromoteTargetBoolean(Mask, DataVT);
1491  // Update in place.
1492  SmallVector<SDValue, 4> NewOps(N->op_begin(), N->op_end());
1493  NewOps[3] = Mask;
1494  return SDValue(DAG.UpdateNodeOperands(N, NewOps), 0);
1495  } else { // Data operand
1496  assert(OpNo == 1 && "Unexpected operand for promotion");
1497  DataOp = GetPromotedInteger(DataOp);
1498  TruncateStore = true;
1499  }
1500 
1501  return DAG.getMaskedStore(N->getChain(), dl, DataOp, N->getBasePtr(), Mask,
1502  N->getMemoryVT(), N->getMemOperand(),
1503  TruncateStore, N->isCompressingStore());
1504 }
1505 
1506 SDValue DAGTypeLegalizer::PromoteIntOp_MLOAD(MaskedLoadSDNode *N,
1507  unsigned OpNo) {
1508  assert(OpNo == 2 && "Only know how to promote the mask!");
1509  EVT DataVT = N->getValueType(0);
1510  SDValue Mask = PromoteTargetBoolean(N->getOperand(OpNo), DataVT);
1511  SmallVector<SDValue, 4> NewOps(N->op_begin(), N->op_end());
1512  NewOps[OpNo] = Mask;
1513  return SDValue(DAG.UpdateNodeOperands(N, NewOps), 0);
1514 }
1515 
1516 SDValue DAGTypeLegalizer::PromoteIntOp_MGATHER(MaskedGatherSDNode *N,
1517  unsigned OpNo) {
1518 
1519  SmallVector<SDValue, 5> NewOps(N->op_begin(), N->op_end());
1520  if (OpNo == 2) {
1521  // The Mask
1522  EVT DataVT = N->getValueType(0);
1523  NewOps[OpNo] = PromoteTargetBoolean(N->getOperand(OpNo), DataVT);
1524  } else if (OpNo == 4) {
1525  // The Index
1526  if (N->isIndexSigned())
1527  // Need to sign extend the index since the bits will likely be used.
1528  NewOps[OpNo] = SExtPromotedInteger(N->getOperand(OpNo));
1529  else
1530  NewOps[OpNo] = ZExtPromotedInteger(N->getOperand(OpNo));
1531  } else
1532  NewOps[OpNo] = GetPromotedInteger(N->getOperand(OpNo));
1533 
1534  return SDValue(DAG.UpdateNodeOperands(N, NewOps), 0);
1535 }
1536 
1537 SDValue DAGTypeLegalizer::PromoteIntOp_MSCATTER(MaskedScatterSDNode *N,
1538  unsigned OpNo) {
1539  SmallVector<SDValue, 5> NewOps(N->op_begin(), N->op_end());
1540  if (OpNo == 2) {
1541  // The Mask
1542  EVT DataVT = N->getValue().getValueType();
1543  NewOps[OpNo] = PromoteTargetBoolean(N->getOperand(OpNo), DataVT);
1544  } else if (OpNo == 4) {
1545  // The Index
1546  if (N->isIndexSigned())
1547  // Need to sign extend the index since the bits will likely be used.
1548  NewOps[OpNo] = SExtPromotedInteger(N->getOperand(OpNo));
1549  else
1550  NewOps[OpNo] = ZExtPromotedInteger(N->getOperand(OpNo));
1551  } else
1552  NewOps[OpNo] = GetPromotedInteger(N->getOperand(OpNo));
1553  return SDValue(DAG.UpdateNodeOperands(N, NewOps), 0);
1554 }
1555 
1556 SDValue DAGTypeLegalizer::PromoteIntOp_TRUNCATE(SDNode *N) {
1557  SDValue Op = GetPromotedInteger(N->getOperand(0));
1558  return DAG.getNode(ISD::TRUNCATE, SDLoc(N), N->getValueType(0), Op);
1559 }
1560 
1561 SDValue DAGTypeLegalizer::PromoteIntOp_UINT_TO_FP(SDNode *N) {
1562  return SDValue(DAG.UpdateNodeOperands(N,
1563  ZExtPromotedInteger(N->getOperand(0))), 0);
1564 }
1565 
1566 SDValue DAGTypeLegalizer::PromoteIntOp_ZERO_EXTEND(SDNode *N) {
1567  SDLoc dl(N);
1568  SDValue Op = GetPromotedInteger(N->getOperand(0));
1569  Op = DAG.getNode(ISD::ANY_EXTEND, dl, N->getValueType(0), Op);
1570  return DAG.getZeroExtendInReg(Op, dl,
1572 }
1573 
1574 SDValue DAGTypeLegalizer::PromoteIntOp_ADDSUBCARRY(SDNode *N, unsigned OpNo) {
1575  assert(OpNo == 2 && "Don't know how to promote this operand!");
1576 
1577  SDValue LHS = N->getOperand(0);
1578  SDValue RHS = N->getOperand(1);
1579  SDValue Carry = N->getOperand(2);
1580  SDLoc DL(N);
1581 
1582  Carry = PromoteTargetBoolean(Carry, LHS.getValueType());
1583 
1584  return SDValue(DAG.UpdateNodeOperands(N, LHS, RHS, Carry), 0);
1585 }
1586 
1587 SDValue DAGTypeLegalizer::PromoteIntOp_MULFIX(SDNode *N) {
1588  SDValue Op2 = ZExtPromotedInteger(N->getOperand(2));
1589  return SDValue(
1590  DAG.UpdateNodeOperands(N, N->getOperand(0), N->getOperand(1), Op2), 0);
1591 }
1592 
1593 SDValue DAGTypeLegalizer::PromoteIntOp_FRAMERETURNADDR(SDNode *N) {
1594  // Promote the RETURNADDR/FRAMEADDR argument to a supported integer width.
1595  SDValue Op = ZExtPromotedInteger(N->getOperand(0));
1596  return SDValue(DAG.UpdateNodeOperands(N, Op), 0);
1597 }
1598 
1599 SDValue DAGTypeLegalizer::PromoteIntOp_PREFETCH(SDNode *N, unsigned OpNo) {
1600  assert(OpNo > 1 && "Don't know how to promote this operand!");
1601  // Promote the rw, locality, and cache type arguments to a supported integer
1602  // width.
1603  SDValue Op2 = ZExtPromotedInteger(N->getOperand(2));
1604  SDValue Op3 = ZExtPromotedInteger(N->getOperand(3));
1605  SDValue Op4 = ZExtPromotedInteger(N->getOperand(4));
1606  return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0), N->getOperand(1),
1607  Op2, Op3, Op4),
1608  0);
1609 }
1610 
1611 SDValue DAGTypeLegalizer::PromoteIntOp_FPOWI(SDNode *N) {
1612  SDValue Op = SExtPromotedInteger(N->getOperand(1));
1613  return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0), Op), 0);
1614 }
1615 
1616 SDValue DAGTypeLegalizer::PromoteIntOp_VECREDUCE(SDNode *N) {
1617  SDLoc dl(N);
1618  SDValue Op;
1619  switch (N->getOpcode()) {
1620  default: llvm_unreachable("Expected integer vector reduction");
1621  case ISD::VECREDUCE_ADD:
1622  case ISD::VECREDUCE_MUL:
1623  case ISD::VECREDUCE_AND:
1624  case ISD::VECREDUCE_OR:
1625  case ISD::VECREDUCE_XOR:
1626  Op = GetPromotedInteger(N->getOperand(0));
1627  break;
1628  case ISD::VECREDUCE_SMAX:
1629  case ISD::VECREDUCE_SMIN:
1630  Op = SExtPromotedInteger(N->getOperand(0));
1631  break;
1632  case ISD::VECREDUCE_UMAX:
1633  case ISD::VECREDUCE_UMIN:
1634  Op = ZExtPromotedInteger(N->getOperand(0));
1635  break;
1636  }
1637 
1638  EVT EltVT = Op.getValueType().getVectorElementType();
1639  EVT VT = N->getValueType(0);
1640  if (VT.bitsGE(EltVT))
1641  return DAG.getNode(N->getOpcode(), SDLoc(N), VT, Op);
1642 
1643  // Result size must be >= element size. If this is not the case after
1644  // promotion, also promote the result type and then truncate.
1645  SDValue Reduce = DAG.getNode(N->getOpcode(), dl, EltVT, Op);
1646  return DAG.getNode(ISD::TRUNCATE, dl, VT, Reduce);
1647 }
1648 
1649 //===----------------------------------------------------------------------===//
1650 // Integer Result Expansion
1651 //===----------------------------------------------------------------------===//
1652 
1653 /// ExpandIntegerResult - This method is called when the specified result of the
1654 /// specified node is found to need expansion. At this point, the node may also
1655 /// have invalid operands or may have other results that need promotion, we just
1656 /// know that (at least) one result needs expansion.
1657 void DAGTypeLegalizer::ExpandIntegerResult(SDNode *N, unsigned ResNo) {
1658  LLVM_DEBUG(dbgs() << "Expand integer result: "; N->dump(&DAG);
1659  dbgs() << "\n");
1660  SDValue Lo, Hi;
1661  Lo = Hi = SDValue();
1662 
1663  // See if the target wants to custom expand this node.
1664  if (CustomLowerNode(N, N->getValueType(ResNo), true))
1665  return;
1666 
1667  switch (N->getOpcode()) {
1668  default:
1669 #ifndef NDEBUG
1670  dbgs() << "ExpandIntegerResult #" << ResNo << ": ";
1671  N->dump(&DAG); dbgs() << "\n";
1672 #endif
1673  report_fatal_error("Do not know how to expand the result of this "
1674  "operator!");
1675 
1676  case ISD::MERGE_VALUES: SplitRes_MERGE_VALUES(N, ResNo, Lo, Hi); break;
1677  case ISD::SELECT: SplitRes_SELECT(N, Lo, Hi); break;
1678  case ISD::SELECT_CC: SplitRes_SELECT_CC(N, Lo, Hi); break;
1679  case ISD::UNDEF: SplitRes_UNDEF(N, Lo, Hi); break;
1680 
1681  case ISD::BITCAST: ExpandRes_BITCAST(N, Lo, Hi); break;
1682  case ISD::BUILD_PAIR: ExpandRes_BUILD_PAIR(N, Lo, Hi); break;
1683  case ISD::EXTRACT_ELEMENT: ExpandRes_EXTRACT_ELEMENT(N, Lo, Hi); break;
1684  case ISD::EXTRACT_VECTOR_ELT: ExpandRes_EXTRACT_VECTOR_ELT(N, Lo, Hi); break;
1685  case ISD::VAARG: ExpandRes_VAARG(N, Lo, Hi); break;
1686 
1687  case ISD::ANY_EXTEND: ExpandIntRes_ANY_EXTEND(N, Lo, Hi); break;
1688  case ISD::AssertSext: ExpandIntRes_AssertSext(N, Lo, Hi); break;
1689  case ISD::AssertZext: ExpandIntRes_AssertZext(N, Lo, Hi); break;
1690  case ISD::BITREVERSE: ExpandIntRes_BITREVERSE(N, Lo, Hi); break;
1691  case ISD::BSWAP: ExpandIntRes_BSWAP(N, Lo, Hi); break;
1692  case ISD::Constant: ExpandIntRes_Constant(N, Lo, Hi); break;
1693  case ISD::ABS: ExpandIntRes_ABS(N, Lo, Hi); break;
1694  case ISD::CTLZ_ZERO_UNDEF:
1695  case ISD::CTLZ: ExpandIntRes_CTLZ(N, Lo, Hi); break;
1696  case ISD::CTPOP: ExpandIntRes_CTPOP(N, Lo, Hi); break;
1697  case ISD::CTTZ_ZERO_UNDEF:
1698  case ISD::CTTZ: ExpandIntRes_CTTZ(N, Lo, Hi); break;
1699  case ISD::FLT_ROUNDS_: ExpandIntRes_FLT_ROUNDS(N, Lo, Hi); break;
1700  case ISD::FP_TO_SINT: ExpandIntRes_FP_TO_SINT(N, Lo, Hi); break;
1701  case ISD::FP_TO_UINT: ExpandIntRes_FP_TO_UINT(N, Lo, Hi); break;
1702  case ISD::LLROUND: ExpandIntRes_LLROUND(N, Lo, Hi); break;
1703  case ISD::LLRINT: ExpandIntRes_LLRINT(N, Lo, Hi); break;
1704  case ISD::LOAD: ExpandIntRes_LOAD(cast<LoadSDNode>(N), Lo, Hi); break;
1705  case ISD::MUL: ExpandIntRes_MUL(N, Lo, Hi); break;
1706  case ISD::READCYCLECOUNTER: ExpandIntRes_READCYCLECOUNTER(N, Lo, Hi); break;
1707  case ISD::SDIV: ExpandIntRes_SDIV(N, Lo, Hi); break;
1708  case ISD::SIGN_EXTEND: ExpandIntRes_SIGN_EXTEND(N, Lo, Hi); break;
1709  case ISD::SIGN_EXTEND_INREG: ExpandIntRes_SIGN_EXTEND_INREG(N, Lo, Hi); break;
1710  case ISD::SREM: ExpandIntRes_SREM(N, Lo, Hi); break;
1711  case ISD::TRUNCATE: ExpandIntRes_TRUNCATE(N, Lo, Hi); break;
1712  case ISD::UDIV: ExpandIntRes_UDIV(N, Lo, Hi); break;
1713  case ISD::UREM: ExpandIntRes_UREM(N, Lo, Hi); break;
1714  case ISD::ZERO_EXTEND: ExpandIntRes_ZERO_EXTEND(N, Lo, Hi); break;
1715  case ISD::ATOMIC_LOAD: ExpandIntRes_ATOMIC_LOAD(N, Lo, Hi); break;
1716 
1717  case ISD::ATOMIC_LOAD_ADD:
1718  case ISD::ATOMIC_LOAD_SUB:
1719  case ISD::ATOMIC_LOAD_AND:
1720  case ISD::ATOMIC_LOAD_CLR:
1721  case ISD::ATOMIC_LOAD_OR:
1722  case ISD::ATOMIC_LOAD_XOR:
1723  case ISD::ATOMIC_LOAD_NAND:
1724  case ISD::ATOMIC_LOAD_MIN:
1725  case ISD::ATOMIC_LOAD_MAX:
1726  case ISD::ATOMIC_LOAD_UMIN:
1727  case ISD::ATOMIC_LOAD_UMAX:
1728  case ISD::ATOMIC_SWAP:
1729  case ISD::ATOMIC_CMP_SWAP: {
1730  std::pair<SDValue, SDValue> Tmp = ExpandAtomic(N);
1731  SplitInteger(Tmp.first, Lo, Hi);
1732  ReplaceValueWith(SDValue(N, 1), Tmp.second);
1733  break;
1734  }
1736  AtomicSDNode *AN = cast<AtomicSDNode>(N);
1737  SDVTList VTs = DAG.getVTList(N->getValueType(0), MVT::Other);
1738  SDValue Tmp = DAG.getAtomicCmpSwap(
1739  ISD::ATOMIC_CMP_SWAP, SDLoc(N), AN->getMemoryVT(), VTs,
1740  N->getOperand(0), N->getOperand(1), N->getOperand(2), N->getOperand(3),
1741  AN->getMemOperand());
1742 
1743  // Expanding to the strong ATOMIC_CMP_SWAP node means we can determine
1744  // success simply by comparing the loaded value against the ingoing
1745  // comparison.
1746  SDValue Success = DAG.getSetCC(SDLoc(N), N->getValueType(1), Tmp,
1747  N->getOperand(2), ISD::SETEQ);
1748 
1749  SplitInteger(Tmp, Lo, Hi);
1750  ReplaceValueWith(SDValue(N, 1), Success);
1751  ReplaceValueWith(SDValue(N, 2), Tmp.getValue(1));
1752  break;
1753  }
1754 
1755  case ISD::AND:
1756  case ISD::OR:
1757  case ISD::XOR: ExpandIntRes_Logical(N, Lo, Hi); break;
1758 
1759  case ISD::UMAX:
1760  case ISD::SMAX:
1761  case ISD::UMIN:
1762  case ISD::SMIN: ExpandIntRes_MINMAX(N, Lo, Hi); break;
1763 
1764  case ISD::ADD:
1765  case ISD::SUB: ExpandIntRes_ADDSUB(N, Lo, Hi); break;
1766 
1767  case ISD::ADDC:
1768  case ISD::SUBC: ExpandIntRes_ADDSUBC(N, Lo, Hi); break;
1769 
1770  case ISD::ADDE:
1771  case ISD::SUBE: ExpandIntRes_ADDSUBE(N, Lo, Hi); break;
1772 
1773  case ISD::ADDCARRY:
1774  case ISD::SUBCARRY: ExpandIntRes_ADDSUBCARRY(N, Lo, Hi); break;
1775 
1776  case ISD::SHL:
1777  case ISD::SRA:
1778  case ISD::SRL: ExpandIntRes_Shift(N, Lo, Hi); break;
1779 
1780  case ISD::SADDO:
1781  case ISD::SSUBO: ExpandIntRes_SADDSUBO(N, Lo, Hi); break;
1782  case ISD::UADDO:
1783  case ISD::USUBO: ExpandIntRes_UADDSUBO(N, Lo, Hi); break;
1784  case ISD::UMULO:
1785  case ISD::SMULO: ExpandIntRes_XMULO(N, Lo, Hi); break;
1786 
1787  case ISD::SADDSAT:
1788  case ISD::UADDSAT:
1789  case ISD::SSUBSAT:
1790  case ISD::USUBSAT: ExpandIntRes_ADDSUBSAT(N, Lo, Hi); break;
1791 
1792  case ISD::SMULFIX:
1793  case ISD::SMULFIXSAT:
1794  case ISD::UMULFIX:
1795  case ISD::UMULFIXSAT: ExpandIntRes_MULFIX(N, Lo, Hi); break;
1796 
1797  case ISD::VECREDUCE_ADD:
1798  case ISD::VECREDUCE_MUL:
1799  case ISD::VECREDUCE_AND:
1800  case ISD::VECREDUCE_OR:
1801  case ISD::VECREDUCE_XOR:
1802  case ISD::VECREDUCE_SMAX:
1803  case ISD::VECREDUCE_SMIN:
1804  case ISD::VECREDUCE_UMAX:
1805  case ISD::VECREDUCE_UMIN: ExpandIntRes_VECREDUCE(N, Lo, Hi); break;
1806  }
1807 
1808  // If Lo/Hi is null, the sub-method took care of registering results etc.
1809  if (Lo.getNode())
1810  SetExpandedInteger(SDValue(N, ResNo), Lo, Hi);
1811 }
1812 
1813 /// Lower an atomic node to the appropriate builtin call.
1814 std::pair <SDValue, SDValue> DAGTypeLegalizer::ExpandAtomic(SDNode *Node) {
1815  unsigned Opc = Node->getOpcode();
1816  MVT VT = cast<AtomicSDNode>(Node)->getMemoryVT().getSimpleVT();
1817  RTLIB::Libcall LC = RTLIB::getSYNC(Opc, VT);
1818  assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected atomic op or value type!");
1819 
1820  return ExpandChainLibCall(LC, Node, false);
1821 }
1822 
1823 /// N is a shift by a value that needs to be expanded,
1824 /// and the shift amount is a constant 'Amt'. Expand the operation.
1825 void DAGTypeLegalizer::ExpandShiftByConstant(SDNode *N, const APInt &Amt,
1826  SDValue &Lo, SDValue &Hi) {
1827  SDLoc DL(N);
1828  // Expand the incoming operand to be shifted, so that we have its parts
1829  SDValue InL, InH;
1830  GetExpandedInteger(N->getOperand(0), InL, InH);
1831 
1832  // Though Amt shouldn't usually be 0, it's possible. E.g. when legalization
1833  // splitted a vector shift, like this: <op1, op2> SHL <0, 2>.
1834  if (!Amt) {
1835  Lo = InL;
1836  Hi = InH;
1837  return;
1838  }
1839 
1840  EVT NVT = InL.getValueType();
1841  unsigned VTBits = N->getValueType(0).getSizeInBits();
1842  unsigned NVTBits = NVT.getSizeInBits();
1843  EVT ShTy = N->getOperand(1).getValueType();
1844 
1845  if (N->getOpcode() == ISD::SHL) {
1846  if (Amt.ugt(VTBits)) {
1847  Lo = Hi = DAG.getConstant(0, DL, NVT);
1848  } else if (Amt.ugt(NVTBits)) {
1849  Lo = DAG.getConstant(0, DL, NVT);
1850  Hi = DAG.getNode(ISD::SHL, DL,
1851  NVT, InL, DAG.getConstant(Amt - NVTBits, DL, ShTy));
1852  } else if (Amt == NVTBits) {
1853  Lo = DAG.getConstant(0, DL, NVT);
1854  Hi = InL;
1855  } else {
1856  Lo = DAG.getNode(ISD::SHL, DL, NVT, InL, DAG.getConstant(Amt, DL, ShTy));
1857  Hi = DAG.getNode(ISD::OR, DL, NVT,
1858  DAG.getNode(ISD::SHL, DL, NVT, InH,
1859  DAG.getConstant(Amt, DL, ShTy)),
1860  DAG.getNode(ISD::SRL, DL, NVT, InL,
1861  DAG.getConstant(-Amt + NVTBits, DL, ShTy)));
1862  }
1863  return;
1864  }
1865 
1866  if (N->getOpcode() == ISD::SRL) {
1867  if (Amt.ugt(VTBits)) {
1868  Lo = Hi = DAG.getConstant(0, DL, NVT);
1869  } else if (Amt.ugt(NVTBits)) {
1870  Lo = DAG.getNode(ISD::SRL, DL,
1871  NVT, InH, DAG.getConstant(Amt - NVTBits, DL, ShTy));
1872  Hi = DAG.getConstant(0, DL, NVT);
1873  } else if (Amt == NVTBits) {
1874  Lo = InH;
1875  Hi = DAG.getConstant(0, DL, NVT);
1876  } else {
1877  Lo = DAG.getNode(ISD::OR, DL, NVT,
1878  DAG.getNode(ISD::SRL, DL, NVT, InL,
1879  DAG.getConstant(Amt, DL, ShTy)),
1880  DAG.getNode(ISD::SHL, DL, NVT, InH,
1881  DAG.getConstant(-Amt + NVTBits, DL, ShTy)));
1882  Hi = DAG.getNode(ISD::SRL, DL, NVT, InH, DAG.getConstant(Amt, DL, ShTy));
1883  }
1884  return;
1885  }
1886 
1887  assert(N->getOpcode() == ISD::SRA && "Unknown shift!");
1888  if (Amt.ugt(VTBits)) {
1889  Hi = Lo = DAG.getNode(ISD::SRA, DL, NVT, InH,
1890  DAG.getConstant(NVTBits - 1, DL, ShTy));
1891  } else if (Amt.ugt(NVTBits)) {
1892  Lo = DAG.getNode(ISD::SRA, DL, NVT, InH,
1893  DAG.getConstant(Amt - NVTBits, DL, ShTy));
1894  Hi = DAG.getNode(ISD::SRA, DL, NVT, InH,
1895  DAG.getConstant(NVTBits - 1, DL, ShTy));
1896  } else if (Amt == NVTBits) {
1897  Lo = InH;
1898  Hi = DAG.getNode(ISD::SRA, DL, NVT, InH,
1899  DAG.getConstant(NVTBits - 1, DL, ShTy));
1900  } else {
1901  Lo = DAG.getNode(ISD::OR, DL, NVT,
1902  DAG.getNode(ISD::SRL, DL, NVT, InL,
1903  DAG.getConstant(Amt, DL, ShTy)),
1904  DAG.getNode(ISD::SHL, DL, NVT, InH,
1905  DAG.getConstant(-Amt + NVTBits, DL, ShTy)));
1906  Hi = DAG.getNode(ISD::SRA, DL, NVT, InH, DAG.getConstant(Amt, DL, ShTy));
1907  }
1908 }
1909 
1910 /// ExpandShiftWithKnownAmountBit - Try to determine whether we can simplify
1911 /// this shift based on knowledge of the high bit of the shift amount. If we
1912 /// can tell this, we know that it is >= 32 or < 32, without knowing the actual
1913 /// shift amount.
1914 bool DAGTypeLegalizer::
1915 ExpandShiftWithKnownAmountBit(SDNode *N, SDValue &Lo, SDValue &Hi) {
1916  SDValue Amt = N->getOperand(1);
1917  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
1918  EVT ShTy = Amt.getValueType();
1919  unsigned ShBits = ShTy.getScalarSizeInBits();
1920  unsigned NVTBits = NVT.getScalarSizeInBits();
1921  assert(isPowerOf2_32(NVTBits) &&
1922  "Expanded integer type size not a power of two!");
1923  SDLoc dl(N);
1924 
1925  APInt HighBitMask = APInt::getHighBitsSet(ShBits, ShBits - Log2_32(NVTBits));
1926  KnownBits Known = DAG.computeKnownBits(N->getOperand(1));
1927 
1928  // If we don't know anything about the high bits, exit.
1929  if (((Known.Zero|Known.One) & HighBitMask) == 0)
1930  return false;
1931 
1932  // Get the incoming operand to be shifted.
1933  SDValue InL, InH;
1934  GetExpandedInteger(N->getOperand(0), InL, InH);
1935 
1936  // If we know that any of the high bits of the shift amount are one, then we
1937  // can do this as a couple of simple shifts.
1938  if (Known.One.intersects(HighBitMask)) {
1939  // Mask out the high bit, which we know is set.
1940  Amt = DAG.getNode(ISD::AND, dl, ShTy, Amt,
1941  DAG.getConstant(~HighBitMask, dl, ShTy));
1942 
1943  switch (N->getOpcode()) {
1944  default: llvm_unreachable("Unknown shift");
1945  case ISD::SHL:
1946  Lo = DAG.getConstant(0, dl, NVT); // Low part is zero.
1947  Hi = DAG.getNode(ISD::SHL, dl, NVT, InL, Amt); // High part from Lo part.
1948  return true;
1949  case ISD::SRL:
1950  Hi = DAG.getConstant(0, dl, NVT); // Hi part is zero.
1951  Lo = DAG.getNode(ISD::SRL, dl, NVT, InH, Amt); // Lo part from Hi part.
1952  return true;
1953  case ISD::SRA:
1954  Hi = DAG.getNode(ISD::SRA, dl, NVT, InH, // Sign extend high part.
1955  DAG.getConstant(NVTBits - 1, dl, ShTy));
1956  Lo = DAG.getNode(ISD::SRA, dl, NVT, InH, Amt); // Lo part from Hi part.
1957  return true;
1958  }
1959  }
1960 
1961  // If we know that all of the high bits of the shift amount are zero, then we
1962  // can do this as a couple of simple shifts.
1963  if (HighBitMask.isSubsetOf(Known.Zero)) {
1964  // Calculate 31-x. 31 is used instead of 32 to avoid creating an undefined
1965  // shift if x is zero. We can use XOR here because x is known to be smaller
1966  // than 32.
1967  SDValue Amt2 = DAG.getNode(ISD::XOR, dl, ShTy, Amt,
1968  DAG.getConstant(NVTBits - 1, dl, ShTy));
1969 
1970  unsigned Op1, Op2;
1971  switch (N->getOpcode()) {
1972  default: llvm_unreachable("Unknown shift");
1973  case ISD::SHL: Op1 = ISD::SHL; Op2 = ISD::SRL; break;
1974  case ISD::SRL:
1975  case ISD::SRA: Op1 = ISD::SRL; Op2 = ISD::SHL; break;
1976  }
1977 
1978  // When shifting right the arithmetic for Lo and Hi is swapped.
1979  if (N->getOpcode() != ISD::SHL)
1980  std::swap(InL, InH);
1981 
1982  // Use a little trick to get the bits that move from Lo to Hi. First
1983  // shift by one bit.
1984  SDValue Sh1 = DAG.getNode(Op2, dl, NVT, InL, DAG.getConstant(1, dl, ShTy));
1985  // Then compute the remaining shift with amount-1.
1986  SDValue Sh2 = DAG.getNode(Op2, dl, NVT, Sh1, Amt2);
1987 
1988  Lo = DAG.getNode(N->getOpcode(), dl, NVT, InL, Amt);
1989  Hi = DAG.getNode(ISD::OR, dl, NVT, DAG.getNode(Op1, dl, NVT, InH, Amt),Sh2);
1990 
1991  if (N->getOpcode() != ISD::SHL)
1992  std::swap(Hi, Lo);
1993  return true;
1994  }
1995 
1996  return false;
1997 }
1998 
1999 /// ExpandShiftWithUnknownAmountBit - Fully general expansion of integer shift
2000 /// of any size.
2001 bool DAGTypeLegalizer::
2002 ExpandShiftWithUnknownAmountBit(SDNode *N, SDValue &Lo, SDValue &Hi) {
2003  SDValue Amt = N->getOperand(1);
2004  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
2005  EVT ShTy = Amt.getValueType();
2006  unsigned NVTBits = NVT.getSizeInBits();
2007  assert(isPowerOf2_32(NVTBits) &&
2008  "Expanded integer type size not a power of two!");
2009  SDLoc dl(N);
2010 
2011  // Get the incoming operand to be shifted.
2012  SDValue InL, InH;
2013  GetExpandedInteger(N->getOperand(0), InL, InH);
2014 
2015  SDValue NVBitsNode = DAG.getConstant(NVTBits, dl, ShTy);
2016  SDValue AmtExcess = DAG.getNode(ISD::SUB, dl, ShTy, Amt, NVBitsNode);
2017  SDValue AmtLack = DAG.getNode(ISD::SUB, dl, ShTy, NVBitsNode, Amt);
2018  SDValue isShort = DAG.getSetCC(dl, getSetCCResultType(ShTy),
2019  Amt, NVBitsNode, ISD::SETULT);
2020  SDValue isZero = DAG.getSetCC(dl, getSetCCResultType(ShTy),
2021  Amt, DAG.getConstant(0, dl, ShTy),
2022  ISD::SETEQ);
2023 
2024  SDValue LoS, HiS, LoL, HiL;
2025  switch (N->getOpcode()) {
2026  default: llvm_unreachable("Unknown shift");
2027  case ISD::SHL:
2028  // Short: ShAmt < NVTBits
2029  LoS = DAG.getNode(ISD::SHL, dl, NVT, InL, Amt);
2030  HiS = DAG.getNode(ISD::OR, dl, NVT,
2031  DAG.getNode(ISD::SHL, dl, NVT, InH, Amt),
2032  DAG.getNode(ISD::SRL, dl, NVT, InL, AmtLack));
2033 
2034  // Long: ShAmt >= NVTBits
2035  LoL = DAG.getConstant(0, dl, NVT); // Lo part is zero.
2036  HiL = DAG.getNode(ISD::SHL, dl, NVT, InL, AmtExcess); // Hi from Lo part.
2037 
2038  Lo = DAG.getSelect(dl, NVT, isShort, LoS, LoL);
2039  Hi = DAG.getSelect(dl, NVT, isZero, InH,
2040  DAG.getSelect(dl, NVT, isShort, HiS, HiL));
2041  return true;
2042  case ISD::SRL:
2043  // Short: ShAmt < NVTBits
2044  HiS = DAG.getNode(ISD::SRL, dl, NVT, InH, Amt);
2045  LoS = DAG.getNode(ISD::OR, dl, NVT,
2046  DAG.getNode(ISD::SRL, dl, NVT, InL, Amt),
2047  // FIXME: If Amt is zero, the following shift generates an undefined result
2048  // on some architectures.
2049  DAG.getNode(ISD::SHL, dl, NVT, InH, AmtLack));
2050 
2051  // Long: ShAmt >= NVTBits
2052  HiL = DAG.getConstant(0, dl, NVT); // Hi part is zero.
2053  LoL = DAG.getNode(ISD::SRL, dl, NVT, InH, AmtExcess); // Lo from Hi part.
2054 
2055  Lo = DAG.getSelect(dl, NVT, isZero, InL,
2056  DAG.getSelect(dl, NVT, isShort, LoS, LoL));
2057  Hi = DAG.getSelect(dl, NVT, isShort, HiS, HiL);
2058  return true;
2059  case ISD::SRA:
2060  // Short: ShAmt < NVTBits
2061  HiS = DAG.getNode(ISD::SRA, dl, NVT, InH, Amt);
2062  LoS = DAG.getNode(ISD::OR, dl, NVT,
2063  DAG.getNode(ISD::SRL, dl, NVT, InL, Amt),
2064  DAG.getNode(ISD::SHL, dl, NVT, InH, AmtLack));
2065 
2066  // Long: ShAmt >= NVTBits
2067  HiL = DAG.getNode(ISD::SRA, dl, NVT, InH, // Sign of Hi part.
2068  DAG.getConstant(NVTBits - 1, dl, ShTy));
2069  LoL = DAG.getNode(ISD::SRA, dl, NVT, InH, AmtExcess); // Lo from Hi part.
2070 
2071  Lo = DAG.getSelect(dl, NVT, isZero, InL,
2072  DAG.getSelect(dl, NVT, isShort, LoS, LoL));
2073  Hi = DAG.getSelect(dl, NVT, isShort, HiS, HiL);
2074  return true;
2075  }
2076 }
2077 
2078 static std::pair<ISD::CondCode, ISD::NodeType> getExpandedMinMaxOps(int Op) {
2079 
2080  switch (Op) {
2081  default: llvm_unreachable("invalid min/max opcode");
2082  case ISD::SMAX:
2083  return std::make_pair(ISD::SETGT, ISD::UMAX);
2084  case ISD::UMAX:
2085  return std::make_pair(ISD::SETUGT, ISD::UMAX);
2086  case ISD::SMIN:
2087  return std::make_pair(ISD::SETLT, ISD::UMIN);
2088  case ISD::UMIN:
2089  return std::make_pair(ISD::SETULT, ISD::UMIN);
2090  }
2091 }
2092 
2093 void DAGTypeLegalizer::ExpandIntRes_MINMAX(SDNode *N,
2094  SDValue &Lo, SDValue &Hi) {
2095  SDLoc DL(N);
2096  ISD::NodeType LoOpc;
2097  ISD::CondCode CondC;
2098  std::tie(CondC, LoOpc) = getExpandedMinMaxOps(N->getOpcode());
2099 
2100  // Expand the subcomponents.
2101  SDValue LHSL, LHSH, RHSL, RHSH;
2102  GetExpandedInteger(N->getOperand(0), LHSL, LHSH);
2103  GetExpandedInteger(N->getOperand(1), RHSL, RHSH);
2104 
2105  // Value types
2106  EVT NVT = LHSL.getValueType();
2107  EVT CCT = getSetCCResultType(NVT);
2108 
2109  // Hi part is always the same op
2110  Hi = DAG.getNode(N->getOpcode(), DL, NVT, {LHSH, RHSH});
2111 
2112  // We need to know whether to select Lo part that corresponds to 'winning'
2113  // Hi part or if Hi parts are equal.
2114  SDValue IsHiLeft = DAG.getSetCC(DL, CCT, LHSH, RHSH, CondC);
2115  SDValue IsHiEq = DAG.getSetCC(DL, CCT, LHSH, RHSH, ISD::SETEQ);
2116 
2117  // Lo part corresponding to the 'winning' Hi part
2118  SDValue LoCmp = DAG.getSelect(DL, NVT, IsHiLeft, LHSL, RHSL);
2119 
2120  // Recursed Lo part if Hi parts are equal, this uses unsigned version
2121  SDValue LoMinMax = DAG.getNode(LoOpc, DL, NVT, {LHSL, RHSL});
2122 
2123  Lo = DAG.getSelect(DL, NVT, IsHiEq, LoMinMax, LoCmp);
2124 }
2125 
2126 void DAGTypeLegalizer::ExpandIntRes_ADDSUB(SDNode *N,
2127  SDValue &Lo, SDValue &Hi) {
2128  SDLoc dl(N);
2129  // Expand the subcomponents.
2130  SDValue LHSL, LHSH, RHSL, RHSH;
2131  GetExpandedInteger(N->getOperand(0), LHSL, LHSH);
2132  GetExpandedInteger(N->getOperand(1), RHSL, RHSH);
2133 
2134  EVT NVT = LHSL.getValueType();
2135  SDValue LoOps[2] = { LHSL, RHSL };
2136  SDValue HiOps[3] = { LHSH, RHSH };
2137 
2138  bool HasOpCarry = TLI.isOperationLegalOrCustom(
2140  TLI.getTypeToExpandTo(*DAG.getContext(), NVT));
2141  if (HasOpCarry) {
2142  SDVTList VTList = DAG.getVTList(NVT, getSetCCResultType(NVT));
2143  if (N->getOpcode() == ISD::ADD) {
2144  Lo = DAG.getNode(ISD::UADDO, dl, VTList, LoOps);
2145  HiOps[2] = Lo.getValue(1);
2146  Hi = DAG.getNode(ISD::ADDCARRY, dl, VTList, HiOps);
2147  } else {
2148  Lo = DAG.getNode(ISD::USUBO, dl, VTList, LoOps);
2149  HiOps[2] = Lo.getValue(1);
2150  Hi = DAG.getNode(ISD::SUBCARRY, dl, VTList, HiOps);
2151  }
2152  return;
2153  }
2154 
2155  // Do not generate ADDC/ADDE or SUBC/SUBE if the target does not support
2156  // them. TODO: Teach operation legalization how to expand unsupported
2157  // ADDC/ADDE/SUBC/SUBE. The problem is that these operations generate
2158  // a carry of type MVT::Glue, but there doesn't seem to be any way to
2159  // generate a value of this type in the expanded code sequence.
2160  bool hasCarry =
2162  ISD::ADDC : ISD::SUBC,
2163  TLI.getTypeToExpandTo(*DAG.getContext(), NVT));
2164 
2165  if (hasCarry) {
2166  SDVTList VTList = DAG.getVTList(NVT, MVT::Glue);
2167  if (N->getOpcode() == ISD::ADD) {
2168  Lo = DAG.getNode(ISD::ADDC, dl, VTList, LoOps);
2169  HiOps[2] = Lo.getValue(1);
2170  Hi = DAG.getNode(ISD::ADDE, dl, VTList, HiOps);
2171  } else {
2172  Lo = DAG.getNode(ISD::SUBC, dl, VTList, LoOps);
2173  HiOps[2] = Lo.getValue(1);
2174  Hi = DAG.getNode(ISD::SUBE, dl, VTList, HiOps);
2175  }
2176  return;
2177  }
2178 
2179  bool hasOVF =
2182  TLI.getTypeToExpandTo(*DAG.getContext(), NVT));
2184 
2185  if (hasOVF) {
2186  EVT OvfVT = getSetCCResultType(NVT);
2187  SDVTList VTList = DAG.getVTList(NVT, OvfVT);
2188  int RevOpc;
2189  if (N->getOpcode() == ISD::ADD) {
2190  RevOpc = ISD::SUB;
2191  Lo = DAG.getNode(ISD::UADDO, dl, VTList, LoOps);
2192  Hi = DAG.getNode(ISD::ADD, dl, NVT, makeArrayRef(HiOps, 2));
2193  } else {
2194  RevOpc = ISD::ADD;
2195  Lo = DAG.getNode(ISD::USUBO, dl, VTList, LoOps);
2196  Hi = DAG.getNode(ISD::SUB, dl, NVT, makeArrayRef(HiOps, 2));
2197  }
2198  SDValue OVF = Lo.getValue(1);
2199 
2200  switch (BoolType) {
2202  OVF = DAG.getNode(ISD::AND, dl, OvfVT, DAG.getConstant(1, dl, OvfVT), OVF);
2205  OVF = DAG.getZExtOrTrunc(OVF, dl, NVT);
2206  Hi = DAG.getNode(N->getOpcode(), dl, NVT, Hi, OVF);
2207  break;
2209  OVF = DAG.getSExtOrTrunc(OVF, dl, NVT);
2210  Hi = DAG.getNode(RevOpc, dl, NVT, Hi, OVF);
2211  }
2212  return;
2213  }
2214 
2215  if (N->getOpcode() == ISD::ADD) {
2216  Lo = DAG.getNode(ISD::ADD, dl, NVT, LoOps);
2217  Hi = DAG.getNode(ISD::ADD, dl, NVT, makeArrayRef(HiOps, 2));
2218  SDValue Cmp1 = DAG.getSetCC(dl, getSetCCResultType(NVT), Lo, LoOps[0],
2219  ISD::SETULT);
2220 
2222  SDValue Carry = DAG.getZExtOrTrunc(Cmp1, dl, NVT);
2223  Hi = DAG.getNode(ISD::ADD, dl, NVT, Hi, Carry);
2224  return;
2225  }
2226 
2227  SDValue Carry1 = DAG.getSelect(dl, NVT, Cmp1,
2228  DAG.getConstant(1, dl, NVT),
2229  DAG.getConstant(0, dl, NVT));
2230  SDValue Cmp2 = DAG.getSetCC(dl, getSetCCResultType(NVT), Lo, LoOps[1],
2231  ISD::SETULT);
2232  SDValue Carry2 = DAG.getSelect(dl, NVT, Cmp2,
2233  DAG.getConstant(1, dl, NVT), Carry1);
2234  Hi = DAG.getNode(ISD::ADD, dl, NVT, Hi, Carry2);
2235  } else {
2236  Lo = DAG.getNode(ISD::SUB, dl, NVT, LoOps);
2237  Hi = DAG.getNode(ISD::SUB, dl, NVT, makeArrayRef(HiOps, 2));
2238  SDValue Cmp =
2239  DAG.getSetCC(dl, getSetCCResultType(LoOps[0].getValueType()),
2240  LoOps[0], LoOps[1], ISD::SETULT);
2241 
2242  SDValue Borrow;
2244  Borrow = DAG.getZExtOrTrunc(Cmp, dl, NVT);
2245  else
2246  Borrow = DAG.getSelect(dl, NVT, Cmp, DAG.getConstant(1, dl, NVT),
2247  DAG.getConstant(0, dl, NVT));
2248 
2249  Hi = DAG.getNode(ISD::SUB, dl, NVT, Hi, Borrow);
2250  }
2251 }
2252 
2253 void DAGTypeLegalizer::ExpandIntRes_ADDSUBC(SDNode *N,
2254  SDValue &Lo, SDValue &Hi) {
2255  // Expand the subcomponents.
2256  SDValue LHSL, LHSH, RHSL, RHSH;
2257  SDLoc dl(N);
2258  GetExpandedInteger(N->getOperand(0), LHSL, LHSH);
2259  GetExpandedInteger(N->getOperand(1), RHSL, RHSH);
2260  SDVTList VTList = DAG.getVTList(LHSL.getValueType(), MVT::Glue);
2261  SDValue LoOps[2] = { LHSL, RHSL };
2262  SDValue HiOps[3] = { LHSH, RHSH };
2263 
2264  if (N->getOpcode() == ISD::ADDC) {
2265  Lo = DAG.getNode(ISD::ADDC, dl, VTList, LoOps);
2266  HiOps[2] = Lo.getValue(1);
2267  Hi = DAG.getNode(ISD::ADDE, dl, VTList, HiOps);
2268  } else {
2269  Lo = DAG.getNode(ISD::SUBC, dl, VTList, LoOps);
2270  HiOps[2] = Lo.getValue(1);
2271  Hi = DAG.getNode(ISD::SUBE, dl, VTList, HiOps);
2272  }
2273 
2274  // Legalized the flag result - switch anything that used the old flag to
2275  // use the new one.
2276  ReplaceValueWith(SDValue(N, 1), Hi.getValue(1));
2277 }
2278 
2279 void DAGTypeLegalizer::ExpandIntRes_ADDSUBE(SDNode *N,
2280  SDValue &Lo, SDValue &Hi) {
2281  // Expand the subcomponents.
2282  SDValue LHSL, LHSH, RHSL, RHSH;
2283  SDLoc dl(N);
2284  GetExpandedInteger(N->getOperand(0), LHSL, LHSH);
2285  GetExpandedInteger(N->getOperand(1), RHSL, RHSH);
2286  SDVTList VTList = DAG.getVTList(LHSL.getValueType(), MVT::Glue);
2287  SDValue LoOps[3] = { LHSL, RHSL, N->getOperand(2) };
2288  SDValue HiOps[3] = { LHSH, RHSH };
2289 
2290  Lo = DAG.getNode(N->getOpcode(), dl, VTList, LoOps);
2291  HiOps[2] = Lo.getValue(1);
2292  Hi = DAG.getNode(N->getOpcode(), dl, VTList, HiOps);
2293 
2294  // Legalized the flag result - switch anything that used the old flag to
2295  // use the new one.
2296  ReplaceValueWith(SDValue(N, 1), Hi.getValue(1));
2297 }
2298 
2299 void DAGTypeLegalizer::ExpandIntRes_UADDSUBO(SDNode *N,
2300  SDValue &Lo, SDValue &Hi) {
2301  SDValue LHS = N->getOperand(0);
2302  SDValue RHS = N->getOperand(1);
2303  SDLoc dl(N);
2304 
2305  SDValue Ovf;
2306 
2307  bool HasOpCarry = TLI.isOperationLegalOrCustom(
2309  TLI.getTypeToExpandTo(*DAG.getContext(), LHS.getValueType()));
2310 
2311  if (HasOpCarry) {
2312  // Expand the subcomponents.
2313  SDValue LHSL, LHSH, RHSL, RHSH;
2314  GetExpandedInteger(LHS, LHSL, LHSH);
2315  GetExpandedInteger(RHS, RHSL, RHSH);
2316  SDVTList VTList = DAG.getVTList(LHSL.getValueType(), N->getValueType(1));
2317  SDValue LoOps[2] = { LHSL, RHSL };
2318  SDValue HiOps[3] = { LHSH, RHSH };
2319 
2320  unsigned Opc = N->getOpcode() == ISD::UADDO ? ISD::ADDCARRY : ISD::SUBCARRY;
2321  Lo = DAG.getNode(N->getOpcode(), dl, VTList, LoOps);
2322  HiOps[2] = Lo.getValue(1);
2323  Hi = DAG.getNode(Opc, dl, VTList, HiOps);
2324 
2325  Ovf = Hi.getValue(1);
2326  } else {
2327  // Expand the result by simply replacing it with the equivalent
2328  // non-overflow-checking operation.
2329  auto Opc = N->getOpcode() == ISD::UADDO ? ISD::ADD : ISD::SUB;
2330  SDValue Sum = DAG.getNode(Opc, dl, LHS.getValueType(), LHS, RHS);
2331  SplitInteger(Sum, Lo, Hi);
2332 
2333  // Calculate the overflow: addition overflows iff a + b < a, and subtraction
2334  // overflows iff a - b > a.
2335  auto Cond = N->getOpcode() == ISD::UADDO ? ISD::SETULT : ISD::SETUGT;
2336  Ovf = DAG.getSetCC(dl, N->getValueType(1), Sum, LHS, Cond);
2337  }
2338 
2339  // Legalized the flag result - switch anything that used the old flag to
2340  // use the new one.
2341  ReplaceValueWith(SDValue(N, 1), Ovf);
2342 }
2343 
2344 void DAGTypeLegalizer::ExpandIntRes_ADDSUBCARRY(SDNode *N,
2345  SDValue &Lo, SDValue &Hi) {
2346  // Expand the subcomponents.
2347  SDValue LHSL, LHSH, RHSL, RHSH;
2348  SDLoc dl(N);
2349  GetExpandedInteger(N->getOperand(0), LHSL, LHSH);
2350  GetExpandedInteger(N->getOperand(1), RHSL, RHSH);
2351  SDVTList VTList = DAG.getVTList(LHSL.getValueType(), N->getValueType(1));
2352  SDValue LoOps[3] = { LHSL, RHSL, N->getOperand(2) };
2353  SDValue HiOps[3] = { LHSH, RHSH, SDValue() };
2354 
2355  Lo = DAG.getNode(N->getOpcode(), dl, VTList, LoOps);
2356  HiOps[2] = Lo.getValue(1);
2357  Hi = DAG.getNode(N->getOpcode(), dl, VTList, HiOps);
2358 
2359  // Legalized the flag result - switch anything that used the old flag to
2360  // use the new one.
2361  ReplaceValueWith(SDValue(N, 1), Hi.getValue(1));
2362 }
2363 
2364 void DAGTypeLegalizer::ExpandIntRes_ANY_EXTEND(SDNode *N,
2365  SDValue &Lo, SDValue &Hi) {
2366  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
2367  SDLoc dl(N);
2368  SDValue Op = N->getOperand(0);
2369  if (Op.getValueType().bitsLE(NVT)) {
2370  // The low part is any extension of the input (which degenerates to a copy).
2371  Lo = DAG.getNode(ISD::ANY_EXTEND, dl, NVT, Op);
2372  Hi = DAG.getUNDEF(NVT); // The high part is undefined.
2373  } else {
2374  // For example, extension of an i48 to an i64. The operand type necessarily
2375  // promotes to the result type, so will end up being expanded too.
2376  assert(getTypeAction(Op.getValueType()) ==
2378  "Only know how to promote this result!");
2379  SDValue Res = GetPromotedInteger(Op);
2380  assert(Res.getValueType() == N->getValueType(0) &&
2381  "Operand over promoted?");
2382  // Split the promoted operand. This will simplify when it is expanded.
2383  SplitInteger(Res, Lo, Hi);
2384  }
2385 }
2386 
2387 void DAGTypeLegalizer::ExpandIntRes_AssertSext(SDNode *N,
2388  SDValue &Lo, SDValue &Hi) {
2389  SDLoc dl(N);
2390  GetExpandedInteger(N->getOperand(0), Lo, Hi);
2391  EVT NVT = Lo.getValueType();
2392  EVT EVT = cast<VTSDNode>(N->getOperand(1))->getVT();
2393  unsigned NVTBits = NVT.getSizeInBits();
2394  unsigned EVTBits = EVT.getSizeInBits();
2395 
2396  if (NVTBits < EVTBits) {
2397  Hi = DAG.getNode(ISD::AssertSext, dl, NVT, Hi,
2399  EVTBits - NVTBits)));
2400  } else {
2401  Lo = DAG.getNode(ISD::AssertSext, dl, NVT, Lo, DAG.getValueType(EVT));
2402  // The high part replicates the sign bit of Lo, make it explicit.
2403  Hi = DAG.getNode(ISD::SRA, dl, NVT, Lo,
2404  DAG.getConstant(NVTBits - 1, dl,
2405  TLI.getPointerTy(DAG.getDataLayout())));
2406  }
2407 }
2408 
2409 void DAGTypeLegalizer::ExpandIntRes_AssertZext(SDNode *N,
2410  SDValue &Lo, SDValue &Hi) {
2411  SDLoc dl(N);
2412  GetExpandedInteger(N->getOperand(0), Lo, Hi);
2413  EVT NVT = Lo.getValueType();
2414  EVT EVT = cast<VTSDNode>(N->getOperand(1))->getVT();
2415  unsigned NVTBits = NVT.getSizeInBits();
2416  unsigned EVTBits = EVT.getSizeInBits();
2417 
2418  if (NVTBits < EVTBits) {
2419  Hi = DAG.getNode(ISD::AssertZext, dl, NVT, Hi,
2421  EVTBits - NVTBits)));
2422  } else {
2423  Lo = DAG.getNode(ISD::AssertZext, dl, NVT, Lo, DAG.getValueType(EVT));
2424  // The high part must be zero, make it explicit.
2425  Hi = DAG.getConstant(0, dl, NVT);
2426  }
2427 }
2428 
2429 void DAGTypeLegalizer::ExpandIntRes_BITREVERSE(SDNode *N,
2430  SDValue &Lo, SDValue &Hi) {
2431  SDLoc dl(N);
2432  GetExpandedInteger(N->getOperand(0), Hi, Lo); // Note swapped operands.
2433  Lo = DAG.getNode(ISD::BITREVERSE, dl, Lo.getValueType(), Lo);
2434  Hi = DAG.getNode(ISD::BITREVERSE, dl, Hi.getValueType(), Hi);
2435 }
2436 
2437 void DAGTypeLegalizer::ExpandIntRes_BSWAP(SDNode *N,
2438  SDValue &Lo, SDValue &Hi) {
2439  SDLoc dl(N);
2440  GetExpandedInteger(N->getOperand(0), Hi, Lo); // Note swapped operands.
2441  Lo = DAG.getNode(ISD::BSWAP, dl, Lo.getValueType(), Lo);
2442  Hi = DAG.getNode(ISD::BSWAP, dl, Hi.getValueType(), Hi);
2443 }
2444 
2445 void DAGTypeLegalizer::ExpandIntRes_Constant(SDNode *N,
2446  SDValue &Lo, SDValue &Hi) {
2447  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
2448  unsigned NBitWidth = NVT.getSizeInBits();
2449  auto Constant = cast<ConstantSDNode>(N);
2450  const APInt &Cst = Constant->getAPIntValue();
2451  bool IsTarget = Constant->isTargetOpcode();
2452  bool IsOpaque = Constant->isOpaque();
2453  SDLoc dl(N);
2454  Lo = DAG.getConstant(Cst.trunc(NBitWidth), dl, NVT, IsTarget, IsOpaque);
2455  Hi = DAG.getConstant(Cst.lshr(NBitWidth).trunc(NBitWidth), dl, NVT, IsTarget,
2456  IsOpaque);
2457 }
2458 
2459 void DAGTypeLegalizer::ExpandIntRes_ABS(SDNode *N, SDValue &Lo, SDValue &Hi) {
2460  SDLoc dl(N);
2461 
2462  // abs(HiLo) -> (Hi < 0 ? -HiLo : HiLo)
2463  EVT VT = N->getValueType(0);
2464  SDValue N0 = N->getOperand(0);
2465  SDValue Neg = DAG.getNode(ISD::SUB, dl, VT,
2466  DAG.getConstant(0, dl, VT), N0);
2467  SDValue NegLo, NegHi;
2468  SplitInteger(Neg, NegLo, NegHi);
2469 
2470  GetExpandedInteger(N0, Lo, Hi);
2471  EVT NVT = Lo.getValueType();
2472  SDValue HiIsNeg = DAG.getSetCC(dl, getSetCCResultType(NVT),
2473  DAG.getConstant(0, dl, NVT), Hi, ISD::SETGT);
2474  Lo = DAG.getSelect(dl, NVT, HiIsNeg, NegLo, Lo);
2475  Hi = DAG.getSelect(dl, NVT, HiIsNeg, NegHi, Hi);
2476 }
2477 
2478 void DAGTypeLegalizer::ExpandIntRes_CTLZ(SDNode *N,
2479  SDValue &Lo, SDValue &Hi) {
2480  SDLoc dl(N);
2481  // ctlz (HiLo) -> Hi != 0 ? ctlz(Hi) : (ctlz(Lo)+32)
2482  GetExpandedInteger(N->getOperand(0), Lo, Hi);
2483  EVT NVT = Lo.getValueType();
2484 
2485  SDValue HiNotZero = DAG.getSetCC(dl, getSetCCResultType(NVT), Hi,
2486  DAG.getConstant(0, dl, NVT), ISD::SETNE);
2487 
2488  SDValue LoLZ = DAG.getNode(N->getOpcode(), dl, NVT, Lo);
2489  SDValue HiLZ = DAG.getNode(ISD::CTLZ_ZERO_UNDEF, dl, NVT, Hi);
2490 
2491  Lo = DAG.getSelect(dl, NVT, HiNotZero, HiLZ,
2492  DAG.getNode(ISD::ADD, dl, NVT, LoLZ,
2493  DAG.getConstant(NVT.getSizeInBits(), dl,
2494  NVT)));
2495  Hi = DAG.getConstant(0, dl, NVT);
2496 }
2497 
2498 void DAGTypeLegalizer::ExpandIntRes_CTPOP(SDNode *N,
2499  SDValue &Lo, SDValue &Hi) {
2500  SDLoc dl(N);
2501  // ctpop(HiLo) -> ctpop(Hi)+ctpop(Lo)
2502  GetExpandedInteger(N->getOperand(0), Lo, Hi);
2503  EVT NVT = Lo.getValueType();
2504  Lo = DAG.getNode(ISD::ADD, dl, NVT, DAG.getNode(ISD::CTPOP, dl, NVT, Lo),
2505  DAG.getNode(ISD::CTPOP, dl, NVT, Hi));
2506  Hi = DAG.getConstant(0, dl, NVT);
2507 }
2508 
2509 void DAGTypeLegalizer::ExpandIntRes_CTTZ(SDNode *N,
2510  SDValue &Lo, SDValue &Hi) {
2511  SDLoc dl(N);
2512  // cttz (HiLo) -> Lo != 0 ? cttz(Lo) : (cttz(Hi)+32)
2513  GetExpandedInteger(N->getOperand(0), Lo, Hi);
2514  EVT NVT = Lo.getValueType();
2515 
2516  SDValue LoNotZero = DAG.getSetCC(dl, getSetCCResultType(NVT), Lo,
2517  DAG.getConstant(0, dl, NVT), ISD::SETNE);
2518 
2519  SDValue LoLZ = DAG.getNode(ISD::CTTZ_ZERO_UNDEF, dl, NVT, Lo);
2520  SDValue HiLZ = DAG.getNode(N->getOpcode(), dl, NVT, Hi);
2521 
2522  Lo = DAG.getSelect(dl, NVT, LoNotZero, LoLZ,
2523  DAG.getNode(ISD::ADD, dl, NVT, HiLZ,
2524  DAG.getConstant(NVT.getSizeInBits(), dl,
2525  NVT)));
2526  Hi = DAG.getConstant(0, dl, NVT);
2527 }
2528 
2529 void DAGTypeLegalizer::ExpandIntRes_FLT_ROUNDS(SDNode *N, SDValue &Lo,
2530  SDValue &Hi) {
2531  SDLoc dl(N);
2532  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
2533  unsigned NBitWidth = NVT.getSizeInBits();
2534 
2535  EVT ShiftAmtTy = TLI.getShiftAmountTy(NVT, DAG.getDataLayout());
2536  Lo = DAG.getNode(ISD::FLT_ROUNDS_, dl, NVT);
2537  // The high part is the sign of Lo, as -1 is a valid value for FLT_ROUNDS
2538  Hi = DAG.getNode(ISD::SRA, dl, NVT, Lo,
2539  DAG.getConstant(NBitWidth - 1, dl, ShiftAmtTy));
2540 }
2541 
2542 void DAGTypeLegalizer::ExpandIntRes_FP_TO_SINT(SDNode *N, SDValue &Lo,
2543  SDValue &Hi) {
2544  SDLoc dl(N);
2545  EVT VT = N->getValueType(0);
2546 
2547  SDValue Op = N->getOperand(0);
2548  if (getTypeAction(Op.getValueType()) == TargetLowering::TypePromoteFloat)
2549  Op = GetPromotedFloat(Op);
2550 
2552  assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected fp-to-sint conversion!");
2554  CallOptions.setSExt(true);
2555  SplitInteger(TLI.makeLibCall(DAG, LC, VT, Op, CallOptions, dl).first,
2556  Lo, Hi);
2557 }
2558 
2559 void DAGTypeLegalizer::ExpandIntRes_FP_TO_UINT(SDNode *N, SDValue &Lo,
2560  SDValue &Hi) {
2561  SDLoc dl(N);
2562  EVT VT = N->getValueType(0);
2563 
2564  SDValue Op = N->getOperand(0);
2565  if (getTypeAction(Op.getValueType()) == TargetLowering::TypePromoteFloat)
2566  Op = GetPromotedFloat(Op);
2567 
2569  assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected fp-to-uint conversion!");
2571  SplitInteger(TLI.makeLibCall(DAG, LC, VT, Op, CallOptions, dl).first,
2572  Lo, Hi);
2573 }
2574 
2575 void DAGTypeLegalizer::ExpandIntRes_LLROUND(SDNode *N, SDValue &Lo,
2576  SDValue &Hi) {
2577  RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
2579  if (VT == MVT::f32)
2580  LC = RTLIB::LLROUND_F32;
2581  else if (VT == MVT::f64)
2582  LC = RTLIB::LLROUND_F64;
2583  else if (VT == MVT::f80)
2584  LC = RTLIB::LLROUND_F80;
2585  else if (VT == MVT::f128)
2586  LC = RTLIB::LLROUND_F128;
2587  else if (VT == MVT::ppcf128)
2588  LC = RTLIB::LLROUND_PPCF128;
2589  assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected llround input type!");
2590 
2591  SDValue Op = N->getOperand(0);
2592  if (getTypeAction(Op.getValueType()) == TargetLowering::TypePromoteFloat)
2593  Op = GetPromotedFloat(Op);
2594 
2595  SDLoc dl(N);
2596  EVT RetVT = N->getValueType(0);
2598  CallOptions.setSExt(true);
2599  SplitInteger(TLI.makeLibCall(DAG, LC, RetVT, Op, CallOptions, dl).first,
2600  Lo, Hi);
2601 }
2602 
2603 void DAGTypeLegalizer::ExpandIntRes_LLRINT(SDNode *N, SDValue &Lo,
2604  SDValue &Hi) {
2605  RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
2607  if (VT == MVT::f32)
2608  LC = RTLIB::LLRINT_F32;
2609  else if (VT == MVT::f64)
2610  LC = RTLIB::LLRINT_F64;
2611  else if (VT == MVT::f80)
2612  LC = RTLIB::LLRINT_F80;
2613  else if (VT == MVT::f128)
2614  LC = RTLIB::LLRINT_F128;
2615  else if (VT == MVT::ppcf128)
2616  LC = RTLIB::LLRINT_PPCF128;
2617  assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected llrint input type!");
2618 
2619  SDValue Op = N->getOperand(0);
2620  if (getTypeAction(Op.getValueType()) == TargetLowering::TypePromoteFloat)
2621  Op = GetPromotedFloat(Op);
2622 
2623  SDLoc dl(N);
2624  EVT RetVT = N->getValueType(0);
2626  CallOptions.setSExt(true);
2627  SplitInteger(TLI.makeLibCall(DAG, LC, RetVT, Op, CallOptions, dl).first,
2628  Lo, Hi);
2629 }
2630 
2631 void DAGTypeLegalizer::ExpandIntRes_LOAD(LoadSDNode *N,
2632  SDValue &Lo, SDValue &Hi) {
2633  if (ISD::isNormalLoad(N)) {
2634  ExpandRes_NormalLoad(N, Lo, Hi);
2635  return;
2636  }
2637 
2638  assert(ISD::isUNINDEXEDLoad(N) && "Indexed load during type legalization!");
2639 
2640  EVT VT = N->getValueType(0);
2641  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
2642  SDValue Ch = N->getChain();
2643  SDValue Ptr = N->getBasePtr();
2645  unsigned Alignment = N->getAlignment();
2646  MachineMemOperand::Flags MMOFlags = N->getMemOperand()->getFlags();
2647  AAMDNodes AAInfo = N->getAAInfo();
2648  SDLoc dl(N);
2649 
2650  assert(NVT.isByteSized() && "Expanded type not byte sized!");
2651 
2652  if (N->getMemoryVT().bitsLE(NVT)) {
2653  EVT MemVT = N->getMemoryVT();
2654 
2655  Lo = DAG.getExtLoad(ExtType, dl, NVT, Ch, Ptr, N->getPointerInfo(), MemVT,
2656  Alignment, MMOFlags, AAInfo);
2657 
2658  // Remember the chain.
2659  Ch = Lo.getValue(1);
2660 
2661  if (ExtType == ISD::SEXTLOAD) {
2662  // The high part is obtained by SRA'ing all but one of the bits of the
2663  // lo part.
2664  unsigned LoSize = Lo.getValueSizeInBits();
2665  Hi = DAG.getNode(ISD::SRA, dl, NVT, Lo,
2666  DAG.getConstant(LoSize - 1, dl,
2667  TLI.getPointerTy(DAG.getDataLayout())));
2668  } else if (ExtType == ISD::ZEXTLOAD) {
2669  // The high part is just a zero.
2670  Hi = DAG.getConstant(0, dl, NVT);
2671  } else {
2672  assert(ExtType == ISD::EXTLOAD && "Unknown extload!");
2673  // The high part is undefined.
2674  Hi = DAG.getUNDEF(NVT);
2675  }
2676  } else if (DAG.getDataLayout().isLittleEndian()) {
2677  // Little-endian - low bits are at low addresses.
2678  Lo = DAG.getLoad(NVT, dl, Ch, Ptr, N->getPointerInfo(), Alignment, MMOFlags,
2679  AAInfo);
2680 
2681  unsigned ExcessBits =
2682  N->getMemoryVT().getSizeInBits() - NVT.getSizeInBits();
2683  EVT NEVT = EVT::getIntegerVT(*DAG.getContext(), ExcessBits);
2684 
2685  // Increment the pointer to the other half.
2686  unsigned IncrementSize = NVT.getSizeInBits()/8;
2687  Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr,
2688  DAG.getConstant(IncrementSize, dl, Ptr.getValueType()));
2689  Hi = DAG.getExtLoad(ExtType, dl, NVT, Ch, Ptr,
2690  N->getPointerInfo().getWithOffset(IncrementSize), NEVT,
2691  MinAlign(Alignment, IncrementSize), MMOFlags, AAInfo);
2692 
2693  // Build a factor node to remember that this load is independent of the
2694  // other one.
2695  Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1),
2696  Hi.getValue(1));
2697  } else {
2698  // Big-endian - high bits are at low addresses. Favor aligned loads at
2699  // the cost of some bit-fiddling.
2700  EVT MemVT = N->getMemoryVT();
2701  unsigned EBytes = MemVT.getStoreSize();
2702  unsigned IncrementSize = NVT.getSizeInBits()/8;
2703  unsigned ExcessBits = (EBytes - IncrementSize)*8;
2704 
2705  // Load both the high bits and maybe some of the low bits.
2706  Hi = DAG.getExtLoad(ExtType, dl, NVT, Ch, Ptr, N->getPointerInfo(),
2708  MemVT.getSizeInBits() - ExcessBits),
2709  Alignment, MMOFlags, AAInfo);
2710 
2711  // Increment the pointer to the other half.
2712  Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr,
2713  DAG.getConstant(IncrementSize, dl, Ptr.getValueType()));
2714  // Load the rest of the low bits.
2715  Lo = DAG.getExtLoad(ISD::ZEXTLOAD, dl, NVT, Ch, Ptr,
2716  N->getPointerInfo().getWithOffset(IncrementSize),
2717  EVT::getIntegerVT(*DAG.getContext(), ExcessBits),
2718  MinAlign(Alignment, IncrementSize), MMOFlags, AAInfo);
2719 
2720  // Build a factor node to remember that this load is independent of the
2721  // other one.
2722  Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1),
2723  Hi.getValue(1));
2724 
2725  if (ExcessBits < NVT.getSizeInBits()) {
2726  // Transfer low bits from the bottom of Hi to the top of Lo.
2727  Lo = DAG.getNode(
2728  ISD::OR, dl, NVT, Lo,
2729  DAG.getNode(ISD::SHL, dl, NVT, Hi,
2730  DAG.getConstant(ExcessBits, dl,
2731  TLI.getPointerTy(DAG.getDataLayout()))));
2732  // Move high bits to the right position in Hi.
2733  Hi = DAG.getNode(ExtType == ISD::SEXTLOAD ? ISD::SRA : ISD::SRL, dl, NVT,
2734  Hi,
2735  DAG.getConstant(NVT.getSizeInBits() - ExcessBits, dl,
2736  TLI.getPointerTy(DAG.getDataLayout())));
2737  }
2738  }
2739 
2740  // Legalize the chain result - switch anything that used the old chain to
2741  // use the new one.
2742  ReplaceValueWith(SDValue(N, 1), Ch);
2743 }
2744 
2745 void DAGTypeLegalizer::ExpandIntRes_Logical(SDNode *N,
2746  SDValue &Lo, SDValue &Hi) {
2747  SDLoc dl(N);
2748  SDValue LL, LH, RL, RH;
2749  GetExpandedInteger(N->getOperand(0), LL, LH);
2750  GetExpandedInteger(N->getOperand(1), RL, RH);
2751  Lo = DAG.getNode(N->getOpcode(), dl, LL.getValueType(), LL, RL);
2752  Hi = DAG.getNode(N->getOpcode(), dl, LL.getValueType(), LH, RH);
2753 }
2754 
2755 void DAGTypeLegalizer::ExpandIntRes_MUL(SDNode *N,
2756  SDValue &Lo, SDValue &Hi) {
2757  EVT VT = N->getValueType(0);
2758  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
2759  SDLoc dl(N);
2760 
2761  SDValue LL, LH, RL, RH;
2762  GetExpandedInteger(N->getOperand(0), LL, LH);
2763  GetExpandedInteger(N->getOperand(1), RL, RH);
2764 
2765  if (TLI.expandMUL(N, Lo, Hi, NVT, DAG,
2767  LL, LH, RL, RH))
2768  return;
2769 
2770  // If nothing else, we can make a libcall.
2771  RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
2772  if (VT == MVT::i16)
2773  LC = RTLIB::MUL_I16;
2774  else if (VT == MVT::i32)
2775  LC = RTLIB::MUL_I32;
2776  else if (VT == MVT::i64)
2777  LC = RTLIB::MUL_I64;
2778  else if (VT == MVT::i128)
2779  LC = RTLIB::MUL_I128;
2780 
2781  if (LC == RTLIB::UNKNOWN_LIBCALL || !TLI.getLibcallName(LC)) {
2782  // We'll expand the multiplication by brute force because we have no other
2783  // options. This is a trivially-generalized version of the code from
2784  // Hacker's Delight (itself derived from Knuth's Algorithm M from section
2785  // 4.3.1).
2786  unsigned Bits = NVT.getSizeInBits();
2787  unsigned HalfBits = Bits >> 1;
2788  SDValue Mask = DAG.getConstant(APInt::getLowBitsSet(Bits, HalfBits), dl,
2789  NVT);
2790  SDValue LLL = DAG.getNode(ISD::AND, dl, NVT, LL, Mask);
2791  SDValue RLL = DAG.getNode(ISD::AND, dl, NVT, RL, Mask);
2792 
2793  SDValue T = DAG.getNode(ISD::MUL, dl, NVT, LLL, RLL);
2794  SDValue TL = DAG.getNode(ISD::AND, dl, NVT, T, Mask);
2795 
2796  EVT ShiftAmtTy = TLI.getShiftAmountTy(NVT, DAG.getDataLayout());
2797  if (APInt::getMaxValue(ShiftAmtTy.getSizeInBits()).ult(HalfBits)) {
2798  // The type from TLI is too small to fit the shift amount we want.
2799  // Override it with i32. The shift will have to be legalized.
2800  ShiftAmtTy = MVT::i32;
2801  }
2802  SDValue Shift = DAG.getConstant(HalfBits, dl, ShiftAmtTy);
2803  SDValue TH = DAG.getNode(ISD::SRL, dl, NVT, T, Shift);
2804  SDValue LLH = DAG.getNode(ISD::SRL, dl, NVT, LL, Shift);
2805  SDValue RLH = DAG.getNode(ISD::SRL, dl, NVT, RL, Shift);
2806 
2807  SDValue U = DAG.getNode(ISD::ADD, dl, NVT,
2808  DAG.getNode(ISD::MUL, dl, NVT, LLH, RLL), TH);
2809  SDValue UL = DAG.getNode(ISD::AND, dl, NVT, U, Mask);
2810  SDValue UH = DAG.getNode(ISD::SRL, dl, NVT, U, Shift);
2811 
2812  SDValue V = DAG.getNode(ISD::ADD, dl, NVT,
2813  DAG.getNode(ISD::MUL, dl, NVT, LLL, RLH), UL);
2814  SDValue VH = DAG.getNode(ISD::SRL, dl, NVT, V, Shift);
2815 
2816  SDValue W = DAG.getNode(ISD::ADD, dl, NVT,
2817  DAG.getNode(ISD::MUL, dl, NVT, LLH, RLH),
2818  DAG.getNode(ISD::ADD, dl, NVT, UH, VH));
2819  Lo = DAG.getNode(ISD::ADD, dl, NVT, TL,
2820  DAG.getNode(ISD::SHL, dl, NVT, V, Shift));
2821 
2822  Hi = DAG.getNode(ISD::ADD, dl, NVT, W,
2823  DAG.getNode(ISD::ADD, dl, NVT,
2824  DAG.getNode(ISD::MUL, dl, NVT, RH, LL),
2825  DAG.getNode(ISD::MUL, dl, NVT, RL, LH)));
2826  return;
2827  }
2828 
2829  SDValue Ops[2] = { N->getOperand(0), N->getOperand(1) };
2831  CallOptions.setSExt(true);
2832  SplitInteger(TLI.makeLibCall(DAG, LC, VT, Ops, CallOptions, dl).first,
2833  Lo, Hi);
2834 }
2835 
2836 void DAGTypeLegalizer::ExpandIntRes_READCYCLECOUNTER(SDNode *N, SDValue &Lo,
2837  SDValue &Hi) {
2838  SDLoc DL(N);
2839  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
2840  SDVTList VTs = DAG.getVTList(NVT, NVT, MVT::Other);
2841  SDValue R = DAG.getNode(N->getOpcode(), DL, VTs, N->getOperand(0));
2842  Lo = R.getValue(0);
2843  Hi = R.getValue(1);
2844  ReplaceValueWith(SDValue(N, 1), R.getValue(2));
2845 }
2846 
2847 void DAGTypeLegalizer::ExpandIntRes_ADDSUBSAT(SDNode *N, SDValue &Lo,
2848  SDValue &Hi) {
2849  SDValue Result = TLI.expandAddSubSat(N, DAG);
2850  SplitInteger(Result, Lo, Hi);
2851 }
2852 
2853 /// This performs an expansion of the integer result for a fixed point
2854 /// multiplication. The default expansion performs rounding down towards
2855 /// negative infinity, though targets that do care about rounding should specify
2856 /// a target hook for rounding and provide their own expansion or lowering of
2857 /// fixed point multiplication to be consistent with rounding.
2858 void DAGTypeLegalizer::ExpandIntRes_MULFIX(SDNode *N, SDValue &Lo,
2859  SDValue &Hi) {
2860  SDLoc dl(N);
2861  EVT VT = N->getValueType(0);
2862  unsigned VTSize = VT.getScalarSizeInBits();
2863  SDValue LHS = N->getOperand(0);
2864  SDValue RHS = N->getOperand(1);
2865  uint64_t Scale = N->getConstantOperandVal(2);
2866  bool Saturating = (N->getOpcode() == ISD::SMULFIXSAT ||
2867  N->getOpcode() == ISD::UMULFIXSAT);
2868  bool Signed = (N->getOpcode() == ISD::SMULFIX ||
2869  N->getOpcode() == ISD::SMULFIXSAT);
2870 
2871  // Handle special case when scale is equal to zero.
2872  if (!Scale) {
2873  SDValue Result;
2874  if (!Saturating) {
2875  Result = DAG.getNode(ISD::MUL, dl, VT, LHS, RHS);
2876  } else {
2877  EVT BoolVT = getSetCCResultType(VT);
2878  unsigned MulOp = Signed ? ISD::SMULO : ISD::UMULO;
2879  Result = DAG.getNode(MulOp, dl, DAG.getVTList(VT, BoolVT), LHS, RHS);
2880  SDValue Product = Result.getValue(0);
2881  SDValue Overflow = Result.getValue(1);
2882  if (Signed) {
2883  APInt MinVal = APInt::getSignedMinValue(VTSize);
2884  APInt MaxVal = APInt::getSignedMaxValue(VTSize);
2885  SDValue SatMin = DAG.getConstant(MinVal, dl, VT);
2886  SDValue SatMax = DAG.getConstant(MaxVal, dl, VT);
2887  SDValue Zero = DAG.getConstant(0, dl, VT);
2888  SDValue ProdNeg = DAG.getSetCC(dl, BoolVT, Product, Zero, ISD::SETLT);
2889  Result = DAG.getSelect(dl, VT, ProdNeg, SatMax, SatMin);
2890  Result = DAG.getSelect(dl, VT, Overflow, Result, Product);
2891  } else {
2892  // For unsigned multiplication, we only need to check the max since we
2893  // can't really overflow towards zero.
2894  APInt MaxVal = APInt::getMaxValue(VTSize);
2895  SDValue SatMax = DAG.getConstant(MaxVal, dl, VT);
2896  Result = DAG.getSelect(dl, VT, Overflow, SatMax, Product);
2897  }
2898  }
2899  SplitInteger(Result, Lo, Hi);
2900  return;
2901  }
2902 
2903  // For SMULFIX[SAT] we only expect to find Scale<VTSize, but this assert will
2904  // cover for unhandled cases below, while still being valid for UMULFIX[SAT].
2905  assert(Scale <= VTSize && "Scale can't be larger than the value type size.");
2906 
2907  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
2908  SDValue LL, LH, RL, RH;
2909  GetExpandedInteger(LHS, LL, LH);
2910  GetExpandedInteger(RHS, RL, RH);
2911  SmallVector<SDValue, 4> Result;
2912 
2913  unsigned LoHiOp = Signed ? ISD::SMUL_LOHI : ISD::UMUL_LOHI;
2914  if (!TLI.expandMUL_LOHI(LoHiOp, VT, dl, LHS, RHS, Result, NVT, DAG,
2916  LL, LH, RL, RH)) {
2917  report_fatal_error("Unable to expand MUL_FIX using MUL_LOHI.");
2918  return;
2919  }
2920 
2921  unsigned NVTSize = NVT.getScalarSizeInBits();
2922  assert((VTSize == NVTSize * 2) && "Expected the new value type to be half "
2923  "the size of the current value type");
2924  EVT ShiftTy = TLI.getShiftAmountTy(NVT, DAG.getDataLayout());
2925 
2926  // After getting the multiplication result in 4 parts, we need to perform a
2927  // shift right by the amount of the scale to get the result in that scale.
2928  //
2929  // Let's say we multiply 2 64 bit numbers. The resulting value can be held in
2930  // 128 bits that are cut into 4 32-bit parts:
2931  //
2932  // HH HL LH LL
2933  // |---32---|---32---|---32---|---32---|
2934  // 128 96 64 32 0
2935  //
2936  // |------VTSize-----|
2937  //
2938  // |NVTSize-|
2939  //
2940  // The resulting Lo and Hi would normally be in LL and LH after the shift. But
2941  // to avoid unneccessary shifting of all 4 parts, we can adjust the shift
2942  // amount and get Lo and Hi using two funnel shifts. Or for the special case
2943  // when Scale is a multiple of NVTSize we can just pick the result without
2944  // shifting.
2945  uint64_t Part0 = Scale / NVTSize; // Part holding lowest bit needed.
2946  if (Scale % NVTSize) {
2947  SDValue ShiftAmount = DAG.getConstant(Scale % NVTSize, dl, ShiftTy);
2948  Lo = DAG.getNode(ISD::FSHR, dl, NVT, Result[Part0 + 1], Result[Part0],
2949  ShiftAmount);
2950  Hi = DAG.getNode(ISD::FSHR, dl, NVT, Result[Part0 + 2], Result[Part0 + 1],
2951  ShiftAmount);
2952  } else {
2953  Lo = Result[Part0];
2954  Hi = Result[Part0 + 1];
2955  }
2956 
2957  // Unless saturation is requested we are done. The result is in <Hi,Lo>.
2958  if (!Saturating)
2959  return;
2960 
2961  // Can not overflow when there is no integer part.
2962  if (Scale == VTSize)
2963  return;
2964 
2965  // To handle saturation we must check for overflow in the multiplication.
2966  //
2967  // Unsigned overflow happened if the upper (VTSize - Scale) bits (of Result)
2968  // aren't all zeroes.
2969  //
2970  // Signed overflow happened if the upper (VTSize - Scale + 1) bits (of Result)
2971  // aren't all ones or all zeroes.
2972  //
2973  // We cannot overflow past HH when multiplying 2 ints of size VTSize, so the
2974  // highest bit of HH determines saturation direction in the event of signed
2975  // saturation.
2976 
2977  SDValue ResultHL = Result[2];
2978  SDValue ResultHH = Result[3];
2979 
2980  SDValue SatMax, SatMin;
2981  SDValue NVTZero = DAG.getConstant(0, dl, NVT);
2982  SDValue NVTNeg1 = DAG.getConstant(-1, dl, NVT);
2983  EVT BoolNVT = getSetCCResultType(NVT);
2984 
2985  if (!Signed) {
2986  if (Scale < NVTSize) {
2987  // Overflow happened if ((HH | (HL >> Scale)) != 0).
2988  SDValue HLAdjusted = DAG.getNode(ISD::SRL, dl, NVT, ResultHL,
2989  DAG.getConstant(Scale, dl, ShiftTy));
2990  SDValue Tmp = DAG.getNode(ISD::OR, dl, NVT, HLAdjusted, ResultHH);
2991  SatMax = DAG.getSetCC(dl, BoolNVT, Tmp, NVTZero, ISD::SETNE);
2992  } else if (Scale == NVTSize) {
2993  // Overflow happened if (HH != 0).
2994  SatMax = DAG.getSetCC(dl, BoolNVT, ResultHH, NVTZero, ISD::SETNE);
2995  } else if (Scale < VTSize) {
2996  // Overflow happened if ((HH >> (Scale - NVTSize)) != 0).
2997  SDValue HLAdjusted = DAG.getNode(ISD::SRL, dl, NVT, ResultHL,
2998  DAG.getConstant(Scale - NVTSize, dl,
2999  ShiftTy));
3000  SatMax = DAG.getSetCC(dl, BoolNVT, HLAdjusted, NVTZero, ISD::SETNE);
3001  } else
3002  llvm_unreachable("Scale must be less or equal to VTSize for UMULFIXSAT"
3003  "(and saturation can't happen with Scale==VTSize).");
3004 
3005  Hi = DAG.getSelect(dl, NVT, SatMax, NVTNeg1, Hi);
3006  Lo = DAG.getSelect(dl, NVT, SatMax, NVTNeg1, Lo);
3007  return;
3008  }
3009 
3010  if (Scale < NVTSize) {
3011  // The number of overflow bits we can check are VTSize - Scale + 1 (we
3012  // include the sign bit). If these top bits are > 0, then we overflowed past
3013  // the max value. If these top bits are < -1, then we overflowed past the
3014  // min value. Otherwise, we did not overflow.
3015  unsigned OverflowBits = VTSize - Scale + 1;
3016  assert(OverflowBits <= VTSize && OverflowBits > NVTSize &&
3017  "Extent of overflow bits must start within HL");
3018  SDValue HLHiMask = DAG.getConstant(
3019  APInt::getHighBitsSet(NVTSize, OverflowBits - NVTSize), dl, NVT);
3020  SDValue HLLoMask = DAG.getConstant(
3021  APInt::getLowBitsSet(NVTSize, VTSize - OverflowBits), dl, NVT);
3022  // We overflow max if HH > 0 or (HH == 0 && HL > HLLoMask).
3023  SDValue HHGT0 = DAG.getSetCC(dl, BoolNVT, ResultHH, NVTZero, ISD::SETGT);
3024  SDValue HHEQ0 = DAG.getSetCC(dl, BoolNVT, ResultHH, NVTZero, ISD::SETEQ);
3025  SDValue HLUGT = DAG.getSetCC(dl, BoolNVT, ResultHL, HLLoMask, ISD::SETUGT);
3026  SatMax = DAG.getNode(ISD::OR, dl, BoolNVT, HHGT0,
3027  DAG.getNode(ISD::AND, dl, BoolNVT, HHEQ0, HLUGT));
3028  // We overflow min if HH < -1 or (HH == -1 && HL < HLHiMask).
3029  SDValue HHLT = DAG.getSetCC(dl, BoolNVT, ResultHH, NVTNeg1, ISD::SETLT);
3030  SDValue HHEQ = DAG.getSetCC(dl, BoolNVT, ResultHH, NVTNeg1, ISD::SETEQ);
3031  SDValue HLULT = DAG.getSetCC(dl, BoolNVT, ResultHL, HLHiMask, ISD::SETULT);
3032  SatMin = DAG.getNode(ISD::OR, dl, BoolNVT, HHLT,
3033  DAG.getNode(ISD::AND, dl, BoolNVT, HHEQ, HLULT));
3034  } else if (Scale == NVTSize) {
3035  // We overflow max if HH > 0 or (HH == 0 && HL sign bit is 1).
3036  SDValue HHGT0 = DAG.getSetCC(dl, BoolNVT, ResultHH, NVTZero, ISD::SETGT);
3037  SDValue HHEQ0 = DAG.getSetCC(dl, BoolNVT, ResultHH, NVTZero, ISD::SETEQ);
3038  SDValue HLNeg = DAG.getSetCC(dl, BoolNVT, ResultHL, NVTZero, ISD::SETLT);
3039  SatMax = DAG.getNode(ISD::OR, dl, BoolNVT, HHGT0,
3040  DAG.getNode(ISD::AND, dl, BoolNVT, HHEQ0, HLNeg));
3041  // We overflow min if HH < -1 or (HH == -1 && HL sign bit is 0).
3042  SDValue HHLT = DAG.getSetCC(dl, BoolNVT, ResultHH, NVTNeg1, ISD::SETLT);
3043  SDValue HHEQ = DAG.getSetCC(dl, BoolNVT, ResultHH, NVTNeg1, ISD::SETEQ);
3044  SDValue HLPos = DAG.getSetCC(dl, BoolNVT, ResultHL, NVTZero, ISD::SETGE);
3045  SatMin = DAG.getNode(ISD::OR, dl, BoolNVT, HHLT,
3046  DAG.getNode(ISD::AND, dl, BoolNVT, HHEQ, HLPos));
3047  } else if (Scale < VTSize) {
3048  // This is similar to the case when we saturate if Scale < NVTSize, but we
3049  // only need to check HH.
3050  unsigned OverflowBits = VTSize - Scale + 1;
3051  SDValue HHHiMask = DAG.getConstant(
3052  APInt::getHighBitsSet(NVTSize, OverflowBits), dl, NVT);
3053  SDValue HHLoMask = DAG.getConstant(
3054  APInt::getLowBitsSet(NVTSize, NVTSize - OverflowBits), dl, NVT);
3055  SatMax = DAG.getSetCC(dl, BoolNVT, ResultHH, HHLoMask, ISD::SETGT);
3056  SatMin = DAG.getSetCC(dl, BoolNVT, ResultHH, HHHiMask, ISD::SETLT);
3057  } else
3058  llvm_unreachable("Illegal scale for signed fixed point mul.");
3059 
3060  // Saturate to signed maximum.
3061  APInt MaxHi = APInt::getSignedMaxValue(NVTSize);
3062  APInt MaxLo = APInt::getAllOnesValue(NVTSize);
3063  Hi = DAG.getSelect(dl, NVT, SatMax, DAG.getConstant(MaxHi, dl, NVT), Hi);
3064  Lo = DAG.getSelect(dl, NVT, SatMax, DAG.getConstant(MaxLo, dl, NVT), Lo);
3065  // Saturate to signed minimum.
3066  APInt MinHi = APInt::getSignedMinValue(NVTSize);
3067  Hi = DAG.getSelect(dl, NVT, SatMin, DAG.getConstant(MinHi, dl, NVT), Hi);
3068  Lo = DAG.getSelect(dl, NVT, SatMin, NVTZero, Lo);
3069 }
3070 
3071 void DAGTypeLegalizer::ExpandIntRes_SADDSUBO(SDNode *Node,
3072  SDValue &Lo, SDValue &Hi) {
3073  SDValue LHS = Node->getOperand(0);
3074  SDValue RHS = Node->getOperand(1);
3075  SDLoc dl(Node);
3076 
3077  // Expand the result by simply replacing it with the equivalent
3078  // non-overflow-checking operation.
3079  SDValue Sum = DAG.getNode(Node->getOpcode() == ISD::SADDO ?
3080  ISD::ADD : ISD::SUB, dl, LHS.getValueType(),
3081  LHS, RHS);
3082  SplitInteger(Sum, Lo, Hi);
3083 
3084  // Compute the overflow.
3085  //
3086  // LHSSign -> LHS >= 0
3087  // RHSSign -> RHS >= 0
3088  // SumSign -> Sum >= 0
3089  //
3090  // Add:
3091  // Overflow -> (LHSSign == RHSSign) && (LHSSign != SumSign)
3092  // Sub:
3093  // Overflow -> (LHSSign != RHSSign) && (LHSSign != SumSign)
3094  //
3095  EVT OType = Node->getValueType(1);
3096  SDValue Zero = DAG.getConstant(0, dl, LHS.getValueType());
3097 
3098  SDValue LHSSign = DAG.getSetCC(dl, OType, LHS, Zero, ISD::SETGE);
3099  SDValue RHSSign = DAG.getSetCC(dl, OType, RHS, Zero, ISD::SETGE);
3100  SDValue SignsMatch = DAG.getSetCC(dl, OType, LHSSign, RHSSign,
3101  Node->getOpcode() == ISD::SADDO ?
3103 
3104  SDValue SumSign = DAG.getSetCC(dl, OType, Sum, Zero, ISD::SETGE);
3105  SDValue SumSignNE = DAG.getSetCC(dl, OType, LHSSign, SumSign, ISD::SETNE);
3106 
3107  SDValue Cmp = DAG.getNode(ISD::AND, dl, OType, SignsMatch, SumSignNE);
3108 
3109  // Use the calculated overflow everywhere.
3110  ReplaceValueWith(SDValue(Node, 1), Cmp);
3111 }
3112 
3113 void DAGTypeLegalizer::ExpandIntRes_SDIV(SDNode *N,
3114  SDValue &Lo, SDValue &Hi) {
3115  EVT VT = N->getValueType(0);
3116  SDLoc dl(N);
3117  SDValue Ops[2] = { N->getOperand(0), N->getOperand(1) };
3118 
3120  SDValue Res = DAG.getNode(ISD::SDIVREM, dl, DAG.getVTList(VT, VT), Ops);
3121  SplitInteger(Res.getValue(0), Lo, Hi);
3122  return;
3123  }
3124 
3125  RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
3126  if (VT == MVT::i16)
3127  LC = RTLIB::SDIV_I16;
3128  else if (VT == MVT::i32)
3129  LC = RTLIB::SDIV_I32;
3130  else if (VT == MVT::i64)
3131  LC = RTLIB::SDIV_I64;
3132  else if (VT == MVT::i128)
3133  LC = RTLIB::SDIV_I128;
3134  assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported SDIV!");
3135 
3137  CallOptions.setSExt(true);
3138  SplitInteger(TLI.makeLibCall(DAG, LC, VT, Ops, CallOptions, dl).first, Lo, Hi);
3139 }
3140 
3141 void DAGTypeLegalizer::ExpandIntRes_Shift(SDNode *N,
3142  SDValue &Lo, SDValue &Hi) {
3143  EVT VT = N->getValueType(0);
3144  SDLoc dl(N);
3145 
3146  // If we can emit an efficient shift operation, do so now. Check to see if
3147  // the RHS is a constant.
3148  if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(N->getOperand(1)))
3149  return ExpandShiftByConstant(N, CN->getAPIntValue(), Lo, Hi);
3150 
3151  // If we can determine that the high bit of the shift is zero or one, even if
3152  // the low bits are variable, emit this shift in an optimized form.
3153  if (ExpandShiftWithKnownAmountBit(N, Lo, Hi))
3154  return;
3155 
3156  // If this target supports shift_PARTS, use it. First, map to the _PARTS opc.
3157  unsigned PartsOpc;
3158  if (N->getOpcode() == ISD::SHL) {
3159  PartsOpc = ISD::SHL_PARTS;
3160  } else if (N->getOpcode() == ISD::SRL) {
3161  PartsOpc = ISD::SRL_PARTS;
3162  } else {
3163  assert(N->getOpcode() == ISD::SRA && "Unknown shift!");
3164  PartsOpc = ISD::SRA_PARTS;
3165  }
3166 
3167  // Next check to see if the target supports this SHL_PARTS operation or if it
3168  // will custom expand it. Don't lower this to SHL_PARTS when we optimise for
3169  // size, but create a libcall instead.
3170  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
3171  TargetLowering::LegalizeAction Action = TLI.getOperationAction(PartsOpc, NVT);
3172  const bool LegalOrCustom =
3173  (Action == TargetLowering::Legal && TLI.isTypeLegal(NVT)) ||
3174  Action == TargetLowering::Custom;
3175 
3176  if (LegalOrCustom && TLI.shouldExpandShift(DAG, N)) {
3177  // Expand the subcomponents.
3178  SDValue LHSL, LHSH;
3179  GetExpandedInteger(N->getOperand(0), LHSL, LHSH);
3180  EVT VT = LHSL.getValueType();
3181 
3182  // If the shift amount operand is coming from a vector legalization it may
3183  // have an illegal type. Fix that first by casting the operand, otherwise
3184  // the new SHL_PARTS operation would need further legalization.
3185  SDValue ShiftOp = N->getOperand(1);
3186  EVT ShiftTy = TLI.getShiftAmountTy(VT, DAG.getDataLayout());
3187  assert(ShiftTy.getScalarSizeInBits() >=
3188  Log2_32_Ceil(VT.getScalarSizeInBits()) &&
3189  "ShiftAmountTy is too small to cover the range of this type!");
3190  if (ShiftOp.getValueType() != ShiftTy)
3191  ShiftOp = DAG.getZExtOrTrunc(ShiftOp, dl, ShiftTy);
3192 
3193  SDValue Ops[] = { LHSL, LHSH, ShiftOp };
3194  Lo = DAG.getNode(PartsOpc, dl, DAG.getVTList(VT, VT), Ops);
3195  Hi = Lo.getValue(1);
3196  return;
3197  }
3198 
3199  // Otherwise, emit a libcall.
3200  RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
3201  bool isSigned;
3202  if (N->getOpcode() == ISD::SHL) {
3203  isSigned = false; /*sign irrelevant*/
3204  if (VT == MVT::i16)
3205  LC = RTLIB::SHL_I16;
3206  else if (VT == MVT::i32)
3207  LC = RTLIB::SHL_I32;
3208  else if (VT == MVT::i64)
3209  LC = RTLIB::SHL_I64;
3210  else if (VT == MVT::i128)
3211  LC = RTLIB::SHL_I128;
3212  } else if (N->getOpcode() == ISD::SRL) {
3213  isSigned = false;
3214  if (VT == MVT::i16)
3215  LC = RTLIB::SRL_I16;
3216  else if (VT == MVT::i32)
3217  LC = RTLIB::SRL_I32;
3218  else if (VT == MVT::i64)
3219  LC = RTLIB::SRL_I64;
3220  else if (VT == MVT::i128)
3221  LC = RTLIB::SRL_I128;
3222  } else {
3223  assert(N->getOpcode() == ISD::SRA && "Unknown shift!");
3224  isSigned = true;
3225  if (VT == MVT::i16)
3226  LC = RTLIB::SRA_I16;
3227  else if (VT == MVT::i32)
3228  LC = RTLIB::SRA_I32;
3229  else if (VT == MVT::i64)
3230  LC = RTLIB::SRA_I64;
3231  else if (VT == MVT::i128)
3232  LC = RTLIB::SRA_I128;
3233  }
3234 
3235  if (LC != RTLIB::UNKNOWN_LIBCALL && TLI.getLibcallName(LC)) {
3236  SDValue Ops[2] = { N->getOperand(0), N->getOperand(1) };
3238  CallOptions.setSExt(isSigned);
3239  SplitInteger(TLI.makeLibCall(DAG, LC, VT, Ops, CallOptions, dl).first, Lo, Hi);
3240  return;
3241  }
3242 
3243  if (!ExpandShiftWithUnknownAmountBit(N, Lo, Hi))
3244  llvm_unreachable("Unsupported shift!");
3245 }
3246 
3247 void DAGTypeLegalizer::ExpandIntRes_SIGN_EXTEND(SDNode *N,
3248  SDValue &Lo, SDValue &Hi) {
3249  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
3250  SDLoc dl(N);
3251  SDValue Op = N->getOperand(0);
3252  if (Op.getValueType().bitsLE(NVT)) {
3253  // The low part is sign extension of the input (degenerates to a copy).
3254  Lo = DAG.getNode(ISD::SIGN_EXTEND, dl, NVT, N->getOperand(0));
3255  // The high part is obtained by SRA'ing all but one of the bits of low part.
3256  unsigned LoSize = NVT.getSizeInBits();
3257  Hi = DAG.getNode(
3258  ISD::SRA, dl, NVT, Lo,
3259  DAG.getConstant(LoSize - 1, dl, TLI.getPointerTy(DAG.getDataLayout())));
3260  } else {
3261  // For example, extension of an i48 to an i64. The operand type necessarily
3262  // promotes to the result type, so will end up being expanded too.
3263  assert(getTypeAction(Op.getValueType()) ==
3265  "Only know how to promote this result!");
3266  SDValue Res = GetPromotedInteger(Op);
3267  assert(Res.getValueType() == N->getValueType(0) &&
3268  "Operand over promoted?");
3269  // Split the promoted operand. This will simplify when it is expanded.
3270  SplitInteger(Res, Lo, Hi);
3271  unsigned ExcessBits = Op.getValueSizeInBits() - NVT.getSizeInBits();
3272  Hi = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, Hi.getValueType(), Hi,
3274  ExcessBits)));
3275  }
3276 }
3277 
3278 void DAGTypeLegalizer::
3279 ExpandIntRes_SIGN_EXTEND_INREG(SDNode *N, SDValue &Lo, SDValue &Hi) {
3280  SDLoc dl(N);
3281  GetExpandedInteger(N->getOperand(0), Lo, Hi);
3282  EVT EVT = cast<VTSDNode>(N->getOperand(1))->getVT();
3283 
3284  if (EVT.bitsLE(Lo.getValueType())) {
3285  // sext_inreg the low part if needed.
3286  Lo = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, Lo.getValueType(), Lo,
3287  N->getOperand(1));
3288 
3289  // The high part gets the sign extension from the lo-part. This handles
3290  // things like sextinreg V:i64 from i8.
3291  Hi = DAG.getNode(ISD::SRA, dl, Hi.getValueType(), Lo,
3292  DAG.getConstant(Hi.getValueSizeInBits() - 1, dl,
3293  TLI.getPointerTy(DAG.getDataLayout())));
3294  } else {
3295  // For example, extension of an i48 to an i64. Leave the low part alone,
3296  // sext_inreg the high part.
3297  unsigned ExcessBits = EVT.getSizeInBits() - Lo.getValueSizeInBits();
3298  Hi = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, Hi.getValueType(), Hi,
3300  ExcessBits)));
3301  }
3302 }
3303 
3304 void DAGTypeLegalizer::ExpandIntRes_SREM(SDNode *N,
3305  SDValue &Lo, SDValue &Hi) {
3306  EVT VT = N->getValueType(0);
3307  SDLoc dl(N);
3308  SDValue Ops[2] = { N->getOperand(0), N->getOperand(1) };
3309 
3311  SDValue Res = DAG.getNode(ISD::SDIVREM, dl, DAG.getVTList(VT, VT), Ops);
3312  SplitInteger(Res.getValue(1), Lo, Hi);
3313  return;
3314  }
3315 
3316  RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
3317  if (VT == MVT::i16)
3318  LC = RTLIB::SREM_I16;
3319  else if (VT == MVT::i32)
3320  LC = RTLIB::SREM_I32;
3321  else if (VT == MVT::i64)
3322  LC = RTLIB::SREM_I64;
3323  else if (VT == MVT::i128)
3324  LC = RTLIB::SREM_I128;
3325  assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported SREM!");
3326 
3328  CallOptions.setSExt(true);
3329  SplitInteger(TLI.makeLibCall(DAG, LC, VT, Ops, CallOptions, dl).first, Lo, Hi);
3330 }
3331 
3332 void DAGTypeLegalizer::ExpandIntRes_TRUNCATE(SDNode *N,
3333  SDValue &Lo, SDValue &Hi) {
3334  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
3335  SDLoc dl(N);
3336  Lo = DAG.getNode(ISD::TRUNCATE, dl, NVT, N->getOperand(0));
3337  Hi = DAG.getNode(ISD::SRL, dl, N->getOperand(0).getValueType(),
3338  N->getOperand(0),
3339  DAG.getConstant(NVT.getSizeInBits(), dl,
3340  TLI.getPointerTy(DAG.getDataLayout())));
3341  Hi = DAG.getNode(ISD::TRUNCATE, dl, NVT, Hi);
3342 }
3343 
3344 void DAGTypeLegalizer::ExpandIntRes_XMULO(SDNode *N,
3345  SDValue &Lo, SDValue &Hi) {
3346  EVT VT = N->getValueType(0);
3347  SDLoc dl(N);
3348 
3349  if (N->getOpcode() == ISD::UMULO) {
3350  // This section expands the operation into the following sequence of
3351  // instructions. `iNh` here refers to a type which has half the bit width of
3352  // the type the original operation operated on.
3353  //
3354  // %0 = %LHS.HI != 0 && %RHS.HI != 0
3355  // %1 = { iNh, i1 } @umul.with.overflow.iNh(iNh %LHS.HI, iNh %RHS.LO)
3356  // %2 = { iNh, i1 } @umul.with.overflow.iNh(iNh %RHS.HI, iNh %LHS.LO)
3357  // %3 = mul nuw iN (%LHS.LOW as iN), (%RHS.LOW as iN)
3358  // %4 = add iN (%1.0 as iN) << Nh, (%2.0 as iN) << Nh
3359  // %5 = { iN, i1 } @uadd.with.overflow.iN( %4, %3 )
3360  //
3361  // %res = { %5.0, %0 || %1.1 || %2.1 || %5.1 }
3362  SDValue LHS = N->getOperand(0), RHS = N->getOperand(1);
3363  SDValue LHSHigh, LHSLow, RHSHigh, RHSLow;
3364  SplitInteger(LHS, LHSLow, LHSHigh);
3365  SplitInteger(RHS, RHSLow, RHSHigh);
3366  EVT HalfVT = LHSLow.getValueType()
3367  , BitVT = N->getValueType(1);
3368  SDVTList VTHalfMulO = DAG.getVTList(HalfVT, BitVT);
3369  SDVTList VTFullAddO = DAG.getVTList(VT, BitVT);
3370 
3371  SDValue HalfZero = DAG.getConstant(0, dl, HalfVT);
3372  SDValue Overflow = DAG.getNode(ISD::AND, dl, BitVT,
3373  DAG.getSetCC(dl, BitVT, LHSHigh, HalfZero, ISD::SETNE),
3374  DAG.getSetCC(dl, BitVT, RHSHigh, HalfZero, ISD::SETNE));
3375 
3376  SDValue One = DAG.getNode(ISD::UMULO, dl, VTHalfMulO, LHSHigh, RHSLow);
3377  Overflow = DAG.getNode(ISD::OR, dl, BitVT, Overflow, One.getValue(1));
3378  SDValue OneInHigh = DAG.getNode(ISD::BUILD_PAIR, dl, VT, HalfZero,
3379  One.getValue(0));
3380 
3381  SDValue Two = DAG.getNode(ISD::UMULO, dl, VTHalfMulO, RHSHigh, LHSLow);
3382  Overflow = DAG.getNode(ISD::OR, dl, BitVT, Overflow, Two.getValue(1));
3383  SDValue TwoInHigh = DAG.getNode(ISD::BUILD_PAIR, dl, VT, HalfZero,
3384  Two.getValue(0));
3385 
3386  // Cannot use `UMUL_LOHI` directly, because some 32-bit targets (ARM) do not
3387  // know how to expand `i64,i64 = umul_lohi a, b` and abort (why isn’t this
3388  // operation recursively legalized?).
3389  //
3390  // Many backends understand this pattern and will convert into LOHI
3391  // themselves, if applicable.
3392  SDValue Three = DAG.getNode(ISD::MUL, dl, VT,
3393  DAG.getNode(ISD::ZERO_EXTEND, dl, VT, LHSLow),
3394  DAG.getNode(ISD::ZERO_EXTEND, dl, VT, RHSLow));
3395  SDValue Four = DAG.getNode(ISD::ADD, dl, VT, OneInHigh, TwoInHigh);
3396  SDValue Five = DAG.getNode(ISD::UADDO, dl, VTFullAddO, Three, Four);
3397  Overflow = DAG.getNode(ISD::OR, dl, BitVT, Overflow, Five.getValue(1));
3398  SplitInteger(Five, Lo, Hi);
3399  ReplaceValueWith(SDValue(N, 1), Overflow);
3400  return;
3401  }
3402 
3403  Type *RetTy = VT.getTypeForEVT(*DAG.getContext());
3404  EVT PtrVT = TLI.getPointerTy(DAG.getDataLayout());
3405  Type *PtrTy = PtrVT.getTypeForEVT(*DAG.getContext());
3406 
3407  // Replace this with a libcall that will check overflow.
3408  RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
3409  if (VT == MVT::i32)
3410  LC = RTLIB::MULO_I32;
3411  else if (VT == MVT::i64)
3412  LC = RTLIB::MULO_I64;
3413  else if (VT == MVT::i128)
3414  LC = RTLIB::MULO_I128;
3415  assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported XMULO!");
3416 
3417  SDValue Temp = DAG.CreateStackTemporary(PtrVT);
3418  // Temporary for the overflow value, default it to zero.
3419  SDValue Chain =
3420  DAG.getStore(DAG.getEntryNode(), dl, DAG.getConstant(0, dl, PtrVT), Temp,
3421  MachinePointerInfo());
3422 
3424  TargetLowering::ArgListEntry Entry;
3425  for (const SDValue &Op : N->op_values()) {
3426  EVT ArgVT = Op.getValueType();
3427  Type *ArgTy = ArgVT.getTypeForEVT(*DAG.getContext());
3428  Entry.Node = Op;
3429  Entry.Ty = ArgTy;
3430  Entry.IsSExt = true;
3431  Entry.IsZExt = false;
3432  Args.push_back(Entry);
3433  }
3434 
3435  // Also pass the address of the overflow check.
3436  Entry.Node = Temp;
3437  Entry.Ty = PtrTy->getPointerTo();
3438  Entry.IsSExt = true;
3439  Entry.IsZExt = false;
3440  Args.push_back(Entry);
3441 
3442  SDValue Func = DAG.getExternalSymbol(TLI.getLibcallName(LC), PtrVT);
3443 
3445  CLI.setDebugLoc(dl)
3446  .setChain(Chain)
3447  .setLibCallee(TLI.getLibcallCallingConv(LC), RetTy, Func, std::move(Args))
3448  .setSExtResult();
3449 
3450  std::pair<SDValue, SDValue> CallInfo = TLI.LowerCallTo(CLI);
3451 
3452  SplitInteger(CallInfo.first, Lo, Hi);
3453  SDValue Temp2 =
3454  DAG.getLoad(PtrVT, dl, CallInfo.second, Temp, MachinePointerInfo());
3455  SDValue Ofl = DAG.getSetCC(dl, N->getValueType(1), Temp2,
3456  DAG.getConstant(0, dl, PtrVT),
3457  ISD::SETNE);
3458  // Use the overflow from the libcall everywhere.
3459  ReplaceValueWith(SDValue(N, 1), Ofl);
3460 }
3461 
3462 void DAGTypeLegalizer::ExpandIntRes_UDIV(SDNode *N,
3463  SDValue &Lo, SDValue &Hi) {
3464  EVT VT = N->getValueType(0);
3465  SDLoc dl(N);
3466  SDValue Ops[2] = { N->getOperand(0), N->getOperand(1) };
3467 
3469  SDValue Res = DAG.getNode(ISD::UDIVREM, dl, DAG.getVTList(VT, VT), Ops);
3470  SplitInteger(Res.getValue(0), Lo, Hi);
3471  return;
3472  }
3473 
3474  RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
3475  if (VT == MVT::i16)
3476  LC = RTLIB::UDIV_I16;
3477  else if (VT == MVT::i32)
3478  LC = RTLIB::UDIV_I32;
3479  else if (VT == MVT::i64)
3480  LC = RTLIB::UDIV_I64;
3481  else if (VT == MVT::i128)
3482  LC = RTLIB::UDIV_I128;
3483  assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported UDIV!");
3484 
3486  SplitInteger(TLI.makeLibCall(DAG, LC, VT, Ops, CallOptions, dl).first, Lo, Hi);
3487 }
3488 
3489 void DAGTypeLegalizer::ExpandIntRes_UREM(SDNode *N,
3490  SDValue &Lo, SDValue &Hi) {
3491  EVT VT = N->getValueType(0);
3492  SDLoc dl(N);
3493  SDValue Ops[2] = { N->getOperand(0), N->getOperand(1) };
3494 
3496  SDValue Res = DAG.getNode(ISD::UDIVREM, dl, DAG.getVTList(VT, VT), Ops);
3497  SplitInteger(Res.getValue(1), Lo, Hi);
3498  return;
3499  }
3500 
3501  RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
3502  if (VT == MVT::i16)
3503  LC = RTLIB::UREM_I16;
3504  else if (VT == MVT::i32)
3505  LC = RTLIB::UREM_I32;
3506  else if (VT == MVT::i64)
3507  LC = RTLIB::UREM_I64;
3508  else if (VT == MVT::i128)
3509  LC = RTLIB::UREM_I128;
3510  assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported UREM!");
3511 
3513  SplitInteger(TLI.makeLibCall(DAG, LC, VT, Ops, CallOptions, dl).first, Lo, Hi);
3514 }
3515 
3516 void DAGTypeLegalizer::ExpandIntRes_ZERO_EXTEND(SDNode *N,
3517  SDValue &Lo, SDValue &Hi) {
3518  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
3519  SDLoc dl(N);
3520  SDValue Op = N->getOperand(0);
3521  if (Op.getValueType().bitsLE(NVT)) {
3522  // The low part is zero extension of the input (degenerates to a copy).
3523  Lo = DAG.getNode(ISD::ZERO_EXTEND, dl, NVT, N->getOperand(0));
3524  Hi = DAG.getConstant(0, dl, NVT); // The high part is just a zero.
3525  } else {
3526  // For example, extension of an i48 to an i64. The operand type necessarily
3527  // promotes to the result type, so will end up being expanded too.
3528  assert(getTypeAction(Op.getValueType()) ==
3530  "Only know how to promote this result!");
3531  SDValue Res = GetPromotedInteger(Op);
3532  assert(Res.getValueType() == N->getValueType(0) &&
3533  "Operand over promoted?");
3534  // Split the promoted operand. This will simplify when it is expanded.
3535  SplitInteger(Res, Lo, Hi);
3536  unsigned ExcessBits = Op.getValueSizeInBits() - NVT.getSizeInBits();
3537  Hi = DAG.getZeroExtendInReg(Hi, dl,
3539  ExcessBits));
3540  }
3541 }
3542 
3543 void DAGTypeLegalizer::ExpandIntRes_ATOMIC_LOAD(SDNode *N,
3544  SDValue &Lo, SDValue &Hi) {
3545  SDLoc dl(N);
3546  EVT VT = cast<AtomicSDNode>(N)->getMemoryVT();
3547  SDVTList VTs = DAG.getVTList(VT, MVT::i1, MVT::Other);
3548  SDValue Zero = DAG.getConstant(0, dl, VT);
3549  SDValue Swap = DAG.getAtomicCmpSwap(
3551  cast<AtomicSDNode>(N)->getMemoryVT(), VTs, N->getOperand(0),
3552  N->getOperand(1), Zero, Zero, cast<AtomicSDNode>(N)->getMemOperand());
3553 
3554  ReplaceValueWith(SDValue(N, 0), Swap.getValue(0));
3555  ReplaceValueWith(SDValue(N, 1), Swap.getValue(2));
3556 }
3557 
3558 void DAGTypeLegalizer::ExpandIntRes_VECREDUCE(SDNode *N,
3559  SDValue &Lo, SDValue &Hi) {
3560  // TODO For VECREDUCE_(AND|OR|XOR) we could split the vector and calculate
3561  // both halves independently.
3562  SDValue Res = TLI.expandVecReduce(N, DAG);
3563  SplitInteger(Res, Lo, Hi);
3564 }
3565 
3566 //===----------------------------------------------------------------------===//
3567 // Integer Operand Expansion
3568 //===----------------------------------------------------------------------===//
3569 
3570 /// ExpandIntegerOperand - This method is called when the specified operand of
3571 /// the specified node is found to need expansion. At this point, all of the
3572 /// result types of the node are known to be legal, but other operands of the
3573 /// node may need promotion or expansion as well as the specified one.
3574 bool DAGTypeLegalizer::ExpandIntegerOperand(SDNode *N, unsigned OpNo) {
3575  LLVM_DEBUG(dbgs() << "Expand integer operand: "; N->dump(&DAG);
3576  dbgs() << "\n");
3577  SDValue Res = SDValue();
3578 
3579  if (CustomLowerNode(N, N->getOperand(OpNo).getValueType(), false))
3580  return false;
3581 
3582  switch (N->getOpcode()) {
3583  default:
3584  #ifndef NDEBUG
3585  dbgs() << "ExpandIntegerOperand Op #" << OpNo << ": ";
3586  N->dump(&DAG); dbgs() << "\n";
3587  #endif
3588  report_fatal_error("Do not know how to expand this operator's operand!");
3589 
3590  case ISD::BITCAST: Res = ExpandOp_BITCAST(N); break;
3591  case ISD::BR_CC: Res = ExpandIntOp_BR_CC(N); break;
3592  case ISD::BUILD_VECTOR: Res = ExpandOp_BUILD_VECTOR(N); break;
3593  case ISD::EXTRACT_ELEMENT: Res = ExpandOp_EXTRACT_ELEMENT(N); break;
3594  case ISD::INSERT_VECTOR_ELT: Res = ExpandOp_INSERT_VECTOR_ELT(N); break;
3595  case ISD::SCALAR_TO_VECTOR: Res = ExpandOp_SCALAR_TO_VECTOR(N); break;
3596  case ISD::SELECT_CC: Res = ExpandIntOp_SELECT_CC(N); break;
3597  case ISD::SETCC: Res = ExpandIntOp_SETCC(N); break;
3598  case ISD::SETCCCARRY: Res = ExpandIntOp_SETCCCARRY(N); break;
3599  case ISD::SINT_TO_FP: Res = ExpandIntOp_SINT_TO_FP(N); break;
3600  case ISD::STORE: Res = ExpandIntOp_STORE(cast<StoreSDNode>(N), OpNo); break;
3601  case ISD::TRUNCATE: Res = ExpandIntOp_TRUNCATE(N); break;
3602  case ISD::UINT_TO_FP: Res = ExpandIntOp_UINT_TO_FP(N); break;
3603 
3604  case ISD::SHL:
3605  case ISD::SRA:
3606  case ISD::SRL:
3607  case ISD::ROTL:
3608  case ISD::ROTR: Res = ExpandIntOp_Shift(N); break;
3609  case ISD::RETURNADDR:
3610  case ISD::FRAMEADDR: Res = ExpandIntOp_RETURNADDR(N); break;
3611 
3612  case ISD::ATOMIC_STORE: Res = ExpandIntOp_ATOMIC_STORE(N); break;
3613  }
3614 
3615  // If the result is null, the sub-method took care of registering results etc.
3616  if (!Res.getNode()) return false;
3617 
3618  // If the result is N, the sub-method updated N in place. Tell the legalizer
3619  // core about this.
3620  if (Res.getNode() == N)
3621  return true;
3622 
3623  assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 &&
3624  "Invalid operand expansion");
3625 
3626  ReplaceValueWith(SDValue(N, 0), Res);
3627  return false;
3628 }
3629 
3630 /// IntegerExpandSetCCOperands - Expand the operands of a comparison. This code
3631 /// is shared among BR_CC, SELECT_CC, and SETCC handlers.
3632 void DAGTypeLegalizer::IntegerExpandSetCCOperands(SDValue &NewLHS,
3633  SDValue &NewRHS,
3634  ISD::CondCode &CCCode,
3635  const SDLoc &dl) {
3636  SDValue LHSLo, LHSHi, RHSLo, RHSHi;
3637  GetExpandedInteger(NewLHS, LHSLo, LHSHi);
3638  GetExpandedInteger(NewRHS, RHSLo, RHSHi);
3639 
3640  if (CCCode == ISD::SETEQ || CCCode == ISD::SETNE) {
3641  if (RHSLo == RHSHi) {
3642  if (ConstantSDNode *RHSCST = dyn_cast<ConstantSDNode>(RHSLo)) {
3643  if (RHSCST->isAllOnesValue()) {
3644  // Equality comparison to -1.
3645  NewLHS = DAG.getNode(ISD::AND, dl,
3646  LHSLo.getValueType(), LHSLo, LHSHi);
3647  NewRHS = RHSLo;
3648  return;
3649  }
3650  }
3651  }
3652 
3653  NewLHS = DAG.getNode(ISD::XOR, dl, LHSLo.getValueType(), LHSLo, RHSLo);
3654  NewRHS = DAG.getNode(ISD::XOR, dl, LHSLo.getValueType(), LHSHi, RHSHi);
3655  NewLHS = DAG.getNode(ISD::OR, dl, NewLHS.getValueType(), NewLHS, NewRHS);
3656  NewRHS = DAG.getConstant(0, dl, NewLHS.getValueType());
3657  return;
3658  }
3659 
3660  // If this is a comparison of the sign bit, just look at the top part.
3661  // X > -1, x < 0
3662  if (ConstantSDNode *CST = dyn_cast<ConstantSDNode>(NewRHS))
3663  if ((CCCode == ISD::SETLT && CST->isNullValue()) || // X < 0
3664  (CCCode == ISD::SETGT && CST->isAllOnesValue())) { // X > -1
3665  NewLHS = LHSHi;
3666  NewRHS = RHSHi;
3667  return;
3668  }
3669 
3670  // FIXME: This generated code sucks.
3671  ISD::CondCode LowCC;
3672  switch (CCCode) {
3673  default: llvm_unreachable("Unknown integer setcc!");
3674  case ISD::SETLT:
3675  case ISD::SETULT: LowCC = ISD::SETULT; break;
3676  case ISD::SETGT:
3677  case ISD::SETUGT: LowCC = ISD::SETUGT; break;
3678  case ISD::SETLE:
3679  case ISD::SETULE: LowCC = ISD::SETULE; break;
3680  case ISD::SETGE:
3681  case ISD::SETUGE: LowCC = ISD::SETUGE; break;
3682  }
3683 
3684  // LoCmp = lo(op1) < lo(op2) // Always unsigned comparison
3685  // HiCmp = hi(op1) < hi(op2) // Signedness depends on operands
3686  // dest = hi(op1) == hi(op2) ? LoCmp : HiCmp;
3687 
3688  // NOTE: on targets without efficient SELECT of bools, we can always use
3689  // this identity: (B1 ? B2 : B3) --> (B1 & B2)|(!B1&B3)
3690  TargetLowering::DAGCombinerInfo DagCombineInfo(DAG, AfterLegalizeTypes, true,
3691  nullptr);
3692  SDValue LoCmp, HiCmp;
3693  if (TLI.isTypeLegal(LHSLo.getValueType()) &&
3694  TLI.isTypeLegal(RHSLo.getValueType()))
3695  LoCmp = TLI.SimplifySetCC(getSetCCResultType(LHSLo.getValueType()), LHSLo,
3696  RHSLo, LowCC, false, DagCombineInfo, dl);
3697  if (!LoCmp.getNode())
3698  LoCmp = DAG.getSetCC(dl, getSetCCResultType(LHSLo.getValueType()), LHSLo,
3699  RHSLo, LowCC);
3700  if (TLI.isTypeLegal(LHSHi.getValueType()) &&
3701  TLI.isTypeLegal(RHSHi.getValueType()))
3702  HiCmp = TLI.SimplifySetCC(getSetCCResultType(LHSHi.getValueType()), LHSHi,
3703  RHSHi, CCCode, false, DagCombineInfo, dl);
3704  if (!HiCmp.getNode())
3705  HiCmp =
3706  DAG.getNode(ISD::SETCC, dl, getSetCCResultType(LHSHi.getValueType()),
3707  LHSHi, RHSHi, DAG.getCondCode(CCCode));
3708 
3709  ConstantSDNode *LoCmpC = dyn_cast<ConstantSDNode>(LoCmp.getNode());
3710  ConstantSDNode *HiCmpC = dyn_cast<ConstantSDNode>(HiCmp.getNode());
3711 
3712  bool EqAllowed = (CCCode == ISD::SETLE || CCCode == ISD::SETGE ||
3713  CCCode == ISD::SETUGE || CCCode == ISD::SETULE);
3714 
3715  if ((EqAllowed && (HiCmpC && HiCmpC->isNullValue())) ||
3716  (!EqAllowed && ((HiCmpC && (HiCmpC->getAPIntValue() == 1)) ||
3717  (LoCmpC && LoCmpC->isNullValue())))) {
3718  // For LE / GE, if high part is known false, ignore the low part.
3719  // For LT / GT: if low part is known false, return the high part.
3720  // if high part is known true, ignore the low part.
3721  NewLHS = HiCmp;
3722  NewRHS = SDValue();
3723  return;
3724  }
3725 
3726  if (LHSHi == RHSHi) {
3727  // Comparing the low bits is enough.
3728  NewLHS = LoCmp;
3729  NewRHS = SDValue();
3730  return;
3731  }
3732 
3733  // Lower with SETCCCARRY if the target supports it.
3734  EVT HiVT = LHSHi.getValueType();
3735  EVT ExpandVT = TLI.getTypeToExpandTo(*DAG.getContext(), HiVT);
3736  bool HasSETCCCARRY = TLI.isOperationLegalOrCustom(ISD::SETCCCARRY, ExpandVT);
3737 
3738  // FIXME: Make all targets support this, then remove the other lowering.
3739  if (HasSETCCCARRY) {
3740  // SETCCCARRY can detect < and >= directly. For > and <=, flip
3741  // operands and condition code.
3742  bool FlipOperands = false;
3743  switch (CCCode) {
3744  case ISD::SETGT: CCCode = ISD::SETLT; FlipOperands = true; break;
3745  case ISD::SETUGT: CCCode = ISD::SETULT; FlipOperands = true; break;
3746  case ISD::SETLE: CCCode = ISD::SETGE; FlipOperands = true; break;
3747  case ISD::SETULE: CCCode = ISD::SETUGE; FlipOperands = true; break;
3748  default: break;
3749  }
3750  if (FlipOperands) {
3751  std::swap(LHSLo, RHSLo);
3752  std::swap(LHSHi, RHSHi);
3753  }
3754  // Perform a wide subtraction, feeding the carry from the low part into
3755  // SETCCCARRY. The SETCCCARRY operation is essentially looking at the high
3756  // part of the result of LHS - RHS. It is negative iff LHS < RHS. It is
3757  // zero or positive iff LHS >= RHS.
3758  EVT LoVT = LHSLo.getValueType();
3759  SDVTList VTList = DAG.getVTList(LoVT, getSetCCResultType(LoVT));
3760  SDValue LowCmp = DAG.getNode(ISD::USUBO, dl, VTList, LHSLo, RHSLo);
3761  SDValue Res = DAG.getNode(ISD::SETCCCARRY, dl, getSetCCResultType(HiVT),
3762  LHSHi, RHSHi, LowCmp.getValue(1),
3763  DAG.getCondCode(CCCode));
3764  NewLHS = Res;
3765  NewRHS = SDValue();
3766  return;
3767  }
3768 
3769  NewLHS = TLI.SimplifySetCC(getSetCCResultType(HiVT), LHSHi, RHSHi, ISD::SETEQ,
3770  false, DagCombineInfo, dl);
3771  if (!NewLHS.getNode())
3772  NewLHS =
3773  DAG.getSetCC(dl, getSetCCResultType(HiVT), LHSHi, RHSHi, ISD::SETEQ);
3774  NewLHS = DAG.getSelect(dl, LoCmp.getValueType(), NewLHS, LoCmp, HiCmp);
3775  NewRHS = SDValue();
3776 }
3777 
3778 SDValue DAGTypeLegalizer::ExpandIntOp_BR_CC(SDNode *N) {
3779  SDValue NewLHS = N->getOperand(2), NewRHS = N->getOperand(3);
3780  ISD::CondCode CCCode = cast<CondCodeSDNode>(N->getOperand(1))->get();
3781  IntegerExpandSetCCOperands(NewLHS, NewRHS, CCCode, SDLoc(N));
3782 
3783  // If ExpandSetCCOperands returned a scalar, we need to compare the result
3784  // against zero to select between true and false values.
3785  if (!NewRHS.getNode()) {
3786  NewRHS = DAG.getConstant(0, SDLoc(N), NewLHS.getValueType());
3787  CCCode = ISD::SETNE;
3788  }
3789 
3790  // Update N to have the operands specified.
3791  return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0),
3792  DAG.getCondCode(CCCode), NewLHS, NewRHS,
3793  N->getOperand(4)), 0);
3794 }
3795 
3796 SDValue DAGTypeLegalizer::ExpandIntOp_SELECT_CC(SDNode *N) {
3797  SDValue NewLHS = N->getOperand(0), NewRHS = N->getOperand(1);
3798  ISD::CondCode CCCode = cast<CondCodeSDNode>(N->getOperand(4))->get();
3799  IntegerExpandSetCCOperands(NewLHS, NewRHS, CCCode, SDLoc(N));
3800 
3801  // If ExpandSetCCOperands returned a scalar, we need to compare the result
3802  // against zero to select between true and false values.
3803  if (!NewRHS.getNode()) {
3804  NewRHS = DAG.getConstant(0, SDLoc(N), NewLHS.getValueType());
3805  CCCode = ISD::SETNE;
3806  }
3807 
3808  // Update N to have the operands specified.
3809  return SDValue(DAG.UpdateNodeOperands(N, NewLHS, NewRHS,
3810  N->getOperand(2), N->getOperand(3),
3811  DAG.getCondCode(CCCode)), 0);
3812 }
3813 
3814 SDValue DAGTypeLegalizer::ExpandIntOp_SETCC(SDNode *N) {
3815  SDValue NewLHS = N->getOperand(0), NewRHS = N->getOperand(1);
3816  ISD::CondCode CCCode = cast<CondCodeSDNode>(N->getOperand(2))->get();
3817  IntegerExpandSetCCOperands(NewLHS, NewRHS, CCCode, SDLoc(N));
3818 
3819  // If ExpandSetCCOperands returned a scalar, use it.
3820  if (!NewRHS.getNode()) {
3821  assert(NewLHS.getValueType() == N->getValueType(0) &&
3822  "Unexpected setcc expansion!");
3823  return NewLHS;
3824  }
3825 
3826  // Otherwise, update N to have the operands specified.
3827  return SDValue(
3828  DAG.UpdateNodeOperands(N, NewLHS, NewRHS, DAG.getCondCode(CCCode)), 0);
3829 }
3830 
3831 SDValue DAGTypeLegalizer::ExpandIntOp_SETCCCARRY(SDNode *N) {
3832  SDValue LHS = N->getOperand(0);
3833  SDValue RHS = N->getOperand(1);
3834  SDValue Carry = N->getOperand(2);
3835  SDValue Cond = N->getOperand(3);
3836  SDLoc dl = SDLoc(N);
3837 
3838  SDValue LHSLo, LHSHi, RHSLo, RHSHi;
3839  GetExpandedInteger(LHS, LHSLo, LHSHi);
3840  GetExpandedInteger(RHS, RHSLo, RHSHi);
3841 
3842  // Expand to a SUBE for the low part and a smaller SETCCCARRY for the high.
3843  SDVTList VTList = DAG.getVTList(LHSLo.getValueType(), Carry.getValueType());
3844  SDValue LowCmp = DAG.getNode(ISD::SUBCARRY, dl, VTList, LHSLo, RHSLo, Carry);
3845  return DAG.getNode(ISD::SETCCCARRY, dl, N->getValueType(0), LHSHi, RHSHi,
3846  LowCmp.getValue(1), Cond);
3847 }
3848 
3849 SDValue DAGTypeLegalizer::ExpandIntOp_Shift(SDNode *N) {
3850  // The value being shifted is legal, but the shift amount is too big.
3851  // It follows that either the result of the shift is undefined, or the
3852  // upper half of the shift amount is zero. Just use the lower half.
3853  SDValue Lo, Hi;
3854  GetExpandedInteger(N->getOperand(1), Lo, Hi);
3855  return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0), Lo), 0);
3856 }
3857 
3858 SDValue DAGTypeLegalizer::ExpandIntOp_RETURNADDR(SDNode *N) {
3859  // The argument of RETURNADDR / FRAMEADDR builtin is 32 bit contant. This
3860  // surely makes pretty nice problems on 8/16 bit targets. Just truncate this
3861  // constant to valid type.
3862  SDValue Lo, Hi;
3863  GetExpandedInteger(N->getOperand(0), Lo, Hi);
3864  return SDValue(DAG.UpdateNodeOperands(N, Lo), 0);
3865 }
3866 
3867 SDValue DAGTypeLegalizer::ExpandIntOp_SINT_TO_FP(SDNode *N) {
3868  SDValue Op = N->getOperand(0);
3869  EVT DstVT = N->getValueType(0);
3871  assert(LC != RTLIB::UNKNOWN_LIBCALL &&
3872  "Don't know how to expand this SINT_TO_FP!");
3874  CallOptions.setSExt(true);
3875  return TLI.makeLibCall(DAG, LC, DstVT, Op, CallOptions, SDLoc(N)).first;
3876 }
3877 
3878 SDValue DAGTypeLegalizer::ExpandIntOp_STORE(StoreSDNode *N, unsigned OpNo) {
3879  if (ISD::isNormalStore(N))
3880  return ExpandOp_NormalStore(N, OpNo);
3881 
3882  assert(ISD::isUNINDEXEDStore(N) && "Indexed store during type legalization!");
3883  assert(OpNo == 1 && "Can only expand the stored value so far");
3884 
3885  EVT VT = N->getOperand(1).getValueType();
3886  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
3887  SDValue Ch = N->getChain();
3888  SDValue Ptr = N->getBasePtr();
3889  unsigned Alignment = N->getAlignment();
3890  MachineMemOperand::Flags MMOFlags = N->getMemOperand()->getFlags();
3891  AAMDNodes AAInfo = N->getAAInfo();
3892  SDLoc dl(N);
3893  SDValue Lo, Hi;
3894 
3895  assert(NVT.isByteSized() && "Expanded type not byte sized!");
3896 
3897  if (N->getMemoryVT().bitsLE(NVT)) {
3898  GetExpandedInteger(N->getValue(), Lo, Hi);
3899  return DAG.getTruncStore(Ch, dl, Lo, Ptr, N->getPointerInfo(),
3900  N->getMemoryVT(), Alignment, MMOFlags, AAInfo);
3901  }
3902 
3903  if (DAG.getDataLayout().isLittleEndian()) {
3904  // Little-endian - low bits are at low addresses.
3905  GetExpandedInteger(N->getValue(), Lo, Hi);
3906 
3907  Lo = DAG.getStore(Ch, dl, Lo, Ptr, N->getPointerInfo(), Alignment, MMOFlags,
3908  AAInfo);
3909 
3910  unsigned ExcessBits =
3911  N->getMemoryVT().getSizeInBits() - NVT.getSizeInBits();
3912  EVT NEVT = EVT::getIntegerVT(*DAG.getContext(), ExcessBits);
3913 
3914  // Increment the pointer to the other half.
3915  unsigned IncrementSize = NVT.getSizeInBits()/8;
3916  Ptr = DAG.getObjectPtrOffset(dl, Ptr, IncrementSize);
3917  Hi = DAG.getTruncStore(
3918  Ch, dl, Hi, Ptr, N->getPointerInfo().getWithOffset(IncrementSize), NEVT,
3919  MinAlign(Alignment, IncrementSize), MMOFlags, AAInfo);
3920  return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo, Hi);
3921  }
3922 
3923  // Big-endian - high bits are at low addresses. Favor aligned stores at
3924  // the cost of some bit-fiddling.
3925  GetExpandedInteger(N->getValue(), Lo, Hi);
3926 
3927  EVT ExtVT = N->getMemoryVT();
3928  unsigned EBytes = ExtVT.getStoreSize();
3929  unsigned IncrementSize = NVT.getSizeInBits()/8;
3930  unsigned ExcessBits = (EBytes - IncrementSize)*8;
3931  EVT HiVT = EVT::getIntegerVT(*DAG.getContext(),
3932  ExtVT.getSizeInBits() - ExcessBits);
3933 
3934  if (ExcessBits < NVT.getSizeInBits()) {
3935  // Transfer high bits from the top of Lo to the bottom of Hi.
3936  Hi = DAG.getNode(ISD::SHL, dl, NVT, Hi,
3937  DAG.getConstant(NVT.getSizeInBits() - ExcessBits, dl,
3938  TLI.getPointerTy(DAG.getDataLayout())));
3939  Hi = DAG.getNode(
3940  ISD::OR, dl, NVT, Hi,
3941  DAG.getNode(ISD::SRL, dl, NVT, Lo,
3942  DAG.getConstant(ExcessBits, dl,
3943  TLI.getPointerTy(DAG.getDataLayout()))));
3944  }
3945 
3946  // Store both the high bits and maybe some of the low bits.
3947  Hi = DAG.getTruncStore(Ch, dl, Hi, Ptr, N->getPointerInfo(), HiVT, Alignment,
3948  MMOFlags, AAInfo);
3949 
3950  // Increment the pointer to the other half.
3951  Ptr = DAG.getObjectPtrOffset(dl, Ptr, IncrementSize);
3952  // Store the lowest ExcessBits bits in the second half.
3953  Lo = DAG.getTruncStore(Ch, dl, Lo, Ptr,
3954  N->getPointerInfo().getWithOffset(IncrementSize),
3955  EVT::getIntegerVT(*DAG.getContext(), ExcessBits),
3956  MinAlign(Alignment, IncrementSize), MMOFlags, AAInfo);
3957  return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo, Hi);
3958 }
3959 
3960 SDValue DAGTypeLegalizer::ExpandIntOp_TRUNCATE(SDNode *N) {
3961  SDValue InL, InH;
3962  GetExpandedInteger(N->getOperand(0), InL, InH);
3963  // Just truncate the low part of the source.
3964  return DAG.getNode(ISD::TRUNCATE, SDLoc(N), N->getValueType(0), InL);
3965 }
3966 
3967 SDValue DAGTypeLegalizer::ExpandIntOp_UINT_TO_FP(SDNode *N) {
3968  SDValue Op = N->getOperand(0);
3969  EVT SrcVT = Op.getValueType();
3970  EVT DstVT = N->getValueType(0);
3971  SDLoc dl(N);
3972 
3973  // The following optimization is valid only if every value in SrcVT (when
3974  // treated as signed) is representable in DstVT. Check that the mantissa
3975  // size of DstVT is >= than the number of bits in SrcVT -1.
3976  const fltSemantics &sem = DAG.EVTToAPFloatSemantics(DstVT);
3977  if (APFloat::semanticsPrecision(sem) >= SrcVT.getSizeInBits()-1 &&
3979  // Do a signed conversion then adjust the result.
3980  SDValue SignedConv = DAG.getNode(ISD::SINT_TO_FP, dl, DstVT, Op);
3981  SignedConv = TLI.LowerOperation(SignedConv, DAG);
3982 
3983  // The result of the signed conversion needs adjusting if the 'sign bit' of
3984  // the incoming integer was set. To handle this, we dynamically test to see
3985  // if it is set, and, if so, add a fudge factor.
3986 
3987  const uint64_t F32TwoE32 = 0x4F800000ULL;
3988  const uint64_t F32TwoE64 = 0x5F800000ULL;
3989  const uint64_t F32TwoE128 = 0x7F800000ULL;
3990 
3991  APInt FF(32, 0);
3992  if (SrcVT == MVT::i32)
3993  FF = APInt(32, F32TwoE32);
3994  else if (SrcVT == MVT::i64)
3995  FF = APInt(32, F32TwoE64);
3996  else if (SrcVT == MVT::i128)
3997  FF = APInt(32, F32TwoE128);
3998  else
3999  llvm_unreachable("Unsupported UINT_TO_FP!");
4000 
4001  // Check whether the sign bit is set.
4002  SDValue Lo, Hi;
4003  GetExpandedInteger(Op, Lo, Hi);
4004  SDValue SignSet = DAG.getSetCC(dl,
4005  getSetCCResultType(Hi.getValueType()),
4006  Hi,
4007  DAG.getConstant(0, dl, Hi.getValueType()),
4008  ISD::SETLT);
4009 
4010  // Build a 64 bit pair (0, FF) in the constant pool, with FF in the lo bits.
4011  SDValue FudgePtr =
4012  DAG.getConstantPool(ConstantInt::get(*DAG.getContext(), FF.zext(64)),
4013  TLI.getPointerTy(DAG.getDataLayout()));
4014 
4015  // Get a pointer to FF if the sign bit was set, or to 0 otherwise.
4016  SDValue Zero = DAG.getIntPtrConstant(0, dl);
4017  SDValue Four = DAG.getIntPtrConstant(4, dl);
4018  if (DAG.getDataLayout().isBigEndian())
4019  std::swap(Zero, Four);
4020  SDValue Offset = DAG.getSelect(dl, Zero.getValueType(), SignSet,
4021  Zero, Four);
4022  unsigned Alignment = cast<ConstantPoolSDNode>(FudgePtr)->getAlignment();
4023  FudgePtr = DAG.getNode(ISD::ADD, dl, FudgePtr.getValueType(),
4024  FudgePtr, Offset);
4025  Alignment = std::min(Alignment, 4u);
4026 
4027  // Load the value out, extending it from f32 to the destination float type.
4028  // FIXME: Avoid the extend by constructing the right constant pool?
4029  SDValue Fudge = DAG.getExtLoad(
4030  ISD::EXTLOAD, dl, DstVT, DAG.getEntryNode(), FudgePtr,
4032  Alignment);
4033  return DAG.getNode(ISD::FADD, dl, DstVT, SignedConv, Fudge);
4034  }
4035 
4036  // Otherwise, use a libcall.
4037  RTLIB::Libcall LC = RTLIB::getUINTTOFP(SrcVT, DstVT);
4038  assert(LC != RTLIB::UNKNOWN_LIBCALL &&
4039  "Don't know how to expand this UINT_TO_FP!");
4041  CallOptions.setSExt(true);
4042  return TLI.makeLibCall(DAG, LC, DstVT, Op, CallOptions, dl).first;
4043 }
4044 
4045 SDValue DAGTypeLegalizer::ExpandIntOp_ATOMIC_STORE(SDNode *N) {
4046  SDLoc dl(N);
4047  SDValue Swap = DAG.getAtomic(ISD::ATOMIC_SWAP, dl,
4048  cast<AtomicSDNode>(N)->getMemoryVT(),
4049  N->getOperand(0),
4050  N->getOperand(1), N->getOperand(2),
4051  cast<AtomicSDNode>(N)->getMemOperand());
4052  return Swap.getValue(1);
4053 }
4054 
4055 
4056 SDValue DAGTypeLegalizer::PromoteIntRes_EXTRACT_SUBVECTOR(SDNode *N) {
4057 
4058  EVT OutVT = N->getValueType(0);
4059  EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT);
4060  assert(NOutVT.isVector() && "This type must be promoted to a vector type");
4061  unsigned OutNumElems = OutVT.getVectorNumElements();
4062  EVT NOutVTElem = NOutVT.getVectorElementType();
4063 
4064  SDLoc dl(N);
4065  SDValue BaseIdx = N->getOperand(1);
4066 
4067  SDValue InOp0 = N->getOperand(0);
4068  if (getTypeAction(InOp0.getValueType()) == TargetLowering::TypePromoteInteger)
4069  InOp0 = GetPromotedInteger(N->getOperand(0));
4070 
4071  EVT InVT = InOp0.getValueType();
4072 
4074  Ops.reserve(OutNumElems);
4075  for (unsigned i = 0; i != OutNumElems; ++i) {
4076 
4077  // Extract the element from the original vector.
4078  SDValue Index = DAG.getNode(ISD::ADD, dl, BaseIdx.getValueType(),
4079  BaseIdx, DAG.getConstant(i, dl, BaseIdx.getValueType()));
4081  InVT.getVectorElementType(), N->getOperand(0), Index);
4082 
4083  SDValue Op = DAG.getAnyExtOrTrunc(Ext, dl, NOutVTElem);
4084  // Insert the converted element to the new vector.
4085  Ops.push_back(Op);
4086  }
4087 
4088  return DAG.getBuildVector(NOutVT, dl, Ops);
4089 }
4090 
4091 
4092 SDValue DAGTypeLegalizer::PromoteIntRes_VECTOR_SHUFFLE(SDNode *N) {
4093  ShuffleVectorSDNode *SV = cast<ShuffleVectorSDNode>(N);
4094  EVT VT = N->getValueType(0);
4095  SDLoc dl(N);
4096 
4097  ArrayRef<int> NewMask = SV->getMask().slice(0, VT.getVectorNumElements());
4098 
4099  SDValue V0 = GetPromotedInteger(N->getOperand(0));
4100  SDValue V1 = GetPromotedInteger(N->getOperand(1));
4101  EVT OutVT = V0.getValueType();
4102 
4103  return DAG.getVectorShuffle(OutVT, dl, V0, V1, NewMask);
4104 }
4105 
4106 
4107 SDValue DAGTypeLegalizer::PromoteIntRes_BUILD_VECTOR(SDNode *N) {
4108  EVT OutVT = N->getValueType(0);
4109  EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT);
4110  assert(NOutVT.isVector() && "This type must be promoted to a vector type");
4111  unsigned NumElems = N->getNumOperands();
4112  EVT NOutVTElem = NOutVT.getVectorElementType();
4113 
4114  SDLoc dl(N);
4115 
4117  Ops.reserve(NumElems);
4118  for (unsigned i = 0; i != NumElems; ++i) {
4119  SDValue Op;
4120  // BUILD_VECTOR integer operand types are allowed to be larger than the
4121  // result's element type. This may still be true after the promotion. For
4122  // example, we might be promoting (<v?i1> = BV <i32>, <i32>, ...) to
4123  // (v?i16 = BV <i32>, <i32>, ...), and we can't any_extend <i32> to <i16>.
4124  if (N->getOperand(i).getValueType().bitsLT(NOutVTElem))
4125  Op = DAG.getNode(ISD::ANY_EXTEND, dl, NOutVTElem, N->getOperand(i));
4126  else
4127  Op = N->getOperand(i);
4128  Ops.push_back(Op);
4129  }
4130 
4131  return DAG.getBuildVector(NOutVT, dl, Ops);
4132 }
4133 
4134 SDValue DAGTypeLegalizer::PromoteIntRes_SCALAR_TO_VECTOR(SDNode *N) {
4135 
4136  SDLoc dl(N);
4137 
4138  assert(!N->getOperand(0).getValueType().isVector() &&
4139  "Input must be a scalar");
4140 
4141  EVT OutVT = N->getValueType(0);
4142  EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT);
4143  assert(NOutVT.isVector() && "This type must be promoted to a vector type");
4144  EVT NOutVTElem = NOutVT.getVectorElementType();
4145 
4146  SDValue Op = DAG.getNode(ISD::ANY_EXTEND, dl, NOutVTElem, N->getOperand(0));
4147 
4148  return DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, NOutVT, Op);
4149 }
4150 
4151 SDValue DAGTypeLegalizer::PromoteIntRes_SPLAT_VECTOR(SDNode *N) {
4152  SDLoc dl(N);
4153 
4154  SDValue SplatVal = N->getOperand(0);
4155 
4156  assert(!SplatVal.getValueType().isVector() && "Input must be a scalar");
4157 
4158  EVT OutVT = N->getValueType(0);
4159  EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT);
4160  assert(NOutVT.isVector() && "Type must be promoted to a vector type");
4161  EVT NOutElemVT = NOutVT.getVectorElementType();
4162 
4163  SDValue Op = DAG.getNode(ISD::ANY_EXTEND, dl, NOutElemVT, SplatVal);
4164 
4165  return DAG.getNode(ISD::SPLAT_VECTOR, dl, NOutVT, Op);
4166 }
4167 
4168 SDValue DAGTypeLegalizer::PromoteIntRes_CONCAT_VECTORS(SDNode *N) {
4169  SDLoc dl(N);
4170 
4171  EVT OutVT = N->getValueType(0);
4172  EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT);
4173  assert(NOutVT.isVector() && "This type must be promoted to a vector type");
4174 
4175  EVT OutElemTy = NOutVT.getVectorElementType();
4176 
4177  unsigned NumElem = N->getOperand(0).getValueType().getVectorNumElements();
4178  unsigned NumOutElem = NOutVT.getVectorNumElements();
4179  unsigned NumOperands = N->getNumOperands();
4180  assert(NumElem * NumOperands == NumOutElem &&
4181  "Unexpected number of elements");
4182 
4183  // Take the elements from the first vector.
4184  SmallVector<SDValue, 8> Ops(NumOutElem);
4185  for (unsigned i = 0; i < NumOperands; ++i) {
4186  SDValue Op = N->getOperand(i);
4187  if (getTypeAction(Op.getValueType()) == TargetLowering::TypePromoteInteger)
4188  Op = GetPromotedInteger(Op);
4189  EVT SclrTy = Op.getValueType().getVectorElementType();
4190  assert(NumElem == Op.getValueType().getVectorNumElements() &&
4191  "Unexpected number of elements");
4192 
4193  for (unsigned j = 0; j < NumElem; ++j) {
4194  SDValue Ext = DAG.getNode(
4195  ISD::EXTRACT_VECTOR_ELT, dl, SclrTy, Op,
4196  DAG.getConstant(j, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
4197  Ops[i * NumElem + j] = DAG.getAnyExtOrTrunc(Ext, dl, OutElemTy);
4198  }
4199  }
4200 
4201  return DAG.getBuildVector(NOutVT, dl, Ops);
4202 }
4203 
4204 SDValue DAGTypeLegalizer::PromoteIntRes_EXTEND_VECTOR_INREG(SDNode *N) {
4205  EVT VT = N->getValueType(0);
4206  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
4207  assert(NVT.isVector() && "This type must be promoted to a vector type");
4208 
4209  SDLoc dl(N);
4210 
4211  // For operands whose TypeAction is to promote, extend the promoted node
4212  // appropriately (ZERO_EXTEND or SIGN_EXTEND) from the original pre-promotion
4213  // type, and then construct a new *_EXTEND_VECTOR_INREG node to the promote-to
4214  // type..
4215  if (getTypeAction(N->getOperand(0).getValueType())
4217  SDValue Promoted;
4218 
4219  switch(N->getOpcode()) {
4221  Promoted = SExtPromotedInteger(N->getOperand(0));
4222  break;
4224  Promoted = ZExtPromotedInteger(N->getOperand(0));
4225  break;
4227  Promoted = GetPromotedInteger(N->getOperand(0));
4228  break;
4229  default:
4230  llvm_unreachable("Node has unexpected Opcode");
4231  }
4232  return DAG.getNode(N->getOpcode(), dl, NVT, Promoted);
4233  }
4234 
4235  // Directly extend to the appropriate transform-to type.
4236  return DAG.getNode(N->getOpcode(), dl, NVT, N->getOperand(0));
4237 }
4238 
4239 SDValue DAGTypeLegalizer::PromoteIntRes_INSERT_VECTOR_ELT(SDNode *N) {
4240  EVT OutVT = N->getValueType(0);
4241  EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT);
4242  assert(NOutVT.isVector() && "This type must be promoted to a vector type");
4243 
4244  EVT NOutVTElem = NOutVT.getVectorElementType();
4245 
4246  SDLoc dl(N);
4247  SDValue V0 = GetPromotedInteger(N->getOperand(0));
4248 
4249  SDValue ConvElem = DAG.getNode(ISD::ANY_EXTEND, dl,
4250  NOutVTElem, N->getOperand(1));
4251  return DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, NOutVT,
4252  V0, ConvElem, N->getOperand(2));
4253 }
4254 
4255 SDValue DAGTypeLegalizer::PromoteIntRes_VECREDUCE(SDNode *N) {
4256  // The VECREDUCE result size may be larger than the element size, so
4257  // we can simply change the result type.
4258  SDLoc dl(N);
4259  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
4260  return DAG.getNode(N->getOpcode(), dl, NVT, N->getOperand(0));
4261 }
4262 
4263 SDValue DAGTypeLegalizer::PromoteIntOp_EXTRACT_VECTOR_ELT(SDNode *N) {
4264  SDLoc dl(N);
4265  SDValue V0 = GetPromotedInteger(N->getOperand(0));
4266  SDValue V1 = DAG.getZExtOrTrunc(N->getOperand(1), dl,
4267  TLI.getVectorIdxTy(DAG.getDataLayout()));
4269  V0->getValueType(0).getScalarType(), V0, V1);
4270 
4271  // EXTRACT_VECTOR_ELT can return types which are wider than the incoming
4272  // element types. If this is the case then we need to expand the outgoing
4273  // value and not truncate it.
4274  return DAG.getAnyExtOrTrunc(Ext, dl, N->getValueType(0));
4275 }
4276 
4277 SDValue DAGTypeLegalizer::PromoteIntOp_EXTRACT_SUBVECTOR(SDNode *N) {
4278  SDLoc dl(N);
4279  SDValue V0 = GetPromotedInteger(N->getOperand(0));
4280  MVT InVT = V0.getValueType().getSimpleVT();
4281  MVT OutVT = MVT::getVectorVT(InVT.getVectorElementType(),
4283  SDValue Ext = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, OutVT, V0, N->getOperand(1));
4284  return DAG.getNode(ISD::TRUNCATE, dl, N->getValueType(0), Ext);
4285 }
4286 
4287 SDValue DAGTypeLegalizer::PromoteIntOp_CONCAT_VECTORS(SDNode *N) {
4288  SDLoc dl(N);
4289  unsigned NumElems = N->getNumOperands();
4290 
4291  EVT RetSclrTy = N->getValueType(0).getVectorElementType();
4292 
4293  SmallVector<SDValue, 8> NewOps;
4294  NewOps.reserve(NumElems);
4295 
4296  // For each incoming vector
4297  for (unsigned VecIdx = 0; VecIdx != NumElems; ++VecIdx) {
4298  SDValue Incoming = GetPromotedInteger(N->getOperand(VecIdx));
4299  EVT SclrTy = Incoming->getValueType(0).getVectorElementType();
4300  unsigned NumElem = Incoming->getValueType(0).getVectorNumElements();
4301 
4302  for (unsigned i=0; i<NumElem; ++i) {
4303  // Extract element from incoming vector
4304  SDValue Ex = DAG.getNode(
4305  ISD::EXTRACT_VECTOR_ELT, dl, SclrTy, Incoming,
4306  DAG.getConstant(i, dl, TLI.getVectorIdxTy(DAG.getDataLayout())));
4307  SDValue Tr = DAG.getNode(ISD::TRUNCATE, dl, RetSclrTy, Ex);
4308  NewOps.push_back(Tr);
4309  }
4310  }
4311 
4312  return DAG.getBuildVector(N->getValueType(0), dl, NewOps);
4313 }
SDValue getStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr, MachinePointerInfo PtrInfo, unsigned Alignment=0, MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes())
Helper function to build ISD::STORE nodes.
BITCAST - This operator converts between integer, vector and FP values, as if the value was stored to...
Definition: ISDOpcodes.h:603
unsigned Log2_32_Ceil(uint32_t Value)
Return the ceil log base 2 of the specified value, 32 if the value is zero.
Definition: MathExtras.h:598
static EVT getShiftAmountTyForConstant(unsigned Val, EVT VT, const TargetLowering &TLI, SelectionDAG &DAG)
virtual MVT getVectorIdxTy(const DataLayout &DL) const
Returns the type to be used for the index operand of: ISD::INSERT_VECTOR_ELT, ISD::EXTRACT_VECTOR_ELT...
EVT getValueType() const
Return the ValueType of the referenced return value.
static std::pair< ISD::CondCode, ISD::NodeType > getExpandedMinMaxOps(int Op)
SDValue getBoolExtOrTrunc(SDValue Op, const SDLoc &SL, EVT VT, EVT OpVT)
Convert Op, which must be of integer type, to the integer type VT, by using an extension appropriate ...
EXTRACT_ELEMENT - This is used to get the lower or upper (determined by a Constant, which is required to be operand #1) half of the integer or float value specified as operand #0.
Definition: ISDOpcodes.h:183
APInt sext(unsigned width) const
Sign extend to a new width.
Definition: APInt.cpp:888
static APInt getAllOnesValue(unsigned numBits)
Get the all-ones value.
Definition: APInt.h:561
unsigned getOpcode() const
Return the SelectionDAG opcode value for this node.
SDValue getSetCC(const SDLoc &DL, EVT VT, SDValue LHS, SDValue RHS, ISD::CondCode Cond)
Helper function to make it easier to build SetCC&#39;s if you just have an ISD::CondCode instead of an SD...
Definition: SelectionDAG.h:963
EXTRACT_SUBVECTOR(VECTOR, IDX) - Returns a subvector from VECTOR (an vector value) starting with the ...
Definition: ISDOpcodes.h:390
LLVM_ATTRIBUTE_NORETURN void report_fatal_error(Error Err, bool gen_crash_diag=true)
Report a serious error, calling any installed error handler.
Definition: Error.cpp:139
BR_CC - Conditional branch.
Definition: ISDOpcodes.h:686
This class represents lattice values for constants.
Definition: AllocatorList.h:23
static MVT getVectorVT(MVT VT, unsigned NumElements)
VECTOR_SHUFFLE(VEC1, VEC2) - Returns a vector, of the same type as VEC1/VEC2.
Definition: ISDOpcodes.h:399
EVT getScalarType() const
If this is a vector type, return the element type, otherwise return this.
Definition: ValueTypes.h:259
This structure is used to pass arguments to makeLibCall function.
Same as the corresponding unsaturated fixed point instructions, but the result is clamped between the...
Definition: ISDOpcodes.h:284
ZERO_EXTEND_VECTOR_INREG(Vector) - This operator represents an in-register zero-extension of the low ...
Definition: ISDOpcodes.h:558
Libcall getSYNC(unsigned Opc, MVT VT)
Return the SYNC_FETCH_AND_* value for the given opcode and type, or UNKNOWN_LIBCALL if there is none...
const SDValue & getBasePtr() const
Carry-setting nodes for multiple precision addition and subtraction.
Definition: ISDOpcodes.h:222
void push_back(const T &Elt)
Definition: SmallVector.h:211
EVT getValueType(unsigned ResNo) const
Return the type of a specified result.
APInt zext(unsigned width) const
Zero extend to a new width.
Definition: APInt.cpp:912
const SDValue & getValue() const
bool isCompressingStore() const
Returns true if the op does a compression to the vector before storing.
static APInt getLowBitsSet(unsigned numBits, unsigned loBitsSet)
Get a value with low bits set.
Definition: APInt.h:647
MVT getSimpleVT() const
Return the SimpleValueType held in the specified simple EVT.
Definition: ValueTypes.h:252
Libcall
RTLIB::Libcall enum - This enum defines all of the runtime library calls the backend can emit...
AAMDNodes getAAInfo() const
Returns the AA info that describes the dereference.
RESULT, BOOL = [SU]ADDO(LHS, RHS) - Overflow-aware nodes for addition.
Definition: ISDOpcodes.h:250
bool isSubsetOf(const APInt &RHS) const
This operation checks that all bits set in this APInt are also set in RHS.
Definition: APInt.h:1328
const SDValue & getBasePtr() const
const SDValue & getChain() const
SPLAT_VECTOR(VAL) - Returns a vector with the scalar value VAL duplicated in all lanes.
Definition: ISDOpcodes.h:414
static const fltSemantics & EVTToAPFloatSemantics(EVT VT)
Returns an APFloat semantics tag appropriate for the given type.
unsigned getAlignment() const
Val, Success, OUTCHAIN = ATOMIC_CMP_SWAP_WITH_SUCCESS(INCHAIN, ptr, cmp, swap) N.b.
Definition: ISDOpcodes.h:841
CallLoweringInfo & setDebugLoc(const SDLoc &dl)
SDValue getSelect(const SDLoc &DL, EVT VT, SDValue Cond, SDValue LHS, SDValue RHS)
Helper function to make it easier to build Select&#39;s if you just have operands and don&#39;t want to check...
Definition: SelectionDAG.h:976
SIGN_EXTEND_VECTOR_INREG(Vector) - This operator represents an in-register sign-extension of the low ...
Definition: ISDOpcodes.h:547
[US]{MIN/MAX} - Binary minimum or maximum or signed or unsigned integers.
Definition: ISDOpcodes.h:423
BooleanContent getBooleanContents(bool isVec, bool isFloat) const
For targets without i1 registers, this gives the nature of the high-bits of boolean values held in ty...
SDNode * getNode() const
get the SDNode which holds the desired result
SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT, ArrayRef< SDUse > Ops)
Gets or creates the specified node.
SDValue getConstantPool(const Constant *C, EVT VT, unsigned Align=0, int Offs=0, bool isT=false, unsigned TargetFlags=0)
Same for subtraction.
Definition: ISDOpcodes.h:253
void reserve(size_type N)
Definition: SmallVector.h:369
bool isOperationLegalOrCustom(unsigned Op, EVT VT) const
Return true if the specified operation is legal on this target or can be made legal with custom lower...
bool isByteSized() const
Return true if the bit size is a multiple of 8.
Definition: ValueTypes.h:211
MachineMemOperand * getMemOperand() const
Return a MachineMemOperand object describing the memory reference performed by operation.
unsigned getValueSizeInBits() const
Returns the size of the value in bits.
Select with condition operator - This selects between a true value and a false value (ops #2 and #3) ...
Definition: ISDOpcodes.h:474
NodeType
ISD::NodeType enum - This enum defines the target-independent operators for a SelectionDAG.
Definition: ISDOpcodes.h:38
static APInt getSignedMaxValue(unsigned numBits)
Gets maximum signed value of APInt for a specific bit width.
Definition: APInt.h:534
This SDNode is used to implement the code generator support for the llvm IR shufflevector instruction...
SDValue getExternalSymbol(const char *Sym, EVT VT)
Libcall getUINTTOFP(EVT OpVT, EVT RetVT)
getUINTTOFP - Return the UINTTOFP_*_* value for the given types, or UNKNOWN_LIBCALL if there is none...
virtual bool shouldExpandShift(SelectionDAG &DAG, SDNode *N) const
Return true if SHIFT instructions should be expanded to SHIFT_PARTS instructions, and false if a libr...
MVT getRegisterType(MVT VT) const
Return the type of registers that this ValueType will eventually require.
SDIVREM/UDIVREM - Divide two integers and produce both a quotient and remainder result.
Definition: ISDOpcodes.h:209
SHL_PARTS/SRA_PARTS/SRL_PARTS - These operators are used for expanded integer shift operations...
Definition: ISDOpcodes.h:495
SDValue getIntPtrConstant(uint64_t Val, const SDLoc &DL, bool isTarget=false)
static uint32_t getAlignment(const MCSectionCOFF &Sec)
SDValue SimplifySetCC(EVT VT, SDValue N0, SDValue N1, ISD::CondCode Cond, bool foldBooleans, DAGCombinerInfo &DCI, const SDLoc &dl) const
Try to simplify a setcc built with the specified operands and cc.
ArrayRef< T > makeArrayRef(const T &OneElt)
Construct an ArrayRef from a single element.
Definition: ArrayRef.h:450
Shift and rotation operations.
Definition: ISDOpcodes.h:449
bool isNormalStore(const SDNode *N)
Returns true if the specified node is a non-truncating and unindexed store.
Type * getTypeForEVT(LLVMContext &Context) const
This method returns an LLVM type corresponding to the specified EVT.
Definition: ValueTypes.cpp:264
ABS - Determine the unsigned absolute value of a signed integer value of the same bitwidth...
Definition: ISDOpcodes.h:432
PointerType * getPointerTo(unsigned AddrSpace=0) const
Return a pointer to the current type.
Definition: Type.cpp:659
CallLoweringInfo & setChain(SDValue InChain)
BUILD_PAIR - This is the opposite of EXTRACT_ELEMENT in some ways.
Definition: ISDOpcodes.h:190
RESULT = [US]MULFIX(LHS, RHS, SCALE) - Perform fixed point multiplication on 2 integers with the same...
Definition: ISDOpcodes.h:279
const SDValue & getPassThru() const
op_iterator op_end() const
unsigned getScalarValueSizeInBits() const
ISD::LoadExtType getExtensionType() const
Return whether this is a plain node, or one of the varieties of value-extending loads.
FLT_ROUNDS_ - Returns current rounding mode: -1 Undefined 0 Round to 0 1 Round to nearest 2 Round to ...
Definition: ISDOpcodes.h:585
SimpleValueType SimpleTy
unsigned getStoreSize() const
Return the number of bytes overwritten by a store of the specified value type.
Definition: ValueTypes.h:303
Val, OUTCHAIN = ATOMIC_SWAP(INCHAIN, ptr, amt) Val, OUTCHAIN = ATOMIC_LOAD_[OpName](INCHAIN, ptr, amt) For double-word atomic operations: ValLo, ValHi, OUTCHAIN = ATOMIC_SWAP(INCHAIN, ptr, amtLo, amtHi) ValLo, ValHi, OUTCHAIN = ATOMIC_LOAD_[OpName](INCHAIN, ptr, amtLo, amtHi) These correspond to the atomicrmw instruction.
Definition: ISDOpcodes.h:849
const SDValue & getValue() const
SDValue getEntryNode() const
Return the token chain corresponding to the entry of the function.
Definition: SelectionDAG.h:477
const DataLayout & getDataLayout() const
Definition: SelectionDAG.h:417
SDVTList getVTList(EVT VT)
Return an SDVTList that represents the list of values specified.
FRAMEADDR, RETURNADDR - These nodes represent llvm.frameaddress and llvm.returnaddress on the DAG...
Definition: ISDOpcodes.h:72
This is an SDNode representing atomic operations.
bool expandMUL(SDNode *N, SDValue &Lo, SDValue &Hi, EVT HiLoVT, SelectionDAG &DAG, MulExpansionKind Kind, SDValue LL=SDValue(), SDValue LH=SDValue(), SDValue RL=SDValue(), SDValue RH=SDValue()) const
Expand a MUL into two nodes.
This class defines information used to lower LLVM code to legal SelectionDAG operators that the targe...
This represents a list of ValueType&#39;s that has been intern&#39;d by a SelectionDAG.
SDValue getExtLoad(ISD::LoadExtType ExtType, const SDLoc &dl, EVT VT, SDValue Chain, SDValue Ptr, MachinePointerInfo PtrInfo, EVT MemVT, unsigned Alignment=0, MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes())
This class is used to represent an MSTORE node.
unsigned getSizeInBits() const
auto reverse(ContainerTy &&C, typename std::enable_if< has_rbegin< ContainerTy >::value >::type *=nullptr) -> decltype(make_range(C.rbegin(), C.rend()))
Definition: STLExtras.h:261
SDValue getSExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT)
Convert Op, which must be of integer type, to the integer type VT, by either sign-extending or trunca...
unsigned getScalarSizeInBits() const
Definition: ValueTypes.h:297
unsigned getSizeInBits() const
Return the size of the specified value type in bits.
Definition: ValueTypes.h:291
MachineFunction & getMachineFunction() const
Definition: SelectionDAG.h:414
[SU]INT_TO_FP - These operators convert integers (whose interpreted sign depends on the first letter)...
Definition: ISDOpcodes.h:517
bool isUNINDEXEDStore(const SDNode *N)
Returns true if the specified node is an unindexed store.
const SDValue & getScale() const
CallingConv::ID getLibcallCallingConv(RTLIB::Libcall Call) const
Get the CallingConv that should be used for the specified libcall.
Definition: