LLVM  14.0.0git
LegalizeIntegerTypes.cpp
Go to the documentation of this file.
1 //===----- LegalizeIntegerTypes.cpp - Legalization of integer types -------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file implements integer type expansion and promotion for LegalizeTypes.
10 // Promotion is the act of changing a computation in an illegal type into a
11 // computation in a larger type. For example, implementing i8 arithmetic in an
12 // i32 register (often needed on powerpc).
13 // Expansion is the act of changing a computation in an illegal type into a
14 // computation in two identical registers of a smaller type. For example,
15 // implementing i64 arithmetic in two i32 registers (often needed on 32-bit
16 // targets).
17 //
18 //===----------------------------------------------------------------------===//
19 
20 #include "LegalizeTypes.h"
22 #include "llvm/IR/DerivedTypes.h"
24 #include "llvm/Support/KnownBits.h"
26 using namespace llvm;
27 
28 #define DEBUG_TYPE "legalize-types"
29 
30 //===----------------------------------------------------------------------===//
31 // Integer Result Promotion
32 //===----------------------------------------------------------------------===//
33 
34 /// PromoteIntegerResult - This method is called when a result of a node is
35 /// found to be in need of promotion to a larger type. At this point, the node
36 /// may also have invalid operands or may have other results that need
37 /// expansion, we just know that (at least) one result needs promotion.
38 void DAGTypeLegalizer::PromoteIntegerResult(SDNode *N, unsigned ResNo) {
39  LLVM_DEBUG(dbgs() << "Promote integer result: "; N->dump(&DAG);
40  dbgs() << "\n");
41  SDValue Res = SDValue();
42 
43  // See if the target wants to custom expand this node.
44  if (CustomLowerNode(N, N->getValueType(ResNo), true)) {
45  LLVM_DEBUG(dbgs() << "Node has been custom expanded, done\n");
46  return;
47  }
48 
49  switch (N->getOpcode()) {
50  default:
51 #ifndef NDEBUG
52  dbgs() << "PromoteIntegerResult #" << ResNo << ": ";
53  N->dump(&DAG); dbgs() << "\n";
54 #endif
55  llvm_unreachable("Do not know how to promote this operator!");
56  case ISD::MERGE_VALUES:Res = PromoteIntRes_MERGE_VALUES(N, ResNo); break;
57  case ISD::AssertSext: Res = PromoteIntRes_AssertSext(N); break;
58  case ISD::AssertZext: Res = PromoteIntRes_AssertZext(N); break;
59  case ISD::BITCAST: Res = PromoteIntRes_BITCAST(N); break;
60  case ISD::BITREVERSE: Res = PromoteIntRes_BITREVERSE(N); break;
61  case ISD::BSWAP: Res = PromoteIntRes_BSWAP(N); break;
62  case ISD::BUILD_PAIR: Res = PromoteIntRes_BUILD_PAIR(N); break;
63  case ISD::Constant: Res = PromoteIntRes_Constant(N); break;
65  case ISD::CTLZ: Res = PromoteIntRes_CTLZ(N); break;
66  case ISD::PARITY:
67  case ISD::CTPOP: Res = PromoteIntRes_CTPOP_PARITY(N); break;
69  case ISD::CTTZ: Res = PromoteIntRes_CTTZ(N); break;
71  Res = PromoteIntRes_EXTRACT_VECTOR_ELT(N); break;
72  case ISD::LOAD: Res = PromoteIntRes_LOAD(cast<LoadSDNode>(N)); break;
73  case ISD::MLOAD: Res = PromoteIntRes_MLOAD(cast<MaskedLoadSDNode>(N));
74  break;
75  case ISD::MGATHER: Res = PromoteIntRes_MGATHER(cast<MaskedGatherSDNode>(N));
76  break;
77  case ISD::SELECT: Res = PromoteIntRes_SELECT(N); break;
78  case ISD::VSELECT: Res = PromoteIntRes_VSELECT(N); break;
79  case ISD::SELECT_CC: Res = PromoteIntRes_SELECT_CC(N); break;
80  case ISD::STRICT_FSETCC:
82  case ISD::SETCC: Res = PromoteIntRes_SETCC(N); break;
83  case ISD::SMIN:
84  case ISD::SMAX:
85  Res = PromoteIntRes_SExtIntBinOp(N, /*IsVP*/ false);
86  break;
87  case ISD::UMIN:
88  case ISD::UMAX: Res = PromoteIntRes_UMINUMAX(N); break;
89 
90  case ISD::SHL:
91  Res = PromoteIntRes_SHL(N, /*IsVP*/ false);
92  break;
94  Res = PromoteIntRes_SIGN_EXTEND_INREG(N); break;
95  case ISD::SRA:
96  Res = PromoteIntRes_SRA(N, /*IsVP*/ false);
97  break;
98  case ISD::SRL:
99  Res = PromoteIntRes_SRL(N, /*IsVP*/ false);
100  break;
101  case ISD::TRUNCATE: Res = PromoteIntRes_TRUNCATE(N); break;
102  case ISD::UNDEF: Res = PromoteIntRes_UNDEF(N); break;
103  case ISD::VAARG: Res = PromoteIntRes_VAARG(N); break;
104  case ISD::VSCALE: Res = PromoteIntRes_VSCALE(N); break;
105 
107  Res = PromoteIntRes_EXTRACT_SUBVECTOR(N); break;
109  Res = PromoteIntRes_INSERT_SUBVECTOR(N); break;
110  case ISD::VECTOR_REVERSE:
111  Res = PromoteIntRes_VECTOR_REVERSE(N); break;
112  case ISD::VECTOR_SHUFFLE:
113  Res = PromoteIntRes_VECTOR_SHUFFLE(N); break;
114  case ISD::VECTOR_SPLICE:
115  Res = PromoteIntRes_VECTOR_SPLICE(N); break;
117  Res = PromoteIntRes_INSERT_VECTOR_ELT(N); break;
118  case ISD::BUILD_VECTOR:
119  Res = PromoteIntRes_BUILD_VECTOR(N); break;
121  Res = PromoteIntRes_SCALAR_TO_VECTOR(N); break;
122  case ISD::SPLAT_VECTOR:
123  Res = PromoteIntRes_SPLAT_VECTOR(N); break;
124  case ISD::STEP_VECTOR: Res = PromoteIntRes_STEP_VECTOR(N); break;
125  case ISD::CONCAT_VECTORS:
126  Res = PromoteIntRes_CONCAT_VECTORS(N); break;
127 
131  Res = PromoteIntRes_EXTEND_VECTOR_INREG(N); break;
132 
133  case ISD::SIGN_EXTEND:
134  case ISD::ZERO_EXTEND:
135  case ISD::ANY_EXTEND: Res = PromoteIntRes_INT_EXTEND(N); break;
136 
139  case ISD::FP_TO_SINT:
140  case ISD::FP_TO_UINT: Res = PromoteIntRes_FP_TO_XINT(N); break;
141 
142  case ISD::FP_TO_SINT_SAT:
143  case ISD::FP_TO_UINT_SAT:
144  Res = PromoteIntRes_FP_TO_XINT_SAT(N); break;
145 
146  case ISD::FP_TO_FP16: Res = PromoteIntRes_FP_TO_FP16(N); break;
147 
148  case ISD::FLT_ROUNDS_: Res = PromoteIntRes_FLT_ROUNDS(N); break;
149 
150  case ISD::AND:
151  case ISD::OR:
152  case ISD::XOR:
153  case ISD::ADD:
154  case ISD::SUB:
155  case ISD::MUL:
156  Res = PromoteIntRes_SimpleIntBinOp(N, /*IsVP*/ false);
157  break;
158 
159  case ISD::SDIV:
160  case ISD::SREM:
161  Res = PromoteIntRes_SExtIntBinOp(N, /*IsVP*/ false);
162  break;
163 
164  case ISD::UDIV:
165  case ISD::UREM:
166  Res = PromoteIntRes_ZExtIntBinOp(N, /*IsVP*/ false);
167  break;
168 
169  case ISD::SADDO:
170  case ISD::SSUBO: Res = PromoteIntRes_SADDSUBO(N, ResNo); break;
171  case ISD::UADDO:
172  case ISD::USUBO: Res = PromoteIntRes_UADDSUBO(N, ResNo); break;
173  case ISD::SMULO:
174  case ISD::UMULO: Res = PromoteIntRes_XMULO(N, ResNo); break;
175 
176  case ISD::ADDE:
177  case ISD::SUBE:
178  case ISD::ADDCARRY:
179  case ISD::SUBCARRY: Res = PromoteIntRes_ADDSUBCARRY(N, ResNo); break;
180 
181  case ISD::SADDO_CARRY:
182  case ISD::SSUBO_CARRY: Res = PromoteIntRes_SADDSUBO_CARRY(N, ResNo); break;
183 
184  case ISD::SADDSAT:
185  case ISD::UADDSAT:
186  case ISD::SSUBSAT:
187  case ISD::USUBSAT:
188  case ISD::SSHLSAT:
189  case ISD::USHLSAT: Res = PromoteIntRes_ADDSUBSHLSAT(N); break;
190 
191  case ISD::SMULFIX:
192  case ISD::SMULFIXSAT:
193  case ISD::UMULFIX:
194  case ISD::UMULFIXSAT: Res = PromoteIntRes_MULFIX(N); break;
195 
196  case ISD::SDIVFIX:
197  case ISD::SDIVFIXSAT:
198  case ISD::UDIVFIX:
199  case ISD::UDIVFIXSAT: Res = PromoteIntRes_DIVFIX(N); break;
200 
201  case ISD::ABS: Res = PromoteIntRes_ABS(N); break;
202 
203  case ISD::ATOMIC_LOAD:
204  Res = PromoteIntRes_Atomic0(cast<AtomicSDNode>(N)); break;
205 
210  case ISD::ATOMIC_LOAD_OR:
217  case ISD::ATOMIC_SWAP:
218  Res = PromoteIntRes_Atomic1(cast<AtomicSDNode>(N)); break;
219 
222  Res = PromoteIntRes_AtomicCmpSwap(cast<AtomicSDNode>(N), ResNo);
223  break;
224 
225  case ISD::VECREDUCE_ADD:
226  case ISD::VECREDUCE_MUL:
227  case ISD::VECREDUCE_AND:
228  case ISD::VECREDUCE_OR:
229  case ISD::VECREDUCE_XOR:
230  case ISD::VECREDUCE_SMAX:
231  case ISD::VECREDUCE_SMIN:
232  case ISD::VECREDUCE_UMAX:
233  case ISD::VECREDUCE_UMIN:
234  Res = PromoteIntRes_VECREDUCE(N);
235  break;
236 
237  case ISD::VP_REDUCE_ADD:
238  case ISD::VP_REDUCE_MUL:
239  case ISD::VP_REDUCE_AND:
240  case ISD::VP_REDUCE_OR:
241  case ISD::VP_REDUCE_XOR:
242  case ISD::VP_REDUCE_SMAX:
243  case ISD::VP_REDUCE_SMIN:
244  case ISD::VP_REDUCE_UMAX:
245  case ISD::VP_REDUCE_UMIN:
246  Res = PromoteIntRes_VP_REDUCE(N);
247  break;
248 
249  case ISD::FREEZE:
250  Res = PromoteIntRes_FREEZE(N);
251  break;
252 
253  case ISD::ROTL:
254  case ISD::ROTR:
255  Res = PromoteIntRes_Rotate(N);
256  break;
257 
258  case ISD::FSHL:
259  case ISD::FSHR:
260  Res = PromoteIntRes_FunnelShift(N);
261  break;
262 
263  case ISD::VP_AND:
264  case ISD::VP_OR:
265  case ISD::VP_XOR:
266  case ISD::VP_ADD:
267  case ISD::VP_SUB:
268  case ISD::VP_MUL:
269  Res = PromoteIntRes_SimpleIntBinOp(N, /*IsVP*/ true);
270  break;
271  case ISD::VP_SDIV:
272  case ISD::VP_SREM:
273  Res = PromoteIntRes_SExtIntBinOp(N, /*IsVP*/ true);
274  break;
275  case ISD::VP_UDIV:
276  case ISD::VP_UREM:
277  Res = PromoteIntRes_ZExtIntBinOp(N, /*IsVP*/ true);
278  break;
279  case ISD::VP_SHL:
280  Res = PromoteIntRes_SHL(N, /*IsVP*/ true);
281  break;
282  case ISD::VP_ASHR:
283  Res = PromoteIntRes_SRA(N, /*IsVP*/ true);
284  break;
285  case ISD::VP_LSHR:
286  Res = PromoteIntRes_SRL(N, /*IsVP*/ true);
287  break;
288  }
289 
290  // If the result is null then the sub-method took care of registering it.
291  if (Res.getNode())
292  SetPromotedInteger(SDValue(N, ResNo), Res);
293 }
294 
295 SDValue DAGTypeLegalizer::PromoteIntRes_MERGE_VALUES(SDNode *N,
296  unsigned ResNo) {
297  SDValue Op = DisintegrateMERGE_VALUES(N, ResNo);
298  return GetPromotedInteger(Op);
299 }
300 
301 SDValue DAGTypeLegalizer::PromoteIntRes_AssertSext(SDNode *N) {
302  // Sign-extend the new bits, and continue the assertion.
303  SDValue Op = SExtPromotedInteger(N->getOperand(0));
304  return DAG.getNode(ISD::AssertSext, SDLoc(N),
305  Op.getValueType(), Op, N->getOperand(1));
306 }
307 
308 SDValue DAGTypeLegalizer::PromoteIntRes_AssertZext(SDNode *N) {
309  // Zero the new bits, and continue the assertion.
310  SDValue Op = ZExtPromotedInteger(N->getOperand(0));
311  return DAG.getNode(ISD::AssertZext, SDLoc(N),
312  Op.getValueType(), Op, N->getOperand(1));
313 }
314 
315 SDValue DAGTypeLegalizer::PromoteIntRes_Atomic0(AtomicSDNode *N) {
316  EVT ResVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
317  SDValue Res = DAG.getAtomic(N->getOpcode(), SDLoc(N),
318  N->getMemoryVT(), ResVT,
319  N->getChain(), N->getBasePtr(),
320  N->getMemOperand());
321  // Legalize the chain result - switch anything that used the old chain to
322  // use the new one.
323  ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
324  return Res;
325 }
326 
327 SDValue DAGTypeLegalizer::PromoteIntRes_Atomic1(AtomicSDNode *N) {
328  SDValue Op2 = GetPromotedInteger(N->getOperand(2));
329  SDValue Res = DAG.getAtomic(N->getOpcode(), SDLoc(N),
330  N->getMemoryVT(),
331  N->getChain(), N->getBasePtr(),
332  Op2, N->getMemOperand());
333  // Legalize the chain result - switch anything that used the old chain to
334  // use the new one.
335  ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
336  return Res;
337 }
338 
339 SDValue DAGTypeLegalizer::PromoteIntRes_AtomicCmpSwap(AtomicSDNode *N,
340  unsigned ResNo) {
341  if (ResNo == 1) {
342  assert(N->getOpcode() == ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS);
343  EVT SVT = getSetCCResultType(N->getOperand(2).getValueType());
344  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(1));
345 
346  // Only use the result of getSetCCResultType if it is legal,
347  // otherwise just use the promoted result type (NVT).
348  if (!TLI.isTypeLegal(SVT))
349  SVT = NVT;
350 
351  SDVTList VTs = DAG.getVTList(N->getValueType(0), SVT, MVT::Other);
352  SDValue Res = DAG.getAtomicCmpSwap(
353  ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS, SDLoc(N), N->getMemoryVT(), VTs,
354  N->getChain(), N->getBasePtr(), N->getOperand(2), N->getOperand(3),
355  N->getMemOperand());
356  ReplaceValueWith(SDValue(N, 0), Res.getValue(0));
357  ReplaceValueWith(SDValue(N, 2), Res.getValue(2));
358  return Res.getValue(1);
359  }
360 
361  // Op2 is used for the comparison and thus must be extended according to the
362  // target's atomic operations. Op3 is merely stored and so can be left alone.
363  SDValue Op2 = N->getOperand(2);
364  SDValue Op3 = GetPromotedInteger(N->getOperand(3));
365  switch (TLI.getExtendForAtomicCmpSwapArg()) {
366  case ISD::SIGN_EXTEND:
367  Op2 = SExtPromotedInteger(Op2);
368  break;
369  case ISD::ZERO_EXTEND:
370  Op2 = ZExtPromotedInteger(Op2);
371  break;
372  case ISD::ANY_EXTEND:
373  Op2 = GetPromotedInteger(Op2);
374  break;
375  default:
376  llvm_unreachable("Invalid atomic op extension");
377  }
378 
379  SDVTList VTs =
380  DAG.getVTList(Op2.getValueType(), N->getValueType(1), MVT::Other);
381  SDValue Res = DAG.getAtomicCmpSwap(
382  N->getOpcode(), SDLoc(N), N->getMemoryVT(), VTs, N->getChain(),
383  N->getBasePtr(), Op2, Op3, N->getMemOperand());
384  // Update the use to N with the newly created Res.
385  for (unsigned i = 1, NumResults = N->getNumValues(); i < NumResults; ++i)
386  ReplaceValueWith(SDValue(N, i), Res.getValue(i));
387  return Res;
388 }
389 
390 SDValue DAGTypeLegalizer::PromoteIntRes_BITCAST(SDNode *N) {
391  SDValue InOp = N->getOperand(0);
392  EVT InVT = InOp.getValueType();
393  EVT NInVT = TLI.getTypeToTransformTo(*DAG.getContext(), InVT);
394  EVT OutVT = N->getValueType(0);
395  EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT);
396  SDLoc dl(N);
397 
398  switch (getTypeAction(InVT)) {
400  break;
402  if (NOutVT.bitsEq(NInVT) && !NOutVT.isVector() && !NInVT.isVector())
403  // The input promotes to the same size. Convert the promoted value.
404  return DAG.getNode(ISD::BITCAST, dl, NOutVT, GetPromotedInteger(InOp));
405  break;
407  // Promote the integer operand by hand.
408  return DAG.getNode(ISD::ANY_EXTEND, dl, NOutVT, GetSoftenedFloat(InOp));
410  // Promote the integer operand by hand.
411  return DAG.getNode(ISD::ANY_EXTEND, dl, NOutVT, GetSoftPromotedHalf(InOp));
413  // Convert the promoted float by hand.
414  if (!NOutVT.isVector())
415  return DAG.getNode(ISD::FP_TO_FP16, dl, NOutVT, GetPromotedFloat(InOp));
416  break;
417  }
420  break;
422  // Convert the element to an integer and promote it by hand.
423  if (!NOutVT.isVector())
424  return DAG.getNode(ISD::ANY_EXTEND, dl, NOutVT,
425  BitConvertToInteger(GetScalarizedVector(InOp)));
426  break;
428  report_fatal_error("Scalarization of scalable vectors is not supported.");
430  if (!NOutVT.isVector()) {
431  // For example, i32 = BITCAST v2i16 on alpha. Convert the split
432  // pieces of the input into integers and reassemble in the final type.
433  SDValue Lo, Hi;
434  GetSplitVector(N->getOperand(0), Lo, Hi);
435  Lo = BitConvertToInteger(Lo);
436  Hi = BitConvertToInteger(Hi);
437 
438  if (DAG.getDataLayout().isBigEndian())
439  std::swap(Lo, Hi);
440 
441  InOp = DAG.getNode(ISD::ANY_EXTEND, dl,
443  NOutVT.getSizeInBits()),
444  JoinIntegers(Lo, Hi));
445  return DAG.getNode(ISD::BITCAST, dl, NOutVT, InOp);
446  }
447  break;
448  }
450  // The input is widened to the same size. Convert to the widened value.
451  // Make sure that the outgoing value is not a vector, because this would
452  // make us bitcast between two vectors which are legalized in different ways.
453  if (NOutVT.bitsEq(NInVT) && !NOutVT.isVector()) {
454  SDValue Res =
455  DAG.getNode(ISD::BITCAST, dl, NOutVT, GetWidenedVector(InOp));
456 
457  // For big endian targets we need to shift the casted value or the
458  // interesting bits will end up at the wrong place.
459  if (DAG.getDataLayout().isBigEndian()) {
460  unsigned ShiftAmt = NInVT.getSizeInBits() - InVT.getSizeInBits();
461  EVT ShiftAmtTy = TLI.getShiftAmountTy(NOutVT, DAG.getDataLayout());
462  assert(ShiftAmt < NOutVT.getSizeInBits() && "Too large shift amount!");
463  Res = DAG.getNode(ISD::SRL, dl, NOutVT, Res,
464  DAG.getConstant(ShiftAmt, dl, ShiftAmtTy));
465  }
466  return Res;
467  }
468  // If the output type is also a vector and widening it to the same size
469  // as the widened input type would be a legal type, we can widen the bitcast
470  // and handle the promotion after.
471  if (NOutVT.isVector()) {
472  unsigned WidenInSize = NInVT.getSizeInBits();
473  unsigned OutSize = OutVT.getSizeInBits();
474  if (WidenInSize % OutSize == 0) {
475  unsigned Scale = WidenInSize / OutSize;
476  EVT WideOutVT = EVT::getVectorVT(*DAG.getContext(),
477  OutVT.getVectorElementType(),
478  OutVT.getVectorNumElements() * Scale);
479  if (isTypeLegal(WideOutVT)) {
480  InOp = DAG.getBitcast(WideOutVT, GetWidenedVector(InOp));
481  InOp = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, OutVT, InOp,
482  DAG.getVectorIdxConstant(0, dl));
483  return DAG.getNode(ISD::ANY_EXTEND, dl, NOutVT, InOp);
484  }
485  }
486  }
487  }
488 
489  return DAG.getNode(ISD::ANY_EXTEND, dl, NOutVT,
490  CreateStackStoreLoad(InOp, OutVT));
491 }
492 
493 // Helper for BSWAP/BITREVERSE promotion to ensure we can fit any shift amount
494 // in the VT returned by getShiftAmountTy and to return a safe VT if we can't.
496  SelectionDAG &DAG) {
497  EVT ShiftVT = TLI.getShiftAmountTy(VT, DAG.getDataLayout());
498  // If any possible shift value won't fit in the prefered type, just use
499  // something safe. It will be legalized when the shift is expanded.
500  if (!ShiftVT.isVector() &&
501  ShiftVT.getSizeInBits() < Log2_32_Ceil(VT.getSizeInBits()))
502  ShiftVT = MVT::i32;
503  return ShiftVT;
504 }
505 
506 SDValue DAGTypeLegalizer::PromoteIntRes_FREEZE(SDNode *N) {
507  SDValue V = GetPromotedInteger(N->getOperand(0));
508  return DAG.getNode(ISD::FREEZE, SDLoc(N),
509  V.getValueType(), V);
510 }
511 
512 SDValue DAGTypeLegalizer::PromoteIntRes_BSWAP(SDNode *N) {
513  SDValue Op = GetPromotedInteger(N->getOperand(0));
514  EVT OVT = N->getValueType(0);
515  EVT NVT = Op.getValueType();
516  SDLoc dl(N);
517 
518  // If the larger BSWAP isn't supported by the target, try to expand now.
519  // If we expand later we'll end up with more operations since we lost the
520  // original type. We only do this for scalars since we have a shuffle
521  // based lowering for vectors in LegalizeVectorOps.
522  if (!OVT.isVector() &&
524  if (SDValue Res = TLI.expandBSWAP(N, DAG))
525  return DAG.getNode(ISD::ANY_EXTEND, dl, NVT, Res);
526  }
527 
528  unsigned DiffBits = NVT.getScalarSizeInBits() - OVT.getScalarSizeInBits();
529  EVT ShiftVT = getShiftAmountTyForConstant(NVT, TLI, DAG);
530  return DAG.getNode(ISD::SRL, dl, NVT, DAG.getNode(ISD::BSWAP, dl, NVT, Op),
531  DAG.getConstant(DiffBits, dl, ShiftVT));
532 }
533 
534 SDValue DAGTypeLegalizer::PromoteIntRes_BITREVERSE(SDNode *N) {
535  SDValue Op = GetPromotedInteger(N->getOperand(0));
536  EVT OVT = N->getValueType(0);
537  EVT NVT = Op.getValueType();
538  SDLoc dl(N);
539 
540  // If the larger BITREVERSE isn't supported by the target, try to expand now.
541  // If we expand later we'll end up with more operations since we lost the
542  // original type. We only do this for scalars since we have a shuffle
543  // based lowering for vectors in LegalizeVectorOps.
544  if (!OVT.isVector() && OVT.isSimple() &&
546  if (SDValue Res = TLI.expandBITREVERSE(N, DAG))
547  return DAG.getNode(ISD::ANY_EXTEND, dl, NVT, Res);
548  }
549 
550  unsigned DiffBits = NVT.getScalarSizeInBits() - OVT.getScalarSizeInBits();
551  EVT ShiftVT = getShiftAmountTyForConstant(NVT, TLI, DAG);
552  return DAG.getNode(ISD::SRL, dl, NVT,
553  DAG.getNode(ISD::BITREVERSE, dl, NVT, Op),
554  DAG.getConstant(DiffBits, dl, ShiftVT));
555 }
556 
557 SDValue DAGTypeLegalizer::PromoteIntRes_BUILD_PAIR(SDNode *N) {
558  // The pair element type may be legal, or may not promote to the same type as
559  // the result, for example i14 = BUILD_PAIR (i7, i7). Handle all cases.
560  return DAG.getNode(ISD::ANY_EXTEND, SDLoc(N),
561  TLI.getTypeToTransformTo(*DAG.getContext(),
562  N->getValueType(0)), JoinIntegers(N->getOperand(0),
563  N->getOperand(1)));
564 }
565 
566 SDValue DAGTypeLegalizer::PromoteIntRes_Constant(SDNode *N) {
567  EVT VT = N->getValueType(0);
568  // FIXME there is no actual debug info here
569  SDLoc dl(N);
570  // Zero extend things like i1, sign extend everything else. It shouldn't
571  // matter in theory which one we pick, but this tends to give better code?
572  unsigned Opc = VT.isByteSized() ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND;
573  SDValue Result = DAG.getNode(Opc, dl,
574  TLI.getTypeToTransformTo(*DAG.getContext(), VT),
575  SDValue(N, 0));
576  assert(isa<ConstantSDNode>(Result) && "Didn't constant fold ext?");
577  return Result;
578 }
579 
580 SDValue DAGTypeLegalizer::PromoteIntRes_CTLZ(SDNode *N) {
581  // Zero extend to the promoted type and do the count there.
582  SDValue Op = ZExtPromotedInteger(N->getOperand(0));
583  SDLoc dl(N);
584  EVT OVT = N->getValueType(0);
585  EVT NVT = Op.getValueType();
586  Op = DAG.getNode(N->getOpcode(), dl, NVT, Op);
587  // Subtract off the extra leading bits in the bigger type.
588  return DAG.getNode(
589  ISD::SUB, dl, NVT, Op,
591  NVT));
592 }
593 
594 SDValue DAGTypeLegalizer::PromoteIntRes_CTPOP_PARITY(SDNode *N) {
595  // Zero extend to the promoted type and do the count or parity there.
596  SDValue Op = ZExtPromotedInteger(N->getOperand(0));
597  return DAG.getNode(N->getOpcode(), SDLoc(N), Op.getValueType(), Op);
598 }
599 
600 SDValue DAGTypeLegalizer::PromoteIntRes_CTTZ(SDNode *N) {
601  SDValue Op = GetPromotedInteger(N->getOperand(0));
602  EVT OVT = N->getValueType(0);
603  EVT NVT = Op.getValueType();
604  SDLoc dl(N);
605  if (N->getOpcode() == ISD::CTTZ) {
606  // The count is the same in the promoted type except if the original
607  // value was zero. This can be handled by setting the bit just off
608  // the top of the original type.
609  auto TopBit = APInt::getOneBitSet(NVT.getScalarSizeInBits(),
610  OVT.getScalarSizeInBits());
611  Op = DAG.getNode(ISD::OR, dl, NVT, Op, DAG.getConstant(TopBit, dl, NVT));
612  }
613  return DAG.getNode(N->getOpcode(), dl, NVT, Op);
614 }
615 
616 SDValue DAGTypeLegalizer::PromoteIntRes_EXTRACT_VECTOR_ELT(SDNode *N) {
617  SDLoc dl(N);
618  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
619 
620  SDValue Op0 = N->getOperand(0);
621  SDValue Op1 = N->getOperand(1);
622 
623  // If the input also needs to be promoted, do that first so we can get a
624  // get a good idea for the output type.
625  if (TLI.getTypeAction(*DAG.getContext(), Op0.getValueType())
627  SDValue In = GetPromotedInteger(Op0);
628 
629  // If the new type is larger than NVT, use it. We probably won't need to
630  // promote it again.
631  EVT SVT = In.getValueType().getScalarType();
632  if (SVT.bitsGE(NVT)) {
633  SDValue Ext = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, SVT, In, Op1);
634  return DAG.getAnyExtOrTrunc(Ext, dl, NVT);
635  }
636  }
637 
638  return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, NVT, Op0, Op1);
639 }
640 
641 SDValue DAGTypeLegalizer::PromoteIntRes_FP_TO_XINT(SDNode *N) {
642  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
643  unsigned NewOpc = N->getOpcode();
644  SDLoc dl(N);
645 
646  // If we're promoting a UINT to a larger size and the larger FP_TO_UINT is
647  // not Legal, check to see if we can use FP_TO_SINT instead. (If both UINT
648  // and SINT conversions are Custom, there is no way to tell which is
649  // preferable. We choose SINT because that's the right thing on PPC.)
650  if (N->getOpcode() == ISD::FP_TO_UINT &&
651  !TLI.isOperationLegal(ISD::FP_TO_UINT, NVT) &&
653  NewOpc = ISD::FP_TO_SINT;
654 
655  if (N->getOpcode() == ISD::STRICT_FP_TO_UINT &&
658  NewOpc = ISD::STRICT_FP_TO_SINT;
659 
660  SDValue Res;
661  if (N->isStrictFPOpcode()) {
662  Res = DAG.getNode(NewOpc, dl, {NVT, MVT::Other},
663  {N->getOperand(0), N->getOperand(1)});
664  // Legalize the chain result - switch anything that used the old chain to
665  // use the new one.
666  ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
667  } else
668  Res = DAG.getNode(NewOpc, dl, NVT, N->getOperand(0));
669 
670  // Assert that the converted value fits in the original type. If it doesn't
671  // (eg: because the value being converted is too big), then the result of the
672  // original operation was undefined anyway, so the assert is still correct.
673  //
674  // NOTE: fp-to-uint to fp-to-sint promotion guarantees zero extend. For example:
675  // before legalization: fp-to-uint16, 65534. -> 0xfffe
676  // after legalization: fp-to-sint32, 65534. -> 0x0000fffe
677  return DAG.getNode((N->getOpcode() == ISD::FP_TO_UINT ||
678  N->getOpcode() == ISD::STRICT_FP_TO_UINT) ?
679  ISD::AssertZext : ISD::AssertSext, dl, NVT, Res,
680  DAG.getValueType(N->getValueType(0).getScalarType()));
681 }
682 
683 SDValue DAGTypeLegalizer::PromoteIntRes_FP_TO_XINT_SAT(SDNode *N) {
684  // Promote the result type, while keeping the original width in Op1.
685  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
686  SDLoc dl(N);
687  return DAG.getNode(N->getOpcode(), dl, NVT, N->getOperand(0),
688  N->getOperand(1));
689 }
690 
691 SDValue DAGTypeLegalizer::PromoteIntRes_FP_TO_FP16(SDNode *N) {
692  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
693  SDLoc dl(N);
694 
695  return DAG.getNode(N->getOpcode(), dl, NVT, N->getOperand(0));
696 }
697 
698 SDValue DAGTypeLegalizer::PromoteIntRes_FLT_ROUNDS(SDNode *N) {
699  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
700  SDLoc dl(N);
701 
702  SDValue Res =
703  DAG.getNode(N->getOpcode(), dl, {NVT, MVT::Other}, N->getOperand(0));
704 
705  // Legalize the chain result - switch anything that used the old chain to
706  // use the new one.
707  ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
708  return Res;
709 }
710 
711 SDValue DAGTypeLegalizer::PromoteIntRes_INT_EXTEND(SDNode *N) {
712  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
713  SDLoc dl(N);
714 
715  if (getTypeAction(N->getOperand(0).getValueType())
717  SDValue Res = GetPromotedInteger(N->getOperand(0));
718  assert(Res.getValueType().bitsLE(NVT) && "Extension doesn't make sense!");
719 
720  // If the result and operand types are the same after promotion, simplify
721  // to an in-register extension.
722  if (NVT == Res.getValueType()) {
723  // The high bits are not guaranteed to be anything. Insert an extend.
724  if (N->getOpcode() == ISD::SIGN_EXTEND)
725  return DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, NVT, Res,
726  DAG.getValueType(N->getOperand(0).getValueType()));
727  if (N->getOpcode() == ISD::ZERO_EXTEND)
728  return DAG.getZeroExtendInReg(Res, dl, N->getOperand(0).getValueType());
729  assert(N->getOpcode() == ISD::ANY_EXTEND && "Unknown integer extension!");
730  return Res;
731  }
732  }
733 
734  // Otherwise, just extend the original operand all the way to the larger type.
735  return DAG.getNode(N->getOpcode(), dl, NVT, N->getOperand(0));
736 }
737 
738 SDValue DAGTypeLegalizer::PromoteIntRes_LOAD(LoadSDNode *N) {
739  assert(ISD::isUNINDEXEDLoad(N) && "Indexed load during type legalization!");
740  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
741  ISD::LoadExtType ExtType =
742  ISD::isNON_EXTLoad(N) ? ISD::EXTLOAD : N->getExtensionType();
743  SDLoc dl(N);
744  SDValue Res = DAG.getExtLoad(ExtType, dl, NVT, N->getChain(), N->getBasePtr(),
745  N->getMemoryVT(), N->getMemOperand());
746 
747  // Legalize the chain result - switch anything that used the old chain to
748  // use the new one.
749  ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
750  return Res;
751 }
752 
753 SDValue DAGTypeLegalizer::PromoteIntRes_MLOAD(MaskedLoadSDNode *N) {
754  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
755  SDValue ExtPassThru = GetPromotedInteger(N->getPassThru());
756 
757  SDLoc dl(N);
758  SDValue Res = DAG.getMaskedLoad(NVT, dl, N->getChain(), N->getBasePtr(),
759  N->getOffset(), N->getMask(), ExtPassThru,
760  N->getMemoryVT(), N->getMemOperand(),
761  N->getAddressingMode(), ISD::EXTLOAD);
762  // Legalize the chain result - switch anything that used the old chain to
763  // use the new one.
764  ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
765  return Res;
766 }
767 
768 SDValue DAGTypeLegalizer::PromoteIntRes_MGATHER(MaskedGatherSDNode *N) {
769  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
770  SDValue ExtPassThru = GetPromotedInteger(N->getPassThru());
771  assert(NVT == ExtPassThru.getValueType() &&
772  "Gather result type and the passThru argument type should be the same");
773 
774  ISD::LoadExtType ExtType = N->getExtensionType();
775  if (ExtType == ISD::NON_EXTLOAD)
776  ExtType = ISD::EXTLOAD;
777 
778  SDLoc dl(N);
779  SDValue Ops[] = {N->getChain(), ExtPassThru, N->getMask(), N->getBasePtr(),
780  N->getIndex(), N->getScale() };
781  SDValue Res = DAG.getMaskedGather(DAG.getVTList(NVT, MVT::Other),
782  N->getMemoryVT(), dl, Ops,
783  N->getMemOperand(), N->getIndexType(),
784  ExtType);
785  // Legalize the chain result - switch anything that used the old chain to
786  // use the new one.
787  ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
788  return Res;
789 }
790 
791 /// Promote the overflow flag of an overflowing arithmetic node.
792 SDValue DAGTypeLegalizer::PromoteIntRes_Overflow(SDNode *N) {
793  // Change the return type of the boolean result while obeying
794  // getSetCCResultType.
795  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(1));
796  EVT VT = N->getValueType(0);
797  EVT SVT = getSetCCResultType(VT);
798  SDValue Ops[3] = { N->getOperand(0), N->getOperand(1) };
799  unsigned NumOps = N->getNumOperands();
800  assert(NumOps <= 3 && "Too many operands");
801  if (NumOps == 3)
802  Ops[2] = N->getOperand(2);
803 
804  SDLoc dl(N);
805  SDValue Res = DAG.getNode(N->getOpcode(), dl, DAG.getVTList(VT, SVT),
806  makeArrayRef(Ops, NumOps));
807 
808  // Modified the sum result - switch anything that used the old sum to use
809  // the new one.
810  ReplaceValueWith(SDValue(N, 0), Res);
811 
812  // Convert to the expected type.
813  return DAG.getBoolExtOrTrunc(Res.getValue(1), dl, NVT, VT);
814 }
815 
816 SDValue DAGTypeLegalizer::PromoteIntRes_ADDSUBSHLSAT(SDNode *N) {
817  // If the promoted type is legal, we can convert this to:
818  // 1. ANY_EXTEND iN to iM
819  // 2. SHL by M-N
820  // 3. [US][ADD|SUB|SHL]SAT
821  // 4. L/ASHR by M-N
822  // Else it is more efficient to convert this to a min and a max
823  // operation in the higher precision arithmetic.
824  SDLoc dl(N);
825  SDValue Op1 = N->getOperand(0);
826  SDValue Op2 = N->getOperand(1);
827  unsigned OldBits = Op1.getScalarValueSizeInBits();
828 
829  unsigned Opcode = N->getOpcode();
830  bool IsShift = Opcode == ISD::USHLSAT || Opcode == ISD::SSHLSAT;
831 
832  SDValue Op1Promoted, Op2Promoted;
833  if (IsShift) {
834  Op1Promoted = GetPromotedInteger(Op1);
835  Op2Promoted = ZExtPromotedInteger(Op2);
836  } else if (Opcode == ISD::UADDSAT || Opcode == ISD::USUBSAT) {
837  Op1Promoted = ZExtPromotedInteger(Op1);
838  Op2Promoted = ZExtPromotedInteger(Op2);
839  } else {
840  Op1Promoted = SExtPromotedInteger(Op1);
841  Op2Promoted = SExtPromotedInteger(Op2);
842  }
843  EVT PromotedType = Op1Promoted.getValueType();
844  unsigned NewBits = PromotedType.getScalarSizeInBits();
845 
846  if (Opcode == ISD::UADDSAT) {
847  APInt MaxVal = APInt::getAllOnes(OldBits).zext(NewBits);
848  SDValue SatMax = DAG.getConstant(MaxVal, dl, PromotedType);
849  SDValue Add =
850  DAG.getNode(ISD::ADD, dl, PromotedType, Op1Promoted, Op2Promoted);
851  return DAG.getNode(ISD::UMIN, dl, PromotedType, Add, SatMax);
852  }
853 
854  // USUBSAT can always be promoted as long as we have zero-extended the args.
855  if (Opcode == ISD::USUBSAT)
856  return DAG.getNode(ISD::USUBSAT, dl, PromotedType, Op1Promoted,
857  Op2Promoted);
858 
859  // Shift cannot use a min/max expansion, we can't detect overflow if all of
860  // the bits have been shifted out.
861  if (IsShift || TLI.isOperationLegal(Opcode, PromotedType)) {
862  unsigned ShiftOp;
863  switch (Opcode) {
864  case ISD::SADDSAT:
865  case ISD::SSUBSAT:
866  case ISD::SSHLSAT:
867  ShiftOp = ISD::SRA;
868  break;
869  case ISD::USHLSAT:
870  ShiftOp = ISD::SRL;
871  break;
872  default:
873  llvm_unreachable("Expected opcode to be signed or unsigned saturation "
874  "addition, subtraction or left shift");
875  }
876 
877  unsigned SHLAmount = NewBits - OldBits;
878  EVT SHVT = TLI.getShiftAmountTy(PromotedType, DAG.getDataLayout());
879  SDValue ShiftAmount = DAG.getConstant(SHLAmount, dl, SHVT);
880  Op1Promoted =
881  DAG.getNode(ISD::SHL, dl, PromotedType, Op1Promoted, ShiftAmount);
882  if (!IsShift)
883  Op2Promoted =
884  DAG.getNode(ISD::SHL, dl, PromotedType, Op2Promoted, ShiftAmount);
885 
886  SDValue Result =
887  DAG.getNode(Opcode, dl, PromotedType, Op1Promoted, Op2Promoted);
888  return DAG.getNode(ShiftOp, dl, PromotedType, Result, ShiftAmount);
889  }
890 
891  unsigned AddOp = Opcode == ISD::SADDSAT ? ISD::ADD : ISD::SUB;
892  APInt MinVal = APInt::getSignedMinValue(OldBits).sext(NewBits);
893  APInt MaxVal = APInt::getSignedMaxValue(OldBits).sext(NewBits);
894  SDValue SatMin = DAG.getConstant(MinVal, dl, PromotedType);
895  SDValue SatMax = DAG.getConstant(MaxVal, dl, PromotedType);
896  SDValue Result =
897  DAG.getNode(AddOp, dl, PromotedType, Op1Promoted, Op2Promoted);
898  Result = DAG.getNode(ISD::SMIN, dl, PromotedType, Result, SatMax);
899  Result = DAG.getNode(ISD::SMAX, dl, PromotedType, Result, SatMin);
900  return Result;
901 }
902 
903 SDValue DAGTypeLegalizer::PromoteIntRes_MULFIX(SDNode *N) {
904  // Can just promote the operands then continue with operation.
905  SDLoc dl(N);
906  SDValue Op1Promoted, Op2Promoted;
907  bool Signed =
908  N->getOpcode() == ISD::SMULFIX || N->getOpcode() == ISD::SMULFIXSAT;
909  bool Saturating =
910  N->getOpcode() == ISD::SMULFIXSAT || N->getOpcode() == ISD::UMULFIXSAT;
911  if (Signed) {
912  Op1Promoted = SExtPromotedInteger(N->getOperand(0));
913  Op2Promoted = SExtPromotedInteger(N->getOperand(1));
914  } else {
915  Op1Promoted = ZExtPromotedInteger(N->getOperand(0));
916  Op2Promoted = ZExtPromotedInteger(N->getOperand(1));
917  }
918  EVT OldType = N->getOperand(0).getValueType();
919  EVT PromotedType = Op1Promoted.getValueType();
920  unsigned DiffSize =
921  PromotedType.getScalarSizeInBits() - OldType.getScalarSizeInBits();
922 
923  if (Saturating) {
924  // Promoting the operand and result values changes the saturation width,
925  // which is extends the values that we clamp to on saturation. This could be
926  // resolved by shifting one of the operands the same amount, which would
927  // also shift the result we compare against, then shifting back.
928  EVT ShiftTy = TLI.getShiftAmountTy(PromotedType, DAG.getDataLayout());
929  Op1Promoted = DAG.getNode(ISD::SHL, dl, PromotedType, Op1Promoted,
930  DAG.getConstant(DiffSize, dl, ShiftTy));
931  SDValue Result = DAG.getNode(N->getOpcode(), dl, PromotedType, Op1Promoted,
932  Op2Promoted, N->getOperand(2));
933  unsigned ShiftOp = Signed ? ISD::SRA : ISD::SRL;
934  return DAG.getNode(ShiftOp, dl, PromotedType, Result,
935  DAG.getConstant(DiffSize, dl, ShiftTy));
936  }
937  return DAG.getNode(N->getOpcode(), dl, PromotedType, Op1Promoted, Op2Promoted,
938  N->getOperand(2));
939 }
940 
942  unsigned SatW, bool Signed,
943  const TargetLowering &TLI,
944  SelectionDAG &DAG) {
945  EVT VT = V.getValueType();
946  unsigned VTW = VT.getScalarSizeInBits();
947 
948  if (!Signed) {
949  // Saturate to the unsigned maximum by getting the minimum of V and the
950  // maximum.
951  return DAG.getNode(ISD::UMIN, dl, VT, V,
952  DAG.getConstant(APInt::getLowBitsSet(VTW, SatW),
953  dl, VT));
954  }
955 
956  // Saturate to the signed maximum (the low SatW - 1 bits) by taking the
957  // signed minimum of it and V.
958  V = DAG.getNode(ISD::SMIN, dl, VT, V,
959  DAG.getConstant(APInt::getLowBitsSet(VTW, SatW - 1),
960  dl, VT));
961  // Saturate to the signed minimum (the high SatW + 1 bits) by taking the
962  // signed maximum of it and V.
963  V = DAG.getNode(ISD::SMAX, dl, VT, V,
964  DAG.getConstant(APInt::getHighBitsSet(VTW, VTW - SatW + 1),
965  dl, VT));
966  return V;
967 }
968 
970  unsigned Scale, const TargetLowering &TLI,
971  SelectionDAG &DAG, unsigned SatW = 0) {
972  EVT VT = LHS.getValueType();
973  unsigned VTSize = VT.getScalarSizeInBits();
974  bool Signed = N->getOpcode() == ISD::SDIVFIX ||
975  N->getOpcode() == ISD::SDIVFIXSAT;
976  bool Saturating = N->getOpcode() == ISD::SDIVFIXSAT ||
977  N->getOpcode() == ISD::UDIVFIXSAT;
978 
979  SDLoc dl(N);
980  // Widen the types by a factor of two. This is guaranteed to expand, since it
981  // will always have enough high bits in the LHS to shift into.
982  EVT WideVT = EVT::getIntegerVT(*DAG.getContext(), VTSize * 2);
983  if (VT.isVector())
984  WideVT = EVT::getVectorVT(*DAG.getContext(), WideVT,
985  VT.getVectorElementCount());
986  if (Signed) {
987  LHS = DAG.getSExtOrTrunc(LHS, dl, WideVT);
988  RHS = DAG.getSExtOrTrunc(RHS, dl, WideVT);
989  } else {
990  LHS = DAG.getZExtOrTrunc(LHS, dl, WideVT);
991  RHS = DAG.getZExtOrTrunc(RHS, dl, WideVT);
992  }
993 
994  SDValue Res = TLI.expandFixedPointDiv(N->getOpcode(), dl, LHS, RHS, Scale,
995  DAG);
996  assert(Res && "Expanding DIVFIX with wide type failed?");
997  if (Saturating) {
998  // If the caller has told us to saturate at something less, use that width
999  // instead of the type before doubling. However, it cannot be more than
1000  // what we just widened!
1001  assert(SatW <= VTSize &&
1002  "Tried to saturate to more than the original type?");
1003  Res = SaturateWidenedDIVFIX(Res, dl, SatW == 0 ? VTSize : SatW, Signed,
1004  TLI, DAG);
1005  }
1006  return DAG.getZExtOrTrunc(Res, dl, VT);
1007 }
1008 
1009 SDValue DAGTypeLegalizer::PromoteIntRes_DIVFIX(SDNode *N) {
1010  SDLoc dl(N);
1011  SDValue Op1Promoted, Op2Promoted;
1012  bool Signed = N->getOpcode() == ISD::SDIVFIX ||
1013  N->getOpcode() == ISD::SDIVFIXSAT;
1014  bool Saturating = N->getOpcode() == ISD::SDIVFIXSAT ||
1015  N->getOpcode() == ISD::UDIVFIXSAT;
1016  if (Signed) {
1017  Op1Promoted = SExtPromotedInteger(N->getOperand(0));
1018  Op2Promoted = SExtPromotedInteger(N->getOperand(1));
1019  } else {
1020  Op1Promoted = ZExtPromotedInteger(N->getOperand(0));
1021  Op2Promoted = ZExtPromotedInteger(N->getOperand(1));
1022  }
1023  EVT PromotedType = Op1Promoted.getValueType();
1024  unsigned Scale = N->getConstantOperandVal(2);
1025 
1026  // If the type is already legal and the operation is legal in that type, we
1027  // should not early expand.
1028  if (TLI.isTypeLegal(PromotedType)) {
1030  TLI.getFixedPointOperationAction(N->getOpcode(), PromotedType, Scale);
1031  if (Action == TargetLowering::Legal || Action == TargetLowering::Custom) {
1032  EVT ShiftTy = TLI.getShiftAmountTy(PromotedType, DAG.getDataLayout());
1033  unsigned Diff = PromotedType.getScalarSizeInBits() -
1034  N->getValueType(0).getScalarSizeInBits();
1035  if (Saturating)
1036  Op1Promoted = DAG.getNode(ISD::SHL, dl, PromotedType, Op1Promoted,
1037  DAG.getConstant(Diff, dl, ShiftTy));
1038  SDValue Res = DAG.getNode(N->getOpcode(), dl, PromotedType, Op1Promoted,
1039  Op2Promoted, N->getOperand(2));
1040  if (Saturating)
1041  Res = DAG.getNode(Signed ? ISD::SRA : ISD::SRL, dl, PromotedType, Res,
1042  DAG.getConstant(Diff, dl, ShiftTy));
1043  return Res;
1044  }
1045  }
1046 
1047  // See if we can perform the division in this type without expanding.
1048  if (SDValue Res = TLI.expandFixedPointDiv(N->getOpcode(), dl, Op1Promoted,
1049  Op2Promoted, Scale, DAG)) {
1050  if (Saturating)
1051  Res = SaturateWidenedDIVFIX(Res, dl,
1052  N->getValueType(0).getScalarSizeInBits(),
1053  Signed, TLI, DAG);
1054  return Res;
1055  }
1056  // If we cannot, expand it to twice the type width. If we are saturating, give
1057  // it the original width as a saturating width so we don't need to emit
1058  // two saturations.
1059  return earlyExpandDIVFIX(N, Op1Promoted, Op2Promoted, Scale, TLI, DAG,
1060  N->getValueType(0).getScalarSizeInBits());
1061 }
1062 
1063 SDValue DAGTypeLegalizer::PromoteIntRes_SADDSUBO(SDNode *N, unsigned ResNo) {
1064  if (ResNo == 1)
1065  return PromoteIntRes_Overflow(N);
1066 
1067  // The operation overflowed iff the result in the larger type is not the
1068  // sign extension of its truncation to the original type.
1069  SDValue LHS = SExtPromotedInteger(N->getOperand(0));
1070  SDValue RHS = SExtPromotedInteger(N->getOperand(1));
1071  EVT OVT = N->getOperand(0).getValueType();
1072  EVT NVT = LHS.getValueType();
1073  SDLoc dl(N);
1074 
1075  // Do the arithmetic in the larger type.
1076  unsigned Opcode = N->getOpcode() == ISD::SADDO ? ISD::ADD : ISD::SUB;
1077  SDValue Res = DAG.getNode(Opcode, dl, NVT, LHS, RHS);
1078 
1079  // Calculate the overflow flag: sign extend the arithmetic result from
1080  // the original type.
1081  SDValue Ofl = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, NVT, Res,
1082  DAG.getValueType(OVT));
1083  // Overflowed if and only if this is not equal to Res.
1084  Ofl = DAG.getSetCC(dl, N->getValueType(1), Ofl, Res, ISD::SETNE);
1085 
1086  // Use the calculated overflow everywhere.
1087  ReplaceValueWith(SDValue(N, 1), Ofl);
1088 
1089  return Res;
1090 }
1091 
1092 SDValue DAGTypeLegalizer::PromoteIntRes_SELECT(SDNode *N) {
1093  SDValue LHS = GetPromotedInteger(N->getOperand(1));
1094  SDValue RHS = GetPromotedInteger(N->getOperand(2));
1095  return DAG.getSelect(SDLoc(N),
1096  LHS.getValueType(), N->getOperand(0), LHS, RHS);
1097 }
1098 
1099 SDValue DAGTypeLegalizer::PromoteIntRes_VSELECT(SDNode *N) {
1100  SDValue Mask = N->getOperand(0);
1101 
1102  SDValue LHS = GetPromotedInteger(N->getOperand(1));
1103  SDValue RHS = GetPromotedInteger(N->getOperand(2));
1104  return DAG.getNode(ISD::VSELECT, SDLoc(N),
1105  LHS.getValueType(), Mask, LHS, RHS);
1106 }
1107 
1108 SDValue DAGTypeLegalizer::PromoteIntRes_SELECT_CC(SDNode *N) {
1109  SDValue LHS = GetPromotedInteger(N->getOperand(2));
1110  SDValue RHS = GetPromotedInteger(N->getOperand(3));
1111  return DAG.getNode(ISD::SELECT_CC, SDLoc(N),
1112  LHS.getValueType(), N->getOperand(0),
1113  N->getOperand(1), LHS, RHS, N->getOperand(4));
1114 }
1115 
1116 SDValue DAGTypeLegalizer::PromoteIntRes_SETCC(SDNode *N) {
1117  unsigned OpNo = N->isStrictFPOpcode() ? 1 : 0;
1118  EVT InVT = N->getOperand(OpNo).getValueType();
1119  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
1120 
1121  EVT SVT = getSetCCResultType(InVT);
1122 
1123  // If we got back a type that needs to be promoted, this likely means the
1124  // the input type also needs to be promoted. So get the promoted type for
1125  // the input and try the query again.
1126  if (getTypeAction(SVT) == TargetLowering::TypePromoteInteger) {
1127  if (getTypeAction(InVT) == TargetLowering::TypePromoteInteger) {
1128  InVT = TLI.getTypeToTransformTo(*DAG.getContext(), InVT);
1129  SVT = getSetCCResultType(InVT);
1130  } else {
1131  // Input type isn't promoted, just use the default promoted type.
1132  SVT = NVT;
1133  }
1134  }
1135 
1136  SDLoc dl(N);
1137  assert(SVT.isVector() == N->getOperand(OpNo).getValueType().isVector() &&
1138  "Vector compare must return a vector result!");
1139 
1140  // Get the SETCC result using the canonical SETCC type.
1141  SDValue SetCC;
1142  if (N->isStrictFPOpcode()) {
1143  EVT VTs[] = {SVT, MVT::Other};
1144  SDValue Opers[] = {N->getOperand(0), N->getOperand(1),
1145  N->getOperand(2), N->getOperand(3)};
1146  SetCC = DAG.getNode(N->getOpcode(), dl, VTs, Opers);
1147  // Legalize the chain result - switch anything that used the old chain to
1148  // use the new one.
1149  ReplaceValueWith(SDValue(N, 1), SetCC.getValue(1));
1150  } else
1151  SetCC = DAG.getNode(N->getOpcode(), dl, SVT, N->getOperand(0),
1152  N->getOperand(1), N->getOperand(2));
1153 
1154  // Convert to the expected type.
1155  return DAG.getSExtOrTrunc(SetCC, dl, NVT);
1156 }
1157 
1158 SDValue DAGTypeLegalizer::PromoteIntRes_SHL(SDNode *N, bool IsVP) {
1159  SDValue LHS = GetPromotedInteger(N->getOperand(0));
1160  SDValue RHS = N->getOperand(1);
1161  if (getTypeAction(RHS.getValueType()) == TargetLowering::TypePromoteInteger)
1162  RHS = ZExtPromotedInteger(RHS);
1163  if (!IsVP)
1164  return DAG.getNode(N->getOpcode(), SDLoc(N), LHS.getValueType(), LHS, RHS);
1165  return DAG.getNode(N->getOpcode(), SDLoc(N), LHS.getValueType(), LHS, RHS,
1166  N->getOperand(2), N->getOperand(3));
1167 }
1168 
1169 SDValue DAGTypeLegalizer::PromoteIntRes_SIGN_EXTEND_INREG(SDNode *N) {
1170  SDValue Op = GetPromotedInteger(N->getOperand(0));
1171  return DAG.getNode(ISD::SIGN_EXTEND_INREG, SDLoc(N),
1172  Op.getValueType(), Op, N->getOperand(1));
1173 }
1174 
1175 SDValue DAGTypeLegalizer::PromoteIntRes_SimpleIntBinOp(SDNode *N, bool IsVP) {
1176  // The input may have strange things in the top bits of the registers, but
1177  // these operations don't care. They may have weird bits going out, but
1178  // that too is okay if they are integer operations.
1179  SDValue LHS = GetPromotedInteger(N->getOperand(0));
1180  SDValue RHS = GetPromotedInteger(N->getOperand(1));
1181  if (!IsVP)
1182  return DAG.getNode(N->getOpcode(), SDLoc(N), LHS.getValueType(), LHS, RHS);
1183  return DAG.getNode(N->getOpcode(), SDLoc(N), LHS.getValueType(), LHS, RHS,
1184  N->getOperand(2), N->getOperand(3));
1185 }
1186 
1187 SDValue DAGTypeLegalizer::PromoteIntRes_SExtIntBinOp(SDNode *N, bool IsVP) {
1188  // Sign extend the input.
1189  SDValue LHS = SExtPromotedInteger(N->getOperand(0));
1190  SDValue RHS = SExtPromotedInteger(N->getOperand(1));
1191  if (!IsVP)
1192  return DAG.getNode(N->getOpcode(), SDLoc(N), LHS.getValueType(), LHS, RHS);
1193  return DAG.getNode(N->getOpcode(), SDLoc(N), LHS.getValueType(), LHS, RHS,
1194  N->getOperand(2), N->getOperand(3));
1195 }
1196 
1197 SDValue DAGTypeLegalizer::PromoteIntRes_ZExtIntBinOp(SDNode *N, bool IsVP) {
1198  // Zero extend the input.
1199  SDValue LHS = ZExtPromotedInteger(N->getOperand(0));
1200  SDValue RHS = ZExtPromotedInteger(N->getOperand(1));
1201  if (!IsVP)
1202  return DAG.getNode(N->getOpcode(), SDLoc(N), LHS.getValueType(), LHS, RHS);
1203  return DAG.getNode(N->getOpcode(), SDLoc(N), LHS.getValueType(), LHS, RHS,
1204  N->getOperand(2), N->getOperand(3));
1205 }
1206 
1207 SDValue DAGTypeLegalizer::PromoteIntRes_UMINUMAX(SDNode *N) {
1208  // It doesn't matter if we sign extend or zero extend in the inputs. So do
1209  // whatever is best for the target.
1210  SDValue LHS = SExtOrZExtPromotedInteger(N->getOperand(0));
1211  SDValue RHS = SExtOrZExtPromotedInteger(N->getOperand(1));
1212  return DAG.getNode(N->getOpcode(), SDLoc(N),
1213  LHS.getValueType(), LHS, RHS);
1214 }
1215 
1216 SDValue DAGTypeLegalizer::PromoteIntRes_SRA(SDNode *N, bool IsVP) {
1217  // The input value must be properly sign extended.
1218  SDValue LHS = SExtPromotedInteger(N->getOperand(0));
1219  SDValue RHS = N->getOperand(1);
1220  if (getTypeAction(RHS.getValueType()) == TargetLowering::TypePromoteInteger)
1221  RHS = ZExtPromotedInteger(RHS);
1222  if (!IsVP)
1223  return DAG.getNode(N->getOpcode(), SDLoc(N), LHS.getValueType(), LHS, RHS);
1224  return DAG.getNode(N->getOpcode(), SDLoc(N), LHS.getValueType(), LHS, RHS,
1225  N->getOperand(2), N->getOperand(3));
1226 }
1227 
1228 SDValue DAGTypeLegalizer::PromoteIntRes_SRL(SDNode *N, bool IsVP) {
1229  // The input value must be properly zero extended.
1230  SDValue LHS = ZExtPromotedInteger(N->getOperand(0));
1231  SDValue RHS = N->getOperand(1);
1232  if (getTypeAction(RHS.getValueType()) == TargetLowering::TypePromoteInteger)
1233  RHS = ZExtPromotedInteger(RHS);
1234  if (!IsVP)
1235  return DAG.getNode(N->getOpcode(), SDLoc(N), LHS.getValueType(), LHS, RHS);
1236  return DAG.getNode(N->getOpcode(), SDLoc(N), LHS.getValueType(), LHS, RHS,
1237  N->getOperand(2), N->getOperand(3));
1238 }
1239 
1240 SDValue DAGTypeLegalizer::PromoteIntRes_Rotate(SDNode *N) {
1241  // Lower the rotate to shifts and ORs which can be promoted.
1242  SDValue Res;
1243  TLI.expandROT(N, true /*AllowVectorOps*/, Res, DAG);
1244  ReplaceValueWith(SDValue(N, 0), Res);
1245  return SDValue();
1246 }
1247 
1248 SDValue DAGTypeLegalizer::PromoteIntRes_FunnelShift(SDNode *N) {
1249  SDValue Hi = GetPromotedInteger(N->getOperand(0));
1250  SDValue Lo = GetPromotedInteger(N->getOperand(1));
1251  SDValue Amount = GetPromotedInteger(N->getOperand(2));
1252 
1253  SDLoc DL(N);
1254  EVT OldVT = N->getOperand(0).getValueType();
1255  EVT VT = Lo.getValueType();
1256  unsigned Opcode = N->getOpcode();
1257  bool IsFSHR = Opcode == ISD::FSHR;
1258  unsigned OldBits = OldVT.getScalarSizeInBits();
1259  unsigned NewBits = VT.getScalarSizeInBits();
1260 
1261  // Amount has to be interpreted modulo the old bit width.
1262  Amount =
1263  DAG.getNode(ISD::UREM, DL, VT, Amount, DAG.getConstant(OldBits, DL, VT));
1264 
1265  // If the promoted type is twice the size (or more), then we use the
1266  // traditional funnel 'double' shift codegen. This isn't necessary if the
1267  // shift amount is constant.
1268  // fshl(x,y,z) -> (((aext(x) << bw) | zext(y)) << (z % bw)) >> bw.
1269  // fshr(x,y,z) -> (((aext(x) << bw) | zext(y)) >> (z % bw)).
1270  if (NewBits >= (2 * OldBits) && !isa<ConstantSDNode>(Amount) &&
1271  !TLI.isOperationLegalOrCustom(Opcode, VT)) {
1272  SDValue HiShift = DAG.getConstant(OldBits, DL, VT);
1273  Hi = DAG.getNode(ISD::SHL, DL, VT, Hi, HiShift);
1274  Lo = DAG.getZeroExtendInReg(Lo, DL, OldVT);
1275  SDValue Res = DAG.getNode(ISD::OR, DL, VT, Hi, Lo);
1276  Res = DAG.getNode(IsFSHR ? ISD::SRL : ISD::SHL, DL, VT, Res, Amount);
1277  if (!IsFSHR)
1278  Res = DAG.getNode(ISD::SRL, DL, VT, Res, HiShift);
1279  return Res;
1280  }
1281 
1282  // Shift Lo up to occupy the upper bits of the promoted type.
1283  SDValue ShiftOffset = DAG.getConstant(NewBits - OldBits, DL, VT);
1284  Lo = DAG.getNode(ISD::SHL, DL, VT, Lo, ShiftOffset);
1285 
1286  // Increase Amount to shift the result into the lower bits of the promoted
1287  // type.
1288  if (IsFSHR)
1289  Amount = DAG.getNode(ISD::ADD, DL, VT, Amount, ShiftOffset);
1290 
1291  return DAG.getNode(Opcode, DL, VT, Hi, Lo, Amount);
1292 }
1293 
1294 SDValue DAGTypeLegalizer::PromoteIntRes_TRUNCATE(SDNode *N) {
1295  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
1296  SDValue Res;
1297  SDValue InOp = N->getOperand(0);
1298  SDLoc dl(N);
1299 
1300  switch (getTypeAction(InOp.getValueType())) {
1301  default: llvm_unreachable("Unknown type action!");
1304  Res = InOp;
1305  break;
1307  Res = GetPromotedInteger(InOp);
1308  break;
1310  EVT InVT = InOp.getValueType();
1311  assert(InVT.isVector() && "Cannot split scalar types");
1312  ElementCount NumElts = InVT.getVectorElementCount();
1313  assert(NumElts == NVT.getVectorElementCount() &&
1314  "Dst and Src must have the same number of elements");
1315  assert(isPowerOf2_32(NumElts.getKnownMinValue()) &&
1316  "Promoted vector type must be a power of two");
1317 
1318  SDValue EOp1, EOp2;
1319  GetSplitVector(InOp, EOp1, EOp2);
1320 
1321  EVT HalfNVT = EVT::getVectorVT(*DAG.getContext(), NVT.getScalarType(),
1322  NumElts.divideCoefficientBy(2));
1323  EOp1 = DAG.getNode(ISD::TRUNCATE, dl, HalfNVT, EOp1);
1324  EOp2 = DAG.getNode(ISD::TRUNCATE, dl, HalfNVT, EOp2);
1325 
1326  return DAG.getNode(ISD::CONCAT_VECTORS, dl, NVT, EOp1, EOp2);
1327  }
1329  SDValue WideInOp = GetWidenedVector(InOp);
1330 
1331  // Truncate widened InOp.
1332  unsigned NumElem = WideInOp.getValueType().getVectorNumElements();
1333  EVT TruncVT = EVT::getVectorVT(*DAG.getContext(),
1334  N->getValueType(0).getScalarType(), NumElem);
1335  SDValue WideTrunc = DAG.getNode(ISD::TRUNCATE, dl, TruncVT, WideInOp);
1336 
1337  // Zero extend so that the elements are of same type as those of NVT
1338  EVT ExtVT = EVT::getVectorVT(*DAG.getContext(), NVT.getVectorElementType(),
1339  NumElem);
1340  SDValue WideExt = DAG.getNode(ISD::ZERO_EXTEND, dl, ExtVT, WideTrunc);
1341 
1342  // Extract the low NVT subvector.
1343  SDValue ZeroIdx = DAG.getVectorIdxConstant(0, dl);
1344  return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, NVT, WideExt, ZeroIdx);
1345  }
1346  }
1347 
1348  // Truncate to NVT instead of VT
1349  return DAG.getNode(ISD::TRUNCATE, dl, NVT, Res);
1350 }
1351 
1352 SDValue DAGTypeLegalizer::PromoteIntRes_UADDSUBO(SDNode *N, unsigned ResNo) {
1353  if (ResNo == 1)
1354  return PromoteIntRes_Overflow(N);
1355 
1356  // The operation overflowed iff the result in the larger type is not the
1357  // zero extension of its truncation to the original type.
1358  SDValue LHS = ZExtPromotedInteger(N->getOperand(0));
1359  SDValue RHS = ZExtPromotedInteger(N->getOperand(1));
1360  EVT OVT = N->getOperand(0).getValueType();
1361  EVT NVT = LHS.getValueType();
1362  SDLoc dl(N);
1363 
1364  // Do the arithmetic in the larger type.
1365  unsigned Opcode = N->getOpcode() == ISD::UADDO ? ISD::ADD : ISD::SUB;
1366  SDValue Res = DAG.getNode(Opcode, dl, NVT, LHS, RHS);
1367 
1368  // Calculate the overflow flag: zero extend the arithmetic result from
1369  // the original type.
1370  SDValue Ofl = DAG.getZeroExtendInReg(Res, dl, OVT);
1371  // Overflowed if and only if this is not equal to Res.
1372  Ofl = DAG.getSetCC(dl, N->getValueType(1), Ofl, Res, ISD::SETNE);
1373 
1374  // Use the calculated overflow everywhere.
1375  ReplaceValueWith(SDValue(N, 1), Ofl);
1376 
1377  return Res;
1378 }
1379 
1380 // Handle promotion for the ADDE/SUBE/ADDCARRY/SUBCARRY nodes. Notice that
1381 // the third operand of ADDE/SUBE nodes is carry flag, which differs from
1382 // the ADDCARRY/SUBCARRY nodes in that the third operand is carry Boolean.
1383 SDValue DAGTypeLegalizer::PromoteIntRes_ADDSUBCARRY(SDNode *N, unsigned ResNo) {
1384  if (ResNo == 1)
1385  return PromoteIntRes_Overflow(N);
1386 
1387  // We need to sign-extend the operands so the carry value computed by the
1388  // wide operation will be equivalent to the carry value computed by the
1389  // narrow operation.
1390  // An ADDCARRY can generate carry only if any of the operands has its
1391  // most significant bit set. Sign extension propagates the most significant
1392  // bit into the higher bits which means the extra bit that the narrow
1393  // addition would need (i.e. the carry) will be propagated through the higher
1394  // bits of the wide addition.
1395  // A SUBCARRY can generate borrow only if LHS < RHS and this property will be
1396  // preserved by sign extension.
1397  SDValue LHS = SExtPromotedInteger(N->getOperand(0));
1398  SDValue RHS = SExtPromotedInteger(N->getOperand(1));
1399 
1400  EVT ValueVTs[] = {LHS.getValueType(), N->getValueType(1)};
1401 
1402  // Do the arithmetic in the wide type.
1403  SDValue Res = DAG.getNode(N->getOpcode(), SDLoc(N), DAG.getVTList(ValueVTs),
1404  LHS, RHS, N->getOperand(2));
1405 
1406  // Update the users of the original carry/borrow value.
1407  ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
1408 
1409  return SDValue(Res.getNode(), 0);
1410 }
1411 
1412 SDValue DAGTypeLegalizer::PromoteIntRes_SADDSUBO_CARRY(SDNode *N,
1413  unsigned ResNo) {
1414  assert(ResNo == 1 && "Don't know how to promote other results yet.");
1415  return PromoteIntRes_Overflow(N);
1416 }
1417 
1418 SDValue DAGTypeLegalizer::PromoteIntRes_ABS(SDNode *N) {
1419  SDValue Op0 = SExtPromotedInteger(N->getOperand(0));
1420  return DAG.getNode(ISD::ABS, SDLoc(N), Op0.getValueType(), Op0);
1421 }
1422 
1423 SDValue DAGTypeLegalizer::PromoteIntRes_XMULO(SDNode *N, unsigned ResNo) {
1424  // Promote the overflow bit trivially.
1425  if (ResNo == 1)
1426  return PromoteIntRes_Overflow(N);
1427 
1428  SDValue LHS = N->getOperand(0), RHS = N->getOperand(1);
1429  SDLoc DL(N);
1430  EVT SmallVT = LHS.getValueType();
1431 
1432  // To determine if the result overflowed in a larger type, we extend the
1433  // input to the larger type, do the multiply (checking if it overflows),
1434  // then also check the high bits of the result to see if overflow happened
1435  // there.
1436  if (N->getOpcode() == ISD::SMULO) {
1437  LHS = SExtPromotedInteger(LHS);
1438  RHS = SExtPromotedInteger(RHS);
1439  } else {
1440  LHS = ZExtPromotedInteger(LHS);
1441  RHS = ZExtPromotedInteger(RHS);
1442  }
1443  SDVTList VTs = DAG.getVTList(LHS.getValueType(), N->getValueType(1));
1444  SDValue Mul = DAG.getNode(N->getOpcode(), DL, VTs, LHS, RHS);
1445 
1446  // Overflow occurred if it occurred in the larger type, or if the high part
1447  // of the result does not zero/sign-extend the low part. Check this second
1448  // possibility first.
1449  SDValue Overflow;
1450  if (N->getOpcode() == ISD::UMULO) {
1451  // Unsigned overflow occurred if the high part is non-zero.
1452  unsigned Shift = SmallVT.getScalarSizeInBits();
1453  EVT ShiftTy = getShiftAmountTyForConstant(Mul.getValueType(), TLI, DAG);
1454  SDValue Hi = DAG.getNode(ISD::SRL, DL, Mul.getValueType(), Mul,
1455  DAG.getConstant(Shift, DL, ShiftTy));
1456  Overflow = DAG.getSetCC(DL, N->getValueType(1), Hi,
1457  DAG.getConstant(0, DL, Hi.getValueType()),
1458  ISD::SETNE);
1459  } else {
1460  // Signed overflow occurred if the high part does not sign extend the low.
1461  SDValue SExt = DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, Mul.getValueType(),
1462  Mul, DAG.getValueType(SmallVT));
1463  Overflow = DAG.getSetCC(DL, N->getValueType(1), SExt, Mul, ISD::SETNE);
1464  }
1465 
1466  // The only other way for overflow to occur is if the multiplication in the
1467  // larger type itself overflowed.
1468  Overflow = DAG.getNode(ISD::OR, DL, N->getValueType(1), Overflow,
1469  SDValue(Mul.getNode(), 1));
1470 
1471  // Use the calculated overflow everywhere.
1472  ReplaceValueWith(SDValue(N, 1), Overflow);
1473  return Mul;
1474 }
1475 
1476 SDValue DAGTypeLegalizer::PromoteIntRes_UNDEF(SDNode *N) {
1477  return DAG.getUNDEF(TLI.getTypeToTransformTo(*DAG.getContext(),
1478  N->getValueType(0)));
1479 }
1480 
1481 SDValue DAGTypeLegalizer::PromoteIntRes_VSCALE(SDNode *N) {
1482  EVT VT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
1483 
1484  APInt MulImm = cast<ConstantSDNode>(N->getOperand(0))->getAPIntValue();
1485  return DAG.getVScale(SDLoc(N), VT, MulImm.sextOrSelf(VT.getSizeInBits()));
1486 }
1487 
1488 SDValue DAGTypeLegalizer::PromoteIntRes_VAARG(SDNode *N) {
1489  SDValue Chain = N->getOperand(0); // Get the chain.
1490  SDValue Ptr = N->getOperand(1); // Get the pointer.
1491  EVT VT = N->getValueType(0);
1492  SDLoc dl(N);
1493 
1494  MVT RegVT = TLI.getRegisterType(*DAG.getContext(), VT);
1495  unsigned NumRegs = TLI.getNumRegisters(*DAG.getContext(), VT);
1496  // The argument is passed as NumRegs registers of type RegVT.
1497 
1498  SmallVector<SDValue, 8> Parts(NumRegs);
1499  for (unsigned i = 0; i < NumRegs; ++i) {
1500  Parts[i] = DAG.getVAArg(RegVT, dl, Chain, Ptr, N->getOperand(2),
1501  N->getConstantOperandVal(3));
1502  Chain = Parts[i].getValue(1);
1503  }
1504 
1505  // Handle endianness of the load.
1506  if (DAG.getDataLayout().isBigEndian())
1507  std::reverse(Parts.begin(), Parts.end());
1508 
1509  // Assemble the parts in the promoted type.
1510  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
1511  SDValue Res = DAG.getNode(ISD::ZERO_EXTEND, dl, NVT, Parts[0]);
1512  for (unsigned i = 1; i < NumRegs; ++i) {
1513  SDValue Part = DAG.getNode(ISD::ZERO_EXTEND, dl, NVT, Parts[i]);
1514  // Shift it to the right position and "or" it in.
1515  Part = DAG.getNode(ISD::SHL, dl, NVT, Part,
1516  DAG.getConstant(i * RegVT.getSizeInBits(), dl,
1517  TLI.getPointerTy(DAG.getDataLayout())));
1518  Res = DAG.getNode(ISD::OR, dl, NVT, Res, Part);
1519  }
1520 
1521  // Modified the chain result - switch anything that used the old chain to
1522  // use the new one.
1523  ReplaceValueWith(SDValue(N, 1), Chain);
1524 
1525  return Res;
1526 }
1527 
1528 //===----------------------------------------------------------------------===//
1529 // Integer Operand Promotion
1530 //===----------------------------------------------------------------------===//
1531 
1532 /// PromoteIntegerOperand - This method is called when the specified operand of
1533 /// the specified node is found to need promotion. At this point, all of the
1534 /// result types of the node are known to be legal, but other operands of the
1535 /// node may need promotion or expansion as well as the specified one.
1536 bool DAGTypeLegalizer::PromoteIntegerOperand(SDNode *N, unsigned OpNo) {
1537  LLVM_DEBUG(dbgs() << "Promote integer operand: "; N->dump(&DAG);
1538  dbgs() << "\n");
1539  SDValue Res = SDValue();
1540  if (CustomLowerNode(N, N->getOperand(OpNo).getValueType(), false)) {
1541  LLVM_DEBUG(dbgs() << "Node has been custom lowered, done\n");
1542  return false;
1543  }
1544 
1545  switch (N->getOpcode()) {
1546  default:
1547  #ifndef NDEBUG
1548  dbgs() << "PromoteIntegerOperand Op #" << OpNo << ": ";
1549  N->dump(&DAG); dbgs() << "\n";
1550  #endif
1551  llvm_unreachable("Do not know how to promote this operator's operand!");
1552 
1553  case ISD::ANY_EXTEND: Res = PromoteIntOp_ANY_EXTEND(N); break;
1554  case ISD::ATOMIC_STORE:
1555  Res = PromoteIntOp_ATOMIC_STORE(cast<AtomicSDNode>(N));
1556  break;
1557  case ISD::BITCAST: Res = PromoteIntOp_BITCAST(N); break;
1558  case ISD::BR_CC: Res = PromoteIntOp_BR_CC(N, OpNo); break;
1559  case ISD::BRCOND: Res = PromoteIntOp_BRCOND(N, OpNo); break;
1560  case ISD::BUILD_PAIR: Res = PromoteIntOp_BUILD_PAIR(N); break;
1561  case ISD::BUILD_VECTOR: Res = PromoteIntOp_BUILD_VECTOR(N); break;
1562  case ISD::CONCAT_VECTORS: Res = PromoteIntOp_CONCAT_VECTORS(N); break;
1563  case ISD::EXTRACT_VECTOR_ELT: Res = PromoteIntOp_EXTRACT_VECTOR_ELT(N); break;
1565  Res = PromoteIntOp_INSERT_VECTOR_ELT(N, OpNo);break;
1566  case ISD::SCALAR_TO_VECTOR:
1567  Res = PromoteIntOp_SCALAR_TO_VECTOR(N); break;
1568  case ISD::SPLAT_VECTOR:
1569  Res = PromoteIntOp_SPLAT_VECTOR(N); break;
1570  case ISD::VSELECT:
1571  case ISD::SELECT: Res = PromoteIntOp_SELECT(N, OpNo); break;
1572  case ISD::SELECT_CC: Res = PromoteIntOp_SELECT_CC(N, OpNo); break;
1573  case ISD::SETCC: Res = PromoteIntOp_SETCC(N, OpNo); break;
1574  case ISD::SIGN_EXTEND: Res = PromoteIntOp_SIGN_EXTEND(N); break;
1575  case ISD::SINT_TO_FP: Res = PromoteIntOp_SINT_TO_FP(N); break;
1576  case ISD::STRICT_SINT_TO_FP: Res = PromoteIntOp_STRICT_SINT_TO_FP(N); break;
1577  case ISD::STORE: Res = PromoteIntOp_STORE(cast<StoreSDNode>(N),
1578  OpNo); break;
1579  case ISD::MSTORE: Res = PromoteIntOp_MSTORE(cast<MaskedStoreSDNode>(N),
1580  OpNo); break;
1581  case ISD::MLOAD: Res = PromoteIntOp_MLOAD(cast<MaskedLoadSDNode>(N),
1582  OpNo); break;
1583  case ISD::MGATHER: Res = PromoteIntOp_MGATHER(cast<MaskedGatherSDNode>(N),
1584  OpNo); break;
1585  case ISD::MSCATTER: Res = PromoteIntOp_MSCATTER(cast<MaskedScatterSDNode>(N),
1586  OpNo); break;
1587  case ISD::TRUNCATE: Res = PromoteIntOp_TRUNCATE(N); break;
1588  case ISD::FP16_TO_FP:
1589  case ISD::UINT_TO_FP: Res = PromoteIntOp_UINT_TO_FP(N); break;
1590  case ISD::STRICT_UINT_TO_FP: Res = PromoteIntOp_STRICT_UINT_TO_FP(N); break;
1591  case ISD::ZERO_EXTEND: Res = PromoteIntOp_ZERO_EXTEND(N); break;
1592  case ISD::INSERT_SUBVECTOR: Res = PromoteIntOp_INSERT_SUBVECTOR(N); break;
1593 
1594  case ISD::SHL:
1595  case ISD::SRA:
1596  case ISD::SRL:
1597  case ISD::ROTL:
1598  case ISD::ROTR: Res = PromoteIntOp_Shift(N); break;
1599 
1600  case ISD::SADDO_CARRY:
1601  case ISD::SSUBO_CARRY:
1602  case ISD::ADDCARRY:
1603  case ISD::SUBCARRY: Res = PromoteIntOp_ADDSUBCARRY(N, OpNo); break;
1604 
1605  case ISD::FRAMEADDR:
1606  case ISD::RETURNADDR: Res = PromoteIntOp_FRAMERETURNADDR(N); break;
1607 
1608  case ISD::PREFETCH: Res = PromoteIntOp_PREFETCH(N, OpNo); break;
1609 
1610  case ISD::SMULFIX:
1611  case ISD::SMULFIXSAT:
1612  case ISD::UMULFIX:
1613  case ISD::UMULFIXSAT:
1614  case ISD::SDIVFIX:
1615  case ISD::SDIVFIXSAT:
1616  case ISD::UDIVFIX:
1617  case ISD::UDIVFIXSAT: Res = PromoteIntOp_FIX(N); break;
1618 
1619  case ISD::FPOWI: Res = PromoteIntOp_FPOWI(N); break;
1620 
1621  case ISD::VECREDUCE_ADD:
1622  case ISD::VECREDUCE_MUL:
1623  case ISD::VECREDUCE_AND:
1624  case ISD::VECREDUCE_OR:
1625  case ISD::VECREDUCE_XOR:
1626  case ISD::VECREDUCE_SMAX:
1627  case ISD::VECREDUCE_SMIN:
1628  case ISD::VECREDUCE_UMAX:
1629  case ISD::VECREDUCE_UMIN: Res = PromoteIntOp_VECREDUCE(N); break;
1630  case ISD::VP_REDUCE_ADD:
1631  case ISD::VP_REDUCE_MUL:
1632  case ISD::VP_REDUCE_AND:
1633  case ISD::VP_REDUCE_OR:
1634  case ISD::VP_REDUCE_XOR:
1635  case ISD::VP_REDUCE_SMAX:
1636  case ISD::VP_REDUCE_SMIN:
1637  case ISD::VP_REDUCE_UMAX:
1638  case ISD::VP_REDUCE_UMIN:
1639  Res = PromoteIntOp_VP_REDUCE(N, OpNo);
1640  break;
1641 
1642  case ISD::SET_ROUNDING: Res = PromoteIntOp_SET_ROUNDING(N); break;
1643  }
1644 
1645  // If the result is null, the sub-method took care of registering results etc.
1646  if (!Res.getNode()) return false;
1647 
1648  // If the result is N, the sub-method updated N in place. Tell the legalizer
1649  // core about this.
1650  if (Res.getNode() == N)
1651  return true;
1652 
1653  const bool IsStrictFp = N->isStrictFPOpcode();
1654  assert(Res.getValueType() == N->getValueType(0) &&
1655  N->getNumValues() == (IsStrictFp ? 2 : 1) &&
1656  "Invalid operand expansion");
1657  LLVM_DEBUG(dbgs() << "Replacing: "; N->dump(&DAG); dbgs() << " with: ";
1658  Res.dump());
1659 
1660  ReplaceValueWith(SDValue(N, 0), Res);
1661  if (IsStrictFp)
1662  ReplaceValueWith(SDValue(N, 1), SDValue(Res.getNode(), 1));
1663 
1664  return false;
1665 }
1666 
1667 /// PromoteSetCCOperands - Promote the operands of a comparison. This code is
1668 /// shared among BR_CC, SELECT_CC, and SETCC handlers.
1669 void DAGTypeLegalizer::PromoteSetCCOperands(SDValue &NewLHS,SDValue &NewRHS,
1670  ISD::CondCode CCCode) {
1671  // We have to insert explicit sign or zero extends. Note that we could
1672  // insert sign extends for ALL conditions. For those operations where either
1673  // zero or sign extension would be valid, use SExtOrZExtPromotedInteger
1674  // which will choose the cheapest for the target.
1675  switch (CCCode) {
1676  default: llvm_unreachable("Unknown integer comparison!");
1677  case ISD::SETEQ:
1678  case ISD::SETNE: {
1679  SDValue OpL = GetPromotedInteger(NewLHS);
1680  SDValue OpR = GetPromotedInteger(NewRHS);
1681 
1682  // We would prefer to promote the comparison operand with sign extension.
1683  // If the width of OpL/OpR excluding the duplicated sign bits is no greater
1684  // than the width of NewLHS/NewRH, we can avoid inserting real truncate
1685  // instruction, which is redundant eventually.
1686  unsigned OpLEffectiveBits =
1687  OpL.getScalarValueSizeInBits() - DAG.ComputeNumSignBits(OpL) + 1;
1688  unsigned OpREffectiveBits =
1689  OpR.getScalarValueSizeInBits() - DAG.ComputeNumSignBits(OpR) + 1;
1690  if (OpLEffectiveBits <= NewLHS.getScalarValueSizeInBits() &&
1691  OpREffectiveBits <= NewRHS.getScalarValueSizeInBits()) {
1692  NewLHS = OpL;
1693  NewRHS = OpR;
1694  } else {
1695  NewLHS = SExtOrZExtPromotedInteger(NewLHS);
1696  NewRHS = SExtOrZExtPromotedInteger(NewRHS);
1697  }
1698  break;
1699  }
1700  case ISD::SETUGE:
1701  case ISD::SETUGT:
1702  case ISD::SETULE:
1703  case ISD::SETULT:
1704  NewLHS = SExtOrZExtPromotedInteger(NewLHS);
1705  NewRHS = SExtOrZExtPromotedInteger(NewRHS);
1706  break;
1707  case ISD::SETGE:
1708  case ISD::SETGT:
1709  case ISD::SETLT:
1710  case ISD::SETLE:
1711  NewLHS = SExtPromotedInteger(NewLHS);
1712  NewRHS = SExtPromotedInteger(NewRHS);
1713  break;
1714  }
1715 }
1716 
1717 SDValue DAGTypeLegalizer::PromoteIntOp_ANY_EXTEND(SDNode *N) {
1718  SDValue Op = GetPromotedInteger(N->getOperand(0));
1719  return DAG.getNode(ISD::ANY_EXTEND, SDLoc(N), N->getValueType(0), Op);
1720 }
1721 
1722 SDValue DAGTypeLegalizer::PromoteIntOp_ATOMIC_STORE(AtomicSDNode *N) {
1723  SDValue Op2 = GetPromotedInteger(N->getOperand(2));
1724  return DAG.getAtomic(N->getOpcode(), SDLoc(N), N->getMemoryVT(),
1725  N->getChain(), N->getBasePtr(), Op2, N->getMemOperand());
1726 }
1727 
1728 SDValue DAGTypeLegalizer::PromoteIntOp_BITCAST(SDNode *N) {
1729  // This should only occur in unusual situations like bitcasting to an
1730  // x86_fp80, so just turn it into a store+load
1731  return CreateStackStoreLoad(N->getOperand(0), N->getValueType(0));
1732 }
1733 
1734 SDValue DAGTypeLegalizer::PromoteIntOp_BR_CC(SDNode *N, unsigned OpNo) {
1735  assert(OpNo == 2 && "Don't know how to promote this operand!");
1736 
1737  SDValue LHS = N->getOperand(2);
1738  SDValue RHS = N->getOperand(3);
1739  PromoteSetCCOperands(LHS, RHS, cast<CondCodeSDNode>(N->getOperand(1))->get());
1740 
1741  // The chain (Op#0), CC (#1) and basic block destination (Op#4) are always
1742  // legal types.
1743  return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0),
1744  N->getOperand(1), LHS, RHS, N->getOperand(4)),
1745  0);
1746 }
1747 
1748 SDValue DAGTypeLegalizer::PromoteIntOp_BRCOND(SDNode *N, unsigned OpNo) {
1749  assert(OpNo == 1 && "only know how to promote condition");
1750 
1751  // Promote all the way up to the canonical SetCC type.
1752  SDValue Cond = PromoteTargetBoolean(N->getOperand(1), MVT::Other);
1753 
1754  // The chain (Op#0) and basic block destination (Op#2) are always legal types.
1755  return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0), Cond,
1756  N->getOperand(2)), 0);
1757 }
1758 
1759 SDValue DAGTypeLegalizer::PromoteIntOp_BUILD_PAIR(SDNode *N) {
1760  // Since the result type is legal, the operands must promote to it.
1761  EVT OVT = N->getOperand(0).getValueType();
1762  SDValue Lo = ZExtPromotedInteger(N->getOperand(0));
1763  SDValue Hi = GetPromotedInteger(N->getOperand(1));
1764  assert(Lo.getValueType() == N->getValueType(0) && "Operand over promoted?");
1765  SDLoc dl(N);
1766 
1767  Hi = DAG.getNode(ISD::SHL, dl, N->getValueType(0), Hi,
1768  DAG.getConstant(OVT.getSizeInBits(), dl,
1769  TLI.getPointerTy(DAG.getDataLayout())));
1770  return DAG.getNode(ISD::OR, dl, N->getValueType(0), Lo, Hi);
1771 }
1772 
1773 SDValue DAGTypeLegalizer::PromoteIntOp_BUILD_VECTOR(SDNode *N) {
1774  // The vector type is legal but the element type is not. This implies
1775  // that the vector is a power-of-two in length and that the element
1776  // type does not have a strange size (eg: it is not i1).
1777  EVT VecVT = N->getValueType(0);
1778  unsigned NumElts = VecVT.getVectorNumElements();
1779  assert(!((NumElts & 1) && (!TLI.isTypeLegal(VecVT))) &&
1780  "Legal vector of one illegal element?");
1781 
1782  // Promote the inserted value. The type does not need to match the
1783  // vector element type. Check that any extra bits introduced will be
1784  // truncated away.
1785  assert(N->getOperand(0).getValueSizeInBits() >=
1786  N->getValueType(0).getScalarSizeInBits() &&
1787  "Type of inserted value narrower than vector element type!");
1788 
1789  SmallVector<SDValue, 16> NewOps;
1790  for (unsigned i = 0; i < NumElts; ++i)
1791  NewOps.push_back(GetPromotedInteger(N->getOperand(i)));
1792 
1793  return SDValue(DAG.UpdateNodeOperands(N, NewOps), 0);
1794 }
1795 
1796 SDValue DAGTypeLegalizer::PromoteIntOp_INSERT_VECTOR_ELT(SDNode *N,
1797  unsigned OpNo) {
1798  if (OpNo == 1) {
1799  // Promote the inserted value. This is valid because the type does not
1800  // have to match the vector element type.
1801 
1802  // Check that any extra bits introduced will be truncated away.
1803  assert(N->getOperand(1).getValueSizeInBits() >=
1804  N->getValueType(0).getScalarSizeInBits() &&
1805  "Type of inserted value narrower than vector element type!");
1806  return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0),
1807  GetPromotedInteger(N->getOperand(1)),
1808  N->getOperand(2)),
1809  0);
1810  }
1811 
1812  assert(OpNo == 2 && "Different operand and result vector types?");
1813 
1814  // Promote the index.
1815  SDValue Idx = DAG.getZExtOrTrunc(N->getOperand(2), SDLoc(N),
1816  TLI.getVectorIdxTy(DAG.getDataLayout()));
1817  return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0),
1818  N->getOperand(1), Idx), 0);
1819 }
1820 
1821 SDValue DAGTypeLegalizer::PromoteIntOp_SCALAR_TO_VECTOR(SDNode *N) {
1822  // Integer SCALAR_TO_VECTOR operands are implicitly truncated, so just promote
1823  // the operand in place.
1824  return SDValue(DAG.UpdateNodeOperands(N,
1825  GetPromotedInteger(N->getOperand(0))), 0);
1826 }
1827 
1828 SDValue DAGTypeLegalizer::PromoteIntOp_SPLAT_VECTOR(SDNode *N) {
1829  // Integer SPLAT_VECTOR operands are implicitly truncated, so just promote the
1830  // operand in place.
1831  return SDValue(
1832  DAG.UpdateNodeOperands(N, GetPromotedInteger(N->getOperand(0))), 0);
1833 }
1834 
1835 SDValue DAGTypeLegalizer::PromoteIntOp_SELECT(SDNode *N, unsigned OpNo) {
1836  assert(OpNo == 0 && "Only know how to promote the condition!");
1837  SDValue Cond = N->getOperand(0);
1838  EVT OpTy = N->getOperand(1).getValueType();
1839 
1840  if (N->getOpcode() == ISD::VSELECT)
1841  if (SDValue Res = WidenVSELECTMask(N))
1842  return DAG.getNode(N->getOpcode(), SDLoc(N), N->getValueType(0),
1843  Res, N->getOperand(1), N->getOperand(2));
1844 
1845  // Promote all the way up to the canonical SetCC type.
1846  EVT OpVT = N->getOpcode() == ISD::SELECT ? OpTy.getScalarType() : OpTy;
1847  Cond = PromoteTargetBoolean(Cond, OpVT);
1848 
1849  return SDValue(DAG.UpdateNodeOperands(N, Cond, N->getOperand(1),
1850  N->getOperand(2)), 0);
1851 }
1852 
1853 SDValue DAGTypeLegalizer::PromoteIntOp_SELECT_CC(SDNode *N, unsigned OpNo) {
1854  assert(OpNo == 0 && "Don't know how to promote this operand!");
1855 
1856  SDValue LHS = N->getOperand(0);
1857  SDValue RHS = N->getOperand(1);
1858  PromoteSetCCOperands(LHS, RHS, cast<CondCodeSDNode>(N->getOperand(4))->get());
1859 
1860  // The CC (#4) and the possible return values (#2 and #3) have legal types.
1861  return SDValue(DAG.UpdateNodeOperands(N, LHS, RHS, N->getOperand(2),
1862  N->getOperand(3), N->getOperand(4)), 0);
1863 }
1864 
1865 SDValue DAGTypeLegalizer::PromoteIntOp_SETCC(SDNode *N, unsigned OpNo) {
1866  assert(OpNo == 0 && "Don't know how to promote this operand!");
1867 
1868  SDValue LHS = N->getOperand(0);
1869  SDValue RHS = N->getOperand(1);
1870  PromoteSetCCOperands(LHS, RHS, cast<CondCodeSDNode>(N->getOperand(2))->get());
1871 
1872  // The CC (#2) is always legal.
1873  return SDValue(DAG.UpdateNodeOperands(N, LHS, RHS, N->getOperand(2)), 0);
1874 }
1875 
1876 SDValue DAGTypeLegalizer::PromoteIntOp_Shift(SDNode *N) {
1877  return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0),
1878  ZExtPromotedInteger(N->getOperand(1))), 0);
1879 }
1880 
1881 SDValue DAGTypeLegalizer::PromoteIntOp_SIGN_EXTEND(SDNode *N) {
1882  SDValue Op = GetPromotedInteger(N->getOperand(0));
1883  SDLoc dl(N);
1884  Op = DAG.getNode(ISD::ANY_EXTEND, dl, N->getValueType(0), Op);
1885  return DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, Op.getValueType(),
1886  Op, DAG.getValueType(N->getOperand(0).getValueType()));
1887 }
1888 
1889 SDValue DAGTypeLegalizer::PromoteIntOp_SINT_TO_FP(SDNode *N) {
1890  return SDValue(DAG.UpdateNodeOperands(N,
1891  SExtPromotedInteger(N->getOperand(0))), 0);
1892 }
1893 
1894 SDValue DAGTypeLegalizer::PromoteIntOp_STRICT_SINT_TO_FP(SDNode *N) {
1895  return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0),
1896  SExtPromotedInteger(N->getOperand(1))), 0);
1897 }
1898 
1899 SDValue DAGTypeLegalizer::PromoteIntOp_STORE(StoreSDNode *N, unsigned OpNo){
1900  assert(ISD::isUNINDEXEDStore(N) && "Indexed store during type legalization!");
1901  SDValue Ch = N->getChain(), Ptr = N->getBasePtr();
1902  SDLoc dl(N);
1903 
1904  SDValue Val = GetPromotedInteger(N->getValue()); // Get promoted value.
1905 
1906  // Truncate the value and store the result.
1907  return DAG.getTruncStore(Ch, dl, Val, Ptr,
1908  N->getMemoryVT(), N->getMemOperand());
1909 }
1910 
1911 SDValue DAGTypeLegalizer::PromoteIntOp_MSTORE(MaskedStoreSDNode *N,
1912  unsigned OpNo) {
1913  SDValue DataOp = N->getValue();
1914  SDValue Mask = N->getMask();
1915 
1916  if (OpNo == 4) {
1917  // The Mask. Update in place.
1918  EVT DataVT = DataOp.getValueType();
1919  Mask = PromoteTargetBoolean(Mask, DataVT);
1920  SmallVector<SDValue, 4> NewOps(N->op_begin(), N->op_end());
1921  NewOps[4] = Mask;
1922  return SDValue(DAG.UpdateNodeOperands(N, NewOps), 0);
1923  }
1924 
1925  assert(OpNo == 1 && "Unexpected operand for promotion");
1926  DataOp = GetPromotedInteger(DataOp);
1927 
1928  return DAG.getMaskedStore(N->getChain(), SDLoc(N), DataOp, N->getBasePtr(),
1929  N->getOffset(), Mask, N->getMemoryVT(),
1930  N->getMemOperand(), N->getAddressingMode(),
1931  /*IsTruncating*/ true, N->isCompressingStore());
1932 }
1933 
1934 SDValue DAGTypeLegalizer::PromoteIntOp_MLOAD(MaskedLoadSDNode *N,
1935  unsigned OpNo) {
1936  assert(OpNo == 3 && "Only know how to promote the mask!");
1937  EVT DataVT = N->getValueType(0);
1938  SDValue Mask = PromoteTargetBoolean(N->getOperand(OpNo), DataVT);
1939  SmallVector<SDValue, 4> NewOps(N->op_begin(), N->op_end());
1940  NewOps[OpNo] = Mask;
1941  SDNode *Res = DAG.UpdateNodeOperands(N, NewOps);
1942  if (Res == N)
1943  return SDValue(Res, 0);
1944 
1945  // Update triggered CSE, do our own replacement since caller can't.
1946  ReplaceValueWith(SDValue(N, 0), SDValue(Res, 0));
1947  ReplaceValueWith(SDValue(N, 1), SDValue(Res, 1));
1948  return SDValue();
1949 }
1950 
1951 SDValue DAGTypeLegalizer::PromoteIntOp_MGATHER(MaskedGatherSDNode *N,
1952  unsigned OpNo) {
1953 
1954  SmallVector<SDValue, 5> NewOps(N->op_begin(), N->op_end());
1955  if (OpNo == 2) {
1956  // The Mask
1957  EVT DataVT = N->getValueType(0);
1958  NewOps[OpNo] = PromoteTargetBoolean(N->getOperand(OpNo), DataVT);
1959  } else if (OpNo == 4) {
1960  // The Index
1961  if (N->isIndexSigned())
1962  // Need to sign extend the index since the bits will likely be used.
1963  NewOps[OpNo] = SExtPromotedInteger(N->getOperand(OpNo));
1964  else
1965  NewOps[OpNo] = ZExtPromotedInteger(N->getOperand(OpNo));
1966  } else
1967  NewOps[OpNo] = GetPromotedInteger(N->getOperand(OpNo));
1968 
1969  SDNode *Res = DAG.UpdateNodeOperands(N, NewOps);
1970  if (Res == N)
1971  return SDValue(Res, 0);
1972 
1973  // Update triggered CSE, do our own replacement since caller can't.
1974  ReplaceValueWith(SDValue(N, 0), SDValue(Res, 0));
1975  ReplaceValueWith(SDValue(N, 1), SDValue(Res, 1));
1976  return SDValue();
1977 }
1978 
1979 SDValue DAGTypeLegalizer::PromoteIntOp_MSCATTER(MaskedScatterSDNode *N,
1980  unsigned OpNo) {
1981  bool TruncateStore = N->isTruncatingStore();
1982  SmallVector<SDValue, 5> NewOps(N->op_begin(), N->op_end());
1983  if (OpNo == 2) {
1984  // The Mask
1985  EVT DataVT = N->getValue().getValueType();
1986  NewOps[OpNo] = PromoteTargetBoolean(N->getOperand(OpNo), DataVT);
1987  } else if (OpNo == 4) {
1988  // The Index
1989  if (N->isIndexSigned())
1990  // Need to sign extend the index since the bits will likely be used.
1991  NewOps[OpNo] = SExtPromotedInteger(N->getOperand(OpNo));
1992  else
1993  NewOps[OpNo] = ZExtPromotedInteger(N->getOperand(OpNo));
1994 
1995  N->setIndexType(TLI.getCanonicalIndexType(N->getIndexType(),
1996  N->getMemoryVT(), NewOps[OpNo]));
1997  } else {
1998  NewOps[OpNo] = GetPromotedInteger(N->getOperand(OpNo));
1999  TruncateStore = true;
2000  }
2001 
2002  return DAG.getMaskedScatter(DAG.getVTList(MVT::Other), N->getMemoryVT(),
2003  SDLoc(N), NewOps, N->getMemOperand(),
2004  N->getIndexType(), TruncateStore);
2005 }
2006 
2007 SDValue DAGTypeLegalizer::PromoteIntOp_TRUNCATE(SDNode *N) {
2008  SDValue Op = GetPromotedInteger(N->getOperand(0));
2009  return DAG.getNode(ISD::TRUNCATE, SDLoc(N), N->getValueType(0), Op);
2010 }
2011 
2012 SDValue DAGTypeLegalizer::PromoteIntOp_UINT_TO_FP(SDNode *N) {
2013  return SDValue(DAG.UpdateNodeOperands(N,
2014  ZExtPromotedInteger(N->getOperand(0))), 0);
2015 }
2016 
2017 SDValue DAGTypeLegalizer::PromoteIntOp_STRICT_UINT_TO_FP(SDNode *N) {
2018  return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0),
2019  ZExtPromotedInteger(N->getOperand(1))), 0);
2020 }
2021 
2022 SDValue DAGTypeLegalizer::PromoteIntOp_ZERO_EXTEND(SDNode *N) {
2023  SDLoc dl(N);
2024  SDValue Op = GetPromotedInteger(N->getOperand(0));
2025  Op = DAG.getNode(ISD::ANY_EXTEND, dl, N->getValueType(0), Op);
2026  return DAG.getZeroExtendInReg(Op, dl, N->getOperand(0).getValueType());
2027 }
2028 
2029 SDValue DAGTypeLegalizer::PromoteIntOp_ADDSUBCARRY(SDNode *N, unsigned OpNo) {
2030  assert(OpNo == 2 && "Don't know how to promote this operand!");
2031 
2032  SDValue LHS = N->getOperand(0);
2033  SDValue RHS = N->getOperand(1);
2034  SDValue Carry = N->getOperand(2);
2035  SDLoc DL(N);
2036 
2037  Carry = PromoteTargetBoolean(Carry, LHS.getValueType());
2038 
2039  return SDValue(DAG.UpdateNodeOperands(N, LHS, RHS, Carry), 0);
2040 }
2041 
2042 SDValue DAGTypeLegalizer::PromoteIntOp_FIX(SDNode *N) {
2043  SDValue Op2 = ZExtPromotedInteger(N->getOperand(2));
2044  return SDValue(
2045  DAG.UpdateNodeOperands(N, N->getOperand(0), N->getOperand(1), Op2), 0);
2046 }
2047 
2048 SDValue DAGTypeLegalizer::PromoteIntOp_FRAMERETURNADDR(SDNode *N) {
2049  // Promote the RETURNADDR/FRAMEADDR argument to a supported integer width.
2050  SDValue Op = ZExtPromotedInteger(N->getOperand(0));
2051  return SDValue(DAG.UpdateNodeOperands(N, Op), 0);
2052 }
2053 
2054 SDValue DAGTypeLegalizer::PromoteIntOp_PREFETCH(SDNode *N, unsigned OpNo) {
2055  assert(OpNo > 1 && "Don't know how to promote this operand!");
2056  // Promote the rw, locality, and cache type arguments to a supported integer
2057  // width.
2058  SDValue Op2 = ZExtPromotedInteger(N->getOperand(2));
2059  SDValue Op3 = ZExtPromotedInteger(N->getOperand(3));
2060  SDValue Op4 = ZExtPromotedInteger(N->getOperand(4));
2061  return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0), N->getOperand(1),
2062  Op2, Op3, Op4),
2063  0);
2064 }
2065 
2066 SDValue DAGTypeLegalizer::PromoteIntOp_FPOWI(SDNode *N) {
2067  // FIXME: Support for promotion of STRICT_FPOWI is not implemented yet.
2068  assert(N->getOpcode() == ISD::FPOWI && "No STRICT_FPOWI support here yet.");
2069 
2070  // The integer operand is the last operand in FPOWI (so the result and
2071  // floating point operand is already type legalized).
2072 
2073  // We can't just promote the exponent type in FPOWI, since we want to lower
2074  // the node to a libcall and we if we promote to a type larger than
2075  // sizeof(int) the libcall might not be according to the targets ABI. Instead
2076  // we rewrite to a libcall here directly, letting makeLibCall handle promotion
2077  // if the target accepts it according to shouldSignExtendTypeInLibCall.
2078  RTLIB::Libcall LC = RTLIB::getPOWI(N->getValueType(0));
2079  assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected fpowi.");
2080  if (!TLI.getLibcallName(LC)) {
2081  // Some targets don't have a powi libcall; use pow instead.
2082  // FIXME: Implement this if some target needs it.
2083  DAG.getContext()->emitError("Don't know how to promote fpowi to fpow");
2084  return DAG.getUNDEF(N->getValueType(0));
2085  }
2086  // The exponent should fit in a sizeof(int) type for the libcall to be valid.
2087  assert(DAG.getLibInfo().getIntSize() ==
2088  N->getOperand(1).getValueType().getSizeInBits() &&
2089  "POWI exponent should match with sizeof(int) when doing the libcall.");
2091  CallOptions.setSExt(true);
2092  SDValue Ops[2] = { N->getOperand(0), N->getOperand(1) };
2093  std::pair<SDValue, SDValue> Tmp =
2094  TLI.makeLibCall(DAG, LC, N->getValueType(0), Ops,
2095  CallOptions, SDLoc(N), SDValue());
2096  ReplaceValueWith(SDValue(N, 0), Tmp.first);
2097  return SDValue();
2098 }
2099 
2101  switch (N->getOpcode()) {
2102  default:
2103  llvm_unreachable("Expected integer vector reduction");
2104  case ISD::VECREDUCE_ADD:
2105  case ISD::VECREDUCE_MUL:
2106  case ISD::VECREDUCE_AND:
2107  case ISD::VECREDUCE_OR:
2108  case ISD::VECREDUCE_XOR:
2109  case ISD::VP_REDUCE_ADD:
2110  case ISD::VP_REDUCE_MUL:
2111  case ISD::VP_REDUCE_AND:
2112  case ISD::VP_REDUCE_OR:
2113  case ISD::VP_REDUCE_XOR:
2114  return ISD::ANY_EXTEND;
2115  case ISD::VECREDUCE_SMAX:
2116  case ISD::VECREDUCE_SMIN:
2117  case ISD::VP_REDUCE_SMAX:
2118  case ISD::VP_REDUCE_SMIN:
2119  return ISD::SIGN_EXTEND;
2120  case ISD::VECREDUCE_UMAX:
2121  case ISD::VECREDUCE_UMIN:
2122  case ISD::VP_REDUCE_UMAX:
2123  case ISD::VP_REDUCE_UMIN:
2124  return ISD::ZERO_EXTEND;
2125  }
2126 }
2127 
2128 SDValue DAGTypeLegalizer::PromoteIntOpVectorReduction(SDNode *N, SDValue V) {
2129  switch (getExtendForIntVecReduction(N)) {
2130  default:
2131  llvm_unreachable("Impossible extension kind for integer reduction");
2132  case ISD::ANY_EXTEND:
2133  return GetPromotedInteger(V);
2134  case ISD::SIGN_EXTEND:
2135  return SExtPromotedInteger(V);
2136  case ISD::ZERO_EXTEND:
2137  return ZExtPromotedInteger(V);
2138  }
2139 }
2140 
2141 SDValue DAGTypeLegalizer::PromoteIntOp_VECREDUCE(SDNode *N) {
2142  SDLoc dl(N);
2143  SDValue Op = PromoteIntOpVectorReduction(N, N->getOperand(0));
2144 
2145  EVT EltVT = Op.getValueType().getVectorElementType();
2146  EVT VT = N->getValueType(0);
2147 
2148  if (VT.bitsGE(EltVT))
2149  return DAG.getNode(N->getOpcode(), SDLoc(N), VT, Op);
2150 
2151  // Result size must be >= element size. If this is not the case after
2152  // promotion, also promote the result type and then truncate.
2153  SDValue Reduce = DAG.getNode(N->getOpcode(), dl, EltVT, Op);
2154  return DAG.getNode(ISD::TRUNCATE, dl, VT, Reduce);
2155 }
2156 
2157 SDValue DAGTypeLegalizer::PromoteIntOp_VP_REDUCE(SDNode *N, unsigned OpNo) {
2158  SDLoc DL(N);
2159  SDValue Op = N->getOperand(OpNo);
2160  SmallVector<SDValue, 4> NewOps(N->op_begin(), N->op_end());
2161 
2162  if (OpNo == 2) { // Mask
2163  // Update in place.
2164  NewOps[2] = PromoteTargetBoolean(Op, N->getOperand(1).getValueType());
2165  return SDValue(DAG.UpdateNodeOperands(N, NewOps), 0);
2166  }
2167 
2168  assert(OpNo == 1 && "Unexpected operand for promotion");
2169 
2170  Op = PromoteIntOpVectorReduction(N, Op);
2171 
2172  NewOps[OpNo] = Op;
2173 
2174  EVT VT = N->getValueType(0);
2175  EVT EltVT = Op.getValueType().getScalarType();
2176 
2177  if (VT.bitsGE(EltVT))
2178  return DAG.getNode(N->getOpcode(), SDLoc(N), VT, NewOps);
2179 
2180  // Result size must be >= element/start-value size. If this is not the case
2181  // after promotion, also promote both the start value and result type and
2182  // then truncate.
2183  NewOps[0] =
2184  DAG.getNode(getExtendForIntVecReduction(N), DL, EltVT, N->getOperand(0));
2185  SDValue Reduce = DAG.getNode(N->getOpcode(), DL, EltVT, NewOps);
2186  return DAG.getNode(ISD::TRUNCATE, DL, VT, Reduce);
2187 }
2188 
2189 SDValue DAGTypeLegalizer::PromoteIntOp_SET_ROUNDING(SDNode *N) {
2190  SDValue Op = ZExtPromotedInteger(N->getOperand(1));
2191  return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0), Op), 0);
2192 }
2193 
2194 //===----------------------------------------------------------------------===//
2195 // Integer Result Expansion
2196 //===----------------------------------------------------------------------===//
2197 
2198 /// ExpandIntegerResult - This method is called when the specified result of the
2199 /// specified node is found to need expansion. At this point, the node may also
2200 /// have invalid operands or may have other results that need promotion, we just
2201 /// know that (at least) one result needs expansion.
2202 void DAGTypeLegalizer::ExpandIntegerResult(SDNode *N, unsigned ResNo) {
2203  LLVM_DEBUG(dbgs() << "Expand integer result: "; N->dump(&DAG);
2204  dbgs() << "\n");
2205  SDValue Lo, Hi;
2206  Lo = Hi = SDValue();
2207 
2208  // See if the target wants to custom expand this node.
2209  if (CustomLowerNode(N, N->getValueType(ResNo), true))
2210  return;
2211 
2212  switch (N->getOpcode()) {
2213  default:
2214 #ifndef NDEBUG
2215  dbgs() << "ExpandIntegerResult #" << ResNo << ": ";
2216  N->dump(&DAG); dbgs() << "\n";
2217 #endif
2218  report_fatal_error("Do not know how to expand the result of this "
2219  "operator!");
2220 
2221  case ISD::MERGE_VALUES: SplitRes_MERGE_VALUES(N, ResNo, Lo, Hi); break;
2222  case ISD::SELECT: SplitRes_SELECT(N, Lo, Hi); break;
2223  case ISD::SELECT_CC: SplitRes_SELECT_CC(N, Lo, Hi); break;
2224  case ISD::UNDEF: SplitRes_UNDEF(N, Lo, Hi); break;
2225  case ISD::FREEZE: SplitRes_FREEZE(N, Lo, Hi); break;
2226 
2227  case ISD::BITCAST: ExpandRes_BITCAST(N, Lo, Hi); break;
2228  case ISD::BUILD_PAIR: ExpandRes_BUILD_PAIR(N, Lo, Hi); break;
2229  case ISD::EXTRACT_ELEMENT: ExpandRes_EXTRACT_ELEMENT(N, Lo, Hi); break;
2230  case ISD::EXTRACT_VECTOR_ELT: ExpandRes_EXTRACT_VECTOR_ELT(N, Lo, Hi); break;
2231  case ISD::VAARG: ExpandRes_VAARG(N, Lo, Hi); break;
2232 
2233  case ISD::ANY_EXTEND: ExpandIntRes_ANY_EXTEND(N, Lo, Hi); break;
2234  case ISD::AssertSext: ExpandIntRes_AssertSext(N, Lo, Hi); break;
2235  case ISD::AssertZext: ExpandIntRes_AssertZext(N, Lo, Hi); break;
2236  case ISD::BITREVERSE: ExpandIntRes_BITREVERSE(N, Lo, Hi); break;
2237  case ISD::BSWAP: ExpandIntRes_BSWAP(N, Lo, Hi); break;
2238  case ISD::PARITY: ExpandIntRes_PARITY(N, Lo, Hi); break;
2239  case ISD::Constant: ExpandIntRes_Constant(N, Lo, Hi); break;
2240  case ISD::ABS: ExpandIntRes_ABS(N, Lo, Hi); break;
2241  case ISD::CTLZ_ZERO_UNDEF:
2242  case ISD::CTLZ: ExpandIntRes_CTLZ(N, Lo, Hi); break;
2243  case ISD::CTPOP: ExpandIntRes_CTPOP(N, Lo, Hi); break;
2244  case ISD::CTTZ_ZERO_UNDEF:
2245  case ISD::CTTZ: ExpandIntRes_CTTZ(N, Lo, Hi); break;
2246  case ISD::FLT_ROUNDS_: ExpandIntRes_FLT_ROUNDS(N, Lo, Hi); break;
2248  case ISD::FP_TO_SINT: ExpandIntRes_FP_TO_SINT(N, Lo, Hi); break;
2250  case ISD::FP_TO_UINT: ExpandIntRes_FP_TO_UINT(N, Lo, Hi); break;
2251  case ISD::FP_TO_SINT_SAT:
2252  case ISD::FP_TO_UINT_SAT: ExpandIntRes_FP_TO_XINT_SAT(N, Lo, Hi); break;
2253  case ISD::STRICT_LLROUND:
2254  case ISD::STRICT_LLRINT:
2255  case ISD::LLROUND:
2256  case ISD::LLRINT: ExpandIntRes_LLROUND_LLRINT(N, Lo, Hi); break;
2257  case ISD::LOAD: ExpandIntRes_LOAD(cast<LoadSDNode>(N), Lo, Hi); break;
2258  case ISD::MUL: ExpandIntRes_MUL(N, Lo, Hi); break;
2259  case ISD::READCYCLECOUNTER: ExpandIntRes_READCYCLECOUNTER(N, Lo, Hi); break;
2260  case ISD::SDIV: ExpandIntRes_SDIV(N, Lo, Hi); break;
2261  case ISD::SIGN_EXTEND: ExpandIntRes_SIGN_EXTEND(N, Lo, Hi); break;
2262  case ISD::SIGN_EXTEND_INREG: ExpandIntRes_SIGN_EXTEND_INREG(N, Lo, Hi); break;
2263  case ISD::SREM: ExpandIntRes_SREM(N, Lo, Hi); break;
2264  case ISD::TRUNCATE: ExpandIntRes_TRUNCATE(N, Lo, Hi); break;
2265  case ISD::UDIV: ExpandIntRes_UDIV(N, Lo, Hi); break;
2266  case ISD::UREM: ExpandIntRes_UREM(N, Lo, Hi); break;
2267  case ISD::ZERO_EXTEND: ExpandIntRes_ZERO_EXTEND(N, Lo, Hi); break;
2268  case ISD::ATOMIC_LOAD: ExpandIntRes_ATOMIC_LOAD(N, Lo, Hi); break;
2269 
2270  case ISD::ATOMIC_LOAD_ADD:
2271  case ISD::ATOMIC_LOAD_SUB:
2272  case ISD::ATOMIC_LOAD_AND:
2273  case ISD::ATOMIC_LOAD_CLR:
2274  case ISD::ATOMIC_LOAD_OR:
2275  case ISD::ATOMIC_LOAD_XOR:
2276  case ISD::ATOMIC_LOAD_NAND:
2277  case ISD::ATOMIC_LOAD_MIN:
2278  case ISD::ATOMIC_LOAD_MAX:
2279  case ISD::ATOMIC_LOAD_UMIN:
2280  case ISD::ATOMIC_LOAD_UMAX:
2281  case ISD::ATOMIC_SWAP:
2282  case ISD::ATOMIC_CMP_SWAP: {
2283  std::pair<SDValue, SDValue> Tmp = ExpandAtomic(N);
2284  SplitInteger(Tmp.first, Lo, Hi);
2285  ReplaceValueWith(SDValue(N, 1), Tmp.second);
2286  break;
2287  }
2289  AtomicSDNode *AN = cast<AtomicSDNode>(N);
2290  SDVTList VTs = DAG.getVTList(N->getValueType(0), MVT::Other);
2291  SDValue Tmp = DAG.getAtomicCmpSwap(
2292  ISD::ATOMIC_CMP_SWAP, SDLoc(N), AN->getMemoryVT(), VTs,
2293  N->getOperand(0), N->getOperand(1), N->getOperand(2), N->getOperand(3),
2294  AN->getMemOperand());
2295 
2296  // Expanding to the strong ATOMIC_CMP_SWAP node means we can determine
2297  // success simply by comparing the loaded value against the ingoing
2298  // comparison.
2299  SDValue Success = DAG.getSetCC(SDLoc(N), N->getValueType(1), Tmp,
2300  N->getOperand(2), ISD::SETEQ);
2301 
2302  SplitInteger(Tmp, Lo, Hi);
2303  ReplaceValueWith(SDValue(N, 1), Success);
2304  ReplaceValueWith(SDValue(N, 2), Tmp.getValue(1));
2305  break;
2306  }
2307 
2308  case ISD::AND:
2309  case ISD::OR:
2310  case ISD::XOR: ExpandIntRes_Logical(N, Lo, Hi); break;
2311 
2312  case ISD::UMAX:
2313  case ISD::SMAX:
2314  case ISD::UMIN:
2315  case ISD::SMIN: ExpandIntRes_MINMAX(N, Lo, Hi); break;
2316 
2317  case ISD::ADD:
2318  case ISD::SUB: ExpandIntRes_ADDSUB(N, Lo, Hi); break;
2319 
2320  case ISD::ADDC:
2321  case ISD::SUBC: ExpandIntRes_ADDSUBC(N, Lo, Hi); break;
2322 
2323  case ISD::ADDE:
2324  case ISD::SUBE: ExpandIntRes_ADDSUBE(N, Lo, Hi); break;
2325 
2326  case ISD::ADDCARRY:
2327  case ISD::SUBCARRY: ExpandIntRes_ADDSUBCARRY(N, Lo, Hi); break;
2328 
2329  case ISD::SADDO_CARRY:
2330  case ISD::SSUBO_CARRY: ExpandIntRes_SADDSUBO_CARRY(N, Lo, Hi); break;
2331 
2332  case ISD::SHL:
2333  case ISD::SRA:
2334  case ISD::SRL: ExpandIntRes_Shift(N, Lo, Hi); break;
2335 
2336  case ISD::SADDO:
2337  case ISD::SSUBO: ExpandIntRes_SADDSUBO(N, Lo, Hi); break;
2338  case ISD::UADDO:
2339  case ISD::USUBO: ExpandIntRes_UADDSUBO(N, Lo, Hi); break;
2340  case ISD::UMULO:
2341  case ISD::SMULO: ExpandIntRes_XMULO(N, Lo, Hi); break;
2342 
2343  case ISD::SADDSAT:
2344  case ISD::UADDSAT:
2345  case ISD::SSUBSAT:
2346  case ISD::USUBSAT: ExpandIntRes_ADDSUBSAT(N, Lo, Hi); break;
2347 
2348  case ISD::SSHLSAT:
2349  case ISD::USHLSAT: ExpandIntRes_SHLSAT(N, Lo, Hi); break;
2350 
2351  case ISD::SMULFIX:
2352  case ISD::SMULFIXSAT:
2353  case ISD::UMULFIX:
2354  case ISD::UMULFIXSAT: ExpandIntRes_MULFIX(N, Lo, Hi); break;
2355 
2356  case ISD::SDIVFIX:
2357  case ISD::SDIVFIXSAT:
2358  case ISD::UDIVFIX:
2359  case ISD::UDIVFIXSAT: ExpandIntRes_DIVFIX(N, Lo, Hi); break;
2360 
2361  case ISD::VECREDUCE_ADD:
2362  case ISD::VECREDUCE_MUL:
2363  case ISD::VECREDUCE_AND:
2364  case ISD::VECREDUCE_OR:
2365  case ISD::VECREDUCE_XOR:
2366  case ISD::VECREDUCE_SMAX:
2367  case ISD::VECREDUCE_SMIN:
2368  case ISD::VECREDUCE_UMAX:
2369  case ISD::VECREDUCE_UMIN: ExpandIntRes_VECREDUCE(N, Lo, Hi); break;
2370 
2371  case ISD::ROTL:
2372  case ISD::ROTR:
2373  ExpandIntRes_Rotate(N, Lo, Hi);
2374  break;
2375 
2376  case ISD::FSHL:
2377  case ISD::FSHR:
2378  ExpandIntRes_FunnelShift(N, Lo, Hi);
2379  break;
2380 
2381  case ISD::VSCALE:
2382  ExpandIntRes_VSCALE(N, Lo, Hi);
2383  break;
2384  }
2385 
2386  // If Lo/Hi is null, the sub-method took care of registering results etc.
2387  if (Lo.getNode())
2388  SetExpandedInteger(SDValue(N, ResNo), Lo, Hi);
2389 }
2390 
2391 /// Lower an atomic node to the appropriate builtin call.
2392 std::pair <SDValue, SDValue> DAGTypeLegalizer::ExpandAtomic(SDNode *Node) {
2393  unsigned Opc = Node->getOpcode();
2394  MVT VT = cast<AtomicSDNode>(Node)->getMemoryVT().getSimpleVT();
2395  AtomicOrdering order = cast<AtomicSDNode>(Node)->getMergedOrdering();
2396  // Lower to outline atomic libcall if outline atomics enabled,
2397  // or to sync libcall otherwise
2398  RTLIB::Libcall LC = RTLIB::getOUTLINE_ATOMIC(Opc, order, VT);
2399  EVT RetVT = Node->getValueType(0);
2402  if (TLI.getLibcallName(LC)) {
2403  Ops.append(Node->op_begin() + 2, Node->op_end());
2404  Ops.push_back(Node->getOperand(1));
2405  } else {
2406  LC = RTLIB::getSYNC(Opc, VT);
2407  assert(LC != RTLIB::UNKNOWN_LIBCALL &&
2408  "Unexpected atomic op or value type!");
2409  Ops.append(Node->op_begin() + 1, Node->op_end());
2410  }
2411  return TLI.makeLibCall(DAG, LC, RetVT, Ops, CallOptions, SDLoc(Node),
2412  Node->getOperand(0));
2413 }
2414 
2415 /// N is a shift by a value that needs to be expanded,
2416 /// and the shift amount is a constant 'Amt'. Expand the operation.
2417 void DAGTypeLegalizer::ExpandShiftByConstant(SDNode *N, const APInt &Amt,
2418  SDValue &Lo, SDValue &Hi) {
2419  SDLoc DL(N);
2420  // Expand the incoming operand to be shifted, so that we have its parts
2421  SDValue InL, InH;
2422  GetExpandedInteger(N->getOperand(0), InL, InH);
2423 
2424  // Though Amt shouldn't usually be 0, it's possible. E.g. when legalization
2425  // splitted a vector shift, like this: <op1, op2> SHL <0, 2>.
2426  if (!Amt) {
2427  Lo = InL;
2428  Hi = InH;
2429  return;
2430  }
2431 
2432  EVT NVT = InL.getValueType();
2433  unsigned VTBits = N->getValueType(0).getSizeInBits();
2434  unsigned NVTBits = NVT.getSizeInBits();
2435  EVT ShTy = N->getOperand(1).getValueType();
2436 
2437  if (N->getOpcode() == ISD::SHL) {
2438  if (Amt.ugt(VTBits)) {
2439  Lo = Hi = DAG.getConstant(0, DL, NVT);
2440  } else if (Amt.ugt(NVTBits)) {
2441  Lo = DAG.getConstant(0, DL, NVT);
2442  Hi = DAG.getNode(ISD::SHL, DL,
2443  NVT, InL, DAG.getConstant(Amt - NVTBits, DL, ShTy));
2444  } else if (Amt == NVTBits) {
2445  Lo = DAG.getConstant(0, DL, NVT);
2446  Hi = InL;
2447  } else {
2448  Lo = DAG.getNode(ISD::SHL, DL, NVT, InL, DAG.getConstant(Amt, DL, ShTy));
2449  Hi = DAG.getNode(ISD::OR, DL, NVT,
2450  DAG.getNode(ISD::SHL, DL, NVT, InH,
2451  DAG.getConstant(Amt, DL, ShTy)),
2452  DAG.getNode(ISD::SRL, DL, NVT, InL,
2453  DAG.getConstant(-Amt + NVTBits, DL, ShTy)));
2454  }
2455  return;
2456  }
2457 
2458  if (N->getOpcode() == ISD::SRL) {
2459  if (Amt.ugt(VTBits)) {
2460  Lo = Hi = DAG.getConstant(0, DL, NVT);
2461  } else if (Amt.ugt(NVTBits)) {
2462  Lo = DAG.getNode(ISD::SRL, DL,
2463  NVT, InH, DAG.getConstant(Amt - NVTBits, DL, ShTy));
2464  Hi = DAG.getConstant(0, DL, NVT);
2465  } else if (Amt == NVTBits) {
2466  Lo = InH;
2467  Hi = DAG.getConstant(0, DL, NVT);
2468  } else {
2469  Lo = DAG.getNode(ISD::OR, DL, NVT,
2470  DAG.getNode(ISD::SRL, DL, NVT, InL,
2471  DAG.getConstant(Amt, DL, ShTy)),
2472  DAG.getNode(ISD::SHL, DL, NVT, InH,
2473  DAG.getConstant(-Amt + NVTBits, DL, ShTy)));
2474  Hi = DAG.getNode(ISD::SRL, DL, NVT, InH, DAG.getConstant(Amt, DL, ShTy));
2475  }
2476  return;
2477  }
2478 
2479  assert(N->getOpcode() == ISD::SRA && "Unknown shift!");
2480  if (Amt.ugt(VTBits)) {
2481  Hi = Lo = DAG.getNode(ISD::SRA, DL, NVT, InH,
2482  DAG.getConstant(NVTBits - 1, DL, ShTy));
2483  } else if (Amt.ugt(NVTBits)) {
2484  Lo = DAG.getNode(ISD::SRA, DL, NVT, InH,
2485  DAG.getConstant(Amt - NVTBits, DL, ShTy));
2486  Hi = DAG.getNode(ISD::SRA, DL, NVT, InH,
2487  DAG.getConstant(NVTBits - 1, DL, ShTy));
2488  } else if (Amt == NVTBits) {
2489  Lo = InH;
2490  Hi = DAG.getNode(ISD::SRA, DL, NVT, InH,
2491  DAG.getConstant(NVTBits - 1, DL, ShTy));
2492  } else {
2493  Lo = DAG.getNode(ISD::OR, DL, NVT,
2494  DAG.getNode(ISD::SRL, DL, NVT, InL,
2495  DAG.getConstant(Amt, DL, ShTy)),
2496  DAG.getNode(ISD::SHL, DL, NVT, InH,
2497  DAG.getConstant(-Amt + NVTBits, DL, ShTy)));
2498  Hi = DAG.getNode(ISD::SRA, DL, NVT, InH, DAG.getConstant(Amt, DL, ShTy));
2499  }
2500 }
2501 
2502 /// ExpandShiftWithKnownAmountBit - Try to determine whether we can simplify
2503 /// this shift based on knowledge of the high bit of the shift amount. If we
2504 /// can tell this, we know that it is >= 32 or < 32, without knowing the actual
2505 /// shift amount.
2506 bool DAGTypeLegalizer::
2507 ExpandShiftWithKnownAmountBit(SDNode *N, SDValue &Lo, SDValue &Hi) {
2508  SDValue Amt = N->getOperand(1);
2509  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
2510  EVT ShTy = Amt.getValueType();
2511  unsigned ShBits = ShTy.getScalarSizeInBits();
2512  unsigned NVTBits = NVT.getScalarSizeInBits();
2513  assert(isPowerOf2_32(NVTBits) &&
2514  "Expanded integer type size not a power of two!");
2515  SDLoc dl(N);
2516 
2517  APInt HighBitMask = APInt::getHighBitsSet(ShBits, ShBits - Log2_32(NVTBits));
2518  KnownBits Known = DAG.computeKnownBits(N->getOperand(1));
2519 
2520  // If we don't know anything about the high bits, exit.
2521  if (((Known.Zero|Known.One) & HighBitMask) == 0)
2522  return false;
2523 
2524  // Get the incoming operand to be shifted.
2525  SDValue InL, InH;
2526  GetExpandedInteger(N->getOperand(0), InL, InH);
2527 
2528  // If we know that any of the high bits of the shift amount are one, then we
2529  // can do this as a couple of simple shifts.
2530  if (Known.One.intersects(HighBitMask)) {
2531  // Mask out the high bit, which we know is set.
2532  Amt = DAG.getNode(ISD::AND, dl, ShTy, Amt,
2533  DAG.getConstant(~HighBitMask, dl, ShTy));
2534 
2535  switch (N->getOpcode()) {
2536  default: llvm_unreachable("Unknown shift");
2537  case ISD::SHL:
2538  Lo = DAG.getConstant(0, dl, NVT); // Low part is zero.
2539  Hi = DAG.getNode(ISD::SHL, dl, NVT, InL, Amt); // High part from Lo part.
2540  return true;
2541  case ISD::SRL:
2542  Hi = DAG.getConstant(0, dl, NVT); // Hi part is zero.
2543  Lo = DAG.getNode(ISD::SRL, dl, NVT, InH, Amt); // Lo part from Hi part.
2544  return true;
2545  case ISD::SRA:
2546  Hi = DAG.getNode(ISD::SRA, dl, NVT, InH, // Sign extend high part.
2547  DAG.getConstant(NVTBits - 1, dl, ShTy));
2548  Lo = DAG.getNode(ISD::SRA, dl, NVT, InH, Amt); // Lo part from Hi part.
2549  return true;
2550  }
2551  }
2552 
2553  // If we know that all of the high bits of the shift amount are zero, then we
2554  // can do this as a couple of simple shifts.
2555  if (HighBitMask.isSubsetOf(Known.Zero)) {
2556  // Calculate 31-x. 31 is used instead of 32 to avoid creating an undefined
2557  // shift if x is zero. We can use XOR here because x is known to be smaller
2558  // than 32.
2559  SDValue Amt2 = DAG.getNode(ISD::XOR, dl, ShTy, Amt,
2560  DAG.getConstant(NVTBits - 1, dl, ShTy));
2561 
2562  unsigned Op1, Op2;
2563  switch (N->getOpcode()) {
2564  default: llvm_unreachable("Unknown shift");
2565  case ISD::SHL: Op1 = ISD::SHL; Op2 = ISD::SRL; break;
2566  case ISD::SRL:
2567  case ISD::SRA: Op1 = ISD::SRL; Op2 = ISD::SHL; break;
2568  }
2569 
2570  // When shifting right the arithmetic for Lo and Hi is swapped.
2571  if (N->getOpcode() != ISD::SHL)
2572  std::swap(InL, InH);
2573 
2574  // Use a little trick to get the bits that move from Lo to Hi. First
2575  // shift by one bit.
2576  SDValue Sh1 = DAG.getNode(Op2, dl, NVT, InL, DAG.getConstant(1, dl, ShTy));
2577  // Then compute the remaining shift with amount-1.
2578  SDValue Sh2 = DAG.getNode(Op2, dl, NVT, Sh1, Amt2);
2579 
2580  Lo = DAG.getNode(N->getOpcode(), dl, NVT, InL, Amt);
2581  Hi = DAG.getNode(ISD::OR, dl, NVT, DAG.getNode(Op1, dl, NVT, InH, Amt),Sh2);
2582 
2583  if (N->getOpcode() != ISD::SHL)
2584  std::swap(Hi, Lo);
2585  return true;
2586  }
2587 
2588  return false;
2589 }
2590 
2591 /// ExpandShiftWithUnknownAmountBit - Fully general expansion of integer shift
2592 /// of any size.
2593 bool DAGTypeLegalizer::
2594 ExpandShiftWithUnknownAmountBit(SDNode *N, SDValue &Lo, SDValue &Hi) {
2595  SDValue Amt = N->getOperand(1);
2596  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
2597  EVT ShTy = Amt.getValueType();
2598  unsigned NVTBits = NVT.getSizeInBits();
2599  assert(isPowerOf2_32(NVTBits) &&
2600  "Expanded integer type size not a power of two!");
2601  SDLoc dl(N);
2602 
2603  // Get the incoming operand to be shifted.
2604  SDValue InL, InH;
2605  GetExpandedInteger(N->getOperand(0), InL, InH);
2606 
2607  SDValue NVBitsNode = DAG.getConstant(NVTBits, dl, ShTy);
2608  SDValue AmtExcess = DAG.getNode(ISD::SUB, dl, ShTy, Amt, NVBitsNode);
2609  SDValue AmtLack = DAG.getNode(ISD::SUB, dl, ShTy, NVBitsNode, Amt);
2610  SDValue isShort = DAG.getSetCC(dl, getSetCCResultType(ShTy),
2611  Amt, NVBitsNode, ISD::SETULT);
2612  SDValue isZero = DAG.getSetCC(dl, getSetCCResultType(ShTy),
2613  Amt, DAG.getConstant(0, dl, ShTy),
2614  ISD::SETEQ);
2615 
2616  SDValue LoS, HiS, LoL, HiL;
2617  switch (N->getOpcode()) {
2618  default: llvm_unreachable("Unknown shift");
2619  case ISD::SHL:
2620  // Short: ShAmt < NVTBits
2621  LoS = DAG.getNode(ISD::SHL, dl, NVT, InL, Amt);
2622  HiS = DAG.getNode(ISD::OR, dl, NVT,
2623  DAG.getNode(ISD::SHL, dl, NVT, InH, Amt),
2624  DAG.getNode(ISD::SRL, dl, NVT, InL, AmtLack));
2625 
2626  // Long: ShAmt >= NVTBits
2627  LoL = DAG.getConstant(0, dl, NVT); // Lo part is zero.
2628  HiL = DAG.getNode(ISD::SHL, dl, NVT, InL, AmtExcess); // Hi from Lo part.
2629 
2630  Lo = DAG.getSelect(dl, NVT, isShort, LoS, LoL);
2631  Hi = DAG.getSelect(dl, NVT, isZero, InH,
2632  DAG.getSelect(dl, NVT, isShort, HiS, HiL));
2633  return true;
2634  case ISD::SRL:
2635  // Short: ShAmt < NVTBits
2636  HiS = DAG.getNode(ISD::SRL, dl, NVT, InH, Amt);
2637  LoS = DAG.getNode(ISD::OR, dl, NVT,
2638  DAG.getNode(ISD::SRL, dl, NVT, InL, Amt),
2639  // FIXME: If Amt is zero, the following shift generates an undefined result
2640  // on some architectures.
2641  DAG.getNode(ISD::SHL, dl, NVT, InH, AmtLack));
2642 
2643  // Long: ShAmt >= NVTBits
2644  HiL = DAG.getConstant(0, dl, NVT); // Hi part is zero.
2645  LoL = DAG.getNode(ISD::SRL, dl, NVT, InH, AmtExcess); // Lo from Hi part.
2646 
2647  Lo = DAG.getSelect(dl, NVT, isZero, InL,
2648  DAG.getSelect(dl, NVT, isShort, LoS, LoL));
2649  Hi = DAG.getSelect(dl, NVT, isShort, HiS, HiL);
2650  return true;
2651  case ISD::SRA:
2652  // Short: ShAmt < NVTBits
2653  HiS = DAG.getNode(ISD::SRA, dl, NVT, InH, Amt);
2654  LoS = DAG.getNode(ISD::OR, dl, NVT,
2655  DAG.getNode(ISD::SRL, dl, NVT, InL, Amt),
2656  DAG.getNode(ISD::SHL, dl, NVT, InH, AmtLack));
2657 
2658  // Long: ShAmt >= NVTBits
2659  HiL = DAG.getNode(ISD::SRA, dl, NVT, InH, // Sign of Hi part.
2660  DAG.getConstant(NVTBits - 1, dl, ShTy));
2661  LoL = DAG.getNode(ISD::SRA, dl, NVT, InH, AmtExcess); // Lo from Hi part.
2662 
2663  Lo = DAG.getSelect(dl, NVT, isZero, InL,
2664  DAG.getSelect(dl, NVT, isShort, LoS, LoL));
2665  Hi = DAG.getSelect(dl, NVT, isShort, HiS, HiL);
2666  return true;
2667  }
2668 }
2669 
2670 static std::pair<ISD::CondCode, ISD::NodeType> getExpandedMinMaxOps(int Op) {
2671 
2672  switch (Op) {
2673  default: llvm_unreachable("invalid min/max opcode");
2674  case ISD::SMAX:
2675  return std::make_pair(ISD::SETGT, ISD::UMAX);
2676  case ISD::UMAX:
2677  return std::make_pair(ISD::SETUGT, ISD::UMAX);
2678  case ISD::SMIN:
2679  return std::make_pair(ISD::SETLT, ISD::UMIN);
2680  case ISD::UMIN:
2681  return std::make_pair(ISD::SETULT, ISD::UMIN);
2682  }
2683 }
2684 
2685 void DAGTypeLegalizer::ExpandIntRes_MINMAX(SDNode *N,
2686  SDValue &Lo, SDValue &Hi) {
2687  SDLoc DL(N);
2688  ISD::NodeType LoOpc;
2689  ISD::CondCode CondC;
2690  std::tie(CondC, LoOpc) = getExpandedMinMaxOps(N->getOpcode());
2691 
2692  // Expand the subcomponents.
2693  SDValue LHSL, LHSH, RHSL, RHSH;
2694  GetExpandedInteger(N->getOperand(0), LHSL, LHSH);
2695  GetExpandedInteger(N->getOperand(1), RHSL, RHSH);
2696 
2697  // Value types
2698  EVT NVT = LHSL.getValueType();
2699  EVT CCT = getSetCCResultType(NVT);
2700 
2701  // Hi part is always the same op
2702  Hi = DAG.getNode(N->getOpcode(), DL, NVT, {LHSH, RHSH});
2703 
2704  // We need to know whether to select Lo part that corresponds to 'winning'
2705  // Hi part or if Hi parts are equal.
2706  SDValue IsHiLeft = DAG.getSetCC(DL, CCT, LHSH, RHSH, CondC);
2707  SDValue IsHiEq = DAG.getSetCC(DL, CCT, LHSH, RHSH, ISD::SETEQ);
2708 
2709  // Lo part corresponding to the 'winning' Hi part
2710  SDValue LoCmp = DAG.getSelect(DL, NVT, IsHiLeft, LHSL, RHSL);
2711 
2712  // Recursed Lo part if Hi parts are equal, this uses unsigned version
2713  SDValue LoMinMax = DAG.getNode(LoOpc, DL, NVT, {LHSL, RHSL});
2714 
2715  Lo = DAG.getSelect(DL, NVT, IsHiEq, LoMinMax, LoCmp);
2716 }
2717 
2718 void DAGTypeLegalizer::ExpandIntRes_ADDSUB(SDNode *N,
2719  SDValue &Lo, SDValue &Hi) {
2720  SDLoc dl(N);
2721  // Expand the subcomponents.
2722  SDValue LHSL, LHSH, RHSL, RHSH;
2723  GetExpandedInteger(N->getOperand(0), LHSL, LHSH);
2724  GetExpandedInteger(N->getOperand(1), RHSL, RHSH);
2725 
2726  EVT NVT = LHSL.getValueType();
2727  SDValue LoOps[2] = { LHSL, RHSL };
2728  SDValue HiOps[3] = { LHSH, RHSH };
2729 
2730  bool HasOpCarry = TLI.isOperationLegalOrCustom(
2731  N->getOpcode() == ISD::ADD ? ISD::ADDCARRY : ISD::SUBCARRY,
2732  TLI.getTypeToExpandTo(*DAG.getContext(), NVT));
2733  if (HasOpCarry) {
2734  SDVTList VTList = DAG.getVTList(NVT, getSetCCResultType(NVT));
2735  if (N->getOpcode() == ISD::ADD) {
2736  Lo = DAG.getNode(ISD::UADDO, dl, VTList, LoOps);
2737  HiOps[2] = Lo.getValue(1);
2738  Hi = DAG.getNode(ISD::ADDCARRY, dl, VTList, HiOps);
2739  } else {
2740  Lo = DAG.getNode(ISD::USUBO, dl, VTList, LoOps);
2741  HiOps[2] = Lo.getValue(1);
2742  Hi = DAG.getNode(ISD::SUBCARRY, dl, VTList, HiOps);
2743  }
2744  return;
2745  }
2746 
2747  // Do not generate ADDC/ADDE or SUBC/SUBE if the target does not support
2748  // them. TODO: Teach operation legalization how to expand unsupported
2749  // ADDC/ADDE/SUBC/SUBE. The problem is that these operations generate
2750  // a carry of type MVT::Glue, but there doesn't seem to be any way to
2751  // generate a value of this type in the expanded code sequence.
2752  bool hasCarry =
2753  TLI.isOperationLegalOrCustom(N->getOpcode() == ISD::ADD ?
2754  ISD::ADDC : ISD::SUBC,
2755  TLI.getTypeToExpandTo(*DAG.getContext(), NVT));
2756 
2757  if (hasCarry) {
2758  SDVTList VTList = DAG.getVTList(NVT, MVT::Glue);
2759  if (N->getOpcode() == ISD::ADD) {
2760  Lo = DAG.getNode(ISD::ADDC, dl, VTList, LoOps);
2761  HiOps[2] = Lo.getValue(1);
2762  Hi = DAG.getNode(ISD::ADDE, dl, VTList, HiOps);
2763  } else {
2764  Lo = DAG.getNode(ISD::SUBC, dl, VTList, LoOps);
2765  HiOps[2] = Lo.getValue(1);
2766  Hi = DAG.getNode(ISD::SUBE, dl, VTList, HiOps);
2767  }
2768  return;
2769  }
2770 
2771  bool hasOVF =
2772  TLI.isOperationLegalOrCustom(N->getOpcode() == ISD::ADD ?
2774  TLI.getTypeToExpandTo(*DAG.getContext(), NVT));
2776 
2777  if (hasOVF) {
2778  EVT OvfVT = getSetCCResultType(NVT);
2779  SDVTList VTList = DAG.getVTList(NVT, OvfVT);
2780  int RevOpc;
2781  if (N->getOpcode() == ISD::ADD) {
2782  RevOpc = ISD::SUB;
2783  Lo = DAG.getNode(ISD::UADDO, dl, VTList, LoOps);
2784  Hi = DAG.getNode(ISD::ADD, dl, NVT, makeArrayRef(HiOps, 2));
2785  } else {
2786  RevOpc = ISD::ADD;
2787  Lo = DAG.getNode(ISD::USUBO, dl, VTList, LoOps);
2788  Hi = DAG.getNode(ISD::SUB, dl, NVT, makeArrayRef(HiOps, 2));
2789  }
2790  SDValue OVF = Lo.getValue(1);
2791 
2792  switch (BoolType) {
2794  OVF = DAG.getNode(ISD::AND, dl, OvfVT, DAG.getConstant(1, dl, OvfVT), OVF);
2797  OVF = DAG.getZExtOrTrunc(OVF, dl, NVT);
2798  Hi = DAG.getNode(N->getOpcode(), dl, NVT, Hi, OVF);
2799  break;
2801  OVF = DAG.getSExtOrTrunc(OVF, dl, NVT);
2802  Hi = DAG.getNode(RevOpc, dl, NVT, Hi, OVF);
2803  }
2804  return;
2805  }
2806 
2807  if (N->getOpcode() == ISD::ADD) {
2808  Lo = DAG.getNode(ISD::ADD, dl, NVT, LoOps);
2809  Hi = DAG.getNode(ISD::ADD, dl, NVT, makeArrayRef(HiOps, 2));
2810  SDValue Cmp1 = DAG.getSetCC(dl, getSetCCResultType(NVT), Lo, LoOps[0],
2811  ISD::SETULT);
2812 
2814  SDValue Carry = DAG.getZExtOrTrunc(Cmp1, dl, NVT);
2815  Hi = DAG.getNode(ISD::ADD, dl, NVT, Hi, Carry);
2816  return;
2817  }
2818 
2819  SDValue Carry1 = DAG.getSelect(dl, NVT, Cmp1,
2820  DAG.getConstant(1, dl, NVT),
2821  DAG.getConstant(0, dl, NVT));
2822  SDValue Cmp2 = DAG.getSetCC(dl, getSetCCResultType(NVT), Lo, LoOps[1],
2823  ISD::SETULT);
2824  SDValue Carry2 = DAG.getSelect(dl, NVT, Cmp2,
2825  DAG.getConstant(1, dl, NVT), Carry1);
2826  Hi = DAG.getNode(ISD::ADD, dl, NVT, Hi, Carry2);
2827  } else {
2828  Lo = DAG.getNode(ISD::SUB, dl, NVT, LoOps);
2829  Hi = DAG.getNode(ISD::SUB, dl, NVT, makeArrayRef(HiOps, 2));
2830  SDValue Cmp =
2831  DAG.getSetCC(dl, getSetCCResultType(LoOps[0].getValueType()),
2832  LoOps[0], LoOps[1], ISD::SETULT);
2833 
2834  SDValue Borrow;
2836  Borrow = DAG.getZExtOrTrunc(Cmp, dl, NVT);
2837  else
2838  Borrow = DAG.getSelect(dl, NVT, Cmp, DAG.getConstant(1, dl, NVT),
2839  DAG.getConstant(0, dl, NVT));
2840 
2841  Hi = DAG.getNode(ISD::SUB, dl, NVT, Hi, Borrow);
2842  }
2843 }
2844 
2845 void DAGTypeLegalizer::ExpandIntRes_ADDSUBC(SDNode *N,
2846  SDValue &Lo, SDValue &Hi) {
2847  // Expand the subcomponents.
2848  SDValue LHSL, LHSH, RHSL, RHSH;
2849  SDLoc dl(N);
2850  GetExpandedInteger(N->getOperand(0), LHSL, LHSH);
2851  GetExpandedInteger(N->getOperand(1), RHSL, RHSH);
2852  SDVTList VTList = DAG.getVTList(LHSL.getValueType(), MVT::Glue);
2853  SDValue LoOps[2] = { LHSL, RHSL };
2854  SDValue HiOps[3] = { LHSH, RHSH };
2855 
2856  if (N->getOpcode() == ISD::ADDC) {
2857  Lo = DAG.getNode(ISD::ADDC, dl, VTList, LoOps);
2858  HiOps[2] = Lo.getValue(1);
2859  Hi = DAG.getNode(ISD::ADDE, dl, VTList, HiOps);
2860  } else {
2861  Lo = DAG.getNode(ISD::SUBC, dl, VTList, LoOps);
2862  HiOps[2] = Lo.getValue(1);
2863  Hi = DAG.getNode(ISD::SUBE, dl, VTList, HiOps);
2864  }
2865 
2866  // Legalized the flag result - switch anything that used the old flag to
2867  // use the new one.
2868  ReplaceValueWith(SDValue(N, 1), Hi.getValue(1));
2869 }
2870 
2871 void DAGTypeLegalizer::ExpandIntRes_ADDSUBE(SDNode *N,
2872  SDValue &Lo, SDValue &Hi) {
2873  // Expand the subcomponents.
2874  SDValue LHSL, LHSH, RHSL, RHSH;
2875  SDLoc dl(N);
2876  GetExpandedInteger(N->getOperand(0), LHSL, LHSH);
2877  GetExpandedInteger(N->getOperand(1), RHSL, RHSH);
2878  SDVTList VTList = DAG.getVTList(LHSL.getValueType(), MVT::Glue);
2879  SDValue LoOps[3] = { LHSL, RHSL, N->getOperand(2) };
2880  SDValue HiOps[3] = { LHSH, RHSH };
2881 
2882  Lo = DAG.getNode(N->getOpcode(), dl, VTList, LoOps);
2883  HiOps[2] = Lo.getValue(1);
2884  Hi = DAG.getNode(N->getOpcode(), dl, VTList, HiOps);
2885 
2886  // Legalized the flag result - switch anything that used the old flag to
2887  // use the new one.
2888  ReplaceValueWith(SDValue(N, 1), Hi.getValue(1));
2889 }
2890 
2891 void DAGTypeLegalizer::ExpandIntRes_UADDSUBO(SDNode *N,
2892  SDValue &Lo, SDValue &Hi) {
2893  SDValue LHS = N->getOperand(0);
2894  SDValue RHS = N->getOperand(1);
2895  SDLoc dl(N);
2896 
2897  SDValue Ovf;
2898 
2899  unsigned CarryOp, NoCarryOp;
2901  switch(N->getOpcode()) {
2902  case ISD::UADDO:
2903  CarryOp = ISD::ADDCARRY;
2904  NoCarryOp = ISD::ADD;
2905  Cond = ISD::SETULT;
2906  break;
2907  case ISD::USUBO:
2908  CarryOp = ISD::SUBCARRY;
2909  NoCarryOp = ISD::SUB;
2910  Cond = ISD::SETUGT;
2911  break;
2912  default:
2913  llvm_unreachable("Node has unexpected Opcode");
2914  }
2915 
2916  bool HasCarryOp = TLI.isOperationLegalOrCustom(
2917  CarryOp, TLI.getTypeToExpandTo(*DAG.getContext(), LHS.getValueType()));
2918 
2919  if (HasCarryOp) {
2920  // Expand the subcomponents.
2921  SDValue LHSL, LHSH, RHSL, RHSH;
2922  GetExpandedInteger(LHS, LHSL, LHSH);
2923  GetExpandedInteger(RHS, RHSL, RHSH);
2924  SDVTList VTList = DAG.getVTList(LHSL.getValueType(), N->getValueType(1));
2925  SDValue LoOps[2] = { LHSL, RHSL };
2926  SDValue HiOps[3] = { LHSH, RHSH };
2927 
2928  Lo = DAG.getNode(N->getOpcode(), dl, VTList, LoOps);
2929  HiOps[2] = Lo.getValue(1);
2930  Hi = DAG.getNode(CarryOp, dl, VTList, HiOps);
2931 
2932  Ovf = Hi.getValue(1);
2933  } else {
2934  // Expand the result by simply replacing it with the equivalent
2935  // non-overflow-checking operation.
2936  SDValue Sum = DAG.getNode(NoCarryOp, dl, LHS.getValueType(), LHS, RHS);
2937  SplitInteger(Sum, Lo, Hi);
2938 
2939  // Calculate the overflow: addition overflows iff a + b < a, and subtraction
2940  // overflows iff a - b > a.
2941  Ovf = DAG.getSetCC(dl, N->getValueType(1), Sum, LHS, Cond);
2942  }
2943 
2944  // Legalized the flag result - switch anything that used the old flag to
2945  // use the new one.
2946  ReplaceValueWith(SDValue(N, 1), Ovf);
2947 }
2948 
2949 void DAGTypeLegalizer::ExpandIntRes_ADDSUBCARRY(SDNode *N,
2950  SDValue &Lo, SDValue &Hi) {
2951  // Expand the subcomponents.
2952  SDValue LHSL, LHSH, RHSL, RHSH;
2953  SDLoc dl(N);
2954  GetExpandedInteger(N->getOperand(0), LHSL, LHSH);
2955  GetExpandedInteger(N->getOperand(1), RHSL, RHSH);
2956  SDVTList VTList = DAG.getVTList(LHSL.getValueType(), N->getValueType(1));
2957  SDValue LoOps[3] = { LHSL, RHSL, N->getOperand(2) };
2958  SDValue HiOps[3] = { LHSH, RHSH, SDValue() };
2959 
2960  Lo = DAG.getNode(N->getOpcode(), dl, VTList, LoOps);
2961  HiOps[2] = Lo.getValue(1);
2962  Hi = DAG.getNode(N->getOpcode(), dl, VTList, HiOps);
2963 
2964  // Legalized the flag result - switch anything that used the old flag to
2965  // use the new one.
2966  ReplaceValueWith(SDValue(N, 1), Hi.getValue(1));
2967 }
2968 
2969 void DAGTypeLegalizer::ExpandIntRes_SADDSUBO_CARRY(SDNode *N,
2970  SDValue &Lo, SDValue &Hi) {
2971  // Expand the subcomponents.
2972  SDValue LHSL, LHSH, RHSL, RHSH;
2973  SDLoc dl(N);
2974  GetExpandedInteger(N->getOperand(0), LHSL, LHSH);
2975  GetExpandedInteger(N->getOperand(1), RHSL, RHSH);
2976  SDVTList VTList = DAG.getVTList(LHSL.getValueType(), N->getValueType(1));
2977 
2978  // We need to use an unsigned carry op for the lo part.
2979  unsigned CarryOp = N->getOpcode() == ISD::SADDO_CARRY ? ISD::ADDCARRY
2980  : ISD::SUBCARRY;
2981  Lo = DAG.getNode(CarryOp, dl, VTList, { LHSL, RHSL, N->getOperand(2) });
2982  Hi = DAG.getNode(N->getOpcode(), dl, VTList, { LHSH, RHSH, Lo.getValue(1) });
2983 
2984  // Legalized the flag result - switch anything that used the old flag to
2985  // use the new one.
2986  ReplaceValueWith(SDValue(N, 1), Hi.getValue(1));
2987 }
2988 
2989 void DAGTypeLegalizer::ExpandIntRes_ANY_EXTEND(SDNode *N,
2990  SDValue &Lo, SDValue &Hi) {
2991  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
2992  SDLoc dl(N);
2993  SDValue Op = N->getOperand(0);
2994  if (Op.getValueType().bitsLE(NVT)) {
2995  // The low part is any extension of the input (which degenerates to a copy).
2996  Lo = DAG.getNode(ISD::ANY_EXTEND, dl, NVT, Op);
2997  Hi = DAG.getUNDEF(NVT); // The high part is undefined.
2998  } else {
2999  // For example, extension of an i48 to an i64. The operand type necessarily
3000  // promotes to the result type, so will end up being expanded too.
3001  assert(getTypeAction(Op.getValueType()) ==
3003  "Only know how to promote this result!");
3004  SDValue Res = GetPromotedInteger(Op);
3005  assert(Res.getValueType() == N->getValueType(0) &&
3006  "Operand over promoted?");
3007  // Split the promoted operand. This will simplify when it is expanded.
3008  SplitInteger(Res, Lo, Hi);
3009  }
3010 }
3011 
3012 void DAGTypeLegalizer::ExpandIntRes_AssertSext(SDNode *N,
3013  SDValue &Lo, SDValue &Hi) {
3014  SDLoc dl(N);
3015  GetExpandedInteger(N->getOperand(0), Lo, Hi);
3016  EVT NVT = Lo.getValueType();
3017  EVT EVT = cast<VTSDNode>(N->getOperand(1))->getVT();
3018  unsigned NVTBits = NVT.getSizeInBits();
3019  unsigned EVTBits = EVT.getSizeInBits();
3020 
3021  if (NVTBits < EVTBits) {
3022  Hi = DAG.getNode(ISD::AssertSext, dl, NVT, Hi,
3024  EVTBits - NVTBits)));
3025  } else {
3026  Lo = DAG.getNode(ISD::AssertSext, dl, NVT, Lo, DAG.getValueType(EVT));
3027  // The high part replicates the sign bit of Lo, make it explicit.
3028  Hi = DAG.getNode(ISD::SRA, dl, NVT, Lo,
3029  DAG.getConstant(NVTBits - 1, dl,
3030  TLI.getPointerTy(DAG.getDataLayout())));
3031  }
3032 }
3033 
3034 void DAGTypeLegalizer::ExpandIntRes_AssertZext(SDNode *N,
3035  SDValue &Lo, SDValue &Hi) {
3036  SDLoc dl(N);
3037  GetExpandedInteger(N->getOperand(0), Lo, Hi);
3038  EVT NVT = Lo.getValueType();
3039  EVT EVT = cast<VTSDNode>(N->getOperand(1))->getVT();
3040  unsigned NVTBits = NVT.getSizeInBits();
3041  unsigned EVTBits = EVT.getSizeInBits();
3042 
3043  if (NVTBits < EVTBits) {
3044  Hi = DAG.getNode(ISD::AssertZext, dl, NVT, Hi,
3046  EVTBits - NVTBits)));
3047  } else {
3048  Lo = DAG.getNode(ISD::AssertZext, dl, NVT, Lo, DAG.getValueType(EVT));
3049  // The high part must be zero, make it explicit.
3050  Hi = DAG.getConstant(0, dl, NVT);
3051  }
3052 }
3053 
3054 void DAGTypeLegalizer::ExpandIntRes_BITREVERSE(SDNode *N,
3055  SDValue &Lo, SDValue &Hi) {
3056  SDLoc dl(N);
3057  GetExpandedInteger(N->getOperand(0), Hi, Lo); // Note swapped operands.
3058  Lo = DAG.getNode(ISD::BITREVERSE, dl, Lo.getValueType(), Lo);
3059  Hi = DAG.getNode(ISD::BITREVERSE, dl, Hi.getValueType(), Hi);
3060 }
3061 
3062 void DAGTypeLegalizer::ExpandIntRes_BSWAP(SDNode *N,
3063  SDValue &Lo, SDValue &Hi) {
3064  SDLoc dl(N);
3065  GetExpandedInteger(N->getOperand(0), Hi, Lo); // Note swapped operands.
3066  Lo = DAG.getNode(ISD::BSWAP, dl, Lo.getValueType(), Lo);
3067  Hi = DAG.getNode(ISD::BSWAP, dl, Hi.getValueType(), Hi);
3068 }
3069 
3070 void DAGTypeLegalizer::ExpandIntRes_PARITY(SDNode *N, SDValue &Lo,
3071  SDValue &Hi) {
3072  SDLoc dl(N);
3073  // parity(HiLo) -> parity(Lo^Hi)
3074  GetExpandedInteger(N->getOperand(0), Lo, Hi);
3075  EVT NVT = Lo.getValueType();
3076  Lo =
3077  DAG.getNode(ISD::PARITY, dl, NVT, DAG.getNode(ISD::XOR, dl, NVT, Lo, Hi));
3078  Hi = DAG.getConstant(0, dl, NVT);
3079 }
3080 
3081 void DAGTypeLegalizer::ExpandIntRes_Constant(SDNode *N,
3082  SDValue &Lo, SDValue &Hi) {
3083  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
3084  unsigned NBitWidth = NVT.getSizeInBits();
3085  auto Constant = cast<ConstantSDNode>(N);
3086  const APInt &Cst = Constant->getAPIntValue();
3087  bool IsTarget = Constant->isTargetOpcode();
3088  bool IsOpaque = Constant->isOpaque();
3089  SDLoc dl(N);
3090  Lo = DAG.getConstant(Cst.trunc(NBitWidth), dl, NVT, IsTarget, IsOpaque);
3091  Hi = DAG.getConstant(Cst.lshr(NBitWidth).trunc(NBitWidth), dl, NVT, IsTarget,
3092  IsOpaque);
3093 }
3094 
3095 void DAGTypeLegalizer::ExpandIntRes_ABS(SDNode *N, SDValue &Lo, SDValue &Hi) {
3096  SDLoc dl(N);
3097 
3098  SDValue N0 = N->getOperand(0);
3099  GetExpandedInteger(N0, Lo, Hi);
3100  EVT NVT = Lo.getValueType();
3101 
3102  // If we have ADDCARRY, use the expanded form of the sra+add+xor sequence we
3103  // use in LegalizeDAG. The ADD part of the expansion is based on
3104  // ExpandIntRes_ADDSUB which also uses ADDCARRY/UADDO after checking that
3105  // ADDCARRY is LegalOrCustom. Each of the pieces here can be further expanded
3106  // if needed. Shift expansion has a special case for filling with sign bits
3107  // so that we will only end up with one SRA.
3108  bool HasAddCarry = TLI.isOperationLegalOrCustom(
3109  ISD::ADDCARRY, TLI.getTypeToExpandTo(*DAG.getContext(), NVT));
3110  if (HasAddCarry) {
3111  EVT ShiftAmtTy = getShiftAmountTyForConstant(NVT, TLI, DAG);
3112  SDValue Sign =
3113  DAG.getNode(ISD::SRA, dl, NVT, Hi,
3114  DAG.getConstant(NVT.getSizeInBits() - 1, dl, ShiftAmtTy));
3115  SDVTList VTList = DAG.getVTList(NVT, getSetCCResultType(NVT));
3116  Lo = DAG.getNode(ISD::UADDO, dl, VTList, Lo, Sign);
3117  Hi = DAG.getNode(ISD::ADDCARRY, dl, VTList, Hi, Sign, Lo.getValue(1));
3118  Lo = DAG.getNode(ISD::XOR, dl, NVT, Lo, Sign);
3119  Hi = DAG.getNode(ISD::XOR, dl, NVT, Hi, Sign);
3120  return;
3121  }
3122 
3123  // abs(HiLo) -> (Hi < 0 ? -HiLo : HiLo)
3124  EVT VT = N->getValueType(0);
3125  SDValue Neg = DAG.getNode(ISD::SUB, dl, VT,
3126  DAG.getConstant(0, dl, VT), N0);
3127  SDValue NegLo, NegHi;
3128  SplitInteger(Neg, NegLo, NegHi);
3129 
3130  SDValue HiIsNeg = DAG.getSetCC(dl, getSetCCResultType(NVT),
3131  DAG.getConstant(0, dl, NVT), Hi, ISD::SETGT);
3132  Lo = DAG.getSelect(dl, NVT, HiIsNeg, NegLo, Lo);
3133  Hi = DAG.getSelect(dl, NVT, HiIsNeg, NegHi, Hi);
3134 }
3135 
3136 void DAGTypeLegalizer::ExpandIntRes_CTLZ(SDNode *N,
3137  SDValue &Lo, SDValue &Hi) {
3138  SDLoc dl(N);
3139  // ctlz (HiLo) -> Hi != 0 ? ctlz(Hi) : (ctlz(Lo)+32)
3140  GetExpandedInteger(N->getOperand(0), Lo, Hi);
3141  EVT NVT = Lo.getValueType();
3142 
3143  SDValue HiNotZero = DAG.getSetCC(dl, getSetCCResultType(NVT), Hi,
3144  DAG.getConstant(0, dl, NVT), ISD::SETNE);
3145 
3146  SDValue LoLZ = DAG.getNode(N->getOpcode(), dl, NVT, Lo);
3147  SDValue HiLZ = DAG.getNode(ISD::CTLZ_ZERO_UNDEF, dl, NVT, Hi);
3148 
3149  Lo = DAG.getSelect(dl, NVT, HiNotZero, HiLZ,
3150  DAG.getNode(ISD::ADD, dl, NVT, LoLZ,
3151  DAG.getConstant(NVT.getSizeInBits(), dl,
3152  NVT)));
3153  Hi = DAG.getConstant(0, dl, NVT);
3154 }
3155 
3156 void DAGTypeLegalizer::ExpandIntRes_CTPOP(SDNode *N,
3157  SDValue &Lo, SDValue &Hi) {
3158  SDLoc dl(N);
3159  // ctpop(HiLo) -> ctpop(Hi)+ctpop(Lo)
3160  GetExpandedInteger(N->getOperand(0), Lo, Hi);
3161  EVT NVT = Lo.getValueType();
3162  Lo = DAG.getNode(ISD::ADD, dl, NVT, DAG.getNode(ISD::CTPOP, dl, NVT, Lo),
3163  DAG.getNode(ISD::CTPOP, dl, NVT, Hi));
3164  Hi = DAG.getConstant(0, dl, NVT);
3165 }
3166 
3167 void DAGTypeLegalizer::ExpandIntRes_CTTZ(SDNode *N,
3168  SDValue &Lo, SDValue &Hi) {
3169  SDLoc dl(N);
3170  // cttz (HiLo) -> Lo != 0 ? cttz(Lo) : (cttz(Hi)+32)
3171  GetExpandedInteger(N->getOperand(0), Lo, Hi);
3172  EVT NVT = Lo.getValueType();
3173 
3174  SDValue LoNotZero = DAG.getSetCC(dl, getSetCCResultType(NVT), Lo,
3175  DAG.getConstant(0, dl, NVT), ISD::SETNE);
3176 
3177  SDValue LoLZ = DAG.getNode(ISD::CTTZ_ZERO_UNDEF, dl, NVT, Lo);
3178  SDValue HiLZ = DAG.getNode(N->getOpcode(), dl, NVT, Hi);
3179 
3180  Lo = DAG.getSelect(dl, NVT, LoNotZero, LoLZ,
3181  DAG.getNode(ISD::ADD, dl, NVT, HiLZ,
3182  DAG.getConstant(NVT.getSizeInBits(), dl,
3183  NVT)));
3184  Hi = DAG.getConstant(0, dl, NVT);
3185 }
3186 
3187 void DAGTypeLegalizer::ExpandIntRes_FLT_ROUNDS(SDNode *N, SDValue &Lo,
3188  SDValue &Hi) {
3189  SDLoc dl(N);
3190  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
3191  unsigned NBitWidth = NVT.getSizeInBits();
3192 
3193  EVT ShiftAmtTy = TLI.getShiftAmountTy(NVT, DAG.getDataLayout());
3194  Lo = DAG.getNode(ISD::FLT_ROUNDS_, dl, {NVT, MVT::Other}, N->getOperand(0));
3195  SDValue Chain = Lo.getValue(1);
3196  // The high part is the sign of Lo, as -1 is a valid value for FLT_ROUNDS
3197  Hi = DAG.getNode(ISD::SRA, dl, NVT, Lo,
3198  DAG.getConstant(NBitWidth - 1, dl, ShiftAmtTy));
3199 
3200  // Legalize the chain result - switch anything that used the old chain to
3201  // use the new one.
3202  ReplaceValueWith(SDValue(N, 1), Chain);
3203 }
3204 
3205 void DAGTypeLegalizer::ExpandIntRes_FP_TO_SINT(SDNode *N, SDValue &Lo,
3206  SDValue &Hi) {
3207  SDLoc dl(N);
3208  EVT VT = N->getValueType(0);
3209 
3210  bool IsStrict = N->isStrictFPOpcode();
3211  SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
3212  SDValue Op = N->getOperand(IsStrict ? 1 : 0);
3213  if (getTypeAction(Op.getValueType()) == TargetLowering::TypePromoteFloat)
3214  Op = GetPromotedFloat(Op);
3215 
3216  if (getTypeAction(Op.getValueType()) == TargetLowering::TypeSoftPromoteHalf) {
3217  EVT NFPVT = TLI.getTypeToTransformTo(*DAG.getContext(), Op.getValueType());
3218  Op = GetSoftPromotedHalf(Op);
3219  Op = DAG.getNode(ISD::FP16_TO_FP, dl, NFPVT, Op);
3220  Op = DAG.getNode(ISD::FP_TO_SINT, dl, VT, Op);
3221  SplitInteger(Op, Lo, Hi);
3222  return;
3223  }
3224 
3225  RTLIB::Libcall LC = RTLIB::getFPTOSINT(Op.getValueType(), VT);
3226  assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected fp-to-sint conversion!");
3228  CallOptions.setSExt(true);
3229  std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, VT, Op,
3230  CallOptions, dl, Chain);
3231  SplitInteger(Tmp.first, Lo, Hi);
3232 
3233  if (IsStrict)
3234  ReplaceValueWith(SDValue(N, 1), Tmp.second);
3235 }
3236 
3237 void DAGTypeLegalizer::ExpandIntRes_FP_TO_UINT(SDNode *N, SDValue &Lo,
3238  SDValue &Hi) {
3239  SDLoc dl(N);
3240  EVT VT = N->getValueType(0);
3241 
3242  bool IsStrict = N->isStrictFPOpcode();
3243  SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
3244  SDValue Op = N->getOperand(IsStrict ? 1 : 0);
3245  if (getTypeAction(Op.getValueType()) == TargetLowering::TypePromoteFloat)
3246  Op = GetPromotedFloat(Op);
3247 
3248  if (getTypeAction(Op.getValueType()) == TargetLowering::TypeSoftPromoteHalf) {
3249  EVT NFPVT = TLI.getTypeToTransformTo(*DAG.getContext(), Op.getValueType());
3250  Op = GetSoftPromotedHalf(Op);
3251  Op = DAG.getNode(ISD::FP16_TO_FP, dl, NFPVT, Op);
3252  Op = DAG.getNode(ISD::FP_TO_UINT, dl, VT, Op);
3253  SplitInteger(Op, Lo, Hi);
3254  return;
3255  }
3256 
3257  RTLIB::Libcall LC = RTLIB::getFPTOUINT(Op.getValueType(), VT);
3258  assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected fp-to-uint conversion!");
3260  std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, VT, Op,
3261  CallOptions, dl, Chain);
3262  SplitInteger(Tmp.first, Lo, Hi);
3263 
3264  if (IsStrict)
3265  ReplaceValueWith(SDValue(N, 1), Tmp.second);
3266 }
3267 
3268 void DAGTypeLegalizer::ExpandIntRes_FP_TO_XINT_SAT(SDNode *N, SDValue &Lo,
3269  SDValue &Hi) {
3270  SDValue Res = TLI.expandFP_TO_INT_SAT(N, DAG);
3271  SplitInteger(Res, Lo, Hi);
3272 }
3273 
3274 void DAGTypeLegalizer::ExpandIntRes_LLROUND_LLRINT(SDNode *N, SDValue &Lo,
3275  SDValue &Hi) {
3276  SDValue Op = N->getOperand(N->isStrictFPOpcode() ? 1 : 0);
3277 
3278  assert(getTypeAction(Op.getValueType()) != TargetLowering::TypePromoteFloat &&
3279  "Input type needs to be promoted!");
3280 
3281  EVT VT = Op.getValueType();
3282 
3283  RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
3284  if (N->getOpcode() == ISD::LLROUND ||
3285  N->getOpcode() == ISD::STRICT_LLROUND) {
3286  if (VT == MVT::f32)
3287  LC = RTLIB::LLROUND_F32;
3288  else if (VT == MVT::f64)
3289  LC = RTLIB::LLROUND_F64;
3290  else if (VT == MVT::f80)
3291  LC = RTLIB::LLROUND_F80;
3292  else if (VT == MVT::f128)
3293  LC = RTLIB::LLROUND_F128;
3294  else if (VT == MVT::ppcf128)
3295  LC = RTLIB::LLROUND_PPCF128;
3296  assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected llround input type!");
3297  } else if (N->getOpcode() == ISD::LLRINT ||
3298  N->getOpcode() == ISD::STRICT_LLRINT) {
3299  if (VT == MVT::f32)
3300  LC = RTLIB::LLRINT_F32;
3301  else if (VT == MVT::f64)
3302  LC = RTLIB::LLRINT_F64;
3303  else if (VT == MVT::f80)
3304  LC = RTLIB::LLRINT_F80;
3305  else if (VT == MVT::f128)
3306  LC = RTLIB::LLRINT_F128;
3307  else if (VT == MVT::ppcf128)
3308  LC = RTLIB::LLRINT_PPCF128;
3309  assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected llrint input type!");
3310  } else
3311  llvm_unreachable("Unexpected opcode!");
3312 
3313  SDLoc dl(N);
3314  EVT RetVT = N->getValueType(0);
3315  SDValue Chain = N->isStrictFPOpcode() ? N->getOperand(0) : SDValue();
3316 
3318  CallOptions.setSExt(true);
3319  std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, RetVT,
3320  Op, CallOptions, dl,
3321  Chain);
3322  SplitInteger(Tmp.first, Lo, Hi);
3323 
3324  if (N->isStrictFPOpcode())
3325  ReplaceValueWith(SDValue(N, 1), Tmp.second);
3326 }
3327 
3328 void DAGTypeLegalizer::ExpandIntRes_LOAD(LoadSDNode *N,
3329  SDValue &Lo, SDValue &Hi) {
3330  if (N->isAtomic()) {
3331  // It's typical to have larger CAS than atomic load instructions.
3332  SDLoc dl(N);
3333  EVT VT = N->getMemoryVT();
3334  SDVTList VTs = DAG.getVTList(VT, MVT::i1, MVT::Other);
3335  SDValue Zero = DAG.getConstant(0, dl, VT);
3336  SDValue Swap = DAG.getAtomicCmpSwap(
3338  VT, VTs, N->getOperand(0),
3339  N->getOperand(1), Zero, Zero, N->getMemOperand());
3340  ReplaceValueWith(SDValue(N, 0), Swap.getValue(0));
3341  ReplaceValueWith(SDValue(N, 1), Swap.getValue(2));
3342  return;
3343  }
3344 
3345  if (ISD::isNormalLoad(N)) {
3346  ExpandRes_NormalLoad(N, Lo, Hi);
3347  return;
3348  }
3349 
3350  assert(ISD::isUNINDEXEDLoad(N) && "Indexed load during type legalization!");
3351 
3352  EVT VT = N->getValueType(0);
3353  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
3354  SDValue Ch = N->getChain();
3355  SDValue Ptr = N->getBasePtr();
3356  ISD::LoadExtType ExtType = N->getExtensionType();
3357  MachineMemOperand::Flags MMOFlags = N->getMemOperand()->getFlags();
3358  AAMDNodes AAInfo = N->getAAInfo();
3359  SDLoc dl(N);
3360 
3361  assert(NVT.isByteSized() && "Expanded type not byte sized!");
3362 
3363  if (N->getMemoryVT().bitsLE(NVT)) {
3364  EVT MemVT = N->getMemoryVT();
3365 
3366  Lo = DAG.getExtLoad(ExtType, dl, NVT, Ch, Ptr, N->getPointerInfo(), MemVT,
3367  N->getOriginalAlign(), MMOFlags, AAInfo);
3368 
3369  // Remember the chain.
3370  Ch = Lo.getValue(1);
3371 
3372  if (ExtType == ISD::SEXTLOAD) {
3373  // The high part is obtained by SRA'ing all but one of the bits of the
3374  // lo part.
3375  unsigned LoSize = Lo.getValueSizeInBits();
3376  Hi = DAG.getNode(ISD::SRA, dl, NVT, Lo,
3377  DAG.getConstant(LoSize - 1, dl,
3378  TLI.getPointerTy(DAG.getDataLayout())));
3379  } else if (ExtType == ISD::ZEXTLOAD) {
3380  // The high part is just a zero.
3381  Hi = DAG.getConstant(0, dl, NVT);
3382  } else {
3383  assert(ExtType == ISD::EXTLOAD && "Unknown extload!");
3384  // The high part is undefined.
3385  Hi = DAG.getUNDEF(NVT);
3386  }
3387  } else if (DAG.getDataLayout().isLittleEndian()) {
3388  // Little-endian - low bits are at low addresses.
3389  Lo = DAG.getLoad(NVT, dl, Ch, Ptr, N->getPointerInfo(),
3390  N->getOriginalAlign(), MMOFlags, AAInfo);
3391 
3392  unsigned ExcessBits =
3393  N->getMemoryVT().getSizeInBits() - NVT.getSizeInBits();
3394  EVT NEVT = EVT::getIntegerVT(*DAG.getContext(), ExcessBits);
3395 
3396  // Increment the pointer to the other half.
3397  unsigned IncrementSize = NVT.getSizeInBits()/8;
3398  Ptr = DAG.getMemBasePlusOffset(Ptr, TypeSize::Fixed(IncrementSize), dl);
3399  Hi = DAG.getExtLoad(ExtType, dl, NVT, Ch, Ptr,
3400  N->getPointerInfo().getWithOffset(IncrementSize), NEVT,
3401  N->getOriginalAlign(), MMOFlags, AAInfo);
3402 
3403  // Build a factor node to remember that this load is independent of the
3404  // other one.
3405  Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1),
3406  Hi.getValue(1));
3407  } else {
3408  // Big-endian - high bits are at low addresses. Favor aligned loads at
3409  // the cost of some bit-fiddling.
3410  EVT MemVT = N->getMemoryVT();
3411  unsigned EBytes = MemVT.getStoreSize();
3412  unsigned IncrementSize = NVT.getSizeInBits()/8;
3413  unsigned ExcessBits = (EBytes - IncrementSize)*8;
3414 
3415  // Load both the high bits and maybe some of the low bits.
3416  Hi = DAG.getExtLoad(ExtType, dl, NVT, Ch, Ptr, N->getPointerInfo(),
3418  MemVT.getSizeInBits() - ExcessBits),
3419  N->getOriginalAlign(), MMOFlags, AAInfo);
3420 
3421  // Increment the pointer to the other half.
3422  Ptr = DAG.getMemBasePlusOffset(Ptr, TypeSize::Fixed(IncrementSize), dl);
3423  // Load the rest of the low bits.
3424  Lo = DAG.getExtLoad(ISD::ZEXTLOAD, dl, NVT, Ch, Ptr,
3425  N->getPointerInfo().getWithOffset(IncrementSize),
3426  EVT::getIntegerVT(*DAG.getContext(), ExcessBits),
3427  N->getOriginalAlign(), MMOFlags, AAInfo);
3428 
3429  // Build a factor node to remember that this load is independent of the
3430  // other one.
3431  Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1),
3432  Hi.getValue(1));
3433 
3434  if (ExcessBits < NVT.getSizeInBits()) {
3435  // Transfer low bits from the bottom of Hi to the top of Lo.
3436  Lo = DAG.getNode(
3437  ISD::OR, dl, NVT, Lo,
3438  DAG.getNode(ISD::SHL, dl, NVT, Hi,
3439  DAG.getConstant(ExcessBits, dl,
3440  TLI.getPointerTy(DAG.getDataLayout()))));
3441  // Move high bits to the right position in Hi.
3442  Hi = DAG.getNode(ExtType == ISD::SEXTLOAD ? ISD::SRA : ISD::SRL, dl, NVT,
3443  Hi,
3444  DAG.getConstant(NVT.getSizeInBits() - ExcessBits, dl,
3445  TLI.getPointerTy(DAG.getDataLayout())));
3446  }
3447  }
3448 
3449  // Legalize the chain result - switch anything that used the old chain to
3450  // use the new one.
3451  ReplaceValueWith(SDValue(N, 1), Ch);
3452 }
3453 
3454 void DAGTypeLegalizer::ExpandIntRes_Logical(SDNode *N,
3455  SDValue &Lo, SDValue &Hi) {
3456  SDLoc dl(N);
3457  SDValue LL, LH, RL, RH;
3458  GetExpandedInteger(N->getOperand(0), LL, LH);
3459  GetExpandedInteger(N->getOperand(1), RL, RH);
3460  Lo = DAG.getNode(N->getOpcode(), dl, LL.getValueType(), LL, RL);
3461  Hi = DAG.getNode(N->getOpcode(), dl, LL.getValueType(), LH, RH);
3462 }
3463 
3464 void DAGTypeLegalizer::ExpandIntRes_MUL(SDNode *N,
3465  SDValue &Lo, SDValue &Hi) {
3466  EVT VT = N->getValueType(0);
3467  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
3468  SDLoc dl(N);
3469 
3470  SDValue LL, LH, RL, RH;
3471  GetExpandedInteger(N->getOperand(0), LL, LH);
3472  GetExpandedInteger(N->getOperand(1), RL, RH);
3473 
3474  if (TLI.expandMUL(N, Lo, Hi, NVT, DAG,
3476  LL, LH, RL, RH))
3477  return;
3478 
3479  // If nothing else, we can make a libcall.
3480  RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
3481  if (VT == MVT::i16)
3482  LC = RTLIB::MUL_I16;
3483  else if (VT == MVT::i32)
3484  LC = RTLIB::MUL_I32;
3485  else if (VT == MVT::i64)
3486  LC = RTLIB::MUL_I64;
3487  else if (VT == MVT::i128)
3488  LC = RTLIB::MUL_I128;
3489 
3490  if (LC == RTLIB::UNKNOWN_LIBCALL || !TLI.getLibcallName(LC)) {
3491  // We'll expand the multiplication by brute force because we have no other
3492  // options. This is a trivially-generalized version of the code from
3493  // Hacker's Delight (itself derived from Knuth's Algorithm M from section
3494  // 4.3.1).
3495  unsigned Bits = NVT.getSizeInBits();
3496  unsigned HalfBits = Bits >> 1;
3497  SDValue Mask = DAG.getConstant(APInt::getLowBitsSet(Bits, HalfBits), dl,
3498  NVT);
3499  SDValue LLL = DAG.getNode(ISD::AND, dl, NVT, LL, Mask);
3500  SDValue RLL = DAG.getNode(ISD::AND, dl, NVT, RL, Mask);
3501 
3502  SDValue T = DAG.getNode(ISD::MUL, dl, NVT, LLL, RLL);
3503  SDValue TL = DAG.getNode(ISD::AND, dl, NVT, T, Mask);
3504 
3505  EVT ShiftAmtTy = TLI.getShiftAmountTy(NVT, DAG.getDataLayout());
3506  if (APInt::getMaxValue(ShiftAmtTy.getSizeInBits()).ult(HalfBits)) {
3507  // The type from TLI is too small to fit the shift amount we want.
3508  // Override it with i32. The shift will have to be legalized.
3509  ShiftAmtTy = MVT::i32;
3510  }
3511  SDValue Shift = DAG.getConstant(HalfBits, dl, ShiftAmtTy);
3512  SDValue TH = DAG.getNode(ISD::SRL, dl, NVT, T, Shift);
3513  SDValue LLH = DAG.getNode(ISD::SRL, dl, NVT, LL, Shift);
3514  SDValue RLH = DAG.getNode(ISD::SRL, dl, NVT, RL, Shift);
3515 
3516  SDValue U = DAG.getNode(ISD::ADD, dl, NVT,
3517  DAG.getNode(ISD::MUL, dl, NVT, LLH, RLL), TH);
3518  SDValue UL = DAG.getNode(ISD::AND, dl, NVT, U, Mask);
3519  SDValue UH = DAG.getNode(ISD::SRL, dl, NVT, U, Shift);
3520 
3521  SDValue V = DAG.getNode(ISD::ADD, dl, NVT,
3522  DAG.getNode(ISD::MUL, dl, NVT, LLL, RLH), UL);
3523  SDValue VH = DAG.getNode(ISD::SRL, dl, NVT, V, Shift);
3524 
3525  SDValue W = DAG.getNode(ISD::ADD, dl, NVT,
3526  DAG.getNode(ISD::MUL, dl, NVT, LLH, RLH),
3527  DAG.getNode(ISD::ADD, dl, NVT, UH, VH));
3528  Lo = DAG.getNode(ISD::ADD, dl, NVT, TL,
3529  DAG.getNode(ISD::SHL, dl, NVT, V, Shift));
3530 
3531  Hi = DAG.getNode(ISD::ADD, dl, NVT, W,
3532  DAG.getNode(ISD::ADD, dl, NVT,
3533  DAG.getNode(ISD::MUL, dl, NVT, RH, LL),
3534  DAG.getNode(ISD::MUL, dl, NVT, RL, LH)));
3535  return;
3536  }
3537 
3538  SDValue Ops[2] = { N->getOperand(0), N->getOperand(1) };
3540  CallOptions.setSExt(true);
3541  SplitInteger(TLI.makeLibCall(DAG, LC, VT, Ops, CallOptions, dl).first,
3542  Lo, Hi);
3543 }
3544 
3545 void DAGTypeLegalizer::ExpandIntRes_READCYCLECOUNTER(SDNode *N, SDValue &Lo,
3546  SDValue &Hi) {
3547  SDLoc DL(N);
3548  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
3549  SDVTList VTs = DAG.getVTList(NVT, NVT, MVT::Other);
3550  SDValue R = DAG.getNode(N->getOpcode(), DL, VTs, N->getOperand(0));
3551  Lo = R.getValue(0);
3552  Hi = R.getValue(1);
3553  ReplaceValueWith(SDValue(N, 1), R.getValue(2));
3554 }
3555 
3556 void DAGTypeLegalizer::ExpandIntRes_ADDSUBSAT(SDNode *N, SDValue &Lo,
3557  SDValue &Hi) {
3558  SDValue Result = TLI.expandAddSubSat(N, DAG);
3559  SplitInteger(Result, Lo, Hi);
3560 }
3561 
3562 void DAGTypeLegalizer::ExpandIntRes_SHLSAT(SDNode *N, SDValue &Lo,
3563  SDValue &Hi) {
3564  SDValue Result = TLI.expandShlSat(N, DAG);
3565  SplitInteger(Result, Lo, Hi);
3566 }
3567 
3568 /// This performs an expansion of the integer result for a fixed point
3569 /// multiplication. The default expansion performs rounding down towards
3570 /// negative infinity, though targets that do care about rounding should specify
3571 /// a target hook for rounding and provide their own expansion or lowering of
3572 /// fixed point multiplication to be consistent with rounding.
3573 void DAGTypeLegalizer::ExpandIntRes_MULFIX(SDNode *N, SDValue &Lo,
3574  SDValue &Hi) {
3575  SDLoc dl(N);
3576  EVT VT = N->getValueType(0);
3577  unsigned VTSize = VT.getScalarSizeInBits();
3578  SDValue LHS = N->getOperand(0);
3579  SDValue RHS = N->getOperand(1);
3580  uint64_t Scale = N->getConstantOperandVal(2);
3581  bool Saturating = (N->getOpcode() == ISD::SMULFIXSAT ||
3582  N->getOpcode() == ISD::UMULFIXSAT);
3583  bool Signed = (N->getOpcode() == ISD::SMULFIX ||
3584  N->getOpcode() == ISD::SMULFIXSAT);
3585 
3586  // Handle special case when scale is equal to zero.
3587  if (!Scale) {
3588  SDValue Result;
3589  if (!Saturating) {
3590  Result = DAG.getNode(ISD::MUL, dl, VT, LHS, RHS);
3591  } else {
3592  EVT BoolVT = getSetCCResultType(VT);
3593  unsigned MulOp = Signed ? ISD::SMULO : ISD::UMULO;
3594  Result = DAG.getNode(MulOp, dl, DAG.getVTList(VT, BoolVT), LHS, RHS);
3595  SDValue Product = Result.getValue(0);
3596  SDValue Overflow = Result.getValue(1);
3597  if (Signed) {
3598  APInt MinVal = APInt::getSignedMinValue(VTSize);
3599  APInt MaxVal = APInt::getSignedMaxValue(VTSize);
3600  SDValue SatMin = DAG.getConstant(MinVal, dl, VT);
3601  SDValue SatMax = DAG.getConstant(MaxVal, dl, VT);
3602  SDValue Zero = DAG.getConstant(0, dl, VT);
3603  // Xor the inputs, if resulting sign bit is 0 the product will be
3604  // positive, else negative.
3605  SDValue Xor = DAG.getNode(ISD::XOR, dl, VT, LHS, RHS);
3606  SDValue ProdNeg = DAG.getSetCC(dl, BoolVT, Xor, Zero, ISD::SETLT);
3607  Result = DAG.getSelect(dl, VT, ProdNeg, SatMin, SatMax);
3608  Result = DAG.getSelect(dl, VT, Overflow, Result, Product);
3609  } else {
3610  // For unsigned multiplication, we only need to check the max since we
3611  // can't really overflow towards zero.
3612  APInt MaxVal = APInt::getMaxValue(VTSize);
3613  SDValue SatMax = DAG.getConstant(MaxVal, dl, VT);
3614  Result = DAG.getSelect(dl, VT, Overflow, SatMax, Product);
3615  }
3616  }
3617  SplitInteger(Result, Lo, Hi);
3618  return;
3619  }
3620 
3621  // For SMULFIX[SAT] we only expect to find Scale<VTSize, but this assert will
3622  // cover for unhandled cases below, while still being valid for UMULFIX[SAT].
3623  assert(Scale <= VTSize && "Scale can't be larger than the value type size.");
3624 
3625  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
3626  SDValue LL, LH, RL, RH;
3627  GetExpandedInteger(LHS, LL, LH);
3628  GetExpandedInteger(RHS, RL, RH);
3630 
3631  unsigned LoHiOp = Signed ? ISD::SMUL_LOHI : ISD::UMUL_LOHI;
3632  if (!TLI.expandMUL_LOHI(LoHiOp, VT, dl, LHS, RHS, Result, NVT, DAG,
3634  LL, LH, RL, RH)) {
3635  report_fatal_error("Unable to expand MUL_FIX using MUL_LOHI.");
3636  return;
3637  }
3638 
3639  unsigned NVTSize = NVT.getScalarSizeInBits();
3640  assert((VTSize == NVTSize * 2) && "Expected the new value type to be half "
3641  "the size of the current value type");
3642  EVT ShiftTy = TLI.getShiftAmountTy(NVT, DAG.getDataLayout());
3643 
3644  // After getting the multiplication result in 4 parts, we need to perform a
3645  // shift right by the amount of the scale to get the result in that scale.
3646  //
3647  // Let's say we multiply 2 64 bit numbers. The resulting value can be held in
3648  // 128 bits that are cut into 4 32-bit parts:
3649  //
3650  // HH HL LH LL
3651  // |---32---|---32---|---32---|---32---|
3652  // 128 96 64 32 0
3653  //
3654  // |------VTSize-----|
3655  //
3656  // |NVTSize-|
3657  //
3658  // The resulting Lo and Hi would normally be in LL and LH after the shift. But
3659  // to avoid unneccessary shifting of all 4 parts, we can adjust the shift
3660  // amount and get Lo and Hi using two funnel shifts. Or for the special case
3661  // when Scale is a multiple of NVTSize we can just pick the result without
3662  // shifting.
3663  uint64_t Part0 = Scale / NVTSize; // Part holding lowest bit needed.
3664  if (Scale % NVTSize) {
3665  SDValue ShiftAmount = DAG.getConstant(Scale % NVTSize, dl, ShiftTy);
3666  Lo = DAG.getNode(ISD::FSHR, dl, NVT, Result[Part0 + 1], Result[Part0],
3667  ShiftAmount);
3668  Hi = DAG.getNode(ISD::FSHR, dl, NVT, Result[Part0 + 2], Result[Part0 + 1],
3669  ShiftAmount);
3670  } else {
3671  Lo = Result[Part0];
3672  Hi = Result[Part0 + 1];
3673  }
3674 
3675  // Unless saturation is requested we are done. The result is in <Hi,Lo>.
3676  if (!Saturating)
3677  return;
3678 
3679  // Can not overflow when there is no integer part.
3680  if (Scale == VTSize)
3681  return;
3682 
3683  // To handle saturation we must check for overflow in the multiplication.
3684  //
3685  // Unsigned overflow happened if the upper (VTSize - Scale) bits (of Result)
3686  // aren't all zeroes.
3687  //
3688  // Signed overflow happened if the upper (VTSize - Scale + 1) bits (of Result)
3689  // aren't all ones or all zeroes.
3690  //
3691  // We cannot overflow past HH when multiplying 2 ints of size VTSize, so the
3692  // highest bit of HH determines saturation direction in the event of signed
3693  // saturation.
3694 
3695  SDValue ResultHL = Result[2];
3696  SDValue ResultHH = Result[3];
3697 
3698  SDValue SatMax, SatMin;
3699  SDValue NVTZero = DAG.getConstant(0, dl, NVT);
3700  SDValue NVTNeg1 = DAG.getConstant(-1, dl, NVT);
3701  EVT BoolNVT = getSetCCResultType(NVT);
3702 
3703  if (!Signed) {
3704  if (Scale < NVTSize) {
3705  // Overflow happened if ((HH | (HL >> Scale)) != 0).
3706  SDValue HLAdjusted = DAG.getNode(ISD::SRL, dl, NVT, ResultHL,
3707  DAG.getConstant(Scale, dl, ShiftTy));
3708  SDValue Tmp = DAG.getNode(ISD::OR, dl, NVT, HLAdjusted, ResultHH);
3709  SatMax = DAG.getSetCC(dl, BoolNVT, Tmp, NVTZero, ISD::SETNE);
3710  } else if (Scale == NVTSize) {
3711  // Overflow happened if (HH != 0).
3712  SatMax = DAG.getSetCC(dl, BoolNVT, ResultHH, NVTZero, ISD::SETNE);
3713  } else if (Scale < VTSize) {
3714  // Overflow happened if ((HH >> (Scale - NVTSize)) != 0).
3715  SDValue HLAdjusted = DAG.getNode(ISD::SRL, dl, NVT, ResultHL,
3716  DAG.getConstant(Scale - NVTSize, dl,
3717  ShiftTy));
3718  SatMax = DAG.getSetCC(dl, BoolNVT, HLAdjusted, NVTZero, ISD::SETNE);
3719  } else
3720  llvm_unreachable("Scale must be less or equal to VTSize for UMULFIXSAT"
3721  "(and saturation can't happen with Scale==VTSize).");
3722 
3723  Hi = DAG.getSelect(dl, NVT, SatMax, NVTNeg1, Hi);
3724  Lo = DAG.getSelect(dl, NVT, SatMax, NVTNeg1, Lo);
3725  return;
3726  }
3727 
3728  if (Scale < NVTSize) {
3729  // The number of overflow bits we can check are VTSize - Scale + 1 (we
3730  // include the sign bit). If these top bits are > 0, then we overflowed past
3731  // the max value. If these top bits are < -1, then we overflowed past the
3732  // min value. Otherwise, we did not overflow.
3733  unsigned OverflowBits = VTSize - Scale + 1;
3734  assert(OverflowBits <= VTSize && OverflowBits > NVTSize &&
3735  "Extent of overflow bits must start within HL");
3736  SDValue HLHiMask = DAG.getConstant(
3737  APInt::getHighBitsSet(NVTSize, OverflowBits - NVTSize), dl, NVT);
3738  SDValue HLLoMask = DAG.getConstant(
3739  APInt::getLowBitsSet(NVTSize, VTSize - OverflowBits), dl, NVT);
3740  // We overflow max if HH > 0 or (HH == 0 && HL > HLLoMask).
3741  SDValue HHGT0 = DAG.getSetCC(dl, BoolNVT, ResultHH, NVTZero, ISD::SETGT);
3742  SDValue HHEQ0 = DAG.getSetCC(dl, BoolNVT, ResultHH, NVTZero, ISD::SETEQ);
3743  SDValue HLUGT = DAG.getSetCC(dl, BoolNVT, ResultHL, HLLoMask, ISD::SETUGT);
3744  SatMax = DAG.getNode(ISD::OR, dl, BoolNVT, HHGT0,
3745  DAG.getNode(ISD::AND, dl, BoolNVT, HHEQ0, HLUGT));
3746  // We overflow min if HH < -1 or (HH == -1 && HL < HLHiMask).
3747  SDValue HHLT = DAG.getSetCC(dl, BoolNVT, ResultHH, NVTNeg1, ISD::SETLT);
3748  SDValue HHEQ = DAG.getSetCC(dl, BoolNVT, ResultHH, NVTNeg1, ISD::SETEQ);
3749  SDValue HLULT = DAG.getSetCC(dl, BoolNVT, ResultHL, HLHiMask, ISD::SETULT);
3750  SatMin = DAG.getNode(ISD::OR, dl, BoolNVT, HHLT,
3751  DAG.getNode(ISD::AND, dl, BoolNVT, HHEQ, HLULT));
3752  } else if (Scale == NVTSize) {
3753  // We overflow max if HH > 0 or (HH == 0 && HL sign bit is 1).
3754  SDValue HHGT0 = DAG.getSetCC(dl, BoolNVT, ResultHH, NVTZero, ISD::SETGT);
3755  SDValue HHEQ0 = DAG.getSetCC(dl, BoolNVT, ResultHH, NVTZero, ISD::SETEQ);
3756  SDValue HLNeg = DAG.getSetCC(dl, BoolNVT, ResultHL, NVTZero, ISD::SETLT);
3757  SatMax = DAG.getNode(ISD::OR, dl, BoolNVT, HHGT0,
3758  DAG.getNode(ISD::AND, dl, BoolNVT, HHEQ0, HLNeg));
3759  // We overflow min if HH < -1 or (HH == -1 && HL sign bit is 0).
3760  SDValue HHLT = DAG.getSetCC(dl, BoolNVT, ResultHH, NVTNeg1, ISD::SETLT);
3761  SDValue HHEQ = DAG.getSetCC(dl, BoolNVT, ResultHH, NVTNeg1, ISD::SETEQ);
3762  SDValue HLPos = DAG.getSetCC(dl, BoolNVT, ResultHL, NVTZero, ISD::SETGE);
3763  SatMin = DAG.getNode(ISD::OR, dl, BoolNVT, HHLT,
3764  DAG.getNode(ISD::AND, dl, BoolNVT, HHEQ, HLPos));
3765  } else if (Scale < VTSize) {
3766  // This is similar to the case when we saturate if Scale < NVTSize, but we
3767  // only need to check HH.
3768  unsigned OverflowBits = VTSize - Scale + 1;
3769  SDValue HHHiMask = DAG.getConstant(
3770  APInt::getHighBitsSet(NVTSize, OverflowBits), dl, NVT);
3771  SDValue HHLoMask = DAG.getConstant(
3772  APInt::getLowBitsSet(NVTSize, NVTSize - OverflowBits), dl, NVT);
3773  SatMax = DAG.getSetCC(dl, BoolNVT, ResultHH, HHLoMask, ISD::SETGT);
3774  SatMin = DAG.getSetCC(dl, BoolNVT, ResultHH, HHHiMask, ISD::SETLT);
3775  } else
3776  llvm_unreachable("Illegal scale for signed fixed point mul.");
3777 
3778  // Saturate to signed maximum.
3779  APInt MaxHi = APInt::getSignedMaxValue(NVTSize);
3780  APInt MaxLo = APInt::getAllOnes(NVTSize);
3781  Hi = DAG.getSelect(dl, NVT, SatMax, DAG.getConstant(MaxHi, dl, NVT), Hi);
3782  Lo = DAG.getSelect(dl, NVT, SatMax, DAG.getConstant(MaxLo, dl, NVT), Lo);
3783  // Saturate to signed minimum.
3784  APInt MinHi = APInt::getSignedMinValue(NVTSize);
3785  Hi = DAG.getSelect(dl, NVT, SatMin, DAG.getConstant(MinHi, dl, NVT), Hi);
3786  Lo = DAG.getSelect(dl, NVT, SatMin, NVTZero, Lo);
3787 }
3788 
3789 void DAGTypeLegalizer::ExpandIntRes_DIVFIX(SDNode *N, SDValue &Lo,
3790  SDValue &Hi) {
3791  SDLoc dl(N);
3792  // Try expanding in the existing type first.
3793  SDValue Res = TLI.expandFixedPointDiv(N->getOpcode(), dl, N->getOperand(0),
3794  N->getOperand(1),
3795  N->getConstantOperandVal(2), DAG);
3796 
3797  if (!Res)
3798  Res = earlyExpandDIVFIX(N, N->getOperand(0), N->getOperand(1),
3799  N->getConstantOperandVal(2), TLI, DAG);
3800  SplitInteger(Res, Lo, Hi);
3801 }
3802 
3803 void DAGTypeLegalizer::ExpandIntRes_SADDSUBO(SDNode *Node,
3804  SDValue &Lo, SDValue &Hi) {
3805  assert((Node->getOpcode() == ISD::SADDO || Node->getOpcode() == ISD::SSUBO) &&
3806  "Node has unexpected Opcode");
3807  SDValue LHS = Node->getOperand(0);
3808  SDValue RHS = Node->getOperand(1);
3809  SDLoc dl(Node);
3810 
3811  SDValue Ovf;
3812 
3813  bool IsAdd = Node->getOpcode() == ISD::SADDO;
3814  unsigned CarryOp = IsAdd ? ISD::SADDO_CARRY : ISD::SSUBO_CARRY;
3815 
3816  bool HasCarryOp = TLI.isOperationLegalOrCustom(
3817  CarryOp, TLI.getTypeToExpandTo(*DAG.getContext(), LHS.getValueType()));
3818 
3819  if (HasCarryOp) {
3820  // Expand the subcomponents.
3821  SDValue LHSL, LHSH, RHSL, RHSH;
3822  GetExpandedInteger(LHS, LHSL, LHSH);
3823  GetExpandedInteger(RHS, RHSL, RHSH);
3824  SDVTList VTList = DAG.getVTList(LHSL.getValueType(), Node->getValueType(1));
3825 
3826  Lo = DAG.getNode(IsAdd ? ISD::UADDO : ISD::USUBO, dl, VTList, {LHSL, RHSL});
3827  Hi = DAG.getNode(CarryOp, dl, VTList, { LHSH, RHSH, Lo.getValue(1) });
3828 
3829  Ovf = Hi.getValue(1);
3830  } else {
3831  // Expand the result by simply replacing it with the equivalent
3832  // non-overflow-checking operation.
3833  SDValue Sum = DAG.getNode(Node->getOpcode() == ISD::SADDO ?
3834  ISD::ADD : ISD::SUB, dl, LHS.getValueType(),
3835  LHS, RHS);
3836  SplitInteger(Sum, Lo, Hi);
3837 
3838  // Compute the overflow.
3839  //
3840  // LHSSign -> LHS < 0
3841  // RHSSign -> RHS < 0
3842  // SumSign -> Sum < 0
3843  //
3844  // Add:
3845  // Overflow -> (LHSSign == RHSSign) && (LHSSign != SumSign)
3846  // Sub:
3847  // Overflow -> (LHSSign != RHSSign) && (LHSSign != SumSign)
3848  //
3849  // To get better codegen we can rewrite this by doing bitwise math on
3850  // the integers and extract the final sign bit at the end. So the
3851  // above becomes:
3852  //
3853  // Add:
3854  // Overflow -> (~(LHS ^ RHS) & (LHS ^ Sum)) < 0
3855  // Sub:
3856  // Overflow -> ((LHS ^ RHS) & (LHS ^ Sum)) < 0
3857  //
3858  // NOTE: This is different than the expansion we do in expandSADDSUBO
3859  // because it is more costly to determine the RHS is > 0 for SSUBO with the
3860  // integers split.
3861  EVT VT = LHS.getValueType();
3862  SDValue SignsMatch = DAG.getNode(ISD::XOR, dl, VT, LHS, RHS);
3863  if (IsAdd)
3864  SignsMatch = DAG.getNOT(dl, SignsMatch, VT);
3865 
3866  SDValue SumSignNE = DAG.getNode(ISD::XOR, dl, VT, LHS, Sum);
3867  Ovf = DAG.getNode(ISD::AND, dl, VT, SignsMatch, SumSignNE);
3868  EVT OType = Node->getValueType(1);
3869  Ovf = DAG.getSetCC(dl, OType, Ovf, DAG.getConstant(0, dl, VT), ISD::SETLT);
3870  }
3871 
3872  // Use the calculated overflow everywhere.
3873  ReplaceValueWith(SDValue(Node, 1), Ovf);
3874 }
3875 
3876 void DAGTypeLegalizer::ExpandIntRes_SDIV(SDNode *N,
3877  SDValue &Lo, SDValue &Hi) {
3878  EVT VT = N->getValueType(0);
3879  SDLoc dl(N);
3880  SDValue Ops[2] = { N->getOperand(0), N->getOperand(1) };
3881 
3883  SDValue Res = DAG.getNode(ISD::SDIVREM, dl, DAG.getVTList(VT, VT), Ops);
3884  SplitInteger(Res.getValue(0), Lo, Hi);
3885  return;
3886  }
3887 
3888  RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
3889  if (VT == MVT::i16)
3890  LC = RTLIB::SDIV_I16;
3891  else if (VT == MVT::i32)
3892  LC = RTLIB::SDIV_I32;
3893  else if (VT == MVT::i64)
3894  LC = RTLIB::SDIV_I64;
3895  else if (VT == MVT::i128)
3896  LC = RTLIB::SDIV_I128;
3897  assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported SDIV!");
3898 
3900  CallOptions.setSExt(true);
3901  SplitInteger(TLI.makeLibCall(DAG, LC, VT, Ops, CallOptions, dl).first, Lo, Hi);
3902 }
3903 
3904 void DAGTypeLegalizer::ExpandIntRes_Shift(SDNode *N,
3905  SDValue &Lo, SDValue &Hi) {
3906  EVT VT = N->getValueType(0);
3907  SDLoc dl(N);
3908 
3909  // If we can emit an efficient shift operation, do so now. Check to see if
3910  // the RHS is a constant.
3911  if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(N->getOperand(1)))
3912  return ExpandShiftByConstant(N, CN->getAPIntValue(), Lo, Hi);
3913 
3914  // If we can determine that the high bit of the shift is zero or one, even if
3915  // the low bits are variable, emit this shift in an optimized form.
3916  if (ExpandShiftWithKnownAmountBit(N, Lo, Hi))
3917  return;
3918 
3919  // If this target supports shift_PARTS, use it. First, map to the _PARTS opc.
3920  unsigned PartsOpc;
3921  if (N->getOpcode() == ISD::SHL) {
3922  PartsOpc = ISD::SHL_PARTS;
3923  } else if (N->getOpcode() == ISD::SRL) {
3924  PartsOpc = ISD::SRL_PARTS;
3925  } else {
3926  assert(N->getOpcode() == ISD::SRA && "Unknown shift!");
3927  PartsOpc = ISD::SRA_PARTS;
3928  }
3929 
3930  // Next check to see if the target supports this SHL_PARTS operation or if it
3931  // will custom expand it. Don't lower this to SHL_PARTS when we optimise for
3932  // size, but create a libcall instead.
3933  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
3934  TargetLowering::LegalizeAction Action = TLI.getOperationAction(PartsOpc, NVT);
3935  const bool LegalOrCustom =
3936  (Action == TargetLowering::Legal && TLI.isTypeLegal(NVT)) ||
3937  Action == TargetLowering::Custom;
3938 
3939  if (LegalOrCustom && TLI.shouldExpandShift(DAG, N)) {
3940  // Expand the subcomponents.
3941  SDValue LHSL, LHSH;
3942  GetExpandedInteger(N->getOperand(0), LHSL, LHSH);
3943  EVT VT = LHSL.getValueType();
3944 
3945  // If the shift amount operand is coming from a vector legalization it may
3946  // have an illegal type. Fix that first by casting the operand, otherwise
3947  // the new SHL_PARTS operation would need further legalization.
3948  SDValue ShiftOp = N->getOperand(1);
3949  EVT ShiftTy = TLI.getShiftAmountTy(VT, DAG.getDataLayout());
3950  assert(ShiftTy.getScalarSizeInBits() >=
3952  "ShiftAmountTy is too small to cover the range of this type!");
3953  if (ShiftOp.getValueType() != ShiftTy)
3954  ShiftOp = DAG.getZExtOrTrunc(ShiftOp, dl, ShiftTy);
3955 
3956  SDValue Ops[] = { LHSL, LHSH, ShiftOp };
3957  Lo = DAG.getNode(PartsOpc, dl, DAG.getVTList(VT, VT), Ops);
3958  Hi = Lo.getValue(1);
3959  return;
3960  }
3961 
3962  // Otherwise, emit a libcall.
3963  RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
3964  bool isSigned;
3965  if (N->getOpcode() == ISD::SHL) {
3966  isSigned = false; /*sign irrelevant*/
3967  if (VT == MVT::i16)
3968  LC = RTLIB::SHL_I16;
3969  else if (VT == MVT::i32)
3970  LC = RTLIB::SHL_I32;
3971  else if (VT == MVT::i64)
3972  LC = RTLIB::SHL_I64;
3973  else if (VT == MVT::i128)
3974  LC = RTLIB::SHL_I128;
3975  } else if (N->getOpcode() == ISD::SRL) {
3976  isSigned = false;
3977  if (VT == MVT::i16)
3978  LC = RTLIB::SRL_I16;
3979  else if (VT == MVT::i32)
3980  LC = RTLIB::SRL_I32;
3981  else if (VT == MVT::i64)
3982  LC = RTLIB::SRL_I64;
3983  else if (VT == MVT::i128)
3984  LC = RTLIB::SRL_I128;
3985  } else {
3986  assert(N->getOpcode() == ISD::SRA && "Unknown shift!");
3987  isSigned = true;
3988  if (VT == MVT::i16)
3989  LC = RTLIB::SRA_I16;
3990  else if (VT == MVT::i32)
3991  LC = RTLIB::SRA_I32;
3992  else if (VT == MVT::i64)
3993  LC = RTLIB::SRA_I64;
3994  else if (VT == MVT::i128)
3995  LC = RTLIB::SRA_I128;
3996  }
3997 
3998  if (LC != RTLIB::UNKNOWN_LIBCALL && TLI.getLibcallName(LC)) {
3999  SDValue Ops[2] = { N->getOperand(0), N->getOperand(1) };
4001  CallOptions.setSExt(isSigned);
4002  SplitInteger(TLI.makeLibCall(DAG, LC, VT, Ops, CallOptions, dl).first, Lo, Hi);
4003  return;
4004  }
4005 
4006  if (!ExpandShiftWithUnknownAmountBit(N, Lo, Hi))
4007  llvm_unreachable("Unsupported shift!");
4008 }
4009 
4010 void DAGTypeLegalizer::ExpandIntRes_SIGN_EXTEND(SDNode *N,
4011  SDValue &Lo, SDValue &Hi) {
4012  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
4013  SDLoc dl(N);
4014  SDValue Op = N->getOperand(0);
4015  if (Op.getValueType().bitsLE(NVT)) {
4016  // The low part is sign extension of the input (degenerates to a copy).
4017  Lo = DAG.getNode(ISD::SIGN_EXTEND, dl, NVT, N->getOperand(0));
4018  // The high part is obtained by SRA'ing all but one of the bits of low part.
4019  unsigned LoSize = NVT.getSizeInBits();
4020  Hi = DAG.getNode(
4021  ISD::SRA, dl, NVT, Lo,
4022  DAG.getConstant(LoSize - 1, dl, TLI.getPointerTy(DAG.getDataLayout())));
4023  } else {
4024  // For example, extension of an i48 to an i64. The operand type necessarily
4025  // promotes to the result type, so will end up being expanded too.
4026  assert(getTypeAction(Op.getValueType()) ==
4028  "Only know how to promote this result!");
4029  SDValue Res = GetPromotedInteger(Op);
4030  assert(Res.getValueType() == N->getValueType(0) &&
4031  "Operand over promoted?");
4032  // Split the promoted operand. This will simplify when it is expanded.
4033  SplitInteger(Res, Lo, Hi);
4034  unsigned ExcessBits = Op.getValueSizeInBits() - NVT.getSizeInBits();
4035  Hi = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, Hi.getValueType(), Hi,
4037  ExcessBits)));
4038  }
4039 }
4040 
4041 void DAGTypeLegalizer::
4042 ExpandIntRes_SIGN_EXTEND_INREG(SDNode *N, SDValue &Lo, SDValue &Hi) {
4043  SDLoc dl(N);
4044  GetExpandedInteger(N->getOperand(0), Lo, Hi);
4045  EVT EVT = cast<VTSDNode>(N->getOperand(1))->getVT();
4046 
4047  if (EVT.bitsLE(Lo.getValueType())) {
4048  // sext_inreg the low part if needed.
4049  Lo = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, Lo.getValueType(), Lo,
4050  N->getOperand(1));
4051 
4052  // The high part gets the sign extension from the lo-part. This handles
4053  // things like sextinreg V:i64 from i8.
4054  Hi = DAG.getNode(ISD::SRA, dl, Hi.getValueType(), Lo,
4055  DAG.getConstant(Hi.getValueSizeInBits() - 1, dl,
4056  TLI.getPointerTy(DAG.getDataLayout())));
4057  } else {
4058  // For example, extension of an i48 to an i64. Leave the low part alone,
4059  // sext_inreg the high part.
4060  unsigned ExcessBits = EVT.getSizeInBits() - Lo.getValueSizeInBits();
4061  Hi = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, Hi.getValueType(), Hi,
4063  ExcessBits)));
4064  }
4065 }
4066 
4067 void DAGTypeLegalizer::ExpandIntRes_SREM(SDNode *N,
4068  SDValue &Lo, SDValue &Hi) {
4069  EVT VT = N->getValueType(0);
4070  SDLoc dl(N);
4071  SDValue Ops[2] = { N->getOperand(0), N->getOperand(1) };
4072 
4074  SDValue Res = DAG.getNode(ISD::SDIVREM, dl, DAG.getVTList(VT, VT), Ops);
4075  SplitInteger(Res.getValue(1), Lo, Hi);
4076  return;
4077  }
4078 
4079  RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
4080  if (VT == MVT::i16)
4081  LC = RTLIB::SREM_I16;
4082  else if (VT == MVT::i32)
4083  LC = RTLIB::SREM_I32;
4084  else if (VT == MVT::i64)
4085  LC = RTLIB::SREM_I64;
4086  else if (VT == MVT::i128)
4087  LC = RTLIB::SREM_I128;
4088  assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported SREM!");
4089 
4091  CallOptions.setSExt(true);
4092  SplitInteger(TLI.makeLibCall(DAG, LC, VT, Ops, CallOptions, dl).first, Lo, Hi);
4093 }
4094 
4095 void DAGTypeLegalizer::ExpandIntRes_TRUNCATE(SDNode *N,
4096  SDValue &Lo, SDValue &Hi) {
4097  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
4098  SDLoc dl(N);
4099  Lo = DAG.getNode(ISD::TRUNCATE, dl, NVT, N->getOperand(0));
4100  Hi = DAG.getNode(ISD::SRL, dl, N->getOperand(0).getValueType(),
4101  N->getOperand(0),
4102  DAG.getConstant(NVT.getSizeInBits(), dl,
4103  TLI.getPointerTy(DAG.getDataLayout())));
4104  Hi = DAG.getNode(ISD::TRUNCATE, dl, NVT, Hi);
4105 }
4106 
4107 void DAGTypeLegalizer::ExpandIntRes_XMULO(SDNode *N,
4108  SDValue &Lo, SDValue &Hi) {
4109  EVT VT = N->getValueType(0);
4110  SDLoc dl(N);
4111 
4112  if (N->getOpcode() == ISD::UMULO) {
4113  // This section expands the operation into the following sequence of
4114  // instructions. `iNh` here refers to a type which has half the bit width of
4115  // the type the original operation operated on.
4116  //
4117  // %0 = %LHS.HI != 0 && %RHS.HI != 0
4118  // %1 = { iNh, i1 } @umul.with.overflow.iNh(iNh %LHS.HI, iNh %RHS.LO)
4119  // %2 = { iNh, i1 } @umul.with.overflow.iNh(iNh %RHS.HI, iNh %LHS.LO)
4120  // %3 = mul nuw iN (%LHS.LOW as iN), (%RHS.LOW as iN)
4121  // %4 = add iNh %1.0, %2.0 as iN
4122  // %5 = { iNh, i1 } @uadd.with.overflow.iNh(iNh %4, iNh %3.HIGH)
4123  //
4124  // %lo = %3.LO
4125  // %hi = %5.0
4126  // %ovf = %0 || %1.1 || %2.1 || %5.1
4127  SDValue LHS = N->getOperand(0), RHS = N->getOperand(1);
4128  SDValue LHSHigh, LHSLow, RHSHigh, RHSLow;
4129  GetExpandedInteger(LHS, LHSLow, LHSHigh);
4130  GetExpandedInteger(RHS, RHSLow, RHSHigh);
4131  EVT HalfVT = LHSLow.getValueType();
4132  EVT BitVT = N->getValueType(1);
4133  SDVTList VTHalfWithO = DAG.getVTList(HalfVT, BitVT);
4134 
4135  SDValue HalfZero = DAG.getConstant(0, dl, HalfVT);
4136  SDValue Overflow = DAG.getNode(ISD::AND, dl, BitVT,
4137  DAG.getSetCC(dl, BitVT, LHSHigh, HalfZero, ISD::SETNE),
4138  DAG.getSetCC(dl, BitVT, RHSHigh, HalfZero, ISD::SETNE));
4139 
4140  SDValue One = DAG.getNode(ISD::UMULO, dl, VTHalfWithO, LHSHigh, RHSLow);
4141  Overflow = DAG.getNode(ISD::OR, dl, BitVT, Overflow, One.getValue(1));
4142 
4143  SDValue Two = DAG.getNode(ISD::UMULO, dl, VTHalfWithO, RHSHigh, LHSLow);
4144  Overflow = DAG.getNode(ISD::OR, dl, BitVT, Overflow, Two.getValue(1));
4145 
4146  SDValue HighSum = DAG.getNode(ISD::ADD, dl, HalfVT, One, Two);
4147 
4148  // Cannot use `UMUL_LOHI` directly, because some 32-bit targets (ARM) do not
4149  // know how to expand `i64,i64 = umul_lohi a, b` and abort (why isn’t this
4150  // operation recursively legalized?).
4151  //
4152  // Many backends understand this pattern and will convert into LOHI
4153  // themselves, if applicable.
4154  SDValue Three = DAG.getNode(ISD::MUL, dl, VT,
4155  DAG.getNode(ISD::ZERO_EXTEND, dl, VT, LHSLow),
4156  DAG.