LLVM  16.0.0git
LegalizeIntegerTypes.cpp
Go to the documentation of this file.
1 //===----- LegalizeIntegerTypes.cpp - Legalization of integer types -------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file implements integer type expansion and promotion for LegalizeTypes.
10 // Promotion is the act of changing a computation in an illegal type into a
11 // computation in a larger type. For example, implementing i8 arithmetic in an
12 // i32 register (often needed on powerpc).
13 // Expansion is the act of changing a computation in an illegal type into a
14 // computation in two identical registers of a smaller type. For example,
15 // implementing i64 arithmetic in two i32 registers (often needed on 32-bit
16 // targets).
17 //
18 //===----------------------------------------------------------------------===//
19 
20 #include "LegalizeTypes.h"
22 #include "llvm/CodeGen/StackMaps.h"
23 #include "llvm/IR/DerivedTypes.h"
25 #include "llvm/Support/KnownBits.h"
27 #include <algorithm>
28 using namespace llvm;
29 
30 #define DEBUG_TYPE "legalize-types"
31 
32 //===----------------------------------------------------------------------===//
33 // Integer Result Promotion
34 //===----------------------------------------------------------------------===//
35 
36 /// PromoteIntegerResult - This method is called when a result of a node is
37 /// found to be in need of promotion to a larger type. At this point, the node
38 /// may also have invalid operands or may have other results that need
39 /// expansion, we just know that (at least) one result needs promotion.
40 void DAGTypeLegalizer::PromoteIntegerResult(SDNode *N, unsigned ResNo) {
41  LLVM_DEBUG(dbgs() << "Promote integer result: "; N->dump(&DAG);
42  dbgs() << "\n");
43  SDValue Res = SDValue();
44 
45  // See if the target wants to custom expand this node.
46  if (CustomLowerNode(N, N->getValueType(ResNo), true)) {
47  LLVM_DEBUG(dbgs() << "Node has been custom expanded, done\n");
48  return;
49  }
50 
51  switch (N->getOpcode()) {
52  default:
53 #ifndef NDEBUG
54  dbgs() << "PromoteIntegerResult #" << ResNo << ": ";
55  N->dump(&DAG); dbgs() << "\n";
56 #endif
57  llvm_unreachable("Do not know how to promote this operator!");
58  case ISD::MERGE_VALUES:Res = PromoteIntRes_MERGE_VALUES(N, ResNo); break;
59  case ISD::AssertSext: Res = PromoteIntRes_AssertSext(N); break;
60  case ISD::AssertZext: Res = PromoteIntRes_AssertZext(N); break;
61  case ISD::BITCAST: Res = PromoteIntRes_BITCAST(N); break;
62  case ISD::BITREVERSE: Res = PromoteIntRes_BITREVERSE(N); break;
63  case ISD::BSWAP: Res = PromoteIntRes_BSWAP(N); break;
64  case ISD::BUILD_PAIR: Res = PromoteIntRes_BUILD_PAIR(N); break;
65  case ISD::Constant: Res = PromoteIntRes_Constant(N); break;
67  case ISD::CTLZ: Res = PromoteIntRes_CTLZ(N); break;
68  case ISD::PARITY:
69  case ISD::CTPOP: Res = PromoteIntRes_CTPOP_PARITY(N); break;
71  case ISD::CTTZ: Res = PromoteIntRes_CTTZ(N); break;
73  Res = PromoteIntRes_EXTRACT_VECTOR_ELT(N); break;
74  case ISD::LOAD: Res = PromoteIntRes_LOAD(cast<LoadSDNode>(N)); break;
75  case ISD::MLOAD: Res = PromoteIntRes_MLOAD(cast<MaskedLoadSDNode>(N));
76  break;
77  case ISD::MGATHER: Res = PromoteIntRes_MGATHER(cast<MaskedGatherSDNode>(N));
78  break;
79  case ISD::SELECT:
80  case ISD::VSELECT:
81  case ISD::VP_SELECT:
82  case ISD::VP_MERGE:
83  Res = PromoteIntRes_Select(N);
84  break;
85  case ISD::SELECT_CC: Res = PromoteIntRes_SELECT_CC(N); break;
86  case ISD::STRICT_FSETCC:
88  case ISD::SETCC: Res = PromoteIntRes_SETCC(N); break;
89  case ISD::SMIN:
90  case ISD::SMAX: Res = PromoteIntRes_SExtIntBinOp(N); break;
91  case ISD::UMIN:
92  case ISD::UMAX: Res = PromoteIntRes_UMINUMAX(N); break;
93 
94  case ISD::SHL:
95  case ISD::VP_SHL: Res = PromoteIntRes_SHL(N); break;
97  Res = PromoteIntRes_SIGN_EXTEND_INREG(N); break;
98  case ISD::SRA:
99  case ISD::VP_ASHR: Res = PromoteIntRes_SRA(N); break;
100  case ISD::SRL:
101  case ISD::VP_LSHR: Res = PromoteIntRes_SRL(N); break;
102  case ISD::VP_TRUNCATE:
103  case ISD::TRUNCATE: Res = PromoteIntRes_TRUNCATE(N); break;
104  case ISD::UNDEF: Res = PromoteIntRes_UNDEF(N); break;
105  case ISD::VAARG: Res = PromoteIntRes_VAARG(N); break;
106  case ISD::VSCALE: Res = PromoteIntRes_VSCALE(N); break;
107 
109  Res = PromoteIntRes_EXTRACT_SUBVECTOR(N); break;
111  Res = PromoteIntRes_INSERT_SUBVECTOR(N); break;
112  case ISD::VECTOR_REVERSE:
113  Res = PromoteIntRes_VECTOR_REVERSE(N); break;
114  case ISD::VECTOR_SHUFFLE:
115  Res = PromoteIntRes_VECTOR_SHUFFLE(N); break;
116  case ISD::VECTOR_SPLICE:
117  Res = PromoteIntRes_VECTOR_SPLICE(N); break;
119  Res = PromoteIntRes_INSERT_VECTOR_ELT(N); break;
120  case ISD::BUILD_VECTOR:
121  Res = PromoteIntRes_BUILD_VECTOR(N);
122  break;
123  case ISD::SPLAT_VECTOR:
125  Res = PromoteIntRes_ScalarOp(N);
126  break;
127  case ISD::STEP_VECTOR: Res = PromoteIntRes_STEP_VECTOR(N); break;
128  case ISD::CONCAT_VECTORS:
129  Res = PromoteIntRes_CONCAT_VECTORS(N); break;
130 
134  Res = PromoteIntRes_EXTEND_VECTOR_INREG(N); break;
135 
136  case ISD::SIGN_EXTEND:
137  case ISD::ZERO_EXTEND:
138  case ISD::ANY_EXTEND: Res = PromoteIntRes_INT_EXTEND(N); break;
139 
140  case ISD::VP_FP_TO_SINT:
141  case ISD::VP_FP_TO_UINT:
144  case ISD::FP_TO_SINT:
145  case ISD::FP_TO_UINT: Res = PromoteIntRes_FP_TO_XINT(N); break;
146 
147  case ISD::FP_TO_SINT_SAT:
148  case ISD::FP_TO_UINT_SAT:
149  Res = PromoteIntRes_FP_TO_XINT_SAT(N); break;
150 
151  case ISD::FP_TO_FP16: Res = PromoteIntRes_FP_TO_FP16(N); break;
152 
153  case ISD::FLT_ROUNDS_: Res = PromoteIntRes_FLT_ROUNDS(N); break;
154 
155  case ISD::AND:
156  case ISD::OR:
157  case ISD::XOR:
158  case ISD::ADD:
159  case ISD::SUB:
160  case ISD::MUL:
161  case ISD::VP_AND:
162  case ISD::VP_OR:
163  case ISD::VP_XOR:
164  case ISD::VP_ADD:
165  case ISD::VP_SUB:
166  case ISD::VP_MUL: Res = PromoteIntRes_SimpleIntBinOp(N); break;
167 
168  case ISD::SDIV:
169  case ISD::SREM:
170  case ISD::VP_SDIV:
171  case ISD::VP_SREM: Res = PromoteIntRes_SExtIntBinOp(N); break;
172 
173  case ISD::UDIV:
174  case ISD::UREM:
175  case ISD::VP_UDIV:
176  case ISD::VP_UREM: Res = PromoteIntRes_ZExtIntBinOp(N); break;
177 
178  case ISD::SADDO:
179  case ISD::SSUBO: Res = PromoteIntRes_SADDSUBO(N, ResNo); break;
180  case ISD::UADDO:
181  case ISD::USUBO: Res = PromoteIntRes_UADDSUBO(N, ResNo); break;
182  case ISD::SMULO:
183  case ISD::UMULO: Res = PromoteIntRes_XMULO(N, ResNo); break;
184 
185  case ISD::ADDE:
186  case ISD::SUBE:
187  case ISD::ADDCARRY:
188  case ISD::SUBCARRY: Res = PromoteIntRes_ADDSUBCARRY(N, ResNo); break;
189 
190  case ISD::SADDO_CARRY:
191  case ISD::SSUBO_CARRY: Res = PromoteIntRes_SADDSUBO_CARRY(N, ResNo); break;
192 
193  case ISD::SADDSAT:
194  case ISD::UADDSAT:
195  case ISD::SSUBSAT:
196  case ISD::USUBSAT:
197  case ISD::SSHLSAT:
198  case ISD::USHLSAT: Res = PromoteIntRes_ADDSUBSHLSAT(N); break;
199 
200  case ISD::SMULFIX:
201  case ISD::SMULFIXSAT:
202  case ISD::UMULFIX:
203  case ISD::UMULFIXSAT: Res = PromoteIntRes_MULFIX(N); break;
204 
205  case ISD::SDIVFIX:
206  case ISD::SDIVFIXSAT:
207  case ISD::UDIVFIX:
208  case ISD::UDIVFIXSAT: Res = PromoteIntRes_DIVFIX(N); break;
209 
210  case ISD::ABS: Res = PromoteIntRes_ABS(N); break;
211 
212  case ISD::ATOMIC_LOAD:
213  Res = PromoteIntRes_Atomic0(cast<AtomicSDNode>(N)); break;
214 
219  case ISD::ATOMIC_LOAD_OR:
226  case ISD::ATOMIC_SWAP:
227  Res = PromoteIntRes_Atomic1(cast<AtomicSDNode>(N)); break;
228 
231  Res = PromoteIntRes_AtomicCmpSwap(cast<AtomicSDNode>(N), ResNo);
232  break;
233 
234  case ISD::VECREDUCE_ADD:
235  case ISD::VECREDUCE_MUL:
236  case ISD::VECREDUCE_AND:
237  case ISD::VECREDUCE_OR:
238  case ISD::VECREDUCE_XOR:
239  case ISD::VECREDUCE_SMAX:
240  case ISD::VECREDUCE_SMIN:
241  case ISD::VECREDUCE_UMAX:
242  case ISD::VECREDUCE_UMIN:
243  Res = PromoteIntRes_VECREDUCE(N);
244  break;
245 
246  case ISD::VP_REDUCE_ADD:
247  case ISD::VP_REDUCE_MUL:
248  case ISD::VP_REDUCE_AND:
249  case ISD::VP_REDUCE_OR:
250  case ISD::VP_REDUCE_XOR:
251  case ISD::VP_REDUCE_SMAX:
252  case ISD::VP_REDUCE_SMIN:
253  case ISD::VP_REDUCE_UMAX:
254  case ISD::VP_REDUCE_UMIN:
255  Res = PromoteIntRes_VP_REDUCE(N);
256  break;
257 
258  case ISD::FREEZE:
259  Res = PromoteIntRes_FREEZE(N);
260  break;
261 
262  case ISD::ROTL:
263  case ISD::ROTR:
264  Res = PromoteIntRes_Rotate(N);
265  break;
266 
267  case ISD::FSHL:
268  case ISD::FSHR:
269  Res = PromoteIntRes_FunnelShift(N);
270  break;
271 
272  case ISD::IS_FPCLASS:
273  Res = PromoteIntRes_IS_FPCLASS(N);
274  break;
275  }
276 
277  // If the result is null then the sub-method took care of registering it.
278  if (Res.getNode())
279  SetPromotedInteger(SDValue(N, ResNo), Res);
280 }
281 
282 SDValue DAGTypeLegalizer::PromoteIntRes_MERGE_VALUES(SDNode *N,
283  unsigned ResNo) {
284  SDValue Op = DisintegrateMERGE_VALUES(N, ResNo);
285  return GetPromotedInteger(Op);
286 }
287 
288 SDValue DAGTypeLegalizer::PromoteIntRes_AssertSext(SDNode *N) {
289  // Sign-extend the new bits, and continue the assertion.
290  SDValue Op = SExtPromotedInteger(N->getOperand(0));
291  return DAG.getNode(ISD::AssertSext, SDLoc(N),
292  Op.getValueType(), Op, N->getOperand(1));
293 }
294 
295 SDValue DAGTypeLegalizer::PromoteIntRes_AssertZext(SDNode *N) {
296  // Zero the new bits, and continue the assertion.
297  SDValue Op = ZExtPromotedInteger(N->getOperand(0));
298  return DAG.getNode(ISD::AssertZext, SDLoc(N),
299  Op.getValueType(), Op, N->getOperand(1));
300 }
301 
302 SDValue DAGTypeLegalizer::PromoteIntRes_Atomic0(AtomicSDNode *N) {
303  EVT ResVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
304  SDValue Res = DAG.getAtomic(N->getOpcode(), SDLoc(N),
305  N->getMemoryVT(), ResVT,
306  N->getChain(), N->getBasePtr(),
307  N->getMemOperand());
308  // Legalize the chain result - switch anything that used the old chain to
309  // use the new one.
310  ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
311  return Res;
312 }
313 
314 SDValue DAGTypeLegalizer::PromoteIntRes_Atomic1(AtomicSDNode *N) {
315  SDValue Op2 = GetPromotedInteger(N->getOperand(2));
316  SDValue Res = DAG.getAtomic(N->getOpcode(), SDLoc(N),
317  N->getMemoryVT(),
318  N->getChain(), N->getBasePtr(),
319  Op2, N->getMemOperand());
320  // Legalize the chain result - switch anything that used the old chain to
321  // use the new one.
322  ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
323  return Res;
324 }
325 
326 SDValue DAGTypeLegalizer::PromoteIntRes_AtomicCmpSwap(AtomicSDNode *N,
327  unsigned ResNo) {
328  if (ResNo == 1) {
329  assert(N->getOpcode() == ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS);
330  EVT SVT = getSetCCResultType(N->getOperand(2).getValueType());
331  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(1));
332 
333  // Only use the result of getSetCCResultType if it is legal,
334  // otherwise just use the promoted result type (NVT).
335  if (!TLI.isTypeLegal(SVT))
336  SVT = NVT;
337 
338  SDVTList VTs = DAG.getVTList(N->getValueType(0), SVT, MVT::Other);
339  SDValue Res = DAG.getAtomicCmpSwap(
340  ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS, SDLoc(N), N->getMemoryVT(), VTs,
341  N->getChain(), N->getBasePtr(), N->getOperand(2), N->getOperand(3),
342  N->getMemOperand());
343  ReplaceValueWith(SDValue(N, 0), Res.getValue(0));
344  ReplaceValueWith(SDValue(N, 2), Res.getValue(2));
345  return Res.getValue(1);
346  }
347 
348  // Op2 is used for the comparison and thus must be extended according to the
349  // target's atomic operations. Op3 is merely stored and so can be left alone.
350  SDValue Op2 = N->getOperand(2);
351  SDValue Op3 = GetPromotedInteger(N->getOperand(3));
352  switch (TLI.getExtendForAtomicCmpSwapArg()) {
353  case ISD::SIGN_EXTEND:
354  Op2 = SExtPromotedInteger(Op2);
355  break;
356  case ISD::ZERO_EXTEND:
357  Op2 = ZExtPromotedInteger(Op2);
358  break;
359  case ISD::ANY_EXTEND:
360  Op2 = GetPromotedInteger(Op2);
361  break;
362  default:
363  llvm_unreachable("Invalid atomic op extension");
364  }
365 
366  SDVTList VTs =
367  DAG.getVTList(Op2.getValueType(), N->getValueType(1), MVT::Other);
368  SDValue Res = DAG.getAtomicCmpSwap(
369  N->getOpcode(), SDLoc(N), N->getMemoryVT(), VTs, N->getChain(),
370  N->getBasePtr(), Op2, Op3, N->getMemOperand());
371  // Update the use to N with the newly created Res.
372  for (unsigned i = 1, NumResults = N->getNumValues(); i < NumResults; ++i)
373  ReplaceValueWith(SDValue(N, i), Res.getValue(i));
374  return Res;
375 }
376 
377 SDValue DAGTypeLegalizer::PromoteIntRes_BITCAST(SDNode *N) {
378  SDValue InOp = N->getOperand(0);
379  EVT InVT = InOp.getValueType();
380  EVT NInVT = TLI.getTypeToTransformTo(*DAG.getContext(), InVT);
381  EVT OutVT = N->getValueType(0);
382  EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT);
383  SDLoc dl(N);
384 
385  switch (getTypeAction(InVT)) {
387  break;
389  if (NOutVT.bitsEq(NInVT) && !NOutVT.isVector() && !NInVT.isVector())
390  // The input promotes to the same size. Convert the promoted value.
391  return DAG.getNode(ISD::BITCAST, dl, NOutVT, GetPromotedInteger(InOp));
392  break;
394  // Promote the integer operand by hand.
395  return DAG.getNode(ISD::ANY_EXTEND, dl, NOutVT, GetSoftenedFloat(InOp));
397  // Promote the integer operand by hand.
398  return DAG.getNode(ISD::ANY_EXTEND, dl, NOutVT, GetSoftPromotedHalf(InOp));
400  // Convert the promoted float by hand.
401  if (!NOutVT.isVector())
402  return DAG.getNode(ISD::FP_TO_FP16, dl, NOutVT, GetPromotedFloat(InOp));
403  break;
404  }
407  break;
409  // Convert the element to an integer and promote it by hand.
410  if (!NOutVT.isVector())
411  return DAG.getNode(ISD::ANY_EXTEND, dl, NOutVT,
412  BitConvertToInteger(GetScalarizedVector(InOp)));
413  break;
415  report_fatal_error("Scalarization of scalable vectors is not supported.");
417  if (!NOutVT.isVector()) {
418  // For example, i32 = BITCAST v2i16 on alpha. Convert the split
419  // pieces of the input into integers and reassemble in the final type.
420  SDValue Lo, Hi;
421  GetSplitVector(N->getOperand(0), Lo, Hi);
422  Lo = BitConvertToInteger(Lo);
423  Hi = BitConvertToInteger(Hi);
424 
425  if (DAG.getDataLayout().isBigEndian())
426  std::swap(Lo, Hi);
427 
428  InOp = DAG.getNode(ISD::ANY_EXTEND, dl,
430  NOutVT.getSizeInBits()),
431  JoinIntegers(Lo, Hi));
432  return DAG.getNode(ISD::BITCAST, dl, NOutVT, InOp);
433  }
434  break;
435  }
437  // The input is widened to the same size. Convert to the widened value.
438  // Make sure that the outgoing value is not a vector, because this would
439  // make us bitcast between two vectors which are legalized in different ways.
440  if (NOutVT.bitsEq(NInVT) && !NOutVT.isVector()) {
441  SDValue Res =
442  DAG.getNode(ISD::BITCAST, dl, NOutVT, GetWidenedVector(InOp));
443 
444  // For big endian targets we need to shift the casted value or the
445  // interesting bits will end up at the wrong place.
446  if (DAG.getDataLayout().isBigEndian()) {
447  unsigned ShiftAmt = NInVT.getSizeInBits() - InVT.getSizeInBits();
448  assert(ShiftAmt < NOutVT.getSizeInBits() && "Too large shift amount!");
449  Res = DAG.getNode(ISD::SRL, dl, NOutVT, Res,
450  DAG.getShiftAmountConstant(ShiftAmt, NOutVT, dl));
451  }
452  return Res;
453  }
454  // If the output type is also a vector and widening it to the same size
455  // as the widened input type would be a legal type, we can widen the bitcast
456  // and handle the promotion after.
457  if (NOutVT.isVector()) {
458  TypeSize WidenInSize = NInVT.getSizeInBits();
459  TypeSize OutSize = OutVT.getSizeInBits();
460  if (WidenInSize.hasKnownScalarFactor(OutSize)) {
461  unsigned Scale = WidenInSize.getKnownScalarFactor(OutSize);
462  EVT WideOutVT =
464  OutVT.getVectorElementCount() * Scale);
465  if (isTypeLegal(WideOutVT)) {
466  InOp = DAG.getBitcast(WideOutVT, GetWidenedVector(InOp));
467  InOp = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, OutVT, InOp,
468  DAG.getVectorIdxConstant(0, dl));
469  return DAG.getNode(ISD::ANY_EXTEND, dl, NOutVT, InOp);
470  }
471  }
472  }
473  }
474 
475  return DAG.getNode(ISD::ANY_EXTEND, dl, NOutVT,
476  CreateStackStoreLoad(InOp, OutVT));
477 }
478 
479 SDValue DAGTypeLegalizer::PromoteIntRes_FREEZE(SDNode *N) {
480  SDValue V = GetPromotedInteger(N->getOperand(0));
481  return DAG.getNode(ISD::FREEZE, SDLoc(N),
482  V.getValueType(), V);
483 }
484 
485 SDValue DAGTypeLegalizer::PromoteIntRes_BSWAP(SDNode *N) {
486  SDValue Op = GetPromotedInteger(N->getOperand(0));
487  EVT OVT = N->getValueType(0);
488  EVT NVT = Op.getValueType();
489  SDLoc dl(N);
490 
491  // If the larger BSWAP isn't supported by the target, try to expand now.
492  // If we expand later we'll end up with more operations since we lost the
493  // original type. We only do this for scalars since we have a shuffle
494  // based lowering for vectors in LegalizeVectorOps.
495  if (!OVT.isVector() &&
497  if (SDValue Res = TLI.expandBSWAP(N, DAG))
498  return DAG.getNode(ISD::ANY_EXTEND, dl, NVT, Res);
499  }
500 
501  unsigned DiffBits = NVT.getScalarSizeInBits() - OVT.getScalarSizeInBits();
502  return DAG.getNode(ISD::SRL, dl, NVT, DAG.getNode(ISD::BSWAP, dl, NVT, Op),
503  DAG.getShiftAmountConstant(DiffBits, NVT, dl));
504 }
505 
506 SDValue DAGTypeLegalizer::PromoteIntRes_BITREVERSE(SDNode *N) {
507  SDValue Op = GetPromotedInteger(N->getOperand(0));
508  EVT OVT = N->getValueType(0);
509  EVT NVT = Op.getValueType();
510  SDLoc dl(N);
511 
512  // If the larger BITREVERSE isn't supported by the target, try to expand now.
513  // If we expand later we'll end up with more operations since we lost the
514  // original type. We only do this for scalars since we have a shuffle
515  // based lowering for vectors in LegalizeVectorOps.
516  if (!OVT.isVector() && OVT.isSimple() &&
518  if (SDValue Res = TLI.expandBITREVERSE(N, DAG))
519  return DAG.getNode(ISD::ANY_EXTEND, dl, NVT, Res);
520  }
521 
522  unsigned DiffBits = NVT.getScalarSizeInBits() - OVT.getScalarSizeInBits();
523  return DAG.getNode(ISD::SRL, dl, NVT,
524  DAG.getNode(ISD::BITREVERSE, dl, NVT, Op),
525  DAG.getShiftAmountConstant(DiffBits, NVT, dl));
526 }
527 
528 SDValue DAGTypeLegalizer::PromoteIntRes_BUILD_PAIR(SDNode *N) {
529  // The pair element type may be legal, or may not promote to the same type as
530  // the result, for example i14 = BUILD_PAIR (i7, i7). Handle all cases.
531  return DAG.getNode(ISD::ANY_EXTEND, SDLoc(N),
532  TLI.getTypeToTransformTo(*DAG.getContext(),
533  N->getValueType(0)), JoinIntegers(N->getOperand(0),
534  N->getOperand(1)));
535 }
536 
537 SDValue DAGTypeLegalizer::PromoteIntRes_Constant(SDNode *N) {
538  EVT VT = N->getValueType(0);
539  // FIXME there is no actual debug info here
540  SDLoc dl(N);
541  // Zero extend things like i1, sign extend everything else. It shouldn't
542  // matter in theory which one we pick, but this tends to give better code?
543  unsigned Opc = VT.isByteSized() ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND;
544  SDValue Result = DAG.getNode(Opc, dl,
545  TLI.getTypeToTransformTo(*DAG.getContext(), VT),
546  SDValue(N, 0));
547  assert(isa<ConstantSDNode>(Result) && "Didn't constant fold ext?");
548  return Result;
549 }
550 
551 SDValue DAGTypeLegalizer::PromoteIntRes_CTLZ(SDNode *N) {
552  EVT OVT = N->getValueType(0);
553  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), OVT);
554  SDLoc dl(N);
555 
556  // If the larger CTLZ isn't supported by the target, try to expand now.
557  // If we expand later we'll end up with more operations since we lost the
558  // original type.
559  if (!OVT.isVector() && TLI.isTypeLegal(NVT) &&
562  if (SDValue Result = TLI.expandCTLZ(N, DAG)) {
563  Result = DAG.getNode(ISD::ANY_EXTEND, dl, NVT, Result);
564  return Result;
565  }
566  }
567 
568  // Zero extend to the promoted type and do the count there.
569  SDValue Op = ZExtPromotedInteger(N->getOperand(0));
570  Op = DAG.getNode(N->getOpcode(), dl, NVT, Op);
571  // Subtract off the extra leading bits in the bigger type.
572  return DAG.getNode(
573  ISD::SUB, dl, NVT, Op,
575  NVT));
576 }
577 
578 SDValue DAGTypeLegalizer::PromoteIntRes_CTPOP_PARITY(SDNode *N) {
579  EVT OVT = N->getValueType(0);
580  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), OVT);
581 
582  // If the larger CTPOP isn't supported by the target, try to expand now.
583  // If we expand later we'll end up with more operations since we lost the
584  // original type.
585  // TODO: Expand ISD::PARITY. Need to move ExpandPARITY from LegalizeDAG to
586  // TargetLowering.
587  if (N->getOpcode() == ISD::CTPOP && !OVT.isVector() && TLI.isTypeLegal(NVT) &&
589  if (SDValue Result = TLI.expandCTPOP(N, DAG)) {
590  Result = DAG.getNode(ISD::ANY_EXTEND, SDLoc(N), NVT, Result);
591  return Result;
592  }
593  }
594 
595  // Zero extend to the promoted type and do the count or parity there.
596  SDValue Op = ZExtPromotedInteger(N->getOperand(0));
597  return DAG.getNode(N->getOpcode(), SDLoc(N), Op.getValueType(), Op);
598 }
599 
600 SDValue DAGTypeLegalizer::PromoteIntRes_CTTZ(SDNode *N) {
601  SDValue Op = GetPromotedInteger(N->getOperand(0));
602  EVT OVT = N->getValueType(0);
603  EVT NVT = Op.getValueType();
604  SDLoc dl(N);
605 
606  // If the larger CTTZ isn't supported by the target, try to expand now.
607  // If we expand later we'll end up with more operations since we lost the
608  // original type. Don't expand if we can use CTPOP or CTLZ expansion on the
609  // larger type.
610  if (!OVT.isVector() && TLI.isTypeLegal(NVT) &&
613  !TLI.isOperationLegal(ISD::CTPOP, NVT) &&
614  !TLI.isOperationLegal(ISD::CTLZ, NVT)) {
615  if (SDValue Result = TLI.expandCTTZ(N, DAG)) {
616  Result = DAG.getNode(ISD::ANY_EXTEND, dl, NVT, Result);
617  return Result;
618  }
619  }
620 
621  if (N->getOpcode() == ISD::CTTZ) {
622  // The count is the same in the promoted type except if the original
623  // value was zero. This can be handled by setting the bit just off
624  // the top of the original type.
625  auto TopBit = APInt::getOneBitSet(NVT.getScalarSizeInBits(),
626  OVT.getScalarSizeInBits());
627  Op = DAG.getNode(ISD::OR, dl, NVT, Op, DAG.getConstant(TopBit, dl, NVT));
628  }
629  return DAG.getNode(N->getOpcode(), dl, NVT, Op);
630 }
631 
632 SDValue DAGTypeLegalizer::PromoteIntRes_EXTRACT_VECTOR_ELT(SDNode *N) {
633  SDLoc dl(N);
634  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
635 
636  SDValue Op0 = N->getOperand(0);
637  SDValue Op1 = N->getOperand(1);
638 
639  // If the input also needs to be promoted, do that first so we can get a
640  // get a good idea for the output type.
641  if (TLI.getTypeAction(*DAG.getContext(), Op0.getValueType())
643  SDValue In = GetPromotedInteger(Op0);
644 
645  // If the new type is larger than NVT, use it. We probably won't need to
646  // promote it again.
647  EVT SVT = In.getValueType().getScalarType();
648  if (SVT.bitsGE(NVT)) {
649  SDValue Ext = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, SVT, In, Op1);
650  return DAG.getAnyExtOrTrunc(Ext, dl, NVT);
651  }
652  }
653 
654  return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, NVT, Op0, Op1);
655 }
656 
657 SDValue DAGTypeLegalizer::PromoteIntRes_FP_TO_XINT(SDNode *N) {
658  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
659  unsigned NewOpc = N->getOpcode();
660  SDLoc dl(N);
661 
662  // If we're promoting a UINT to a larger size and the larger FP_TO_UINT is
663  // not Legal, check to see if we can use FP_TO_SINT instead. (If both UINT
664  // and SINT conversions are Custom, there is no way to tell which is
665  // preferable. We choose SINT because that's the right thing on PPC.)
666  if (N->getOpcode() == ISD::FP_TO_UINT &&
667  !TLI.isOperationLegal(ISD::FP_TO_UINT, NVT) &&
669  NewOpc = ISD::FP_TO_SINT;
670 
671  if (N->getOpcode() == ISD::STRICT_FP_TO_UINT &&
674  NewOpc = ISD::STRICT_FP_TO_SINT;
675 
676  if (N->getOpcode() == ISD::VP_FP_TO_UINT &&
677  !TLI.isOperationLegal(ISD::VP_FP_TO_UINT, NVT) &&
678  TLI.isOperationLegalOrCustom(ISD::VP_FP_TO_SINT, NVT))
679  NewOpc = ISD::VP_FP_TO_SINT;
680 
681  SDValue Res;
682  if (N->isStrictFPOpcode()) {
683  Res = DAG.getNode(NewOpc, dl, {NVT, MVT::Other},
684  {N->getOperand(0), N->getOperand(1)});
685  // Legalize the chain result - switch anything that used the old chain to
686  // use the new one.
687  ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
688  } else if (NewOpc == ISD::VP_FP_TO_SINT || NewOpc == ISD::VP_FP_TO_UINT) {
689  Res = DAG.getNode(NewOpc, dl, NVT, {N->getOperand(0), N->getOperand(1),
690  N->getOperand(2)});
691  } else {
692  Res = DAG.getNode(NewOpc, dl, NVT, N->getOperand(0));
693  }
694 
695  // Assert that the converted value fits in the original type. If it doesn't
696  // (eg: because the value being converted is too big), then the result of the
697  // original operation was undefined anyway, so the assert is still correct.
698  //
699  // NOTE: fp-to-uint to fp-to-sint promotion guarantees zero extend. For example:
700  // before legalization: fp-to-uint16, 65534. -> 0xfffe
701  // after legalization: fp-to-sint32, 65534. -> 0x0000fffe
702  return DAG.getNode((N->getOpcode() == ISD::FP_TO_UINT ||
703  N->getOpcode() == ISD::STRICT_FP_TO_UINT ||
704  N->getOpcode() == ISD::VP_FP_TO_UINT)
706  : ISD::AssertSext,
707  dl, NVT, Res,
708  DAG.getValueType(N->getValueType(0).getScalarType()));
709 }
710 
711 SDValue DAGTypeLegalizer::PromoteIntRes_FP_TO_XINT_SAT(SDNode *N) {
712  // Promote the result type, while keeping the original width in Op1.
713  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
714  SDLoc dl(N);
715  return DAG.getNode(N->getOpcode(), dl, NVT, N->getOperand(0),
716  N->getOperand(1));
717 }
718 
719 SDValue DAGTypeLegalizer::PromoteIntRes_FP_TO_FP16(SDNode *N) {
720  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
721  SDLoc dl(N);
722 
723  return DAG.getNode(N->getOpcode(), dl, NVT, N->getOperand(0));
724 }
725 
726 SDValue DAGTypeLegalizer::PromoteIntRes_FLT_ROUNDS(SDNode *N) {
727  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
728  SDLoc dl(N);
729 
730  SDValue Res =
731  DAG.getNode(N->getOpcode(), dl, {NVT, MVT::Other}, N->getOperand(0));
732 
733  // Legalize the chain result - switch anything that used the old chain to
734  // use the new one.
735  ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
736  return Res;
737 }
738 
739 SDValue DAGTypeLegalizer::PromoteIntRes_INT_EXTEND(SDNode *N) {
740  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
741  SDLoc dl(N);
742 
743  if (getTypeAction(N->getOperand(0).getValueType())
745  SDValue Res = GetPromotedInteger(N->getOperand(0));
746  assert(Res.getValueType().bitsLE(NVT) && "Extension doesn't make sense!");
747 
748  // If the result and operand types are the same after promotion, simplify
749  // to an in-register extension.
750  if (NVT == Res.getValueType()) {
751  // The high bits are not guaranteed to be anything. Insert an extend.
752  if (N->getOpcode() == ISD::SIGN_EXTEND)
753  return DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, NVT, Res,
754  DAG.getValueType(N->getOperand(0).getValueType()));
755  if (N->getOpcode() == ISD::ZERO_EXTEND)
756  return DAG.getZeroExtendInReg(Res, dl, N->getOperand(0).getValueType());
757  assert(N->getOpcode() == ISD::ANY_EXTEND && "Unknown integer extension!");
758  return Res;
759  }
760  }
761 
762  // Otherwise, just extend the original operand all the way to the larger type.
763  return DAG.getNode(N->getOpcode(), dl, NVT, N->getOperand(0));
764 }
765 
766 SDValue DAGTypeLegalizer::PromoteIntRes_LOAD(LoadSDNode *N) {
767  assert(ISD::isUNINDEXEDLoad(N) && "Indexed load during type legalization!");
768  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
769  ISD::LoadExtType ExtType =
770  ISD::isNON_EXTLoad(N) ? ISD::EXTLOAD : N->getExtensionType();
771  SDLoc dl(N);
772  SDValue Res = DAG.getExtLoad(ExtType, dl, NVT, N->getChain(), N->getBasePtr(),
773  N->getMemoryVT(), N->getMemOperand());
774 
775  // Legalize the chain result - switch anything that used the old chain to
776  // use the new one.
777  ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
778  return Res;
779 }
780 
781 SDValue DAGTypeLegalizer::PromoteIntRes_MLOAD(MaskedLoadSDNode *N) {
782  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
783  SDValue ExtPassThru = GetPromotedInteger(N->getPassThru());
784 
785  ISD::LoadExtType ExtType = N->getExtensionType();
786  if (ExtType == ISD::NON_EXTLOAD)
787  ExtType = ISD::EXTLOAD;
788 
789  SDLoc dl(N);
790  SDValue Res = DAG.getMaskedLoad(NVT, dl, N->getChain(), N->getBasePtr(),
791  N->getOffset(), N->getMask(), ExtPassThru,
792  N->getMemoryVT(), N->getMemOperand(),
793  N->getAddressingMode(), ExtType,
794  N->isExpandingLoad());
795  // Legalize the chain result - switch anything that used the old chain to
796  // use the new one.
797  ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
798  return Res;
799 }
800 
801 SDValue DAGTypeLegalizer::PromoteIntRes_MGATHER(MaskedGatherSDNode *N) {
802  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
803  SDValue ExtPassThru = GetPromotedInteger(N->getPassThru());
804  assert(NVT == ExtPassThru.getValueType() &&
805  "Gather result type and the passThru argument type should be the same");
806 
807  ISD::LoadExtType ExtType = N->getExtensionType();
808  if (ExtType == ISD::NON_EXTLOAD)
809  ExtType = ISD::EXTLOAD;
810 
811  SDLoc dl(N);
812  SDValue Ops[] = {N->getChain(), ExtPassThru, N->getMask(), N->getBasePtr(),
813  N->getIndex(), N->getScale() };
814  SDValue Res = DAG.getMaskedGather(DAG.getVTList(NVT, MVT::Other),
815  N->getMemoryVT(), dl, Ops,
816  N->getMemOperand(), N->getIndexType(),
817  ExtType);
818  // Legalize the chain result - switch anything that used the old chain to
819  // use the new one.
820  ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
821  return Res;
822 }
823 
824 /// Promote the overflow flag of an overflowing arithmetic node.
825 SDValue DAGTypeLegalizer::PromoteIntRes_Overflow(SDNode *N) {
826  // Change the return type of the boolean result while obeying
827  // getSetCCResultType.
828  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(1));
829  EVT VT = N->getValueType(0);
830  EVT SVT = getSetCCResultType(VT);
831  SDValue Ops[3] = { N->getOperand(0), N->getOperand(1) };
832  unsigned NumOps = N->getNumOperands();
833  assert(NumOps <= 3 && "Too many operands");
834  if (NumOps == 3)
835  Ops[2] = N->getOperand(2);
836 
837  SDLoc dl(N);
838  SDValue Res = DAG.getNode(N->getOpcode(), dl, DAG.getVTList(VT, SVT),
839  makeArrayRef(Ops, NumOps));
840 
841  // Modified the sum result - switch anything that used the old sum to use
842  // the new one.
843  ReplaceValueWith(SDValue(N, 0), Res);
844 
845  // Convert to the expected type.
846  return DAG.getBoolExtOrTrunc(Res.getValue(1), dl, NVT, VT);
847 }
848 
849 SDValue DAGTypeLegalizer::PromoteIntRes_ADDSUBSHLSAT(SDNode *N) {
850  // If the promoted type is legal, we can convert this to:
851  // 1. ANY_EXTEND iN to iM
852  // 2. SHL by M-N
853  // 3. [US][ADD|SUB|SHL]SAT
854  // 4. L/ASHR by M-N
855  // Else it is more efficient to convert this to a min and a max
856  // operation in the higher precision arithmetic.
857  SDLoc dl(N);
858  SDValue Op1 = N->getOperand(0);
859  SDValue Op2 = N->getOperand(1);
860  unsigned OldBits = Op1.getScalarValueSizeInBits();
861 
862  unsigned Opcode = N->getOpcode();
863  bool IsShift = Opcode == ISD::USHLSAT || Opcode == ISD::SSHLSAT;
864 
865  SDValue Op1Promoted, Op2Promoted;
866  if (IsShift) {
867  Op1Promoted = GetPromotedInteger(Op1);
868  Op2Promoted = ZExtPromotedInteger(Op2);
869  } else if (Opcode == ISD::UADDSAT || Opcode == ISD::USUBSAT) {
870  Op1Promoted = ZExtPromotedInteger(Op1);
871  Op2Promoted = ZExtPromotedInteger(Op2);
872  } else {
873  Op1Promoted = SExtPromotedInteger(Op1);
874  Op2Promoted = SExtPromotedInteger(Op2);
875  }
876  EVT PromotedType = Op1Promoted.getValueType();
877  unsigned NewBits = PromotedType.getScalarSizeInBits();
878 
879  if (Opcode == ISD::UADDSAT) {
880  APInt MaxVal = APInt::getAllOnes(OldBits).zext(NewBits);
881  SDValue SatMax = DAG.getConstant(MaxVal, dl, PromotedType);
882  SDValue Add =
883  DAG.getNode(ISD::ADD, dl, PromotedType, Op1Promoted, Op2Promoted);
884  return DAG.getNode(ISD::UMIN, dl, PromotedType, Add, SatMax);
885  }
886 
887  // USUBSAT can always be promoted as long as we have zero-extended the args.
888  if (Opcode == ISD::USUBSAT)
889  return DAG.getNode(ISD::USUBSAT, dl, PromotedType, Op1Promoted,
890  Op2Promoted);
891 
892  // Shift cannot use a min/max expansion, we can't detect overflow if all of
893  // the bits have been shifted out.
894  if (IsShift || TLI.isOperationLegal(Opcode, PromotedType)) {
895  unsigned ShiftOp;
896  switch (Opcode) {
897  case ISD::SADDSAT:
898  case ISD::SSUBSAT:
899  case ISD::SSHLSAT:
900  ShiftOp = ISD::SRA;
901  break;
902  case ISD::USHLSAT:
903  ShiftOp = ISD::SRL;
904  break;
905  default:
906  llvm_unreachable("Expected opcode to be signed or unsigned saturation "
907  "addition, subtraction or left shift");
908  }
909 
910  unsigned SHLAmount = NewBits - OldBits;
911  SDValue ShiftAmount =
912  DAG.getShiftAmountConstant(SHLAmount, PromotedType, dl);
913  Op1Promoted =
914  DAG.getNode(ISD::SHL, dl, PromotedType, Op1Promoted, ShiftAmount);
915  if (!IsShift)
916  Op2Promoted =
917  DAG.getNode(ISD::SHL, dl, PromotedType, Op2Promoted, ShiftAmount);
918 
919  SDValue Result =
920  DAG.getNode(Opcode, dl, PromotedType, Op1Promoted, Op2Promoted);
921  return DAG.getNode(ShiftOp, dl, PromotedType, Result, ShiftAmount);
922  }
923 
924  unsigned AddOp = Opcode == ISD::SADDSAT ? ISD::ADD : ISD::SUB;
925  APInt MinVal = APInt::getSignedMinValue(OldBits).sext(NewBits);
926  APInt MaxVal = APInt::getSignedMaxValue(OldBits).sext(NewBits);
927  SDValue SatMin = DAG.getConstant(MinVal, dl, PromotedType);
928  SDValue SatMax = DAG.getConstant(MaxVal, dl, PromotedType);
929  SDValue Result =
930  DAG.getNode(AddOp, dl, PromotedType, Op1Promoted, Op2Promoted);
931  Result = DAG.getNode(ISD::SMIN, dl, PromotedType, Result, SatMax);
932  Result = DAG.getNode(ISD::SMAX, dl, PromotedType, Result, SatMin);
933  return Result;
934 }
935 
936 SDValue DAGTypeLegalizer::PromoteIntRes_MULFIX(SDNode *N) {
937  // Can just promote the operands then continue with operation.
938  SDLoc dl(N);
939  SDValue Op1Promoted, Op2Promoted;
940  bool Signed =
941  N->getOpcode() == ISD::SMULFIX || N->getOpcode() == ISD::SMULFIXSAT;
942  bool Saturating =
943  N->getOpcode() == ISD::SMULFIXSAT || N->getOpcode() == ISD::UMULFIXSAT;
944  if (Signed) {
945  Op1Promoted = SExtPromotedInteger(N->getOperand(0));
946  Op2Promoted = SExtPromotedInteger(N->getOperand(1));
947  } else {
948  Op1Promoted = ZExtPromotedInteger(N->getOperand(0));
949  Op2Promoted = ZExtPromotedInteger(N->getOperand(1));
950  }
951  EVT OldType = N->getOperand(0).getValueType();
952  EVT PromotedType = Op1Promoted.getValueType();
953  unsigned DiffSize =
954  PromotedType.getScalarSizeInBits() - OldType.getScalarSizeInBits();
955 
956  if (Saturating) {
957  // Promoting the operand and result values changes the saturation width,
958  // which is extends the values that we clamp to on saturation. This could be
959  // resolved by shifting one of the operands the same amount, which would
960  // also shift the result we compare against, then shifting back.
961  Op1Promoted =
962  DAG.getNode(ISD::SHL, dl, PromotedType, Op1Promoted,
963  DAG.getShiftAmountConstant(DiffSize, PromotedType, dl));
964  SDValue Result = DAG.getNode(N->getOpcode(), dl, PromotedType, Op1Promoted,
965  Op2Promoted, N->getOperand(2));
966  unsigned ShiftOp = Signed ? ISD::SRA : ISD::SRL;
967  return DAG.getNode(ShiftOp, dl, PromotedType, Result,
968  DAG.getShiftAmountConstant(DiffSize, PromotedType, dl));
969  }
970  return DAG.getNode(N->getOpcode(), dl, PromotedType, Op1Promoted, Op2Promoted,
971  N->getOperand(2));
972 }
973 
975  unsigned SatW, bool Signed,
976  const TargetLowering &TLI,
977  SelectionDAG &DAG) {
978  EVT VT = V.getValueType();
979  unsigned VTW = VT.getScalarSizeInBits();
980 
981  if (!Signed) {
982  // Saturate to the unsigned maximum by getting the minimum of V and the
983  // maximum.
984  return DAG.getNode(ISD::UMIN, dl, VT, V,
985  DAG.getConstant(APInt::getLowBitsSet(VTW, SatW),
986  dl, VT));
987  }
988 
989  // Saturate to the signed maximum (the low SatW - 1 bits) by taking the
990  // signed minimum of it and V.
991  V = DAG.getNode(ISD::SMIN, dl, VT, V,
992  DAG.getConstant(APInt::getLowBitsSet(VTW, SatW - 1),
993  dl, VT));
994  // Saturate to the signed minimum (the high SatW + 1 bits) by taking the
995  // signed maximum of it and V.
996  V = DAG.getNode(ISD::SMAX, dl, VT, V,
997  DAG.getConstant(APInt::getHighBitsSet(VTW, VTW - SatW + 1),
998  dl, VT));
999  return V;
1000 }
1001 
1003  unsigned Scale, const TargetLowering &TLI,
1004  SelectionDAG &DAG, unsigned SatW = 0) {
1005  EVT VT = LHS.getValueType();
1006  unsigned VTSize = VT.getScalarSizeInBits();
1007  bool Signed = N->getOpcode() == ISD::SDIVFIX ||
1008  N->getOpcode() == ISD::SDIVFIXSAT;
1009  bool Saturating = N->getOpcode() == ISD::SDIVFIXSAT ||
1010  N->getOpcode() == ISD::UDIVFIXSAT;
1011 
1012  SDLoc dl(N);
1013  // Widen the types by a factor of two. This is guaranteed to expand, since it
1014  // will always have enough high bits in the LHS to shift into.
1015  EVT WideVT = EVT::getIntegerVT(*DAG.getContext(), VTSize * 2);
1016  if (VT.isVector())
1017  WideVT = EVT::getVectorVT(*DAG.getContext(), WideVT,
1018  VT.getVectorElementCount());
1019  if (Signed) {
1020  LHS = DAG.getSExtOrTrunc(LHS, dl, WideVT);
1021  RHS = DAG.getSExtOrTrunc(RHS, dl, WideVT);
1022  } else {
1023  LHS = DAG.getZExtOrTrunc(LHS, dl, WideVT);
1024  RHS = DAG.getZExtOrTrunc(RHS, dl, WideVT);
1025  }
1026 
1027  SDValue Res = TLI.expandFixedPointDiv(N->getOpcode(), dl, LHS, RHS, Scale,
1028  DAG);
1029  assert(Res && "Expanding DIVFIX with wide type failed?");
1030  if (Saturating) {
1031  // If the caller has told us to saturate at something less, use that width
1032  // instead of the type before doubling. However, it cannot be more than
1033  // what we just widened!
1034  assert(SatW <= VTSize &&
1035  "Tried to saturate to more than the original type?");
1036  Res = SaturateWidenedDIVFIX(Res, dl, SatW == 0 ? VTSize : SatW, Signed,
1037  TLI, DAG);
1038  }
1039  return DAG.getZExtOrTrunc(Res, dl, VT);
1040 }
1041 
1042 SDValue DAGTypeLegalizer::PromoteIntRes_DIVFIX(SDNode *N) {
1043  SDLoc dl(N);
1044  SDValue Op1Promoted, Op2Promoted;
1045  bool Signed = N->getOpcode() == ISD::SDIVFIX ||
1046  N->getOpcode() == ISD::SDIVFIXSAT;
1047  bool Saturating = N->getOpcode() == ISD::SDIVFIXSAT ||
1048  N->getOpcode() == ISD::UDIVFIXSAT;
1049  if (Signed) {
1050  Op1Promoted = SExtPromotedInteger(N->getOperand(0));
1051  Op2Promoted = SExtPromotedInteger(N->getOperand(1));
1052  } else {
1053  Op1Promoted = ZExtPromotedInteger(N->getOperand(0));
1054  Op2Promoted = ZExtPromotedInteger(N->getOperand(1));
1055  }
1056  EVT PromotedType = Op1Promoted.getValueType();
1057  unsigned Scale = N->getConstantOperandVal(2);
1058 
1059  // If the type is already legal and the operation is legal in that type, we
1060  // should not early expand.
1061  if (TLI.isTypeLegal(PromotedType)) {
1063  TLI.getFixedPointOperationAction(N->getOpcode(), PromotedType, Scale);
1064  if (Action == TargetLowering::Legal || Action == TargetLowering::Custom) {
1065  unsigned Diff = PromotedType.getScalarSizeInBits() -
1066  N->getValueType(0).getScalarSizeInBits();
1067  if (Saturating)
1068  Op1Promoted =
1069  DAG.getNode(ISD::SHL, dl, PromotedType, Op1Promoted,
1070  DAG.getShiftAmountConstant(Diff, PromotedType, dl));
1071  SDValue Res = DAG.getNode(N->getOpcode(), dl, PromotedType, Op1Promoted,
1072  Op2Promoted, N->getOperand(2));
1073  if (Saturating)
1074  Res = DAG.getNode(Signed ? ISD::SRA : ISD::SRL, dl, PromotedType, Res,
1075  DAG.getShiftAmountConstant(Diff, PromotedType, dl));
1076  return Res;
1077  }
1078  }
1079 
1080  // See if we can perform the division in this type without expanding.
1081  if (SDValue Res = TLI.expandFixedPointDiv(N->getOpcode(), dl, Op1Promoted,
1082  Op2Promoted, Scale, DAG)) {
1083  if (Saturating)
1084  Res = SaturateWidenedDIVFIX(Res, dl,
1085  N->getValueType(0).getScalarSizeInBits(),
1086  Signed, TLI, DAG);
1087  return Res;
1088  }
1089  // If we cannot, expand it to twice the type width. If we are saturating, give
1090  // it the original width as a saturating width so we don't need to emit
1091  // two saturations.
1092  return earlyExpandDIVFIX(N, Op1Promoted, Op2Promoted, Scale, TLI, DAG,
1093  N->getValueType(0).getScalarSizeInBits());
1094 }
1095 
1096 SDValue DAGTypeLegalizer::PromoteIntRes_SADDSUBO(SDNode *N, unsigned ResNo) {
1097  if (ResNo == 1)
1098  return PromoteIntRes_Overflow(N);
1099 
1100  // The operation overflowed iff the result in the larger type is not the
1101  // sign extension of its truncation to the original type.
1102  SDValue LHS = SExtPromotedInteger(N->getOperand(0));
1103  SDValue RHS = SExtPromotedInteger(N->getOperand(1));
1104  EVT OVT = N->getOperand(0).getValueType();
1105  EVT NVT = LHS.getValueType();
1106  SDLoc dl(N);
1107 
1108  // Do the arithmetic in the larger type.
1109  unsigned Opcode = N->getOpcode() == ISD::SADDO ? ISD::ADD : ISD::SUB;
1110  SDValue Res = DAG.getNode(Opcode, dl, NVT, LHS, RHS);
1111 
1112  // Calculate the overflow flag: sign extend the arithmetic result from
1113  // the original type.
1114  SDValue Ofl = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, NVT, Res,
1115  DAG.getValueType(OVT));
1116  // Overflowed if and only if this is not equal to Res.
1117  Ofl = DAG.getSetCC(dl, N->getValueType(1), Ofl, Res, ISD::SETNE);
1118 
1119  // Use the calculated overflow everywhere.
1120  ReplaceValueWith(SDValue(N, 1), Ofl);
1121 
1122  return Res;
1123 }
1124 
1125 SDValue DAGTypeLegalizer::PromoteIntRes_Select(SDNode *N) {
1126  SDValue Mask = N->getOperand(0);
1127 
1128  SDValue LHS = GetPromotedInteger(N->getOperand(1));
1129  SDValue RHS = GetPromotedInteger(N->getOperand(2));
1130 
1131  unsigned Opcode = N->getOpcode();
1132  if (Opcode == ISD::VP_SELECT || Opcode == ISD::VP_MERGE)
1133  return DAG.getNode(Opcode, SDLoc(N), LHS.getValueType(), Mask, LHS, RHS,
1134  N->getOperand(3));
1135  return DAG.getNode(Opcode, SDLoc(N), LHS.getValueType(), Mask, LHS, RHS);
1136 }
1137 
1138 SDValue DAGTypeLegalizer::PromoteIntRes_SELECT_CC(SDNode *N) {
1139  SDValue LHS = GetPromotedInteger(N->getOperand(2));
1140  SDValue RHS = GetPromotedInteger(N->getOperand(3));
1141  return DAG.getNode(ISD::SELECT_CC, SDLoc(N),
1142  LHS.getValueType(), N->getOperand(0),
1143  N->getOperand(1), LHS, RHS, N->getOperand(4));
1144 }
1145 
1146 SDValue DAGTypeLegalizer::PromoteIntRes_SETCC(SDNode *N) {
1147  unsigned OpNo = N->isStrictFPOpcode() ? 1 : 0;
1148  EVT InVT = N->getOperand(OpNo).getValueType();
1149  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
1150 
1151  EVT SVT = getSetCCResultType(InVT);
1152 
1153  // If we got back a type that needs to be promoted, this likely means the
1154  // the input type also needs to be promoted. So get the promoted type for
1155  // the input and try the query again.
1156  if (getTypeAction(SVT) == TargetLowering::TypePromoteInteger) {
1157  if (getTypeAction(InVT) == TargetLowering::TypePromoteInteger) {
1158  InVT = TLI.getTypeToTransformTo(*DAG.getContext(), InVT);
1159  SVT = getSetCCResultType(InVT);
1160  } else {
1161  // Input type isn't promoted, just use the default promoted type.
1162  SVT = NVT;
1163  }
1164  }
1165 
1166  SDLoc dl(N);
1167  assert(SVT.isVector() == N->getOperand(OpNo).getValueType().isVector() &&
1168  "Vector compare must return a vector result!");
1169 
1170  // Get the SETCC result using the canonical SETCC type.
1171  SDValue SetCC;
1172  if (N->isStrictFPOpcode()) {
1173  EVT VTs[] = {SVT, MVT::Other};
1174  SDValue Opers[] = {N->getOperand(0), N->getOperand(1),
1175  N->getOperand(2), N->getOperand(3)};
1176  SetCC = DAG.getNode(N->getOpcode(), dl, VTs, Opers);
1177  // Legalize the chain result - switch anything that used the old chain to
1178  // use the new one.
1179  ReplaceValueWith(SDValue(N, 1), SetCC.getValue(1));
1180  } else
1181  SetCC = DAG.getNode(N->getOpcode(), dl, SVT, N->getOperand(0),
1182  N->getOperand(1), N->getOperand(2));
1183 
1184  // Convert to the expected type.
1185  return DAG.getSExtOrTrunc(SetCC, dl, NVT);
1186 }
1187 
1188 SDValue DAGTypeLegalizer::PromoteIntRes_IS_FPCLASS(SDNode *N) {
1189  SDLoc DL(N);
1190  SDValue Arg = N->getOperand(0);
1191  SDValue Test = N->getOperand(1);
1192  EVT NResVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
1193  return DAG.getNode(ISD::IS_FPCLASS, DL, NResVT, Arg, Test);
1194 }
1195 
1196 SDValue DAGTypeLegalizer::PromoteIntRes_SHL(SDNode *N) {
1197  SDValue LHS = GetPromotedInteger(N->getOperand(0));
1198  SDValue RHS = N->getOperand(1);
1199  if (getTypeAction(RHS.getValueType()) == TargetLowering::TypePromoteInteger)
1200  RHS = ZExtPromotedInteger(RHS);
1201  if (N->getOpcode() != ISD::VP_SHL)
1202  return DAG.getNode(N->getOpcode(), SDLoc(N), LHS.getValueType(), LHS, RHS);
1203  return DAG.getNode(N->getOpcode(), SDLoc(N), LHS.getValueType(), LHS, RHS,
1204  N->getOperand(2), N->getOperand(3));
1205 }
1206 
1207 SDValue DAGTypeLegalizer::PromoteIntRes_SIGN_EXTEND_INREG(SDNode *N) {
1208  SDValue Op = GetPromotedInteger(N->getOperand(0));
1209  return DAG.getNode(ISD::SIGN_EXTEND_INREG, SDLoc(N),
1210  Op.getValueType(), Op, N->getOperand(1));
1211 }
1212 
1213 SDValue DAGTypeLegalizer::PromoteIntRes_SimpleIntBinOp(SDNode *N) {
1214  // The input may have strange things in the top bits of the registers, but
1215  // these operations don't care. They may have weird bits going out, but
1216  // that too is okay if they are integer operations.
1217  SDValue LHS = GetPromotedInteger(N->getOperand(0));
1218  SDValue RHS = GetPromotedInteger(N->getOperand(1));
1219  if (N->getNumOperands() == 2)
1220  return DAG.getNode(N->getOpcode(), SDLoc(N), LHS.getValueType(), LHS, RHS);
1221  assert(N->getNumOperands() == 4 && "Unexpected number of operands!");
1222  assert(N->isVPOpcode() && "Expected VP opcode");
1223  return DAG.getNode(N->getOpcode(), SDLoc(N), LHS.getValueType(), LHS, RHS,
1224  N->getOperand(2), N->getOperand(3));
1225 }
1226 
1227 SDValue DAGTypeLegalizer::PromoteIntRes_SExtIntBinOp(SDNode *N) {
1228  // Sign extend the input.
1229  SDValue LHS = SExtPromotedInteger(N->getOperand(0));
1230  SDValue RHS = SExtPromotedInteger(N->getOperand(1));
1231  if (N->getNumOperands() == 2)
1232  return DAG.getNode(N->getOpcode(), SDLoc(N), LHS.getValueType(), LHS, RHS);
1233  assert(N->getNumOperands() == 4 && "Unexpected number of operands!");
1234  assert(N->isVPOpcode() && "Expected VP opcode");
1235  return DAG.getNode(N->getOpcode(), SDLoc(N), LHS.getValueType(), LHS, RHS,
1236  N->getOperand(2), N->getOperand(3));
1237 }
1238 
1239 SDValue DAGTypeLegalizer::PromoteIntRes_ZExtIntBinOp(SDNode *N) {
1240  // Zero extend the input.
1241  SDValue LHS = ZExtPromotedInteger(N->getOperand(0));
1242  SDValue RHS = ZExtPromotedInteger(N->getOperand(1));
1243  if (N->getNumOperands() == 2)
1244  return DAG.getNode(N->getOpcode(), SDLoc(N), LHS.getValueType(), LHS, RHS);
1245  assert(N->getNumOperands() == 4 && "Unexpected number of operands!");
1246  assert(N->isVPOpcode() && "Expected VP opcode");
1247  return DAG.getNode(N->getOpcode(), SDLoc(N), LHS.getValueType(), LHS, RHS,
1248  N->getOperand(2), N->getOperand(3));
1249 }
1250 
1251 SDValue DAGTypeLegalizer::PromoteIntRes_UMINUMAX(SDNode *N) {
1252  // It doesn't matter if we sign extend or zero extend in the inputs. So do
1253  // whatever is best for the target.
1254  SDValue LHS = SExtOrZExtPromotedInteger(N->getOperand(0));
1255  SDValue RHS = SExtOrZExtPromotedInteger(N->getOperand(1));
1256  return DAG.getNode(N->getOpcode(), SDLoc(N),
1257  LHS.getValueType(), LHS, RHS);
1258 }
1259 
1260 SDValue DAGTypeLegalizer::PromoteIntRes_SRA(SDNode *N) {
1261  // The input value must be properly sign extended.
1262  SDValue LHS = SExtPromotedInteger(N->getOperand(0));
1263  SDValue RHS = N->getOperand(1);
1264  if (getTypeAction(RHS.getValueType()) == TargetLowering::TypePromoteInteger)
1265  RHS = ZExtPromotedInteger(RHS);
1266  if (N->getOpcode() != ISD::VP_ASHR)
1267  return DAG.getNode(N->getOpcode(), SDLoc(N), LHS.getValueType(), LHS, RHS);
1268  return DAG.getNode(N->getOpcode(), SDLoc(N), LHS.getValueType(), LHS, RHS,
1269  N->getOperand(2), N->getOperand(3));
1270 }
1271 
1272 SDValue DAGTypeLegalizer::PromoteIntRes_SRL(SDNode *N) {
1273  // The input value must be properly zero extended.
1274  SDValue LHS = ZExtPromotedInteger(N->getOperand(0));
1275  SDValue RHS = N->getOperand(1);
1276  if (getTypeAction(RHS.getValueType()) == TargetLowering::TypePromoteInteger)
1277  RHS = ZExtPromotedInteger(RHS);
1278  if (N->getOpcode() != ISD::VP_LSHR)
1279  return DAG.getNode(N->getOpcode(), SDLoc(N), LHS.getValueType(), LHS, RHS);
1280  return DAG.getNode(N->getOpcode(), SDLoc(N), LHS.getValueType(), LHS, RHS,
1281  N->getOperand(2), N->getOperand(3));
1282 }
1283 
1284 SDValue DAGTypeLegalizer::PromoteIntRes_Rotate(SDNode *N) {
1285  // Lower the rotate to shifts and ORs which can be promoted.
1286  SDValue Res = TLI.expandROT(N, true /*AllowVectorOps*/, DAG);
1287  ReplaceValueWith(SDValue(N, 0), Res);
1288  return SDValue();
1289 }
1290 
1291 SDValue DAGTypeLegalizer::PromoteIntRes_FunnelShift(SDNode *N) {
1292  SDValue Hi = GetPromotedInteger(N->getOperand(0));
1293  SDValue Lo = GetPromotedInteger(N->getOperand(1));
1294  SDValue Amt = N->getOperand(2);
1295  if (getTypeAction(Amt.getValueType()) == TargetLowering::TypePromoteInteger)
1296  Amt = ZExtPromotedInteger(Amt);
1297  EVT AmtVT = Amt.getValueType();
1298 
1299  SDLoc DL(N);
1300  EVT OldVT = N->getOperand(0).getValueType();
1301  EVT VT = Lo.getValueType();
1302  unsigned Opcode = N->getOpcode();
1303  bool IsFSHR = Opcode == ISD::FSHR;
1304  unsigned OldBits = OldVT.getScalarSizeInBits();
1305  unsigned NewBits = VT.getScalarSizeInBits();
1306 
1307  // Amount has to be interpreted modulo the old bit width.
1308  Amt = DAG.getNode(ISD::UREM, DL, AmtVT, Amt,
1309  DAG.getConstant(OldBits, DL, AmtVT));
1310 
1311  // If the promoted type is twice the size (or more), then we use the
1312  // traditional funnel 'double' shift codegen. This isn't necessary if the
1313  // shift amount is constant.
1314  // fshl(x,y,z) -> (((aext(x) << bw) | zext(y)) << (z % bw)) >> bw.
1315  // fshr(x,y,z) -> (((aext(x) << bw) | zext(y)) >> (z % bw)).
1316  if (NewBits >= (2 * OldBits) && !isa<ConstantSDNode>(Amt) &&
1317  !TLI.isOperationLegalOrCustom(Opcode, VT)) {
1318  SDValue HiShift = DAG.getConstant(OldBits, DL, VT);
1319  Hi = DAG.getNode(ISD::SHL, DL, VT, Hi, HiShift);
1320  Lo = DAG.getZeroExtendInReg(Lo, DL, OldVT);
1321  SDValue Res = DAG.getNode(ISD::OR, DL, VT, Hi, Lo);
1322  Res = DAG.getNode(IsFSHR ? ISD::SRL : ISD::SHL, DL, VT, Res, Amt);
1323  if (!IsFSHR)
1324  Res = DAG.getNode(ISD::SRL, DL, VT, Res, HiShift);
1325  return Res;
1326  }
1327 
1328  // Shift Lo up to occupy the upper bits of the promoted type.
1329  SDValue ShiftOffset = DAG.getConstant(NewBits - OldBits, DL, AmtVT);
1330  Lo = DAG.getNode(ISD::SHL, DL, VT, Lo, ShiftOffset);
1331 
1332  // Increase Amount to shift the result into the lower bits of the promoted
1333  // type.
1334  if (IsFSHR)
1335  Amt = DAG.getNode(ISD::ADD, DL, AmtVT, Amt, ShiftOffset);
1336 
1337  return DAG.getNode(Opcode, DL, VT, Hi, Lo, Amt);
1338 }
1339 
1340 SDValue DAGTypeLegalizer::PromoteIntRes_TRUNCATE(SDNode *N) {
1341  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
1342  SDValue Res;
1343  SDValue InOp = N->getOperand(0);
1344  SDLoc dl(N);
1345 
1346  switch (getTypeAction(InOp.getValueType())) {
1347  default: llvm_unreachable("Unknown type action!");
1350  Res = InOp;
1351  break;
1353  Res = GetPromotedInteger(InOp);
1354  break;
1356  EVT InVT = InOp.getValueType();
1357  assert(InVT.isVector() && "Cannot split scalar types");
1358  ElementCount NumElts = InVT.getVectorElementCount();
1359  assert(NumElts == NVT.getVectorElementCount() &&
1360  "Dst and Src must have the same number of elements");
1361  assert(isPowerOf2_32(NumElts.getKnownMinValue()) &&
1362  "Promoted vector type must be a power of two");
1363 
1364  SDValue EOp1, EOp2;
1365  GetSplitVector(InOp, EOp1, EOp2);
1366 
1367  EVT HalfNVT = EVT::getVectorVT(*DAG.getContext(), NVT.getScalarType(),
1368  NumElts.divideCoefficientBy(2));
1369  if (N->getOpcode() == ISD::TRUNCATE) {
1370  EOp1 = DAG.getNode(ISD::TRUNCATE, dl, HalfNVT, EOp1);
1371  EOp2 = DAG.getNode(ISD::TRUNCATE, dl, HalfNVT, EOp2);
1372  } else {
1373  assert(N->getOpcode() == ISD::VP_TRUNCATE &&
1374  "Expected VP_TRUNCATE opcode");
1375  SDValue MaskLo, MaskHi, EVLLo, EVLHi;
1376  std::tie(MaskLo, MaskHi) = SplitMask(N->getOperand(1));
1377  std::tie(EVLLo, EVLHi) =
1378  DAG.SplitEVL(N->getOperand(2), N->getValueType(0), dl);
1379  EOp1 = DAG.getNode(ISD::VP_TRUNCATE, dl, HalfNVT, EOp1, MaskLo, EVLLo);
1380  EOp2 = DAG.getNode(ISD::VP_TRUNCATE, dl, HalfNVT, EOp2, MaskHi, EVLHi);
1381  }
1382  return DAG.getNode(ISD::CONCAT_VECTORS, dl, NVT, EOp1, EOp2);
1383  }
1384  // TODO: VP_TRUNCATE need to handle when TypeWidenVector access to some
1385  // targets.
1387  SDValue WideInOp = GetWidenedVector(InOp);
1388 
1389  // Truncate widened InOp.
1390  unsigned NumElem = WideInOp.getValueType().getVectorNumElements();
1391  EVT TruncVT = EVT::getVectorVT(*DAG.getContext(),
1392  N->getValueType(0).getScalarType(), NumElem);
1393  SDValue WideTrunc = DAG.getNode(ISD::TRUNCATE, dl, TruncVT, WideInOp);
1394 
1395  // Zero extend so that the elements are of same type as those of NVT
1396  EVT ExtVT = EVT::getVectorVT(*DAG.getContext(), NVT.getVectorElementType(),
1397  NumElem);
1398  SDValue WideExt = DAG.getNode(ISD::ZERO_EXTEND, dl, ExtVT, WideTrunc);
1399 
1400  // Extract the low NVT subvector.
1401  SDValue ZeroIdx = DAG.getVectorIdxConstant(0, dl);
1402  return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, NVT, WideExt, ZeroIdx);
1403  }
1404  }
1405 
1406  // Truncate to NVT instead of VT
1407  if (N->getOpcode() == ISD::VP_TRUNCATE)
1408  return DAG.getNode(ISD::VP_TRUNCATE, dl, NVT, Res, N->getOperand(1),
1409  N->getOperand(2));
1410  return DAG.getNode(ISD::TRUNCATE, dl, NVT, Res);
1411 }
1412 
1413 SDValue DAGTypeLegalizer::PromoteIntRes_UADDSUBO(SDNode *N, unsigned ResNo) {
1414  if (ResNo == 1)
1415  return PromoteIntRes_Overflow(N);
1416 
1417  // The operation overflowed iff the result in the larger type is not the
1418  // zero extension of its truncation to the original type.
1419  SDValue LHS = ZExtPromotedInteger(N->getOperand(0));
1420  SDValue RHS = ZExtPromotedInteger(N->getOperand(1));
1421  EVT OVT = N->getOperand(0).getValueType();
1422  EVT NVT = LHS.getValueType();
1423  SDLoc dl(N);
1424 
1425  // Do the arithmetic in the larger type.
1426  unsigned Opcode = N->getOpcode() == ISD::UADDO ? ISD::ADD : ISD::SUB;
1427  SDValue Res = DAG.getNode(Opcode, dl, NVT, LHS, RHS);
1428 
1429  // Calculate the overflow flag: zero extend the arithmetic result from
1430  // the original type.
1431  SDValue Ofl = DAG.getZeroExtendInReg(Res, dl, OVT);
1432  // Overflowed if and only if this is not equal to Res.
1433  Ofl = DAG.getSetCC(dl, N->getValueType(1), Ofl, Res, ISD::SETNE);
1434 
1435  // Use the calculated overflow everywhere.
1436  ReplaceValueWith(SDValue(N, 1), Ofl);
1437 
1438  return Res;
1439 }
1440 
1441 // Handle promotion for the ADDE/SUBE/ADDCARRY/SUBCARRY nodes. Notice that
1442 // the third operand of ADDE/SUBE nodes is carry flag, which differs from
1443 // the ADDCARRY/SUBCARRY nodes in that the third operand is carry Boolean.
1444 SDValue DAGTypeLegalizer::PromoteIntRes_ADDSUBCARRY(SDNode *N, unsigned ResNo) {
1445  if (ResNo == 1)
1446  return PromoteIntRes_Overflow(N);
1447 
1448  // We need to sign-extend the operands so the carry value computed by the
1449  // wide operation will be equivalent to the carry value computed by the
1450  // narrow operation.
1451  // An ADDCARRY can generate carry only if any of the operands has its
1452  // most significant bit set. Sign extension propagates the most significant
1453  // bit into the higher bits which means the extra bit that the narrow
1454  // addition would need (i.e. the carry) will be propagated through the higher
1455  // bits of the wide addition.
1456  // A SUBCARRY can generate borrow only if LHS < RHS and this property will be
1457  // preserved by sign extension.
1458  SDValue LHS = SExtPromotedInteger(N->getOperand(0));
1459  SDValue RHS = SExtPromotedInteger(N->getOperand(1));
1460 
1461  EVT ValueVTs[] = {LHS.getValueType(), N->getValueType(1)};
1462 
1463  // Do the arithmetic in the wide type.
1464  SDValue Res = DAG.getNode(N->getOpcode(), SDLoc(N), DAG.getVTList(ValueVTs),
1465  LHS, RHS, N->getOperand(2));
1466 
1467  // Update the users of the original carry/borrow value.
1468  ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
1469 
1470  return SDValue(Res.getNode(), 0);
1471 }
1472 
1473 SDValue DAGTypeLegalizer::PromoteIntRes_SADDSUBO_CARRY(SDNode *N,
1474  unsigned ResNo) {
1475  assert(ResNo == 1 && "Don't know how to promote other results yet.");
1476  return PromoteIntRes_Overflow(N);
1477 }
1478 
1479 SDValue DAGTypeLegalizer::PromoteIntRes_ABS(SDNode *N) {
1480  EVT OVT = N->getValueType(0);
1481  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), OVT);
1482 
1483  // If a larger ABS or SMAX isn't supported by the target, try to expand now.
1484  // If we expand later we'll end up sign extending more than just the sra input
1485  // in sra+xor+sub expansion.
1486  if (!OVT.isVector() &&
1488  !TLI.isOperationLegal(ISD::SMAX, NVT)) {
1489  if (SDValue Res = TLI.expandABS(N, DAG))
1490  return DAG.getNode(ISD::ANY_EXTEND, SDLoc(N), NVT, Res);
1491  }
1492 
1493  SDValue Op0 = SExtPromotedInteger(N->getOperand(0));
1494  return DAG.getNode(ISD::ABS, SDLoc(N), Op0.getValueType(), Op0);
1495 }
1496 
1497 SDValue DAGTypeLegalizer::PromoteIntRes_XMULO(SDNode *N, unsigned ResNo) {
1498  // Promote the overflow bit trivially.
1499  if (ResNo == 1)
1500  return PromoteIntRes_Overflow(N);
1501 
1502  SDValue LHS = N->getOperand(0), RHS = N->getOperand(1);
1503  SDLoc DL(N);
1504  EVT SmallVT = LHS.getValueType();
1505 
1506  // To determine if the result overflowed in a larger type, we extend the
1507  // input to the larger type, do the multiply (checking if it overflows),
1508  // then also check the high bits of the result to see if overflow happened
1509  // there.
1510  if (N->getOpcode() == ISD::SMULO) {
1511  LHS = SExtPromotedInteger(LHS);
1512  RHS = SExtPromotedInteger(RHS);
1513  } else {
1514  LHS = ZExtPromotedInteger(LHS);
1515  RHS = ZExtPromotedInteger(RHS);
1516  }
1517  SDVTList VTs = DAG.getVTList(LHS.getValueType(), N->getValueType(1));
1518  SDValue Mul = DAG.getNode(N->getOpcode(), DL, VTs, LHS, RHS);
1519 
1520  // Overflow occurred if it occurred in the larger type, or if the high part
1521  // of the result does not zero/sign-extend the low part. Check this second
1522  // possibility first.
1523  SDValue Overflow;
1524  if (N->getOpcode() == ISD::UMULO) {
1525  // Unsigned overflow occurred if the high part is non-zero.
1526  unsigned Shift = SmallVT.getScalarSizeInBits();
1527  SDValue Hi =
1528  DAG.getNode(ISD::SRL, DL, Mul.getValueType(), Mul,
1529  DAG.getShiftAmountConstant(Shift, Mul.getValueType(), DL));
1530  Overflow = DAG.getSetCC(DL, N->getValueType(1), Hi,
1531  DAG.getConstant(0, DL, Hi.getValueType()),
1532  ISD::SETNE);
1533  } else {
1534  // Signed overflow occurred if the high part does not sign extend the low.
1535  SDValue SExt = DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, Mul.getValueType(),
1536  Mul, DAG.getValueType(SmallVT));
1537  Overflow = DAG.getSetCC(DL, N->getValueType(1), SExt, Mul, ISD::SETNE);
1538  }
1539 
1540  // The only other way for overflow to occur is if the multiplication in the
1541  // larger type itself overflowed.
1542  Overflow = DAG.getNode(ISD::OR, DL, N->getValueType(1), Overflow,
1543  SDValue(Mul.getNode(), 1));
1544 
1545  // Use the calculated overflow everywhere.
1546  ReplaceValueWith(SDValue(N, 1), Overflow);
1547  return Mul;
1548 }
1549 
1550 SDValue DAGTypeLegalizer::PromoteIntRes_UNDEF(SDNode *N) {
1551  return DAG.getUNDEF(TLI.getTypeToTransformTo(*DAG.getContext(),
1552  N->getValueType(0)));
1553 }
1554 
1555 SDValue DAGTypeLegalizer::PromoteIntRes_VSCALE(SDNode *N) {
1556  EVT VT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
1557 
1558  const APInt &MulImm = N->getConstantOperandAPInt(0);
1559  return DAG.getVScale(SDLoc(N), VT, MulImm.sext(VT.getSizeInBits()));
1560 }
1561 
1562 SDValue DAGTypeLegalizer::PromoteIntRes_VAARG(SDNode *N) {
1563  SDValue Chain = N->getOperand(0); // Get the chain.
1564  SDValue Ptr = N->getOperand(1); // Get the pointer.
1565  EVT VT = N->getValueType(0);
1566  SDLoc dl(N);
1567 
1568  MVT RegVT = TLI.getRegisterType(*DAG.getContext(), VT);
1569  unsigned NumRegs = TLI.getNumRegisters(*DAG.getContext(), VT);
1570  // The argument is passed as NumRegs registers of type RegVT.
1571 
1572  SmallVector<SDValue, 8> Parts(NumRegs);
1573  for (unsigned i = 0; i < NumRegs; ++i) {
1574  Parts[i] = DAG.getVAArg(RegVT, dl, Chain, Ptr, N->getOperand(2),
1575  N->getConstantOperandVal(3));
1576  Chain = Parts[i].getValue(1);
1577  }
1578 
1579  // Handle endianness of the load.
1580  if (DAG.getDataLayout().isBigEndian())
1581  std::reverse(Parts.begin(), Parts.end());
1582 
1583  // Assemble the parts in the promoted type.
1584  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
1585  SDValue Res = DAG.getNode(ISD::ZERO_EXTEND, dl, NVT, Parts[0]);
1586  for (unsigned i = 1; i < NumRegs; ++i) {
1587  SDValue Part = DAG.getNode(ISD::ZERO_EXTEND, dl, NVT, Parts[i]);
1588  // Shift it to the right position and "or" it in.
1589  Part = DAG.getNode(ISD::SHL, dl, NVT, Part,
1590  DAG.getConstant(i * RegVT.getSizeInBits(), dl,
1591  TLI.getPointerTy(DAG.getDataLayout())));
1592  Res = DAG.getNode(ISD::OR, dl, NVT, Res, Part);
1593  }
1594 
1595  // Modified the chain result - switch anything that used the old chain to
1596  // use the new one.
1597  ReplaceValueWith(SDValue(N, 1), Chain);
1598 
1599  return Res;
1600 }
1601 
1602 //===----------------------------------------------------------------------===//
1603 // Integer Operand Promotion
1604 //===----------------------------------------------------------------------===//
1605 
1606 /// PromoteIntegerOperand - This method is called when the specified operand of
1607 /// the specified node is found to need promotion. At this point, all of the
1608 /// result types of the node are known to be legal, but other operands of the
1609 /// node may need promotion or expansion as well as the specified one.
1610 bool DAGTypeLegalizer::PromoteIntegerOperand(SDNode *N, unsigned OpNo) {
1611  LLVM_DEBUG(dbgs() << "Promote integer operand: "; N->dump(&DAG);
1612  dbgs() << "\n");
1613  SDValue Res = SDValue();
1614  if (CustomLowerNode(N, N->getOperand(OpNo).getValueType(), false)) {
1615  LLVM_DEBUG(dbgs() << "Node has been custom lowered, done\n");
1616  return false;
1617  }
1618 
1619  switch (N->getOpcode()) {
1620  default:
1621  #ifndef NDEBUG
1622  dbgs() << "PromoteIntegerOperand Op #" << OpNo << ": ";
1623  N->dump(&DAG); dbgs() << "\n";
1624  #endif
1625  llvm_unreachable("Do not know how to promote this operator's operand!");
1626 
1627  case ISD::ANY_EXTEND: Res = PromoteIntOp_ANY_EXTEND(N); break;
1628  case ISD::ATOMIC_STORE:
1629  Res = PromoteIntOp_ATOMIC_STORE(cast<AtomicSDNode>(N));
1630  break;
1631  case ISD::BITCAST: Res = PromoteIntOp_BITCAST(N); break;
1632  case ISD::BR_CC: Res = PromoteIntOp_BR_CC(N, OpNo); break;
1633  case ISD::BRCOND: Res = PromoteIntOp_BRCOND(N, OpNo); break;
1634  case ISD::BUILD_PAIR: Res = PromoteIntOp_BUILD_PAIR(N); break;
1635  case ISD::BUILD_VECTOR: Res = PromoteIntOp_BUILD_VECTOR(N); break;
1636  case ISD::CONCAT_VECTORS: Res = PromoteIntOp_CONCAT_VECTORS(N); break;
1637  case ISD::EXTRACT_VECTOR_ELT: Res = PromoteIntOp_EXTRACT_VECTOR_ELT(N); break;
1639  Res = PromoteIntOp_INSERT_VECTOR_ELT(N, OpNo);
1640  break;
1641  case ISD::SPLAT_VECTOR:
1642  case ISD::SCALAR_TO_VECTOR:
1643  Res = PromoteIntOp_ScalarOp(N);
1644  break;
1645  case ISD::VSELECT:
1646  case ISD::SELECT: Res = PromoteIntOp_SELECT(N, OpNo); break;
1647  case ISD::SELECT_CC: Res = PromoteIntOp_SELECT_CC(N, OpNo); break;
1648  case ISD::VP_SETCC:
1649  case ISD::SETCC: Res = PromoteIntOp_SETCC(N, OpNo); break;
1650  case ISD::SIGN_EXTEND: Res = PromoteIntOp_SIGN_EXTEND(N); break;
1651  case ISD::VP_SINT_TO_FP:
1652  case ISD::SINT_TO_FP: Res = PromoteIntOp_SINT_TO_FP(N); break;
1653  case ISD::STRICT_SINT_TO_FP: Res = PromoteIntOp_STRICT_SINT_TO_FP(N); break;
1654  case ISD::STORE: Res = PromoteIntOp_STORE(cast<StoreSDNode>(N),
1655  OpNo); break;
1656  case ISD::MSTORE: Res = PromoteIntOp_MSTORE(cast<MaskedStoreSDNode>(N),
1657  OpNo); break;
1658  case ISD::MLOAD: Res = PromoteIntOp_MLOAD(cast<MaskedLoadSDNode>(N),
1659  OpNo); break;
1660  case ISD::MGATHER: Res = PromoteIntOp_MGATHER(cast<MaskedGatherSDNode>(N),
1661  OpNo); break;
1662  case ISD::MSCATTER: Res = PromoteIntOp_MSCATTER(cast<MaskedScatterSDNode>(N),
1663  OpNo); break;
1664  case ISD::VP_TRUNCATE:
1665  case ISD::TRUNCATE: Res = PromoteIntOp_TRUNCATE(N); break;
1666  case ISD::FP16_TO_FP:
1667  case ISD::VP_UINT_TO_FP:
1668  case ISD::UINT_TO_FP: Res = PromoteIntOp_UINT_TO_FP(N); break;
1669  case ISD::STRICT_UINT_TO_FP: Res = PromoteIntOp_STRICT_UINT_TO_FP(N); break;
1670  case ISD::ZERO_EXTEND: Res = PromoteIntOp_ZERO_EXTEND(N); break;
1671  case ISD::EXTRACT_SUBVECTOR: Res = PromoteIntOp_EXTRACT_SUBVECTOR(N); break;
1672  case ISD::INSERT_SUBVECTOR: Res = PromoteIntOp_INSERT_SUBVECTOR(N); break;
1673 
1674  case ISD::SHL:
1675  case ISD::SRA:
1676  case ISD::SRL:
1677  case ISD::ROTL:
1678  case ISD::ROTR: Res = PromoteIntOp_Shift(N); break;
1679 
1680  case ISD::FSHL:
1681  case ISD::FSHR: Res = PromoteIntOp_FunnelShift(N); break;
1682 
1683  case ISD::SADDO_CARRY:
1684  case ISD::SSUBO_CARRY:
1685  case ISD::ADDCARRY:
1686  case ISD::SUBCARRY: Res = PromoteIntOp_ADDSUBCARRY(N, OpNo); break;
1687 
1688  case ISD::FRAMEADDR:
1689  case ISD::RETURNADDR: Res = PromoteIntOp_FRAMERETURNADDR(N); break;
1690 
1691  case ISD::PREFETCH: Res = PromoteIntOp_PREFETCH(N, OpNo); break;
1692 
1693  case ISD::SMULFIX:
1694  case ISD::SMULFIXSAT:
1695  case ISD::UMULFIX:
1696  case ISD::UMULFIXSAT:
1697  case ISD::SDIVFIX:
1698  case ISD::SDIVFIXSAT:
1699  case ISD::UDIVFIX:
1700  case ISD::UDIVFIXSAT: Res = PromoteIntOp_FIX(N); break;
1701 
1702  case ISD::FPOWI:
1703  case ISD::STRICT_FPOWI: Res = PromoteIntOp_FPOWI(N); break;
1704 
1705  case ISD::VECREDUCE_ADD:
1706  case ISD::VECREDUCE_MUL:
1707  case ISD::VECREDUCE_AND:
1708  case ISD::VECREDUCE_OR:
1709  case ISD::VECREDUCE_XOR:
1710  case ISD::VECREDUCE_SMAX:
1711  case ISD::VECREDUCE_SMIN:
1712  case ISD::VECREDUCE_UMAX:
1713  case ISD::VECREDUCE_UMIN: Res = PromoteIntOp_VECREDUCE(N); break;
1714  case ISD::VP_REDUCE_ADD:
1715  case ISD::VP_REDUCE_MUL:
1716  case ISD::VP_REDUCE_AND:
1717  case ISD::VP_REDUCE_OR:
1718  case ISD::VP_REDUCE_XOR:
1719  case ISD::VP_REDUCE_SMAX:
1720  case ISD::VP_REDUCE_SMIN:
1721  case ISD::VP_REDUCE_UMAX:
1722  case ISD::VP_REDUCE_UMIN:
1723  Res = PromoteIntOp_VP_REDUCE(N, OpNo);
1724  break;
1725 
1726  case ISD::SET_ROUNDING: Res = PromoteIntOp_SET_ROUNDING(N); break;
1727  case ISD::STACKMAP:
1728  Res = PromoteIntOp_STACKMAP(N, OpNo);
1729  break;
1730  case ISD::PATCHPOINT:
1731  Res = PromoteIntOp_PATCHPOINT(N, OpNo);
1732  break;
1733  case ISD::EXPERIMENTAL_VP_STRIDED_LOAD:
1734  case ISD::EXPERIMENTAL_VP_STRIDED_STORE:
1735  Res = PromoteIntOp_VP_STRIDED(N, OpNo);
1736  break;
1737  }
1738 
1739  // If the result is null, the sub-method took care of registering results etc.
1740  if (!Res.getNode()) return false;
1741 
1742  // If the result is N, the sub-method updated N in place. Tell the legalizer
1743  // core about this.
1744  if (Res.getNode() == N)
1745  return true;
1746 
1747  const bool IsStrictFp = N->isStrictFPOpcode();
1748  assert(Res.getValueType() == N->getValueType(0) &&
1749  N->getNumValues() == (IsStrictFp ? 2 : 1) &&
1750  "Invalid operand expansion");
1751  LLVM_DEBUG(dbgs() << "Replacing: "; N->dump(&DAG); dbgs() << " with: ";
1752  Res.dump());
1753 
1754  ReplaceValueWith(SDValue(N, 0), Res);
1755  if (IsStrictFp)
1756  ReplaceValueWith(SDValue(N, 1), SDValue(Res.getNode(), 1));
1757 
1758  return false;
1759 }
1760 
1761 /// PromoteSetCCOperands - Promote the operands of a comparison. This code is
1762 /// shared among BR_CC, SELECT_CC, and SETCC handlers.
1763 void DAGTypeLegalizer::PromoteSetCCOperands(SDValue &LHS, SDValue &RHS,
1764  ISD::CondCode CCCode) {
1765  // We have to insert explicit sign or zero extends. Note that we could
1766  // insert sign extends for ALL conditions. For those operations where either
1767  // zero or sign extension would be valid, we ask the target which extension
1768  // it would prefer.
1769 
1770  // Signed comparisons always require sign extension.
1771  if (ISD::isSignedIntSetCC(CCCode)) {
1772  LHS = SExtPromotedInteger(LHS);
1773  RHS = SExtPromotedInteger(RHS);
1774  return;
1775  }
1776 
1778  "Unknown integer comparison!");
1779 
1780  SDValue OpL = GetPromotedInteger(LHS);
1781  SDValue OpR = GetPromotedInteger(RHS);
1782 
1783  if (TLI.isSExtCheaperThanZExt(LHS.getValueType(), OpL.getValueType())) {
1784  // The target would prefer to promote the comparison operand with sign
1785  // extension. Honor that unless the promoted values are already zero
1786  // extended.
1787  unsigned OpLEffectiveBits =
1789  unsigned OpREffectiveBits =
1791  if (OpLEffectiveBits <= LHS.getScalarValueSizeInBits() &&
1792  OpREffectiveBits <= RHS.getScalarValueSizeInBits()) {
1793  LHS = OpL;
1794  RHS = OpR;
1795  return;
1796  }
1797 
1798  // The promoted values aren't zero extended, use a sext_inreg.
1799  LHS = SExtPromotedInteger(LHS);
1800  RHS = SExtPromotedInteger(RHS);
1801  return;
1802  }
1803 
1804  // Prefer to promote the comparison operand with zero extension.
1805 
1806  // If the width of OpL/OpR excluding the duplicated sign bits is no greater
1807  // than the width of LHS/RHS, we can avoid/ inserting a zext_inreg operation
1808  // that we might not be able to remove.
1809  unsigned OpLEffectiveBits = DAG.ComputeMaxSignificantBits(OpL);
1810  unsigned OpREffectiveBits = DAG.ComputeMaxSignificantBits(OpR);
1811  if (OpLEffectiveBits <= LHS.getScalarValueSizeInBits() &&
1812  OpREffectiveBits <= RHS.getScalarValueSizeInBits()) {
1813  LHS = OpL;
1814  RHS = OpR;
1815  return;
1816  }
1817 
1818  // Otherwise, use zext_inreg.
1819  LHS = ZExtPromotedInteger(LHS);
1820  RHS = ZExtPromotedInteger(RHS);
1821 }
1822 
1823 SDValue DAGTypeLegalizer::PromoteIntOp_ANY_EXTEND(SDNode *N) {
1824  SDValue Op = GetPromotedInteger(N->getOperand(0));
1825  return DAG.getNode(ISD::ANY_EXTEND, SDLoc(N), N->getValueType(0), Op);
1826 }
1827 
1828 SDValue DAGTypeLegalizer::PromoteIntOp_ATOMIC_STORE(AtomicSDNode *N) {
1829  SDValue Op2 = GetPromotedInteger(N->getOperand(2));
1830  return DAG.getAtomic(N->getOpcode(), SDLoc(N), N->getMemoryVT(),
1831  N->getChain(), N->getBasePtr(), Op2, N->getMemOperand());
1832 }
1833 
1834 SDValue DAGTypeLegalizer::PromoteIntOp_BITCAST(SDNode *N) {
1835  // This should only occur in unusual situations like bitcasting to an
1836  // x86_fp80, so just turn it into a store+load
1837  return CreateStackStoreLoad(N->getOperand(0), N->getValueType(0));
1838 }
1839 
1840 SDValue DAGTypeLegalizer::PromoteIntOp_BR_CC(SDNode *N, unsigned OpNo) {
1841  assert(OpNo == 2 && "Don't know how to promote this operand!");
1842 
1843  SDValue LHS = N->getOperand(2);
1844  SDValue RHS = N->getOperand(3);
1845  PromoteSetCCOperands(LHS, RHS, cast<CondCodeSDNode>(N->getOperand(1))->get());
1846 
1847  // The chain (Op#0), CC (#1) and basic block destination (Op#4) are always
1848  // legal types.
1849  return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0),
1850  N->getOperand(1), LHS, RHS, N->getOperand(4)),
1851  0);
1852 }
1853 
1854 SDValue DAGTypeLegalizer::PromoteIntOp_BRCOND(SDNode *N, unsigned OpNo) {
1855  assert(OpNo == 1 && "only know how to promote condition");
1856 
1857  // Promote all the way up to the canonical SetCC type.
1858  SDValue Cond = PromoteTargetBoolean(N->getOperand(1), MVT::Other);
1859 
1860  // The chain (Op#0) and basic block destination (Op#2) are always legal types.
1861  return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0), Cond,
1862  N->getOperand(2)), 0);
1863 }
1864 
1865 SDValue DAGTypeLegalizer::PromoteIntOp_BUILD_PAIR(SDNode *N) {
1866  // Since the result type is legal, the operands must promote to it.
1867  EVT OVT = N->getOperand(0).getValueType();
1868  SDValue Lo = ZExtPromotedInteger(N->getOperand(0));
1869  SDValue Hi = GetPromotedInteger(N->getOperand(1));
1870  assert(Lo.getValueType() == N->getValueType(0) && "Operand over promoted?");
1871  SDLoc dl(N);
1872 
1873  Hi = DAG.getNode(ISD::SHL, dl, N->getValueType(0), Hi,
1874  DAG.getConstant(OVT.getSizeInBits(), dl,
1875  TLI.getPointerTy(DAG.getDataLayout())));
1876  return DAG.getNode(ISD::OR, dl, N->getValueType(0), Lo, Hi);
1877 }
1878 
1879 SDValue DAGTypeLegalizer::PromoteIntOp_BUILD_VECTOR(SDNode *N) {
1880  // The vector type is legal but the element type is not. This implies
1881  // that the vector is a power-of-two in length and that the element
1882  // type does not have a strange size (eg: it is not i1).
1883  EVT VecVT = N->getValueType(0);
1884  unsigned NumElts = VecVT.getVectorNumElements();
1885  assert(!((NumElts & 1) && (!TLI.isTypeLegal(VecVT))) &&
1886  "Legal vector of one illegal element?");
1887 
1888  // Promote the inserted value. The type does not need to match the
1889  // vector element type. Check that any extra bits introduced will be
1890  // truncated away.
1891  assert(N->getOperand(0).getValueSizeInBits() >=
1892  N->getValueType(0).getScalarSizeInBits() &&
1893  "Type of inserted value narrower than vector element type!");
1894 
1895  SmallVector<SDValue, 16> NewOps;
1896  for (unsigned i = 0; i < NumElts; ++i)
1897  NewOps.push_back(GetPromotedInteger(N->getOperand(i)));
1898 
1899  return SDValue(DAG.UpdateNodeOperands(N, NewOps), 0);
1900 }
1901 
1902 SDValue DAGTypeLegalizer::PromoteIntOp_INSERT_VECTOR_ELT(SDNode *N,
1903  unsigned OpNo) {
1904  if (OpNo == 1) {
1905  // Promote the inserted value. This is valid because the type does not
1906  // have to match the vector element type.
1907 
1908  // Check that any extra bits introduced will be truncated away.
1909  assert(N->getOperand(1).getValueSizeInBits() >=
1910  N->getValueType(0).getScalarSizeInBits() &&
1911  "Type of inserted value narrower than vector element type!");
1912  return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0),
1913  GetPromotedInteger(N->getOperand(1)),
1914  N->getOperand(2)),
1915  0);
1916  }
1917 
1918  assert(OpNo == 2 && "Different operand and result vector types?");
1919 
1920  // Promote the index.
1921  SDValue Idx = DAG.getZExtOrTrunc(N->getOperand(2), SDLoc(N),
1922  TLI.getVectorIdxTy(DAG.getDataLayout()));
1923  return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0),
1924  N->getOperand(1), Idx), 0);
1925 }
1926 
1927 SDValue DAGTypeLegalizer::PromoteIntOp_ScalarOp(SDNode *N) {
1928  // Integer SPLAT_VECTOR/SCALAR_TO_VECTOR operands are implicitly truncated,
1929  // so just promote the operand in place.
1930  return SDValue(DAG.UpdateNodeOperands(N,
1931  GetPromotedInteger(N->getOperand(0))), 0);
1932 }
1933 
1934 SDValue DAGTypeLegalizer::PromoteIntOp_SELECT(SDNode *N, unsigned OpNo) {
1935  assert(OpNo == 0 && "Only know how to promote the condition!");
1936  SDValue Cond = N->getOperand(0);
1937  EVT OpTy = N->getOperand(1).getValueType();
1938 
1939  if (N->getOpcode() == ISD::VSELECT)
1940  if (SDValue Res = WidenVSELECTMask(N))
1941  return DAG.getNode(N->getOpcode(), SDLoc(N), N->getValueType(0),
1942  Res, N->getOperand(1), N->getOperand(2));
1943 
1944  // Promote all the way up to the canonical SetCC type.
1945  EVT OpVT = N->getOpcode() == ISD::SELECT ? OpTy.getScalarType() : OpTy;
1946  Cond = PromoteTargetBoolean(Cond, OpVT);
1947 
1948  return SDValue(DAG.UpdateNodeOperands(N, Cond, N->getOperand(1),
1949  N->getOperand(2)), 0);
1950 }
1951 
1952 SDValue DAGTypeLegalizer::PromoteIntOp_SELECT_CC(SDNode *N, unsigned OpNo) {
1953  assert(OpNo == 0 && "Don't know how to promote this operand!");
1954 
1955  SDValue LHS = N->getOperand(0);
1956  SDValue RHS = N->getOperand(1);
1957  PromoteSetCCOperands(LHS, RHS, cast<CondCodeSDNode>(N->getOperand(4))->get());
1958 
1959  // The CC (#4) and the possible return values (#2 and #3) have legal types.
1960  return SDValue(DAG.UpdateNodeOperands(N, LHS, RHS, N->getOperand(2),
1961  N->getOperand(3), N->getOperand(4)), 0);
1962 }
1963 
1964 SDValue DAGTypeLegalizer::PromoteIntOp_SETCC(SDNode *N, unsigned OpNo) {
1965  assert(OpNo == 0 && "Don't know how to promote this operand!");
1966 
1967  SDValue LHS = N->getOperand(0);
1968  SDValue RHS = N->getOperand(1);
1969  PromoteSetCCOperands(LHS, RHS, cast<CondCodeSDNode>(N->getOperand(2))->get());
1970 
1971  // The CC (#2) is always legal.
1972  if (N->getOpcode() == ISD::SETCC)
1973  return SDValue(DAG.UpdateNodeOperands(N, LHS, RHS, N->getOperand(2)), 0);
1974 
1975  assert(N->getOpcode() == ISD::VP_SETCC && "Expected VP_SETCC opcode");
1976 
1977  return SDValue(DAG.UpdateNodeOperands(N, LHS, RHS, N->getOperand(2),
1978  N->getOperand(3), N->getOperand(4)),
1979  0);
1980 }
1981 
1982 SDValue DAGTypeLegalizer::PromoteIntOp_Shift(SDNode *N) {
1983  return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0),
1984  ZExtPromotedInteger(N->getOperand(1))), 0);
1985 }
1986 
1987 SDValue DAGTypeLegalizer::PromoteIntOp_FunnelShift(SDNode *N) {
1988  return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0), N->getOperand(1),
1989  ZExtPromotedInteger(N->getOperand(2))), 0);
1990 }
1991 
1992 SDValue DAGTypeLegalizer::PromoteIntOp_SIGN_EXTEND(SDNode *N) {
1993  SDValue Op = GetPromotedInteger(N->getOperand(0));
1994  SDLoc dl(N);
1995  Op = DAG.getNode(ISD::ANY_EXTEND, dl, N->getValueType(0), Op);
1996  return DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, Op.getValueType(),
1997  Op, DAG.getValueType(N->getOperand(0).getValueType()));
1998 }
1999 
2000 SDValue DAGTypeLegalizer::PromoteIntOp_SINT_TO_FP(SDNode *N) {
2001  if (N->getOpcode() == ISD::VP_SINT_TO_FP)
2002  return SDValue(DAG.UpdateNodeOperands(N,
2003  SExtPromotedInteger(N->getOperand(0)),
2004  N->getOperand(1), N->getOperand(2)),
2005  0);
2006  return SDValue(DAG.UpdateNodeOperands(N,
2007  SExtPromotedInteger(N->getOperand(0))), 0);
2008 }
2009 
2010 SDValue DAGTypeLegalizer::PromoteIntOp_STRICT_SINT_TO_FP(SDNode *N) {
2011  return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0),
2012  SExtPromotedInteger(N->getOperand(1))), 0);
2013 }
2014 
2015 SDValue DAGTypeLegalizer::PromoteIntOp_STORE(StoreSDNode *N, unsigned OpNo){
2016  assert(ISD::isUNINDEXEDStore(N) && "Indexed store during type legalization!");
2017  SDValue Ch = N->getChain(), Ptr = N->getBasePtr();
2018  SDLoc dl(N);
2019 
2020  SDValue Val = GetPromotedInteger(N->getValue()); // Get promoted value.
2021 
2022  // Truncate the value and store the result.
2023  return DAG.getTruncStore(Ch, dl, Val, Ptr,
2024  N->getMemoryVT(), N->getMemOperand());
2025 }
2026 
2027 SDValue DAGTypeLegalizer::PromoteIntOp_MSTORE(MaskedStoreSDNode *N,
2028  unsigned OpNo) {
2029  SDValue DataOp = N->getValue();
2030  SDValue Mask = N->getMask();
2031 
2032  if (OpNo == 4) {
2033  // The Mask. Update in place.
2034  EVT DataVT = DataOp.getValueType();
2035  Mask = PromoteTargetBoolean(Mask, DataVT);
2036  SmallVector<SDValue, 4> NewOps(N->op_begin(), N->op_end());
2037  NewOps[4] = Mask;
2038  return SDValue(DAG.UpdateNodeOperands(N, NewOps), 0);
2039  }
2040 
2041  assert(OpNo == 1 && "Unexpected operand for promotion");
2042  DataOp = GetPromotedInteger(DataOp);
2043 
2044  return DAG.getMaskedStore(N->getChain(), SDLoc(N), DataOp, N->getBasePtr(),
2045  N->getOffset(), Mask, N->getMemoryVT(),
2046  N->getMemOperand(), N->getAddressingMode(),
2047  /*IsTruncating*/ true, N->isCompressingStore());
2048 }
2049 
2050 SDValue DAGTypeLegalizer::PromoteIntOp_MLOAD(MaskedLoadSDNode *N,
2051  unsigned OpNo) {
2052  assert(OpNo == 3 && "Only know how to promote the mask!");
2053  EVT DataVT = N->getValueType(0);
2054  SDValue Mask = PromoteTargetBoolean(N->getOperand(OpNo), DataVT);
2055  SmallVector<SDValue, 4> NewOps(N->op_begin(), N->op_end());
2056  NewOps[OpNo] = Mask;
2057  SDNode *Res = DAG.UpdateNodeOperands(N, NewOps);
2058  if (Res == N)
2059  return SDValue(Res, 0);
2060 
2061  // Update triggered CSE, do our own replacement since caller can't.
2062  ReplaceValueWith(SDValue(N, 0), SDValue(Res, 0));
2063  ReplaceValueWith(SDValue(N, 1), SDValue(Res, 1));
2064  return SDValue();
2065 }
2066 
2067 SDValue DAGTypeLegalizer::PromoteIntOp_MGATHER(MaskedGatherSDNode *N,
2068  unsigned OpNo) {
2069  SmallVector<SDValue, 5> NewOps(N->op_begin(), N->op_end());
2070 
2071  if (OpNo == 2) {
2072  // The Mask
2073  EVT DataVT = N->getValueType(0);
2074  NewOps[OpNo] = PromoteTargetBoolean(N->getOperand(OpNo), DataVT);
2075  } else if (OpNo == 4) {
2076  // The Index
2077  if (N->isIndexSigned())
2078  // Need to sign extend the index since the bits will likely be used.
2079  NewOps[OpNo] = SExtPromotedInteger(N->getOperand(OpNo));
2080  else
2081  NewOps[OpNo] = ZExtPromotedInteger(N->getOperand(OpNo));
2082  } else
2083  NewOps[OpNo] = GetPromotedInteger(N->getOperand(OpNo));
2084 
2085  SDNode *Res = DAG.UpdateNodeOperands(N, NewOps);
2086  if (Res == N)
2087  return SDValue(Res, 0);
2088 
2089  // Update triggered CSE, do our own replacement since caller can't.
2090  ReplaceValueWith(SDValue(N, 0), SDValue(Res, 0));
2091  ReplaceValueWith(SDValue(N, 1), SDValue(Res, 1));
2092  return SDValue();
2093 }
2094 
2095 SDValue DAGTypeLegalizer::PromoteIntOp_MSCATTER(MaskedScatterSDNode *N,
2096  unsigned OpNo) {
2097  bool TruncateStore = N->isTruncatingStore();
2098  SmallVector<SDValue, 5> NewOps(N->op_begin(), N->op_end());
2099 
2100  if (OpNo == 2) {
2101  // The Mask
2102  EVT DataVT = N->getValue().getValueType();
2103  NewOps[OpNo] = PromoteTargetBoolean(N->getOperand(OpNo), DataVT);
2104  } else if (OpNo == 4) {
2105  // The Index
2106  if (N->isIndexSigned())
2107  // Need to sign extend the index since the bits will likely be used.
2108  NewOps[OpNo] = SExtPromotedInteger(N->getOperand(OpNo));
2109  else
2110  NewOps[OpNo] = ZExtPromotedInteger(N->getOperand(OpNo));
2111  } else {
2112  NewOps[OpNo] = GetPromotedInteger(N->getOperand(OpNo));
2113  TruncateStore = true;
2114  }
2115 
2116  return DAG.getMaskedScatter(DAG.getVTList(MVT::Other), N->getMemoryVT(),
2117  SDLoc(N), NewOps, N->getMemOperand(),
2118  N->getIndexType(), TruncateStore);
2119 }
2120 
2121 SDValue DAGTypeLegalizer::PromoteIntOp_TRUNCATE(SDNode *N) {
2122  SDValue Op = GetPromotedInteger(N->getOperand(0));
2123  if (N->getOpcode() == ISD::VP_TRUNCATE)
2124  return DAG.getNode(ISD::VP_TRUNCATE, SDLoc(N), N->getValueType(0), Op,
2125  N->getOperand(1), N->getOperand(2));
2126  return DAG.getNode(ISD::TRUNCATE, SDLoc(N), N->getValueType(0), Op);
2127 }
2128 
2129 SDValue DAGTypeLegalizer::PromoteIntOp_UINT_TO_FP(SDNode *N) {
2130  if (N->getOpcode() == ISD::VP_UINT_TO_FP)
2131  return SDValue(DAG.UpdateNodeOperands(N,
2132  ZExtPromotedInteger(N->getOperand(0)),
2133  N->getOperand(1), N->getOperand(2)),
2134  0);
2135  return SDValue(DAG.UpdateNodeOperands(N,
2136  ZExtPromotedInteger(N->getOperand(0))), 0);
2137 }
2138 
2139 SDValue DAGTypeLegalizer::PromoteIntOp_STRICT_UINT_TO_FP(SDNode *N) {
2140  return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0),
2141  ZExtPromotedInteger(N->getOperand(1))), 0);
2142 }
2143 
2144 SDValue DAGTypeLegalizer::PromoteIntOp_ZERO_EXTEND(SDNode *N) {
2145  SDLoc dl(N);
2146  SDValue Op = GetPromotedInteger(N->getOperand(0));
2147  Op = DAG.getNode(ISD::ANY_EXTEND, dl, N->getValueType(0), Op);
2148  return DAG.getZeroExtendInReg(Op, dl, N->getOperand(0).getValueType());
2149 }
2150 
2151 SDValue DAGTypeLegalizer::PromoteIntOp_ADDSUBCARRY(SDNode *N, unsigned OpNo) {
2152  assert(OpNo == 2 && "Don't know how to promote this operand!");
2153 
2154  SDValue LHS = N->getOperand(0);
2155  SDValue RHS = N->getOperand(1);
2156  SDValue Carry = N->getOperand(2);
2157  SDLoc DL(N);
2158 
2159  Carry = PromoteTargetBoolean(Carry, LHS.getValueType());
2160 
2161  return SDValue(DAG.UpdateNodeOperands(N, LHS, RHS, Carry), 0);
2162 }
2163 
2164 SDValue DAGTypeLegalizer::PromoteIntOp_FIX(SDNode *N) {
2165  SDValue Op2 = ZExtPromotedInteger(N->getOperand(2));
2166  return SDValue(
2167  DAG.UpdateNodeOperands(N, N->getOperand(0), N->getOperand(1), Op2), 0);
2168 }
2169 
2170 SDValue DAGTypeLegalizer::PromoteIntOp_FRAMERETURNADDR(SDNode *N) {
2171  // Promote the RETURNADDR/FRAMEADDR argument to a supported integer width.
2172  SDValue Op = ZExtPromotedInteger(N->getOperand(0));
2173  return SDValue(DAG.UpdateNodeOperands(N, Op), 0);
2174 }
2175 
2176 SDValue DAGTypeLegalizer::PromoteIntOp_PREFETCH(SDNode *N, unsigned OpNo) {
2177  assert(OpNo > 1 && "Don't know how to promote this operand!");
2178  // Promote the rw, locality, and cache type arguments to a supported integer
2179  // width.
2180  SDValue Op2 = ZExtPromotedInteger(N->getOperand(2));
2181  SDValue Op3 = ZExtPromotedInteger(N->getOperand(3));
2182  SDValue Op4 = ZExtPromotedInteger(N->getOperand(4));
2183  return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0), N->getOperand(1),
2184  Op2, Op3, Op4),
2185  0);
2186 }
2187 
2188 SDValue DAGTypeLegalizer::PromoteIntOp_FPOWI(SDNode *N) {
2189  bool IsStrict = N->isStrictFPOpcode();
2190  SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
2191 
2192  // The integer operand is the last operand in FPOWI (so the result and
2193  // floating point operand is already type legalized).
2194 
2195  // We can't just promote the exponent type in FPOWI, since we want to lower
2196  // the node to a libcall and we if we promote to a type larger than
2197  // sizeof(int) the libcall might not be according to the targets ABI. Instead
2198  // we rewrite to a libcall here directly, letting makeLibCall handle promotion
2199  // if the target accepts it according to shouldSignExtendTypeInLibCall.
2200  RTLIB::Libcall LC = RTLIB::getPOWI(N->getValueType(0));
2201  assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected fpowi.");
2202  if (!TLI.getLibcallName(LC)) {
2203  // Some targets don't have a powi libcall; use pow instead.
2204  // FIXME: Implement this if some target needs it.
2205  DAG.getContext()->emitError("Don't know how to promote fpowi to fpow");
2206  return DAG.getUNDEF(N->getValueType(0));
2207  }
2208  unsigned OpOffset = IsStrict ? 1 : 0;
2209  // The exponent should fit in a sizeof(int) type for the libcall to be valid.
2210  assert(DAG.getLibInfo().getIntSize() ==
2211  N->getOperand(1 + OpOffset).getValueType().getSizeInBits() &&
2212  "POWI exponent should match with sizeof(int) when doing the libcall.");
2214  CallOptions.setSExt(true);
2215  SDValue Ops[2] = {N->getOperand(0 + OpOffset), N->getOperand(1 + OpOffset)};
2216  std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(
2217  DAG, LC, N->getValueType(0), Ops, CallOptions, SDLoc(N), Chain);
2218  ReplaceValueWith(SDValue(N, 0), Tmp.first);
2219  if (IsStrict)
2220  ReplaceValueWith(SDValue(N, 1), Tmp.second);
2221  return SDValue();
2222 }
2223 
2225  switch (N->getOpcode()) {
2226  default:
2227  llvm_unreachable("Expected integer vector reduction");
2228  case ISD::VECREDUCE_ADD:
2229  case ISD::VECREDUCE_MUL:
2230  case ISD::VECREDUCE_AND:
2231  case ISD::VECREDUCE_OR:
2232  case ISD::VECREDUCE_XOR:
2233  case ISD::VP_REDUCE_ADD:
2234  case ISD::VP_REDUCE_MUL:
2235  case ISD::VP_REDUCE_AND:
2236  case ISD::VP_REDUCE_OR:
2237  case ISD::VP_REDUCE_XOR:
2238  return ISD::ANY_EXTEND;
2239  case ISD::VECREDUCE_SMAX:
2240  case ISD::VECREDUCE_SMIN:
2241  case ISD::VP_REDUCE_SMAX:
2242  case ISD::VP_REDUCE_SMIN:
2243  return ISD::SIGN_EXTEND;
2244  case ISD::VECREDUCE_UMAX:
2245  case ISD::VECREDUCE_UMIN:
2246  case ISD::VP_REDUCE_UMAX:
2247  case ISD::VP_REDUCE_UMIN:
2248  return ISD::ZERO_EXTEND;
2249  }
2250 }
2251 
2252 SDValue DAGTypeLegalizer::PromoteIntOpVectorReduction(SDNode *N, SDValue V) {
2253  switch (getExtendForIntVecReduction(N)) {
2254  default:
2255  llvm_unreachable("Impossible extension kind for integer reduction");
2256  case ISD::ANY_EXTEND:
2257  return GetPromotedInteger(V);
2258  case ISD::SIGN_EXTEND:
2259  return SExtPromotedInteger(V);
2260  case ISD::ZERO_EXTEND:
2261  return ZExtPromotedInteger(V);
2262  }
2263 }
2264 
2265 SDValue DAGTypeLegalizer::PromoteIntOp_VECREDUCE(SDNode *N) {
2266  SDLoc dl(N);
2267  SDValue Op = PromoteIntOpVectorReduction(N, N->getOperand(0));
2268 
2269  EVT OrigEltVT = N->getOperand(0).getValueType().getVectorElementType();
2270  EVT InVT = Op.getValueType();
2271  EVT EltVT = InVT.getVectorElementType();
2272  EVT ResVT = N->getValueType(0);
2273  unsigned Opcode = N->getOpcode();
2274 
2275  // An i1 vecreduce_xor is equivalent to vecreduce_add, use that instead if
2276  // vecreduce_xor is not legal
2277  if (Opcode == ISD::VECREDUCE_XOR && OrigEltVT == MVT::i1 &&
2280  Opcode = ISD::VECREDUCE_ADD;
2281 
2282  // An i1 vecreduce_or is equivalent to vecreduce_umax, use that instead if
2283  // vecreduce_or is not legal
2284  else if (Opcode == ISD::VECREDUCE_OR && OrigEltVT == MVT::i1 &&
2287  Opcode = ISD::VECREDUCE_UMAX;
2288 
2289  // An i1 vecreduce_and is equivalent to vecreduce_umin, use that instead if
2290  // vecreduce_and is not legal
2291  else if (Opcode == ISD::VECREDUCE_AND && OrigEltVT == MVT::i1 &&
2294  Opcode = ISD::VECREDUCE_UMIN;
2295 
2296  if (ResVT.bitsGE(EltVT))
2297  return DAG.getNode(Opcode, SDLoc(N), ResVT, Op);
2298 
2299  // Result size must be >= element size. If this is not the case after
2300  // promotion, also promote the result type and then truncate.
2301  SDValue Reduce = DAG.getNode(Opcode, dl, EltVT, Op);
2302  return DAG.getNode(ISD::TRUNCATE, dl, ResVT, Reduce);
2303 }
2304 
2305 SDValue DAGTypeLegalizer::PromoteIntOp_VP_REDUCE(SDNode *N, unsigned OpNo) {
2306  SDLoc DL(N);
2307  SDValue Op = N->getOperand(OpNo);
2308  SmallVector<SDValue, 4> NewOps(N->op_begin(), N->op_end());
2309 
2310  if (OpNo == 2) { // Mask
2311  // Update in place.
2312  NewOps[2] = PromoteTargetBoolean(Op, N->getOperand(1).getValueType());
2313  return SDValue(DAG.UpdateNodeOperands(N, NewOps), 0);
2314  }
2315 
2316  assert(OpNo == 1 && "Unexpected operand for promotion");
2317 
2318  Op = PromoteIntOpVectorReduction(N, Op);
2319 
2320  NewOps[OpNo] = Op;
2321 
2322  EVT VT = N->getValueType(0);
2323  EVT EltVT = Op.getValueType().getScalarType();
2324 
2325  if (VT.bitsGE(EltVT))
2326  return DAG.getNode(N->getOpcode(), SDLoc(N), VT, NewOps);
2327 
2328  // Result size must be >= element/start-value size. If this is not the case
2329  // after promotion, also promote both the start value and result type and
2330  // then truncate.
2331  NewOps[0] =
2332  DAG.getNode(getExtendForIntVecReduction(N), DL, EltVT, N->getOperand(0));
2333  SDValue Reduce = DAG.getNode(N->getOpcode(), DL, EltVT, NewOps);
2334  return DAG.getNode(ISD::TRUNCATE, DL, VT, Reduce);
2335 }
2336 
2337 SDValue DAGTypeLegalizer::PromoteIntOp_SET_ROUNDING(SDNode *N) {
2338  SDValue Op = ZExtPromotedInteger(N->getOperand(1));
2339  return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0), Op), 0);
2340 }
2341 
2342 SDValue DAGTypeLegalizer::PromoteIntOp_STACKMAP(SDNode *N, unsigned OpNo) {
2343  assert(OpNo > 1); // Because the first two arguments are guaranteed legal.
2344  SmallVector<SDValue> NewOps(N->ops().begin(), N->ops().end());
2345  SDValue Operand = N->getOperand(OpNo);
2346  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), Operand.getValueType());
2347  NewOps[OpNo] = DAG.getNode(ISD::ANY_EXTEND, SDLoc(N), NVT, Operand);
2348  return SDValue(DAG.UpdateNodeOperands(N, NewOps), 0);
2349 }
2350 
2351 SDValue DAGTypeLegalizer::PromoteIntOp_PATCHPOINT(SDNode *N, unsigned OpNo) {
2352  assert(OpNo >= 7);
2353  SmallVector<SDValue> NewOps(N->ops().begin(), N->ops().end());
2354  SDValue Operand = N->getOperand(OpNo);
2355  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), Operand.getValueType());
2356  NewOps[OpNo] = DAG.getNode(ISD::ANY_EXTEND, SDLoc(N), NVT, Operand);
2357  return SDValue(DAG.UpdateNodeOperands(N, NewOps), 0);
2358 }
2359 
2360 SDValue DAGTypeLegalizer::PromoteIntOp_VP_STRIDED(SDNode *N, unsigned OpNo) {
2361  assert((N->getOpcode() == ISD::EXPERIMENTAL_VP_STRIDED_LOAD && OpNo == 3) ||
2362  (N->getOpcode() == ISD::EXPERIMENTAL_VP_STRIDED_STORE && OpNo == 4));
2363 
2364  SmallVector<SDValue, 8> NewOps(N->op_begin(), N->op_end());
2365  NewOps[OpNo] = SExtPromotedInteger(N->getOperand(OpNo));
2366 
2367  return SDValue(DAG.UpdateNodeOperands(N, NewOps), 0);
2368 }
2369 
2370 //===----------------------------------------------------------------------===//
2371 // Integer Result Expansion
2372 //===----------------------------------------------------------------------===//
2373 
2374 /// ExpandIntegerResult - This method is called when the specified result of the
2375 /// specified node is found to need expansion. At this point, the node may also
2376 /// have invalid operands or may have other results that need promotion, we just
2377 /// know that (at least) one result needs expansion.
2378 void DAGTypeLegalizer::ExpandIntegerResult(SDNode *N, unsigned ResNo) {
2379  LLVM_DEBUG(dbgs() << "Expand integer result: "; N->dump(&DAG);
2380  dbgs() << "\n");
2381  SDValue Lo, Hi;
2382  Lo = Hi = SDValue();
2383 
2384  // See if the target wants to custom expand this node.
2385  if (CustomLowerNode(N, N->getValueType(ResNo), true))
2386  return;
2387 
2388  switch (N->getOpcode()) {
2389  default:
2390 #ifndef NDEBUG
2391  dbgs() << "ExpandIntegerResult #" << ResNo << ": ";
2392  N->dump(&DAG); dbgs() << "\n";
2393 #endif
2394  report_fatal_error("Do not know how to expand the result of this "
2395  "operator!");
2396 
2397  case ISD::ARITH_FENCE: SplitRes_ARITH_FENCE(N, Lo, Hi); break;
2398  case ISD::MERGE_VALUES: SplitRes_MERGE_VALUES(N, ResNo, Lo, Hi); break;
2399  case ISD::SELECT: SplitRes_Select(N, Lo, Hi); break;
2400  case ISD::SELECT_CC: SplitRes_SELECT_CC(N, Lo, Hi); break;
2401  case ISD::UNDEF: SplitRes_UNDEF(N, Lo, Hi); break;
2402  case ISD::FREEZE: SplitRes_FREEZE(N, Lo, Hi); break;
2403 
2404  case ISD::BITCAST: ExpandRes_BITCAST(N, Lo, Hi); break;
2405  case ISD::BUILD_PAIR: ExpandRes_BUILD_PAIR(N, Lo, Hi); break;
2406  case ISD::EXTRACT_ELEMENT: ExpandRes_EXTRACT_ELEMENT(N, Lo, Hi); break;
2407  case ISD::EXTRACT_VECTOR_ELT: ExpandRes_EXTRACT_VECTOR_ELT(N, Lo, Hi); break;
2408  case ISD::VAARG: ExpandRes_VAARG(N, Lo, Hi); break;
2409 
2410  case ISD::ANY_EXTEND: ExpandIntRes_ANY_EXTEND(N, Lo, Hi); break;
2411  case ISD::AssertSext: ExpandIntRes_AssertSext(N, Lo, Hi); break;
2412  case ISD::AssertZext: ExpandIntRes_AssertZext(N, Lo, Hi); break;
2413  case ISD::BITREVERSE: ExpandIntRes_BITREVERSE(N, Lo, Hi); break;
2414  case ISD::BSWAP: ExpandIntRes_BSWAP(N, Lo, Hi); break;
2415  case ISD::PARITY: ExpandIntRes_PARITY(N, Lo, Hi); break;
2416  case ISD::Constant: ExpandIntRes_Constant(N, Lo, Hi); break;
2417  case ISD::ABS: ExpandIntRes_ABS(N, Lo, Hi); break;
2418  case ISD::CTLZ_ZERO_UNDEF:
2419  case ISD::CTLZ: ExpandIntRes_CTLZ(N, Lo, Hi); break;
2420  case ISD::CTPOP: ExpandIntRes_CTPOP(N, Lo, Hi); break;
2421  case ISD::CTTZ_ZERO_UNDEF:
2422  case ISD::CTTZ: ExpandIntRes_CTTZ(N, Lo, Hi); break;
2423  case ISD::FLT_ROUNDS_: ExpandIntRes_FLT_ROUNDS(N, Lo, Hi); break;
2425  case ISD::FP_TO_SINT: ExpandIntRes_FP_TO_SINT(N, Lo, Hi); break;
2427  case ISD::FP_TO_UINT: ExpandIntRes_FP_TO_UINT(N, Lo, Hi); break;
2428  case ISD::FP_TO_SINT_SAT:
2429  case ISD::FP_TO_UINT_SAT: ExpandIntRes_FP_TO_XINT_SAT(N, Lo, Hi); break;
2430  case ISD::STRICT_LLROUND:
2431  case ISD::STRICT_LLRINT:
2432  case ISD::LLROUND:
2433  case ISD::LLRINT: ExpandIntRes_LLROUND_LLRINT(N, Lo, Hi); break;
2434  case ISD::LOAD: ExpandIntRes_LOAD(cast<LoadSDNode>(N), Lo, Hi); break;
2435  case ISD::MUL: ExpandIntRes_MUL(N, Lo, Hi); break;
2436  case ISD::READCYCLECOUNTER: ExpandIntRes_READCYCLECOUNTER(N, Lo, Hi); break;
2437  case ISD::SDIV: ExpandIntRes_SDIV(N, Lo, Hi); break;
2438  case ISD::SIGN_EXTEND: ExpandIntRes_SIGN_EXTEND(N, Lo, Hi); break;
2439  case ISD::SIGN_EXTEND_INREG: ExpandIntRes_SIGN_EXTEND_INREG(N, Lo, Hi); break;
2440  case ISD::SREM: ExpandIntRes_SREM(N, Lo, Hi); break;
2441  case ISD::TRUNCATE: ExpandIntRes_TRUNCATE(N, Lo, Hi); break;
2442  case ISD::UDIV: ExpandIntRes_UDIV(N, Lo, Hi); break;
2443  case ISD::UREM: ExpandIntRes_UREM(N, Lo, Hi); break;
2444  case ISD::ZERO_EXTEND: ExpandIntRes_ZERO_EXTEND(N, Lo, Hi); break;
2445  case ISD::ATOMIC_LOAD: ExpandIntRes_ATOMIC_LOAD(N, Lo, Hi); break;
2446 
2447  case ISD::ATOMIC_LOAD_ADD:
2448  case ISD::ATOMIC_LOAD_SUB:
2449  case ISD::ATOMIC_LOAD_AND:
2450  case ISD::ATOMIC_LOAD_CLR:
2451  case ISD::ATOMIC_LOAD_OR:
2452  case ISD::ATOMIC_LOAD_XOR:
2453  case ISD::ATOMIC_LOAD_NAND:
2454  case ISD::ATOMIC_LOAD_MIN:
2455  case ISD::ATOMIC_LOAD_MAX:
2456  case ISD::ATOMIC_LOAD_UMIN:
2457  case ISD::ATOMIC_LOAD_UMAX:
2458  case ISD::ATOMIC_SWAP:
2459  case ISD::ATOMIC_CMP_SWAP: {
2460  std::pair<SDValue, SDValue> Tmp = ExpandAtomic(N);
2461  SplitInteger(Tmp.first, Lo, Hi);
2462  ReplaceValueWith(SDValue(N, 1), Tmp.second);
2463  break;
2464  }
2466  AtomicSDNode *AN = cast<AtomicSDNode>(N);
2467  SDVTList VTs = DAG.getVTList(N->getValueType(0), MVT::Other);
2468  SDValue Tmp = DAG.getAtomicCmpSwap(
2469  ISD::ATOMIC_CMP_SWAP, SDLoc(N), AN->getMemoryVT(), VTs,
2470  N->getOperand(0), N->getOperand(1), N->getOperand(2), N->getOperand(3),
2471  AN->getMemOperand());
2472 
2473  // Expanding to the strong ATOMIC_CMP_SWAP node means we can determine
2474  // success simply by comparing the loaded value against the ingoing
2475  // comparison.
2476  SDValue Success = DAG.getSetCC(SDLoc(N), N->getValueType(1), Tmp,
2477  N->getOperand(2), ISD::SETEQ);
2478 
2479  SplitInteger(Tmp, Lo, Hi);
2480  ReplaceValueWith(SDValue(N, 1), Success);
2481  ReplaceValueWith(SDValue(N, 2), Tmp.getValue(1));
2482  break;
2483  }
2484 
2485  case ISD::AND:
2486  case ISD::OR:
2487  case ISD::XOR: ExpandIntRes_Logical(N, Lo, Hi); break;
2488 
2489  case ISD::UMAX:
2490  case ISD::SMAX:
2491  case ISD::UMIN:
2492  case ISD::SMIN: ExpandIntRes_MINMAX(N, Lo, Hi); break;
2493 
2494  case ISD::ADD:
2495  case ISD::SUB: ExpandIntRes_ADDSUB(N, Lo, Hi); break;
2496 
2497  case ISD::ADDC:
2498  case ISD::SUBC: ExpandIntRes_ADDSUBC(N, Lo, Hi); break;
2499 
2500  case ISD::ADDE:
2501  case ISD::SUBE: ExpandIntRes_ADDSUBE(N, Lo, Hi); break;
2502 
2503  case ISD::ADDCARRY:
2504  case ISD::SUBCARRY: ExpandIntRes_ADDSUBCARRY(N, Lo, Hi); break;
2505 
2506  case ISD::SADDO_CARRY:
2507  case ISD::SSUBO_CARRY: ExpandIntRes_SADDSUBO_CARRY(N, Lo, Hi); break;
2508 
2509  case ISD::SHL:
2510  case ISD::SRA:
2511  case ISD::SRL: ExpandIntRes_Shift(N, Lo, Hi); break;
2512 
2513  case ISD::SADDO:
2514  case ISD::SSUBO: ExpandIntRes_SADDSUBO(N, Lo, Hi); break;
2515  case ISD::UADDO:
2516  case ISD::USUBO: ExpandIntRes_UADDSUBO(N, Lo, Hi); break;
2517  case ISD::UMULO:
2518  case ISD::SMULO: ExpandIntRes_XMULO(N, Lo, Hi); break;
2519 
2520  case ISD::SADDSAT:
2521  case ISD::UADDSAT:
2522  case ISD::SSUBSAT:
2523  case ISD::USUBSAT: ExpandIntRes_ADDSUBSAT(N, Lo, Hi); break;
2524 
2525  case ISD::SSHLSAT:
2526  case ISD::USHLSAT: ExpandIntRes_SHLSAT(N, Lo, Hi); break;
2527 
2528  case ISD::SMULFIX:
2529  case ISD::SMULFIXSAT:
2530  case ISD::UMULFIX:
2531  case ISD::UMULFIXSAT: ExpandIntRes_MULFIX(N, Lo, Hi); break;
2532 
2533  case ISD::SDIVFIX:
2534  case ISD::SDIVFIXSAT:
2535  case ISD::UDIVFIX:
2536  case ISD::UDIVFIXSAT: ExpandIntRes_DIVFIX(N, Lo, Hi); break;
2537 
2538  case ISD::VECREDUCE_ADD:
2539  case ISD::VECREDUCE_MUL:
2540  case ISD::VECREDUCE_AND:
2541  case ISD::VECREDUCE_OR:
2542  case ISD::VECREDUCE_XOR:
2543  case ISD::VECREDUCE_SMAX:
2544  case ISD::VECREDUCE_SMIN:
2545  case ISD::VECREDUCE_UMAX:
2546  case ISD::VECREDUCE_UMIN: ExpandIntRes_VECREDUCE(N, Lo, Hi); break;
2547 
2548  case ISD::ROTL:
2549  case ISD::ROTR:
2550  ExpandIntRes_Rotate(N, Lo, Hi);
2551  break;
2552 
2553  case ISD::FSHL:
2554  case ISD::FSHR:
2555  ExpandIntRes_FunnelShift(N, Lo, Hi);
2556  break;
2557 
2558  case ISD::VSCALE:
2559  ExpandIntRes_VSCALE(N, Lo, Hi);
2560  break;
2561  }
2562 
2563  // If Lo/Hi is null, the sub-method took care of registering results etc.
2564  if (Lo.getNode())
2565  SetExpandedInteger(SDValue(N, ResNo), Lo, Hi);
2566 }
2567 
2568 /// Lower an atomic node to the appropriate builtin call.
2569 std::pair <SDValue, SDValue> DAGTypeLegalizer::ExpandAtomic(SDNode *Node) {
2570  unsigned Opc = Node->getOpcode();
2571  MVT VT = cast<AtomicSDNode>(Node)->getMemoryVT().getSimpleVT();
2572  AtomicOrdering order = cast<AtomicSDNode>(Node)->getMergedOrdering();
2573  // Lower to outline atomic libcall if outline atomics enabled,
2574  // or to sync libcall otherwise
2575  RTLIB::Libcall LC = RTLIB::getOUTLINE_ATOMIC(Opc, order, VT);
2576  EVT RetVT = Node->getValueType(0);
2579  if (TLI.getLibcallName(LC)) {
2580  Ops.append(Node->op_begin() + 2, Node->op_end());
2581  Ops.push_back(Node->getOperand(1));
2582  } else {
2583  LC = RTLIB::getSYNC(Opc, VT);
2584  assert(LC != RTLIB::UNKNOWN_LIBCALL &&
2585  "Unexpected atomic op or value type!");
2586  Ops.append(Node->op_begin() + 1, Node->op_end());
2587  }
2588  return TLI.makeLibCall(DAG, LC, RetVT, Ops, CallOptions, SDLoc(Node),
2589  Node->getOperand(0));
2590 }
2591 
2592 /// N is a shift by a value that needs to be expanded,
2593 /// and the shift amount is a constant 'Amt'. Expand the operation.
2594 void DAGTypeLegalizer::ExpandShiftByConstant(SDNode *N, const APInt &Amt,
2595  SDValue &Lo, SDValue &Hi) {
2596  SDLoc DL(N);
2597  // Expand the incoming operand to be shifted, so that we have its parts
2598  SDValue InL, InH;
2599  GetExpandedInteger(N->getOperand(0), InL, InH);
2600 
2601  // Though Amt shouldn't usually be 0, it's possible. E.g. when legalization
2602  // splitted a vector shift, like this: <op1, op2> SHL <0, 2>.
2603  if (!Amt) {
2604  Lo = InL;
2605  Hi = InH;
2606  return;
2607  }
2608 
2609  EVT NVT = InL.getValueType();
2610  unsigned VTBits = N->getValueType(0).getSizeInBits();
2611  unsigned NVTBits = NVT.getSizeInBits();
2612  EVT ShTy = N->getOperand(1).getValueType();
2613 
2614  if (N->getOpcode() == ISD::SHL) {
2615  if (Amt.uge(VTBits)) {
2616  Lo = Hi = DAG.getConstant(0, DL, NVT);
2617  } else if (Amt.ugt(NVTBits)) {
2618  Lo = DAG.getConstant(0, DL, NVT);
2619  Hi = DAG.getNode(ISD::SHL, DL,
2620  NVT, InL, DAG.getConstant(Amt - NVTBits, DL, ShTy));
2621  } else if (Amt == NVTBits) {
2622  Lo = DAG.getConstant(0, DL, NVT);
2623  Hi = InL;
2624  } else {
2625  Lo = DAG.getNode(ISD::SHL, DL, NVT, InL, DAG.getConstant(Amt, DL, ShTy));
2626  Hi = DAG.getNode(ISD::OR, DL, NVT,
2627  DAG.getNode(ISD::SHL, DL, NVT, InH,
2628  DAG.getConstant(Amt, DL, ShTy)),
2629  DAG.getNode(ISD::SRL, DL, NVT, InL,
2630  DAG.getConstant(-Amt + NVTBits, DL, ShTy)));
2631  }
2632  return;
2633  }
2634 
2635  if (N->getOpcode() == ISD::SRL) {
2636  if (Amt.uge(VTBits)) {
2637  Lo = Hi = DAG.getConstant(0, DL, NVT);
2638  } else if (Amt.ugt(NVTBits)) {
2639  Lo = DAG.getNode(ISD::SRL, DL,
2640  NVT, InH, DAG.getConstant(Amt - NVTBits, DL, ShTy));
2641  Hi = DAG.getConstant(0, DL, NVT);
2642  } else if (Amt == NVTBits) {
2643  Lo = InH;
2644  Hi = DAG.getConstant(0, DL, NVT);
2645  } else {
2646  Lo = DAG.getNode(ISD::OR, DL, NVT,
2647  DAG.getNode(ISD::SRL, DL, NVT, InL,
2648  DAG.getConstant(Amt, DL, ShTy)),
2649  DAG.getNode(ISD::SHL, DL, NVT, InH,
2650  DAG.getConstant(-Amt + NVTBits, DL, ShTy)));
2651  Hi = DAG.getNode(ISD::SRL, DL, NVT, InH, DAG.getConstant(Amt, DL, ShTy));
2652  }
2653  return;
2654  }
2655 
2656  assert(N->getOpcode() == ISD::SRA && "Unknown shift!");
2657  if (Amt.uge(VTBits)) {
2658  Hi = Lo = DAG.getNode(ISD::SRA, DL, NVT, InH,
2659  DAG.getConstant(NVTBits - 1, DL, ShTy));
2660  } else if (Amt.ugt(NVTBits)) {
2661  Lo = DAG.getNode(ISD::SRA, DL, NVT, InH,
2662  DAG.getConstant(Amt - NVTBits, DL, ShTy));
2663  Hi = DAG.getNode(ISD::SRA, DL, NVT, InH,
2664  DAG.getConstant(NVTBits - 1, DL, ShTy));
2665  } else if (Amt == NVTBits) {
2666  Lo = InH;
2667  Hi = DAG.getNode(ISD::SRA, DL, NVT, InH,
2668  DAG.getConstant(NVTBits - 1, DL, ShTy));
2669  } else {
2670  Lo = DAG.getNode(ISD::OR, DL, NVT,
2671  DAG.getNode(ISD::SRL, DL, NVT, InL,
2672  DAG.getConstant(Amt, DL, ShTy)),
2673  DAG.getNode(ISD::SHL, DL, NVT, InH,
2674  DAG.getConstant(-Amt + NVTBits, DL, ShTy)));
2675  Hi = DAG.getNode(ISD::SRA, DL, NVT, InH, DAG.getConstant(Amt, DL, ShTy));
2676  }
2677 }
2678 
2679 /// ExpandShiftWithKnownAmountBit - Try to determine whether we can simplify
2680 /// this shift based on knowledge of the high bit of the shift amount. If we
2681 /// can tell this, we know that it is >= 32 or < 32, without knowing the actual
2682 /// shift amount.
2683 bool DAGTypeLegalizer::
2684 ExpandShiftWithKnownAmountBit(SDNode *N, SDValue &Lo, SDValue &Hi) {
2685  SDValue Amt = N->getOperand(1);
2686  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
2687  EVT ShTy = Amt.getValueType();
2688  unsigned ShBits = ShTy.getScalarSizeInBits();
2689  unsigned NVTBits = NVT.getScalarSizeInBits();
2690  assert(isPowerOf2_32(NVTBits) &&
2691  "Expanded integer type size not a power of two!");
2692  SDLoc dl(N);
2693 
2694  APInt HighBitMask = APInt::getHighBitsSet(ShBits, ShBits - Log2_32(NVTBits));
2695  KnownBits Known = DAG.computeKnownBits(N->getOperand(1));
2696 
2697  // If we don't know anything about the high bits, exit.
2698  if (((Known.Zero|Known.One) & HighBitMask) == 0)
2699  return false;
2700 
2701  // Get the incoming operand to be shifted.
2702  SDValue InL, InH;
2703  GetExpandedInteger(N->getOperand(0), InL, InH);
2704 
2705  // If we know that any of the high bits of the shift amount are one, then we
2706  // can do this as a couple of simple shifts.
2707  if (Known.One.intersects(HighBitMask)) {
2708  // Mask out the high bit, which we know is set.
2709  Amt = DAG.getNode(ISD::AND, dl, ShTy, Amt,
2710  DAG.getConstant(~HighBitMask, dl, ShTy));
2711 
2712  switch (N->getOpcode()) {
2713  default: llvm_unreachable("Unknown shift");
2714  case ISD::SHL:
2715  Lo = DAG.getConstant(0, dl, NVT); // Low part is zero.
2716  Hi = DAG.getNode(ISD::SHL, dl, NVT, InL, Amt); // High part from Lo part.
2717  return true;
2718  case ISD::SRL:
2719  Hi = DAG.getConstant(0, dl, NVT); // Hi part is zero.
2720  Lo = DAG.getNode(ISD::SRL, dl, NVT, InH, Amt); // Lo part from Hi part.
2721  return true;
2722  case ISD::SRA:
2723  Hi = DAG.getNode(ISD::SRA, dl, NVT, InH, // Sign extend high part.
2724  DAG.getConstant(NVTBits - 1, dl, ShTy));
2725  Lo = DAG.getNode(ISD::SRA, dl, NVT, InH, Amt); // Lo part from Hi part.
2726  return true;
2727  }
2728  }
2729 
2730  // If we know that all of the high bits of the shift amount are zero, then we
2731  // can do this as a couple of simple shifts.
2732  if (HighBitMask.isSubsetOf(Known.Zero)) {
2733  // Calculate 31-x. 31 is used instead of 32 to avoid creating an undefined
2734  // shift if x is zero. We can use XOR here because x is known to be smaller
2735  // than 32.
2736  SDValue Amt2 = DAG.getNode(ISD::XOR, dl, ShTy, Amt,
2737  DAG.getConstant(NVTBits - 1, dl, ShTy));
2738 
2739  unsigned Op1, Op2;
2740  switch (N->getOpcode()) {
2741  default: llvm_unreachable("Unknown shift");
2742  case ISD::SHL: Op1 = ISD::SHL; Op2 = ISD::SRL; break;
2743  case ISD::SRL:
2744  case ISD::SRA: Op1 = ISD::SRL; Op2 = ISD::SHL; break;
2745  }
2746 
2747  // When shifting right the arithmetic for Lo and Hi is swapped.
2748  if (N->getOpcode() != ISD::SHL)
2749  std::swap(InL, InH);
2750 
2751  // Use a little trick to get the bits that move from Lo to Hi. First
2752  // shift by one bit.
2753  SDValue Sh1 = DAG.getNode(Op2, dl, NVT, InL, DAG.getConstant(1, dl, ShTy));
2754  // Then compute the remaining shift with amount-1.
2755  SDValue Sh2 = DAG.getNode(Op2, dl, NVT, Sh1, Amt2);
2756 
2757  Lo = DAG.getNode(N->getOpcode(), dl, NVT, InL, Amt);
2758  Hi = DAG.getNode(ISD::OR, dl, NVT, DAG.getNode(Op1, dl, NVT, InH, Amt),Sh2);
2759 
2760  if (N->getOpcode() != ISD::SHL)
2761  std::swap(Hi, Lo);
2762  return true;
2763  }
2764 
2765  return false;
2766 }
2767 
2768 /// ExpandShiftWithUnknownAmountBit - Fully general expansion of integer shift
2769 /// of any size.
2770 bool DAGTypeLegalizer::
2771 ExpandShiftWithUnknownAmountBit(SDNode *N, SDValue &Lo, SDValue &Hi) {
2772  SDValue Amt = N->getOperand(1);
2773  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
2774  EVT ShTy = Amt.getValueType();
2775  unsigned NVTBits = NVT.getSizeInBits();
2776  assert(isPowerOf2_32(NVTBits) &&
2777  "Expanded integer type size not a power of two!");
2778  SDLoc dl(N);
2779 
2780  // Get the incoming operand to be shifted.
2781  SDValue InL, InH;
2782  GetExpandedInteger(N->getOperand(0), InL, InH);
2783 
2784  SDValue NVBitsNode = DAG.getConstant(NVTBits, dl, ShTy);
2785  SDValue AmtExcess = DAG.getNode(ISD::SUB, dl, ShTy, Amt, NVBitsNode);
2786  SDValue AmtLack = DAG.getNode(ISD::SUB, dl, ShTy, NVBitsNode, Amt);
2787  SDValue isShort = DAG.getSetCC(dl, getSetCCResultType(ShTy),
2788  Amt, NVBitsNode, ISD::SETULT);
2789  SDValue isZero = DAG.getSetCC(dl, getSetCCResultType(ShTy),
2790  Amt, DAG.getConstant(0, dl, ShTy),
2791  ISD::SETEQ);
2792 
2793  SDValue LoS, HiS, LoL, HiL;
2794  switch (N->getOpcode()) {
2795  default: llvm_unreachable("Unknown shift");
2796  case ISD::SHL:
2797  // Short: ShAmt < NVTBits
2798  LoS = DAG.getNode(ISD::SHL, dl, NVT, InL, Amt);
2799  HiS = DAG.getNode(ISD::OR, dl, NVT,
2800  DAG.getNode(ISD::SHL, dl, NVT, InH, Amt),
2801  DAG.getNode(ISD::SRL, dl, NVT, InL, AmtLack));
2802 
2803  // Long: ShAmt >= NVTBits
2804  LoL = DAG.getConstant(0, dl, NVT); // Lo part is zero.
2805  HiL = DAG.getNode(ISD::SHL, dl, NVT, InL, AmtExcess); // Hi from Lo part.
2806 
2807  Lo = DAG.getSelect(dl, NVT, isShort, LoS, LoL);
2808  Hi = DAG.getSelect(dl, NVT, isZero, InH,
2809  DAG.getSelect(dl, NVT, isShort, HiS, HiL));
2810  return true;
2811  case ISD::SRL:
2812  // Short: ShAmt < NVTBits
2813  HiS = DAG.getNode(ISD::SRL, dl, NVT, InH, Amt);
2814  LoS = DAG.getNode(ISD::OR, dl, NVT,
2815  DAG.getNode(ISD::SRL, dl, NVT, InL, Amt),
2816  // FIXME: If Amt is zero, the following shift generates an undefined result
2817  // on some architectures.
2818  DAG.getNode(ISD::SHL, dl, NVT, InH, AmtLack));
2819 
2820  // Long: ShAmt >= NVTBits
2821  HiL = DAG.getConstant(0, dl, NVT); // Hi part is zero.
2822  LoL = DAG.getNode(ISD::SRL, dl, NVT, InH, AmtExcess); // Lo from Hi part.
2823 
2824  Lo = DAG.getSelect(dl, NVT, isZero, InL,
2825  DAG.getSelect(dl, NVT, isShort, LoS, LoL));
2826  Hi = DAG.getSelect(dl, NVT, isShort, HiS, HiL);
2827  return true;
2828  case ISD::SRA:
2829  // Short: ShAmt < NVTBits
2830  HiS = DAG.getNode(ISD::SRA, dl, NVT, InH, Amt);
2831  LoS = DAG.getNode(ISD::OR, dl, NVT,
2832  DAG.getNode(ISD::SRL, dl, NVT, InL, Amt),
2833  DAG.getNode(ISD::SHL, dl, NVT, InH, AmtLack));
2834 
2835  // Long: ShAmt >= NVTBits
2836  HiL = DAG.getNode(ISD::SRA, dl, NVT, InH, // Sign of Hi part.
2837  DAG.getConstant(NVTBits - 1, dl, ShTy));
2838  LoL = DAG.getNode(ISD::SRA, dl, NVT, InH, AmtExcess); // Lo from Hi part.
2839 
2840  Lo = DAG.getSelect(dl, NVT, isZero, InL,
2841  DAG.getSelect(dl, NVT, isShort, LoS, LoL));
2842  Hi = DAG.getSelect(dl, NVT, isShort, HiS, HiL);
2843  return true;
2844  }
2845 }
2846 
2847 static std::pair<ISD::CondCode, ISD::NodeType> getExpandedMinMaxOps(int Op) {
2848 
2849  switch (Op) {
2850  default: llvm_unreachable("invalid min/max opcode");
2851  case ISD::SMAX:
2852  return std::make_pair(ISD::SETGT, ISD::UMAX);
2853  case ISD::UMAX:
2854  return std::make_pair(ISD::SETUGT, ISD::UMAX);
2855  case ISD::SMIN:
2856  return std::make_pair(ISD::SETLT, ISD::UMIN);
2857  case ISD::UMIN:
2858  return std::make_pair(ISD::SETULT, ISD::UMIN);
2859  }
2860 }
2861 
2862 void DAGTypeLegalizer::ExpandIntRes_MINMAX(SDNode *N,
2863  SDValue &Lo, SDValue &Hi) {
2864  SDLoc DL(N);
2865  ISD::NodeType LoOpc;
2866  ISD::CondCode CondC;
2867  std::tie(CondC, LoOpc) = getExpandedMinMaxOps(N->getOpcode());
2868 
2869  // Expand the subcomponents.
2870  SDValue LHSL, LHSH, RHSL, RHSH;
2871  GetExpandedInteger(N->getOperand(0), LHSL, LHSH);
2872  GetExpandedInteger(N->getOperand(1), RHSL, RHSH);
2873 
2874  // Value types
2875  EVT NVT = LHSL.getValueType();
2876  EVT CCT = getSetCCResultType(NVT);
2877 
2878  // Hi part is always the same op
2879  Hi = DAG.getNode(N->getOpcode(), DL, NVT, {LHSH, RHSH});
2880 
2881  // We need to know whether to select Lo part that corresponds to 'winning'
2882  // Hi part or if Hi parts are equal.
2883  SDValue IsHiLeft = DAG.getSetCC(DL, CCT, LHSH, RHSH, CondC);
2884  SDValue IsHiEq = DAG.getSetCC(DL, CCT, LHSH, RHSH, ISD::SETEQ);
2885 
2886  // Lo part corresponding to the 'winning' Hi part
2887  SDValue LoCmp = DAG.getSelect(DL, NVT, IsHiLeft, LHSL, RHSL);
2888 
2889  // Recursed Lo part if Hi parts are equal, this uses unsigned version
2890  SDValue LoMinMax = DAG.getNode(LoOpc, DL, NVT, {LHSL, RHSL});
2891 
2892  Lo = DAG.getSelect(DL, NVT, IsHiEq, LoMinMax, LoCmp);
2893 }
2894 
2895 void DAGTypeLegalizer::ExpandIntRes_ADDSUB(SDNode *N,
2896  SDValue &Lo, SDValue &Hi) {
2897  SDLoc dl(N);
2898  // Expand the subcomponents.
2899  SDValue LHSL, LHSH, RHSL, RHSH;
2900  GetExpandedInteger(N->getOperand(0), LHSL, LHSH);
2901  GetExpandedInteger(N->getOperand(1), RHSL, RHSH);
2902 
2903  EVT NVT = LHSL.getValueType();
2904  SDValue LoOps[2] = { LHSL, RHSL };
2905  SDValue HiOps[3] = { LHSH, RHSH };
2906 
2907  bool HasOpCarry = TLI.isOperationLegalOrCustom(
2908  N->getOpcode() == ISD::ADD ? ISD::ADDCARRY : ISD::SUBCARRY,
2909  TLI.getTypeToExpandTo(*DAG.getContext(), NVT));
2910  if (HasOpCarry) {
2911  SDVTList VTList = DAG.getVTList(NVT, getSetCCResultType(NVT));
2912  if (N->getOpcode() == ISD::ADD) {
2913  Lo = DAG.getNode(ISD::UADDO, dl, VTList, LoOps);
2914  HiOps[2] = Lo.getValue(1);
2915  Hi = DAG.computeKnownBits(HiOps[2]).isZero()
2916  ? DAG.getNode(ISD::UADDO, dl, VTList, makeArrayRef(HiOps, 2))
2917  : DAG.getNode(ISD::ADDCARRY, dl, VTList, HiOps);
2918  } else {
2919  Lo = DAG.getNode(ISD::USUBO, dl, VTList, LoOps);
2920  HiOps[2] = Lo.getValue(1);
2921  Hi = DAG.computeKnownBits(HiOps[2]).isZero()
2922  ? DAG.getNode(ISD::USUBO, dl, VTList, makeArrayRef(HiOps, 2))
2923  : DAG.getNode(ISD::SUBCARRY, dl, VTList, HiOps);
2924  }
2925  return;
2926  }
2927 
2928  // Do not generate ADDC/ADDE or SUBC/SUBE if the target does not support
2929  // them. TODO: Teach operation legalization how to expand unsupported
2930  // ADDC/ADDE/SUBC/SUBE. The problem is that these operations generate
2931  // a carry of type MVT::Glue, but there doesn't seem to be any way to
2932  // generate a value of this type in the expanded code sequence.
2933  bool hasCarry =
2934  TLI.isOperationLegalOrCustom(N->getOpcode() == ISD::ADD ?
2935  ISD::ADDC : ISD::SUBC,
2936  TLI.getTypeToExpandTo(*DAG.getContext(), NVT));
2937 
2938  if (hasCarry) {
2939  SDVTList VTList = DAG.getVTList(NVT, MVT::Glue);
2940  if (N->getOpcode() == ISD::ADD) {
2941  Lo = DAG.getNode(ISD::ADDC, dl, VTList, LoOps);
2942  HiOps[2] = Lo.getValue(1);
2943  Hi = DAG.getNode(ISD::ADDE, dl, VTList, HiOps);
2944  } else {
2945  Lo = DAG.getNode(ISD::SUBC, dl, VTList, LoOps);
2946  HiOps[2] = Lo.getValue(1);
2947  Hi = DAG.getNode(ISD::SUBE, dl, VTList, HiOps);
2948  }
2949  return;
2950  }
2951 
2952  bool hasOVF =
2953  TLI.isOperationLegalOrCustom(N->getOpcode() == ISD::ADD ?
2955  TLI.getTypeToExpandTo(*DAG.getContext(), NVT));
2957 
2958  if (hasOVF) {
2959  EVT OvfVT = getSetCCResultType(NVT);
2960  SDVTList VTList = DAG.getVTList(NVT, OvfVT);
2961  int RevOpc;
2962  if (N->getOpcode() == ISD::ADD) {
2963  RevOpc = ISD::SUB;
2964  Lo = DAG.getNode(ISD::UADDO, dl, VTList, LoOps);
2965  Hi = DAG.getNode(ISD::ADD, dl, NVT, makeArrayRef(HiOps, 2));
2966  } else {
2967  RevOpc = ISD::ADD;
2968  Lo = DAG.getNode(ISD::USUBO, dl, VTList, LoOps);
2969  Hi = DAG.getNode(ISD::SUB, dl, NVT, makeArrayRef(HiOps, 2));
2970  }
2971  SDValue OVF = Lo.getValue(1);
2972 
2973  switch (BoolType) {
2975  OVF = DAG.getNode(ISD::AND, dl, OvfVT, DAG.getConstant(1, dl, OvfVT), OVF);
2976  [[fallthrough]];
2978  OVF = DAG.getZExtOrTrunc(OVF, dl, NVT);
2979  Hi = DAG.getNode(N->getOpcode(), dl, NVT, Hi, OVF);
2980  break;
2982  OVF = DAG.getSExtOrTrunc(OVF, dl, NVT);
2983  Hi = DAG.getNode(RevOpc, dl, NVT, Hi, OVF);
2984  }
2985  return;
2986  }
2987 
2988  if (N->getOpcode() == ISD::ADD) {
2989  Lo = DAG.getNode(ISD::ADD, dl, NVT, LoOps);
2990  Hi = DAG.getNode(ISD::ADD, dl, NVT, makeArrayRef(HiOps, 2));
2991  SDValue Cmp = DAG.getSetCC(dl, getSetCCResultType(NVT), Lo, LoOps[0],
2992  ISD::SETULT);
2993 
2994  SDValue Carry;
2996  Carry = DAG.getZExtOrTrunc(Cmp, dl, NVT);
2997  else
2998  Carry = DAG.getSelect(dl, NVT, Cmp, DAG.getConstant(1, dl, NVT),
2999  DAG.getConstant(0, dl, NVT));
3000 
3001  Hi = DAG.getNode(ISD::ADD, dl, NVT, Hi, Carry);
3002  } else {
3003  Lo = DAG.getNode(ISD::SUB, dl, NVT, LoOps);
3004  Hi = DAG.getNode(ISD::SUB, dl, NVT, makeArrayRef(HiOps, 2));
3005  SDValue Cmp =
3006  DAG.getSetCC(dl, getSetCCResultType(LoOps[0].getValueType()),
3007  LoOps[0], LoOps[1], ISD::SETULT);
3008 
3009  SDValue Borrow;
3011  Borrow = DAG.getZExtOrTrunc(Cmp, dl, NVT);
3012  else
3013  Borrow = DAG.getSelect(dl, NVT, Cmp, DAG.getConstant(1, dl, NVT),
3014  DAG.getConstant(0, dl, NVT));
3015 
3016  Hi = DAG.getNode(ISD::SUB, dl, NVT, Hi, Borrow);
3017  }
3018 }
3019 
3020 void DAGTypeLegalizer::ExpandIntRes_ADDSUBC(SDNode *N,
3021  SDValue &Lo, SDValue &Hi) {
3022  // Expand the subcomponents.
3023  SDValue LHSL, LHSH, RHSL, RHSH;
3024  SDLoc dl(N);
3025  GetExpandedInteger(N->getOperand(0), LHSL, LHSH);
3026  GetExpandedInteger(N->getOperand(1), RHSL, RHSH);
3027  SDVTList VTList = DAG.getVTList(LHSL.getValueType(), MVT::Glue);
3028  SDValue LoOps[2] = { LHSL, RHSL };
3029  SDValue HiOps[3] = { LHSH, RHSH };
3030 
3031  if (N->getOpcode() == ISD::ADDC) {
3032  Lo = DAG.getNode(ISD::ADDC, dl, VTList, LoOps);
3033  HiOps[2] = Lo.getValue(1);
3034  Hi = DAG.getNode(ISD::ADDE, dl, VTList, HiOps);
3035  } else {
3036  Lo = DAG.getNode(ISD::SUBC, dl, VTList, LoOps);
3037  HiOps[2] = Lo.getValue(1);
3038  Hi = DAG.getNode(ISD::SUBE, dl, VTList, HiOps);
3039  }
3040 
3041  // Legalized the flag result - switch anything that used the old flag to
3042  // use the new one.
3043  ReplaceValueWith(SDValue(N, 1), Hi.getValue(1));
3044 }
3045 
3046 void DAGTypeLegalizer::ExpandIntRes_ADDSUBE(SDNode *N,
3047  SDValue &Lo, SDValue &Hi) {
3048  // Expand the subcomponents.
3049  SDValue LHSL, LHSH, RHSL, RHSH;
3050  SDLoc dl(N);
3051  GetExpandedInteger(N->getOperand(0), LHSL, LHSH);
3052  GetExpandedInteger(N->getOperand(1), RHSL, RHSH);
3053  SDVTList VTList = DAG.getVTList(LHSL.getValueType(), MVT::Glue);
3054  SDValue LoOps[3] = { LHSL, RHSL, N->getOperand(2) };
3055  SDValue HiOps[3] = { LHSH, RHSH };
3056 
3057  Lo = DAG.getNode(N->getOpcode(), dl, VTList, LoOps);
3058  HiOps[2] = Lo.getValue(1);
3059  Hi = DAG.getNode(N->getOpcode(), dl, VTList, HiOps);
3060 
3061  // Legalized the flag result - switch anything that used the old flag to
3062  // use the new one.
3063  ReplaceValueWith(SDValue(N, 1), Hi.getValue(1));
3064 }
3065 
3066 void DAGTypeLegalizer::ExpandIntRes_UADDSUBO(SDNode *N,
3067  SDValue &Lo, SDValue &Hi) {
3068  SDValue LHS = N->getOperand(0);
3069  SDValue RHS = N->getOperand(1);
3070  SDLoc dl(N);
3071 
3072  SDValue Ovf;
3073 
3074  unsigned CarryOp, NoCarryOp;
3076  switch(N->getOpcode()) {
3077  case ISD::UADDO:
3078  CarryOp = ISD::ADDCARRY;
3079  NoCarryOp = ISD::ADD;
3080  Cond = ISD::SETULT;
3081  break;
3082  case ISD::USUBO:
3083  CarryOp = ISD::SUBCARRY;
3084  NoCarryOp = ISD::SUB;
3085  Cond = ISD::SETUGT;
3086  break;
3087  default:
3088  llvm_unreachable("Node has unexpected Opcode");
3089  }
3090 
3091  bool HasCarryOp = TLI.isOperationLegalOrCustom(
3092  CarryOp, TLI.getTypeToExpandTo(*DAG.getContext(), LHS.getValueType()));
3093 
3094  if (HasCarryOp) {
3095  // Expand the subcomponents.
3096  SDValue LHSL, LHSH, RHSL, RHSH;
3097  GetExpandedInteger(LHS, LHSL, LHSH);
3098  GetExpandedInteger(RHS, RHSL, RHSH);
3099  SDVTList VTList = DAG.getVTList(LHSL.getValueType(), N->getValueType(1));
3100  SDValue LoOps[2] = { LHSL, RHSL };
3101  SDValue HiOps[3] = { LHSH, RHSH };
3102 
3103  Lo = DAG.getNode(N->getOpcode(), dl, VTList, LoOps);
3104  HiOps[2] = Lo.getValue(1);
3105  Hi = DAG.getNode(CarryOp, dl, VTList, HiOps);
3106 
3107  Ovf = Hi.getValue(1);
3108  } else {
3109  // Expand the result by simply replacing it with the equivalent
3110  // non-overflow-checking operation.
3111  SDValue Sum = DAG.getNode(NoCarryOp, dl, LHS.getValueType(), LHS, RHS);
3112  SplitInteger(Sum, Lo, Hi);
3113 
3114  // Calculate the overflow: addition overflows iff a + b < a, and subtraction
3115  // overflows iff a - b > a.
3116  Ovf = DAG.getSetCC(dl, N->getValueType(1), Sum, LHS, Cond);
3117  }
3118 
3119  // Legalized the flag result - switch anything that used the old flag to
3120  // use the new one.
3121  ReplaceValueWith(SDValue(N, 1), Ovf);
3122 }
3123 
3124 void DAGTypeLegalizer::ExpandIntRes_ADDSUBCARRY(SDNode *N,
3125  SDValue &Lo, SDValue &Hi) {
3126  // Expand the subcomponents.
3127  SDValue LHSL, LHSH, RHSL, RHSH;
3128  SDLoc dl(N);
3129  GetExpandedInteger(N->getOperand(0), LHSL, LHSH);
3130  GetExpandedInteger(N->getOperand(1), RHSL, RHSH);
3131  SDVTList VTList = DAG.getVTList(LHSL.getValueType(), N->getValueType(1));
3132  SDValue LoOps[3] = { LHSL, RHSL, N->getOperand(2) };
3133  SDValue HiOps[3] = { LHSH, RHSH, SDValue() };
3134 
3135  Lo = DAG.getNode(N->getOpcode(), dl, VTList, LoOps);
3136  HiOps[2] = Lo.getValue(1);
3137  Hi = DAG.getNode(N->getOpcode(), dl, VTList, HiOps);
3138 
3139  // Legalized the flag result - switch anything that used the old flag to
3140  // use the new one.
3141  ReplaceValueWith(SDValue(N, 1), Hi.getValue(1));
3142 }
3143 
3144 void DAGTypeLegalizer::ExpandIntRes_SADDSUBO_CARRY(SDNode *N,
3145  SDValue &Lo, SDValue &Hi) {
3146  // Expand the subcomponents.
3147  SDValue LHSL, LHSH, RHSL, RHSH;
3148  SDLoc dl(N);
3149  GetExpandedInteger(N->getOperand(0), LHSL, LHSH);
3150  GetExpandedInteger(N->getOperand(1), RHSL, RHSH);
3151  SDVTList VTList = DAG.getVTList(LHSL.getValueType(), N->getValueType(1));
3152 
3153  // We need to use an unsigned carry op for the lo part.
3154  unsigned CarryOp = N->getOpcode() == ISD::SADDO_CARRY ? ISD::ADDCARRY
3155  : ISD::SUBCARRY;
3156  Lo = DAG.getNode(CarryOp, dl, VTList, { LHSL, RHSL, N->getOperand(2) });
3157  Hi = DAG.getNode(N->getOpcode(), dl, VTList, { LHSH, RHSH, Lo.getValue(1) });
3158 
3159  // Legalized the flag result - switch anything that used the old flag to
3160  // use the new one.
3161  ReplaceValueWith(SDValue(N, 1), Hi.getValue(1));
3162 }
3163 
3164 void DAGTypeLegalizer::ExpandIntRes_ANY_EXTEND(SDNode *N,
3165  SDValue &Lo, SDValue &Hi) {
3166  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
3167  SDLoc dl(N);
3168  SDValue Op = N->getOperand(0);
3169  if (Op.getValueType().bitsLE(NVT)) {
3170  // The low part is any extension of the input (which degenerates to a copy).
3171  Lo = DAG.getNode(ISD::ANY_EXTEND, dl, NVT, Op);
3172  Hi = DAG.getUNDEF(NVT); // The high part is undefined.
3173  } else {
3174  // For example, extension of an i48 to an i64. The operand type necessarily
3175  // promotes to the result type, so will end up being expanded too.
3176  assert(getTypeAction(Op.getValueType()) ==
3178  "Only know how to promote this result!");
3179  SDValue Res = GetPromotedInteger(Op);
3180  assert(Res.getValueType() == N->getValueType(0) &&
3181  "Operand over promoted?");
3182  // Split the promoted operand. This will simplify when it is expanded.
3183  SplitInteger(Res, Lo, Hi);
3184  }
3185 }
3186 
3187 void DAGTypeLegalizer::ExpandIntRes_AssertSext(SDNode *N,
3188  SDValue &Lo, SDValue &Hi) {
3189  SDLoc dl(N);
3190  GetExpandedInteger(N->getOperand(0), Lo, Hi);
3191  EVT NVT = Lo.getValueType();
3192  EVT EVT = cast<VTSDNode>(N->getOperand(1))->getVT();
3193  unsigned NVTBits = NVT.getSizeInBits();
3194  unsigned EVTBits = EVT.getSizeInBits();
3195 
3196  if (NVTBits < EVTBits) {
3197  Hi = DAG.getNode(ISD::AssertSext, dl, NVT, Hi,
3199  EVTBits - NVTBits)));
3200  } else {
3201  Lo = DAG.getNode(ISD::AssertSext, dl, NVT, Lo, DAG.getValueType(EVT));
3202  // The high part replicates the sign bit of Lo, make it explicit.
3203  Hi = DAG.getNode(ISD::SRA, dl, NVT, Lo,
3204  DAG.getConstant(NVTBits - 1, dl,
3205  TLI.getPointerTy(DAG.getDataLayout())));
3206  }
3207 }
3208 
3209 void DAGTypeLegalizer::ExpandIntRes_AssertZext(SDNode *N,
3210  SDValue &Lo, SDValue &Hi) {
3211  SDLoc dl(N);
3212  GetExpandedInteger(N->getOperand(0), Lo, Hi);
3213  EVT NVT = Lo.getValueType();
3214  EVT EVT = cast<VTSDNode>(N->getOperand(1))->getVT();
3215  unsigned NVTBits = NVT.getSizeInBits();
3216  unsigned EVTBits = EVT.getSizeInBits();
3217 
3218  if (NVTBits < EVTBits) {
3219  Hi = DAG.getNode(ISD::AssertZext, dl, NVT, Hi,
3221  EVTBits - NVTBits)));
3222  } else {
3223  Lo = DAG.getNode(ISD::AssertZext, dl, NVT, Lo, DAG.getValueType(EVT));
3224  // The high part must be zero, make it explicit.
3225  Hi = DAG.getConstant(0, dl, NVT);
3226  }
3227 }
3228 
3229 void DAGTypeLegalizer::ExpandIntRes_BITREVERSE(SDNode *N,
3230  SDValue &Lo, SDValue &Hi) {
3231  SDLoc dl(N);
3232  GetExpandedInteger(N->getOperand(0), Hi, Lo); // Note swapped operands.
3233  Lo = DAG.getNode(ISD::BITREVERSE, dl, Lo.getValueType(), Lo);
3234  Hi = DAG.getNode(ISD::BITREVERSE, dl, Hi.getValueType(), Hi);
3235 }
3236 
3237 void DAGTypeLegalizer::ExpandIntRes_BSWAP(SDNode *N,
3238  SDValue &Lo, SDValue &Hi) {
3239  SDLoc dl(N);
3240  GetExpandedInteger(N->getOperand(0), Hi, Lo); // Note swapped operands.
3241  Lo = DAG.getNode(ISD::BSWAP, dl, Lo.getValueType(), Lo);
3242  Hi = DAG.getNode(ISD::BSWAP, dl, Hi.getValueType(), Hi);
3243 }
3244 
3245 void DAGTypeLegalizer::ExpandIntRes_PARITY(SDNode *N, SDValue &Lo,
3246  SDValue &Hi) {
3247  SDLoc dl(N);
3248  // parity(HiLo) -> parity(Lo^Hi)
3249  GetExpandedInteger(N->getOperand(0), Lo, Hi);
3250  EVT NVT = Lo.getValueType();
3251  Lo =
3252  DAG.getNode(ISD::PARITY, dl, NVT, DAG.getNode(ISD::XOR, dl, NVT, Lo, Hi));
3253  Hi = DAG.getConstant(0, dl, NVT);
3254 }
3255 
3256 void DAGTypeLegalizer::ExpandIntRes_Constant(SDNode *N,
3257  SDValue &Lo, SDValue &Hi) {
3258  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
3259  unsigned NBitWidth = NVT.getSizeInBits();
3260  auto Constant = cast<ConstantSDNode>(N);
3261  const APInt &Cst = Constant->getAPIntValue();
3262  bool IsTarget = Constant->isTargetOpcode();
3263  bool IsOpaque = Constant->isOpaque();
3264  SDLoc dl(N);
3265  Lo = DAG.getConstant(Cst.trunc(NBitWidth), dl, NVT, IsTarget, IsOpaque);
3266  Hi = DAG.getConstant(Cst.lshr(NBitWidth).trunc(NBitWidth), dl, NVT, IsTarget,
3267  IsOpaque);
3268 }
3269 
3270 void DAGTypeLegalizer::ExpandIntRes_ABS(SDNode *N, SDValue &Lo, SDValue &Hi) {
3271  SDLoc dl(N);
3272 
3273  SDValue N0 = N->getOperand(0);
3274  GetExpandedInteger(N0, Lo, Hi);
3275  EVT NVT = Lo.getValueType();
3276 
3277  // If we have SUBCARRY, use the expanded form of the sra+xor+sub sequence we
3278  // use in LegalizeDAG. The SUB part of the expansion is based on
3279  // ExpandIntRes_ADDSUB which also uses SUBCARRY/USUBO after checking that
3280  // SUBCARRY is LegalOrCustom. Each of the pieces here can be further expanded
3281  // if needed. Shift expansion has a special case for filling with sign bits
3282  // so that we will only end up with one SRA.
3283  bool HasSubCarry = TLI.isOperationLegalOrCustom(
3284  ISD::SUBCARRY, TLI.getTypeToExpandTo(*DAG.getContext(), NVT));
3285  if (HasSubCarry) {
3286  SDValue Sign = DAG.getNode(
3287  ISD::SRA, dl, NVT, Hi,
3288  DAG.getShiftAmountConstant(NVT.getSizeInBits() - 1, NVT, dl));
3289  SDVTList VTList = DAG.getVTList(NVT, getSetCCResultType(NVT));
3290  Lo = DAG.getNode(ISD::XOR, dl, NVT, Lo, Sign);
3291  Hi = DAG.getNode(ISD::XOR, dl, NVT, Hi, Sign);
3292  Lo = DAG.getNode(ISD::USUBO, dl, VTList, Lo, Sign);
3293  Hi = DAG.getNode(ISD::SUBCARRY, dl, VTList, Hi, Sign, Lo.getValue(1));
3294  return;
3295  }
3296 
3297  // abs(HiLo) -> (Hi < 0 ? -HiLo : HiLo)
3298  EVT VT = N->getValueType(0);
3299  SDValue Neg = DAG.getNode(ISD::SUB, dl, VT,
3300  DAG.getConstant(0, dl, VT), N0);
3301  SDValue NegLo, NegHi;
3302  SplitInteger(Neg, NegLo, NegHi);
3303 
3304  SDValue HiIsNeg = DAG.getSetCC(dl, getSetCCResultType(NVT), Hi,
3305  DAG.getConstant(0, dl, NVT), ISD::SETLT);
3306  Lo = DAG.getSelect(dl, NVT, HiIsNeg, NegLo, Lo);
3307  Hi = DAG.getSelect(dl, NVT, HiIsNeg, NegHi, Hi);
3308 }
3309 
3310 void DAGTypeLegalizer::ExpandIntRes_CTLZ(SDNode *N,
3311  SDValue &Lo, SDValue &Hi) {
3312  SDLoc dl(N);
3313  // ctlz (HiLo) -> Hi != 0 ? ctlz(Hi) : (ctlz(Lo)+32)
3314  GetExpandedInteger(N->getOperand(0), Lo, Hi);
3315  EVT NVT = Lo.getValueType();
3316 
3317  SDValue HiNotZero = DAG.getSetCC(dl, getSetCCResultType(NVT), Hi,
3318  DAG.getConstant(0, dl, NVT), ISD::SETNE);
3319 
3320  SDValue LoLZ = DAG.getNode(N->getOpcode(), dl, NVT, Lo);
3321  SDValue HiLZ = DAG.getNode(ISD::CTLZ_ZERO_UNDEF, dl, NVT, Hi);
3322 
3323  Lo = DAG.getSelect(dl, NVT, HiNotZero, HiLZ,
3324  DAG.getNode(ISD::ADD, dl, NVT, LoLZ,
3325  DAG.getConstant(NVT.getSizeInBits(), dl,
3326  NVT)));
3327  Hi = DAG.getConstant(0, dl, NVT);
3328 }
3329 
3330 void DAGTypeLegalizer::ExpandIntRes_CTPOP(SDNode *N,
3331  SDValue &Lo, SDValue &Hi) {
3332  SDLoc dl(N);
3333  // ctpop(HiLo) -> ctpop(Hi)+ctpop(Lo)
3334  GetExpandedInteger(N->getOperand(0), Lo, Hi);
3335  EVT NVT = Lo.getValueType();
3336  Lo = DAG.getNode(ISD::ADD, dl, NVT, DAG.getNode(ISD::CTPOP, dl, NVT, Lo),
3337  DAG.getNode(ISD::CTPOP, dl, NVT, Hi));
3338  Hi = DAG.getConstant(0, dl, NVT);
3339 }
3340 
3341 void DAGTypeLegalizer::ExpandIntRes_CTTZ(SDNode *N,
3342  SDValue &Lo, SDValue &Hi) {
3343  SDLoc dl(N);
3344  // cttz (HiLo) -> Lo != 0 ? cttz(Lo) : (cttz(Hi)+32)
3345  GetExpandedInteger(N->getOperand(0), Lo, Hi);
3346  EVT NVT = Lo.getValueType();
3347 
3348  SDValue LoNotZero = DAG.getSetCC(dl, getSetCCResultType(NVT), Lo,
3349  DAG.getConstant(0, dl, NVT), ISD::SETNE);
3350 
3351  SDValue LoLZ = DAG.getNode(ISD::CTTZ_ZERO_UNDEF, dl, NVT, Lo);
3352  SDValue HiLZ = DAG.getNode(N->getOpcode(), dl, NVT, Hi);
3353 
3354  Lo = DAG.getSelect(dl, NVT, LoNotZero, LoLZ,
3355  DAG.getNode(ISD::ADD, dl, NVT, HiLZ,
3356  DAG.getConstant(NVT.getSizeInBits(), dl,
3357  NVT)));
3358  Hi = DAG.getConstant(0, dl, NVT);
3359 }
3360 
3361 void DAGTypeLegalizer::ExpandIntRes_FLT_ROUNDS(SDNode *N, SDValue &Lo,
3362  SDValue &Hi) {
3363  SDLoc dl(N);
3364  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
3365  unsigned NBitWidth = NVT.getSizeInBits();
3366 
3367  Lo = DAG.getNode(ISD::FLT_ROUNDS_, dl, {NVT, MVT::Other}, N->getOperand(0));
3368  SDValue Chain = Lo.getValue(1);
3369  // The high part is the sign of Lo, as -1 is a valid value for FLT_ROUNDS
3370  Hi = DAG.getNode(ISD::SRA, dl, NVT, Lo,
3371  DAG.getShiftAmountConstant(NBitWidth - 1, NVT, dl));
3372 
3373  // Legalize the chain result - switch anything that used the old chain to
3374  // use the new one.
3375  ReplaceValueWith(SDValue(N, 1), Chain);
3376 }
3377 
3378 void DAGTypeLegalizer::ExpandIntRes_FP_TO_SINT(SDNode *N, SDValue &Lo,
3379  SDValue &Hi) {
3380  SDLoc dl(N);
3381  EVT VT = N->getValueType(0);
3382 
3383  bool IsStrict = N->isStrictFPOpcode();
3384  SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
3385  SDValue Op = N->getOperand(IsStrict ? 1 : 0);
3386  if (getTypeAction(Op.getValueType()) == TargetLowering::TypePromoteFloat)
3387  Op = GetPromotedFloat(Op);
3388 
3389  if (getTypeAction(Op.getValueType()) == TargetLowering::TypeSoftPromoteHalf) {
3390  EVT NFPVT = TLI.getTypeToTransformTo(*DAG.getContext(), Op.getValueType());
3391  Op = GetSoftPromotedHalf(Op);
3392  Op = DAG.getNode(ISD::FP16_TO_FP, dl, NFPVT, Op);
3393  Op = DAG.getNode(ISD::FP_TO_SINT, dl, VT, Op);
3394  SplitInteger(Op, Lo, Hi);
3395  return;
3396  }
3397 
3398  RTLIB::Libcall LC = RTLIB::getFPTOSINT(Op.getValueType(), VT);
3399  assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected fp-to-sint conversion!");
3401  CallOptions.setSExt(true);
3402  std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, VT, Op,
3403  CallOptions, dl, Chain);
3404  SplitInteger(Tmp.first, Lo, Hi);
3405 
3406  if (IsStrict)
3407  ReplaceValueWith(SDValue(N, 1), Tmp.second);
3408 }
3409 
3410 void DAGTypeLegalizer::ExpandIntRes_FP_TO_UINT(SDNode *N, SDValue &Lo,
3411  SDValue &Hi) {
3412  SDLoc dl(N);
3413  EVT VT = N->getValueType(0);
3414 
3415  bool IsStrict = N->isStrictFPOpcode();
3416  SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
3417  SDValue Op = N->getOperand(IsStrict ? 1 : 0);
3418  if (getTypeAction(Op.getValueType()) == TargetLowering::TypePromoteFloat)
3419  Op = GetPromotedFloat(Op);
3420 
3421  if (getTypeAction(Op.getValueType()) == TargetLowering::TypeSoftPromoteHalf) {
3422  EVT NFPVT = TLI.getTypeToTransformTo(*DAG.getContext(), Op.getValueType());
3423  Op = GetSoftPromotedHalf(Op);
3424  Op = DAG.getNode(ISD::FP16_TO_FP, dl, NFPVT, Op);
3425  Op = DAG.getNode(ISD::FP_TO_UINT, dl, VT, Op);
3426  SplitInteger(Op, Lo, Hi);
3427  return;
3428  }
3429 
3430  RTLIB::Libcall LC = RTLIB::getFPTOUINT(Op.getValueType(), VT);
3431  assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected fp-to-uint conversion!");
3433  std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, VT, Op,
3434  CallOptions, dl, Chain);
3435  SplitInteger(Tmp.first, Lo, Hi);
3436 
3437  if (IsStrict)
3438  ReplaceValueWith(SDValue(N, 1), Tmp.second);
3439 }
3440 
3441 void DAGTypeLegalizer::ExpandIntRes_FP_TO_XINT_SAT(SDNode *N, SDValue &Lo,
3442  SDValue &Hi) {
3443  SDValue Res = TLI.expandFP_TO_INT_SAT(N, DAG);
3444  SplitInteger(Res, Lo, Hi);
3445 }
3446 
3447 void DAGTypeLegalizer::ExpandIntRes_LLROUND_LLRINT(SDNode *N, SDValue &Lo,
3448  SDValue &Hi) {
3449  SDValue Op = N->getOperand(N->isStrictFPOpcode() ? 1 : 0);
3450 
3451  assert(getTypeAction(Op.getValueType()) != TargetLowering::TypePromoteFloat &&
3452  "Input type needs to be promoted!");
3453 
3454  EVT VT = Op.getValueType();
3455 
3456  RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
3457  if (N->getOpcode() == ISD::LLROUND ||
3458  N->getOpcode() == ISD::STRICT_LLROUND) {
3459  if (VT == MVT::f32)
3460  LC = RTLIB::LLROUND_F32;
3461  else if (VT == MVT::f64)
3462  LC = RTLIB::LLROUND_F64;
3463  else if (VT == MVT::f80)
3464  LC = RTLIB::LLROUND_F80;
3465  else if (VT == MVT::f128)
3466  LC = RTLIB::LLROUND_F128;
3467  else if (VT == MVT::ppcf128)
3468  LC = RTLIB::LLROUND_PPCF128;
3469  assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected llround input type!");
3470  } else if (N->getOpcode() == ISD::LLRINT ||
3471  N->getOpcode() == ISD::STRICT_LLRINT) {
3472  if (VT == MVT::f32)
3473  LC = RTLIB::LLRINT_F32;
3474  else if (VT == MVT::f64)
3475  LC = RTLIB::LLRINT_F64;
3476  else if (VT == MVT::f80)
3477  LC = RTLIB::LLRINT_F80;
3478  else if (VT == MVT::f128)
3479  LC = RTLIB::LLRINT_F128;
3480  else if (VT == MVT::ppcf128)
3481  LC = RTLIB::LLRINT_PPCF128;
3482  assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected llrint input type!");
3483  } else
3484  llvm_unreachable("Unexpected opcode!");
3485 
3486  SDLoc dl(N);
3487  EVT RetVT = N->getValueType(0);
3488  SDValue Chain = N->isStrictFPOpcode() ? N->getOperand(0) : SDValue();
3489 
3491  CallOptions.setSExt(true);
3492  std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, RetVT,
3493  Op, CallOptions, dl,
3494  Chain);
3495  SplitInteger(Tmp.first, Lo, Hi);
3496 
3497  if (N->isStrictFPOpcode())
3498  ReplaceValueWith(SDValue(N, 1), Tmp.second);
3499 }
3500 
3501 void DAGTypeLegalizer::ExpandIntRes_LOAD(LoadSDNode *N,
3502  SDValue &Lo, SDValue &Hi) {
3503  if (N->isAtomic()) {
3504  // It's typical to have larger CAS than atomic load instructions.
3505  SDLoc dl(N);
3506  EVT VT = N->getMemoryVT();
3507  SDVTList VTs = DAG.getVTList(VT, MVT::i1, MVT::Other);
3508  SDValue Zero = DAG.getConstant(0, dl, VT);
3509  SDValue Swap = DAG.getAtomicCmpSwap(
3511  VT, VTs, N->getOperand(0),
3512  N->getOperand(1), Zero, Zero, N->getMemOperand());
3513  ReplaceValueWith(SDValue(N, 0), Swap.getValue(0));
3514  ReplaceValueWith(SDValue(N, 1), Swap.getValue(2));
3515  return;
3516  }
3517 
3518  if (ISD::isNormalLoad(N)) {
3519  ExpandRes_NormalLoad(N, Lo, Hi);
3520  return;
3521  }
3522 
3523  assert(ISD::isUNINDEXEDLoad(N) && "Indexed load during type legalization!");
3524 
3525  EVT VT = N->getValueType(0);
3526  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
3527  SDValue Ch = N->getChain();
3528  SDValue Ptr = N->getBasePtr();
3529  ISD::LoadExtType ExtType = N->getExtensionType();
3530  MachineMemOperand::Flags MMOFlags = N->getMemOperand()->getFlags();
3531  AAMDNodes AAInfo = N->getAAInfo();
3532  SDLoc dl(N);
3533 
3534  assert(NVT.isByteSized() && "Expanded type not byte sized!");
3535 
3536  if (N->getMemoryVT().bitsLE(NVT)) {
3537  EVT MemVT = N->getMemoryVT();
3538 
3539  Lo = DAG.getExtLoad(ExtType, dl, NVT, Ch, Ptr, N->getPointerInfo(), MemVT,
3540  N->getOriginalAlign(), MMOFlags, AAInfo);
3541 
3542  // Remember the chain.
3543  Ch = Lo.getValue(1);
3544 
3545  if (ExtType == ISD::SEXTLOAD) {
3546  // The high part is obtained by SRA'ing all but one of the bits of the
3547  // lo part.
3548  unsigned LoSize = Lo.getValueSizeInBits();
3549  Hi = DAG.getNode(ISD::SRA, dl, NVT, Lo,
3550  DAG.getConstant(LoSize - 1, dl,
3551  TLI.getPointerTy(DAG.getDataLayout())));
3552  } else if (ExtType == ISD::ZEXTLOAD) {
3553  // The high part is just a zero.
3554  Hi = DAG.getConstant(0, dl, NVT);
3555  } else {
3556  assert(ExtType == ISD::EXTLOAD && "Unknown extload!");
3557  // The high part is undefined.
3558  Hi = DAG.getUNDEF(NVT);
3559  }
3560  } else if (DAG.getDataLayout().isLittleEndian()) {
3561  // Little-endian - low bits are at low addresses.
3562  Lo = DAG.getLoad(NVT, dl, Ch, Ptr, N->getPointerInfo(),
3563  N->getOriginalAlign(), MMOFlags, AAInfo);
3564 
3565  unsigned ExcessBits =
3566  N->getMemoryVT().getSizeInBits() - NVT.getSizeInBits();
3567  EVT NEVT = EVT::getIntegerVT(*DAG.getContext(), ExcessBits);
3568 
3569  // Increment the pointer to the other half.
3570  unsigned IncrementSize = NVT.getSizeInBits()/8;
3571  Ptr = DAG.getMemBasePlusOffset(Ptr, TypeSize::Fixed(IncrementSize), dl);
3572  Hi = DAG.getExtLoad(ExtType, dl, NVT, Ch, Ptr,
3573  N->getPointerInfo().getWithOffset(IncrementSize), NEVT,
3574  N->getOriginalAlign(), MMOFlags, AAInfo);
3575 
3576  // Build a factor node to remember that this load is independent of the
3577  // other one.
3578  Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1),
3579  Hi.getValue(1));
3580  } else {
3581  // Big-endian - high bits are at low addresses. Favor aligned loads at
3582  // the cost of some bit-fiddling.
3583  EVT MemVT = N->getMemoryVT();
3584  unsigned EBytes = MemVT.getStoreSize();
3585  unsigned IncrementSize = NVT.getSizeInBits()/8;
3586  unsigned ExcessBits = (EBytes - IncrementSize)*8;
3587 
3588  // Load both the high bits and maybe some of the low bits.
3589  Hi = DAG.getExtLoad(ExtType, dl, NVT, Ch, Ptr, N->getPointerInfo(),
3591  MemVT.getSizeInBits() - ExcessBits),
3592  N->getOriginalAlign(), MMOFlags, AAInfo);
3593 
3594  // Increment the pointer to the other half.
3595  Ptr = DAG.getMemBasePlusOffset(Ptr, TypeSize::Fixed(IncrementSize), dl);
3596  // Load the rest of the low bits.
3597  Lo = DAG.getExtLoad(ISD::ZEXTLOAD, dl, NVT, Ch, Ptr,
3598  N->getPointerInfo().getWithOffset(IncrementSize),
3599  EVT::getIntegerVT(*DAG.getContext(), ExcessBits),
3600  N->getOriginalAlign(), MMOFlags, AAInfo);
3601 
3602  // Build a factor node to remember that this load is independent of the
3603  // other one.
3604  Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1),
3605  Hi.getValue(1));
3606 
3607  if (ExcessBits < NVT.getSizeInBits()) {
3608  // Transfer low bits from the bottom of Hi to the top of Lo.
3609  Lo = DAG.getNode(
3610  ISD::OR, dl, NVT, Lo,
3611  DAG.getNode(ISD::SHL, dl, NVT, Hi,
3612  DAG.getConstant(ExcessBits, dl,
3613  TLI.getPointerTy(DAG.getDataLayout()))));
3614  // Move high bits to the right position in Hi.
3615  Hi = DAG.getNode(ExtType == ISD::SEXTLOAD ? ISD::SRA : ISD::SRL, dl, NVT,
3616  Hi,
3617  DAG.getConstant(NVT.getSizeInBits() - ExcessBits, dl,
3618  TLI.getPointerTy(DAG.getDataLayout())));
3619  }
3620  }
3621 
3622  // Legalize the chain result - switch anything that used the old chain to
3623  // use the new one.
3624  ReplaceValueWith(SDValue(N, 1), Ch);
3625 }
3626 
3627 void DAGTypeLegalizer::ExpandIntRes_Logical(SDNode *N,
3628  SDValue &Lo, SDValue &Hi) {
3629  SDLoc dl(N);
3630  SDValue LL, LH, RL, RH;
3631  GetExpandedInteger(N->getOperand(0), LL, LH);
3632  GetExpandedInteger(N->getOperand(1), RL, RH);
3633  Lo = DAG.getNode(N->getOpcode(), dl, LL.getValueType(), LL, RL);
3634  Hi = DAG.getNode(N->getOpcode(), dl, LL.getValueType(), LH, RH);
3635 }
3636 
3637 void DAGTypeLegalizer::ExpandIntRes_MUL(SDNode *N,
3638  SDValue &Lo, SDValue &Hi) {
3639  EVT VT = N->getValueType(0);
3640  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
3641  SDLoc dl(N);
3642 
3643  SDValue LL, LH, RL, RH;
3644  GetExpandedInteger(N->getOperand(0), LL, LH);
3645  GetExpandedInteger(N->getOperand(1), RL, RH);
3646 
3647  if (TLI.expandMUL(N, Lo, Hi, NVT, DAG,
3649  LL, LH, RL, RH))
3650  return;
3651 
3652  // If nothing else, we can make a libcall.
3653  RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
3654  if (VT == MVT::i16)
3655  LC = RTLIB::MUL_I16;
3656  else if (VT == MVT::i32)
3657  LC = RTLIB::MUL_I32;
3658  else if (VT == MVT::i64)
3659  LC = RTLIB::MUL_I64;
3660  else if (VT == MVT::i128)
3661  LC = RTLIB::MUL_I128;
3662 
3663  if (LC == RTLIB::UNKNOWN_LIBCALL || !TLI.getLibcallName(LC)) {
3664  // We'll expand the multiplication by brute force because we have no other
3665  // options. This is a trivially-generalized version of the code from
3666  // Hacker's Delight (itself derived from Knuth's Algorithm M from section
3667  // 4.3.1).
3668  unsigned Bits = NVT.getSizeInBits();
3669  unsigned HalfBits = Bits >> 1;
3670  SDValue Mask = DAG.getConstant(APInt::getLowBitsSet(Bits, HalfBits), dl,
3671  NVT);
3672  SDValue LLL = DAG.getNode(ISD::AND, dl, NVT, LL, Mask);
3673  SDValue RLL = DAG.getNode(ISD::AND, dl, NVT, RL, Mask);
3674 
3675  SDValue T = DAG.getNode(ISD::MUL, dl, NVT, LLL, RLL);
3676  SDValue TL = DAG.getNode(ISD::AND, dl, NVT, T, Mask);
3677 
3678  SDValue Shift = DAG.getShiftAmountConstant(HalfBits, NVT, dl);
3679  SDValue TH = DAG.getNode(ISD::SRL, dl, NVT, T, Shift);
3680  SDValue LLH = DAG.getNode(ISD::SRL, dl, NVT, LL, Shift);
3681  SDValue RLH = DAG.getNode(ISD::SRL, dl, NVT, RL, Shift);
3682 
3683  SDValue U = DAG.getNode(ISD::ADD, dl, NVT,
3684  DAG.getNode(ISD::MUL, dl, NVT, LLH, RLL), TH);
3685  SDValue UL = DAG.getNode(ISD::AND, dl, NVT, U, Mask);
3686  SDValue UH = DAG.getNode(ISD::SRL, dl, NVT, U, Shift);
3687 
3688  SDValue V = DAG.getNode(ISD::ADD, dl, NVT,
3689  DAG.getNode(ISD::MUL, dl, NVT, LLL, RLH), UL);
3690  SDValue VH = DAG.getNode(ISD::SRL, dl, NVT, V, Shift);
3691 
3692  SDValue W = DAG.getNode(ISD::ADD, dl, NVT,
3693  DAG.getNode(ISD::MUL, dl, NVT, LLH, RLH),
3694  DAG.getNode(ISD::ADD, dl, NVT, UH, VH));
3695  Lo = DAG.getNode(ISD::ADD, dl, NVT, TL,
3696  DAG.getNode(ISD::SHL, dl, NVT, V, Shift));
3697 
3698  Hi = DAG.getNode(ISD::ADD, dl, NVT, W,
3699  DAG.getNode(ISD::ADD, dl, NVT,
3700  DAG.getNode(ISD::MUL, dl, NVT, RH, LL),
3701  DAG.getNode(ISD::MUL, dl, NVT, RL, LH)));
3702  return;
3703  }
3704 
3705  SDValue Ops[2] = { N->getOperand(0), N->getOperand(1) };
3707  CallOptions.setSExt(true);
3708  SplitInteger(TLI.makeLibCall(DAG, LC, VT, Ops, CallOptions, dl).first,
3709  Lo, Hi);
3710 }
3711 
3712 void DAGTypeLegalizer::ExpandIntRes_READCYCLECOUNTER(SDNode *N, SDValue &Lo,
3713  SDValue &Hi) {
3714  SDLoc DL(N);
3715  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
3716  SDVTList VTs = DAG.getVTList(NVT, NVT, MVT::Other);
3717  SDValue R = DAG.getNode(N->getOpcode(), DL, VTs, N->getOperand(0));
3718  Lo = R.getValue(0);
3719  Hi = R.getValue(1);
3720  ReplaceValueWith(SDValue(N, 1), R.getValue(2));
3721 }
3722 
3723 void DAGTypeLegalizer::ExpandIntRes_ADDSUBSAT(SDNode *N, SDValue &Lo,
3724  SDValue &Hi) {
3725  SDValue Result = TLI.expandAddSubSat(N, DAG);
3726  SplitInteger(Result, Lo, Hi);
3727 }
3728 
3729 void DAGTypeLegalizer::ExpandIntRes_SHLSAT(SDNode *N, SDValue &Lo,
3730  SDValue &Hi) {
3731  SDValue Result = TLI.expandShlSat(N, DAG);
3732  SplitInteger(Result, Lo, Hi);
3733 }
3734 
3735 /// This performs an expansion of the integer result for a fixed point
3736 /// multiplication. The default expansion performs rounding down towards
3737 /// negative infinity, though targets that do care about rounding should specify
3738 /// a target hook for rounding and provide their own expansion or lowering of
3739 /// fixed point multiplication to be consistent with rounding.
3740 void DAGTypeLegalizer::ExpandIntRes_MULFIX(SDNode *N, SDValue &Lo,
3741  SDValue &Hi) {
3742  SDLoc dl(N);
3743  EVT VT = N->getValueType(0);
3744  unsigned VTSize = VT.getScalarSizeInBits();
3745  SDValue LHS = N->getOperand(0);
3746  SDValue RHS = N->getOperand(1);
3747  uint64_t Scale = N->getConstantOperandVal(2);
3748  bool Saturating = (N->getOpcode() == ISD::SMULFIXSAT ||
3749  N->getOpcode() == ISD::UMULFIXSAT);
3750  bool Signed = (N->getOpcode() == ISD::SMULFIX ||
3751  N->getOpcode() == ISD::SMULFIXSAT);
3752 
3753  // Handle special case when scale is equal to zero.
3754  if (!Scale) {
3755  SDValue Result;
3756  if (!Saturating) {
3757  Result = DAG.getNode(ISD::MUL, dl, VT, LHS, RHS);
3758  } else {
3759  EVT BoolVT = getSetCCResultType(VT);
3760  unsigned MulOp = Signed ? ISD::SMULO : ISD::UMULO;
3761  Result = DAG.getNode(MulOp, dl, DAG.getVTList(VT, BoolVT), LHS, RHS);
3762  SDValue Product = Result.getValue(0);
3763  SDValue Overflow = Result.getValue(1);
3764  if (Signed) {
3765  APInt MinVal = APInt::getSignedMinValue(VTSize);
3766  APInt MaxVal = APInt::getSignedMaxValue(VTSize);
3767  SDValue SatMin = DAG.getConstant(MinVal, dl, VT);
3768  SDValue SatMax = DAG.getConstant(MaxVal, dl, VT);
3769  SDValue Zero = DAG.getConstant(0, dl, VT);
3770  // Xor the inputs, if resulting sign bit is 0 the product will be
3771  // positive, else negative.
3772  SDValue Xor = DAG.getNode(ISD::XOR, dl, VT, LHS, RHS);
3773  SDValue ProdNeg = DAG.getSetCC(dl, BoolVT, Xor, Zero, ISD::SETLT);
3774  Result = DAG.getSelect(dl, VT, ProdNeg, SatMin, SatMax);
3775  Result = DAG.getSelect(dl, VT, Overflow, Result, Product);
3776  } else {
3777  // For unsigned multiplication, we only need to check the max since we
3778  // can't really overflow towards zero.
3779  APInt MaxVal = APInt::getMaxValue(VTSize);
3780  SDValue SatMax = DAG.getConstant(MaxVal, dl, VT);
3781  Result = DAG.getSelect(dl, VT, Overflow, SatMax, Product);
3782  }
3783  }
3784  SplitInteger(Result, Lo, Hi);
3785  return;
3786  }
3787 
3788  // For SMULFIX[SAT] we only expect to find Scale<VTSize, but this assert will
3789  // cover for unhandled cases below, while still being valid for UMULFIX[SAT].
3790  assert(Scale <= VTSize && "Scale can't be larger than the value type size.");
3791 
3792  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
3793  SDValue LL, LH, RL, RH;
3794  GetExpandedInteger(LHS, LL, LH);
3795  GetExpandedInteger(RHS, RL, RH);
3797 
3798  unsigned LoHiOp = Signed ? ISD::SMUL_LOHI : ISD::UMUL_LOHI;
3799  if (!TLI.expandMUL_LOHI(LoHiOp, VT, dl, LHS, RHS, Result, NVT, DAG,
3801  LL, LH, RL, RH)) {
3802  report_fatal_error("Unable to expand MUL_FIX using MUL_LOHI.");
3803  return;
3804  }
3805 
3806  unsigned NVTSize = NVT.getScalarSizeInBits();
3807  assert((VTSize == NVTSize * 2) && "Expected the new value type to be half "
3808  "the size of the current value type");
3809 
3810  // After getting the multiplication result in 4 parts, we need to perform a
3811  // shift right by the amount of the scale to get the result in that scale.
3812  //
3813  // Let's say we multiply 2 64 bit numbers. The resulting value can be held in
3814  // 128 bits that are cut into 4 32-bit parts:
3815  //
3816  // HH HL LH LL
3817  // |---32---|---32---|---32---|---32---|
3818  // 128 96 64 32 0
3819  //
3820  // |------VTSize-----|
3821  //
3822  // |NVTSize-|
3823  //
3824  // The resulting Lo and Hi would normally be in LL and LH after the shift. But
3825  // to avoid unneccessary shifting of all 4 parts, we can adjust the shift
3826  // amount and get Lo and Hi using two funnel shifts. Or for the special case
3827  // when Scale is a multiple of NVTSize we can just pick the result without
3828  // shifting.
3829  uint64_t Part0 = Scale / NVTSize; // Part holding lowest bit needed.
3830  if (Scale % NVTSize) {
3831  SDValue ShiftAmount = DAG.getShiftAmountConstant(Scale % NVTSize, NVT, dl);
3832  Lo = DAG.getNode(ISD::FSHR, dl, NVT, Result[Part0 + 1], Result[Part0],
3833  ShiftAmount);
3834  Hi = DAG.getNode(ISD::FSHR, dl, NVT, Result[Part0 + 2], Result[Part0 + 1],
3835  ShiftAmount);
3836  } else {
3837  Lo = Result[Part0];
3838  Hi = Result[Part0 + 1];
3839  }
3840 
3841  // Unless saturation is requested we are done. The result is in <Hi,Lo>.
3842  if (!Saturating)
3843  return;
3844 
3845  // Can not overflow when there is no integer part.
3846  if (Scale == VTSize)
3847  return;
3848 
3849  // To handle saturation we must check for overflow in the multiplication.
3850  //
3851  // Unsigned overflow happened if the upper (VTSize - Scale) bits (of Result)
3852  // aren't all zeroes.
3853  //
3854  // Signed overflow happened if the upper (VTSize - Scale + 1) bits (of Result)
3855  // aren't all ones or all zeroes.
3856  //
3857  // We cannot overflow past HH when multiplying 2 ints of size VTSize, so the
3858  // highest bit of HH determines saturation direction in the event of signed
3859  // saturation.
3860 
3861  SDValue ResultHL = Result[2];
3862  SDValue ResultHH = Result[3];
3863 
3864  SDValue SatMax, SatMin;
3865  SDValue NVTZero = DAG.getConstant(0, dl, NVT);
3866  SDValue NVTNeg1 = DAG.getConstant(-1, dl, NVT);
3867  EVT BoolNVT = getSetCCResultType(NVT);
3868 
3869  if (!Signed) {
3870  if (Scale < NVTSize) {
3871  // Overflow happened if ((HH | (HL >> Scale)) != 0).
3872  SDValue HLAdjusted =
3873  DAG.getNode(ISD::SRL, dl, NVT, ResultHL,
3874  DAG.getShiftAmountConstant(Scale, NVT, dl));
3875  SDValue Tmp = DAG.getNode(ISD::OR, dl, NVT, HLAdjusted, ResultHH);
3876  SatMax = DAG.getSetCC(dl, BoolNVT, Tmp, NVTZero, ISD::SETNE);
3877  } else if (Scale == NVTSize) {
3878  // Overflow happened if (HH != 0).
3879  SatMax = DAG.getSetCC(dl, BoolNVT, ResultHH, NVTZero, ISD::SETNE);
3880  } else if (Scale < VTSize) {
3881  // Overflow happened if ((HH >> (Scale - NVTSize)) != 0).
3882  SDValue HLAdjusted =
3883  DAG.getNode(ISD::SRL, dl, NVT, ResultHL,
3884  DAG.getShiftAmountConstant(Scale - NVTSize, NVT, dl));
3885  SatMax = DAG.getSetCC(dl, BoolNVT, HLAdjusted, NVTZero, ISD::SETNE);
3886  } else
3887  llvm_unreachable("Scale must be less or equal to VTSize for UMULFIXSAT"
3888  "(and saturation can't happen with Scale==VTSize).");
3889 
3890  Hi = DAG.getSelect(dl, NVT, SatMax, NVTNeg1, Hi);
3891  Lo = DAG.getSelect(dl, NVT, SatMax, NVTNeg1, Lo);
3892  return;
3893  }
3894 
3895  if (Scale < NVTSize) {
3896  // The number of overflow bits we can check are VTSize - Scale + 1 (we
3897  // include the sign bit). If these top bits are > 0, then we overflowed past
3898  // the max value. If these top bits are < -1, then we overflowed past the
3899  // min value. Otherwise, we did not overflow.
3900  unsigned OverflowBits = VTSize - Scale + 1;
3901  assert(OverflowBits <= VTSize && OverflowBits > NVTSize &&
3902  "Extent of overflow bits must start within HL");
3903  SDValue HLHiMask = DAG.getConstant(
3904  APInt::getHighBitsSet(NVTSize, OverflowBits - NVTSize), dl, NVT);
3905  SDValue HLLoMask = DAG.getConstant(
3906  APInt::getLowBitsSet(NVTSize, VTSize - OverflowBits), dl, NVT);
3907  // We overflow max if HH > 0 or (HH == 0 && HL > HLLoMask).
3908  SDValue HHGT0 = DAG.getSetCC(dl, BoolNVT, ResultHH, NVTZero, ISD::SETGT);
3909  SDValue HHEQ0 = DAG.getSetCC(dl, BoolNVT, ResultHH, NVTZero, ISD::SETEQ);
3910  SDValue HLUGT = DAG.getSetCC(dl, BoolNVT, ResultHL, HLLoMask, ISD::SETUGT);
3911  SatMax = DAG.getNode(ISD::OR, dl, BoolNVT, HHGT0,
3912  DAG.getNode(ISD::AND, dl, BoolNVT, HHEQ0, HLUGT));
3913  // We overflow min if HH < -1 or (HH == -1 && HL < HLHiMask).
3914  SDValue HHLT = DAG.getSetCC(dl, BoolNVT, ResultHH, NVTNeg1, ISD::SETLT);
3915  SDValue HHEQ = DAG.getSetCC(dl, BoolNVT, ResultHH, NVTNeg1, ISD::SETEQ);
3916  SDValue HLULT = DAG.getSetCC(dl, BoolNVT, ResultHL, HLHiMask, ISD::SETULT);
3917  SatMin = DAG.getNode(ISD::OR, dl, BoolNVT, HHLT,
3918  DAG.getNode(ISD::AND, dl, BoolNVT, HHEQ, HLULT));
3919  } else if (Scale == NVTSize) {
3920  // We overflow max if HH > 0 or (HH == 0 && HL sign bit is 1).
3921  SDValue HHGT0 = DAG.getSetCC(dl, BoolNVT, ResultHH, NVTZero, ISD::SETGT);
3922  SDValue HHEQ0 = DAG.getSetCC(dl, BoolNVT, ResultHH, NVTZero, ISD::SETEQ);
3923  SDValue HLNeg = DAG.getSetCC(dl, BoolNVT, ResultHL, NVTZero, ISD::SETLT);
3924  SatMax = DAG.getNode(ISD::OR, dl, BoolNVT, HHGT0,
3925  DAG.getNode(ISD::AND, dl, BoolNVT, HHEQ0, HLNeg));
3926  // We overflow min if HH < -1 or (HH == -1 && HL sign bit is 0).
3927  SDValue HHLT = DAG.getSetCC(dl, BoolNVT, ResultHH, NVTNeg1, ISD::SETLT);
3928  SDValue HHEQ = DAG.getSetCC(dl, BoolNVT, ResultHH, NVTNeg1, ISD::SETEQ);
3929  SDValue HLPos = DAG.getSetCC(dl, BoolNVT, ResultHL, NVTZero, ISD::SETGE);
3930  SatMin = DAG.getNode(ISD::OR, dl, BoolNVT, HHLT,
3931  DAG.getNode(ISD::AND, dl, BoolNVT, HHEQ, HLPos));
3932  } else if (Scale < VTSize) {
3933  // This is similar to the case when we saturate if Scale < NVTSize, but we
3934  // only need to check HH.
3935  unsigned OverflowBits = VTSize - Scale + 1;
3936  SDValue HHHiMask = DAG.getConstant(
3937  APInt::getHighBitsSet(NVTSize, OverflowBits), dl, NVT);
3938  SDValue HHLoMask = DAG.getConstant(
3939  APInt::getLowBitsSet(NVTSize, NVTSize - OverflowBits), dl, NVT);
3940  SatMax = DAG.getSetCC(dl, BoolNVT, ResultHH, HHLoMask, ISD::SETGT);
3941  SatMin = DAG.getSetCC(dl, BoolNVT, ResultHH, HHHiMask, ISD::SETLT);
3942  } else
3943  llvm_unreachable("Illegal scale for signed fixed point mul.");
3944 
3945  // Saturate to signed maximum.
3946  APInt MaxHi = APInt::getSignedMaxValue(NVTSize);
3947  APInt MaxLo = APInt::getAllOnes(NVTSize);
3948  Hi = DAG.getSelect(dl, NVT, SatMax, DAG.getConstant(MaxHi, dl, NVT), Hi);
3949  Lo = DAG.getSelect(dl, NVT, SatMax, DAG.getConstant(MaxLo, dl, NVT), Lo);
3950  // Saturate to signed minimum.
3951  APInt MinHi = APInt::getSignedMinValue(NVTSize);
3952  Hi = DAG.getSelect(dl, NVT, SatMin, DAG.getConstant(MinHi, dl, NVT), Hi);
3953  Lo = DAG.getSelect(dl, NVT, SatMin, NVTZero, Lo);
3954 }
3955 
3956 void DAGTypeLegalizer::ExpandIntRes_DIVFIX(SDNode *N, SDValue &Lo,
3957  SDValue &Hi) {
3958  SDLoc dl(N);
3959  // Try expanding in the existing type first.
3960  SDValue Res = TLI.expandFixedPointDiv(N->getOpcode(), dl, N->getOperand(0),
3961  N->getOperand(1),
3962  N->getConstantOperandVal(2), DAG);
3963 
3964  if (!Res)
3965  Res = earlyExpandDIVFIX(N, N->getOperand(0), N->getOperand(1),
3966  N->getConstantOperandVal(2), TLI, DAG);
3967  SplitInteger(Res, Lo, Hi);
3968 }
3969 
3970 void DAGTypeLegalizer::ExpandIntRes_SADDSUBO(SDNode *Node,
3971  SDValue &Lo, SDValue &Hi) {
3972  assert((Node->getOpcode() == ISD::SADDO || Node->getOpcode() == ISD::SSUBO) &&
3973  "Node has unexpected Opcode");
3974  SDValue LHS = Node->getOperand(0);
3975  SDValue RHS = Node->getOperand(1);
3976  SDLoc dl(Node);
3977 
3978  SDValue Ovf;
3979 
3980  bool IsAdd = Node->getOpcode() == ISD::SADDO;
3981  unsigned CarryOp = IsAdd ? ISD::SADDO_CARRY : ISD::SSUBO_CARRY;
3982 
3983  bool HasCarryOp = TLI.isOperationLegalOrCustom(
3984  CarryOp, TLI.getTypeToExpandTo(*DAG.getContext(), LHS.getValueType()));
3985 
3986  if (HasCarryOp) {
3987  // Expand the subcomponents.
3988  SDValue LHSL, LHSH, RHSL, RHSH;
3989  GetExpandedInteger(LHS, LHSL, LHSH);
3990  GetExpandedInteger(RHS, RHSL, RHSH);
3991  SDVTList VTList = DAG.getVTList(LHSL.getValueType(), Node->getValueType(1));
3992 
3993  Lo = DAG.getNode(IsAdd ? ISD::UADDO : ISD::USUBO, dl, VTList, {LHSL, RHSL});
3994  Hi = DAG.getNode(CarryOp, dl, VTList, { LHSH, RHSH, Lo.getValue(1) });
3995 
3996  Ovf = Hi.getValue(1);
3997  } else {
3998  // Expand the result by simply replacing it with the equivalent
3999  // non-overflow-checking operation.
4000  SDValue Sum = DAG.getNode(Node->getOpcode() == ISD::SADDO ?
4001  ISD::ADD : ISD::SUB, dl, LHS.getValueType(),
4002  LHS, RHS);
4003  SplitInteger(Sum, Lo, Hi);
4004 
4005  // Compute the overflow.
4006  //
4007  // LHSSign -> LHS < 0
4008  // RHSSign -> RHS < 0
4009  // SumSign -> Sum < 0
4010  //
4011  // Add:
4012  // Overflow -> (LHSSign == RHSSign) && (LHSSign != SumSign)
4013  // Sub:
4014  // Overflow -> (LHSSign != RHSSign) && (LHSSign != SumSign)
4015  //
4016  // To get better codegen we can rewrite this by doing bitwise math on
4017  // the integers and extract the final sign bit at the end. So the
4018  // above becomes:
4019  //
4020  // Add:
4021  // Overflow -> (~(LHS ^ RHS) & (LHS ^ Sum)) < 0
4022  // Sub:
4023  // Overflow -> ((LHS ^ RHS) & (LHS ^ Sum)) < 0
4024  //
4025  // NOTE: This is different than the expansion we do in expandSADDSUBO
4026  // because it is more costly to determine the RHS is > 0 for SSUBO with the
4027  // integers split.
4028  EVT VT = LHS.getValueType();
4029  SDValue SignsMatch = DAG.getNode(ISD::XOR, dl, VT, LHS, RHS);
4030  if (IsAdd)
4031  SignsMatch = DAG.getNOT(dl, SignsMatch, VT);
4032 
4033  SDValue SumSignNE = DAG.getNode(ISD::XOR, dl, VT, LHS, Sum);
4034  Ovf = DAG.getNode(ISD::AND, dl, VT, SignsMatch, SumSignNE);
4035  EVT OType = Node->getValueType(1);
4036  Ovf = DAG.getSetCC(dl, OType, Ovf, DAG.getConstant(0, dl, VT), ISD::SETLT);
4037  }
4038 
4039  // Use the calculated overflow everywhere.
4040  ReplaceValueWith(SDValue(Node, 1), Ovf);
4041 }
4042 
4043 void DAGTypeLegalizer::ExpandIntRes_SDIV(SDNode *N,
4044  SDValue &Lo, SDValue &Hi) {
4045  EVT VT = N->getValueType(0);
4046  SDLoc dl(N);
4047  SDValue Ops[2] = { N->getOperand(0), N->getOperand(1) };
4048 
4050  SDValue Res = DAG.getNode(ISD::SDIVREM, dl, DAG.getVTList(VT, VT), Ops);
4051  SplitInteger(Res.getValue(0), Lo, Hi);
4052  return;
4053  }
4054 
4055  RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
4056  if (VT == MVT::i16)
4057  LC = RTLIB::SDIV_I16;
4058  else if (VT == MVT::i32)
4059  LC = RTLIB::SDIV_I32;
4060  else if (VT == MVT::i64)
4061  LC = RTLIB::SDIV_I64;
4062  else if (VT == MVT::i128)
4063  LC = RTLIB::SDIV_I128;
4064  assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported SDIV!");
4065 
4067  CallOptions.setSExt(true);
4068  SplitInteger(TLI.makeLibCall(DAG, LC, VT, Ops, CallOptions, dl).first, Lo, Hi);
4069 }
4070 
4071 void DAGTypeLegalizer::ExpandIntRes_Shift(SDNode *N,
4072  SDValue &Lo, SDValue &Hi) {
4073  EVT VT = N->getValueType(0);
4074  SDLoc dl(N);
4075 
4076  // If we can emit an efficient shift operation, do so now. Check to see if
4077  // the RHS is a constant.
4078  if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(N->getOperand(1)))
4079  return ExpandShiftByConstant(N, CN->getAPIntValue(), Lo, Hi);
4080 
4081  // If we can determine that the high bit of the shift is zero or one, even if
4082  // the low bits are variable, emit this shift in an optimized form.
4083  if (ExpandShiftWithKnownAmountBit(N, Lo, Hi))
4084  return;
4085 
4086  // If this target supports shift_PARTS, use it. First, map to the _PARTS opc.
4087  unsigned PartsOpc;
4088  if (N->getOpcode() == ISD::SHL) {
4089  PartsOpc = ISD::SHL_PARTS;
4090  } else if (N->getOpcode() == ISD::SRL) {
4091  PartsOpc = ISD::SRL_PARTS;
4092  } else {
4093  assert(N->getOpcode() == ISD::SRA && "Unknown shift!");
4094  PartsOpc = ISD::SRA_PARTS;
4095  }
4096 
4097  // Next check to see if the target supports this SHL_PARTS operation or if it
4098  // will custom expand it. Don't lower this to SHL_PARTS when we optimise for
4099  // size, but create a libcall instead.
4100  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
4101  TargetLowering::LegalizeAction Action = TLI.getOperationAction(PartsOpc, NVT);
4102  const bool LegalOrCustom =
4103  (Action == TargetLowering::Legal && TLI.isTypeLegal(NVT)) ||
4104  Action == TargetLowering::Custom;
4105 
4106  if (LegalOrCustom && TLI.shouldExpandShift(DAG, N)) {
4107  // Expand the subcomponents.
4108  SDValue LHSL, LHSH;
4109  GetExpandedInteger(N->getOperand(0), LHSL, LHSH);
4110  EVT VT = LHSL.getValueType();
4111 
4112  // If the shift amount operand is coming from a vector legalization it may
4113  // have an illegal type. Fix that first by casting the operand, otherwise
4114  // the new SHL_PARTS operation would need further legalization.
4115  SDValue ShiftOp = N->getOperand(1);
4116  EVT ShiftTy = TLI.getShiftAmountTy(VT, DAG.getDataLayout());
4117  if (ShiftOp.getValueType() != ShiftTy)
4118  ShiftOp = DAG.getZExtOrTrunc(ShiftOp, dl, ShiftTy);
4119 
4120  SDValue Ops[] = { LHSL, LHSH, ShiftOp };
4121  Lo = DAG.getNode(PartsOpc, dl, DAG.getVTList(VT, VT), Ops);
4122  Hi = Lo.getValue(1);
4123  return;
4124  }
4125 
4126  // Otherwise, emit a libcall.
4127  RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
4128  bool isSigned;
4129  if (N->getOpcode() == ISD::SHL) {
4130  isSigned = false; /*sign irrelevant*/
4131  if (VT == MVT::i16)
4132  LC = RTLIB::SHL_I16;
4133  else if (VT == MVT::i32)
4134  LC = RTLIB::SHL_I32;
4135  else if (VT == MVT::i64)
4136  LC = RTLIB::SHL_I64;
4137  else if (VT == MVT::i128)
4138  LC = RTLIB::SHL_I128;
4139  } else if (N->getOpcode() == ISD::SRL) {
4140  isSigned = false;
4141  if (VT == MVT::i16)
4142  LC = RTLIB::SRL_I16;
4143  else if (VT == MVT::i32)
4144  LC = RTLIB::SRL_I32;
4145  else if (VT == MVT::i64)
4146  LC = RTLIB::SRL_I64;
4147  else if (VT == MVT::i128)
4148  LC = RTLIB::SRL_I128;
4149  } else {
4150  assert(N->getOpcode() == ISD::SRA && "Unknown shift!");
4151  isSigned = true;
4152  if (VT == MVT::i16)
4153  LC = RTLIB::SRA_I16;
4154  else if (VT == MVT::i32)
4155  LC = RTLIB::SRA_I32;
4156  else if (VT == MVT::i64)
4157  LC = RTLIB::SRA_I64;
4158  else if (VT == MVT::i128)
4159  LC = RTLIB::SRA_I128;
4160  }
4161 
4162  if (LC != RTLIB::UNKNOWN_LIBCALL && TLI.getLibcallName(LC)) {
4163  EVT ShAmtTy =
4165  SDValue ShAmt = DAG.getZExtOrTrunc(N->getOperand(1), dl, ShAmtTy);
4166  SDValue Ops[2] = {N->getOperand(0), ShAmt};
4168  CallOptions.setSExt(isSigned);
4169  SplitInteger(TLI.makeLibCall(DAG, LC, VT, Ops, CallOptions, dl).first, Lo, Hi);
4170  return;
4171  }
4172 
4173  if (!ExpandShiftWithUnknownAmountBit(N, Lo, Hi))
4174  llvm_unreachable("Unsupported shift!");
4175 }
4176 
4177 void DAGTypeLegalizer::ExpandIntRes_SIGN_EXTEND(SDNode *N,
4178  SDValue &Lo, SDValue &Hi) {
4179  EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
4180  SDLoc dl(N);
4181  SDValue Op = N->getOperand(0);
4182  if (Op.getValueType().bitsLE(NVT)) {
4183  // The low part is sign extension of the input (degenerates to a copy).
4184  Lo = DAG.getNode(ISD::SIGN_EXTEND, dl, NVT, N->getOperand(0));
4185  // The high part is obtained by SRA'ing all but one of the bits of low part.
4186  unsigned LoSize = NVT.getSizeInBits();
4187  Hi = DAG.getNode(
4188  ISD::SRA, dl, NVT, Lo,
4189  DAG.getConstant(LoSize - 1, dl, TLI.getPointerTy(DAG.getDataLayout())));
4190  } else {
4191  // For example, extension of an i48 to an i64. The operand type necessarily
4192  // promotes to the result type, so will end up being expanded too.
4193  assert(getTypeAction(Op.getValueType()) ==
4195  "Only know how to promote this result!");
4196  SDValue Res = GetPromotedInteger(Op);
4197  assert(Res.getValueType() == N->getValueType(0) &&
4198  "Operand over promoted?");
4199  // Split the promoted operand. This will simplify when it is expanded.
4200  SplitInteger(Res, Lo, Hi);
4201  unsigned ExcessBits = Op.getValueSizeInBits() - NVT.getSizeInBits();
4202  Hi = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, Hi.getValueType(), Hi,
4204  ExcessBits)));
4205  }
4206 }
4207 
4208 void DAGTypeLegalizer::
4209 ExpandIntRes_SIGN_EXTEND_INREG(SDNode *N, SDValue &Lo, SDValue &Hi) {
4210  SDLoc dl(N);
4211  GetExpandedInteger(N->getOperand(0), Lo, Hi);
4212  EVT EVT = cast<VTSDNode>(N->getOperand(1))->getVT();
4213 
4214  if (EVT.bitsLE(Lo.getValueType())) {
4215  // sext_inreg the low part if needed.
4216  Lo = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, Lo.getValueType(), Lo,
4217  N->getOperand(1));
4218 
4219  // The high part gets the sign extension from the lo-part. This handles
4220  // things like sextinreg V:i64 from i8.
4221  Hi = DAG.getNode(ISD::SRA, dl, Hi.getValueType(), Lo,
4222  DAG.getConstant(Hi.getValueSizeInBits() - 1, dl,
4223  TLI.getPointerTy(DAG.getDataLayout())));
4224  } else {
4225  // For example, extension of an i48 to an i64. Leave the low part alone,
4226  // sext_inreg the high part.
4227  unsigned ExcessBits = EVT.getSizeInBits() - Lo.getValueSizeInBits();
4228  Hi = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, Hi.getValueType(), Hi,
4230  ExcessBits)));
4231  }
4232 }
4233 
4234 void DAGTypeLegalizer::ExpandIntRes_SREM(SDNode *N,
4235  SDValue &Lo, SDValue &Hi) {
4236  EVT VT = N->getValueType(0);
4237  SDLoc dl(N);
4238  SDValue Ops[2] = { N->getOperand(0), N->getOperand(1) };
4239 
4241  SDValue Res = DAG.getNode(ISD::SDIVREM, dl, DAG.getVTList(VT, VT), Ops);
4242  SplitInteger(Res.getValue(1), Lo, Hi);
4243  return;
4244  }
4245 
4246  RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
4247  if (VT == MVT::i16)
4248  LC = RTLIB::SREM_I16;
4249  else if (VT == MVT::i32)
4250  LC = RTLIB::SREM_I32;
4251  else if (VT == MVT::i64)
4252  LC = RTLIB::SREM_I64;
4253  else if (VT == MVT::i128)
4254  LC = RTLIB::SREM_I128;
4255  assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported SREM!");
4256 
4258  CallOptions.setSExt(true);
4259  SplitInteger(TLI.makeLibCall(DAG, LC, VT, Ops, CallOptions, dl).first, Lo, Hi);
4260 }