LLVM 19.0.0git
LegalizeIntegerTypes.cpp
Go to the documentation of this file.
1//===----- LegalizeIntegerTypes.cpp - Legalization of integer types -------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements integer type expansion and promotion for LegalizeTypes.
10// Promotion is the act of changing a computation in an illegal type into a
11// computation in a larger type. For example, implementing i8 arithmetic in an
12// i32 register (often needed on powerpc).
13// Expansion is the act of changing a computation in an illegal type into a
14// computation in two identical registers of a smaller type. For example,
15// implementing i64 arithmetic in two i32 registers (often needed on 32-bit
16// targets).
17//
18//===----------------------------------------------------------------------===//
19
20#include "LegalizeTypes.h"
28#include <algorithm>
29using namespace llvm;
30
31#define DEBUG_TYPE "legalize-types"
32
33//===----------------------------------------------------------------------===//
34// Integer Result Promotion
35//===----------------------------------------------------------------------===//
36
37/// PromoteIntegerResult - This method is called when a result of a node is
38/// found to be in need of promotion to a larger type. At this point, the node
39/// may also have invalid operands or may have other results that need
40/// expansion, we just know that (at least) one result needs promotion.
41void DAGTypeLegalizer::PromoteIntegerResult(SDNode *N, unsigned ResNo) {
42 LLVM_DEBUG(dbgs() << "Promote integer result: "; N->dump(&DAG));
43 SDValue Res = SDValue();
44
45 // See if the target wants to custom expand this node.
46 if (CustomLowerNode(N, N->getValueType(ResNo), true)) {
47 LLVM_DEBUG(dbgs() << "Node has been custom expanded, done\n");
48 return;
49 }
50
51 switch (N->getOpcode()) {
52 default:
53#ifndef NDEBUG
54 dbgs() << "PromoteIntegerResult #" << ResNo << ": ";
55 N->dump(&DAG); dbgs() << "\n";
56#endif
57 report_fatal_error("Do not know how to promote this operator!");
58 case ISD::MERGE_VALUES:Res = PromoteIntRes_MERGE_VALUES(N, ResNo); break;
59 case ISD::AssertSext: Res = PromoteIntRes_AssertSext(N); break;
60 case ISD::AssertZext: Res = PromoteIntRes_AssertZext(N); break;
61 case ISD::BITCAST: Res = PromoteIntRes_BITCAST(N); break;
62 case ISD::VP_BITREVERSE:
63 case ISD::BITREVERSE: Res = PromoteIntRes_BITREVERSE(N); break;
64 case ISD::VP_BSWAP:
65 case ISD::BSWAP: Res = PromoteIntRes_BSWAP(N); break;
66 case ISD::BUILD_PAIR: Res = PromoteIntRes_BUILD_PAIR(N); break;
67 case ISD::Constant: Res = PromoteIntRes_Constant(N); break;
68 case ISD::VP_CTLZ_ZERO_UNDEF:
69 case ISD::VP_CTLZ:
71 case ISD::CTLZ: Res = PromoteIntRes_CTLZ(N); break;
72 case ISD::PARITY:
73 case ISD::VP_CTPOP:
74 case ISD::CTPOP: Res = PromoteIntRes_CTPOP_PARITY(N); break;
75 case ISD::VP_CTTZ_ZERO_UNDEF:
76 case ISD::VP_CTTZ:
78 case ISD::CTTZ: Res = PromoteIntRes_CTTZ(N); break;
80 Res = PromoteIntRes_EXTRACT_VECTOR_ELT(N); break;
81 case ISD::LOAD: Res = PromoteIntRes_LOAD(cast<LoadSDNode>(N)); break;
82 case ISD::MLOAD: Res = PromoteIntRes_MLOAD(cast<MaskedLoadSDNode>(N));
83 break;
84 case ISD::MGATHER: Res = PromoteIntRes_MGATHER(cast<MaskedGatherSDNode>(N));
85 break;
86 case ISD::SELECT:
87 case ISD::VSELECT:
88 case ISD::VP_SELECT:
89 case ISD::VP_MERGE:
90 Res = PromoteIntRes_Select(N);
91 break;
92 case ISD::SELECT_CC: Res = PromoteIntRes_SELECT_CC(N); break;
95 case ISD::SETCC: Res = PromoteIntRes_SETCC(N); break;
96 case ISD::SMIN:
97 case ISD::SMAX: Res = PromoteIntRes_SExtIntBinOp(N); break;
98 case ISD::UMIN:
99 case ISD::UMAX: Res = PromoteIntRes_UMINUMAX(N); break;
100
101 case ISD::SHL:
102 case ISD::VP_SHL: Res = PromoteIntRes_SHL(N); break;
104 Res = PromoteIntRes_SIGN_EXTEND_INREG(N); break;
105 case ISD::SRA:
106 case ISD::VP_ASHR: Res = PromoteIntRes_SRA(N); break;
107 case ISD::SRL:
108 case ISD::VP_LSHR: Res = PromoteIntRes_SRL(N); break;
109 case ISD::VP_TRUNCATE:
110 case ISD::TRUNCATE: Res = PromoteIntRes_TRUNCATE(N); break;
111 case ISD::UNDEF: Res = PromoteIntRes_UNDEF(N); break;
112 case ISD::VAARG: Res = PromoteIntRes_VAARG(N); break;
113 case ISD::VSCALE: Res = PromoteIntRes_VSCALE(N); break;
114
116 Res = PromoteIntRes_EXTRACT_SUBVECTOR(N); break;
118 Res = PromoteIntRes_INSERT_SUBVECTOR(N); break;
120 Res = PromoteIntRes_VECTOR_REVERSE(N); break;
122 Res = PromoteIntRes_VECTOR_SHUFFLE(N); break;
124 Res = PromoteIntRes_VECTOR_SPLICE(N); break;
127 Res = PromoteIntRes_VECTOR_INTERLEAVE_DEINTERLEAVE(N);
128 return;
130 Res = PromoteIntRes_INSERT_VECTOR_ELT(N); break;
132 Res = PromoteIntRes_BUILD_VECTOR(N);
133 break;
136 Res = PromoteIntRes_ScalarOp(N);
137 break;
138 case ISD::STEP_VECTOR: Res = PromoteIntRes_STEP_VECTOR(N); break;
140 Res = PromoteIntRes_CONCAT_VECTORS(N); break;
141
145 Res = PromoteIntRes_EXTEND_VECTOR_INREG(N); break;
146
147 case ISD::SIGN_EXTEND:
148 case ISD::VP_SIGN_EXTEND:
149 case ISD::ZERO_EXTEND:
150 case ISD::VP_ZERO_EXTEND:
151 case ISD::ANY_EXTEND: Res = PromoteIntRes_INT_EXTEND(N); break;
152
153 case ISD::VP_FP_TO_SINT:
154 case ISD::VP_FP_TO_UINT:
157 case ISD::FP_TO_SINT:
158 case ISD::FP_TO_UINT: Res = PromoteIntRes_FP_TO_XINT(N); break;
159
162 Res = PromoteIntRes_FP_TO_XINT_SAT(N); break;
163
164 case ISD::FP_TO_BF16:
165 case ISD::FP_TO_FP16:
166 Res = PromoteIntRes_FP_TO_FP16_BF16(N);
167 break;
170 Res = PromoteIntRes_STRICT_FP_TO_FP16_BF16(N);
171 break;
172 case ISD::GET_ROUNDING: Res = PromoteIntRes_GET_ROUNDING(N); break;
173
174 case ISD::AND:
175 case ISD::OR:
176 case ISD::XOR:
177 case ISD::ADD:
178 case ISD::SUB:
179 case ISD::MUL:
180 case ISD::VP_AND:
181 case ISD::VP_OR:
182 case ISD::VP_XOR:
183 case ISD::VP_ADD:
184 case ISD::VP_SUB:
185 case ISD::VP_MUL: Res = PromoteIntRes_SimpleIntBinOp(N); break;
186
187 case ISD::VP_SMIN:
188 case ISD::VP_SMAX:
189 case ISD::SDIV:
190 case ISD::SREM:
191 case ISD::VP_SDIV:
192 case ISD::VP_SREM: Res = PromoteIntRes_SExtIntBinOp(N); break;
193
194 case ISD::VP_UMIN:
195 case ISD::VP_UMAX:
196 case ISD::UDIV:
197 case ISD::UREM:
198 case ISD::VP_UDIV:
199 case ISD::VP_UREM: Res = PromoteIntRes_ZExtIntBinOp(N); break;
200
201 case ISD::SADDO:
202 case ISD::SSUBO: Res = PromoteIntRes_SADDSUBO(N, ResNo); break;
203 case ISD::UADDO:
204 case ISD::USUBO: Res = PromoteIntRes_UADDSUBO(N, ResNo); break;
205 case ISD::SMULO:
206 case ISD::UMULO: Res = PromoteIntRes_XMULO(N, ResNo); break;
207
208 case ISD::ADDE:
209 case ISD::SUBE:
210 case ISD::UADDO_CARRY:
211 case ISD::USUBO_CARRY: Res = PromoteIntRes_UADDSUBO_CARRY(N, ResNo); break;
212
213 case ISD::SADDO_CARRY:
214 case ISD::SSUBO_CARRY: Res = PromoteIntRes_SADDSUBO_CARRY(N, ResNo); break;
215
216 case ISD::SADDSAT:
217 case ISD::UADDSAT:
218 case ISD::SSUBSAT:
219 case ISD::USUBSAT:
220 case ISD::SSHLSAT:
221 case ISD::USHLSAT:
222 Res = PromoteIntRes_ADDSUBSHLSAT<EmptyMatchContext>(N);
223 break;
224 case ISD::VP_SADDSAT:
225 case ISD::VP_UADDSAT:
226 case ISD::VP_SSUBSAT:
227 case ISD::VP_USUBSAT:
228 Res = PromoteIntRes_ADDSUBSHLSAT<VPMatchContext>(N);
229 break;
230
231 case ISD::SMULFIX:
232 case ISD::SMULFIXSAT:
233 case ISD::UMULFIX:
234 case ISD::UMULFIXSAT: Res = PromoteIntRes_MULFIX(N); break;
235
236 case ISD::SDIVFIX:
237 case ISD::SDIVFIXSAT:
238 case ISD::UDIVFIX:
239 case ISD::UDIVFIXSAT: Res = PromoteIntRes_DIVFIX(N); break;
240
241 case ISD::ABS: Res = PromoteIntRes_ABS(N); break;
242
243 case ISD::ATOMIC_LOAD:
244 Res = PromoteIntRes_Atomic0(cast<AtomicSDNode>(N)); break;
245
257 case ISD::ATOMIC_SWAP:
258 Res = PromoteIntRes_Atomic1(cast<AtomicSDNode>(N)); break;
259
262 Res = PromoteIntRes_AtomicCmpSwap(cast<AtomicSDNode>(N), ResNo);
263 break;
264
274 Res = PromoteIntRes_VECREDUCE(N);
275 break;
276
277 case ISD::VP_REDUCE_ADD:
278 case ISD::VP_REDUCE_MUL:
279 case ISD::VP_REDUCE_AND:
280 case ISD::VP_REDUCE_OR:
281 case ISD::VP_REDUCE_XOR:
282 case ISD::VP_REDUCE_SMAX:
283 case ISD::VP_REDUCE_SMIN:
284 case ISD::VP_REDUCE_UMAX:
285 case ISD::VP_REDUCE_UMIN:
286 Res = PromoteIntRes_VP_REDUCE(N);
287 break;
288
289 case ISD::FREEZE:
290 Res = PromoteIntRes_FREEZE(N);
291 break;
292
293 case ISD::ROTL:
294 case ISD::ROTR:
295 Res = PromoteIntRes_Rotate(N);
296 break;
297
298 case ISD::FSHL:
299 case ISD::FSHR:
300 Res = PromoteIntRes_FunnelShift(N);
301 break;
302
303 case ISD::VP_FSHL:
304 case ISD::VP_FSHR:
305 Res = PromoteIntRes_VPFunnelShift(N);
306 break;
307
308 case ISD::IS_FPCLASS:
309 Res = PromoteIntRes_IS_FPCLASS(N);
310 break;
311 case ISD::FFREXP:
312 Res = PromoteIntRes_FFREXP(N);
313 break;
314
315 case ISD::LRINT:
316 case ISD::LLRINT:
317 Res = PromoteIntRes_XRINT(N);
318 break;
319 }
320
321 // If the result is null then the sub-method took care of registering it.
322 if (Res.getNode())
323 SetPromotedInteger(SDValue(N, ResNo), Res);
324}
325
326SDValue DAGTypeLegalizer::PromoteIntRes_MERGE_VALUES(SDNode *N,
327 unsigned ResNo) {
328 SDValue Op = DisintegrateMERGE_VALUES(N, ResNo);
329 return GetPromotedInteger(Op);
330}
331
332SDValue DAGTypeLegalizer::PromoteIntRes_AssertSext(SDNode *N) {
333 // Sign-extend the new bits, and continue the assertion.
334 SDValue Op = SExtPromotedInteger(N->getOperand(0));
335 return DAG.getNode(ISD::AssertSext, SDLoc(N),
336 Op.getValueType(), Op, N->getOperand(1));
337}
338
339SDValue DAGTypeLegalizer::PromoteIntRes_AssertZext(SDNode *N) {
340 // Zero the new bits, and continue the assertion.
341 SDValue Op = ZExtPromotedInteger(N->getOperand(0));
342 return DAG.getNode(ISD::AssertZext, SDLoc(N),
343 Op.getValueType(), Op, N->getOperand(1));
344}
345
346SDValue DAGTypeLegalizer::PromoteIntRes_Atomic0(AtomicSDNode *N) {
347 EVT ResVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
348 SDValue Res = DAG.getAtomic(N->getOpcode(), SDLoc(N),
349 N->getMemoryVT(), ResVT,
350 N->getChain(), N->getBasePtr(),
351 N->getMemOperand());
352 if (N->getOpcode() == ISD::ATOMIC_LOAD) {
353 ISD::LoadExtType ETy = cast<AtomicSDNode>(N)->getExtensionType();
354 if (ETy == ISD::NON_EXTLOAD) {
355 switch (TLI.getExtendForAtomicOps()) {
356 case ISD::SIGN_EXTEND:
357 ETy = ISD::SEXTLOAD;
358 break;
359 case ISD::ZERO_EXTEND:
360 ETy = ISD::ZEXTLOAD;
361 break;
362 case ISD::ANY_EXTEND:
363 ETy = ISD::EXTLOAD;
364 break;
365 default:
366 llvm_unreachable("Invalid atomic op extension");
367 }
368 }
369 cast<AtomicSDNode>(Res)->setExtensionType(ETy);
370 }
371
372 // Legalize the chain result - switch anything that used the old chain to
373 // use the new one.
374 ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
375 return Res;
376}
377
378SDValue DAGTypeLegalizer::PromoteIntRes_Atomic1(AtomicSDNode *N) {
379 SDValue Op2 = GetPromotedInteger(N->getOperand(2));
380 SDValue Res = DAG.getAtomic(N->getOpcode(), SDLoc(N),
381 N->getMemoryVT(),
382 N->getChain(), N->getBasePtr(),
383 Op2, N->getMemOperand());
384 // Legalize the chain result - switch anything that used the old chain to
385 // use the new one.
386 ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
387 return Res;
388}
389
390SDValue DAGTypeLegalizer::PromoteIntRes_AtomicCmpSwap(AtomicSDNode *N,
391 unsigned ResNo) {
392 if (ResNo == 1) {
394 EVT SVT = getSetCCResultType(N->getOperand(2).getValueType());
395 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(1));
396
397 // Only use the result of getSetCCResultType if it is legal,
398 // otherwise just use the promoted result type (NVT).
399 if (!TLI.isTypeLegal(SVT))
400 SVT = NVT;
401
402 SDVTList VTs = DAG.getVTList(N->getValueType(0), SVT, MVT::Other);
403 SDValue Res = DAG.getAtomicCmpSwap(
404 ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS, SDLoc(N), N->getMemoryVT(), VTs,
405 N->getChain(), N->getBasePtr(), N->getOperand(2), N->getOperand(3),
406 N->getMemOperand());
407 ReplaceValueWith(SDValue(N, 0), Res.getValue(0));
408 ReplaceValueWith(SDValue(N, 2), Res.getValue(2));
409 return DAG.getSExtOrTrunc(Res.getValue(1), SDLoc(N), NVT);
410 }
411
412 // Op2 is used for the comparison and thus must be extended according to the
413 // target's atomic operations. Op3 is merely stored and so can be left alone.
414 SDValue Op2 = N->getOperand(2);
415 SDValue Op3 = GetPromotedInteger(N->getOperand(3));
416 switch (TLI.getExtendForAtomicCmpSwapArg()) {
417 case ISD::SIGN_EXTEND:
418 Op2 = SExtPromotedInteger(Op2);
419 break;
420 case ISD::ZERO_EXTEND:
421 Op2 = ZExtPromotedInteger(Op2);
422 break;
423 case ISD::ANY_EXTEND:
424 Op2 = GetPromotedInteger(Op2);
425 break;
426 default:
427 llvm_unreachable("Invalid atomic op extension");
428 }
429
430 SDVTList VTs =
431 DAG.getVTList(Op2.getValueType(), N->getValueType(1), MVT::Other);
432 SDValue Res = DAG.getAtomicCmpSwap(
433 N->getOpcode(), SDLoc(N), N->getMemoryVT(), VTs, N->getChain(),
434 N->getBasePtr(), Op2, Op3, N->getMemOperand());
435 // Update the use to N with the newly created Res.
436 for (unsigned i = 1, NumResults = N->getNumValues(); i < NumResults; ++i)
437 ReplaceValueWith(SDValue(N, i), Res.getValue(i));
438 return Res;
439}
440
441SDValue DAGTypeLegalizer::PromoteIntRes_BITCAST(SDNode *N) {
442 SDValue InOp = N->getOperand(0);
443 EVT InVT = InOp.getValueType();
444 EVT NInVT = TLI.getTypeToTransformTo(*DAG.getContext(), InVT);
445 EVT OutVT = N->getValueType(0);
446 EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT);
447 SDLoc dl(N);
448
449 switch (getTypeAction(InVT)) {
451 break;
453 if (NOutVT.bitsEq(NInVT) && !NOutVT.isVector() && !NInVT.isVector())
454 // The input promotes to the same size. Convert the promoted value.
455 return DAG.getNode(ISD::BITCAST, dl, NOutVT, GetPromotedInteger(InOp));
456 break;
458 // Promote the integer operand by hand.
459 return DAG.getNode(ISD::ANY_EXTEND, dl, NOutVT, GetSoftenedFloat(InOp));
461 // Promote the integer operand by hand.
462 return DAG.getNode(ISD::ANY_EXTEND, dl, NOutVT, GetSoftPromotedHalf(InOp));
464 // Convert the promoted float by hand.
465 if (!NOutVT.isVector())
466 return DAG.getNode(ISD::FP_TO_FP16, dl, NOutVT, GetPromotedFloat(InOp));
467 break;
468 }
471 break;
473 // Convert the element to an integer and promote it by hand.
474 if (!NOutVT.isVector())
475 return DAG.getNode(ISD::ANY_EXTEND, dl, NOutVT,
476 BitConvertToInteger(GetScalarizedVector(InOp)));
477 break;
479 report_fatal_error("Scalarization of scalable vectors is not supported.");
481 if (!NOutVT.isVector()) {
482 // For example, i32 = BITCAST v2i16 on alpha. Convert the split
483 // pieces of the input into integers and reassemble in the final type.
484 SDValue Lo, Hi;
485 GetSplitVector(N->getOperand(0), Lo, Hi);
486 Lo = BitConvertToInteger(Lo);
487 Hi = BitConvertToInteger(Hi);
488
489 if (DAG.getDataLayout().isBigEndian())
490 std::swap(Lo, Hi);
491
492 InOp = DAG.getNode(ISD::ANY_EXTEND, dl,
494 NOutVT.getSizeInBits()),
495 JoinIntegers(Lo, Hi));
496 return DAG.getNode(ISD::BITCAST, dl, NOutVT, InOp);
497 }
498 break;
499 }
501 // The input is widened to the same size. Convert to the widened value.
502 // Make sure that the outgoing value is not a vector, because this would
503 // make us bitcast between two vectors which are legalized in different ways.
504 if (NOutVT.bitsEq(NInVT) && !NOutVT.isVector()) {
505 SDValue Res =
506 DAG.getNode(ISD::BITCAST, dl, NOutVT, GetWidenedVector(InOp));
507
508 // For big endian targets we need to shift the casted value or the
509 // interesting bits will end up at the wrong place.
510 if (DAG.getDataLayout().isBigEndian()) {
511 unsigned ShiftAmt = NInVT.getSizeInBits() - InVT.getSizeInBits();
512 assert(ShiftAmt < NOutVT.getSizeInBits() && "Too large shift amount!");
513 Res = DAG.getNode(ISD::SRL, dl, NOutVT, Res,
514 DAG.getShiftAmountConstant(ShiftAmt, NOutVT, dl));
515 }
516 return Res;
517 }
518 // If the output type is also a vector and widening it to the same size
519 // as the widened input type would be a legal type, we can widen the bitcast
520 // and handle the promotion after.
521 if (NOutVT.isVector()) {
522 TypeSize WidenInSize = NInVT.getSizeInBits();
523 TypeSize OutSize = OutVT.getSizeInBits();
524 if (WidenInSize.hasKnownScalarFactor(OutSize)) {
525 unsigned Scale = WidenInSize.getKnownScalarFactor(OutSize);
526 EVT WideOutVT =
528 OutVT.getVectorElementCount() * Scale);
529 if (isTypeLegal(WideOutVT)) {
530 InOp = DAG.getBitcast(WideOutVT, GetWidenedVector(InOp));
531 InOp = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, OutVT, InOp,
532 DAG.getVectorIdxConstant(0, dl));
533 return DAG.getNode(ISD::ANY_EXTEND, dl, NOutVT, InOp);
534 }
535 }
536 }
537 }
538
539 return DAG.getNode(ISD::ANY_EXTEND, dl, NOutVT,
540 CreateStackStoreLoad(InOp, OutVT));
541}
542
543SDValue DAGTypeLegalizer::PromoteIntRes_FREEZE(SDNode *N) {
544 SDValue V = GetPromotedInteger(N->getOperand(0));
545 return DAG.getNode(ISD::FREEZE, SDLoc(N),
546 V.getValueType(), V);
547}
548
549SDValue DAGTypeLegalizer::PromoteIntRes_BSWAP(SDNode *N) {
550 SDValue Op = GetPromotedInteger(N->getOperand(0));
551 EVT OVT = N->getValueType(0);
552 EVT NVT = Op.getValueType();
553 SDLoc dl(N);
554
555 // If the larger BSWAP isn't supported by the target, try to expand now.
556 // If we expand later we'll end up with more operations since we lost the
557 // original type. We only do this for scalars since we have a shuffle
558 // based lowering for vectors in LegalizeVectorOps.
559 if (!OVT.isVector() &&
561 if (SDValue Res = TLI.expandBSWAP(N, DAG))
562 return DAG.getNode(ISD::ANY_EXTEND, dl, NVT, Res);
563 }
564
565 unsigned DiffBits = NVT.getScalarSizeInBits() - OVT.getScalarSizeInBits();
566 SDValue ShAmt = DAG.getShiftAmountConstant(DiffBits, NVT, dl);
567 if (N->getOpcode() == ISD::BSWAP)
568 return DAG.getNode(ISD::SRL, dl, NVT, DAG.getNode(ISD::BSWAP, dl, NVT, Op),
569 ShAmt);
570 SDValue Mask = N->getOperand(1);
571 SDValue EVL = N->getOperand(2);
572 return DAG.getNode(ISD::VP_LSHR, dl, NVT,
573 DAG.getNode(ISD::VP_BSWAP, dl, NVT, Op, Mask, EVL), ShAmt,
574 Mask, EVL);
575}
576
577SDValue DAGTypeLegalizer::PromoteIntRes_BITREVERSE(SDNode *N) {
578 SDValue Op = GetPromotedInteger(N->getOperand(0));
579 EVT OVT = N->getValueType(0);
580 EVT NVT = Op.getValueType();
581 SDLoc dl(N);
582
583 // If the larger BITREVERSE isn't supported by the target, try to expand now.
584 // If we expand later we'll end up with more operations since we lost the
585 // original type. We only do this for scalars since we have a shuffle
586 // based lowering for vectors in LegalizeVectorOps.
587 if (!OVT.isVector() && OVT.isSimple() &&
589 if (SDValue Res = TLI.expandBITREVERSE(N, DAG))
590 return DAG.getNode(ISD::ANY_EXTEND, dl, NVT, Res);
591 }
592
593 unsigned DiffBits = NVT.getScalarSizeInBits() - OVT.getScalarSizeInBits();
594 SDValue ShAmt = DAG.getShiftAmountConstant(DiffBits, NVT, dl);
595 if (N->getOpcode() == ISD::BITREVERSE)
596 return DAG.getNode(ISD::SRL, dl, NVT,
597 DAG.getNode(ISD::BITREVERSE, dl, NVT, Op), ShAmt);
598 SDValue Mask = N->getOperand(1);
599 SDValue EVL = N->getOperand(2);
600 return DAG.getNode(ISD::VP_LSHR, dl, NVT,
601 DAG.getNode(ISD::VP_BITREVERSE, dl, NVT, Op, Mask, EVL),
602 ShAmt, Mask, EVL);
603}
604
605SDValue DAGTypeLegalizer::PromoteIntRes_BUILD_PAIR(SDNode *N) {
606 // The pair element type may be legal, or may not promote to the same type as
607 // the result, for example i14 = BUILD_PAIR (i7, i7). Handle all cases.
608 return DAG.getNode(ISD::ANY_EXTEND, SDLoc(N),
610 N->getValueType(0)), JoinIntegers(N->getOperand(0),
611 N->getOperand(1)));
612}
613
614SDValue DAGTypeLegalizer::PromoteIntRes_Constant(SDNode *N) {
615 EVT VT = N->getValueType(0);
616 // FIXME there is no actual debug info here
617 SDLoc dl(N);
618 // Zero extend things like i1, sign extend everything else. It shouldn't
619 // matter in theory which one we pick, but this tends to give better code?
620 unsigned Opc = VT.isByteSized() ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND;
621 SDValue Result = DAG.getNode(Opc, dl,
622 TLI.getTypeToTransformTo(*DAG.getContext(), VT),
623 SDValue(N, 0));
624 assert(isa<ConstantSDNode>(Result) && "Didn't constant fold ext?");
625 return Result;
626}
627
628SDValue DAGTypeLegalizer::PromoteIntRes_CTLZ(SDNode *N) {
629 EVT OVT = N->getValueType(0);
630 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), OVT);
631 SDLoc dl(N);
632
633 // If the larger CTLZ isn't supported by the target, try to expand now.
634 // If we expand later we'll end up with more operations since we lost the
635 // original type.
636 if (!OVT.isVector() && TLI.isTypeLegal(NVT) &&
639 if (SDValue Result = TLI.expandCTLZ(N, DAG)) {
640 Result = DAG.getNode(ISD::ANY_EXTEND, dl, NVT, Result);
641 return Result;
642 }
643 }
644
645 // Zero extend to the promoted type and do the count there.
646 SDValue Op = ZExtPromotedInteger(N->getOperand(0));
647
648 // Subtract off the extra leading bits in the bigger type.
649 SDValue ExtractLeadingBits = DAG.getConstant(
650 NVT.getScalarSizeInBits() - OVT.getScalarSizeInBits(), dl, NVT);
651 if (!N->isVPOpcode())
652 return DAG.getNode(ISD::SUB, dl, NVT,
653 DAG.getNode(N->getOpcode(), dl, NVT, Op),
654 ExtractLeadingBits);
655 SDValue Mask = N->getOperand(1);
656 SDValue EVL = N->getOperand(2);
657 return DAG.getNode(ISD::VP_SUB, dl, NVT,
658 DAG.getNode(N->getOpcode(), dl, NVT, Op, Mask, EVL),
659 ExtractLeadingBits, Mask, EVL);
660}
661
662SDValue DAGTypeLegalizer::PromoteIntRes_CTPOP_PARITY(SDNode *N) {
663 EVT OVT = N->getValueType(0);
664 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), OVT);
665
666 // If the larger CTPOP isn't supported by the target, try to expand now.
667 // If we expand later we'll end up with more operations since we lost the
668 // original type.
669 // TODO: Expand ISD::PARITY. Need to move ExpandPARITY from LegalizeDAG to
670 // TargetLowering.
671 if (N->getOpcode() == ISD::CTPOP && !OVT.isVector() && TLI.isTypeLegal(NVT) &&
673 if (SDValue Result = TLI.expandCTPOP(N, DAG)) {
674 Result = DAG.getNode(ISD::ANY_EXTEND, SDLoc(N), NVT, Result);
675 return Result;
676 }
677 }
678
679 // Zero extend to the promoted type and do the count or parity there.
680 SDValue Op = ZExtPromotedInteger(N->getOperand(0));
681 if (!N->isVPOpcode())
682 return DAG.getNode(N->getOpcode(), SDLoc(N), Op.getValueType(), Op);
683 return DAG.getNode(N->getOpcode(), SDLoc(N), Op.getValueType(), Op,
684 N->getOperand(1), N->getOperand(2));
685}
686
687SDValue DAGTypeLegalizer::PromoteIntRes_CTTZ(SDNode *N) {
688 SDValue Op = GetPromotedInteger(N->getOperand(0));
689 EVT OVT = N->getValueType(0);
690 EVT NVT = Op.getValueType();
691 SDLoc dl(N);
692
693 // If the larger CTTZ isn't supported by the target, try to expand now.
694 // If we expand later we'll end up with more operations since we lost the
695 // original type. Don't expand if we can use CTPOP or CTLZ expansion on the
696 // larger type.
697 if (!OVT.isVector() && TLI.isTypeLegal(NVT) &&
700 !TLI.isOperationLegal(ISD::CTPOP, NVT) &&
701 !TLI.isOperationLegal(ISD::CTLZ, NVT)) {
702 if (SDValue Result = TLI.expandCTTZ(N, DAG)) {
703 Result = DAG.getNode(ISD::ANY_EXTEND, dl, NVT, Result);
704 return Result;
705 }
706 }
707
708 if (N->getOpcode() == ISD::CTTZ || N->getOpcode() == ISD::VP_CTTZ) {
709 // The count is the same in the promoted type except if the original
710 // value was zero. This can be handled by setting the bit just off
711 // the top of the original type.
712 auto TopBit = APInt::getOneBitSet(NVT.getScalarSizeInBits(),
713 OVT.getScalarSizeInBits());
714 if (N->getOpcode() == ISD::CTTZ)
715 Op = DAG.getNode(ISD::OR, dl, NVT, Op, DAG.getConstant(TopBit, dl, NVT));
716 else
717 Op =
718 DAG.getNode(ISD::VP_OR, dl, NVT, Op, DAG.getConstant(TopBit, dl, NVT),
719 N->getOperand(1), N->getOperand(2));
720 }
721 if (!N->isVPOpcode())
722 return DAG.getNode(N->getOpcode(), dl, NVT, Op);
723 return DAG.getNode(N->getOpcode(), dl, NVT, Op, N->getOperand(1),
724 N->getOperand(2));
725}
726
727SDValue DAGTypeLegalizer::PromoteIntRes_EXTRACT_VECTOR_ELT(SDNode *N) {
728 SDLoc dl(N);
729 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
730
731 SDValue Op0 = N->getOperand(0);
732 SDValue Op1 = N->getOperand(1);
733
734 // If the input also needs to be promoted, do that first so we can get a
735 // get a good idea for the output type.
736 if (TLI.getTypeAction(*DAG.getContext(), Op0.getValueType())
738 SDValue In = GetPromotedInteger(Op0);
739
740 // If the new type is larger than NVT, use it. We probably won't need to
741 // promote it again.
742 EVT SVT = In.getValueType().getScalarType();
743 if (SVT.bitsGE(NVT)) {
744 SDValue Ext = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, SVT, In, Op1);
745 return DAG.getAnyExtOrTrunc(Ext, dl, NVT);
746 }
747 }
748
749 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, NVT, Op0, Op1);
750}
751
752SDValue DAGTypeLegalizer::PromoteIntRes_FP_TO_XINT(SDNode *N) {
753 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
754 unsigned NewOpc = N->getOpcode();
755 SDLoc dl(N);
756
757 // If we're promoting a UINT to a larger size and the larger FP_TO_UINT is
758 // not Legal, check to see if we can use FP_TO_SINT instead. (If both UINT
759 // and SINT conversions are Custom, there is no way to tell which is
760 // preferable. We choose SINT because that's the right thing on PPC.)
761 if (N->getOpcode() == ISD::FP_TO_UINT &&
764 NewOpc = ISD::FP_TO_SINT;
765
766 if (N->getOpcode() == ISD::STRICT_FP_TO_UINT &&
769 NewOpc = ISD::STRICT_FP_TO_SINT;
770
771 if (N->getOpcode() == ISD::VP_FP_TO_UINT &&
772 !TLI.isOperationLegal(ISD::VP_FP_TO_UINT, NVT) &&
773 TLI.isOperationLegalOrCustom(ISD::VP_FP_TO_SINT, NVT))
774 NewOpc = ISD::VP_FP_TO_SINT;
775
776 SDValue Res;
777 if (N->isStrictFPOpcode()) {
778 Res = DAG.getNode(NewOpc, dl, {NVT, MVT::Other},
779 {N->getOperand(0), N->getOperand(1)});
780 // Legalize the chain result - switch anything that used the old chain to
781 // use the new one.
782 ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
783 } else if (NewOpc == ISD::VP_FP_TO_SINT || NewOpc == ISD::VP_FP_TO_UINT) {
784 Res = DAG.getNode(NewOpc, dl, NVT, {N->getOperand(0), N->getOperand(1),
785 N->getOperand(2)});
786 } else {
787 Res = DAG.getNode(NewOpc, dl, NVT, N->getOperand(0));
788 }
789
790 // Assert that the converted value fits in the original type. If it doesn't
791 // (eg: because the value being converted is too big), then the result of the
792 // original operation was undefined anyway, so the assert is still correct.
793 //
794 // NOTE: fp-to-uint to fp-to-sint promotion guarantees zero extend. For example:
795 // before legalization: fp-to-uint16, 65534. -> 0xfffe
796 // after legalization: fp-to-sint32, 65534. -> 0x0000fffe
797 return DAG.getNode((N->getOpcode() == ISD::FP_TO_UINT ||
798 N->getOpcode() == ISD::STRICT_FP_TO_UINT ||
799 N->getOpcode() == ISD::VP_FP_TO_UINT)
802 dl, NVT, Res,
803 DAG.getValueType(N->getValueType(0).getScalarType()));
804}
805
806SDValue DAGTypeLegalizer::PromoteIntRes_FP_TO_XINT_SAT(SDNode *N) {
807 // Promote the result type, while keeping the original width in Op1.
808 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
809 SDLoc dl(N);
810 return DAG.getNode(N->getOpcode(), dl, NVT, N->getOperand(0),
811 N->getOperand(1));
812}
813
814SDValue DAGTypeLegalizer::PromoteIntRes_FP_TO_FP16_BF16(SDNode *N) {
815 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
816 SDLoc dl(N);
817
818 return DAG.getNode(N->getOpcode(), dl, NVT, N->getOperand(0));
819}
820
821SDValue DAGTypeLegalizer::PromoteIntRes_STRICT_FP_TO_FP16_BF16(SDNode *N) {
822 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
823 SDLoc dl(N);
824
825 SDValue Res = DAG.getNode(N->getOpcode(), dl, DAG.getVTList(NVT, MVT::Other),
826 N->getOperand(0), N->getOperand(1));
827 ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
828 return Res;
829}
830
831SDValue DAGTypeLegalizer::PromoteIntRes_XRINT(SDNode *N) {
832 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
833 SDLoc dl(N);
834 return DAG.getNode(N->getOpcode(), dl, NVT, N->getOperand(0));
835}
836
837SDValue DAGTypeLegalizer::PromoteIntRes_GET_ROUNDING(SDNode *N) {
838 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
839 SDLoc dl(N);
840
841 SDValue Res =
842 DAG.getNode(N->getOpcode(), dl, {NVT, MVT::Other}, N->getOperand(0));
843
844 // Legalize the chain result - switch anything that used the old chain to
845 // use the new one.
846 ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
847 return Res;
848}
849
850SDValue DAGTypeLegalizer::PromoteIntRes_INT_EXTEND(SDNode *N) {
851 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
852 SDLoc dl(N);
853
854 if (getTypeAction(N->getOperand(0).getValueType())
856 SDValue Res = GetPromotedInteger(N->getOperand(0));
857 assert(Res.getValueType().bitsLE(NVT) && "Extension doesn't make sense!");
858
859 // If the result and operand types are the same after promotion, simplify
860 // to an in-register extension. Unless this is a VP_*_EXTEND.
861 if (NVT == Res.getValueType() && N->getNumOperands() == 1) {
862 // The high bits are not guaranteed to be anything. Insert an extend.
863 if (N->getOpcode() == ISD::SIGN_EXTEND)
864 return DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, NVT, Res,
865 DAG.getValueType(N->getOperand(0).getValueType()));
866 if (N->getOpcode() == ISD::ZERO_EXTEND)
867 return DAG.getZeroExtendInReg(Res, dl, N->getOperand(0).getValueType());
868 assert(N->getOpcode() == ISD::ANY_EXTEND && "Unknown integer extension!");
869 return Res;
870 }
871 }
872
873 // Otherwise, just extend the original operand all the way to the larger type.
874 if (N->getNumOperands() != 1) {
875 assert(N->getNumOperands() == 3 && "Unexpected number of operands!");
876 assert(N->isVPOpcode() && "Expected VP opcode");
877 return DAG.getNode(N->getOpcode(), dl, NVT, N->getOperand(0),
878 N->getOperand(1), N->getOperand(2));
879 }
880 return DAG.getNode(N->getOpcode(), dl, NVT, N->getOperand(0));
881}
882
883SDValue DAGTypeLegalizer::PromoteIntRes_LOAD(LoadSDNode *N) {
884 assert(ISD::isUNINDEXEDLoad(N) && "Indexed load during type legalization!");
885 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
886 ISD::LoadExtType ExtType =
887 ISD::isNON_EXTLoad(N) ? ISD::EXTLOAD : N->getExtensionType();
888 SDLoc dl(N);
889 SDValue Res = DAG.getExtLoad(ExtType, dl, NVT, N->getChain(), N->getBasePtr(),
890 N->getMemoryVT(), N->getMemOperand());
891
892 // Legalize the chain result - switch anything that used the old chain to
893 // use the new one.
894 ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
895 return Res;
896}
897
898SDValue DAGTypeLegalizer::PromoteIntRes_MLOAD(MaskedLoadSDNode *N) {
899 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
900 SDValue ExtPassThru = GetPromotedInteger(N->getPassThru());
901
902 ISD::LoadExtType ExtType = N->getExtensionType();
903 if (ExtType == ISD::NON_EXTLOAD)
904 ExtType = ISD::EXTLOAD;
905
906 SDLoc dl(N);
907 SDValue Res = DAG.getMaskedLoad(NVT, dl, N->getChain(), N->getBasePtr(),
908 N->getOffset(), N->getMask(), ExtPassThru,
909 N->getMemoryVT(), N->getMemOperand(),
910 N->getAddressingMode(), ExtType,
911 N->isExpandingLoad());
912 // Legalize the chain result - switch anything that used the old chain to
913 // use the new one.
914 ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
915 return Res;
916}
917
918SDValue DAGTypeLegalizer::PromoteIntRes_MGATHER(MaskedGatherSDNode *N) {
919 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
920 SDValue ExtPassThru = GetPromotedInteger(N->getPassThru());
921 assert(NVT == ExtPassThru.getValueType() &&
922 "Gather result type and the passThru argument type should be the same");
923
924 ISD::LoadExtType ExtType = N->getExtensionType();
925 if (ExtType == ISD::NON_EXTLOAD)
926 ExtType = ISD::EXTLOAD;
927
928 SDLoc dl(N);
929 SDValue Ops[] = {N->getChain(), ExtPassThru, N->getMask(), N->getBasePtr(),
930 N->getIndex(), N->getScale() };
931 SDValue Res = DAG.getMaskedGather(DAG.getVTList(NVT, MVT::Other),
932 N->getMemoryVT(), dl, Ops,
933 N->getMemOperand(), N->getIndexType(),
934 ExtType);
935 // Legalize the chain result - switch anything that used the old chain to
936 // use the new one.
937 ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
938 return Res;
939}
940
941/// Promote the overflow flag of an overflowing arithmetic node.
942SDValue DAGTypeLegalizer::PromoteIntRes_Overflow(SDNode *N) {
943 // Change the return type of the boolean result while obeying
944 // getSetCCResultType.
945 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(1));
946 EVT VT = N->getValueType(0);
947 EVT SVT = getSetCCResultType(VT);
948 SDValue Ops[3] = { N->getOperand(0), N->getOperand(1) };
949 unsigned NumOps = N->getNumOperands();
950 assert(NumOps <= 3 && "Too many operands");
951 if (NumOps == 3)
952 Ops[2] = N->getOperand(2);
953
954 SDLoc dl(N);
955 SDValue Res = DAG.getNode(N->getOpcode(), dl, DAG.getVTList(VT, SVT),
956 ArrayRef(Ops, NumOps));
957
958 // Modified the sum result - switch anything that used the old sum to use
959 // the new one.
960 ReplaceValueWith(SDValue(N, 0), Res);
961
962 // Convert to the expected type.
963 return DAG.getBoolExtOrTrunc(Res.getValue(1), dl, NVT, VT);
964}
965
966template <class MatchContextClass>
967SDValue DAGTypeLegalizer::PromoteIntRes_ADDSUBSHLSAT(SDNode *N) {
968 // If the promoted type is legal, we can convert this to:
969 // 1. ANY_EXTEND iN to iM
970 // 2. SHL by M-N
971 // 3. [US][ADD|SUB|SHL]SAT
972 // 4. L/ASHR by M-N
973 // Else it is more efficient to convert this to a min and a max
974 // operation in the higher precision arithmetic.
975 SDLoc dl(N);
976 SDValue Op1 = N->getOperand(0);
977 SDValue Op2 = N->getOperand(1);
978 MatchContextClass matcher(DAG, TLI, N);
979 unsigned OldBits = Op1.getScalarValueSizeInBits();
980
981 unsigned Opcode = matcher.getRootBaseOpcode();
982 bool IsShift = Opcode == ISD::USHLSAT || Opcode == ISD::SSHLSAT;
983
984 // FIXME: We need vp-aware PromotedInteger functions.
985 SDValue Op1Promoted, Op2Promoted;
986 if (IsShift) {
987 Op1Promoted = GetPromotedInteger(Op1);
988 Op2Promoted = ZExtPromotedInteger(Op2);
989 } else if (Opcode == ISD::UADDSAT || Opcode == ISD::USUBSAT) {
990 Op1Promoted = ZExtPromotedInteger(Op1);
991 Op2Promoted = ZExtPromotedInteger(Op2);
992 } else {
993 Op1Promoted = SExtPromotedInteger(Op1);
994 Op2Promoted = SExtPromotedInteger(Op2);
995 }
996 EVT PromotedType = Op1Promoted.getValueType();
997 unsigned NewBits = PromotedType.getScalarSizeInBits();
998
999 if (Opcode == ISD::UADDSAT) {
1000 APInt MaxVal = APInt::getAllOnes(OldBits).zext(NewBits);
1001 SDValue SatMax = DAG.getConstant(MaxVal, dl, PromotedType);
1002 SDValue Add =
1003 matcher.getNode(ISD::ADD, dl, PromotedType, Op1Promoted, Op2Promoted);
1004 return matcher.getNode(ISD::UMIN, dl, PromotedType, Add, SatMax);
1005 }
1006
1007 // USUBSAT can always be promoted as long as we have zero-extended the args.
1008 if (Opcode == ISD::USUBSAT)
1009 return matcher.getNode(ISD::USUBSAT, dl, PromotedType, Op1Promoted,
1010 Op2Promoted);
1011
1012 // Shift cannot use a min/max expansion, we can't detect overflow if all of
1013 // the bits have been shifted out.
1014 if (IsShift || matcher.isOperationLegal(Opcode, PromotedType)) {
1015 unsigned ShiftOp;
1016 switch (Opcode) {
1017 case ISD::SADDSAT:
1018 case ISD::SSUBSAT:
1019 case ISD::SSHLSAT:
1020 ShiftOp = ISD::SRA;
1021 break;
1022 case ISD::USHLSAT:
1023 ShiftOp = ISD::SRL;
1024 break;
1025 default:
1026 llvm_unreachable("Expected opcode to be signed or unsigned saturation "
1027 "addition, subtraction or left shift");
1028 }
1029
1030 unsigned SHLAmount = NewBits - OldBits;
1031 SDValue ShiftAmount =
1032 DAG.getShiftAmountConstant(SHLAmount, PromotedType, dl);
1033 Op1Promoted =
1034 DAG.getNode(ISD::SHL, dl, PromotedType, Op1Promoted, ShiftAmount);
1035 if (!IsShift)
1036 Op2Promoted =
1037 matcher.getNode(ISD::SHL, dl, PromotedType, Op2Promoted, ShiftAmount);
1038
1039 SDValue Result =
1040 matcher.getNode(Opcode, dl, PromotedType, Op1Promoted, Op2Promoted);
1041 return matcher.getNode(ShiftOp, dl, PromotedType, Result, ShiftAmount);
1042 }
1043
1044 unsigned AddOp = Opcode == ISD::SADDSAT ? ISD::ADD : ISD::SUB;
1045 APInt MinVal = APInt::getSignedMinValue(OldBits).sext(NewBits);
1046 APInt MaxVal = APInt::getSignedMaxValue(OldBits).sext(NewBits);
1047 SDValue SatMin = DAG.getConstant(MinVal, dl, PromotedType);
1048 SDValue SatMax = DAG.getConstant(MaxVal, dl, PromotedType);
1049 SDValue Result =
1050 matcher.getNode(AddOp, dl, PromotedType, Op1Promoted, Op2Promoted);
1051 Result = matcher.getNode(ISD::SMIN, dl, PromotedType, Result, SatMax);
1052 Result = matcher.getNode(ISD::SMAX, dl, PromotedType, Result, SatMin);
1053 return Result;
1054}
1055
1056SDValue DAGTypeLegalizer::PromoteIntRes_MULFIX(SDNode *N) {
1057 // Can just promote the operands then continue with operation.
1058 SDLoc dl(N);
1059 SDValue Op1Promoted, Op2Promoted;
1060 bool Signed =
1061 N->getOpcode() == ISD::SMULFIX || N->getOpcode() == ISD::SMULFIXSAT;
1062 bool Saturating =
1063 N->getOpcode() == ISD::SMULFIXSAT || N->getOpcode() == ISD::UMULFIXSAT;
1064 if (Signed) {
1065 Op1Promoted = SExtPromotedInteger(N->getOperand(0));
1066 Op2Promoted = SExtPromotedInteger(N->getOperand(1));
1067 } else {
1068 Op1Promoted = ZExtPromotedInteger(N->getOperand(0));
1069 Op2Promoted = ZExtPromotedInteger(N->getOperand(1));
1070 }
1071 EVT OldType = N->getOperand(0).getValueType();
1072 EVT PromotedType = Op1Promoted.getValueType();
1073 unsigned DiffSize =
1074 PromotedType.getScalarSizeInBits() - OldType.getScalarSizeInBits();
1075
1076 if (Saturating) {
1077 // Promoting the operand and result values changes the saturation width,
1078 // which is extends the values that we clamp to on saturation. This could be
1079 // resolved by shifting one of the operands the same amount, which would
1080 // also shift the result we compare against, then shifting back.
1081 Op1Promoted =
1082 DAG.getNode(ISD::SHL, dl, PromotedType, Op1Promoted,
1083 DAG.getShiftAmountConstant(DiffSize, PromotedType, dl));
1084 SDValue Result = DAG.getNode(N->getOpcode(), dl, PromotedType, Op1Promoted,
1085 Op2Promoted, N->getOperand(2));
1086 unsigned ShiftOp = Signed ? ISD::SRA : ISD::SRL;
1087 return DAG.getNode(ShiftOp, dl, PromotedType, Result,
1088 DAG.getShiftAmountConstant(DiffSize, PromotedType, dl));
1089 }
1090 return DAG.getNode(N->getOpcode(), dl, PromotedType, Op1Promoted, Op2Promoted,
1091 N->getOperand(2));
1092}
1093
1095 unsigned SatW, bool Signed,
1096 const TargetLowering &TLI,
1097 SelectionDAG &DAG) {
1098 EVT VT = V.getValueType();
1099 unsigned VTW = VT.getScalarSizeInBits();
1100
1101 if (!Signed) {
1102 // Saturate to the unsigned maximum by getting the minimum of V and the
1103 // maximum.
1104 return DAG.getNode(ISD::UMIN, dl, VT, V,
1105 DAG.getConstant(APInt::getLowBitsSet(VTW, SatW),
1106 dl, VT));
1107 }
1108
1109 // Saturate to the signed maximum (the low SatW - 1 bits) by taking the
1110 // signed minimum of it and V.
1111 V = DAG.getNode(ISD::SMIN, dl, VT, V,
1112 DAG.getConstant(APInt::getLowBitsSet(VTW, SatW - 1),
1113 dl, VT));
1114 // Saturate to the signed minimum (the high SatW + 1 bits) by taking the
1115 // signed maximum of it and V.
1116 V = DAG.getNode(ISD::SMAX, dl, VT, V,
1117 DAG.getConstant(APInt::getHighBitsSet(VTW, VTW - SatW + 1),
1118 dl, VT));
1119 return V;
1120}
1121
1123 unsigned Scale, const TargetLowering &TLI,
1124 SelectionDAG &DAG, unsigned SatW = 0) {
1125 EVT VT = LHS.getValueType();
1126 unsigned VTSize = VT.getScalarSizeInBits();
1127 bool Signed = N->getOpcode() == ISD::SDIVFIX ||
1128 N->getOpcode() == ISD::SDIVFIXSAT;
1129 bool Saturating = N->getOpcode() == ISD::SDIVFIXSAT ||
1130 N->getOpcode() == ISD::UDIVFIXSAT;
1131
1132 SDLoc dl(N);
1133 // Widen the types by a factor of two. This is guaranteed to expand, since it
1134 // will always have enough high bits in the LHS to shift into.
1135 EVT WideVT = EVT::getIntegerVT(*DAG.getContext(), VTSize * 2);
1136 if (VT.isVector())
1137 WideVT = EVT::getVectorVT(*DAG.getContext(), WideVT,
1139 LHS = DAG.getExtOrTrunc(Signed, LHS, dl, WideVT);
1140 RHS = DAG.getExtOrTrunc(Signed, RHS, dl, WideVT);
1141 SDValue Res = TLI.expandFixedPointDiv(N->getOpcode(), dl, LHS, RHS, Scale,
1142 DAG);
1143 assert(Res && "Expanding DIVFIX with wide type failed?");
1144 if (Saturating) {
1145 // If the caller has told us to saturate at something less, use that width
1146 // instead of the type before doubling. However, it cannot be more than
1147 // what we just widened!
1148 assert(SatW <= VTSize &&
1149 "Tried to saturate to more than the original type?");
1150 Res = SaturateWidenedDIVFIX(Res, dl, SatW == 0 ? VTSize : SatW, Signed,
1151 TLI, DAG);
1152 }
1153 return DAG.getZExtOrTrunc(Res, dl, VT);
1154}
1155
1156SDValue DAGTypeLegalizer::PromoteIntRes_DIVFIX(SDNode *N) {
1157 SDLoc dl(N);
1158 SDValue Op1Promoted, Op2Promoted;
1159 bool Signed = N->getOpcode() == ISD::SDIVFIX ||
1160 N->getOpcode() == ISD::SDIVFIXSAT;
1161 bool Saturating = N->getOpcode() == ISD::SDIVFIXSAT ||
1162 N->getOpcode() == ISD::UDIVFIXSAT;
1163 if (Signed) {
1164 Op1Promoted = SExtPromotedInteger(N->getOperand(0));
1165 Op2Promoted = SExtPromotedInteger(N->getOperand(1));
1166 } else {
1167 Op1Promoted = ZExtPromotedInteger(N->getOperand(0));
1168 Op2Promoted = ZExtPromotedInteger(N->getOperand(1));
1169 }
1170 EVT PromotedType = Op1Promoted.getValueType();
1171 unsigned Scale = N->getConstantOperandVal(2);
1172
1173 // If the type is already legal and the operation is legal in that type, we
1174 // should not early expand.
1175 if (TLI.isTypeLegal(PromotedType)) {
1177 TLI.getFixedPointOperationAction(N->getOpcode(), PromotedType, Scale);
1178 if (Action == TargetLowering::Legal || Action == TargetLowering::Custom) {
1179 unsigned Diff = PromotedType.getScalarSizeInBits() -
1180 N->getValueType(0).getScalarSizeInBits();
1181 if (Saturating)
1182 Op1Promoted =
1183 DAG.getNode(ISD::SHL, dl, PromotedType, Op1Promoted,
1184 DAG.getShiftAmountConstant(Diff, PromotedType, dl));
1185 SDValue Res = DAG.getNode(N->getOpcode(), dl, PromotedType, Op1Promoted,
1186 Op2Promoted, N->getOperand(2));
1187 if (Saturating)
1188 Res = DAG.getNode(Signed ? ISD::SRA : ISD::SRL, dl, PromotedType, Res,
1189 DAG.getShiftAmountConstant(Diff, PromotedType, dl));
1190 return Res;
1191 }
1192 }
1193
1194 // See if we can perform the division in this type without expanding.
1195 if (SDValue Res = TLI.expandFixedPointDiv(N->getOpcode(), dl, Op1Promoted,
1196 Op2Promoted, Scale, DAG)) {
1197 if (Saturating)
1198 Res = SaturateWidenedDIVFIX(Res, dl,
1199 N->getValueType(0).getScalarSizeInBits(),
1200 Signed, TLI, DAG);
1201 return Res;
1202 }
1203 // If we cannot, expand it to twice the type width. If we are saturating, give
1204 // it the original width as a saturating width so we don't need to emit
1205 // two saturations.
1206 return earlyExpandDIVFIX(N, Op1Promoted, Op2Promoted, Scale, TLI, DAG,
1207 N->getValueType(0).getScalarSizeInBits());
1208}
1209
1210SDValue DAGTypeLegalizer::PromoteIntRes_SADDSUBO(SDNode *N, unsigned ResNo) {
1211 if (ResNo == 1)
1212 return PromoteIntRes_Overflow(N);
1213
1214 // The operation overflowed iff the result in the larger type is not the
1215 // sign extension of its truncation to the original type.
1216 SDValue LHS = SExtPromotedInteger(N->getOperand(0));
1217 SDValue RHS = SExtPromotedInteger(N->getOperand(1));
1218 EVT OVT = N->getOperand(0).getValueType();
1219 EVT NVT = LHS.getValueType();
1220 SDLoc dl(N);
1221
1222 // Do the arithmetic in the larger type.
1223 unsigned Opcode = N->getOpcode() == ISD::SADDO ? ISD::ADD : ISD::SUB;
1224 SDValue Res = DAG.getNode(Opcode, dl, NVT, LHS, RHS);
1225
1226 // Calculate the overflow flag: sign extend the arithmetic result from
1227 // the original type.
1228 SDValue Ofl = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, NVT, Res,
1229 DAG.getValueType(OVT));
1230 // Overflowed if and only if this is not equal to Res.
1231 Ofl = DAG.getSetCC(dl, N->getValueType(1), Ofl, Res, ISD::SETNE);
1232
1233 // Use the calculated overflow everywhere.
1234 ReplaceValueWith(SDValue(N, 1), Ofl);
1235
1236 return Res;
1237}
1238
1239SDValue DAGTypeLegalizer::PromoteIntRes_Select(SDNode *N) {
1240 SDValue Mask = N->getOperand(0);
1241
1242 SDValue LHS = GetPromotedInteger(N->getOperand(1));
1243 SDValue RHS = GetPromotedInteger(N->getOperand(2));
1244
1245 unsigned Opcode = N->getOpcode();
1246 if (Opcode == ISD::VP_SELECT || Opcode == ISD::VP_MERGE)
1247 return DAG.getNode(Opcode, SDLoc(N), LHS.getValueType(), Mask, LHS, RHS,
1248 N->getOperand(3));
1249 return DAG.getNode(Opcode, SDLoc(N), LHS.getValueType(), Mask, LHS, RHS);
1250}
1251
1252SDValue DAGTypeLegalizer::PromoteIntRes_SELECT_CC(SDNode *N) {
1253 SDValue LHS = GetPromotedInteger(N->getOperand(2));
1254 SDValue RHS = GetPromotedInteger(N->getOperand(3));
1255 return DAG.getNode(ISD::SELECT_CC, SDLoc(N),
1256 LHS.getValueType(), N->getOperand(0),
1257 N->getOperand(1), LHS, RHS, N->getOperand(4));
1258}
1259
1260SDValue DAGTypeLegalizer::PromoteIntRes_SETCC(SDNode *N) {
1261 unsigned OpNo = N->isStrictFPOpcode() ? 1 : 0;
1262 EVT InVT = N->getOperand(OpNo).getValueType();
1263 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
1264
1265 EVT SVT = getSetCCResultType(InVT);
1266
1267 // If we got back a type that needs to be promoted, this likely means the
1268 // the input type also needs to be promoted. So get the promoted type for
1269 // the input and try the query again.
1270 if (getTypeAction(SVT) == TargetLowering::TypePromoteInteger) {
1271 if (getTypeAction(InVT) == TargetLowering::TypePromoteInteger) {
1272 InVT = TLI.getTypeToTransformTo(*DAG.getContext(), InVT);
1273 SVT = getSetCCResultType(InVT);
1274 } else {
1275 // Input type isn't promoted, just use the default promoted type.
1276 SVT = NVT;
1277 }
1278 }
1279
1280 SDLoc dl(N);
1281 assert(SVT.isVector() == N->getOperand(OpNo).getValueType().isVector() &&
1282 "Vector compare must return a vector result!");
1283
1284 // Get the SETCC result using the canonical SETCC type.
1285 SDValue SetCC;
1286 if (N->isStrictFPOpcode()) {
1287 SDVTList VTs = DAG.getVTList({SVT, MVT::Other});
1288 SDValue Opers[] = {N->getOperand(0), N->getOperand(1),
1289 N->getOperand(2), N->getOperand(3)};
1290 SetCC = DAG.getNode(N->getOpcode(), dl, VTs, Opers, N->getFlags());
1291 // Legalize the chain result - switch anything that used the old chain to
1292 // use the new one.
1293 ReplaceValueWith(SDValue(N, 1), SetCC.getValue(1));
1294 } else
1295 SetCC = DAG.getNode(N->getOpcode(), dl, SVT, N->getOperand(0),
1296 N->getOperand(1), N->getOperand(2), N->getFlags());
1297
1298 // Convert to the expected type.
1299 return DAG.getSExtOrTrunc(SetCC, dl, NVT);
1300}
1301
1302SDValue DAGTypeLegalizer::PromoteIntRes_IS_FPCLASS(SDNode *N) {
1303 SDLoc DL(N);
1304 SDValue Arg = N->getOperand(0);
1305 SDValue Test = N->getOperand(1);
1306 EVT NResVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
1307 return DAG.getNode(ISD::IS_FPCLASS, DL, NResVT, Arg, Test);
1308}
1309
1310SDValue DAGTypeLegalizer::PromoteIntRes_FFREXP(SDNode *N) {
1311 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(1));
1312 EVT VT = N->getValueType(0);
1313
1314 SDLoc dl(N);
1315 SDValue Res =
1316 DAG.getNode(N->getOpcode(), dl, DAG.getVTList(VT, NVT), N->getOperand(0));
1317
1318 ReplaceValueWith(SDValue(N, 0), Res);
1319 return Res.getValue(1);
1320}
1321
1322SDValue DAGTypeLegalizer::PromoteIntRes_SHL(SDNode *N) {
1323 SDValue LHS = GetPromotedInteger(N->getOperand(0));
1324 SDValue RHS = N->getOperand(1);
1325 if (getTypeAction(RHS.getValueType()) == TargetLowering::TypePromoteInteger)
1326 RHS = ZExtPromotedInteger(RHS);
1327 if (N->getOpcode() != ISD::VP_SHL)
1328 return DAG.getNode(N->getOpcode(), SDLoc(N), LHS.getValueType(), LHS, RHS);
1329 return DAG.getNode(N->getOpcode(), SDLoc(N), LHS.getValueType(), LHS, RHS,
1330 N->getOperand(2), N->getOperand(3));
1331}
1332
1333SDValue DAGTypeLegalizer::PromoteIntRes_SIGN_EXTEND_INREG(SDNode *N) {
1334 SDValue Op = GetPromotedInteger(N->getOperand(0));
1336 Op.getValueType(), Op, N->getOperand(1));
1337}
1338
1339SDValue DAGTypeLegalizer::PromoteIntRes_SimpleIntBinOp(SDNode *N) {
1340 // The input may have strange things in the top bits of the registers, but
1341 // these operations don't care. They may have weird bits going out, but
1342 // that too is okay if they are integer operations.
1343 SDValue LHS = GetPromotedInteger(N->getOperand(0));
1344 SDValue RHS = GetPromotedInteger(N->getOperand(1));
1345 if (N->getNumOperands() == 2)
1346 return DAG.getNode(N->getOpcode(), SDLoc(N), LHS.getValueType(), LHS, RHS);
1347 assert(N->getNumOperands() == 4 && "Unexpected number of operands!");
1348 assert(N->isVPOpcode() && "Expected VP opcode");
1349 return DAG.getNode(N->getOpcode(), SDLoc(N), LHS.getValueType(), LHS, RHS,
1350 N->getOperand(2), N->getOperand(3));
1351}
1352
1353SDValue DAGTypeLegalizer::PromoteIntRes_SExtIntBinOp(SDNode *N) {
1354 // Sign extend the input.
1355 SDValue LHS = SExtPromotedInteger(N->getOperand(0));
1356 SDValue RHS = SExtPromotedInteger(N->getOperand(1));
1357 if (N->getNumOperands() == 2)
1358 return DAG.getNode(N->getOpcode(), SDLoc(N), LHS.getValueType(), LHS, RHS);
1359 assert(N->getNumOperands() == 4 && "Unexpected number of operands!");
1360 assert(N->isVPOpcode() && "Expected VP opcode");
1361 return DAG.getNode(N->getOpcode(), SDLoc(N), LHS.getValueType(), LHS, RHS,
1362 N->getOperand(2), N->getOperand(3));
1363}
1364
1365SDValue DAGTypeLegalizer::PromoteIntRes_ZExtIntBinOp(SDNode *N) {
1366 // Zero extend the input.
1367 SDValue LHS = ZExtPromotedInteger(N->getOperand(0));
1368 SDValue RHS = ZExtPromotedInteger(N->getOperand(1));
1369 if (N->getNumOperands() == 2)
1370 return DAG.getNode(N->getOpcode(), SDLoc(N), LHS.getValueType(), LHS, RHS);
1371 assert(N->getNumOperands() == 4 && "Unexpected number of operands!");
1372 assert(N->isVPOpcode() && "Expected VP opcode");
1373 return DAG.getNode(N->getOpcode(), SDLoc(N), LHS.getValueType(), LHS, RHS,
1374 N->getOperand(2), N->getOperand(3));
1375}
1376
1377SDValue DAGTypeLegalizer::PromoteIntRes_UMINUMAX(SDNode *N) {
1378 SDValue LHS = N->getOperand(0);
1379 SDValue RHS = N->getOperand(1);
1380
1381 // It doesn't matter if we sign extend or zero extend in the inputs. So do
1382 // whatever is best for the target and the promoted operands.
1383 SExtOrZExtPromotedOperands(LHS, RHS);
1384
1385 return DAG.getNode(N->getOpcode(), SDLoc(N),
1386 LHS.getValueType(), LHS, RHS);
1387}
1388
1389SDValue DAGTypeLegalizer::PromoteIntRes_SRA(SDNode *N) {
1390 // The input value must be properly sign extended.
1391 SDValue LHS = SExtPromotedInteger(N->getOperand(0));
1392 SDValue RHS = N->getOperand(1);
1393 if (getTypeAction(RHS.getValueType()) == TargetLowering::TypePromoteInteger)
1394 RHS = ZExtPromotedInteger(RHS);
1395 if (N->getOpcode() != ISD::VP_ASHR)
1396 return DAG.getNode(N->getOpcode(), SDLoc(N), LHS.getValueType(), LHS, RHS);
1397 return DAG.getNode(N->getOpcode(), SDLoc(N), LHS.getValueType(), LHS, RHS,
1398 N->getOperand(2), N->getOperand(3));
1399}
1400
1401SDValue DAGTypeLegalizer::PromoteIntRes_SRL(SDNode *N) {
1402 // The input value must be properly zero extended.
1403 SDValue LHS = ZExtPromotedInteger(N->getOperand(0));
1404 SDValue RHS = N->getOperand(1);
1405 if (getTypeAction(RHS.getValueType()) == TargetLowering::TypePromoteInteger)
1406 RHS = ZExtPromotedInteger(RHS);
1407 if (N->getOpcode() != ISD::VP_LSHR)
1408 return DAG.getNode(N->getOpcode(), SDLoc(N), LHS.getValueType(), LHS, RHS);
1409 return DAG.getNode(N->getOpcode(), SDLoc(N), LHS.getValueType(), LHS, RHS,
1410 N->getOperand(2), N->getOperand(3));
1411}
1412
1413SDValue DAGTypeLegalizer::PromoteIntRes_Rotate(SDNode *N) {
1414 // Lower the rotate to shifts and ORs which can be promoted.
1415 SDValue Res = TLI.expandROT(N, true /*AllowVectorOps*/, DAG);
1416 ReplaceValueWith(SDValue(N, 0), Res);
1417 return SDValue();
1418}
1419
1420SDValue DAGTypeLegalizer::PromoteIntRes_FunnelShift(SDNode *N) {
1421 SDValue Hi = GetPromotedInteger(N->getOperand(0));
1422 SDValue Lo = GetPromotedInteger(N->getOperand(1));
1423 SDValue Amt = N->getOperand(2);
1424 if (getTypeAction(Amt.getValueType()) == TargetLowering::TypePromoteInteger)
1425 Amt = ZExtPromotedInteger(Amt);
1426 EVT AmtVT = Amt.getValueType();
1427
1428 SDLoc DL(N);
1429 EVT OldVT = N->getOperand(0).getValueType();
1430 EVT VT = Lo.getValueType();
1431 unsigned Opcode = N->getOpcode();
1432 bool IsFSHR = Opcode == ISD::FSHR;
1433 unsigned OldBits = OldVT.getScalarSizeInBits();
1434 unsigned NewBits = VT.getScalarSizeInBits();
1435
1436 // Amount has to be interpreted modulo the old bit width.
1437 Amt = DAG.getNode(ISD::UREM, DL, AmtVT, Amt,
1438 DAG.getConstant(OldBits, DL, AmtVT));
1439
1440 // If the promoted type is twice the size (or more), then we use the
1441 // traditional funnel 'double' shift codegen. This isn't necessary if the
1442 // shift amount is constant.
1443 // fshl(x,y,z) -> (((aext(x) << bw) | zext(y)) << (z % bw)) >> bw.
1444 // fshr(x,y,z) -> (((aext(x) << bw) | zext(y)) >> (z % bw)).
1445 if (NewBits >= (2 * OldBits) && !isa<ConstantSDNode>(Amt) &&
1446 !TLI.isOperationLegalOrCustom(Opcode, VT)) {
1447 SDValue HiShift = DAG.getConstant(OldBits, DL, VT);
1448 Hi = DAG.getNode(ISD::SHL, DL, VT, Hi, HiShift);
1449 Lo = DAG.getZeroExtendInReg(Lo, DL, OldVT);
1450 SDValue Res = DAG.getNode(ISD::OR, DL, VT, Hi, Lo);
1451 Res = DAG.getNode(IsFSHR ? ISD::SRL : ISD::SHL, DL, VT, Res, Amt);
1452 if (!IsFSHR)
1453 Res = DAG.getNode(ISD::SRL, DL, VT, Res, HiShift);
1454 return Res;
1455 }
1456
1457 // Shift Lo up to occupy the upper bits of the promoted type.
1458 SDValue ShiftOffset = DAG.getConstant(NewBits - OldBits, DL, AmtVT);
1459 Lo = DAG.getNode(ISD::SHL, DL, VT, Lo, ShiftOffset);
1460
1461 // Increase Amount to shift the result into the lower bits of the promoted
1462 // type.
1463 if (IsFSHR)
1464 Amt = DAG.getNode(ISD::ADD, DL, AmtVT, Amt, ShiftOffset);
1465
1466 return DAG.getNode(Opcode, DL, VT, Hi, Lo, Amt);
1467}
1468
1469// A vp version of PromoteIntRes_FunnelShift.
1470SDValue DAGTypeLegalizer::PromoteIntRes_VPFunnelShift(SDNode *N) {
1471 SDValue Hi = GetPromotedInteger(N->getOperand(0));
1472 SDValue Lo = GetPromotedInteger(N->getOperand(1));
1473 SDValue Amt = N->getOperand(2);
1474 SDValue Mask = N->getOperand(3);
1475 SDValue EVL = N->getOperand(4);
1476 if (getTypeAction(Amt.getValueType()) == TargetLowering::TypePromoteInteger)
1477 Amt = ZExtPromotedInteger(Amt);
1478 EVT AmtVT = Amt.getValueType();
1479
1480 SDLoc DL(N);
1481 EVT OldVT = N->getOperand(0).getValueType();
1482 EVT VT = Lo.getValueType();
1483 unsigned Opcode = N->getOpcode();
1484 bool IsFSHR = Opcode == ISD::VP_FSHR;
1485 unsigned OldBits = OldVT.getScalarSizeInBits();
1486 unsigned NewBits = VT.getScalarSizeInBits();
1487
1488 // Amount has to be interpreted modulo the old bit width.
1489 Amt = DAG.getNode(ISD::VP_UREM, DL, AmtVT, Amt,
1490 DAG.getConstant(OldBits, DL, AmtVT), Mask, EVL);
1491
1492 // If the promoted type is twice the size (or more), then we use the
1493 // traditional funnel 'double' shift codegen. This isn't necessary if the
1494 // shift amount is constant.
1495 // fshl(x,y,z) -> (((aext(x) << bw) | zext(y)) << (z % bw)) >> bw.
1496 // fshr(x,y,z) -> (((aext(x) << bw) | zext(y)) >> (z % bw)).
1497 if (NewBits >= (2 * OldBits) && !isa<ConstantSDNode>(Amt) &&
1498 !TLI.isOperationLegalOrCustom(Opcode, VT)) {
1499 SDValue HiShift = DAG.getConstant(OldBits, DL, VT);
1500 Hi = DAG.getNode(ISD::VP_SHL, DL, VT, Hi, HiShift, Mask, EVL);
1501 // FIXME: Replace it by vp operations.
1502 Lo = DAG.getZeroExtendInReg(Lo, DL, OldVT);
1503 SDValue Res = DAG.getNode(ISD::VP_OR, DL, VT, Hi, Lo, Mask, EVL);
1504 Res = DAG.getNode(IsFSHR ? ISD::VP_LSHR : ISD::VP_SHL, DL, VT, Res, Amt,
1505 Mask, EVL);
1506 if (!IsFSHR)
1507 Res = DAG.getNode(ISD::VP_LSHR, DL, VT, Res, HiShift, Mask, EVL);
1508 return Res;
1509 }
1510
1511 // Shift Lo up to occupy the upper bits of the promoted type.
1512 SDValue ShiftOffset = DAG.getConstant(NewBits - OldBits, DL, AmtVT);
1513 Lo = DAG.getNode(ISD::VP_SHL, DL, VT, Lo, ShiftOffset, Mask, EVL);
1514
1515 // Increase Amount to shift the result into the lower bits of the promoted
1516 // type.
1517 if (IsFSHR)
1518 Amt = DAG.getNode(ISD::VP_ADD, DL, AmtVT, Amt, ShiftOffset, Mask, EVL);
1519
1520 return DAG.getNode(Opcode, DL, VT, Hi, Lo, Amt, Mask, EVL);
1521}
1522
1523SDValue DAGTypeLegalizer::PromoteIntRes_TRUNCATE(SDNode *N) {
1524 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
1525 SDValue Res;
1526 SDValue InOp = N->getOperand(0);
1527 SDLoc dl(N);
1528
1529 switch (getTypeAction(InOp.getValueType())) {
1530 default: llvm_unreachable("Unknown type action!");
1533 Res = InOp;
1534 break;
1536 Res = GetPromotedInteger(InOp);
1537 break;
1539 EVT InVT = InOp.getValueType();
1540 assert(InVT.isVector() && "Cannot split scalar types");
1541 ElementCount NumElts = InVT.getVectorElementCount();
1542 assert(NumElts == NVT.getVectorElementCount() &&
1543 "Dst and Src must have the same number of elements");
1545 "Promoted vector type must be a power of two");
1546
1547 SDValue EOp1, EOp2;
1548 GetSplitVector(InOp, EOp1, EOp2);
1549
1550 EVT HalfNVT = EVT::getVectorVT(*DAG.getContext(), NVT.getScalarType(),
1551 NumElts.divideCoefficientBy(2));
1552 if (N->getOpcode() == ISD::TRUNCATE) {
1553 EOp1 = DAG.getNode(ISD::TRUNCATE, dl, HalfNVT, EOp1);
1554 EOp2 = DAG.getNode(ISD::TRUNCATE, dl, HalfNVT, EOp2);
1555 } else {
1556 assert(N->getOpcode() == ISD::VP_TRUNCATE &&
1557 "Expected VP_TRUNCATE opcode");
1558 SDValue MaskLo, MaskHi, EVLLo, EVLHi;
1559 std::tie(MaskLo, MaskHi) = SplitMask(N->getOperand(1));
1560 std::tie(EVLLo, EVLHi) =
1561 DAG.SplitEVL(N->getOperand(2), N->getValueType(0), dl);
1562 EOp1 = DAG.getNode(ISD::VP_TRUNCATE, dl, HalfNVT, EOp1, MaskLo, EVLLo);
1563 EOp2 = DAG.getNode(ISD::VP_TRUNCATE, dl, HalfNVT, EOp2, MaskHi, EVLHi);
1564 }
1565 return DAG.getNode(ISD::CONCAT_VECTORS, dl, NVT, EOp1, EOp2);
1566 }
1567 // TODO: VP_TRUNCATE need to handle when TypeWidenVector access to some
1568 // targets.
1570 SDValue WideInOp = GetWidenedVector(InOp);
1571
1572 // Truncate widened InOp.
1573 unsigned NumElem = WideInOp.getValueType().getVectorNumElements();
1574 EVT TruncVT = EVT::getVectorVT(*DAG.getContext(),
1575 N->getValueType(0).getScalarType(), NumElem);
1576 SDValue WideTrunc = DAG.getNode(ISD::TRUNCATE, dl, TruncVT, WideInOp);
1577
1578 // Zero extend so that the elements are of same type as those of NVT
1580 NumElem);
1581 SDValue WideExt = DAG.getNode(ISD::ZERO_EXTEND, dl, ExtVT, WideTrunc);
1582
1583 // Extract the low NVT subvector.
1584 SDValue ZeroIdx = DAG.getVectorIdxConstant(0, dl);
1585 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, NVT, WideExt, ZeroIdx);
1586 }
1587 }
1588
1589 // Truncate to NVT instead of VT
1590 if (N->getOpcode() == ISD::VP_TRUNCATE)
1591 return DAG.getNode(ISD::VP_TRUNCATE, dl, NVT, Res, N->getOperand(1),
1592 N->getOperand(2));
1593 return DAG.getNode(ISD::TRUNCATE, dl, NVT, Res);
1594}
1595
1596SDValue DAGTypeLegalizer::PromoteIntRes_UADDSUBO(SDNode *N, unsigned ResNo) {
1597 if (ResNo == 1)
1598 return PromoteIntRes_Overflow(N);
1599
1600 // The operation overflowed iff the result in the larger type is not the
1601 // zero extension of its truncation to the original type.
1602 SDValue LHS = ZExtPromotedInteger(N->getOperand(0));
1603 SDValue RHS = ZExtPromotedInteger(N->getOperand(1));
1604 EVT OVT = N->getOperand(0).getValueType();
1605 EVT NVT = LHS.getValueType();
1606 SDLoc dl(N);
1607
1608 // Do the arithmetic in the larger type.
1609 unsigned Opcode = N->getOpcode() == ISD::UADDO ? ISD::ADD : ISD::SUB;
1610 SDValue Res = DAG.getNode(Opcode, dl, NVT, LHS, RHS);
1611
1612 // Calculate the overflow flag: zero extend the arithmetic result from
1613 // the original type.
1614 SDValue Ofl = DAG.getZeroExtendInReg(Res, dl, OVT);
1615 // Overflowed if and only if this is not equal to Res.
1616 Ofl = DAG.getSetCC(dl, N->getValueType(1), Ofl, Res, ISD::SETNE);
1617
1618 // Use the calculated overflow everywhere.
1619 ReplaceValueWith(SDValue(N, 1), Ofl);
1620
1621 return Res;
1622}
1623
1624// Handle promotion for the ADDE/SUBE/UADDO_CARRY/USUBO_CARRY nodes. Notice that
1625// the third operand of ADDE/SUBE nodes is carry flag, which differs from
1626// the UADDO_CARRY/USUBO_CARRY nodes in that the third operand is carry Boolean.
1627SDValue DAGTypeLegalizer::PromoteIntRes_UADDSUBO_CARRY(SDNode *N,
1628 unsigned ResNo) {
1629 if (ResNo == 1)
1630 return PromoteIntRes_Overflow(N);
1631
1632 // We need to sign-extend the operands so the carry value computed by the
1633 // wide operation will be equivalent to the carry value computed by the
1634 // narrow operation.
1635 // An UADDO_CARRY can generate carry only if any of the operands has its
1636 // most significant bit set. Sign extension propagates the most significant
1637 // bit into the higher bits which means the extra bit that the narrow
1638 // addition would need (i.e. the carry) will be propagated through the higher
1639 // bits of the wide addition.
1640 // A USUBO_CARRY can generate borrow only if LHS < RHS and this property will
1641 // be preserved by sign extension.
1642 SDValue LHS = SExtPromotedInteger(N->getOperand(0));
1643 SDValue RHS = SExtPromotedInteger(N->getOperand(1));
1644
1645 EVT ValueVTs[] = {LHS.getValueType(), N->getValueType(1)};
1646
1647 // Do the arithmetic in the wide type.
1648 SDValue Res = DAG.getNode(N->getOpcode(), SDLoc(N), DAG.getVTList(ValueVTs),
1649 LHS, RHS, N->getOperand(2));
1650
1651 // Update the users of the original carry/borrow value.
1652 ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
1653
1654 return SDValue(Res.getNode(), 0);
1655}
1656
1657SDValue DAGTypeLegalizer::PromoteIntRes_SADDSUBO_CARRY(SDNode *N,
1658 unsigned ResNo) {
1659 assert(ResNo == 1 && "Don't know how to promote other results yet.");
1660 return PromoteIntRes_Overflow(N);
1661}
1662
1663SDValue DAGTypeLegalizer::PromoteIntRes_ABS(SDNode *N) {
1664 EVT OVT = N->getValueType(0);
1665 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), OVT);
1666
1667 // If a larger ABS or SMAX isn't supported by the target, try to expand now.
1668 // If we expand later we'll end up sign extending more than just the sra input
1669 // in sra+xor+sub expansion.
1670 if (!OVT.isVector() &&
1672 !TLI.isOperationLegal(ISD::SMAX, NVT)) {
1673 if (SDValue Res = TLI.expandABS(N, DAG))
1674 return DAG.getNode(ISD::ANY_EXTEND, SDLoc(N), NVT, Res);
1675 }
1676
1677 SDValue Op0 = SExtPromotedInteger(N->getOperand(0));
1678 return DAG.getNode(ISD::ABS, SDLoc(N), Op0.getValueType(), Op0);
1679}
1680
1681SDValue DAGTypeLegalizer::PromoteIntRes_XMULO(SDNode *N, unsigned ResNo) {
1682 // Promote the overflow bit trivially.
1683 if (ResNo == 1)
1684 return PromoteIntRes_Overflow(N);
1685
1686 SDValue LHS = N->getOperand(0), RHS = N->getOperand(1);
1687 SDLoc DL(N);
1688 EVT SmallVT = LHS.getValueType();
1689
1690 // To determine if the result overflowed in a larger type, we extend the
1691 // input to the larger type, do the multiply (checking if it overflows),
1692 // then also check the high bits of the result to see if overflow happened
1693 // there.
1694 if (N->getOpcode() == ISD::SMULO) {
1695 LHS = SExtPromotedInteger(LHS);
1696 RHS = SExtPromotedInteger(RHS);
1697 } else {
1698 LHS = ZExtPromotedInteger(LHS);
1699 RHS = ZExtPromotedInteger(RHS);
1700 }
1701 SDVTList VTs = DAG.getVTList(LHS.getValueType(), N->getValueType(1));
1702 SDValue Mul = DAG.getNode(N->getOpcode(), DL, VTs, LHS, RHS);
1703
1704 // Overflow occurred if it occurred in the larger type, or if the high part
1705 // of the result does not zero/sign-extend the low part. Check this second
1706 // possibility first.
1707 SDValue Overflow;
1708 if (N->getOpcode() == ISD::UMULO) {
1709 // Unsigned overflow occurred if the high part is non-zero.
1710 unsigned Shift = SmallVT.getScalarSizeInBits();
1711 SDValue Hi =
1712 DAG.getNode(ISD::SRL, DL, Mul.getValueType(), Mul,
1713 DAG.getShiftAmountConstant(Shift, Mul.getValueType(), DL));
1714 Overflow = DAG.getSetCC(DL, N->getValueType(1), Hi,
1715 DAG.getConstant(0, DL, Hi.getValueType()),
1716 ISD::SETNE);
1717 } else {
1718 // Signed overflow occurred if the high part does not sign extend the low.
1719 SDValue SExt = DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, Mul.getValueType(),
1720 Mul, DAG.getValueType(SmallVT));
1721 Overflow = DAG.getSetCC(DL, N->getValueType(1), SExt, Mul, ISD::SETNE);
1722 }
1723
1724 // The only other way for overflow to occur is if the multiplication in the
1725 // larger type itself overflowed.
1726 Overflow = DAG.getNode(ISD::OR, DL, N->getValueType(1), Overflow,
1727 SDValue(Mul.getNode(), 1));
1728
1729 // Use the calculated overflow everywhere.
1730 ReplaceValueWith(SDValue(N, 1), Overflow);
1731 return Mul;
1732}
1733
1734SDValue DAGTypeLegalizer::PromoteIntRes_UNDEF(SDNode *N) {
1735 return DAG.getUNDEF(TLI.getTypeToTransformTo(*DAG.getContext(),
1736 N->getValueType(0)));
1737}
1738
1739SDValue DAGTypeLegalizer::PromoteIntRes_VSCALE(SDNode *N) {
1740 EVT VT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
1741
1742 const APInt &MulImm = N->getConstantOperandAPInt(0);
1743 return DAG.getVScale(SDLoc(N), VT, MulImm.sext(VT.getSizeInBits()));
1744}
1745
1746SDValue DAGTypeLegalizer::PromoteIntRes_VAARG(SDNode *N) {
1747 SDValue Chain = N->getOperand(0); // Get the chain.
1748 SDValue Ptr = N->getOperand(1); // Get the pointer.
1749 EVT VT = N->getValueType(0);
1750 SDLoc dl(N);
1751
1752 MVT RegVT = TLI.getRegisterType(*DAG.getContext(), VT);
1753 unsigned NumRegs = TLI.getNumRegisters(*DAG.getContext(), VT);
1754 // The argument is passed as NumRegs registers of type RegVT.
1755
1756 SmallVector<SDValue, 8> Parts(NumRegs);
1757 for (unsigned i = 0; i < NumRegs; ++i) {
1758 Parts[i] = DAG.getVAArg(RegVT, dl, Chain, Ptr, N->getOperand(2),
1759 N->getConstantOperandVal(3));
1760 Chain = Parts[i].getValue(1);
1761 }
1762
1763 // Handle endianness of the load.
1764 if (DAG.getDataLayout().isBigEndian())
1765 std::reverse(Parts.begin(), Parts.end());
1766
1767 // Assemble the parts in the promoted type.
1768 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
1769 SDValue Res = DAG.getNode(ISD::ZERO_EXTEND, dl, NVT, Parts[0]);
1770 for (unsigned i = 1; i < NumRegs; ++i) {
1771 SDValue Part = DAG.getNode(ISD::ZERO_EXTEND, dl, NVT, Parts[i]);
1772 // Shift it to the right position and "or" it in.
1773 Part = DAG.getNode(ISD::SHL, dl, NVT, Part,
1774 DAG.getConstant(i * RegVT.getSizeInBits(), dl,
1775 TLI.getPointerTy(DAG.getDataLayout())));
1776 Res = DAG.getNode(ISD::OR, dl, NVT, Res, Part);
1777 }
1778
1779 // Modified the chain result - switch anything that used the old chain to
1780 // use the new one.
1781 ReplaceValueWith(SDValue(N, 1), Chain);
1782
1783 return Res;
1784}
1785
1786//===----------------------------------------------------------------------===//
1787// Integer Operand Promotion
1788//===----------------------------------------------------------------------===//
1789
1790/// PromoteIntegerOperand - This method is called when the specified operand of
1791/// the specified node is found to need promotion. At this point, all of the
1792/// result types of the node are known to be legal, but other operands of the
1793/// node may need promotion or expansion as well as the specified one.
1794bool DAGTypeLegalizer::PromoteIntegerOperand(SDNode *N, unsigned OpNo) {
1795 LLVM_DEBUG(dbgs() << "Promote integer operand: "; N->dump(&DAG));
1796 SDValue Res = SDValue();
1797 if (CustomLowerNode(N, N->getOperand(OpNo).getValueType(), false)) {
1798 LLVM_DEBUG(dbgs() << "Node has been custom lowered, done\n");
1799 return false;
1800 }
1801
1802 switch (N->getOpcode()) {
1803 default:
1804 #ifndef NDEBUG
1805 dbgs() << "PromoteIntegerOperand Op #" << OpNo << ": ";
1806 N->dump(&DAG); dbgs() << "\n";
1807 #endif
1808 report_fatal_error("Do not know how to promote this operator's operand!");
1809
1810 case ISD::ANY_EXTEND: Res = PromoteIntOp_ANY_EXTEND(N); break;
1811 case ISD::ATOMIC_STORE:
1812 Res = PromoteIntOp_ATOMIC_STORE(cast<AtomicSDNode>(N));
1813 break;
1814 case ISD::BITCAST: Res = PromoteIntOp_BITCAST(N); break;
1815 case ISD::BR_CC: Res = PromoteIntOp_BR_CC(N, OpNo); break;
1816 case ISD::BRCOND: Res = PromoteIntOp_BRCOND(N, OpNo); break;
1817 case ISD::BUILD_PAIR: Res = PromoteIntOp_BUILD_PAIR(N); break;
1818 case ISD::BUILD_VECTOR: Res = PromoteIntOp_BUILD_VECTOR(N); break;
1819 case ISD::CONCAT_VECTORS: Res = PromoteIntOp_CONCAT_VECTORS(N); break;
1820 case ISD::EXTRACT_VECTOR_ELT: Res = PromoteIntOp_EXTRACT_VECTOR_ELT(N); break;
1822 Res = PromoteIntOp_INSERT_VECTOR_ELT(N, OpNo);
1823 break;
1824 case ISD::SPLAT_VECTOR:
1826 Res = PromoteIntOp_ScalarOp(N);
1827 break;
1828 case ISD::VSELECT:
1829 case ISD::SELECT: Res = PromoteIntOp_SELECT(N, OpNo); break;
1830 case ISD::SELECT_CC: Res = PromoteIntOp_SELECT_CC(N, OpNo); break;
1831 case ISD::VP_SETCC:
1832 case ISD::SETCC: Res = PromoteIntOp_SETCC(N, OpNo); break;
1833 case ISD::SIGN_EXTEND: Res = PromoteIntOp_SIGN_EXTEND(N); break;
1834 case ISD::VP_SIGN_EXTEND: Res = PromoteIntOp_VP_SIGN_EXTEND(N); break;
1835 case ISD::VP_SINT_TO_FP:
1836 case ISD::SINT_TO_FP: Res = PromoteIntOp_SINT_TO_FP(N); break;
1837 case ISD::STRICT_SINT_TO_FP: Res = PromoteIntOp_STRICT_SINT_TO_FP(N); break;
1838 case ISD::STORE: Res = PromoteIntOp_STORE(cast<StoreSDNode>(N),
1839 OpNo); break;
1840 case ISD::MSTORE: Res = PromoteIntOp_MSTORE(cast<MaskedStoreSDNode>(N),
1841 OpNo); break;
1842 case ISD::MLOAD: Res = PromoteIntOp_MLOAD(cast<MaskedLoadSDNode>(N),
1843 OpNo); break;
1844 case ISD::MGATHER: Res = PromoteIntOp_MGATHER(cast<MaskedGatherSDNode>(N),
1845 OpNo); break;
1846 case ISD::MSCATTER: Res = PromoteIntOp_MSCATTER(cast<MaskedScatterSDNode>(N),
1847 OpNo); break;
1848 case ISD::VP_TRUNCATE:
1849 case ISD::TRUNCATE: Res = PromoteIntOp_TRUNCATE(N); break;
1850 case ISD::BF16_TO_FP:
1851 case ISD::FP16_TO_FP:
1852 case ISD::VP_UINT_TO_FP:
1853 case ISD::UINT_TO_FP: Res = PromoteIntOp_UINT_TO_FP(N); break;
1855 case ISD::STRICT_UINT_TO_FP: Res = PromoteIntOp_STRICT_UINT_TO_FP(N); break;
1856 case ISD::ZERO_EXTEND: Res = PromoteIntOp_ZERO_EXTEND(N); break;
1857 case ISD::VP_ZERO_EXTEND: Res = PromoteIntOp_VP_ZERO_EXTEND(N); break;
1858 case ISD::EXTRACT_SUBVECTOR: Res = PromoteIntOp_EXTRACT_SUBVECTOR(N); break;
1859 case ISD::INSERT_SUBVECTOR: Res = PromoteIntOp_INSERT_SUBVECTOR(N); break;
1860
1861 case ISD::SHL:
1862 case ISD::SRA:
1863 case ISD::SRL:
1864 case ISD::ROTL:
1865 case ISD::ROTR: Res = PromoteIntOp_Shift(N); break;
1866
1867 case ISD::FSHL:
1868 case ISD::FSHR: Res = PromoteIntOp_FunnelShift(N); break;
1869
1870 case ISD::SADDO_CARRY:
1871 case ISD::SSUBO_CARRY:
1872 case ISD::UADDO_CARRY:
1873 case ISD::USUBO_CARRY: Res = PromoteIntOp_ADDSUBO_CARRY(N, OpNo); break;
1874
1875 case ISD::FRAMEADDR:
1876 case ISD::RETURNADDR: Res = PromoteIntOp_FRAMERETURNADDR(N); break;
1877
1878 case ISD::SMULFIX:
1879 case ISD::SMULFIXSAT:
1880 case ISD::UMULFIX:
1881 case ISD::UMULFIXSAT:
1882 case ISD::SDIVFIX:
1883 case ISD::SDIVFIXSAT:
1884 case ISD::UDIVFIX:
1885 case ISD::UDIVFIXSAT: Res = PromoteIntOp_FIX(N); break;
1886 case ISD::FPOWI:
1887 case ISD::STRICT_FPOWI:
1888 case ISD::FLDEXP:
1889 case ISD::STRICT_FLDEXP: Res = PromoteIntOp_ExpOp(N); break;
1890 case ISD::VECREDUCE_ADD:
1891 case ISD::VECREDUCE_MUL:
1892 case ISD::VECREDUCE_AND:
1893 case ISD::VECREDUCE_OR:
1894 case ISD::VECREDUCE_XOR:
1898 case ISD::VECREDUCE_UMIN: Res = PromoteIntOp_VECREDUCE(N); break;
1899 case ISD::VP_REDUCE_ADD:
1900 case ISD::VP_REDUCE_MUL:
1901 case ISD::VP_REDUCE_AND:
1902 case ISD::VP_REDUCE_OR:
1903 case ISD::VP_REDUCE_XOR:
1904 case ISD::VP_REDUCE_SMAX:
1905 case ISD::VP_REDUCE_SMIN:
1906 case ISD::VP_REDUCE_UMAX:
1907 case ISD::VP_REDUCE_UMIN:
1908 Res = PromoteIntOp_VP_REDUCE(N, OpNo);
1909 break;
1910
1911 case ISD::SET_ROUNDING: Res = PromoteIntOp_SET_ROUNDING(N); break;
1912 case ISD::STACKMAP:
1913 Res = PromoteIntOp_STACKMAP(N, OpNo);
1914 break;
1915 case ISD::PATCHPOINT:
1916 Res = PromoteIntOp_PATCHPOINT(N, OpNo);
1917 break;
1918 case ISD::EXPERIMENTAL_VP_STRIDED_LOAD:
1919 case ISD::EXPERIMENTAL_VP_STRIDED_STORE:
1920 Res = PromoteIntOp_VP_STRIDED(N, OpNo);
1921 break;
1922 case ISD::EXPERIMENTAL_VP_SPLICE:
1923 Res = PromoteIntOp_VP_SPLICE(N, OpNo);
1924 break;
1925 }
1926
1927 // If the result is null, the sub-method took care of registering results etc.
1928 if (!Res.getNode()) return false;
1929
1930 // If the result is N, the sub-method updated N in place. Tell the legalizer
1931 // core about this.
1932 if (Res.getNode() == N)
1933 return true;
1934
1935 const bool IsStrictFp = N->isStrictFPOpcode();
1936 assert(Res.getValueType() == N->getValueType(0) &&
1937 N->getNumValues() == (IsStrictFp ? 2 : 1) &&
1938 "Invalid operand expansion");
1939 LLVM_DEBUG(dbgs() << "Replacing: "; N->dump(&DAG); dbgs() << " with: ";
1940 Res.dump());
1941
1942 ReplaceValueWith(SDValue(N, 0), Res);
1943 if (IsStrictFp)
1944 ReplaceValueWith(SDValue(N, 1), SDValue(Res.getNode(), 1));
1945
1946 return false;
1947}
1948
1949// These operands can be either sign extended or zero extended as long as we
1950// treat them the same. If an extension is free, choose that. Otherwise, follow
1951// target preference.
1952void DAGTypeLegalizer::SExtOrZExtPromotedOperands(SDValue &LHS, SDValue &RHS) {
1953 SDValue OpL = GetPromotedInteger(LHS);
1954 SDValue OpR = GetPromotedInteger(RHS);
1955
1956 if (TLI.isSExtCheaperThanZExt(LHS.getValueType(), OpL.getValueType())) {
1957 // The target would prefer to promote the comparison operand with sign
1958 // extension. Honor that unless the promoted values are already zero
1959 // extended.
1960 unsigned OpLEffectiveBits =
1962 unsigned OpREffectiveBits =
1964 if (OpLEffectiveBits <= LHS.getScalarValueSizeInBits() &&
1965 OpREffectiveBits <= RHS.getScalarValueSizeInBits()) {
1966 LHS = OpL;
1967 RHS = OpR;
1968 return;
1969 }
1970
1971 // The promoted values aren't zero extended, use a sext_inreg.
1972 LHS = SExtPromotedInteger(LHS);
1973 RHS = SExtPromotedInteger(RHS);
1974 return;
1975 }
1976
1977 // Prefer to promote the comparison operand with zero extension.
1978
1979 // If the width of OpL/OpR excluding the duplicated sign bits is no greater
1980 // than the width of LHS/RHS, we can avoid/ inserting a zext_inreg operation
1981 // that we might not be able to remove.
1982 unsigned OpLEffectiveBits = DAG.ComputeMaxSignificantBits(OpL);
1983 unsigned OpREffectiveBits = DAG.ComputeMaxSignificantBits(OpR);
1984 if (OpLEffectiveBits <= LHS.getScalarValueSizeInBits() &&
1985 OpREffectiveBits <= RHS.getScalarValueSizeInBits()) {
1986 LHS = OpL;
1987 RHS = OpR;
1988 return;
1989 }
1990
1991 // Otherwise, use zext_inreg.
1992 LHS = ZExtPromotedInteger(LHS);
1993 RHS = ZExtPromotedInteger(RHS);
1994}
1995
1996/// PromoteSetCCOperands - Promote the operands of a comparison. This code is
1997/// shared among BR_CC, SELECT_CC, and SETCC handlers.
1998void DAGTypeLegalizer::PromoteSetCCOperands(SDValue &LHS, SDValue &RHS,
1999 ISD::CondCode CCCode) {
2000 // We have to insert explicit sign or zero extends. Note that we could
2001 // insert sign extends for ALL conditions. For those operations where either
2002 // zero or sign extension would be valid, we ask the target which extension
2003 // it would prefer.
2004
2005 // Signed comparisons always require sign extension.
2006 if (ISD::isSignedIntSetCC(CCCode)) {
2007 LHS = SExtPromotedInteger(LHS);
2008 RHS = SExtPromotedInteger(RHS);
2009 return;
2010 }
2011
2013 "Unknown integer comparison!");
2014
2015 SExtOrZExtPromotedOperands(LHS, RHS);
2016}
2017
2018SDValue DAGTypeLegalizer::PromoteIntOp_ANY_EXTEND(SDNode *N) {
2019 SDValue Op = GetPromotedInteger(N->getOperand(0));
2020 return DAG.getNode(ISD::ANY_EXTEND, SDLoc(N), N->getValueType(0), Op);
2021}
2022
2023SDValue DAGTypeLegalizer::PromoteIntOp_ATOMIC_STORE(AtomicSDNode *N) {
2024 SDValue Op1 = GetPromotedInteger(N->getOperand(1));
2025 return DAG.getAtomic(N->getOpcode(), SDLoc(N), N->getMemoryVT(),
2026 N->getChain(), Op1, N->getBasePtr(), N->getMemOperand());
2027}
2028
2029SDValue DAGTypeLegalizer::PromoteIntOp_BITCAST(SDNode *N) {
2030 // This should only occur in unusual situations like bitcasting to an
2031 // x86_fp80, so just turn it into a store+load
2032 return CreateStackStoreLoad(N->getOperand(0), N->getValueType(0));
2033}
2034
2035SDValue DAGTypeLegalizer::PromoteIntOp_BR_CC(SDNode *N, unsigned OpNo) {
2036 assert(OpNo == 2 && "Don't know how to promote this operand!");
2037
2038 SDValue LHS = N->getOperand(2);
2039 SDValue RHS = N->getOperand(3);
2040 PromoteSetCCOperands(LHS, RHS, cast<CondCodeSDNode>(N->getOperand(1))->get());
2041
2042 // The chain (Op#0), CC (#1) and basic block destination (Op#4) are always
2043 // legal types.
2044 return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0),
2045 N->getOperand(1), LHS, RHS, N->getOperand(4)),
2046 0);
2047}
2048
2049SDValue DAGTypeLegalizer::PromoteIntOp_BRCOND(SDNode *N, unsigned OpNo) {
2050 assert(OpNo == 1 && "only know how to promote condition");
2051
2052 // Promote all the way up to the canonical SetCC type.
2053 SDValue Cond = PromoteTargetBoolean(N->getOperand(1), MVT::Other);
2054
2055 // The chain (Op#0) and basic block destination (Op#2) are always legal types.
2056 return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0), Cond,
2057 N->getOperand(2)), 0);
2058}
2059
2060SDValue DAGTypeLegalizer::PromoteIntOp_BUILD_PAIR(SDNode *N) {
2061 // Since the result type is legal, the operands must promote to it.
2062 EVT OVT = N->getOperand(0).getValueType();
2063 SDValue Lo = ZExtPromotedInteger(N->getOperand(0));
2064 SDValue Hi = GetPromotedInteger(N->getOperand(1));
2065 assert(Lo.getValueType() == N->getValueType(0) && "Operand over promoted?");
2066 SDLoc dl(N);
2067
2068 Hi = DAG.getNode(ISD::SHL, dl, N->getValueType(0), Hi,
2069 DAG.getConstant(OVT.getSizeInBits(), dl,
2070 TLI.getPointerTy(DAG.getDataLayout())));
2071 return DAG.getNode(ISD::OR, dl, N->getValueType(0), Lo, Hi);
2072}
2073
2074SDValue DAGTypeLegalizer::PromoteIntOp_BUILD_VECTOR(SDNode *N) {
2075 // The vector type is legal but the element type is not. This implies
2076 // that the vector is a power-of-two in length and that the element
2077 // type does not have a strange size (eg: it is not i1).
2078 EVT VecVT = N->getValueType(0);
2079 unsigned NumElts = VecVT.getVectorNumElements();
2080 assert(!((NumElts & 1) && (!TLI.isTypeLegal(VecVT))) &&
2081 "Legal vector of one illegal element?");
2082
2083 // Promote the inserted value. The type does not need to match the
2084 // vector element type. Check that any extra bits introduced will be
2085 // truncated away.
2086 assert(N->getOperand(0).getValueSizeInBits() >=
2087 N->getValueType(0).getScalarSizeInBits() &&
2088 "Type of inserted value narrower than vector element type!");
2089
2091 for (unsigned i = 0; i < NumElts; ++i)
2092 NewOps.push_back(GetPromotedInteger(N->getOperand(i)));
2093
2094 return SDValue(DAG.UpdateNodeOperands(N, NewOps), 0);
2095}
2096
2097SDValue DAGTypeLegalizer::PromoteIntOp_INSERT_VECTOR_ELT(SDNode *N,
2098 unsigned OpNo) {
2099 if (OpNo == 1) {
2100 // Promote the inserted value. This is valid because the type does not
2101 // have to match the vector element type.
2102
2103 // Check that any extra bits introduced will be truncated away.
2104 assert(N->getOperand(1).getValueSizeInBits() >=
2105 N->getValueType(0).getScalarSizeInBits() &&
2106 "Type of inserted value narrower than vector element type!");
2107 return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0),
2108 GetPromotedInteger(N->getOperand(1)),
2109 N->getOperand(2)),
2110 0);
2111 }
2112
2113 assert(OpNo == 2 && "Different operand and result vector types?");
2114
2115 // Promote the index.
2116 SDValue Idx = DAG.getZExtOrTrunc(N->getOperand(2), SDLoc(N),
2117 TLI.getVectorIdxTy(DAG.getDataLayout()));
2118 return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0),
2119 N->getOperand(1), Idx), 0);
2120}
2121
2122SDValue DAGTypeLegalizer::PromoteIntOp_ScalarOp(SDNode *N) {
2123 // Integer SPLAT_VECTOR/SCALAR_TO_VECTOR operands are implicitly truncated,
2124 // so just promote the operand in place.
2125 return SDValue(DAG.UpdateNodeOperands(N,
2126 GetPromotedInteger(N->getOperand(0))), 0);
2127}
2128
2129SDValue DAGTypeLegalizer::PromoteIntOp_SELECT(SDNode *N, unsigned OpNo) {
2130 assert(OpNo == 0 && "Only know how to promote the condition!");
2131 SDValue Cond = N->getOperand(0);
2132 EVT OpTy = N->getOperand(1).getValueType();
2133
2134 if (N->getOpcode() == ISD::VSELECT)
2135 if (SDValue Res = WidenVSELECTMask(N))
2136 return DAG.getNode(N->getOpcode(), SDLoc(N), N->getValueType(0),
2137 Res, N->getOperand(1), N->getOperand(2));
2138
2139 // Promote all the way up to the canonical SetCC type.
2140 EVT OpVT = N->getOpcode() == ISD::SELECT ? OpTy.getScalarType() : OpTy;
2141 Cond = PromoteTargetBoolean(Cond, OpVT);
2142
2143 return SDValue(DAG.UpdateNodeOperands(N, Cond, N->getOperand(1),
2144 N->getOperand(2)), 0);
2145}
2146
2147SDValue DAGTypeLegalizer::PromoteIntOp_SELECT_CC(SDNode *N, unsigned OpNo) {
2148 assert(OpNo == 0 && "Don't know how to promote this operand!");
2149
2150 SDValue LHS = N->getOperand(0);
2151 SDValue RHS = N->getOperand(1);
2152 PromoteSetCCOperands(LHS, RHS, cast<CondCodeSDNode>(N->getOperand(4))->get());
2153
2154 // The CC (#4) and the possible return values (#2 and #3) have legal types.
2155 return SDValue(DAG.UpdateNodeOperands(N, LHS, RHS, N->getOperand(2),
2156 N->getOperand(3), N->getOperand(4)), 0);
2157}
2158
2159SDValue DAGTypeLegalizer::PromoteIntOp_SETCC(SDNode *N, unsigned OpNo) {
2160 assert(OpNo == 0 && "Don't know how to promote this operand!");
2161
2162 SDValue LHS = N->getOperand(0);
2163 SDValue RHS = N->getOperand(1);
2164 PromoteSetCCOperands(LHS, RHS, cast<CondCodeSDNode>(N->getOperand(2))->get());
2165
2166 // The CC (#2) is always legal.
2167 if (N->getOpcode() == ISD::SETCC)
2168 return SDValue(DAG.UpdateNodeOperands(N, LHS, RHS, N->getOperand(2)), 0);
2169
2170 assert(N->getOpcode() == ISD::VP_SETCC && "Expected VP_SETCC opcode");
2171
2172 return SDValue(DAG.UpdateNodeOperands(N, LHS, RHS, N->getOperand(2),
2173 N->getOperand(3), N->getOperand(4)),
2174 0);
2175}
2176
2177SDValue DAGTypeLegalizer::PromoteIntOp_Shift(SDNode *N) {
2178 return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0),
2179 ZExtPromotedInteger(N->getOperand(1))), 0);
2180}
2181
2182SDValue DAGTypeLegalizer::PromoteIntOp_FunnelShift(SDNode *N) {
2183 return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0), N->getOperand(1),
2184 ZExtPromotedInteger(N->getOperand(2))), 0);
2185}
2186
2187SDValue DAGTypeLegalizer::PromoteIntOp_SIGN_EXTEND(SDNode *N) {
2188 SDValue Op = GetPromotedInteger(N->getOperand(0));
2189 SDLoc dl(N);
2190 Op = DAG.getNode(ISD::ANY_EXTEND, dl, N->getValueType(0), Op);
2191 return DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, Op.getValueType(),
2192 Op, DAG.getValueType(N->getOperand(0).getValueType()));
2193}
2194
2195SDValue DAGTypeLegalizer::PromoteIntOp_VP_SIGN_EXTEND(SDNode *N) {
2196 SDLoc dl(N);
2197 EVT VT = N->getValueType(0);
2198 SDValue Op = GetPromotedInteger(N->getOperand(0));
2199 // FIXME: There is no VP_ANY_EXTEND yet.
2200 Op = DAG.getNode(ISD::VP_ZERO_EXTEND, dl, VT, Op, N->getOperand(1),
2201 N->getOperand(2));
2202 unsigned Diff =
2203 VT.getScalarSizeInBits() - N->getOperand(0).getScalarValueSizeInBits();
2204 SDValue ShAmt = DAG.getShiftAmountConstant(Diff, VT, dl);
2205 // FIXME: There is no VP_SIGN_EXTEND_INREG so use a pair of shifts.
2206 SDValue Shl = DAG.getNode(ISD::VP_SHL, dl, VT, Op, ShAmt, N->getOperand(1),
2207 N->getOperand(2));
2208 return DAG.getNode(ISD::VP_ASHR, dl, VT, Shl, ShAmt, N->getOperand(1),
2209 N->getOperand(2));
2210}
2211
2212SDValue DAGTypeLegalizer::PromoteIntOp_SINT_TO_FP(SDNode *N) {
2213 if (N->getOpcode() == ISD::VP_SINT_TO_FP)
2214 return SDValue(DAG.UpdateNodeOperands(N,
2215 SExtPromotedInteger(N->getOperand(0)),
2216 N->getOperand(1), N->getOperand(2)),
2217 0);
2218 return SDValue(DAG.UpdateNodeOperands(N,
2219 SExtPromotedInteger(N->getOperand(0))), 0);
2220}
2221
2222SDValue DAGTypeLegalizer::PromoteIntOp_STRICT_SINT_TO_FP(SDNode *N) {
2223 return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0),
2224 SExtPromotedInteger(N->getOperand(1))), 0);
2225}
2226
2227SDValue DAGTypeLegalizer::PromoteIntOp_STORE(StoreSDNode *N, unsigned OpNo){
2228 assert(ISD::isUNINDEXEDStore(N) && "Indexed store during type legalization!");
2229 SDValue Ch = N->getChain(), Ptr = N->getBasePtr();
2230 SDLoc dl(N);
2231
2232 SDValue Val = GetPromotedInteger(N->getValue()); // Get promoted value.
2233
2234 // Truncate the value and store the result.
2235 return DAG.getTruncStore(Ch, dl, Val, Ptr,
2236 N->getMemoryVT(), N->getMemOperand());
2237}
2238
2239SDValue DAGTypeLegalizer::PromoteIntOp_MSTORE(MaskedStoreSDNode *N,
2240 unsigned OpNo) {
2241 SDValue DataOp = N->getValue();
2242 SDValue Mask = N->getMask();
2243
2244 if (OpNo == 4) {
2245 // The Mask. Update in place.
2246 EVT DataVT = DataOp.getValueType();
2247 Mask = PromoteTargetBoolean(Mask, DataVT);
2248 SmallVector<SDValue, 4> NewOps(N->op_begin(), N->op_end());
2249 NewOps[4] = Mask;
2250 return SDValue(DAG.UpdateNodeOperands(N, NewOps), 0);
2251 }
2252
2253 assert(OpNo == 1 && "Unexpected operand for promotion");
2254 DataOp = GetPromotedInteger(DataOp);
2255
2256 return DAG.getMaskedStore(N->getChain(), SDLoc(N), DataOp, N->getBasePtr(),
2257 N->getOffset(), Mask, N->getMemoryVT(),
2258 N->getMemOperand(), N->getAddressingMode(),
2259 /*IsTruncating*/ true, N->isCompressingStore());
2260}
2261
2262SDValue DAGTypeLegalizer::PromoteIntOp_MLOAD(MaskedLoadSDNode *N,
2263 unsigned OpNo) {
2264 assert(OpNo == 3 && "Only know how to promote the mask!");
2265 EVT DataVT = N->getValueType(0);
2266 SDValue Mask = PromoteTargetBoolean(N->getOperand(OpNo), DataVT);
2267 SmallVector<SDValue, 4> NewOps(N->op_begin(), N->op_end());
2268 NewOps[OpNo] = Mask;
2269 SDNode *Res = DAG.UpdateNodeOperands(N, NewOps);
2270 if (Res == N)
2271 return SDValue(Res, 0);
2272
2273 // Update triggered CSE, do our own replacement since caller can't.
2274 ReplaceValueWith(SDValue(N, 0), SDValue(Res, 0));
2275 ReplaceValueWith(SDValue(N, 1), SDValue(Res, 1));
2276 return SDValue();
2277}
2278
2279SDValue DAGTypeLegalizer::PromoteIntOp_MGATHER(MaskedGatherSDNode *N,
2280 unsigned OpNo) {
2281 SmallVector<SDValue, 5> NewOps(N->op_begin(), N->op_end());
2282
2283 if (OpNo == 2) {
2284 // The Mask
2285 EVT DataVT = N->getValueType(0);
2286 NewOps[OpNo] = PromoteTargetBoolean(N->getOperand(OpNo), DataVT);
2287 } else if (OpNo == 4) {
2288 // The Index
2289 if (N->isIndexSigned())
2290 // Need to sign extend the index since the bits will likely be used.
2291 NewOps[OpNo] = SExtPromotedInteger(N->getOperand(OpNo));
2292 else
2293 NewOps[OpNo] = ZExtPromotedInteger(N->getOperand(OpNo));
2294 } else
2295 NewOps[OpNo] = GetPromotedInteger(N->getOperand(OpNo));
2296
2297 SDNode *Res = DAG.UpdateNodeOperands(N, NewOps);
2298 if (Res == N)
2299 return SDValue(Res, 0);
2300
2301 // Update triggered CSE, do our own replacement since caller can't.
2302 ReplaceValueWith(SDValue(N, 0), SDValue(Res, 0));
2303 ReplaceValueWith(SDValue(N, 1), SDValue(Res, 1));
2304 return SDValue();
2305}
2306
2307SDValue DAGTypeLegalizer::PromoteIntOp_MSCATTER(MaskedScatterSDNode *N,
2308 unsigned OpNo) {
2309 bool TruncateStore = N->isTruncatingStore();
2310 SmallVector<SDValue, 5> NewOps(N->op_begin(), N->op_end());
2311
2312 if (OpNo == 2) {
2313 // The Mask
2314 EVT DataVT = N->getValue().getValueType();
2315 NewOps[OpNo] = PromoteTargetBoolean(N->getOperand(OpNo), DataVT);
2316 } else if (OpNo == 4) {
2317 // The Index
2318 if (N->isIndexSigned())
2319 // Need to sign extend the index since the bits will likely be used.
2320 NewOps[OpNo] = SExtPromotedInteger(N->getOperand(OpNo));
2321 else
2322 NewOps[OpNo] = ZExtPromotedInteger(N->getOperand(OpNo));
2323 } else {
2324 NewOps[OpNo] = GetPromotedInteger(N->getOperand(OpNo));
2325 TruncateStore = true;
2326 }
2327
2328 return DAG.getMaskedScatter(DAG.getVTList(MVT::Other), N->getMemoryVT(),
2329 SDLoc(N), NewOps, N->getMemOperand(),
2330 N->getIndexType(), TruncateStore);
2331}
2332
2333SDValue DAGTypeLegalizer::PromoteIntOp_TRUNCATE(SDNode *N) {
2334 SDValue Op = GetPromotedInteger(N->getOperand(0));
2335 if (N->getOpcode() == ISD::VP_TRUNCATE)
2336 return DAG.getNode(ISD::VP_TRUNCATE, SDLoc(N), N->getValueType(0), Op,
2337 N->getOperand(1), N->getOperand(2));
2338 return DAG.getNode(ISD::TRUNCATE, SDLoc(N), N->getValueType(0), Op);
2339}
2340
2341SDValue DAGTypeLegalizer::PromoteIntOp_UINT_TO_FP(SDNode *N) {
2342 if (N->getOpcode() == ISD::VP_UINT_TO_FP)
2343 return SDValue(DAG.UpdateNodeOperands(N,
2344 ZExtPromotedInteger(N->getOperand(0)),
2345 N->getOperand(1), N->getOperand(2)),
2346 0);
2347 return SDValue(DAG.UpdateNodeOperands(N,
2348 ZExtPromotedInteger(N->getOperand(0))), 0);
2349}
2350
2351SDValue DAGTypeLegalizer::PromoteIntOp_STRICT_UINT_TO_FP(SDNode *N) {
2352 return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0),
2353 ZExtPromotedInteger(N->getOperand(1))), 0);
2354}
2355
2356SDValue DAGTypeLegalizer::PromoteIntOp_ZERO_EXTEND(SDNode *N) {
2357 SDLoc dl(N);
2358 SDValue Op = GetPromotedInteger(N->getOperand(0));
2359 Op = DAG.getNode(ISD::ANY_EXTEND, dl, N->getValueType(0), Op);
2360 return DAG.getZeroExtendInReg(Op, dl, N->getOperand(0).getValueType());
2361}
2362
2363SDValue DAGTypeLegalizer::PromoteIntOp_VP_ZERO_EXTEND(SDNode *N) {
2364 SDLoc dl(N);
2365 EVT VT = N->getValueType(0);
2366 SDValue Op = GetPromotedInteger(N->getOperand(0));
2367 // FIXME: There is no VP_ANY_EXTEND yet.
2368 Op = DAG.getNode(ISD::VP_ZERO_EXTEND, dl, VT, Op, N->getOperand(1),
2369 N->getOperand(2));
2371 N->getOperand(0).getScalarValueSizeInBits());
2372 return DAG.getNode(ISD::VP_AND, dl, VT, Op, DAG.getConstant(Imm, dl, VT),
2373 N->getOperand(1), N->getOperand(2));
2374}
2375
2376SDValue DAGTypeLegalizer::PromoteIntOp_ADDSUBO_CARRY(SDNode *N, unsigned OpNo) {
2377 assert(OpNo == 2 && "Don't know how to promote this operand!");
2378
2379 SDValue LHS = N->getOperand(0);
2380 SDValue RHS = N->getOperand(1);
2381 SDValue Carry = N->getOperand(2);
2382 SDLoc DL(N);
2383
2384 Carry = PromoteTargetBoolean(Carry, LHS.getValueType());
2385
2386 return SDValue(DAG.UpdateNodeOperands(N, LHS, RHS, Carry), 0);
2387}
2388
2389SDValue DAGTypeLegalizer::PromoteIntOp_FIX(SDNode *N) {
2390 SDValue Op2 = ZExtPromotedInteger(N->getOperand(2));
2391 return SDValue(
2392 DAG.UpdateNodeOperands(N, N->getOperand(0), N->getOperand(1), Op2), 0);
2393}
2394
2395SDValue DAGTypeLegalizer::PromoteIntOp_FRAMERETURNADDR(SDNode *N) {
2396 // Promote the RETURNADDR/FRAMEADDR argument to a supported integer width.
2397 SDValue Op = ZExtPromotedInteger(N->getOperand(0));
2398 return SDValue(DAG.UpdateNodeOperands(N, Op), 0);
2399}
2400
2401SDValue DAGTypeLegalizer::PromoteIntOp_ExpOp(SDNode *N) {
2402 bool IsStrict = N->isStrictFPOpcode();
2403 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
2404
2405 bool IsPowI =
2406 N->getOpcode() == ISD::FPOWI || N->getOpcode() == ISD::STRICT_FPOWI;
2407
2408 // The integer operand is the last operand in FPOWI (or FLDEXP) (so the result
2409 // and floating point operand is already type legalized).
2410 RTLIB::Libcall LC = IsPowI ? RTLIB::getPOWI(N->getValueType(0))
2411 : RTLIB::getLDEXP(N->getValueType(0));
2412
2413 if (LC == RTLIB::UNKNOWN_LIBCALL || !TLI.getLibcallName(LC)) {
2414 SDValue Op = SExtPromotedInteger(N->getOperand(1));
2415 return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0), Op), 0);
2416 }
2417
2418 // We can't just promote the exponent type in FPOWI, since we want to lower
2419 // the node to a libcall and we if we promote to a type larger than
2420 // sizeof(int) the libcall might not be according to the targets ABI. Instead
2421 // we rewrite to a libcall here directly, letting makeLibCall handle promotion
2422 // if the target accepts it according to shouldSignExtendTypeInLibCall.
2423
2424 unsigned OpOffset = IsStrict ? 1 : 0;
2425 // The exponent should fit in a sizeof(int) type for the libcall to be valid.
2426 assert(DAG.getLibInfo().getIntSize() ==
2427 N->getOperand(1 + OpOffset).getValueType().getSizeInBits() &&
2428 "POWI exponent should match with sizeof(int) when doing the libcall.");
2430 CallOptions.setSExt(true);
2431 SDValue Ops[2] = {N->getOperand(0 + OpOffset), N->getOperand(1 + OpOffset)};
2432 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(
2433 DAG, LC, N->getValueType(0), Ops, CallOptions, SDLoc(N), Chain);
2434 ReplaceValueWith(SDValue(N, 0), Tmp.first);
2435 if (IsStrict)
2436 ReplaceValueWith(SDValue(N, 1), Tmp.second);
2437 return SDValue();
2438}
2439
2441 switch (N->getOpcode()) {
2442 default:
2443 llvm_unreachable("Expected integer vector reduction");
2444 case ISD::VECREDUCE_ADD:
2445 case ISD::VECREDUCE_MUL:
2446 case ISD::VECREDUCE_AND:
2447 case ISD::VECREDUCE_OR:
2448 case ISD::VECREDUCE_XOR:
2449 case ISD::VP_REDUCE_ADD:
2450 case ISD::VP_REDUCE_MUL:
2451 case ISD::VP_REDUCE_AND:
2452 case ISD::VP_REDUCE_OR:
2453 case ISD::VP_REDUCE_XOR:
2454 return ISD::ANY_EXTEND;
2457 case ISD::VP_REDUCE_SMAX:
2458 case ISD::VP_REDUCE_SMIN:
2459 return ISD::SIGN_EXTEND;
2462 case ISD::VP_REDUCE_UMAX:
2463 case ISD::VP_REDUCE_UMIN:
2464 return ISD::ZERO_EXTEND;
2465 }
2466}
2467
2468SDValue DAGTypeLegalizer::PromoteIntOpVectorReduction(SDNode *N, SDValue V) {
2469 switch (getExtendForIntVecReduction(N)) {
2470 default:
2471 llvm_unreachable("Impossible extension kind for integer reduction");
2472 case ISD::ANY_EXTEND:
2473 return GetPromotedInteger(V);
2474 case ISD::SIGN_EXTEND:
2475 return SExtPromotedInteger(V);
2476 case ISD::ZERO_EXTEND:
2477 return ZExtPromotedInteger(V);
2478 }
2479}
2480
2481SDValue DAGTypeLegalizer::PromoteIntOp_VECREDUCE(SDNode *N) {
2482 SDLoc dl(N);
2483 SDValue Op = PromoteIntOpVectorReduction(N, N->getOperand(0));
2484
2485 EVT OrigEltVT = N->getOperand(0).getValueType().getVectorElementType();
2486 EVT InVT = Op.getValueType();
2487 EVT EltVT = InVT.getVectorElementType();
2488 EVT ResVT = N->getValueType(0);
2489 unsigned Opcode = N->getOpcode();
2490
2491 // An i1 vecreduce_xor is equivalent to vecreduce_add, use that instead if
2492 // vecreduce_xor is not legal
2493 if (Opcode == ISD::VECREDUCE_XOR && OrigEltVT == MVT::i1 &&
2496 Opcode = ISD::VECREDUCE_ADD;
2497
2498 // An i1 vecreduce_or is equivalent to vecreduce_umax, use that instead if
2499 // vecreduce_or is not legal
2500 else if (Opcode == ISD::VECREDUCE_OR && OrigEltVT == MVT::i1 &&
2503 Opcode = ISD::VECREDUCE_UMAX;
2504 // Can't use promoteTargetBoolean here because we still need
2505 // to either sign_ext or zero_ext in the undefined case.
2506 switch (TLI.getBooleanContents(InVT)) {
2509 Op = ZExtPromotedInteger(N->getOperand(0));
2510 break;
2512 Op = SExtPromotedInteger(N->getOperand(0));
2513 break;
2514 }
2515 }
2516
2517 // An i1 vecreduce_and is equivalent to vecreduce_umin, use that instead if
2518 // vecreduce_and is not legal
2519 else if (Opcode == ISD::VECREDUCE_AND && OrigEltVT == MVT::i1 &&
2522 Opcode = ISD::VECREDUCE_UMIN;
2523 // Can't use promoteTargetBoolean here because we still need
2524 // to either sign_ext or zero_ext in the undefined case.
2525 switch (TLI.getBooleanContents(InVT)) {
2528 Op = ZExtPromotedInteger(N->getOperand(0));
2529 break;
2531 Op = SExtPromotedInteger(N->getOperand(0));
2532 break;
2533 }
2534 }
2535
2536 if (ResVT.bitsGE(EltVT))
2537 return DAG.getNode(Opcode, SDLoc(N), ResVT, Op);
2538
2539 // Result size must be >= element size. If this is not the case after
2540 // promotion, also promote the result type and then truncate.
2541 SDValue Reduce = DAG.getNode(Opcode, dl, EltVT, Op);
2542 return DAG.getNode(ISD::TRUNCATE, dl, ResVT, Reduce);
2543}
2544
2545SDValue DAGTypeLegalizer::PromoteIntOp_VP_REDUCE(SDNode *N, unsigned OpNo) {
2546 SDLoc DL(N);
2547 SDValue Op = N->getOperand(OpNo);
2548 SmallVector<SDValue, 4> NewOps(N->op_begin(), N->op_end());
2549
2550 if (OpNo == 2) { // Mask
2551 // Update in place.
2552 NewOps[2] = PromoteTargetBoolean(Op, N->getOperand(1).getValueType());
2553 return SDValue(DAG.UpdateNodeOperands(N, NewOps), 0);
2554 }
2555
2556 assert(OpNo == 1 && "Unexpected operand for promotion");
2557
2558 Op = PromoteIntOpVectorReduction(N, Op);
2559
2560 NewOps[OpNo] = Op;
2561
2562 EVT VT = N->getValueType(0);
2563 EVT EltVT = Op.getValueType().getScalarType();
2564
2565 if (VT.bitsGE(EltVT))
2566 return DAG.getNode(N->getOpcode(), SDLoc(N), VT, NewOps);
2567
2568 // Result size must be >= element/start-value size. If this is not the case
2569 // after promotion, also promote both the start value and result type and
2570 // then truncate.
2571 NewOps[0] =
2572 DAG.getNode(getExtendForIntVecReduction(N), DL, EltVT, N->getOperand(0));
2573 SDValue Reduce = DAG.getNode(N->getOpcode(), DL, EltVT, NewOps);
2574 return DAG.getNode(ISD::TRUNCATE, DL, VT, Reduce);
2575}
2576
2577SDValue DAGTypeLegalizer::PromoteIntOp_SET_ROUNDING(SDNode *N) {
2578 SDValue Op = ZExtPromotedInteger(N->getOperand(1));
2579 return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0), Op), 0);
2580}
2581
2582SDValue DAGTypeLegalizer::PromoteIntOp_STACKMAP(SDNode *N, unsigned OpNo) {
2583 assert(OpNo > 1); // Because the first two arguments are guaranteed legal.
2584 SmallVector<SDValue> NewOps(N->ops().begin(), N->ops().end());
2585 SDValue Operand = N->getOperand(OpNo);
2586 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), Operand.getValueType());
2587 NewOps[OpNo] = DAG.getNode(ISD::ANY_EXTEND, SDLoc(N), NVT, Operand);
2588 return SDValue(DAG.UpdateNodeOperands(N, NewOps), 0);
2589}
2590
2591SDValue DAGTypeLegalizer::PromoteIntOp_PATCHPOINT(SDNode *N, unsigned OpNo) {
2592 assert(OpNo >= 7);
2593 SmallVector<SDValue> NewOps(N->ops().begin(), N->ops().end());
2594 SDValue Operand = N->getOperand(OpNo);
2595 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), Operand.getValueType());
2596 NewOps[OpNo] = DAG.getNode(ISD::ANY_EXTEND, SDLoc(N), NVT, Operand);
2597 return SDValue(DAG.UpdateNodeOperands(N, NewOps), 0);
2598}
2599
2600SDValue DAGTypeLegalizer::PromoteIntOp_VP_STRIDED(SDNode *N, unsigned OpNo) {
2601 assert((N->getOpcode() == ISD::EXPERIMENTAL_VP_STRIDED_LOAD && OpNo == 3) ||
2602 (N->getOpcode() == ISD::EXPERIMENTAL_VP_STRIDED_STORE && OpNo == 4));
2603
2604 SmallVector<SDValue, 8> NewOps(N->op_begin(), N->op_end());
2605 NewOps[OpNo] = SExtPromotedInteger(N->getOperand(OpNo));
2606
2607 return SDValue(DAG.UpdateNodeOperands(N, NewOps), 0);
2608}
2609
2610SDValue DAGTypeLegalizer::PromoteIntOp_VP_SPLICE(SDNode *N, unsigned OpNo) {
2611 SmallVector<SDValue, 6> NewOps(N->op_begin(), N->op_end());
2612
2613 if (OpNo == 2) { // Offset operand
2614 NewOps[OpNo] = SExtPromotedInteger(N->getOperand(OpNo));
2615 return SDValue(DAG.UpdateNodeOperands(N, NewOps), 0);
2616 }
2617
2618 assert((OpNo == 4 || OpNo == 5) && "Unexpected operand for promotion");
2619
2620 NewOps[OpNo] = ZExtPromotedInteger(N->getOperand(OpNo));
2621 return SDValue(DAG.UpdateNodeOperands(N, NewOps), 0);
2622}
2623
2624//===----------------------------------------------------------------------===//
2625// Integer Result Expansion
2626//===----------------------------------------------------------------------===//
2627
2628/// ExpandIntegerResult - This method is called when the specified result of the
2629/// specified node is found to need expansion. At this point, the node may also
2630/// have invalid operands or may have other results that need promotion, we just
2631/// know that (at least) one result needs expansion.
2632void DAGTypeLegalizer::ExpandIntegerResult(SDNode *N, unsigned ResNo) {
2633 LLVM_DEBUG(dbgs() << "Expand integer result: "; N->dump(&DAG));
2634 SDValue Lo, Hi;
2635 Lo = Hi = SDValue();
2636
2637 // See if the target wants to custom expand this node.
2638 if (CustomLowerNode(N, N->getValueType(ResNo), true))
2639 return;
2640
2641 switch (N->getOpcode()) {
2642 default:
2643#ifndef NDEBUG
2644 dbgs() << "ExpandIntegerResult #" << ResNo << ": ";
2645 N->dump(&DAG); dbgs() << "\n";
2646#endif
2647 report_fatal_error("Do not know how to expand the result of this "
2648 "operator!");
2649
2650 case ISD::ARITH_FENCE: SplitRes_ARITH_FENCE(N, Lo, Hi); break;
2651 case ISD::MERGE_VALUES: SplitRes_MERGE_VALUES(N, ResNo, Lo, Hi); break;
2652 case ISD::SELECT: SplitRes_Select(N, Lo, Hi); break;
2653 case ISD::SELECT_CC: SplitRes_SELECT_CC(N, Lo, Hi); break;
2654 case ISD::UNDEF: SplitRes_UNDEF(N, Lo, Hi); break;
2655 case ISD::FREEZE: SplitRes_FREEZE(N, Lo, Hi); break;
2656
2657 case ISD::BITCAST: ExpandRes_BITCAST(N, Lo, Hi); break;
2658 case ISD::BUILD_PAIR: ExpandRes_BUILD_PAIR(N, Lo, Hi); break;
2659 case ISD::EXTRACT_ELEMENT: ExpandRes_EXTRACT_ELEMENT(N, Lo, Hi); break;
2660 case ISD::EXTRACT_VECTOR_ELT: ExpandRes_EXTRACT_VECTOR_ELT(N, Lo, Hi); break;
2661 case ISD::VAARG: ExpandRes_VAARG(N, Lo, Hi); break;
2662
2663 case ISD::ANY_EXTEND: ExpandIntRes_ANY_EXTEND(N, Lo, Hi); break;
2664 case ISD::AssertSext: ExpandIntRes_AssertSext(N, Lo, Hi); break;
2665 case ISD::AssertZext: ExpandIntRes_AssertZext(N, Lo, Hi); break;
2666 case ISD::BITREVERSE: ExpandIntRes_BITREVERSE(N, Lo, Hi); break;
2667 case ISD::BSWAP: ExpandIntRes_BSWAP(N, Lo, Hi); break;
2668 case ISD::PARITY: ExpandIntRes_PARITY(N, Lo, Hi); break;
2669 case ISD::Constant: ExpandIntRes_Constant(N, Lo, Hi); break;
2670 case ISD::ABS: ExpandIntRes_ABS(N, Lo, Hi); break;
2672 case ISD::CTLZ: ExpandIntRes_CTLZ(N, Lo, Hi); break;
2673 case ISD::CTPOP: ExpandIntRes_CTPOP(N, Lo, Hi); break;
2675 case ISD::CTTZ: ExpandIntRes_CTTZ(N, Lo, Hi); break;
2676 case ISD::GET_ROUNDING:ExpandIntRes_GET_ROUNDING(N, Lo, Hi); break;
2678 case ISD::FP_TO_SINT:
2680 case ISD::FP_TO_UINT: ExpandIntRes_FP_TO_XINT(N, Lo, Hi); break;
2682 case ISD::FP_TO_UINT_SAT: ExpandIntRes_FP_TO_XINT_SAT(N, Lo, Hi); break;
2683 case ISD::STRICT_LROUND:
2684 case ISD::STRICT_LRINT:
2685 case ISD::LROUND:
2686 case ISD::LRINT:
2688 case ISD::STRICT_LLRINT:
2689 case ISD::LLROUND:
2690 case ISD::LLRINT: ExpandIntRes_XROUND_XRINT(N, Lo, Hi); break;
2691 case ISD::LOAD: ExpandIntRes_LOAD(cast<LoadSDNode>(N), Lo, Hi); break;
2692 case ISD::MUL: ExpandIntRes_MUL(N, Lo, Hi); break;
2694 case ISD::READSTEADYCOUNTER: ExpandIntRes_READCOUNTER(N, Lo, Hi); break;
2695 case ISD::SDIV: ExpandIntRes_SDIV(N, Lo, Hi); break;
2696 case ISD::SIGN_EXTEND: ExpandIntRes_SIGN_EXTEND(N, Lo, Hi); break;
2697 case ISD::SIGN_EXTEND_INREG: ExpandIntRes_SIGN_EXTEND_INREG(N, Lo, Hi); break;
2698 case ISD::SREM: ExpandIntRes_SREM(N, Lo, Hi); break;
2699 case ISD::TRUNCATE: ExpandIntRes_TRUNCATE(N, Lo, Hi); break;
2700 case ISD::UDIV: ExpandIntRes_UDIV(N, Lo, Hi); break;
2701 case ISD::UREM: ExpandIntRes_UREM(N, Lo, Hi); break;
2702 case ISD::ZERO_EXTEND: ExpandIntRes_ZERO_EXTEND(N, Lo, Hi); break;
2703 case ISD::ATOMIC_LOAD: ExpandIntRes_ATOMIC_LOAD(N, Lo, Hi); break;
2704
2716 case ISD::ATOMIC_SWAP:
2717 case ISD::ATOMIC_CMP_SWAP: {
2718 std::pair<SDValue, SDValue> Tmp = ExpandAtomic(N);
2719 SplitInteger(Tmp.first, Lo, Hi);
2720 ReplaceValueWith(SDValue(N, 1), Tmp.second);
2721 break;
2722 }
2724 AtomicSDNode *AN = cast<AtomicSDNode>(N);
2725 SDVTList VTs = DAG.getVTList(N->getValueType(0), MVT::Other);
2726 SDValue Tmp = DAG.getAtomicCmpSwap(
2728 N->getOperand(0), N->getOperand(1), N->getOperand(2), N->getOperand(3),
2729 AN->getMemOperand());
2730
2731 // Expanding to the strong ATOMIC_CMP_SWAP node means we can determine
2732 // success simply by comparing the loaded value against the ingoing
2733 // comparison.
2734 SDValue Success = DAG.getSetCC(SDLoc(N), N->getValueType(1), Tmp,
2735 N->getOperand(2), ISD::SETEQ);
2736
2737 SplitInteger(Tmp, Lo, Hi);
2738 ReplaceValueWith(SDValue(N, 1), Success);
2739 ReplaceValueWith(SDValue(N, 2), Tmp.getValue(1));
2740 break;
2741 }
2742
2743 case ISD::AND:
2744 case ISD::OR:
2745 case ISD::XOR: ExpandIntRes_Logical(N, Lo, Hi); break;
2746
2747 case ISD::UMAX:
2748 case ISD::SMAX:
2749 case ISD::UMIN:
2750 case ISD::SMIN: ExpandIntRes_MINMAX(N, Lo, Hi); break;
2751
2752 case ISD::ADD:
2753 case ISD::SUB: ExpandIntRes_ADDSUB(N, Lo, Hi); break;
2754
2755 case ISD::ADDC:
2756 case ISD::SUBC: ExpandIntRes_ADDSUBC(N, Lo, Hi); break;
2757
2758 case ISD::ADDE:
2759 case ISD::SUBE: ExpandIntRes_ADDSUBE(N, Lo, Hi); break;
2760
2761 case ISD::UADDO_CARRY:
2762 case ISD::USUBO_CARRY: ExpandIntRes_UADDSUBO_CARRY(N, Lo, Hi); break;
2763
2764 case ISD::SADDO_CARRY:
2765 case ISD::SSUBO_CARRY: ExpandIntRes_SADDSUBO_CARRY(N, Lo, Hi); break;
2766
2767 case ISD::SHL:
2768 case ISD::SRA:
2769 case ISD::SRL: ExpandIntRes_Shift(N, Lo, Hi); break;
2770
2771 case ISD::SADDO:
2772 case ISD::SSUBO: ExpandIntRes_SADDSUBO(N, Lo, Hi); break;
2773 case ISD::UADDO:
2774 case ISD::USUBO: ExpandIntRes_UADDSUBO(N, Lo, Hi); break;
2775 case ISD::UMULO:
2776 case ISD::SMULO: ExpandIntRes_XMULO(N, Lo, Hi); break;
2777
2778 case ISD::SADDSAT:
2779 case ISD::UADDSAT:
2780 case ISD::SSUBSAT:
2781 case ISD::USUBSAT: ExpandIntRes_ADDSUBSAT(N, Lo, Hi); break;
2782
2783 case ISD::SSHLSAT:
2784 case ISD::USHLSAT: ExpandIntRes_SHLSAT(N, Lo, Hi); break;
2785
2786 case ISD::SMULFIX:
2787 case ISD::SMULFIXSAT:
2788 case ISD::UMULFIX:
2789 case ISD::UMULFIXSAT: ExpandIntRes_MULFIX(N, Lo, Hi); break;
2790
2791 case ISD::SDIVFIX:
2792 case ISD::SDIVFIXSAT:
2793 case ISD::UDIVFIX:
2794 case ISD::UDIVFIXSAT: ExpandIntRes_DIVFIX(N, Lo, Hi); break;
2795
2796 case ISD::VECREDUCE_ADD:
2797 case ISD::VECREDUCE_MUL:
2798 case ISD::VECREDUCE_AND:
2799 case ISD::VECREDUCE_OR:
2800 case ISD::VECREDUCE_XOR:
2804 case ISD::VECREDUCE_UMIN: ExpandIntRes_VECREDUCE(N, Lo, Hi); break;
2805
2806 case ISD::ROTL:
2807 case ISD::ROTR:
2808 ExpandIntRes_Rotate(N, Lo, Hi);
2809 break;
2810
2811 case ISD::FSHL:
2812 case ISD::FSHR:
2813 ExpandIntRes_FunnelShift(N, Lo, Hi);
2814 break;
2815
2816 case ISD::VSCALE:
2817 ExpandIntRes_VSCALE(N, Lo, Hi);
2818 break;
2819 }
2820
2821 // If Lo/Hi is null, the sub-method took care of registering results etc.
2822 if (Lo.getNode())
2823 SetExpandedInteger(SDValue(N, ResNo), Lo, Hi);
2824}
2825
2826/// Lower an atomic node to the appropriate builtin call.
2827std::pair <SDValue, SDValue> DAGTypeLegalizer::ExpandAtomic(SDNode *Node) {
2828 unsigned Opc = Node->getOpcode();
2829 MVT VT = cast<AtomicSDNode>(Node)->getMemoryVT().getSimpleVT();
2830 AtomicOrdering order = cast<AtomicSDNode>(Node)->getMergedOrdering();
2831 // Lower to outline atomic libcall if outline atomics enabled,
2832 // or to sync libcall otherwise
2833 RTLIB::Libcall LC = RTLIB::getOUTLINE_ATOMIC(Opc, order, VT);
2834 EVT RetVT = Node->getValueType(0);
2837 if (TLI.getLibcallName(LC)) {
2838 Ops.append(Node->op_begin() + 2, Node->op_end());
2839 Ops.push_back(Node->getOperand(1));
2840 } else {
2841 LC = RTLIB::getSYNC(Opc, VT);
2842 assert(LC != RTLIB::UNKNOWN_LIBCALL &&
2843 "Unexpected atomic op or value type!");
2844 Ops.append(Node->op_begin() + 1, Node->op_end());
2845 }
2846 return TLI.makeLibCall(DAG, LC, RetVT, Ops, CallOptions, SDLoc(Node),
2847 Node->getOperand(0));
2848}
2849
2850/// N is a shift by a value that needs to be expanded,
2851/// and the shift amount is a constant 'Amt'. Expand the operation.
2852void DAGTypeLegalizer::ExpandShiftByConstant(SDNode *N, const APInt &Amt,
2853 SDValue &Lo, SDValue &Hi) {
2854 SDLoc DL(N);
2855 // Expand the incoming operand to be shifted, so that we have its parts
2856 SDValue InL, InH;
2857 GetExpandedInteger(N->getOperand(0), InL, InH);
2858
2859 // Though Amt shouldn't usually be 0, it's possible. E.g. when legalization
2860 // splitted a vector shift, like this: <op1, op2> SHL <0, 2>.
2861 if (!Amt) {
2862 Lo = InL;
2863 Hi = InH;
2864 return;
2865 }
2866
2867 EVT NVT = InL.getValueType();
2868 unsigned VTBits = N->getValueType(0).getSizeInBits();
2869 unsigned NVTBits = NVT.getSizeInBits();
2870
2871 if (N->getOpcode() == ISD::SHL) {
2872 if (Amt.uge(VTBits)) {
2873 Lo = Hi = DAG.getConstant(0, DL, NVT);
2874 } else if (Amt.ugt(NVTBits)) {
2875 Lo = DAG.getConstant(0, DL, NVT);
2876 Hi = DAG.getNode(ISD::SHL, DL, NVT, InL,
2877 DAG.getShiftAmountConstant(Amt - NVTBits, NVT, DL));
2878 } else if (Amt == NVTBits) {
2879 Lo = DAG.getConstant(0, DL, NVT);
2880 Hi = InL;
2881 } else {
2882 Lo = DAG.getNode(ISD::SHL, DL, NVT, InL,
2883 DAG.getShiftAmountConstant(Amt, NVT, DL));
2884 Hi = DAG.getNode(
2885 ISD::OR, DL, NVT,
2886 DAG.getNode(ISD::SHL, DL, NVT, InH,
2887 DAG.getShiftAmountConstant(Amt, NVT, DL)),
2888 DAG.getNode(ISD::SRL, DL, NVT, InL,
2889 DAG.getShiftAmountConstant(-Amt + NVTBits, NVT, DL)));
2890 }
2891 return;
2892 }
2893
2894 if (N->getOpcode() == ISD::SRL) {
2895 if (Amt.uge(VTBits)) {
2896 Lo = Hi = DAG.getConstant(0, DL, NVT);
2897 } else if (Amt.ugt(NVTBits)) {
2898 Lo = DAG.getNode(ISD::SRL, DL, NVT, InH,
2899 DAG.getShiftAmountConstant(Amt - NVTBits, NVT, DL));
2900 Hi = DAG.getConstant(0, DL, NVT);
2901 } else if (Amt == NVTBits) {
2902 Lo = InH;
2903 Hi = DAG.getConstant(0, DL, NVT);
2904 } else {
2905 Lo = DAG.getNode(
2906 ISD::OR, DL, NVT,
2907 DAG.getNode(ISD::SRL, DL, NVT, InL,
2908 DAG.getShiftAmountConstant(Amt, NVT, DL)),
2909 DAG.getNode(ISD::SHL, DL, NVT, InH,
2910 DAG.getShiftAmountConstant(-Amt + NVTBits, NVT, DL)));
2911 Hi = DAG.getNode(ISD::SRL, DL, NVT, InH,
2912 DAG.getShiftAmountConstant(Amt, NVT, DL));
2913 }
2914 return;
2915 }
2916
2917 assert(N->getOpcode() == ISD::SRA && "Unknown shift!");
2918 if (Amt.uge(VTBits)) {
2919 Hi = Lo = DAG.getNode(ISD::SRA, DL, NVT, InH,
2920 DAG.getShiftAmountConstant(NVTBits - 1, NVT, DL));
2921 } else if (Amt.ugt(NVTBits)) {
2922 Lo = DAG.getNode(ISD::SRA, DL, NVT, InH,
2923 DAG.getShiftAmountConstant(Amt - NVTBits, NVT, DL));
2924 Hi = DAG.getNode(ISD::SRA, DL, NVT, InH,
2925 DAG.getShiftAmountConstant(NVTBits - 1, NVT, DL));
2926 } else if (Amt == NVTBits) {
2927 Lo = InH;
2928 Hi = DAG.getNode(ISD::SRA, DL, NVT, InH,
2929 DAG.getShiftAmountConstant(NVTBits - 1, NVT, DL));
2930 } else {
2931 Lo = DAG.getNode(
2932 ISD::OR, DL, NVT,
2933 DAG.getNode(ISD::SRL, DL, NVT, InL,
2934 DAG.getShiftAmountConstant(Amt, NVT, DL)),
2935 DAG.getNode(ISD::SHL, DL, NVT, InH,
2936 DAG.getShiftAmountConstant(-Amt + NVTBits, NVT, DL)));
2937 Hi = DAG.getNode(ISD::SRA, DL, NVT, InH,
2938 DAG.getShiftAmountConstant(Amt, NVT, DL));
2939 }
2940}
2941
2942/// ExpandShiftWithKnownAmountBit - Try to determine whether we can simplify
2943/// this shift based on knowledge of the high bit of the shift amount. If we
2944/// can tell this, we know that it is >= 32 or < 32, without knowing the actual
2945/// shift amount.
2946bool DAGTypeLegalizer::
2947ExpandShiftWithKnownAmountBit(SDNode *N, SDValue &Lo, SDValue &Hi) {
2948 unsigned Opc = N->getOpcode();
2949 SDValue In = N->getOperand(0);
2950 SDValue Amt = N->getOperand(1);
2951 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
2952 EVT ShTy = Amt.getValueType();
2953 unsigned ShBits = ShTy.getScalarSizeInBits();
2954 unsigned NVTBits = NVT.getScalarSizeInBits();
2955 assert(isPowerOf2_32(NVTBits) &&
2956 "Expanded integer type size not a power of two!");
2957 SDLoc dl(N);
2958
2959 APInt HighBitMask = APInt::getHighBitsSet(ShBits, ShBits - Log2_32(NVTBits));
2960 KnownBits Known = DAG.computeKnownBits(Amt);
2961
2962 // If we don't know anything about the high bits, exit.
2963 if (((Known.Zero | Known.One) & HighBitMask) == 0)
2964 return false;
2965
2966 // Get the incoming operand to be shifted.
2967 SDValue InL, InH;
2968 GetExpandedInteger(In, InL, InH);
2969
2970 // If we know that any of the high bits of the shift amount are one, then we
2971 // can do this as a couple of simple shifts.
2972 if (Known.One.intersects(HighBitMask)) {
2973 // Mask out the high bit, which we know is set.
2974 Amt = DAG.getNode(ISD::AND, dl, ShTy, Amt,
2975 DAG.getConstant(~HighBitMask, dl, ShTy));
2976
2977 switch (Opc) {
2978 default: llvm_unreachable("Unknown shift");
2979 case ISD::SHL:
2980 Lo = DAG.getConstant(0, dl, NVT); // Low part is zero.
2981 Hi = DAG.getNode(ISD::SHL, dl, NVT, InL, Amt); // High part from Lo part.
2982 return true;
2983 case ISD::SRL:
2984 Hi = DAG.getConstant(0, dl, NVT); // Hi part is zero.
2985 Lo = DAG.getNode(ISD::SRL, dl, NVT, InH, Amt); // Lo part from Hi part.
2986 return true;
2987 case ISD::SRA:
2988 Hi = DAG.getNode(ISD::SRA, dl, NVT, InH, // Sign extend high part.
2989 DAG.getConstant(NVTBits - 1, dl, ShTy));
2990 Lo = DAG.getNode(ISD::SRA, dl, NVT, InH, Amt); // Lo part from Hi part.
2991 return true;
2992 }
2993 }
2994
2995 // If we know that all of the high bits of the shift amount are zero, then we
2996 // can do this as a couple of simple shifts.
2997 if (HighBitMask.isSubsetOf(Known.Zero)) {
2998 // Calculate 31-x. 31 is used instead of 32 to avoid creating an undefined
2999 // shift if x is zero. We can use XOR here because x is known to be smaller
3000 // than 32.
3001 SDValue Amt2 = DAG.getNode(ISD::XOR, dl, ShTy, Amt,
3002 DAG.getConstant(NVTBits - 1, dl, ShTy));
3003
3004 unsigned Op1, Op2;
3005 switch (Opc) {
3006 default: llvm_unreachable("Unknown shift");
3007 case ISD::SHL: Op1 = ISD::SHL; Op2 = ISD::SRL; break;
3008 case ISD::SRL:
3009 case ISD::SRA: Op1 = ISD::SRL; Op2 = ISD::SHL; break;
3010 }
3011
3012 // When shifting right the arithmetic for Lo and Hi is swapped.
3013 if (Opc != ISD::SHL)
3014 std::swap(InL, InH);
3015
3016 // Use a little trick to get the bits that move from Lo to Hi. First
3017 // shift by one bit.
3018 SDValue Sh1 = DAG.getNode(Op2, dl, NVT, InL, DAG.getConstant(1, dl, ShTy));
3019 // Then compute the remaining shift with amount-1.
3020 SDValue Sh2 = DAG.getNode(Op2, dl, NVT, Sh1, Amt2);
3021
3022 Lo = DAG.getNode(Opc, dl, NVT, InL, Amt);
3023 Hi = DAG.getNode(ISD::OR, dl, NVT, DAG.getNode(Op1, dl, NVT, InH, Amt),Sh2);
3024
3025 if (Opc != ISD::SHL)
3026 std::swap(Hi, Lo);
3027 return true;
3028 }
3029
3030 return false;
3031}
3032
3033/// ExpandShiftWithUnknownAmountBit - Fully general expansion of integer shift
3034/// of any size.
3035bool DAGTypeLegalizer::
3036ExpandShiftWithUnknownAmountBit(SDNode *N, SDValue &Lo, SDValue &Hi) {
3037 SDValue Amt = N->getOperand(1);
3038 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
3039 EVT ShTy = Amt.getValueType();
3040 unsigned NVTBits = NVT.getSizeInBits();
3041 assert(isPowerOf2_32(NVTBits) &&
3042 "Expanded integer type size not a power of two!");
3043 SDLoc dl(N);
3044
3045 // Get the incoming operand to be shifted.
3046 SDValue InL, InH;
3047 GetExpandedInteger(N->getOperand(0), InL, InH);
3048
3049 SDValue NVBitsNode = DAG.getConstant(NVTBits, dl, ShTy);
3050 SDValue AmtExcess = DAG.getNode(ISD::SUB, dl, ShTy, Amt, NVBitsNode);
3051 SDValue AmtLack = DAG.getNode(ISD::SUB, dl, ShTy, NVBitsNode, Amt);
3052 SDValue isShort = DAG.getSetCC(dl, getSetCCResultType(ShTy),
3053 Amt, NVBitsNode, ISD::SETULT);
3054 SDValue isZero = DAG.getSetCC(dl, getSetCCResultType(ShTy),
3055 Amt, DAG.getConstant(0, dl, ShTy),
3056 ISD::SETEQ);
3057
3058 SDValue LoS, HiS, LoL, HiL;
3059 switch (N->getOpcode()) {
3060 default: llvm_unreachable("Unknown shift");
3061 case ISD::SHL:
3062 // Short: ShAmt < NVTBits
3063 LoS = DAG.getNode(ISD::SHL, dl, NVT, InL, Amt);
3064 HiS = DAG.getNode(ISD::OR, dl, NVT,
3065 DAG.getNode(ISD::SHL, dl, NVT, InH, Amt),
3066 DAG.getNode(ISD::SRL, dl, NVT, InL, AmtLack));
3067
3068 // Long: ShAmt >= NVTBits
3069 LoL = DAG.getConstant(0, dl, NVT); // Lo part is zero.
3070 HiL = DAG.getNode(ISD::SHL, dl, NVT, InL, AmtExcess); // Hi from Lo part.
3071
3072 Lo = DAG.getSelect(dl, NVT, isShort, LoS, LoL);
3073 Hi = DAG.getSelect(dl, NVT, isZero, InH,
3074 DAG.getSelect(dl, NVT, isShort, HiS, HiL));
3075 return true;
3076 case ISD::SRL:
3077 // Short: ShAmt < NVTBits
3078 HiS = DAG.getNode(ISD::SRL, dl, NVT, InH, Amt);
3079 LoS = DAG.getNode(ISD::OR, dl, NVT,
3080 DAG.getNode(ISD::SRL, dl, NVT, InL, Amt),
3081 // FIXME: If Amt is zero, the following shift generates an undefined result
3082 // on some architectures.
3083 DAG.getNode(ISD::SHL, dl, NVT, InH, AmtLack));
3084
3085 // Long: ShAmt >= NVTBits
3086 HiL = DAG.getConstant(0, dl, NVT); // Hi part is zero.
3087 LoL = DAG.getNode(ISD::SRL, dl, NVT, InH, AmtExcess); // Lo from Hi part.
3088
3089 Lo = DAG.getSelect(dl, NVT, isZero, InL,
3090 DAG.getSelect(dl, NVT, isShort, LoS, LoL));
3091 Hi = DAG.getSelect(dl, NVT, isShort, HiS, HiL);
3092 return true;
3093 case ISD::SRA:
3094 // Short: ShAmt < NVTBits
3095 HiS = DAG.getNode(ISD::SRA, dl, NVT, InH, Amt);
3096 LoS = DAG.getNode(ISD::OR, dl, NVT,
3097 DAG.getNode(ISD::SRL, dl, NVT, InL, Amt),
3098 DAG.getNode(ISD::SHL, dl, NVT, InH, AmtLack));
3099
3100 // Long: ShAmt >= NVTBits
3101 HiL = DAG.getNode(ISD::SRA, dl, NVT, InH, // Sign of Hi part.
3102 DAG.getConstant(NVTBits - 1, dl, ShTy));
3103 LoL = DAG.getNode(ISD::SRA, dl, NVT, InH, AmtExcess); // Lo from Hi part.
3104
3105 Lo = DAG.getSelect(dl, NVT, isZero, InL,
3106 DAG.getSelect(dl, NVT, isShort, LoS, LoL));
3107 Hi = DAG.getSelect(dl, NVT, isShort, HiS, HiL);
3108 return true;
3109 }
3110}
3111
3112static std::pair<ISD::CondCode, ISD::NodeType> getExpandedMinMaxOps(int Op) {
3113
3114 switch (Op) {
3115 default: llvm_unreachable("invalid min/max opcode");
3116 case ISD::SMAX:
3117 return std::make_pair(ISD::SETGT, ISD::UMAX);
3118 case ISD::UMAX:
3119 return std::make_pair(ISD::SETUGT, ISD::UMAX);
3120 case ISD::SMIN:
3121 return std::make_pair(ISD::SETLT, ISD::UMIN);
3122 case ISD::UMIN:
3123 return std::make_pair(ISD::SETULT, ISD::UMIN);
3124 }
3125}
3126
3127void DAGTypeLegalizer::ExpandIntRes_MINMAX(SDNode *N,
3128 SDValue &Lo, SDValue &Hi) {
3129 SDLoc DL(N);
3130
3131 SDValue LHS = N->getOperand(0);
3132 SDValue RHS = N->getOperand(1);
3133
3134 // If the upper halves are all sign bits, then we can perform the MINMAX on
3135 // the lower half and sign-extend the result to the upper half.
3136 unsigned NumBits = N->getValueType(0).getScalarSizeInBits();
3137 unsigned NumHalfBits = NumBits / 2;
3138 if (DAG.ComputeNumSignBits(LHS) > NumHalfBits &&
3139 DAG.ComputeNumSignBits(RHS) > NumHalfBits) {
3140 SDValue LHSL, LHSH, RHSL, RHSH;
3141 GetExpandedInteger(LHS, LHSL, LHSH);
3142 GetExpandedInteger(RHS, RHSL, RHSH);
3143 EVT NVT = LHSL.getValueType();
3144
3145 Lo = DAG.getNode(N->getOpcode(), DL, NVT, LHSL, RHSL);
3146 Hi = DAG.getNode(ISD::SRA, DL, NVT, Lo,
3147 DAG.getShiftAmountConstant(NumHalfBits - 1, NVT, DL));
3148 return;
3149 }
3150
3151 // The Lo of smin(X, -1) is LHSL if X is negative. Otherwise it's -1.
3152 // The Lo of smax(X, 0) is 0 if X is negative. Otherwise it's LHSL.
3153 if ((N->getOpcode() == ISD::SMAX && isNullConstant(RHS)) ||
3154 (N->getOpcode() == ISD::SMIN && isAllOnesConstant(RHS))) {
3155 SDValue LHSL, LHSH, RHSL, RHSH;
3156 GetExpandedInteger(LHS, LHSL, LHSH);
3157 GetExpandedInteger(RHS, RHSL, RHSH);
3158 EVT NVT = LHSL.getValueType();
3159 EVT CCT = getSetCCResultType(NVT);
3160
3161 SDValue HiNeg =
3162 DAG.getSetCC(DL, CCT, LHSH, DAG.getConstant(0, DL, NVT), ISD::SETLT);
3163 if (N->getOpcode() == ISD::SMIN) {
3164 Lo = DAG.getSelect(DL, NVT, HiNeg, LHSL, DAG.getConstant(-1, DL, NVT));
3165 } else {
3166 Lo = DAG.getSelect(DL, NVT, HiNeg, DAG.getConstant(0, DL, NVT), LHSL);
3167 }
3168 Hi = DAG.getNode(N->getOpcode(), DL, NVT, {LHSH, RHSH});
3169 return;
3170 }
3171
3172 const APInt *RHSVal = nullptr;
3173 if (auto *RHSConst = dyn_cast<ConstantSDNode>(RHS))
3174 RHSVal = &RHSConst->getAPIntValue();
3175
3176 // The high half of MIN/MAX is always just the the MIN/MAX of the
3177 // high halves of the operands. Expand this way if it appears profitable.
3178 if (RHSVal && (N->getOpcode() == ISD::UMIN || N->getOpcode() == ISD::UMAX) &&
3179 (RHSVal->countLeadingOnes() >= NumHalfBits ||
3180 RHSVal->countLeadingZeros() >= NumHalfBits)) {
3181 SDValue LHSL, LHSH, RHSL, RHSH;
3182 GetExpandedInteger(LHS, LHSL, LHSH);
3183 GetExpandedInteger(RHS, RHSL, RHSH);
3184 EVT NVT = LHSL.getValueType();
3185 EVT CCT = getSetCCResultType(NVT);
3186
3187 ISD::NodeType LoOpc;
3188 ISD::CondCode CondC;
3189 std::tie(CondC, LoOpc) = getExpandedMinMaxOps(N->getOpcode());
3190
3191 Hi = DAG.getNode(N->getOpcode(), DL, NVT, {LHSH, RHSH});
3192 // We need to know whether to select Lo part that corresponds to 'winning'
3193 // Hi part or if Hi parts are equal.
3194 SDValue IsHiLeft = DAG.getSetCC(DL, CCT, LHSH, RHSH, CondC);
3195 SDValue IsHiEq = DAG.getSetCC(DL, CCT, LHSH, RHSH, ISD::SETEQ);
3196
3197 // Lo part corresponding to the 'winning' Hi part
3198 SDValue LoCmp = DAG.getSelect(DL, NVT, IsHiLeft, LHSL, RHSL);
3199
3200 // Recursed Lo part if Hi parts are equal, this uses unsigned version
3201 SDValue LoMinMax = DAG.getNode(LoOpc, DL, NVT, {LHSL, RHSL});
3202
3203 Lo = DAG.getSelect(DL, NVT, IsHiEq, LoMinMax, LoCmp);
3204 return;
3205 }
3206
3207 // Expand to "a < b ? a : b" etc. Prefer ge/le if that simplifies
3208 // the compare.
3209 ISD::CondCode Pred;
3210 switch (N->getOpcode()) {
3211 default: llvm_unreachable("How did we get here?");
3212 case ISD::SMAX:
3213 if (RHSVal && RHSVal->countTrailingZeros() >= NumHalfBits)
3214 Pred = ISD::SETGE;
3215 else
3216 Pred = ISD::SETGT;
3217 break;
3218 case ISD::SMIN:
3219 if (RHSVal && RHSVal->countTrailingOnes() >= NumHalfBits)
3220 Pred = ISD::SETLE;
3221 else
3222 Pred = ISD::SETLT;
3223 break;
3224 case ISD::UMAX:
3225 if (RHSVal && RHSVal->countTrailingZeros() >= NumHalfBits)
3226 Pred = ISD::SETUGE;
3227 else
3228 Pred = ISD::SETUGT;
3229 break;
3230 case ISD::UMIN:
3231 if (RHSVal && RHSVal->countTrailingOnes() >= NumHalfBits)
3232 Pred = ISD::SETULE;
3233 else
3234 Pred = ISD::SETULT;
3235 break;
3236 }
3237 EVT VT = N->getValueType(0);
3238 EVT CCT = getSetCCResultType(VT);
3239 SDValue Cond = DAG.getSetCC(DL, CCT, LHS, RHS, Pred);
3240 SDValue Result = DAG.getSelect(DL, VT, Cond, LHS, RHS);
3241 SplitInteger(Result, Lo, Hi);
3242}
3243
3244void DAGTypeLegalizer::ExpandIntRes_ADDSUB(SDNode *N,
3245 SDValue &Lo, SDValue &Hi) {
3246 SDLoc dl(N);
3247 // Expand the subcomponents.
3248 SDValue LHSL, LHSH, RHSL, RHSH;
3249 GetExpandedInteger(N->getOperand(0), LHSL, LHSH);
3250 GetExpandedInteger(N->getOperand(1), RHSL, RHSH);
3251
3252 EVT NVT = LHSL.getValueType();
3253 SDValue LoOps[2] = { LHSL, RHSL };
3254 SDValue HiOps[3] = { LHSH, RHSH };
3255
3256 bool HasOpCarry = TLI.isOperationLegalOrCustom(
3257 N->getOpcode() == ISD::ADD ? ISD::UADDO_CARRY : ISD::USUBO_CARRY,
3258 TLI.getTypeToExpandTo(*DAG.getContext(), NVT));
3259 if (HasOpCarry) {
3260 SDVTList VTList = DAG.getVTList(NVT, getSetCCResultType(NVT));
3261 if (N->getOpcode() == ISD::ADD) {
3262 Lo = DAG.getNode(ISD::UADDO, dl, VTList, LoOps);
3263 HiOps[2] = Lo.getValue(1);
3264 Hi = DAG.computeKnownBits(HiOps[2]).isZero()
3265 ? DAG.getNode(ISD::UADDO, dl, VTList, ArrayRef(HiOps, 2))
3266 : DAG.getNode(ISD::UADDO_CARRY, dl, VTList, HiOps);
3267 } else {
3268 Lo = DAG.getNode(ISD::USUBO, dl, VTList, LoOps);
3269 HiOps[2] = Lo.getValue(1);
3270 Hi = DAG.computeKnownBits(HiOps[2]).isZero()
3271 ? DAG.getNode(ISD::USUBO, dl, VTList, ArrayRef(HiOps, 2))
3272 : DAG.getNode(ISD::USUBO_CARRY, dl, VTList, HiOps);
3273 }
3274 return;
3275 }
3276
3277 // Do not generate ADDC/ADDE or SUBC/SUBE if the target does not support
3278 // them. TODO: Teach operation legalization how to expand unsupported
3279 // ADDC/ADDE/SUBC/SUBE. The problem is that these operations generate
3280 // a carry of type MVT::Glue, but there doesn't seem to be any way to
3281 // generate a value of this type in the expanded code sequence.
3282 bool hasCarry =
3283 TLI.isOperationLegalOrCustom(N->getOpcode() == ISD::ADD ?
3285 TLI.getTypeToExpandTo(*DAG.getContext(), NVT));
3286
3287 if (hasCarry) {
3288 SDVTList VTList = DAG.getVTList(NVT, MVT::Glue);
3289 if (N->getOpcode() == ISD::ADD) {
3290 Lo = DAG.getNode(ISD::ADDC, dl, VTList, LoOps);
3291 HiOps[2] = Lo.getValue(1);
3292 Hi = DAG.getNode(ISD::ADDE, dl, VTList, HiOps);
3293 } else {
3294 Lo = DAG.getNode(ISD::SUBC, dl, VTList, LoOps);
3295 HiOps[2] = Lo.getValue(1);
3296 Hi = DAG.getNode(ISD::SUBE, dl, VTList, HiOps);
3297 }
3298 return;
3299 }
3300
3301 bool hasOVF =
3302 TLI.isOperationLegalOrCustom(N->getOpcode() == ISD::ADD ?
3304 TLI.getTypeToExpandTo(*DAG.getContext(), NVT));
3306
3307 if (hasOVF) {
3308 EVT OvfVT = getSetCCResultType(NVT);
3309 SDVTList VTList = DAG.getVTList(NVT, OvfVT);
3310 int RevOpc;
3311 if (N->getOpcode() == ISD::ADD) {
3312 RevOpc = ISD::SUB;
3313 Lo = DAG.getNode(ISD::UADDO, dl, VTList, LoOps);
3314 Hi = DAG.getNode(ISD::ADD, dl, NVT, ArrayRef(HiOps, 2));
3315 } else {
3316 RevOpc = ISD::ADD;
3317 Lo = DAG.getNode(ISD::USUBO, dl, VTList, LoOps);
3318 Hi = DAG.getNode(ISD::SUB, dl, NVT, ArrayRef(HiOps, 2));
3319 }
3320 SDValue OVF = Lo.getValue(1);
3321
3322 switch (BoolType) {
3324 OVF = DAG.getNode(ISD::AND, dl, OvfVT, DAG.getConstant(1, dl, OvfVT), OVF);
3325 [[fallthrough]];
3327 OVF = DAG.getZExtOrTrunc(OVF, dl, NVT);
3328 Hi = DAG.getNode(N->getOpcode(), dl, NVT, Hi, OVF);
3329 break;
3331 OVF = DAG.getSExtOrTrunc(OVF, dl, NVT);
3332 Hi = DAG.getNode(RevOpc, dl, NVT, Hi, OVF);
3333 }
3334 return;
3335 }
3336
3337 if (N->getOpcode() == ISD::ADD) {
3338 Lo = DAG.getNode(ISD::ADD, dl, NVT, LoOps);
3339 Hi = DAG.getNode(ISD::ADD, dl, NVT, ArrayRef(HiOps, 2));
3340 SDValue Cmp;
3341 // Special case: X+1 has a carry out if X+1==0. This may reduce the live
3342 // range of X. We assume comparing with 0 is cheap.
3343 if (isOneConstant(LoOps[1]))
3344 Cmp = DAG.getSetCC(dl, getSetCCResultType(NVT), Lo,
3345 DAG.getConstant(0, dl, NVT), ISD::SETEQ);
3346 else if (isAllOnesConstant(LoOps[1])) {
3347 if (isAllOnesConstant(HiOps[1]))
3348 Cmp = DAG.getSetCC(dl, getSetCCResultType(NVT), LoOps[0],
3349 DAG.getConstant(0, dl, NVT), ISD::SETEQ);
3350 else
3351 Cmp = DAG.getSetCC(dl, getSetCCResultType(NVT), LoOps[0],
3352 DAG.getConstant(0, dl, NVT), ISD::SETNE);
3353 } else
3354 Cmp = DAG.getSetCC(dl, getSetCCResultType(NVT), Lo, LoOps[0],
3355 ISD::SETULT);
3356
3357 SDValue Carry;
3359 Carry = DAG.getZExtOrTrunc(Cmp, dl, NVT);
3360 else
3361 Carry = DAG.getSelect(dl, NVT, Cmp, DAG.getConstant(1, dl, NVT),
3362 DAG.getConstant(0, dl, NVT));
3363
3364 if (isAllOnesConstant(LoOps[1]) && isAllOnesConstant(HiOps[1]))
3365 Hi = DAG.getNode(ISD::SUB, dl, NVT, HiOps[0], Carry);
3366 else
3367 Hi = DAG.getNode(ISD::ADD, dl, NVT, Hi, Carry);
3368 } else {
3369 Lo = DAG.getNode(ISD::SUB, dl, NVT, LoOps);
3370 Hi = DAG.getNode(ISD::SUB, dl, NVT, ArrayRef(HiOps, 2));
3371 SDValue Cmp =
3372 DAG.getSetCC(dl, getSetCCResultType(LoOps[0].getValueType()),
3373 LoOps[0], LoOps[1], ISD::SETULT);
3374
3375 SDValue Borrow;
3377 Borrow = DAG.getZExtOrTrunc(Cmp, dl, NVT);
3378 else
3379 Borrow = DAG.getSelect(dl, NVT, Cmp, DAG.getConstant(1, dl, NVT),
3380 DAG.getConstant(0, dl, NVT));
3381
3382 Hi = DAG.getNode(ISD::SUB, dl, NVT, Hi, Borrow);
3383 }
3384}
3385
3386void DAGTypeLegalizer::ExpandIntRes_ADDSUBC(SDNode *N,
3387 SDValue &Lo, SDValue &Hi) {
3388 // Expand the subcomponents.
3389 SDValue LHSL, LHSH, RHSL, RHSH;
3390 SDLoc dl(N);
3391 GetExpandedInteger(N->getOperand(0), LHSL, LHSH);
3392 GetExpandedInteger(N->getOperand(1), RHSL, RHSH);
3393 SDVTList VTList = DAG.getVTList(LHSL.getValueType(), MVT::Glue);
3394 SDValue LoOps[2] = { LHSL, RHSL };
3395 SDValue HiOps[3] = { LHSH, RHSH };
3396
3397 if (N->getOpcode() == ISD::ADDC) {
3398 Lo = DAG.getNode(ISD::ADDC, dl, VTList, LoOps);
3399 HiOps[2] = Lo.getValue(1);
3400 Hi = DAG.getNode(ISD::ADDE, dl, VTList, HiOps);
3401 } else {
3402 Lo = DAG.getNode(ISD::SUBC, dl, VTList, LoOps);
3403 HiOps[2] = Lo.getValue(1);
3404 Hi = DAG.getNode(ISD::SUBE, dl, VTList, HiOps);
3405 }
3406
3407 // Legalized the flag result - switch anything that used the old flag to
3408 // use the new one.
3409 ReplaceValueWith(SDValue(N, 1), Hi.getValue(1));
3410}
3411
3412void DAGTypeLegalizer::ExpandIntRes_ADDSUBE(SDNode *N,
3413 SDValue &Lo, SDValue &Hi) {
3414 // Expand the subcomponents.
3415 SDValue LHSL, LHSH, RHSL, RHSH;
3416 SDLoc dl(N);
3417 GetExpandedInteger(N->getOperand(0), LHSL, LHSH);
3418 GetExpandedInteger(N->getOperand(1), RHSL, RHSH);
3419 SDVTList VTList = DAG.getVTList(LHSL.getValueType(), MVT::Glue);
3420 SDValue LoOps[3] = { LHSL, RHSL, N->getOperand(2) };
3421 SDValue HiOps[3] = { LHSH, RHSH };
3422
3423 Lo = DAG.getNode(N->getOpcode(), dl, VTList, LoOps);
3424 HiOps[2] = Lo.getValue(1);
3425 Hi = DAG.getNode(N->getOpcode(), dl, VTList, HiOps);
3426
3427 // Legalized the flag result - switch anything that used the old flag to
3428 // use the new one.
3429 ReplaceValueWith(SDValue(N, 1), Hi.getValue(1));
3430}
3431
3432void DAGTypeLegalizer::ExpandIntRes_UADDSUBO(SDNode *N,
3433 SDValue &Lo, SDValue &Hi) {
3434 SDValue LHS = N->getOperand(0);
3435 SDValue RHS = N->getOperand(1);
3436 SDLoc dl(N);
3437
3438 SDValue Ovf;
3439
3440 unsigned CarryOp, NoCarryOp;
3442 switch(N->getOpcode()) {
3443 case ISD::UADDO:
3444 CarryOp = ISD::UADDO_CARRY;
3445 NoCarryOp = ISD::ADD;
3446 Cond = ISD::SETULT;
3447 break;
3448 case ISD::USUBO:
3449 CarryOp = ISD::USUBO_CARRY;
3450 NoCarryOp = ISD::SUB;
3451 Cond = ISD::SETUGT;
3452 break;
3453 default:
3454 llvm_unreachable("Node has unexpected Opcode");
3455 }
3456
3457 bool HasCarryOp = TLI.isOperationLegalOrCustom(
3458 CarryOp, TLI.getTypeToExpandTo(*DAG.getContext(), LHS.getValueType()));
3459
3460 if (HasCarryOp) {
3461 // Expand the subcomponents.
3462 SDValue LHSL, LHSH, RHSL, RHSH;
3463 GetExpandedInteger(LHS, LHSL, LHSH);
3464 GetExpandedInteger(RHS, RHSL, RHSH);
3465 SDVTList VTList = DAG.getVTList(LHSL.getValueType(), N->getValueType(1));
3466 SDValue LoOps[2] = { LHSL, RHSL };
3467 SDValue HiOps[3] = { LHSH, RHSH };
3468
3469 Lo = DAG.getNode(N->getOpcode(), dl, VTList, LoOps);
3470 HiOps[2] = Lo.getValue(1);
3471 Hi = DAG.getNode(CarryOp, dl, VTList, HiOps);
3472
3473 Ovf = Hi.getValue(1);
3474 } else {
3475 // Expand the result by simply replacing it with the equivalent
3476 // non-overflow-checking operation.
3477 SDValue Sum = DAG.getNode(NoCarryOp, dl, LHS.getValueType(), LHS, RHS);
3478 SplitInteger(Sum, Lo, Hi);
3479
3480 if (N->getOpcode() == ISD::UADDO && isOneConstant(RHS)) {
3481 // Special case: uaddo X, 1 overflowed if X+1 == 0. We can detect this
3482 // with (Lo | Hi) == 0.
3483 SDValue Or = DAG.getNode(ISD::OR, dl, Lo.getValueType(), Lo, Hi);
3484 Ovf = DAG.getSetCC(dl, N->getValueType(1), Or,
3485 DAG.getConstant(0, dl, Lo.getValueType()), ISD::SETEQ);
3486 } else if (N->getOpcode() == ISD::UADDO && isAllOnesConstant(RHS)) {
3487 // Special case: uaddo X, -1 overflows if X == 0.
3488 Ovf =
3489 DAG.getSetCC(dl, N->getValueType(1), LHS,
3490 DAG.getConstant(0, dl, LHS.getValueType()), ISD::SETNE);
3491 } else {
3492 // Calculate the overflow: addition overflows iff a + b < a, and
3493 // subtraction overflows iff a - b > a.
3494 Ovf = DAG.getSetCC(dl, N->getValueType(1), Sum, LHS, Cond);
3495 }
3496 }
3497
3498 // Legalized the flag result - switch anything that used the old flag to
3499 // use the new one.
3500 ReplaceValueWith(SDValue(N, 1), Ovf);
3501}
3502
3503void DAGTypeLegalizer::ExpandIntRes_UADDSUBO_CARRY(SDNode *N, SDValue &Lo,
3504 SDValue &Hi) {
3505 // Expand the subcomponents.
3506 SDValue LHSL, LHSH, RHSL, RHSH;
3507 SDLoc dl(N);
3508 GetExpandedInteger(N->getOperand(0), LHSL, LHSH);
3509 GetExpandedInteger(N->getOperand(1), RHSL, RHSH);
3510 SDVTList VTList = DAG.getVTList(LHSL.getValueType(), N->getValueType(1));
3511 SDValue LoOps[3] = { LHSL, RHSL, N->getOperand(2) };
3512 SDValue HiOps[3] = { LHSH, RHSH, SDValue() };
3513
3514 Lo = DAG.getNode(N->getOpcode(), dl, VTList, LoOps);
3515 HiOps[2] = Lo.getValue(1);
3516 Hi = DAG.getNode(N->getOpcode(), dl, VTList, HiOps);
3517
3518 // Legalized the flag result - switch anything that used the old flag to
3519 // use the new one.
3520 ReplaceValueWith(SDValue(N, 1), Hi.getValue(1));
3521}
3522
3523void DAGTypeLegalizer::ExpandIntRes_SADDSUBO_CARRY(SDNode *N,
3524 SDValue &Lo, SDValue &Hi) {
3525 // Expand the subcomponents.
3526 SDValue LHSL, LHSH, RHSL, RHSH;
3527 SDLoc dl(N);
3528 GetExpandedInteger(N->getOperand(0), LHSL, LHSH);
3529 GetExpandedInteger(N->getOperand(1), RHSL, RHSH);
3530 SDVTList VTList = DAG.getVTList(LHSL.getValueType(), N->getValueType(1));
3531
3532 // We need to use an unsigned carry op for the lo part.
3533 unsigned CarryOp =
3535 Lo = DAG.getNode(CarryOp, dl, VTList, { LHSL, RHSL, N->getOperand(2) });
3536 Hi = DAG.getNode(N->getOpcode(), dl, VTList, { LHSH, RHSH, Lo.getValue(1) });
3537
3538 // Legalized the flag result - switch anything that used the old flag to
3539 // use the new one.
3540 ReplaceValueWith(SDValue(N, 1), Hi.getValue(1));
3541}
3542
3543void DAGTypeLegalizer::ExpandIntRes_ANY_EXTEND(SDNode *N,
3544 SDValue &Lo, SDValue &Hi) {
3545 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
3546 SDLoc dl(N);
3547 SDValue Op = N->getOperand(0);
3548 if (Op.getValueType().bitsLE(NVT)) {
3549 // The low part is any extension of the input (which degenerates to a copy).
3550 Lo = DAG.getNode(ISD::ANY_EXTEND, dl, NVT, Op);
3551 Hi = DAG.getUNDEF(NVT); // The high part is undefined.
3552 } else {
3553 // For example, extension of an i48 to an i64. The operand type necessarily
3554 // promotes to the result type, so will end up being expanded too.
3555 assert(getTypeAction(Op.getValueType()) ==
3557 "Only know how to promote this result!");
3558 SDValue Res = GetPromotedInteger(Op);
3559 assert(Res.getValueType() == N->getValueType(0) &&
3560 "Operand over promoted?");
3561 // Split the promoted operand. This will simplify when it is expanded.
3562 SplitInteger(Res, Lo, Hi);
3563 }
3564}
3565
3566void DAGTypeLegalizer::ExpandIntRes_AssertSext(SDNode *N,
3567 SDValue &Lo, SDValue &Hi) {
3568 SDLoc dl(N);
3569 GetExpandedInteger(N->getOperand(0), Lo, Hi);
3570 EVT NVT = Lo.getValueType();
3571 EVT EVT = cast<VTSDNode>(N->getOperand(1))->getVT();
3572 unsigned NVTBits = NVT.getSizeInBits();
3573 unsigned EVTBits = EVT.getSizeInBits();
3574
3575 if (NVTBits < EVTBits) {
3576 Hi = DAG.getNode(ISD::AssertSext, dl, NVT, Hi,
3578 EVTBits - NVTBits)));
3579 } else {
3580 Lo = DAG.getNode(ISD::AssertSext, dl, NVT, Lo, DAG.getValueType(EVT));
3581 // The high part replicates the sign bit of Lo, make it explicit.
3582 Hi = DAG.getNode(ISD::SRA, dl, NVT, Lo,
3583 DAG.getConstant(NVTBits - 1, dl,
3584 TLI.getPointerTy(DAG.getDataLayout())));
3585 }
3586}
3587
3588void DAGTypeLegalizer::ExpandIntRes_AssertZext(SDNode *N,
3589 SDValue &Lo, SDValue &Hi) {
3590 SDLoc dl(N);
3591 GetExpandedInteger(N->getOperand(0), Lo, Hi);
3592 EVT NVT = Lo.getValueType();
3593 EVT EVT = cast<VTSDNode>(N->getOperand(1))->getVT();
3594 unsigned NVTBits = NVT.getSizeInBits();
3595 unsigned EVTBits = EVT.getSizeInBits();
3596
3597 if (NVTBits < EVTBits) {
3598 Hi = DAG.getNode(ISD::AssertZext, dl, NVT, Hi,
3600 EVTBits - NVTBits)));
3601 } else {
3602 Lo = DAG.getNode(ISD::AssertZext, dl, NVT, Lo, DAG.getValueType(EVT));
3603 // The high part must be zero, make it explicit.
3604 Hi = DAG.getConstant(0, dl, NVT);
3605 }
3606}
3607
3608void DAGTypeLegalizer::ExpandIntRes_BITREVERSE(SDNode *N,
3609 SDValue &Lo, SDValue &Hi) {
3610 SDLoc dl(N);
3611 GetExpandedInteger(N->getOperand(0), Hi, Lo); // Note swapped operands.
3612 Lo = DAG.getNode(ISD::BITREVERSE, dl, Lo.getValueType(), Lo);
3613 Hi = DAG.getNode(ISD::BITREVERSE, dl, Hi.getValueType(), Hi);
3614}
3615
3616void DAGTypeLegalizer::ExpandIntRes_BSWAP(SDNode *N,
3617 SDValue &Lo, SDValue &Hi) {
3618 SDLoc dl(N);
3619 GetExpandedInteger(N->getOperand(0), Hi, Lo); // Note swapped operands.
3620 Lo = DAG.getNode(ISD::BSWAP, dl, Lo.getValueType(), Lo);
3621 Hi = DAG.getNode(ISD::BSWAP, dl, Hi.getValueType(), Hi);
3622}
3623
3624void DAGTypeLegalizer::ExpandIntRes_PARITY(SDNode *N, SDValue &Lo,
3625 SDValue &Hi) {
3626 SDLoc dl(N);
3627 // parity(HiLo) -> parity(Lo^Hi)
3628 GetExpandedInteger(N->getOperand(0), Lo, Hi);
3629 EVT NVT = Lo.getValueType();
3630 Lo =
3631 DAG.getNode(ISD::PARITY, dl, NVT, DAG.getNode(ISD::XOR, dl, NVT, Lo, Hi));
3632 Hi = DAG.getConstant(0, dl, NVT);
3633}
3634
3635void DAGTypeLegalizer::ExpandIntRes_Constant(SDNode *N,
3636 SDValue &Lo, SDValue &Hi) {
3637 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
3638 unsigned NBitWidth = NVT.getSizeInBits();
3639 auto Constant = cast<ConstantSDNode>(N);
3640 const APInt &Cst = Constant->getAPIntValue();
3641 bool IsTarget = Constant->isTargetOpcode();
3642 bool IsOpaque = Constant->isOpaque();
3643 SDLoc dl(N);
3644 Lo = DAG.getConstant(Cst.trunc(NBitWidth), dl, NVT, IsTarget, IsOpaque);
3645 Hi = DAG.getConstant(Cst.lshr(NBitWidth).trunc(NBitWidth), dl, NVT, IsTarget,
3646 IsOpaque);
3647}
3648
3649void DAGTypeLegalizer::ExpandIntRes_ABS(SDNode *N, SDValue &Lo, SDValue &Hi) {
3650 SDLoc dl(N);
3651
3652 SDValue N0 = N->getOperand(0);
3653 GetExpandedInteger(N0, Lo, Hi);
3654 EVT NVT = Lo.getValueType();
3655
3656 // If the upper half is all sign bits, then we can perform the ABS on the
3657 // lower half and zero-extend.
3658 if (DAG.ComputeNumSignBits(N0) > NVT.getScalarSizeInBits()) {
3659 Lo = DAG.getNode(ISD::ABS, dl, NVT, Lo);
3660 Hi = DAG.getConstant(0, dl, NVT);
3661 return;
3662 }
3663
3664 // If we have USUBO_CARRY, use the expanded form of the sra+xor+sub sequence
3665 // we use in LegalizeDAG. The SUB part of the expansion is based on
3666 // ExpandIntRes_ADDSUB which also uses USUBO_CARRY/USUBO after checking that
3667 // USUBO_CARRY is LegalOrCustom. Each of the pieces here can be further
3668 // expanded if needed. Shift expansion has a special case for filling with
3669 // sign bits so that we will only end up with one SRA.
3670 bool HasSubCarry = TLI.isOperationLegalOrCustom(
3672 if (HasSubCarry) {
3673 SDValue Sign = DAG.getNode(
3674 ISD::SRA, dl, NVT, Hi,
3675 DAG.getShiftAmountConstant(NVT.getSizeInBits() - 1, NVT, dl));
3676 SDVTList VTList = DAG.getVTList(NVT, getSetCCResultType(NVT));
3677 Lo = DAG.getNode(ISD::XOR, dl, NVT, Lo, Sign);
3678 Hi = DAG.getNode(ISD::XOR, dl, NVT, Hi, Sign);
3679 Lo = DAG.getNode(ISD::USUBO, dl, VTList, Lo, Sign);
3680 Hi = DAG.getNode(ISD::USUBO_CARRY, dl, VTList, Hi, Sign, Lo.getValue(1));
3681 return;
3682 }
3683
3684 // abs(HiLo) -> (Hi < 0 ? -HiLo : HiLo)
3685 EVT VT = N->getValueType(0);
3686 SDValue Neg = DAG.getNode(ISD::SUB, dl, VT,
3687 DAG.getConstant(0, dl, VT), N0);
3688 SDValue NegLo, NegHi;
3689 SplitInteger(Neg, NegLo, NegHi);
3690
3691 SDValue HiIsNeg = DAG.getSetCC(dl, getSetCCResultType(NVT), Hi,
3692 DAG.getConstant(0, dl, NVT), ISD::SETLT);
3693 Lo = DAG.getSelect(dl, NVT, HiIsNeg, NegLo, Lo);
3694 Hi = DAG.getSelect(dl, NVT, HiIsNeg, NegHi, Hi);
3695}
3696
3697void DAGTypeLegalizer::ExpandIntRes_CTLZ(SDNode *N,
3698 SDValue &Lo, SDValue &Hi) {
3699 SDLoc dl(N);
3700 // ctlz (HiLo) -> Hi != 0 ? ctlz(Hi) : (ctlz(Lo)+32)
3701 GetExpandedInteger(N->getOperand(0), Lo, Hi);
3702 EVT NVT = Lo.getValueType();
3703
3704 SDValue HiNotZero = DAG.getSetCC(dl, getSetCCResultType(NVT), Hi,
3705 DAG.getConstant(0, dl, NVT), ISD::SETNE);
3706
3707 SDValue LoLZ = DAG.getNode(N->getOpcode(), dl, NVT, Lo);
3708 SDValue HiLZ = DAG.getNode(ISD::CTLZ_ZERO_UNDEF, dl, NVT, Hi);
3709
3710 Lo = DAG.getSelect(dl, NVT, HiNotZero, HiLZ,
3711 DAG.getNode(ISD::ADD, dl, NVT, LoLZ,
3712 DAG.getConstant(NVT.getSizeInBits(), dl,
3713 NVT)));
3714 Hi = DAG.getConstant(0, dl, NVT);
3715}
3716
3717void DAGTypeLegalizer::ExpandIntRes_CTPOP(SDNode *N,
3718 SDValue &Lo, SDValue &Hi) {
3719 SDLoc dl(N);
3720 // ctpop(HiLo) -> ctpop(Hi)+ctpop(Lo)
3721 GetExpandedInteger(N->getOperand(0), Lo, Hi);
3722 EVT NVT = Lo.getValueType();
3723 Lo = DAG.getNode(ISD::ADD, dl, NVT, DAG.getNode(ISD::CTPOP, dl, NVT, Lo),
3724 DAG.getNode(ISD::CTPOP, dl, NVT, Hi));
3725 Hi = DAG.getConstant(0, dl, NVT);
3726}
3727
3728void DAGTypeLegalizer::ExpandIntRes_CTTZ(SDNode *N,
3729 SDValue &Lo, SDValue &Hi) {
3730 SDLoc dl(N);
3731 // cttz (HiLo) -> Lo != 0 ? cttz(Lo) : (cttz(Hi)+32)
3732 GetExpandedInteger(N->getOperand(0), Lo, Hi);
3733 EVT NVT = Lo.getValueType();
3734
3735 SDValue LoNotZero = DAG.getSetCC(dl, getSetCCResultType(NVT), Lo,
3736 DAG.getConstant(0, dl, NVT), ISD::SETNE);
3737
3738 SDValue LoLZ = DAG.getNode(ISD::CTTZ_ZERO_UNDEF, dl, NVT, Lo);
3739 SDValue HiLZ = DAG.getNode(N->getOpcode(), dl, NVT, Hi);
3740
3741 Lo = DAG.getSelect(dl, NVT, LoNotZero, LoLZ,
3742 DAG.getNode(ISD::ADD, dl, NVT, HiLZ,
3743 DAG.getConstant(NVT.getSizeInBits(), dl,
3744 NVT)));
3745 Hi = DAG.getConstant(0, dl, NVT);
3746}
3747
3748void DAGTypeLegalizer::ExpandIntRes_GET_ROUNDING(SDNode *N, SDValue &Lo,
3749 SDValue &Hi) {
3750 SDLoc dl(N);
3751 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
3752 unsigned NBitWidth = NVT.getSizeInBits();
3753
3754 Lo = DAG.getNode(ISD::GET_ROUNDING, dl, {NVT, MVT::Other}, N->getOperand(0));
3755 SDValue Chain = Lo.getValue(1);
3756 // The high part is the sign of Lo, as -1 is a valid value for GET_ROUNDING
3757 Hi = DAG.getNode(ISD::SRA, dl, NVT, Lo,
3758 DAG.getShiftAmountConstant(NBitWidth - 1, NVT, dl));
3759
3760 // Legalize the chain result - switch anything that used the old chain to
3761 // use the new one.
3762 ReplaceValueWith(SDValue(N, 1), Chain);
3763}
3764
3765// Helper for producing an FP_EXTEND/STRICT_FP_EXTEND of Op.
3766static SDValue fpExtendHelper(SDValue Op, SDValue &Chain, bool IsStrict, EVT VT,
3767 SDLoc DL, SelectionDAG &DAG) {
3768 if (IsStrict) {
3769 Op = DAG.getNode(ISD::STRICT_FP_EXTEND, DL, {VT, MVT::Other}, {Chain, Op});
3770 Chain = Op.getValue(1);
3771 return Op;
3772 }
3773 return DAG.getNode(ISD::FP_EXTEND, DL, VT, Op);
3774}
3775
3776void DAGTypeLegalizer::ExpandIntRes_FP_TO_XINT(SDNode *N, SDValue &Lo,
3777 SDValue &Hi) {
3778 SDLoc dl(N);
3779 EVT VT = N->getValueType(0);
3780
3781 bool IsSigned = N->getOpcode() == ISD::FP_TO_SINT ||
3782 N->getOpcode() == ISD::STRICT_FP_TO_SINT;
3783 bool IsStrict = N->isStrictFPOpcode();
3784 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
3785 SDValue Op = N->getOperand(IsStrict ? 1 : 0);
3786 if (getTypeAction(Op.getValueType()) == TargetLowering::TypePromoteFloat)
3787 Op = GetPromotedFloat(Op);
3788
3789 if (getTypeAction(Op.getValueType()) == TargetLowering::TypeSoftPromoteHalf) {
3790 EVT OFPVT = Op.getValueType();
3791 EVT NFPVT = TLI.getTypeToTransformTo(*DAG.getContext(), OFPVT);
3792 Op = GetSoftPromotedHalf(Op);
3793 Op = DAG.getNode(OFPVT == MVT::f16 ? ISD::FP16_TO_FP : ISD::BF16_TO_FP, dl,
3794 NFPVT, Op);
3795 Op = DAG.getNode(IsSigned ? ISD::FP_TO_SINT : ISD::FP_TO_UINT, dl, VT, Op);
3796 SplitInteger(Op, Lo, Hi);
3797 return;
3798 }
3799
3800 if (Op.getValueType() == MVT::bf16) {
3801 // Extend to f32 as there is no bf16 libcall.
3802 Op = fpExtendHelper(Op, Chain, IsStrict, MVT::f32, dl, DAG);
3803 }
3804
3805 RTLIB::Libcall LC = IsSigned ? RTLIB::getFPTOSINT(Op.getValueType(), VT)
3806 : RTLIB::getFPTOUINT(Op.getValueType(), VT);
3807 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected fp-to-xint conversion!");
3809 CallOptions.setSExt(true);
3810 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, VT, Op,
3811 CallOptions, dl, Chain);
3812 SplitInteger(Tmp.first, Lo, Hi);
3813
3814 if (IsStrict)
3815 ReplaceValueWith(SDValue(N, 1), Tmp.second);
3816}
3817
3818void DAGTypeLegalizer::ExpandIntRes_FP_TO_XINT_SAT(SDNode *N, SDValue &Lo,
3819 SDValue &Hi) {
3820 SDValue Res = TLI.expandFP_TO_INT_SAT(N, DAG);
3821 SplitInteger(Res, Lo, Hi);
3822}
3823
3824void DAGTypeLegalizer::ExpandIntRes_XROUND_XRINT(SDNode *N, SDValue &Lo,
3825 SDValue &Hi) {
3826 SDLoc dl(N);
3827 bool IsStrict = N->isStrictFPOpcode();
3828 SDValue Op = N->getOperand(IsStrict ? 1 : 0);
3829 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
3830
3831 assert(getTypeAction(Op.getValueType()) != TargetLowering::TypePromoteFloat &&
3832 "Input type needs to be promoted!");
3833
3834 EVT VT = Op.getValueType();
3835
3836 if (VT == MVT::f16) {
3837 // Extend to f32.
3838 VT = MVT::f32;
3839 Op = fpExtendHelper(Op, Chain, IsStrict, VT, dl, DAG);
3840 }
3841
3842 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
3843 if (N->getOpcode() == ISD::LROUND ||
3844 N->getOpcode() == ISD::STRICT_LROUND) {
3845 if (VT == MVT::f32)
3846 LC = RTLIB::LROUND_F32;
3847 else if (VT == MVT::f64)
3848 LC = RTLIB::LROUND_F64;
3849 else if (VT == MVT::f80)
3850 LC = RTLIB::LROUND_F80;
3851 else if (VT == MVT::f128)
3852 LC = RTLIB::LROUND_F128;
3853 else if (VT == MVT::ppcf128)
3854 LC = RTLIB::LROUND_PPCF128;
3855 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected lround input type!");
3856 } else if (N->getOpcode() == ISD::LRINT ||
3857 N->getOpcode() == ISD::STRICT_LRINT) {
3858 if (VT == MVT::f32)
3859 LC = RTLIB::LRINT_F32;
3860 else if (VT == MVT::f64)
3861 LC = RTLIB::LRINT_F64;
3862 else if (VT == MVT::f80)
3863 LC = RTLIB::LRINT_F80;
3864 else if (VT == MVT::f128)
3865 LC = RTLIB::LRINT_F128;
3866 else if (VT == MVT::ppcf128)
3867 LC = RTLIB::LRINT_PPCF128;
3868 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected lrint input type!");
3869 } else if (N->getOpcode() == ISD::LLROUND ||
3870 N->getOpcode() == ISD::STRICT_LLROUND) {
3871 if (VT == MVT::f32)
3872 LC = RTLIB::LLROUND_F32;
3873 else if (VT == MVT::f64)
3874 LC = RTLIB::LLROUND_F64;
3875 else if (VT == MVT::f80)
3876 LC = RTLIB::LLROUND_F80;
3877 else if (VT == MVT::f128)
3878 LC = RTLIB::LLROUND_F128;
3879 else if (VT == MVT::ppcf128)
3880 LC = RTLIB::LLROUND_PPCF128;
3881 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected llround input type!");
3882 } else if (N->getOpcode() == ISD::LLRINT ||
3883 N->getOpcode() == ISD::STRICT_LLRINT) {
3884 if (VT == MVT::f32)
3885 LC = RTLIB::LLRINT_F32;
3886 else if (VT == MVT::f64)
3887 LC = RTLIB::LLRINT_F64;
3888 else if (VT == MVT::f80)
3889 LC = RTLIB::LLRINT_F80;
3890 else if (VT == MVT::f128)
3891 LC = RTLIB::LLRINT_F128;
3892 else if (VT == MVT::ppcf128)
3893 LC = RTLIB::LLRINT_PPCF128;
3894 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected llrint input type!");
3895 } else
3896 llvm_unreachable("Unexpected opcode!");
3897
3898 EVT RetVT