LLVM 19.0.0git
LegalizeFloatTypes.cpp
Go to the documentation of this file.
1//===-------- LegalizeFloatTypes.cpp - Legalization of float types --------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements float type expansion and softening for LegalizeTypes.
10// Softening is the act of turning a computation in an illegal floating point
11// type into a computation in an integer type of the same size; also known as
12// "soft float". For example, turning f32 arithmetic into operations using i32.
13// The resulting integer value is the same as what you would get by performing
14// the floating point operation and bitcasting the result to the integer type.
15// Expansion is the act of changing a computation in an illegal type to be a
16// computation in two identical registers of a smaller type. For example,
17// implementing ppcf128 arithmetic in two f64 registers.
18//
19//===----------------------------------------------------------------------===//
20
21#include "LegalizeTypes.h"
25using namespace llvm;
26
27#define DEBUG_TYPE "legalize-types"
28
29/// GetFPLibCall - Return the right libcall for the given floating point type.
30/// FIXME: This is a local version of RTLIB::getFPLibCall that should be
31/// refactored away (see RTLIB::getPOWI for an example).
33 RTLIB::Libcall Call_F32,
34 RTLIB::Libcall Call_F64,
35 RTLIB::Libcall Call_F80,
36 RTLIB::Libcall Call_F128,
37 RTLIB::Libcall Call_PPCF128) {
38 return
39 VT == MVT::f32 ? Call_F32 :
40 VT == MVT::f64 ? Call_F64 :
41 VT == MVT::f80 ? Call_F80 :
42 VT == MVT::f128 ? Call_F128 :
43 VT == MVT::ppcf128 ? Call_PPCF128 :
44 RTLIB::UNKNOWN_LIBCALL;
45}
46
47//===----------------------------------------------------------------------===//
48// Convert Float Results to Integer
49//===----------------------------------------------------------------------===//
50
51void DAGTypeLegalizer::SoftenFloatResult(SDNode *N, unsigned ResNo) {
52 LLVM_DEBUG(dbgs() << "Soften float result " << ResNo << ": "; N->dump(&DAG));
53 SDValue R = SDValue();
54
55 switch (N->getOpcode()) {
56 default:
57#ifndef NDEBUG
58 dbgs() << "SoftenFloatResult #" << ResNo << ": ";
59 N->dump(&DAG); dbgs() << "\n";
60#endif
61 report_fatal_error("Do not know how to soften the result of this "
62 "operator!");
63
64 case ISD::ARITH_FENCE: R = SoftenFloatRes_ARITH_FENCE(N); break;
65 case ISD::MERGE_VALUES:R = SoftenFloatRes_MERGE_VALUES(N, ResNo); break;
66 case ISD::BITCAST: R = SoftenFloatRes_BITCAST(N); break;
67 case ISD::BUILD_PAIR: R = SoftenFloatRes_BUILD_PAIR(N); break;
68 case ISD::ConstantFP: R = SoftenFloatRes_ConstantFP(N); break;
70 R = SoftenFloatRes_EXTRACT_VECTOR_ELT(N, ResNo); break;
71 case ISD::FABS: R = SoftenFloatRes_FABS(N); break;
73 case ISD::FMINNUM: R = SoftenFloatRes_FMINNUM(N); break;
75 case ISD::FMAXNUM: R = SoftenFloatRes_FMAXNUM(N); break;
77 case ISD::FADD: R = SoftenFloatRes_FADD(N); break;
78 case ISD::FCBRT: R = SoftenFloatRes_FCBRT(N); break;
80 case ISD::FCEIL: R = SoftenFloatRes_FCEIL(N); break;
81 case ISD::FCOPYSIGN: R = SoftenFloatRes_FCOPYSIGN(N); break;
83 case ISD::FCOS: R = SoftenFloatRes_FCOS(N); break;
85 case ISD::FDIV: R = SoftenFloatRes_FDIV(N); break;
87 case ISD::FEXP: R = SoftenFloatRes_FEXP(N); break;
89 case ISD::FEXP2: R = SoftenFloatRes_FEXP2(N); break;
90 case ISD::FEXP10: R = SoftenFloatRes_FEXP10(N); break;
92 case ISD::FFLOOR: R = SoftenFloatRes_FFLOOR(N); break;
94 case ISD::FLOG: R = SoftenFloatRes_FLOG(N); break;
96 case ISD::FLOG2: R = SoftenFloatRes_FLOG2(N); break;
98 case ISD::FLOG10: R = SoftenFloatRes_FLOG10(N); break;
99 case ISD::STRICT_FMA:
100 case ISD::FMA: R = SoftenFloatRes_FMA(N); break;
101 case ISD::STRICT_FMUL:
102 case ISD::FMUL: R = SoftenFloatRes_FMUL(N); break;
104 case ISD::FNEARBYINT: R = SoftenFloatRes_FNEARBYINT(N); break;
105 case ISD::FNEG: R = SoftenFloatRes_FNEG(N); break;
107 case ISD::FP_EXTEND: R = SoftenFloatRes_FP_EXTEND(N); break;
109 case ISD::FP_ROUND: R = SoftenFloatRes_FP_ROUND(N); break;
110 case ISD::FP16_TO_FP: R = SoftenFloatRes_FP16_TO_FP(N); break;
111 case ISD::BF16_TO_FP: R = SoftenFloatRes_BF16_TO_FP(N); break;
112 case ISD::STRICT_FPOW:
113 case ISD::FPOW: R = SoftenFloatRes_FPOW(N); break;
115 case ISD::FPOWI:
116 case ISD::FLDEXP:
117 case ISD::STRICT_FLDEXP: R = SoftenFloatRes_ExpOp(N); break;
118 case ISD::FFREXP:
119 R = SoftenFloatRes_FFREXP(N);
120 break;
121 case ISD::STRICT_FREM:
122 case ISD::FREM: R = SoftenFloatRes_FREM(N); break;
124 case ISD::FRINT: R = SoftenFloatRes_FRINT(N); break;
126 case ISD::FROUND: R = SoftenFloatRes_FROUND(N); break;
128 case ISD::FROUNDEVEN: R = SoftenFloatRes_FROUNDEVEN(N); break;
129 case ISD::STRICT_FSIN:
130 case ISD::FSIN: R = SoftenFloatRes_FSIN(N); break;
132 case ISD::FSQRT: R = SoftenFloatRes_FSQRT(N); break;
133 case ISD::STRICT_FSUB:
134 case ISD::FSUB: R = SoftenFloatRes_FSUB(N); break;
136 case ISD::FTRUNC: R = SoftenFloatRes_FTRUNC(N); break;
137 case ISD::LOAD: R = SoftenFloatRes_LOAD(N); break;
138 case ISD::ATOMIC_SWAP: R = BitcastToInt_ATOMIC_SWAP(N); break;
139 case ISD::SELECT: R = SoftenFloatRes_SELECT(N); break;
140 case ISD::SELECT_CC: R = SoftenFloatRes_SELECT_CC(N); break;
141 case ISD::FREEZE: R = SoftenFloatRes_FREEZE(N); break;
144 case ISD::SINT_TO_FP:
145 case ISD::UINT_TO_FP: R = SoftenFloatRes_XINT_TO_FP(N); break;
146 case ISD::UNDEF: R = SoftenFloatRes_UNDEF(N); break;
147 case ISD::VAARG: R = SoftenFloatRes_VAARG(N); break;
154 R = SoftenFloatRes_VECREDUCE(N);
155 break;
158 R = SoftenFloatRes_VECREDUCE_SEQ(N);
159 break;
160 }
161
162 // If R is null, the sub-method took care of registering the result.
163 if (R.getNode()) {
164 assert(R.getNode() != N);
165 SetSoftenedFloat(SDValue(N, ResNo), R);
166 }
167}
168
169SDValue DAGTypeLegalizer::SoftenFloatRes_Unary(SDNode *N, RTLIB::Libcall LC) {
170 bool IsStrict = N->isStrictFPOpcode();
171 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
172 unsigned Offset = IsStrict ? 1 : 0;
173 assert(N->getNumOperands() == (1 + Offset) &&
174 "Unexpected number of operands!");
175 SDValue Op = GetSoftenedFloat(N->getOperand(0 + Offset));
176 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
178 EVT OpVT = N->getOperand(0 + Offset).getValueType();
179 CallOptions.setTypeListBeforeSoften(OpVT, N->getValueType(0), true);
180 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, NVT, Op,
181 CallOptions, SDLoc(N),
182 Chain);
183 if (IsStrict)
184 ReplaceValueWith(SDValue(N, 1), Tmp.second);
185 return Tmp.first;
186}
187
188SDValue DAGTypeLegalizer::SoftenFloatRes_Binary(SDNode *N, RTLIB::Libcall LC) {
189 bool IsStrict = N->isStrictFPOpcode();
190 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
191 unsigned Offset = IsStrict ? 1 : 0;
192 assert(N->getNumOperands() == (2 + Offset) &&
193 "Unexpected number of operands!");
194 SDValue Ops[2] = { GetSoftenedFloat(N->getOperand(0 + Offset)),
195 GetSoftenedFloat(N->getOperand(1 + Offset)) };
196 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
198 EVT OpsVT[2] = { N->getOperand(0 + Offset).getValueType(),
199 N->getOperand(1 + Offset).getValueType() };
200 CallOptions.setTypeListBeforeSoften(OpsVT, N->getValueType(0), true);
201 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, NVT, Ops,
202 CallOptions, SDLoc(N),
203 Chain);
204 if (IsStrict)
205 ReplaceValueWith(SDValue(N, 1), Tmp.second);
206 return Tmp.first;
207}
208
209SDValue DAGTypeLegalizer::SoftenFloatRes_BITCAST(SDNode *N) {
210 return BitConvertToInteger(N->getOperand(0));
211}
212
213SDValue DAGTypeLegalizer::SoftenFloatRes_FREEZE(SDNode *N) {
214 EVT Ty = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
215 return DAG.getNode(ISD::FREEZE, SDLoc(N), Ty,
216 GetSoftenedFloat(N->getOperand(0)));
217}
218
219SDValue DAGTypeLegalizer::SoftenFloatRes_ARITH_FENCE(SDNode *N) {
220 EVT Ty = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
221 SDValue NewFence = DAG.getNode(ISD::ARITH_FENCE, SDLoc(N), Ty,
222 GetSoftenedFloat(N->getOperand(0)));
223 return NewFence;
224}
225
226SDValue DAGTypeLegalizer::SoftenFloatRes_MERGE_VALUES(SDNode *N,
227 unsigned ResNo) {
228 SDValue Op = DisintegrateMERGE_VALUES(N, ResNo);
229 return BitConvertToInteger(Op);
230}
231
232SDValue DAGTypeLegalizer::SoftenFloatRes_BUILD_PAIR(SDNode *N) {
233 // Convert the inputs to integers, and build a new pair out of them.
234 return DAG.getNode(ISD::BUILD_PAIR, SDLoc(N),
236 N->getValueType(0)),
237 BitConvertToInteger(N->getOperand(0)),
238 BitConvertToInteger(N->getOperand(1)));
239}
240
241SDValue DAGTypeLegalizer::SoftenFloatRes_ConstantFP(SDNode *N) {
242 ConstantFPSDNode *CN = cast<ConstantFPSDNode>(N);
243 // In ppcf128, the high 64 bits are always first in memory regardless
244 // of Endianness. LLVM's APFloat representation is not Endian sensitive,
245 // and so always converts into a 128-bit APInt in a non-Endian-sensitive
246 // way. However, APInt's are serialized in an Endian-sensitive fashion,
247 // so on big-Endian targets, the two doubles are output in the wrong
248 // order. Fix this by manually flipping the order of the high 64 bits
249 // and the low 64 bits here.
250 if (DAG.getDataLayout().isBigEndian() &&
251 CN->getValueType(0).getSimpleVT() == llvm::MVT::ppcf128) {
252 uint64_t words[2] = { CN->getValueAPF().bitcastToAPInt().getRawData()[1],
254 APInt Val(128, words);
255 return DAG.getConstant(Val, SDLoc(CN),
257 CN->getValueType(0)));
258 } else {
259 return DAG.getConstant(CN->getValueAPF().bitcastToAPInt(), SDLoc(CN),
261 CN->getValueType(0)));
262 }
263}
264
265SDValue DAGTypeLegalizer::SoftenFloatRes_EXTRACT_VECTOR_ELT(SDNode *N, unsigned ResNo) {
266 SDValue NewOp = BitConvertVectorToIntegerVector(N->getOperand(0));
269 NewOp, N->getOperand(1));
270}
271
272SDValue DAGTypeLegalizer::SoftenFloatRes_FABS(SDNode *N) {
273 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
274 unsigned Size = NVT.getSizeInBits();
275
276 // Mask = ~(1 << (Size-1))
278 API.clearBit(Size - 1);
279 SDValue Mask = DAG.getConstant(API, SDLoc(N), NVT);
280 SDValue Op = GetSoftenedFloat(N->getOperand(0));
281 return DAG.getNode(ISD::AND, SDLoc(N), NVT, Op, Mask);
282}
283
284SDValue DAGTypeLegalizer::SoftenFloatRes_FMINNUM(SDNode *N) {
285 if (SDValue SelCC = TLI.createSelectForFMINNUM_FMAXNUM(N, DAG))
286 return SoftenFloatRes_SELECT_CC(SelCC.getNode());
287 return SoftenFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
288 RTLIB::FMIN_F32,
289 RTLIB::FMIN_F64,
290 RTLIB::FMIN_F80,
291 RTLIB::FMIN_F128,
292 RTLIB::FMIN_PPCF128));
293}
294
295SDValue DAGTypeLegalizer::SoftenFloatRes_FMAXNUM(SDNode *N) {
296 if (SDValue SelCC = TLI.createSelectForFMINNUM_FMAXNUM(N, DAG))
297 return SoftenFloatRes_SELECT_CC(SelCC.getNode());
298 return SoftenFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
299 RTLIB::FMAX_F32,
300 RTLIB::FMAX_F64,
301 RTLIB::FMAX_F80,
302 RTLIB::FMAX_F128,
303 RTLIB::FMAX_PPCF128));
304}
305
306SDValue DAGTypeLegalizer::SoftenFloatRes_FADD(SDNode *N) {
307 return SoftenFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
308 RTLIB::ADD_F32,
309 RTLIB::ADD_F64,
310 RTLIB::ADD_F80,
311 RTLIB::ADD_F128,
312 RTLIB::ADD_PPCF128));
313}
314
315SDValue DAGTypeLegalizer::SoftenFloatRes_FCBRT(SDNode *N) {
316 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
317 RTLIB::CBRT_F32,
318 RTLIB::CBRT_F64,
319 RTLIB::CBRT_F80,
320 RTLIB::CBRT_F128,
321 RTLIB::CBRT_PPCF128));
322}
323
324SDValue DAGTypeLegalizer::SoftenFloatRes_FCEIL(SDNode *N) {
325 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
326 RTLIB::CEIL_F32,
327 RTLIB::CEIL_F64,
328 RTLIB::CEIL_F80,
329 RTLIB::CEIL_F128,
330 RTLIB::CEIL_PPCF128));
331}
332
333SDValue DAGTypeLegalizer::SoftenFloatRes_FCOPYSIGN(SDNode *N) {
334 SDValue LHS = GetSoftenedFloat(N->getOperand(0));
335 SDValue RHS = BitConvertToInteger(N->getOperand(1));
336 SDLoc dl(N);
337
338 EVT LVT = LHS.getValueType();
339 EVT RVT = RHS.getValueType();
340
341 unsigned LSize = LVT.getSizeInBits();
342 unsigned RSize = RVT.getSizeInBits();
343
344 // First get the sign bit of second operand.
345 SDValue SignBit = DAG.getNode(
346 ISD::SHL, dl, RVT, DAG.getConstant(1, dl, RVT),
347 DAG.getConstant(RSize - 1, dl,
348 TLI.getShiftAmountTy(RVT, DAG.getDataLayout())));
349 SignBit = DAG.getNode(ISD::AND, dl, RVT, RHS, SignBit);
350
351 // Shift right or sign-extend it if the two operands have different types.
352 int SizeDiff = RVT.getSizeInBits() - LVT.getSizeInBits();
353 if (SizeDiff > 0) {
354 SignBit =
355 DAG.getNode(ISD::SRL, dl, RVT, SignBit,
356 DAG.getConstant(SizeDiff, dl,
357 TLI.getShiftAmountTy(SignBit.getValueType(),
358 DAG.getDataLayout())));
359 SignBit = DAG.getNode(ISD::TRUNCATE, dl, LVT, SignBit);
360 } else if (SizeDiff < 0) {
361 SignBit = DAG.getNode(ISD::ANY_EXTEND, dl, LVT, SignBit);
362 SignBit =
363 DAG.getNode(ISD::SHL, dl, LVT, SignBit,
364 DAG.getConstant(-SizeDiff, dl,
365 TLI.getShiftAmountTy(SignBit.getValueType(),
366 DAG.getDataLayout())));
367 }
368
369 // Clear the sign bit of the first operand.
370 SDValue Mask = DAG.getNode(
371 ISD::SHL, dl, LVT, DAG.getConstant(1, dl, LVT),
372 DAG.getConstant(LSize - 1, dl,
373 TLI.getShiftAmountTy(LVT, DAG.getDataLayout())));
374 Mask = DAG.getNode(ISD::SUB, dl, LVT, Mask, DAG.getConstant(1, dl, LVT));
375 LHS = DAG.getNode(ISD::AND, dl, LVT, LHS, Mask);
376
377 // Or the value with the sign bit.
378 return DAG.getNode(ISD::OR, dl, LVT, LHS, SignBit);
379}
380
381SDValue DAGTypeLegalizer::SoftenFloatRes_FCOS(SDNode *N) {
382 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
383 RTLIB::COS_F32,
384 RTLIB::COS_F64,
385 RTLIB::COS_F80,
386 RTLIB::COS_F128,
387 RTLIB::COS_PPCF128));
388}
389
390SDValue DAGTypeLegalizer::SoftenFloatRes_FDIV(SDNode *N) {
391 return SoftenFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
392 RTLIB::DIV_F32,
393 RTLIB::DIV_F64,
394 RTLIB::DIV_F80,
395 RTLIB::DIV_F128,
396 RTLIB::DIV_PPCF128));
397}
398
399SDValue DAGTypeLegalizer::SoftenFloatRes_FEXP(SDNode *N) {
400 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
401 RTLIB::EXP_F32,
402 RTLIB::EXP_F64,
403 RTLIB::EXP_F80,
404 RTLIB::EXP_F128,
405 RTLIB::EXP_PPCF128));
406}
407
408SDValue DAGTypeLegalizer::SoftenFloatRes_FEXP2(SDNode *N) {
409 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
410 RTLIB::EXP2_F32,
411 RTLIB::EXP2_F64,
412 RTLIB::EXP2_F80,
413 RTLIB::EXP2_F128,
414 RTLIB::EXP2_PPCF128));
415}
416
417SDValue DAGTypeLegalizer::SoftenFloatRes_FEXP10(SDNode *N) {
418 return SoftenFloatRes_Unary(
419 N,
420 GetFPLibCall(N->getValueType(0), RTLIB::EXP10_F32, RTLIB::EXP10_F64,
421 RTLIB::EXP10_F80, RTLIB::EXP10_F128, RTLIB::EXP10_PPCF128));
422}
423
424SDValue DAGTypeLegalizer::SoftenFloatRes_FFLOOR(SDNode *N) {
425 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
426 RTLIB::FLOOR_F32,
427 RTLIB::FLOOR_F64,
428 RTLIB::FLOOR_F80,
429 RTLIB::FLOOR_F128,
430 RTLIB::FLOOR_PPCF128));
431}
432
433SDValue DAGTypeLegalizer::SoftenFloatRes_FLOG(SDNode *N) {
434 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
435 RTLIB::LOG_F32,
436 RTLIB::LOG_F64,
437 RTLIB::LOG_F80,
438 RTLIB::LOG_F128,
439 RTLIB::LOG_PPCF128));
440}
441
442SDValue DAGTypeLegalizer::SoftenFloatRes_FLOG2(SDNode *N) {
443 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
444 RTLIB::LOG2_F32,
445 RTLIB::LOG2_F64,
446 RTLIB::LOG2_F80,
447 RTLIB::LOG2_F128,
448 RTLIB::LOG2_PPCF128));
449}
450
451SDValue DAGTypeLegalizer::SoftenFloatRes_FLOG10(SDNode *N) {
452 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
453 RTLIB::LOG10_F32,
454 RTLIB::LOG10_F64,
455 RTLIB::LOG10_F80,
456 RTLIB::LOG10_F128,
457 RTLIB::LOG10_PPCF128));
458}
459
460SDValue DAGTypeLegalizer::SoftenFloatRes_FMA(SDNode *N) {
461 bool IsStrict = N->isStrictFPOpcode();
462 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
463 unsigned Offset = IsStrict ? 1 : 0;
464 SDValue Ops[3] = { GetSoftenedFloat(N->getOperand(0 + Offset)),
465 GetSoftenedFloat(N->getOperand(1 + Offset)),
466 GetSoftenedFloat(N->getOperand(2 + Offset)) };
467 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
469 EVT OpsVT[3] = { N->getOperand(0 + Offset).getValueType(),
470 N->getOperand(1 + Offset).getValueType(),
471 N->getOperand(2 + Offset).getValueType() };
472 CallOptions.setTypeListBeforeSoften(OpsVT, N->getValueType(0), true);
473 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG,
474 GetFPLibCall(N->getValueType(0),
475 RTLIB::FMA_F32,
476 RTLIB::FMA_F64,
477 RTLIB::FMA_F80,
478 RTLIB::FMA_F128,
479 RTLIB::FMA_PPCF128),
480 NVT, Ops, CallOptions, SDLoc(N), Chain);
481 if (IsStrict)
482 ReplaceValueWith(SDValue(N, 1), Tmp.second);
483 return Tmp.first;
484}
485
486SDValue DAGTypeLegalizer::SoftenFloatRes_FMUL(SDNode *N) {
487 return SoftenFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
488 RTLIB::MUL_F32,
489 RTLIB::MUL_F64,
490 RTLIB::MUL_F80,
491 RTLIB::MUL_F128,
492 RTLIB::MUL_PPCF128));
493}
494
495SDValue DAGTypeLegalizer::SoftenFloatRes_FNEARBYINT(SDNode *N) {
496 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
497 RTLIB::NEARBYINT_F32,
498 RTLIB::NEARBYINT_F64,
499 RTLIB::NEARBYINT_F80,
500 RTLIB::NEARBYINT_F128,
501 RTLIB::NEARBYINT_PPCF128));
502}
503
504SDValue DAGTypeLegalizer::SoftenFloatRes_FNEG(SDNode *N) {
505 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
506 SDLoc dl(N);
507
508 // Expand Y = FNEG(X) -> Y = X ^ sign mask
509 APInt SignMask = APInt::getSignMask(NVT.getSizeInBits());
510 return DAG.getNode(ISD::XOR, dl, NVT, GetSoftenedFloat(N->getOperand(0)),
511 DAG.getConstant(SignMask, dl, NVT));
512}
513
514SDValue DAGTypeLegalizer::SoftenFloatRes_FP_EXTEND(SDNode *N) {
515 bool IsStrict = N->isStrictFPOpcode();
516 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
517 SDValue Op = N->getOperand(IsStrict ? 1 : 0);
518
519 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
520
521 if (getTypeAction(Op.getValueType()) == TargetLowering::TypePromoteFloat) {
522 Op = GetPromotedFloat(Op);
523 // If the promotion did the FP_EXTEND to the destination type for us,
524 // there's nothing left to do here.
525 if (Op.getValueType() == N->getValueType(0)) {
526 if (IsStrict)
527 ReplaceValueWith(SDValue(N, 1), Chain);
528 return BitConvertToInteger(Op);
529 }
530 }
531
532 // There's only a libcall for f16 -> f32 and shifting is only valid for bf16
533 // -> f32, so proceed in two stages. Also, it's entirely possible for both
534 // f16 and f32 to be legal, so use the fully hard-float FP_EXTEND rather
535 // than FP16_TO_FP.
536 if ((Op.getValueType() == MVT::f16 || Op.getValueType() == MVT::bf16) &&
537 N->getValueType(0) != MVT::f32) {
538 if (IsStrict) {
540 { MVT::f32, MVT::Other }, { Chain, Op });
541 Chain = Op.getValue(1);
542 } else {
543 Op = DAG.getNode(ISD::FP_EXTEND, SDLoc(N), MVT::f32, Op);
544 }
545 }
546
547 if (Op.getValueType() == MVT::bf16) {
548 // FIXME: Need ReplaceValueWith on chain in strict case
549 return SoftenFloatRes_BF16_TO_FP(N);
550 }
551
552 RTLIB::Libcall LC = RTLIB::getFPEXT(Op.getValueType(), N->getValueType(0));
553 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported FP_EXTEND!");
555 EVT OpVT = N->getOperand(IsStrict ? 1 : 0).getValueType();
556 CallOptions.setTypeListBeforeSoften(OpVT, N->getValueType(0), true);
557 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, NVT, Op,
558 CallOptions, SDLoc(N),
559 Chain);
560 if (IsStrict)
561 ReplaceValueWith(SDValue(N, 1), Tmp.second);
562 return Tmp.first;
563}
564
565// FIXME: Should we just use 'normal' FP_EXTEND / FP_TRUNC instead of special
566// nodes?
567SDValue DAGTypeLegalizer::SoftenFloatRes_FP16_TO_FP(SDNode *N) {
568 EVT MidVT = TLI.getTypeToTransformTo(*DAG.getContext(), MVT::f32);
569 SDValue Op = N->getOperand(0);
571 EVT OpsVT[1] = { N->getOperand(0).getValueType() };
572 CallOptions.setTypeListBeforeSoften(OpsVT, N->getValueType(0), true);
573 SDValue Res32 = TLI.makeLibCall(DAG, RTLIB::FPEXT_F16_F32, MidVT, Op,
574 CallOptions, SDLoc(N)).first;
575 if (N->getValueType(0) == MVT::f32)
576 return Res32;
577
578 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
579 RTLIB::Libcall LC = RTLIB::getFPEXT(MVT::f32, N->getValueType(0));
580 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported FP_EXTEND!");
581 return TLI.makeLibCall(DAG, LC, NVT, Res32, CallOptions, SDLoc(N)).first;
582}
583
584// FIXME: Should we just use 'normal' FP_EXTEND / FP_TRUNC instead of special
585// nodes?
586SDValue DAGTypeLegalizer::SoftenFloatRes_BF16_TO_FP(SDNode *N) {
587 assert(N->getValueType(0) == MVT::f32 &&
588 "Can only soften BF16_TO_FP with f32 result");
589 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), MVT::f32);
590 SDValue Op = N->getOperand(0);
591 SDLoc DL(N);
592 Op = DAG.getNode(ISD::ANY_EXTEND, DL, NVT,
593 DAG.getNode(ISD::BITCAST, DL, MVT::i16, Op));
594 SDValue Res = DAG.getNode(ISD::SHL, DL, NVT, Op,
595 DAG.getShiftAmountConstant(16, NVT, DL));
596 return Res;
597}
598
599SDValue DAGTypeLegalizer::SoftenFloatRes_FP_ROUND(SDNode *N) {
600 bool IsStrict = N->isStrictFPOpcode();
601 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
602 SDValue Op = N->getOperand(IsStrict ? 1 : 0);
603 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
604 RTLIB::Libcall LC = RTLIB::getFPROUND(Op.getValueType(), N->getValueType(0));
605 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported FP_ROUND!");
607 EVT OpVT = N->getOperand(IsStrict ? 1 : 0).getValueType();
608 CallOptions.setTypeListBeforeSoften(OpVT, N->getValueType(0), true);
609 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, NVT, Op,
610 CallOptions, SDLoc(N),
611 Chain);
612 if (IsStrict)
613 ReplaceValueWith(SDValue(N, 1), Tmp.second);
614 return Tmp.first;
615}
616
617SDValue DAGTypeLegalizer::SoftenFloatRes_FPOW(SDNode *N) {
618 return SoftenFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
619 RTLIB::POW_F32,
620 RTLIB::POW_F64,
621 RTLIB::POW_F80,
622 RTLIB::POW_F128,
623 RTLIB::POW_PPCF128));
624}
625
626SDValue DAGTypeLegalizer::SoftenFloatRes_ExpOp(SDNode *N) {
627 bool IsStrict = N->isStrictFPOpcode();
628 unsigned Offset = IsStrict ? 1 : 0;
629 assert((N->getOperand(1 + Offset).getValueType() == MVT::i16 ||
630 N->getOperand(1 + Offset).getValueType() == MVT::i32) &&
631 "Unsupported power type!");
632 bool IsPowI =
633 N->getOpcode() == ISD::FPOWI || N->getOpcode() == ISD::STRICT_FPOWI;
634
635 RTLIB::Libcall LC = IsPowI ? RTLIB::getPOWI(N->getValueType(0))
636 : RTLIB::getLDEXP(N->getValueType(0));
637 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected fpowi.");
638 if (!TLI.getLibcallName(LC)) {
639 // Some targets don't have a powi libcall; use pow instead.
640 // FIXME: Implement this if some target needs it.
641 DAG.getContext()->emitError("Don't know how to soften fpowi to fpow");
642 return DAG.getUNDEF(N->getValueType(0));
643 }
644
645 if (DAG.getLibInfo().getIntSize() !=
646 N->getOperand(1 + Offset).getValueType().getSizeInBits()) {
647 // If the exponent does not match with sizeof(int) a libcall to RTLIB::POWI
648 // would use the wrong type for the argument.
649 DAG.getContext()->emitError("POWI exponent does not match sizeof(int)");
650 return DAG.getUNDEF(N->getValueType(0));
651 }
652
653 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
654 SDValue Ops[2] = { GetSoftenedFloat(N->getOperand(0 + Offset)),
655 N->getOperand(1 + Offset) };
656 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
658 EVT OpsVT[2] = { N->getOperand(0 + Offset).getValueType(),
659 N->getOperand(1 + Offset).getValueType() };
660 CallOptions.setTypeListBeforeSoften(OpsVT, N->getValueType(0), true);
661 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, NVT, Ops,
662 CallOptions, SDLoc(N),
663 Chain);
664 if (IsStrict)
665 ReplaceValueWith(SDValue(N, 1), Tmp.second);
666 return Tmp.first;
667}
668
669SDValue DAGTypeLegalizer::SoftenFloatRes_FFREXP(SDNode *N) {
670 assert(!N->isStrictFPOpcode() && "strictfp not implemented for frexp");
671 EVT VT0 = N->getValueType(0);
672 EVT VT1 = N->getValueType(1);
674
675 if (DAG.getLibInfo().getIntSize() != VT1.getSizeInBits()) {
676 // If the exponent does not match with sizeof(int) a libcall would use the
677 // wrong type for the argument.
678 // TODO: Should be able to handle mismatches.
679 DAG.getContext()->emitError("ffrexp exponent does not match sizeof(int)");
680 return DAG.getUNDEF(N->getValueType(0));
681 }
682
683 EVT NVT0 = TLI.getTypeToTransformTo(*DAG.getContext(), VT0);
684 SDValue StackSlot = DAG.CreateStackTemporary(VT1);
685
686 SDLoc DL(N);
687
689 SDValue Ops[2] = {GetSoftenedFloat(N->getOperand(0)), StackSlot};
690 EVT OpsVT[2] = {VT0, StackSlot.getValueType()};
691
692 // TODO: setTypeListBeforeSoften can't properly express multiple return types,
693 // but we only really need to handle the 0th one for softening anyway.
694 CallOptions.setTypeListBeforeSoften({OpsVT}, VT0, true);
695
696 auto [ReturnVal, Chain] = TLI.makeLibCall(DAG, LC, NVT0, Ops, CallOptions, DL,
697 /*Chain=*/SDValue());
698 int FrameIdx = cast<FrameIndexSDNode>(StackSlot)->getIndex();
699 auto PtrInfo =
701
702 SDValue LoadExp = DAG.getLoad(VT1, DL, Chain, StackSlot, PtrInfo);
703
704 ReplaceValueWith(SDValue(N, 1), LoadExp);
705 return ReturnVal;
706}
707
708SDValue DAGTypeLegalizer::SoftenFloatRes_FREM(SDNode *N) {
709 return SoftenFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
710 RTLIB::REM_F32,
711 RTLIB::REM_F64,
712 RTLIB::REM_F80,
713 RTLIB::REM_F128,
714 RTLIB::REM_PPCF128));
715}
716
717SDValue DAGTypeLegalizer::SoftenFloatRes_FRINT(SDNode *N) {
718 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
719 RTLIB::RINT_F32,
720 RTLIB::RINT_F64,
721 RTLIB::RINT_F80,
722 RTLIB::RINT_F128,
723 RTLIB::RINT_PPCF128));
724}
725
726SDValue DAGTypeLegalizer::SoftenFloatRes_FROUND(SDNode *N) {
727 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
728 RTLIB::ROUND_F32,
729 RTLIB::ROUND_F64,
730 RTLIB::ROUND_F80,
731 RTLIB::ROUND_F128,
732 RTLIB::ROUND_PPCF128));
733}
734
735SDValue DAGTypeLegalizer::SoftenFloatRes_FROUNDEVEN(SDNode *N) {
736 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
737 RTLIB::ROUNDEVEN_F32,
738 RTLIB::ROUNDEVEN_F64,
739 RTLIB::ROUNDEVEN_F80,
740 RTLIB::ROUNDEVEN_F128,
741 RTLIB::ROUNDEVEN_PPCF128));
742}
743
744SDValue DAGTypeLegalizer::SoftenFloatRes_FSIN(SDNode *N) {
745 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
746 RTLIB::SIN_F32,
747 RTLIB::SIN_F64,
748 RTLIB::SIN_F80,
749 RTLIB::SIN_F128,
750 RTLIB::SIN_PPCF128));
751}
752
753SDValue DAGTypeLegalizer::SoftenFloatRes_FSQRT(SDNode *N) {
754 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
755 RTLIB::SQRT_F32,
756 RTLIB::SQRT_F64,
757 RTLIB::SQRT_F80,
758 RTLIB::SQRT_F128,
759 RTLIB::SQRT_PPCF128));
760}
761
762SDValue DAGTypeLegalizer::SoftenFloatRes_FSUB(SDNode *N) {
763 return SoftenFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
764 RTLIB::SUB_F32,
765 RTLIB::SUB_F64,
766 RTLIB::SUB_F80,
767 RTLIB::SUB_F128,
768 RTLIB::SUB_PPCF128));
769}
770
771SDValue DAGTypeLegalizer::SoftenFloatRes_FTRUNC(SDNode *N) {
772 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
773 RTLIB::TRUNC_F32,
774 RTLIB::TRUNC_F64,
775 RTLIB::TRUNC_F80,
776 RTLIB::TRUNC_F128,
777 RTLIB::TRUNC_PPCF128));
778}
779
780SDValue DAGTypeLegalizer::SoftenFloatRes_LOAD(SDNode *N) {
781 LoadSDNode *L = cast<LoadSDNode>(N);
782 EVT VT = N->getValueType(0);
783 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
784 SDLoc dl(N);
785
786 auto MMOFlags =
787 L->getMemOperand()->getFlags() &
789 SDValue NewL;
790 if (L->getExtensionType() == ISD::NON_EXTLOAD) {
791 NewL = DAG.getLoad(L->getAddressingMode(), L->getExtensionType(), NVT, dl,
792 L->getChain(), L->getBasePtr(), L->getOffset(),
793 L->getPointerInfo(), NVT, L->getOriginalAlign(),
794 MMOFlags, L->getAAInfo());
795 // Legalized the chain result - switch anything that used the old chain to
796 // use the new one.
797 ReplaceValueWith(SDValue(N, 1), NewL.getValue(1));
798 return NewL;
799 }
800
801 // Do a non-extending load followed by FP_EXTEND.
802 NewL = DAG.getLoad(L->getAddressingMode(), ISD::NON_EXTLOAD, L->getMemoryVT(),
803 dl, L->getChain(), L->getBasePtr(), L->getOffset(),
804 L->getPointerInfo(), L->getMemoryVT(),
805 L->getOriginalAlign(), MMOFlags, L->getAAInfo());
806 // Legalized the chain result - switch anything that used the old chain to
807 // use the new one.
808 ReplaceValueWith(SDValue(N, 1), NewL.getValue(1));
809 auto ExtendNode = DAG.getNode(ISD::FP_EXTEND, dl, VT, NewL);
810 return BitConvertToInteger(ExtendNode);
811}
812
813SDValue DAGTypeLegalizer::SoftenFloatRes_SELECT(SDNode *N) {
814 SDValue LHS = GetSoftenedFloat(N->getOperand(1));
815 SDValue RHS = GetSoftenedFloat(N->getOperand(2));
816 return DAG.getSelect(SDLoc(N),
817 LHS.getValueType(), N->getOperand(0), LHS, RHS);
818}
819
820SDValue DAGTypeLegalizer::SoftenFloatRes_SELECT_CC(SDNode *N) {
821 SDValue LHS = GetSoftenedFloat(N->getOperand(2));
822 SDValue RHS = GetSoftenedFloat(N->getOperand(3));
823 return DAG.getNode(ISD::SELECT_CC, SDLoc(N),
824 LHS.getValueType(), N->getOperand(0),
825 N->getOperand(1), LHS, RHS, N->getOperand(4));
826}
827
828SDValue DAGTypeLegalizer::SoftenFloatRes_UNDEF(SDNode *N) {
829 return DAG.getUNDEF(TLI.getTypeToTransformTo(*DAG.getContext(),
830 N->getValueType(0)));
831}
832
833SDValue DAGTypeLegalizer::SoftenFloatRes_VAARG(SDNode *N) {
834 SDValue Chain = N->getOperand(0); // Get the chain.
835 SDValue Ptr = N->getOperand(1); // Get the pointer.
836 EVT VT = N->getValueType(0);
837 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
838 SDLoc dl(N);
839
840 SDValue NewVAARG;
841 NewVAARG = DAG.getVAArg(NVT, dl, Chain, Ptr, N->getOperand(2),
842 N->getConstantOperandVal(3));
843
844 // Legalized the chain result - switch anything that used the old chain to
845 // use the new one.
846 if (N != NewVAARG.getValue(1).getNode())
847 ReplaceValueWith(SDValue(N, 1), NewVAARG.getValue(1));
848 return NewVAARG;
849}
850
851SDValue DAGTypeLegalizer::SoftenFloatRes_XINT_TO_FP(SDNode *N) {
852 bool IsStrict = N->isStrictFPOpcode();
853 bool Signed = N->getOpcode() == ISD::SINT_TO_FP ||
854 N->getOpcode() == ISD::STRICT_SINT_TO_FP;
855 EVT SVT = N->getOperand(IsStrict ? 1 : 0).getValueType();
856 EVT RVT = N->getValueType(0);
857 EVT NVT = EVT();
858 SDLoc dl(N);
859
860 // If the input is not legal, eg: i1 -> fp, then it needs to be promoted to
861 // a larger type, eg: i8 -> fp. Even if it is legal, no libcall may exactly
862 // match. Look for an appropriate libcall.
863 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
864 for (unsigned t = MVT::FIRST_INTEGER_VALUETYPE;
865 t <= MVT::LAST_INTEGER_VALUETYPE && LC == RTLIB::UNKNOWN_LIBCALL; ++t) {
866 NVT = (MVT::SimpleValueType)t;
867 // The source needs to big enough to hold the operand.
868 if (NVT.bitsGE(SVT))
869 LC = Signed ? RTLIB::getSINTTOFP(NVT, RVT):RTLIB::getUINTTOFP (NVT, RVT);
870 }
871 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported XINT_TO_FP!");
872
873 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
874 // Sign/zero extend the argument if the libcall takes a larger type.
876 NVT, N->getOperand(IsStrict ? 1 : 0));
878 CallOptions.setSExt(Signed);
879 CallOptions.setTypeListBeforeSoften(SVT, RVT, true);
880 std::pair<SDValue, SDValue> Tmp =
881 TLI.makeLibCall(DAG, LC, TLI.getTypeToTransformTo(*DAG.getContext(), RVT),
882 Op, CallOptions, dl, Chain);
883
884 if (IsStrict)
885 ReplaceValueWith(SDValue(N, 1), Tmp.second);
886 return Tmp.first;
887}
888
889SDValue DAGTypeLegalizer::SoftenFloatRes_VECREDUCE(SDNode *N) {
890 // Expand and soften recursively.
891 ReplaceValueWith(SDValue(N, 0), TLI.expandVecReduce(N, DAG));
892 return SDValue();
893}
894
895SDValue DAGTypeLegalizer::SoftenFloatRes_VECREDUCE_SEQ(SDNode *N) {
896 ReplaceValueWith(SDValue(N, 0), TLI.expandVecReduceSeq(N, DAG));
897 return SDValue();
898}
899
900//===----------------------------------------------------------------------===//
901// Convert Float Operand to Integer
902//===----------------------------------------------------------------------===//
903
904bool DAGTypeLegalizer::SoftenFloatOperand(SDNode *N, unsigned OpNo) {
905 LLVM_DEBUG(dbgs() << "Soften float operand " << OpNo << ": "; N->dump(&DAG));
906 SDValue Res = SDValue();
907
908 switch (N->getOpcode()) {
909 default:
910#ifndef NDEBUG
911 dbgs() << "SoftenFloatOperand Op #" << OpNo << ": ";
912 N->dump(&DAG); dbgs() << "\n";
913#endif
914 report_fatal_error("Do not know how to soften this operator's operand!");
915
916 case ISD::BITCAST: Res = SoftenFloatOp_BITCAST(N); break;
917 case ISD::BR_CC: Res = SoftenFloatOp_BR_CC(N); break;
919 case ISD::FP_TO_FP16: // Same as FP_ROUND for softening purposes
920 case ISD::FP_TO_BF16:
923 case ISD::FP_ROUND: Res = SoftenFloatOp_FP_ROUND(N); break;
926 case ISD::FP_TO_SINT:
927 case ISD::FP_TO_UINT: Res = SoftenFloatOp_FP_TO_XINT(N); break;
930 Res = SoftenFloatOp_FP_TO_XINT_SAT(N); break;
932 case ISD::LROUND: Res = SoftenFloatOp_LROUND(N); break;
934 case ISD::LLROUND: Res = SoftenFloatOp_LLROUND(N); break;
936 case ISD::LRINT: Res = SoftenFloatOp_LRINT(N); break;
938 case ISD::LLRINT: Res = SoftenFloatOp_LLRINT(N); break;
939 case ISD::SELECT_CC: Res = SoftenFloatOp_SELECT_CC(N); break;
942 case ISD::SETCC: Res = SoftenFloatOp_SETCC(N); break;
943 case ISD::STORE: Res = SoftenFloatOp_STORE(N, OpNo); break;
944 case ISD::FCOPYSIGN: Res = SoftenFloatOp_FCOPYSIGN(N); break;
945 }
946
947 // If the result is null, the sub-method took care of registering results etc.
948 if (!Res.getNode()) return false;
949
950 // If the result is N, the sub-method updated N in place. Tell the legalizer
951 // core about this to re-analyze.
952 if (Res.getNode() == N)
953 return true;
954
955 assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 &&
956 "Invalid operand softening");
957
958 ReplaceValueWith(SDValue(N, 0), Res);
959 return false;
960}
961
962SDValue DAGTypeLegalizer::SoftenFloatOp_BITCAST(SDNode *N) {
963 SDValue Op0 = GetSoftenedFloat(N->getOperand(0));
964
965 return DAG.getNode(ISD::BITCAST, SDLoc(N), N->getValueType(0), Op0);
966}
967
968SDValue DAGTypeLegalizer::SoftenFloatOp_FP_ROUND(SDNode *N) {
969 // We actually deal with the partially-softened FP_TO_FP16 node too, which
970 // returns an i16 so doesn't meet the constraints necessary for FP_ROUND.
971 assert(N->getOpcode() == ISD::FP_ROUND || N->getOpcode() == ISD::FP_TO_FP16 ||
972 N->getOpcode() == ISD::STRICT_FP_TO_FP16 ||
973 N->getOpcode() == ISD::FP_TO_BF16 ||
974 N->getOpcode() == ISD::STRICT_FP_TO_BF16 ||
975 N->getOpcode() == ISD::STRICT_FP_ROUND);
976
977 bool IsStrict = N->isStrictFPOpcode();
978 SDValue Op = N->getOperand(IsStrict ? 1 : 0);
979 EVT SVT = Op.getValueType();
980 EVT RVT = N->getValueType(0);
981 EVT FloatRVT = RVT;
982 if (N->getOpcode() == ISD::FP_TO_FP16 ||
983 N->getOpcode() == ISD::STRICT_FP_TO_FP16)
984 FloatRVT = MVT::f16;
985 else if (N->getOpcode() == ISD::FP_TO_BF16 ||
986 N->getOpcode() == ISD::STRICT_FP_TO_BF16)
987 FloatRVT = MVT::bf16;
988
989 RTLIB::Libcall LC = RTLIB::getFPROUND(SVT, FloatRVT);
990 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported FP_ROUND libcall");
991
992 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
993 Op = GetSoftenedFloat(Op);
995 CallOptions.setTypeListBeforeSoften(SVT, RVT, true);
996 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, RVT, Op,
997 CallOptions, SDLoc(N),
998 Chain);
999 if (IsStrict) {
1000 ReplaceValueWith(SDValue(N, 1), Tmp.second);
1001 ReplaceValueWith(SDValue(N, 0), Tmp.first);
1002 return SDValue();
1003 }
1004 return Tmp.first;
1005}
1006
1007SDValue DAGTypeLegalizer::SoftenFloatOp_BR_CC(SDNode *N) {
1008 SDValue NewLHS = N->getOperand(2), NewRHS = N->getOperand(3);
1009 ISD::CondCode CCCode = cast<CondCodeSDNode>(N->getOperand(1))->get();
1010
1011 EVT VT = NewLHS.getValueType();
1012 NewLHS = GetSoftenedFloat(NewLHS);
1013 NewRHS = GetSoftenedFloat(NewRHS);
1014 TLI.softenSetCCOperands(DAG, VT, NewLHS, NewRHS, CCCode, SDLoc(N),
1015 N->getOperand(2), N->getOperand(3));
1016
1017 // If softenSetCCOperands returned a scalar, we need to compare the result
1018 // against zero to select between true and false values.
1019 if (!NewRHS.getNode()) {
1020 NewRHS = DAG.getConstant(0, SDLoc(N), NewLHS.getValueType());
1021 CCCode = ISD::SETNE;
1022 }
1023
1024 // Update N to have the operands specified.
1025 return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0),
1026 DAG.getCondCode(CCCode), NewLHS, NewRHS,
1027 N->getOperand(4)),
1028 0);
1029}
1030
1031// Even if the result type is legal, no libcall may exactly match. (e.g. We
1032// don't have FP-i8 conversions) This helper method looks for an appropriate
1033// promoted libcall.
1034static RTLIB::Libcall findFPToIntLibcall(EVT SrcVT, EVT RetVT, EVT &Promoted,
1035 bool Signed) {
1036 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
1037 for (unsigned IntVT = MVT::FIRST_INTEGER_VALUETYPE;
1038 IntVT <= MVT::LAST_INTEGER_VALUETYPE && LC == RTLIB::UNKNOWN_LIBCALL;
1039 ++IntVT) {
1040 Promoted = (MVT::SimpleValueType)IntVT;
1041 // The type needs to big enough to hold the result.
1042 if (Promoted.bitsGE(RetVT))
1043 LC = Signed ? RTLIB::getFPTOSINT(SrcVT, Promoted)
1044 : RTLIB::getFPTOUINT(SrcVT, Promoted);
1045 }
1046 return LC;
1047}
1048
1049SDValue DAGTypeLegalizer::SoftenFloatOp_FP_TO_XINT(SDNode *N) {
1050 bool IsStrict = N->isStrictFPOpcode();
1051 bool Signed = N->getOpcode() == ISD::FP_TO_SINT ||
1052 N->getOpcode() == ISD::STRICT_FP_TO_SINT;
1053
1054 SDValue Op = N->getOperand(IsStrict ? 1 : 0);
1055 EVT SVT = Op.getValueType();
1056 EVT RVT = N->getValueType(0);
1057 EVT NVT = EVT();
1058 SDLoc dl(N);
1059
1060 // If the result is not legal, eg: fp -> i1, then it needs to be promoted to
1061 // a larger type, eg: fp -> i32. Even if it is legal, no libcall may exactly
1062 // match, eg. we don't have fp -> i8 conversions.
1063 // Look for an appropriate libcall.
1064 RTLIB::Libcall LC = findFPToIntLibcall(SVT, RVT, NVT, Signed);
1065 assert(LC != RTLIB::UNKNOWN_LIBCALL && NVT.isSimple() &&
1066 "Unsupported FP_TO_XINT!");
1067
1068 Op = GetSoftenedFloat(Op);
1069 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
1071 CallOptions.setTypeListBeforeSoften(SVT, RVT, true);
1072 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, NVT, Op,
1073 CallOptions, dl, Chain);
1074
1075 // Truncate the result if the libcall returns a larger type.
1076 SDValue Res = DAG.getNode(ISD::TRUNCATE, dl, RVT, Tmp.first);
1077
1078 if (!IsStrict)
1079 return Res;
1080
1081 ReplaceValueWith(SDValue(N, 1), Tmp.second);
1082 ReplaceValueWith(SDValue(N, 0), Res);
1083 return SDValue();
1084}
1085
1086SDValue DAGTypeLegalizer::SoftenFloatOp_FP_TO_XINT_SAT(SDNode *N) {
1087 SDValue Res = TLI.expandFP_TO_INT_SAT(N, DAG);
1088 return Res;
1089}
1090
1091SDValue DAGTypeLegalizer::SoftenFloatOp_SELECT_CC(SDNode *N) {
1092 SDValue NewLHS = N->getOperand(0), NewRHS = N->getOperand(1);
1093 ISD::CondCode CCCode = cast<CondCodeSDNode>(N->getOperand(4))->get();
1094
1095 EVT VT = NewLHS.getValueType();
1096 NewLHS = GetSoftenedFloat(NewLHS);
1097 NewRHS = GetSoftenedFloat(NewRHS);
1098 TLI.softenSetCCOperands(DAG, VT, NewLHS, NewRHS, CCCode, SDLoc(N),
1099 N->getOperand(0), N->getOperand(1));
1100
1101 // If softenSetCCOperands returned a scalar, we need to compare the result
1102 // against zero to select between true and false values.
1103 if (!NewRHS.getNode()) {
1104 NewRHS = DAG.getConstant(0, SDLoc(N), NewLHS.getValueType());
1105 CCCode = ISD::SETNE;
1106 }
1107
1108 // Update N to have the operands specified.
1109 return SDValue(DAG.UpdateNodeOperands(N, NewLHS, NewRHS,
1110 N->getOperand(2), N->getOperand(3),
1111 DAG.getCondCode(CCCode)),
1112 0);
1113}
1114
1115SDValue DAGTypeLegalizer::SoftenFloatOp_SETCC(SDNode *N) {
1116 bool IsStrict = N->isStrictFPOpcode();
1117 SDValue Op0 = N->getOperand(IsStrict ? 1 : 0);
1118 SDValue Op1 = N->getOperand(IsStrict ? 2 : 1);
1119 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
1120 ISD::CondCode CCCode =
1121 cast<CondCodeSDNode>(N->getOperand(IsStrict ? 3 : 2))->get();
1122
1123 EVT VT = Op0.getValueType();
1124 SDValue NewLHS = GetSoftenedFloat(Op0);
1125 SDValue NewRHS = GetSoftenedFloat(Op1);
1126 TLI.softenSetCCOperands(DAG, VT, NewLHS, NewRHS, CCCode, SDLoc(N), Op0, Op1,
1127 Chain, N->getOpcode() == ISD::STRICT_FSETCCS);
1128
1129 // Update N to have the operands specified.
1130 if (NewRHS.getNode()) {
1131 if (IsStrict)
1132 NewLHS = DAG.getNode(ISD::SETCC, SDLoc(N), N->getValueType(0), NewLHS,
1133 NewRHS, DAG.getCondCode(CCCode));
1134 else
1135 return SDValue(DAG.UpdateNodeOperands(N, NewLHS, NewRHS,
1136 DAG.getCondCode(CCCode)), 0);
1137 }
1138
1139 // Otherwise, softenSetCCOperands returned a scalar, use it.
1140 assert((NewRHS.getNode() || NewLHS.getValueType() == N->getValueType(0)) &&
1141 "Unexpected setcc expansion!");
1142
1143 if (IsStrict) {
1144 ReplaceValueWith(SDValue(N, 0), NewLHS);
1145 ReplaceValueWith(SDValue(N, 1), Chain);
1146 return SDValue();
1147 }
1148 return NewLHS;
1149}
1150
1151SDValue DAGTypeLegalizer::SoftenFloatOp_STORE(SDNode *N, unsigned OpNo) {
1152 assert(ISD::isUNINDEXEDStore(N) && "Indexed store during type legalization!");
1153 assert(OpNo == 1 && "Can only soften the stored value!");
1154 StoreSDNode *ST = cast<StoreSDNode>(N);
1155 SDValue Val = ST->getValue();
1156 SDLoc dl(N);
1157
1158 if (ST->isTruncatingStore())
1159 // Do an FP_ROUND followed by a non-truncating store.
1160 Val = BitConvertToInteger(
1161 DAG.getNode(ISD::FP_ROUND, dl, ST->getMemoryVT(), Val,
1162 DAG.getIntPtrConstant(0, dl, /*isTarget=*/true)));
1163 else
1164 Val = GetSoftenedFloat(Val);
1165
1166 return DAG.getStore(ST->getChain(), dl, Val, ST->getBasePtr(),
1167 ST->getMemOperand());
1168}
1169
1170SDValue DAGTypeLegalizer::SoftenFloatOp_FCOPYSIGN(SDNode *N) {
1171 SDValue LHS = N->getOperand(0);
1172 SDValue RHS = BitConvertToInteger(N->getOperand(1));
1173 SDLoc dl(N);
1174
1175 EVT LVT = LHS.getValueType();
1176 EVT ILVT = EVT::getIntegerVT(*DAG.getContext(), LVT.getSizeInBits());
1177 EVT RVT = RHS.getValueType();
1178
1179 unsigned LSize = LVT.getSizeInBits();
1180 unsigned RSize = RVT.getSizeInBits();
1181
1182 // Shift right or sign-extend it if the two operands have different types.
1183 int SizeDiff = RSize - LSize;
1184 if (SizeDiff > 0) {
1185 RHS =
1186 DAG.getNode(ISD::SRL, dl, RVT, RHS,
1187 DAG.getConstant(SizeDiff, dl,
1188 TLI.getShiftAmountTy(RHS.getValueType(),
1189 DAG.getDataLayout())));
1190 RHS = DAG.getNode(ISD::TRUNCATE, dl, ILVT, RHS);
1191 } else if (SizeDiff < 0) {
1192 RHS = DAG.getNode(ISD::ANY_EXTEND, dl, LVT, RHS);
1193 RHS =
1194 DAG.getNode(ISD::SHL, dl, ILVT, RHS,
1195 DAG.getConstant(-SizeDiff, dl,
1196 TLI.getShiftAmountTy(RHS.getValueType(),
1197 DAG.getDataLayout())));
1198 }
1199
1200 RHS = DAG.getBitcast(LVT, RHS);
1201 return DAG.getNode(ISD::FCOPYSIGN, dl, LVT, LHS, RHS);
1202}
1203
1204SDValue DAGTypeLegalizer::SoftenFloatOp_Unary(SDNode *N, RTLIB::Libcall LC) {
1205 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
1206 bool IsStrict = N->isStrictFPOpcode();
1207 unsigned Offset = IsStrict ? 1 : 0;
1208 SDValue Op = GetSoftenedFloat(N->getOperand(0 + Offset));
1209 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
1211 EVT OpVT = N->getOperand(0 + Offset).getValueType();
1212 CallOptions.setTypeListBeforeSoften(OpVT, N->getValueType(0), true);
1213 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, NVT, Op,
1214 CallOptions, SDLoc(N),
1215 Chain);
1216 if (IsStrict) {
1217 ReplaceValueWith(SDValue(N, 1), Tmp.second);
1218 ReplaceValueWith(SDValue(N, 0), Tmp.first);
1219 return SDValue();
1220 }
1221
1222 return Tmp.first;
1223}
1224
1225SDValue DAGTypeLegalizer::SoftenFloatOp_LROUND(SDNode *N) {
1226 EVT OpVT = N->getOperand(N->isStrictFPOpcode() ? 1 : 0).getValueType();
1227 return SoftenFloatOp_Unary(N, GetFPLibCall(OpVT,
1228 RTLIB::LROUND_F32,
1229 RTLIB::LROUND_F64,
1230 RTLIB::LROUND_F80,
1231 RTLIB::LROUND_F128,
1232 RTLIB::LROUND_PPCF128));
1233}
1234
1235SDValue DAGTypeLegalizer::SoftenFloatOp_LLROUND(SDNode *N) {
1236 EVT OpVT = N->getOperand(N->isStrictFPOpcode() ? 1 : 0).getValueType();
1237 return SoftenFloatOp_Unary(N, GetFPLibCall(OpVT,
1238 RTLIB::LLROUND_F32,
1239 RTLIB::LLROUND_F64,
1240 RTLIB::LLROUND_F80,
1241 RTLIB::LLROUND_F128,
1242 RTLIB::LLROUND_PPCF128));
1243}
1244
1245SDValue DAGTypeLegalizer::SoftenFloatOp_LRINT(SDNode *N) {
1246 EVT OpVT = N->getOperand(N->isStrictFPOpcode() ? 1 : 0).getValueType();
1247 return SoftenFloatOp_Unary(N, GetFPLibCall(OpVT,
1248 RTLIB::LRINT_F32,
1249 RTLIB::LRINT_F64,
1250 RTLIB::LRINT_F80,
1251 RTLIB::LRINT_F128,
1252 RTLIB::LRINT_PPCF128));
1253}
1254
1255SDValue DAGTypeLegalizer::SoftenFloatOp_LLRINT(SDNode *N) {
1256 EVT OpVT = N->getOperand(N->isStrictFPOpcode() ? 1 : 0).getValueType();
1257 return SoftenFloatOp_Unary(N, GetFPLibCall(OpVT,
1258 RTLIB::LLRINT_F32,
1259 RTLIB::LLRINT_F64,
1260 RTLIB::LLRINT_F80,
1261 RTLIB::LLRINT_F128,
1262 RTLIB::LLRINT_PPCF128));
1263}
1264
1265//===----------------------------------------------------------------------===//
1266// Float Result Expansion
1267//===----------------------------------------------------------------------===//
1268
1269/// ExpandFloatResult - This method is called when the specified result of the
1270/// specified node is found to need expansion. At this point, the node may also
1271/// have invalid operands or may have other results that need promotion, we just
1272/// know that (at least) one result needs expansion.
1273void DAGTypeLegalizer::ExpandFloatResult(SDNode *N, unsigned ResNo) {
1274 LLVM_DEBUG(dbgs() << "Expand float result: "; N->dump(&DAG));
1275 SDValue Lo, Hi;
1276 Lo = Hi = SDValue();
1277
1278 // See if the target wants to custom expand this node.
1279 if (CustomLowerNode(N, N->getValueType(ResNo), true))
1280 return;
1281
1282 switch (N->getOpcode()) {
1283 default:
1284#ifndef NDEBUG
1285 dbgs() << "ExpandFloatResult #" << ResNo << ": ";
1286 N->dump(&DAG); dbgs() << "\n";
1287#endif
1288 report_fatal_error("Do not know how to expand the result of this "
1289 "operator!");
1290
1291 case ISD::UNDEF: SplitRes_UNDEF(N, Lo, Hi); break;
1292 case ISD::SELECT: SplitRes_Select(N, Lo, Hi); break;
1293 case ISD::SELECT_CC: SplitRes_SELECT_CC(N, Lo, Hi); break;
1294
1295 case ISD::MERGE_VALUES: ExpandRes_MERGE_VALUES(N, ResNo, Lo, Hi); break;
1296 case ISD::BITCAST: ExpandRes_BITCAST(N, Lo, Hi); break;
1297 case ISD::BUILD_PAIR: ExpandRes_BUILD_PAIR(N, Lo, Hi); break;
1298 case ISD::EXTRACT_ELEMENT: ExpandRes_EXTRACT_ELEMENT(N, Lo, Hi); break;
1299 case ISD::EXTRACT_VECTOR_ELT: ExpandRes_EXTRACT_VECTOR_ELT(N, Lo, Hi); break;
1300 case ISD::VAARG: ExpandRes_VAARG(N, Lo, Hi); break;
1301
1302 case ISD::ConstantFP: ExpandFloatRes_ConstantFP(N, Lo, Hi); break;
1303 case ISD::FABS: ExpandFloatRes_FABS(N, Lo, Hi); break;
1305 case ISD::FMINNUM: ExpandFloatRes_FMINNUM(N, Lo, Hi); break;
1307 case ISD::FMAXNUM: ExpandFloatRes_FMAXNUM(N, Lo, Hi); break;
1308 case ISD::STRICT_FADD:
1309 case ISD::FADD: ExpandFloatRes_FADD(N, Lo, Hi); break;
1310 case ISD::FCBRT: ExpandFloatRes_FCBRT(N, Lo, Hi); break;
1311 case ISD::STRICT_FCEIL:
1312 case ISD::FCEIL: ExpandFloatRes_FCEIL(N, Lo, Hi); break;
1313 case ISD::FCOPYSIGN: ExpandFloatRes_FCOPYSIGN(N, Lo, Hi); break;
1314 case ISD::STRICT_FCOS:
1315 case ISD::FCOS: ExpandFloatRes_FCOS(N, Lo, Hi); break;
1316 case ISD::STRICT_FDIV:
1317 case ISD::FDIV: ExpandFloatRes_FDIV(N, Lo, Hi); break;
1318 case ISD::STRICT_FEXP:
1319 case ISD::FEXP: ExpandFloatRes_FEXP(N, Lo, Hi); break;
1320 case ISD::STRICT_FEXP2:
1321 case ISD::FEXP2: ExpandFloatRes_FEXP2(N, Lo, Hi); break;
1322 case ISD::FEXP10: ExpandFloatRes_FEXP10(N, Lo, Hi); break;
1323 case ISD::STRICT_FFLOOR:
1324 case ISD::FFLOOR: ExpandFloatRes_FFLOOR(N, Lo, Hi); break;
1325 case ISD::STRICT_FLOG:
1326 case ISD::FLOG: ExpandFloatRes_FLOG(N, Lo, Hi); break;
1327 case ISD::STRICT_FLOG2:
1328 case ISD::FLOG2: ExpandFloatRes_FLOG2(N, Lo, Hi); break;
1329 case ISD::STRICT_FLOG10:
1330 case ISD::FLOG10: ExpandFloatRes_FLOG10(N, Lo, Hi); break;
1331 case ISD::STRICT_FMA:
1332 case ISD::FMA: ExpandFloatRes_FMA(N, Lo, Hi); break;
1333 case ISD::STRICT_FMUL:
1334 case ISD::FMUL: ExpandFloatRes_FMUL(N, Lo, Hi); break;
1336 case ISD::FNEARBYINT: ExpandFloatRes_FNEARBYINT(N, Lo, Hi); break;
1337 case ISD::FNEG: ExpandFloatRes_FNEG(N, Lo, Hi); break;
1339 case ISD::FP_EXTEND: ExpandFloatRes_FP_EXTEND(N, Lo, Hi); break;
1340 case ISD::STRICT_FPOW:
1341 case ISD::FPOW: ExpandFloatRes_FPOW(N, Lo, Hi); break;
1342 case ISD::STRICT_FPOWI:
1343 case ISD::FPOWI: ExpandFloatRes_FPOWI(N, Lo, Hi); break;
1344 case ISD::FLDEXP:
1345 case ISD::STRICT_FLDEXP: ExpandFloatRes_FLDEXP(N, Lo, Hi); break;
1346 case ISD::FREEZE: ExpandFloatRes_FREEZE(N, Lo, Hi); break;
1347 case ISD::STRICT_FRINT:
1348 case ISD::FRINT: ExpandFloatRes_FRINT(N, Lo, Hi); break;
1349 case ISD::STRICT_FROUND:
1350 case ISD::FROUND: ExpandFloatRes_FROUND(N, Lo, Hi); break;
1352 case ISD::FROUNDEVEN: ExpandFloatRes_FROUNDEVEN(N, Lo, Hi); break;
1353 case ISD::STRICT_FSIN:
1354 case ISD::FSIN: ExpandFloatRes_FSIN(N, Lo, Hi); break;
1355 case ISD::STRICT_FSQRT:
1356 case ISD::FSQRT: ExpandFloatRes_FSQRT(N, Lo, Hi); break;
1357 case ISD::STRICT_FSUB:
1358 case ISD::FSUB: ExpandFloatRes_FSUB(N, Lo, Hi); break;
1359 case ISD::STRICT_FTRUNC:
1360 case ISD::FTRUNC: ExpandFloatRes_FTRUNC(N, Lo, Hi); break;
1361 case ISD::LOAD: ExpandFloatRes_LOAD(N, Lo, Hi); break;
1364 case ISD::SINT_TO_FP:
1365 case ISD::UINT_TO_FP: ExpandFloatRes_XINT_TO_FP(N, Lo, Hi); break;
1366 case ISD::STRICT_FREM:
1367 case ISD::FREM: ExpandFloatRes_FREM(N, Lo, Hi); break;
1368 }
1369
1370 // If Lo/Hi is null, the sub-method took care of registering results etc.
1371 if (Lo.getNode())
1372 SetExpandedFloat(SDValue(N, ResNo), Lo, Hi);
1373}
1374
1375void DAGTypeLegalizer::ExpandFloatRes_ConstantFP(SDNode *N, SDValue &Lo,
1376 SDValue &Hi) {
1377 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
1378 assert(NVT.getSizeInBits() == 64 &&
1379 "Do not know how to expand this float constant!");
1380 APInt C = cast<ConstantFPSDNode>(N)->getValueAPF().bitcastToAPInt();
1381 SDLoc dl(N);
1383 APInt(64, C.getRawData()[1])),
1384 dl, NVT);
1386 APInt(64, C.getRawData()[0])),
1387 dl, NVT);
1388}
1389
1390void DAGTypeLegalizer::ExpandFloatRes_Unary(SDNode *N, RTLIB::Libcall LC,
1391 SDValue &Lo, SDValue &Hi) {
1392 bool IsStrict = N->isStrictFPOpcode();
1393 unsigned Offset = IsStrict ? 1 : 0;
1394 SDValue Op = N->getOperand(0 + Offset);
1395 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
1397 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, N->getValueType(0),
1398 Op, CallOptions, SDLoc(N),
1399 Chain);
1400 if (IsStrict)
1401 ReplaceValueWith(SDValue(N, 1), Tmp.second);
1402 GetPairElements(Tmp.first, Lo, Hi);
1403}
1404
1405void DAGTypeLegalizer::ExpandFloatRes_Binary(SDNode *N, RTLIB::Libcall LC,
1406 SDValue &Lo, SDValue &Hi) {
1407 bool IsStrict = N->isStrictFPOpcode();
1408 unsigned Offset = IsStrict ? 1 : 0;
1409 SDValue Ops[] = { N->getOperand(0 + Offset), N->getOperand(1 + Offset) };
1410 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
1412 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, N->getValueType(0),
1413 Ops, CallOptions, SDLoc(N),
1414 Chain);
1415 if (IsStrict)
1416 ReplaceValueWith(SDValue(N, 1), Tmp.second);
1417 GetPairElements(Tmp.first, Lo, Hi);
1418}
1419
1420void DAGTypeLegalizer::ExpandFloatRes_FABS(SDNode *N, SDValue &Lo,
1421 SDValue &Hi) {
1422 assert(N->getValueType(0) == MVT::ppcf128 &&
1423 "Logic only correct for ppcf128!");
1424 SDLoc dl(N);
1425 SDValue Tmp;
1426 GetExpandedFloat(N->getOperand(0), Lo, Tmp);
1427 Hi = DAG.getNode(ISD::FABS, dl, Tmp.getValueType(), Tmp);
1428 // Lo = Hi==fabs(Hi) ? Lo : -Lo;
1429 Lo = DAG.getSelectCC(dl, Tmp, Hi, Lo,
1430 DAG.getNode(ISD::FNEG, dl, Lo.getValueType(), Lo),
1431 ISD::SETEQ);
1432}
1433
1434void DAGTypeLegalizer::ExpandFloatRes_FMINNUM(SDNode *N, SDValue &Lo,
1435 SDValue &Hi) {
1436 ExpandFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
1437 RTLIB::FMIN_F32, RTLIB::FMIN_F64,
1438 RTLIB::FMIN_F80, RTLIB::FMIN_F128,
1439 RTLIB::FMIN_PPCF128), Lo, Hi);
1440}
1441
1442void DAGTypeLegalizer::ExpandFloatRes_FMAXNUM(SDNode *N, SDValue &Lo,
1443 SDValue &Hi) {
1444 ExpandFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
1445 RTLIB::FMAX_F32, RTLIB::FMAX_F64,
1446 RTLIB::FMAX_F80, RTLIB::FMAX_F128,
1447 RTLIB::FMAX_PPCF128), Lo, Hi);
1448}
1449
1450void DAGTypeLegalizer::ExpandFloatRes_FADD(SDNode *N, SDValue &Lo,
1451 SDValue &Hi) {
1452 ExpandFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
1453 RTLIB::ADD_F32, RTLIB::ADD_F64,
1454 RTLIB::ADD_F80, RTLIB::ADD_F128,
1455 RTLIB::ADD_PPCF128), Lo, Hi);
1456}
1457
1458void DAGTypeLegalizer::ExpandFloatRes_FCBRT(SDNode *N, SDValue &Lo,
1459 SDValue &Hi) {
1460 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0), RTLIB::CBRT_F32,
1461 RTLIB::CBRT_F64, RTLIB::CBRT_F80,
1462 RTLIB::CBRT_F128,
1463 RTLIB::CBRT_PPCF128), Lo, Hi);
1464}
1465
1466void DAGTypeLegalizer::ExpandFloatRes_FCEIL(SDNode *N,
1467 SDValue &Lo, SDValue &Hi) {
1468 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
1469 RTLIB::CEIL_F32, RTLIB::CEIL_F64,
1470 RTLIB::CEIL_F80, RTLIB::CEIL_F128,
1471 RTLIB::CEIL_PPCF128), Lo, Hi);
1472}
1473
1474void DAGTypeLegalizer::ExpandFloatRes_FCOPYSIGN(SDNode *N,
1475 SDValue &Lo, SDValue &Hi) {
1476 ExpandFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
1477 RTLIB::COPYSIGN_F32,
1478 RTLIB::COPYSIGN_F64,
1479 RTLIB::COPYSIGN_F80,
1480 RTLIB::COPYSIGN_F128,
1481 RTLIB::COPYSIGN_PPCF128), Lo, Hi);
1482}
1483
1484void DAGTypeLegalizer::ExpandFloatRes_FCOS(SDNode *N,
1485 SDValue &Lo, SDValue &Hi) {
1486 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
1487 RTLIB::COS_F32, RTLIB::COS_F64,
1488 RTLIB::COS_F80, RTLIB::COS_F128,
1489 RTLIB::COS_PPCF128), Lo, Hi);
1490}
1491
1492void DAGTypeLegalizer::ExpandFloatRes_FDIV(SDNode *N, SDValue &Lo,
1493 SDValue &Hi) {
1494 ExpandFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
1495 RTLIB::DIV_F32,
1496 RTLIB::DIV_F64,
1497 RTLIB::DIV_F80,
1498 RTLIB::DIV_F128,
1499 RTLIB::DIV_PPCF128), Lo, Hi);
1500}
1501
1502void DAGTypeLegalizer::ExpandFloatRes_FEXP(SDNode *N,
1503 SDValue &Lo, SDValue &Hi) {
1504 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
1505 RTLIB::EXP_F32, RTLIB::EXP_F64,
1506 RTLIB::EXP_F80, RTLIB::EXP_F128,
1507 RTLIB::EXP_PPCF128), Lo, Hi);
1508}
1509
1510void DAGTypeLegalizer::ExpandFloatRes_FEXP2(SDNode *N,
1511 SDValue &Lo, SDValue &Hi) {
1512 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
1513 RTLIB::EXP2_F32, RTLIB::EXP2_F64,
1514 RTLIB::EXP2_F80, RTLIB::EXP2_F128,
1515 RTLIB::EXP2_PPCF128), Lo, Hi);
1516}
1517
1518void DAGTypeLegalizer::ExpandFloatRes_FEXP10(SDNode *N, SDValue &Lo,
1519 SDValue &Hi) {
1520 ExpandFloatRes_Unary(N,
1521 GetFPLibCall(N->getValueType(0), RTLIB::EXP10_F32,
1522 RTLIB::EXP10_F64, RTLIB::EXP10_F80,
1523 RTLIB::EXP10_F128, RTLIB::EXP10_PPCF128),
1524 Lo, Hi);
1525}
1526
1527void DAGTypeLegalizer::ExpandFloatRes_FFLOOR(SDNode *N,
1528 SDValue &Lo, SDValue &Hi) {
1529 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
1530 RTLIB::FLOOR_F32, RTLIB::FLOOR_F64,
1531 RTLIB::FLOOR_F80, RTLIB::FLOOR_F128,
1532 RTLIB::FLOOR_PPCF128), Lo, Hi);
1533}
1534
1535void DAGTypeLegalizer::ExpandFloatRes_FLOG(SDNode *N,
1536 SDValue &Lo, SDValue &Hi) {
1537 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
1538 RTLIB::LOG_F32, RTLIB::LOG_F64,
1539 RTLIB::LOG_F80, RTLIB::LOG_F128,
1540 RTLIB::LOG_PPCF128), Lo, Hi);
1541}
1542
1543void DAGTypeLegalizer::ExpandFloatRes_FLOG2(SDNode *N,
1544 SDValue &Lo, SDValue &Hi) {
1545 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
1546 RTLIB::LOG2_F32, RTLIB::LOG2_F64,
1547 RTLIB::LOG2_F80, RTLIB::LOG2_F128,
1548 RTLIB::LOG2_PPCF128), Lo, Hi);
1549}
1550
1551void DAGTypeLegalizer::ExpandFloatRes_FLOG10(SDNode *N,
1552 SDValue &Lo, SDValue &Hi) {
1553 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
1554 RTLIB::LOG10_F32, RTLIB::LOG10_F64,
1555 RTLIB::LOG10_F80, RTLIB::LOG10_F128,
1556 RTLIB::LOG10_PPCF128), Lo, Hi);
1557}
1558
1559void DAGTypeLegalizer::ExpandFloatRes_FMA(SDNode *N, SDValue &Lo,
1560 SDValue &Hi) {
1561 bool IsStrict = N->isStrictFPOpcode();
1562 unsigned Offset = IsStrict ? 1 : 0;
1563 SDValue Ops[3] = { N->getOperand(0 + Offset), N->getOperand(1 + Offset),
1564 N->getOperand(2 + Offset) };
1565 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
1567 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, GetFPLibCall(N->getValueType(0),
1568 RTLIB::FMA_F32,
1569 RTLIB::FMA_F64,
1570 RTLIB::FMA_F80,
1571 RTLIB::FMA_F128,
1572 RTLIB::FMA_PPCF128),
1573 N->getValueType(0), Ops, CallOptions,
1574 SDLoc(N), Chain);
1575 if (IsStrict)
1576 ReplaceValueWith(SDValue(N, 1), Tmp.second);
1577 GetPairElements(Tmp.first, Lo, Hi);
1578}
1579
1580void DAGTypeLegalizer::ExpandFloatRes_FMUL(SDNode *N, SDValue &Lo,
1581 SDValue &Hi) {
1582 ExpandFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
1583 RTLIB::MUL_F32,
1584 RTLIB::MUL_F64,
1585 RTLIB::MUL_F80,
1586 RTLIB::MUL_F128,
1587 RTLIB::MUL_PPCF128), Lo, Hi);
1588}
1589
1590void DAGTypeLegalizer::ExpandFloatRes_FNEARBYINT(SDNode *N,
1591 SDValue &Lo, SDValue &Hi) {
1592 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
1593 RTLIB::NEARBYINT_F32,
1594 RTLIB::NEARBYINT_F64,
1595 RTLIB::NEARBYINT_F80,
1596 RTLIB::NEARBYINT_F128,
1597 RTLIB::NEARBYINT_PPCF128), Lo, Hi);
1598}
1599
1600void DAGTypeLegalizer::ExpandFloatRes_FNEG(SDNode *N, SDValue &Lo,
1601 SDValue &Hi) {
1602 SDLoc dl(N);
1603 GetExpandedFloat(N->getOperand(0), Lo, Hi);
1604 Lo = DAG.getNode(ISD::FNEG, dl, Lo.getValueType(), Lo);
1605 Hi = DAG.getNode(ISD::FNEG, dl, Hi.getValueType(), Hi);
1606}
1607
1608void DAGTypeLegalizer::ExpandFloatRes_FP_EXTEND(SDNode *N, SDValue &Lo,
1609 SDValue &Hi) {
1610 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
1611 SDLoc dl(N);
1612 bool IsStrict = N->isStrictFPOpcode();
1613
1614 SDValue Chain;
1615 if (IsStrict) {
1616 // If the expanded type is the same as the input type, just bypass the node.
1617 if (NVT == N->getOperand(1).getValueType()) {
1618 Hi = N->getOperand(1);
1619 Chain = N->getOperand(0);
1620 } else {
1621 // Other we need to extend.
1622 Hi = DAG.getNode(ISD::STRICT_FP_EXTEND, dl, { NVT, MVT::Other },
1623 { N->getOperand(0), N->getOperand(1) });
1624 Chain = Hi.getValue(1);
1625 }
1626 } else {
1627 Hi = DAG.getNode(ISD::FP_EXTEND, dl, NVT, N->getOperand(0));
1628 }
1629
1631 APInt(NVT.getSizeInBits(), 0)), dl, NVT);
1632
1633 if (IsStrict)
1634 ReplaceValueWith(SDValue(N, 1), Chain);
1635}
1636
1637void DAGTypeLegalizer::ExpandFloatRes_FPOW(SDNode *N,
1638 SDValue &Lo, SDValue &Hi) {
1639 ExpandFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
1640 RTLIB::POW_F32, RTLIB::POW_F64,
1641 RTLIB::POW_F80, RTLIB::POW_F128,
1642 RTLIB::POW_PPCF128), Lo, Hi);
1643}
1644
1645void DAGTypeLegalizer::ExpandFloatRes_FPOWI(SDNode *N,
1646 SDValue &Lo, SDValue &Hi) {
1647 ExpandFloatRes_Binary(N, RTLIB::getPOWI(N->getValueType(0)), Lo, Hi);
1648}
1649
1650void DAGTypeLegalizer::ExpandFloatRes_FLDEXP(SDNode *N, SDValue &Lo,
1651 SDValue &Hi) {
1652 ExpandFloatRes_Binary(N, RTLIB::getLDEXP(N->getValueType(0)), Lo, Hi);
1653}
1654
1655void DAGTypeLegalizer::ExpandFloatRes_FREEZE(SDNode *N,
1656 SDValue &Lo, SDValue &Hi) {
1657 assert(N->getValueType(0) == MVT::ppcf128 &&
1658 "Logic only correct for ppcf128!");
1659
1660 SDLoc dl(N);
1661 GetExpandedFloat(N->getOperand(0), Lo, Hi);
1662 Lo = DAG.getNode(ISD::FREEZE, dl, Lo.getValueType(), Lo);
1663 Hi = DAG.getNode(ISD::FREEZE, dl, Hi.getValueType(), Hi);
1664}
1665
1666void DAGTypeLegalizer::ExpandFloatRes_FREM(SDNode *N,
1667 SDValue &Lo, SDValue &Hi) {
1668 ExpandFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
1669 RTLIB::REM_F32, RTLIB::REM_F64,
1670 RTLIB::REM_F80, RTLIB::REM_F128,
1671 RTLIB::REM_PPCF128), Lo, Hi);
1672}
1673
1674void DAGTypeLegalizer::ExpandFloatRes_FRINT(SDNode *N,
1675 SDValue &Lo, SDValue &Hi) {
1676 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
1677 RTLIB::RINT_F32, RTLIB::RINT_F64,
1678 RTLIB::RINT_F80, RTLIB::RINT_F128,
1679 RTLIB::RINT_PPCF128), Lo, Hi);
1680}
1681
1682void DAGTypeLegalizer::ExpandFloatRes_FROUND(SDNode *N,
1683 SDValue &Lo, SDValue &Hi) {
1684 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
1685 RTLIB::ROUND_F32,
1686 RTLIB::ROUND_F64,
1687 RTLIB::ROUND_F80,
1688 RTLIB::ROUND_F128,
1689 RTLIB::ROUND_PPCF128), Lo, Hi);
1690}
1691
1692void DAGTypeLegalizer::ExpandFloatRes_FROUNDEVEN(SDNode *N,
1693 SDValue &Lo, SDValue &Hi) {
1694 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
1695 RTLIB::ROUNDEVEN_F32,
1696 RTLIB::ROUNDEVEN_F64,
1697 RTLIB::ROUNDEVEN_F80,
1698 RTLIB::ROUNDEVEN_F128,
1699 RTLIB::ROUNDEVEN_PPCF128), Lo, Hi);
1700}
1701
1702void DAGTypeLegalizer::ExpandFloatRes_FSIN(SDNode *N,
1703 SDValue &Lo, SDValue &Hi) {
1704 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
1705 RTLIB::SIN_F32, RTLIB::SIN_F64,
1706 RTLIB::SIN_F80, RTLIB::SIN_F128,
1707 RTLIB::SIN_PPCF128), Lo, Hi);
1708}
1709
1710void DAGTypeLegalizer::ExpandFloatRes_FSQRT(SDNode *N,
1711 SDValue &Lo, SDValue &Hi) {
1712 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
1713 RTLIB::SQRT_F32, RTLIB::SQRT_F64,
1714 RTLIB::SQRT_F80, RTLIB::SQRT_F128,
1715 RTLIB::SQRT_PPCF128), Lo, Hi);
1716}
1717
1718void DAGTypeLegalizer::ExpandFloatRes_FSUB(SDNode *N, SDValue &Lo,
1719 SDValue &Hi) {
1720 ExpandFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
1721 RTLIB::SUB_F32,
1722 RTLIB::SUB_F64,
1723 RTLIB::SUB_F80,
1724 RTLIB::SUB_F128,
1725 RTLIB::SUB_PPCF128), Lo, Hi);
1726}
1727
1728void DAGTypeLegalizer::ExpandFloatRes_FTRUNC(SDNode *N,
1729 SDValue &Lo, SDValue &Hi) {
1730 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
1731 RTLIB::TRUNC_F32, RTLIB::TRUNC_F64,
1732 RTLIB::TRUNC_F80, RTLIB::TRUNC_F128,
1733 RTLIB::TRUNC_PPCF128), Lo, Hi);
1734}
1735
1736void DAGTypeLegalizer::ExpandFloatRes_LOAD(SDNode *N, SDValue &Lo,
1737 SDValue &Hi) {
1738 if (ISD::isNormalLoad(N)) {
1739 ExpandRes_NormalLoad(N, Lo, Hi);
1740 return;
1741 }
1742
1743 assert(ISD::isUNINDEXEDLoad(N) && "Indexed load during type legalization!");
1744 LoadSDNode *LD = cast<LoadSDNode>(N);
1745 SDValue Chain = LD->getChain();
1746 SDValue Ptr = LD->getBasePtr();
1747 SDLoc dl(N);
1748
1749 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), LD->getValueType(0));
1750 assert(NVT.isByteSized() && "Expanded type not byte sized!");
1751 assert(LD->getMemoryVT().bitsLE(NVT) && "Float type not round?");
1752
1753 Hi = DAG.getExtLoad(LD->getExtensionType(), dl, NVT, Chain, Ptr,
1754 LD->getMemoryVT(), LD->getMemOperand());
1755
1756 // Remember the chain.
1757 Chain = Hi.getValue(1);
1758
1759 // The low part is zero.
1761 APInt(NVT.getSizeInBits(), 0)), dl, NVT);
1762
1763 // Modified the chain - switch anything that used the old chain to use the
1764 // new one.
1765 ReplaceValueWith(SDValue(LD, 1), Chain);
1766}
1767
1768void DAGTypeLegalizer::ExpandFloatRes_XINT_TO_FP(SDNode *N, SDValue &Lo,
1769 SDValue &Hi) {
1770 assert(N->getValueType(0) == MVT::ppcf128 && "Unsupported XINT_TO_FP!");
1771 EVT VT = N->getValueType(0);
1772 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
1773 bool Strict = N->isStrictFPOpcode();
1774 SDValue Src = N->getOperand(Strict ? 1 : 0);
1775 EVT SrcVT = Src.getValueType();
1776 bool isSigned = N->getOpcode() == ISD::SINT_TO_FP ||
1777 N->getOpcode() == ISD::STRICT_SINT_TO_FP;
1778 SDLoc dl(N);
1779 SDValue Chain = Strict ? N->getOperand(0) : DAG.getEntryNode();
1780
1781 // TODO: Any other flags to propagate?
1783 Flags.setNoFPExcept(N->getFlags().hasNoFPExcept());
1784
1785 // First do an SINT_TO_FP, whether the original was signed or unsigned.
1786 // When promoting partial word types to i32 we must honor the signedness,
1787 // though.
1788 if (SrcVT.bitsLE(MVT::i32)) {
1789 // The integer can be represented exactly in an f64.
1791 APInt(NVT.getSizeInBits(), 0)), dl, NVT);
1792 if (Strict) {
1793 Hi = DAG.getNode(N->getOpcode(), dl, DAG.getVTList(NVT, MVT::Other),
1794 {Chain, Src}, Flags);
1795 Chain = Hi.getValue(1);
1796 } else
1797 Hi = DAG.getNode(N->getOpcode(), dl, NVT, Src);
1798 } else {
1799 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
1800 if (SrcVT.bitsLE(MVT::i64)) {
1802 MVT::i64, Src);
1803 LC = RTLIB::SINTTOFP_I64_PPCF128;
1804 } else if (SrcVT.bitsLE(MVT::i128)) {
1805 Src = DAG.getNode(ISD::SIGN_EXTEND, dl, MVT::i128, Src);
1806 LC = RTLIB::SINTTOFP_I128_PPCF128;
1807 }
1808 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported XINT_TO_FP!");
1809
1811 CallOptions.setSExt(true);
1812 std::pair<SDValue, SDValue> Tmp =
1813 TLI.makeLibCall(DAG, LC, VT, Src, CallOptions, dl, Chain);
1814 if (Strict)
1815 Chain = Tmp.second;
1816 GetPairElements(Tmp.first, Lo, Hi);
1817 }
1818
1819 // No need to complement for unsigned 32-bit integers
1820 if (isSigned || SrcVT.bitsLE(MVT::i32)) {
1821 if (Strict)
1822 ReplaceValueWith(SDValue(N, 1), Chain);
1823
1824 return;
1825 }
1826
1827 // Unsigned - fix up the SINT_TO_FP value just calculated.
1828 // FIXME: For unsigned i128 to ppc_fp128 conversion, we need to carefully
1829 // keep semantics correctness if the integer is not exactly representable
1830 // here. See ExpandLegalINT_TO_FP.
1831 Hi = DAG.getNode(ISD::BUILD_PAIR, dl, VT, Lo, Hi);
1832 SrcVT = Src.getValueType();
1833
1834 // x>=0 ? (ppcf128)(iN)x : (ppcf128)(iN)x + 2^N; N=32,64,128.
1835 static const uint64_t TwoE32[] = { 0x41f0000000000000LL, 0 };
1836 static const uint64_t TwoE64[] = { 0x43f0000000000000LL, 0 };
1837 static const uint64_t TwoE128[] = { 0x47f0000000000000LL, 0 };
1838 ArrayRef<uint64_t> Parts;
1839
1840 switch (SrcVT.getSimpleVT().SimpleTy) {
1841 default:
1842 llvm_unreachable("Unsupported UINT_TO_FP!");
1843 case MVT::i32:
1844 Parts = TwoE32;
1845 break;
1846 case MVT::i64:
1847 Parts = TwoE64;
1848 break;
1849 case MVT::i128:
1850 Parts = TwoE128;
1851 break;
1852 }
1853
1854 // TODO: Are there other fast-math-flags to propagate to this FADD?
1855 SDValue NewLo = DAG.getConstantFP(
1856 APFloat(APFloat::PPCDoubleDouble(), APInt(128, Parts)), dl, MVT::ppcf128);
1857 if (Strict) {
1858 Lo = DAG.getNode(ISD::STRICT_FADD, dl, DAG.getVTList(VT, MVT::Other),
1859 {Chain, Hi, NewLo}, Flags);
1860 Chain = Lo.getValue(1);
1861 ReplaceValueWith(SDValue(N, 1), Chain);
1862 } else
1863 Lo = DAG.getNode(ISD::FADD, dl, VT, Hi, NewLo);
1864 Lo = DAG.getSelectCC(dl, Src, DAG.getConstant(0, dl, SrcVT),
1865 Lo, Hi, ISD::SETLT);
1866 GetPairElements(Lo, Lo, Hi);
1867}
1868
1869
1870//===----------------------------------------------------------------------===//
1871// Float Operand Expansion
1872//===----------------------------------------------------------------------===//
1873
1874/// ExpandFloatOperand - This method is called when the specified operand of the
1875/// specified node is found to need expansion. At this point, all of the result
1876/// types of the node are known to be legal, but other operands of the node may
1877/// need promotion or expansion as well as the specified one.
1878bool DAGTypeLegalizer::ExpandFloatOperand(SDNode *N, unsigned OpNo) {
1879 LLVM_DEBUG(dbgs() << "Expand float operand: "; N->dump(&DAG));
1880 SDValue Res = SDValue();
1881
1882 // See if the target wants to custom expand this node.
1883 if (CustomLowerNode(N, N->getOperand(OpNo).getValueType(), false))
1884 return false;
1885
1886 switch (N->getOpcode()) {
1887 default:
1888#ifndef NDEBUG
1889 dbgs() << "ExpandFloatOperand Op #" << OpNo << ": ";
1890 N->dump(&DAG); dbgs() << "\n";
1891#endif
1892 report_fatal_error("Do not know how to expand this operator's operand!");
1893
1894 case ISD::BITCAST: Res = ExpandOp_BITCAST(N); break;
1895 case ISD::BUILD_VECTOR: Res = ExpandOp_BUILD_VECTOR(N); break;
1896 case ISD::EXTRACT_ELEMENT: Res = ExpandOp_EXTRACT_ELEMENT(N); break;
1897
1898 case ISD::BR_CC: Res = ExpandFloatOp_BR_CC(N); break;
1899 case ISD::FCOPYSIGN: Res = ExpandFloatOp_FCOPYSIGN(N); break;
1901 case ISD::FP_ROUND: Res = ExpandFloatOp_FP_ROUND(N); break;
1904 case ISD::FP_TO_SINT:
1905 case ISD::FP_TO_UINT: Res = ExpandFloatOp_FP_TO_XINT(N); break;
1906 case ISD::LROUND: Res = ExpandFloatOp_LROUND(N); break;
1907 case ISD::LLROUND: Res = ExpandFloatOp_LLROUND(N); break;
1908 case ISD::LRINT: Res = ExpandFloatOp_LRINT(N); break;
1909 case ISD::LLRINT: Res = ExpandFloatOp_LLRINT(N); break;
1910 case ISD::SELECT_CC: Res = ExpandFloatOp_SELECT_CC(N); break;
1911 case ISD::STRICT_FSETCC:
1913 case ISD::SETCC: Res = ExpandFloatOp_SETCC(N); break;
1914 case ISD::STORE: Res = ExpandFloatOp_STORE(cast<StoreSDNode>(N),
1915 OpNo); break;
1916 }
1917
1918 // If the result is null, the sub-method took care of registering results etc.
1919 if (!Res.getNode()) return false;
1920
1921 // If the result is N, the sub-method updated N in place. Tell the legalizer
1922 // core about this.
1923 if (Res.getNode() == N)
1924 return true;
1925
1926 assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 &&
1927 "Invalid operand expansion");
1928
1929 ReplaceValueWith(SDValue(N, 0), Res);
1930 return false;
1931}
1932
1933/// FloatExpandSetCCOperands - Expand the operands of a comparison. This code
1934/// is shared among BR_CC, SELECT_CC, and SETCC handlers.
1935void DAGTypeLegalizer::FloatExpandSetCCOperands(SDValue &NewLHS,
1936 SDValue &NewRHS,
1937 ISD::CondCode &CCCode,
1938 const SDLoc &dl, SDValue &Chain,
1939 bool IsSignaling) {
1940 SDValue LHSLo, LHSHi, RHSLo, RHSHi;
1941 GetExpandedFloat(NewLHS, LHSLo, LHSHi);
1942 GetExpandedFloat(NewRHS, RHSLo, RHSHi);
1943
1944 assert(NewLHS.getValueType() == MVT::ppcf128 && "Unsupported setcc type!");
1945
1946 // FIXME: This generated code sucks. We want to generate
1947 // FCMPU crN, hi1, hi2
1948 // BNE crN, L:
1949 // FCMPU crN, lo1, lo2
1950 // The following can be improved, but not that much.
1951 SDValue Tmp1, Tmp2, Tmp3, OutputChain;
1952 Tmp1 = DAG.getSetCC(dl, getSetCCResultType(LHSHi.getValueType()), LHSHi,
1953 RHSHi, ISD::SETOEQ, Chain, IsSignaling);
1954 OutputChain = Tmp1->getNumValues() > 1 ? Tmp1.getValue(1) : SDValue();
1955 Tmp2 = DAG.getSetCC(dl, getSetCCResultType(LHSLo.getValueType()), LHSLo,
1956 RHSLo, CCCode, OutputChain, IsSignaling);
1957 OutputChain = Tmp2->getNumValues() > 1 ? Tmp2.getValue(1) : SDValue();
1958 Tmp3 = DAG.getNode(ISD::AND, dl, Tmp1.getValueType(), Tmp1, Tmp2);
1959 Tmp1 =
1960 DAG.getSetCC(dl, getSetCCResultType(LHSHi.getValueType()), LHSHi, RHSHi,
1961 ISD::SETUNE, OutputChain, IsSignaling);
1962 OutputChain = Tmp1->getNumValues() > 1 ? Tmp1.getValue(1) : SDValue();
1963 Tmp2 = DAG.getSetCC(dl, getSetCCResultType(LHSHi.getValueType()), LHSHi,
1964 RHSHi, CCCode, OutputChain, IsSignaling);
1965 OutputChain = Tmp2->getNumValues() > 1 ? Tmp2.getValue(1) : SDValue();
1966 Tmp1 = DAG.getNode(ISD::AND, dl, Tmp1.getValueType(), Tmp1, Tmp2);
1967 NewLHS = DAG.getNode(ISD::OR, dl, Tmp1.getValueType(), Tmp1, Tmp3);
1968 NewRHS = SDValue(); // LHS is the result, not a compare.
1969 Chain = OutputChain;
1970}
1971
1972SDValue DAGTypeLegalizer::ExpandFloatOp_BR_CC(SDNode *N) {
1973 SDValue NewLHS = N->getOperand(2), NewRHS = N->getOperand(3);
1974 ISD::CondCode CCCode = cast<CondCodeSDNode>(N->getOperand(1))->get();
1975 SDValue Chain;
1976 FloatExpandSetCCOperands(NewLHS, NewRHS, CCCode, SDLoc(N), Chain);
1977
1978 // If ExpandSetCCOperands returned a scalar, we need to compare the result
1979 // against zero to select between true and false values.
1980 if (!NewRHS.getNode()) {
1981 NewRHS = DAG.getConstant(0, SDLoc(N), NewLHS.getValueType());
1982 CCCode = ISD::SETNE;
1983 }
1984
1985 // Update N to have the operands specified.
1986 return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0),
1987 DAG.getCondCode(CCCode), NewLHS, NewRHS,
1988 N->getOperand(4)), 0);
1989}
1990
1991SDValue DAGTypeLegalizer::ExpandFloatOp_FCOPYSIGN(SDNode *N) {
1992 assert(N->getOperand(1).getValueType() == MVT::ppcf128 &&
1993 "Logic only correct for ppcf128!");
1994 SDValue Lo, Hi;
1995 GetExpandedFloat(N->getOperand(1), Lo, Hi);
1996 // The ppcf128 value is providing only the sign; take it from the
1997 // higher-order double (which must have the larger magnitude).
1998 return DAG.getNode(ISD::FCOPYSIGN, SDLoc(N),
1999 N->getValueType(0), N->getOperand(0), Hi);
2000}
2001
2002SDValue DAGTypeLegalizer::ExpandFloatOp_FP_ROUND(SDNode *N) {
2003 bool IsStrict = N->isStrictFPOpcode();
2004 assert(N->getOperand(IsStrict ? 1 : 0).getValueType() == MVT::ppcf128 &&
2005 "Logic only correct for ppcf128!");
2006 SDValue Lo, Hi;
2007 GetExpandedFloat(N->getOperand(IsStrict ? 1 : 0), Lo, Hi);
2008
2009 if (!IsStrict)
2010 // Round it the rest of the way (e.g. to f32) if needed.
2011 return DAG.getNode(ISD::FP_ROUND, SDLoc(N),
2012 N->getValueType(0), Hi, N->getOperand(1));
2013
2014 // Eliminate the node if the input float type is the same as the output float
2015 // type.
2016 if (Hi.getValueType() == N->getValueType(0)) {
2017 // Connect the output chain to the input chain, unlinking the node.
2018 ReplaceValueWith(SDValue(N, 1), N->getOperand(0));
2019 ReplaceValueWith(SDValue(N, 0), Hi);
2020 return SDValue();
2021 }
2022
2024 {N->getValueType(0), MVT::Other},
2025 {N->getOperand(0), Hi, N->getOperand(2)});
2026 ReplaceValueWith(SDValue(N, 1), Expansion.getValue(1));
2027 ReplaceValueWith(SDValue(N, 0), Expansion);
2028 return SDValue();
2029}
2030
2031SDValue DAGTypeLegalizer::ExpandFloatOp_FP_TO_XINT(SDNode *N) {
2032 EVT RVT = N->getValueType(0);
2033 SDLoc dl(N);
2034
2035 bool IsStrict = N->isStrictFPOpcode();
2036 bool Signed = N->getOpcode() == ISD::FP_TO_SINT ||
2037 N->getOpcode() == ISD::STRICT_FP_TO_SINT;
2038 SDValue Op = N->getOperand(IsStrict ? 1 : 0);
2039 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
2040
2041 EVT NVT;
2042 RTLIB::Libcall LC = findFPToIntLibcall(Op.getValueType(), RVT, NVT, Signed);
2043 assert(LC != RTLIB::UNKNOWN_LIBCALL && NVT.isSimple() &&
2044 "Unsupported FP_TO_XINT!");
2046 std::pair<SDValue, SDValue> Tmp =
2047 TLI.makeLibCall(DAG, LC, NVT, Op, CallOptions, dl, Chain);
2048 if (!IsStrict)
2049 return Tmp.first;
2050
2051 ReplaceValueWith(SDValue(N, 1), Tmp.second);
2052 ReplaceValueWith(SDValue(N, 0), Tmp.first);
2053 return SDValue();
2054}
2055
2056SDValue DAGTypeLegalizer::ExpandFloatOp_SELECT_CC(SDNode *N) {
2057 SDValue NewLHS = N->getOperand(0), NewRHS = N->getOperand(1);
2058 ISD::CondCode CCCode = cast<CondCodeSDNode>(N->getOperand(4))->get();
2059 SDValue Chain;
2060 FloatExpandSetCCOperands(NewLHS, NewRHS, CCCode, SDLoc(N), Chain);
2061
2062 // If ExpandSetCCOperands returned a scalar, we need to compare the result
2063 // against zero to select between true and false values.
2064 if (!NewRHS.getNode()) {
2065 NewRHS = DAG.getConstant(0, SDLoc(N), NewLHS.getValueType());
2066 CCCode = ISD::SETNE;
2067 }
2068
2069 // Update N to have the operands specified.
2070 return SDValue(DAG.UpdateNodeOperands(N, NewLHS, NewRHS,
2071 N->getOperand(2), N->getOperand(3),
2072 DAG.getCondCode(CCCode)), 0);
2073}
2074
2075SDValue DAGTypeLegalizer::ExpandFloatOp_SETCC(SDNode *N) {
2076 bool IsStrict = N->isStrictFPOpcode();
2077 SDValue NewLHS = N->getOperand(IsStrict ? 1 : 0);
2078 SDValue NewRHS = N->getOperand(IsStrict ? 2 : 1);
2079 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
2080 ISD::CondCode CCCode =
2081 cast<CondCodeSDNode>(N->getOperand(IsStrict ? 3 : 2))->get();
2082 FloatExpandSetCCOperands(NewLHS, NewRHS, CCCode, SDLoc(N), Chain,
2083 N->getOpcode() == ISD::STRICT_FSETCCS);
2084
2085 // FloatExpandSetCCOperands always returned a scalar.
2086 assert(!NewRHS.getNode() && "Expect to return scalar");
2087 assert(NewLHS.getValueType() == N->getValueType(0) &&
2088 "Unexpected setcc expansion!");
2089 if (Chain) {
2090 ReplaceValueWith(SDValue(N, 0), NewLHS);
2091 ReplaceValueWith(SDValue(N, 1), Chain);
2092 return SDValue();
2093 }
2094 return NewLHS;
2095}
2096
2097SDValue DAGTypeLegalizer::ExpandFloatOp_STORE(SDNode *N, unsigned OpNo) {
2098 if (ISD::isNormalStore(N))
2099 return ExpandOp_NormalStore(N, OpNo);
2100
2101 assert(ISD::isUNINDEXEDStore(N) && "Indexed store during type legalization!");
2102 assert(OpNo == 1 && "Can only expand the stored value so far");
2103 StoreSDNode *ST = cast<StoreSDNode>(N);
2104
2105 SDValue Chain = ST->getChain();
2106 SDValue Ptr = ST->getBasePtr();
2107
2108 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(),
2109 ST->getValue().getValueType());
2110 assert(NVT.isByteSized() && "Expanded type not byte sized!");
2111 assert(ST->getMemoryVT().bitsLE(NVT) && "Float type not round?");
2112 (void)NVT;
2113
2114 SDValue Lo, Hi;
2115 GetExpandedOp(ST->getValue(), Lo, Hi);
2116
2117 return DAG.getTruncStore(Chain, SDLoc(N), Hi, Ptr,
2118 ST->getMemoryVT(), ST->getMemOperand());
2119}
2120
2121SDValue DAGTypeLegalizer::ExpandFloatOp_LROUND(SDNode *N) {
2122 EVT RVT = N->getValueType(0);
2123 EVT RetVT = N->getOperand(0).getValueType();
2125 return TLI.makeLibCall(DAG, GetFPLibCall(RetVT,
2126 RTLIB::LROUND_F32,
2127 RTLIB::LROUND_F64,
2128 RTLIB::LROUND_F80,
2129 RTLIB::LROUND_F128,
2130 RTLIB::LROUND_PPCF128),
2131 RVT, N->getOperand(0), CallOptions, SDLoc(N)).first;
2132}
2133
2134SDValue DAGTypeLegalizer::ExpandFloatOp_LLROUND(SDNode *N) {
2135 EVT RVT = N->getValueType(0);
2136 EVT RetVT = N->getOperand(0).getValueType();
2138 return TLI.makeLibCall(DAG, GetFPLibCall(RetVT,
2139 RTLIB::LLROUND_F32,
2140 RTLIB::LLROUND_F64,
2141 RTLIB::LLROUND_F80,
2142 RTLIB::LLROUND_F128,
2143 RTLIB::LLROUND_PPCF128),
2144 RVT, N->getOperand(0), CallOptions, SDLoc(N)).first;
2145}
2146
2147SDValue DAGTypeLegalizer::ExpandFloatOp_LRINT(SDNode *N) {
2148 EVT RVT = N->getValueType(0);
2149 EVT RetVT = N->getOperand(0).getValueType();
2151 return TLI.makeLibCall(DAG, GetFPLibCall(RetVT,
2152 RTLIB::LRINT_F32,
2153 RTLIB::LRINT_F64,
2154 RTLIB::LRINT_F80,
2155 RTLIB::LRINT_F128,
2156 RTLIB::LRINT_PPCF128),
2157 RVT, N->getOperand(0), CallOptions, SDLoc(N)).first;
2158}
2159
2160SDValue DAGTypeLegalizer::ExpandFloatOp_LLRINT(SDNode *N) {
2161 EVT RVT = N->getValueType(0);
2162 EVT RetVT = N->getOperand(0).getValueType();
2164 return TLI.makeLibCall(DAG, GetFPLibCall(RetVT,
2165 RTLIB::LLRINT_F32,
2166 RTLIB::LLRINT_F64,
2167 RTLIB::LLRINT_F80,
2168 RTLIB::LLRINT_F128,
2169 RTLIB::LLRINT_PPCF128),
2170 RVT, N->getOperand(0), CallOptions, SDLoc(N)).first;
2171}
2172
2173//===----------------------------------------------------------------------===//
2174// Float Operand Promotion
2175//===----------------------------------------------------------------------===//
2176//
2177
2179 if (OpVT == MVT::f16) {
2180 return ISD::FP16_TO_FP;
2181 } else if (RetVT == MVT::f16) {
2182 return ISD::FP_TO_FP16;
2183 } else if (OpVT == MVT::bf16) {
2184 return ISD::BF16_TO_FP;
2185 } else if (RetVT == MVT::bf16) {
2186 return ISD::FP_TO_BF16;
2187 }
2188
2189 report_fatal_error("Attempt at an invalid promotion-related conversion");
2190}
2191
2193 if (OpVT == MVT::f16)
2195
2196 if (RetVT == MVT::f16)
2198
2199 if (OpVT == MVT::bf16)
2201
2202 if (RetVT == MVT::bf16)
2204
2205 report_fatal_error("Attempt at an invalid promotion-related conversion");
2206}
2207
2208bool DAGTypeLegalizer::PromoteFloatOperand(SDNode *N, unsigned OpNo) {
2209 LLVM_DEBUG(dbgs() << "Promote float operand " << OpNo << ": "; N->dump(&DAG));
2210 SDValue R = SDValue();
2211
2212 if (CustomLowerNode(N, N->getOperand(OpNo).getValueType(), false)) {
2213 LLVM_DEBUG(dbgs() << "Node has been custom lowered, done\n");
2214 return false;
2215 }
2216
2217 // Nodes that use a promotion-requiring floating point operand, but doesn't
2218 // produce a promotion-requiring floating point result, need to be legalized
2219 // to use the promoted float operand. Nodes that produce at least one
2220 // promotion-requiring floating point result have their operands legalized as
2221 // a part of PromoteFloatResult.
2222 // clang-format off
2223 switch (N->getOpcode()) {
2224 default:
2225 #ifndef NDEBUG
2226 dbgs() << "PromoteFloatOperand Op #" << OpNo << ": ";
2227 N->dump(&DAG); dbgs() << "\n";
2228 #endif
2229 report_fatal_error("Do not know how to promote this operator's operand!");
2230
2231 case ISD::BITCAST: R = PromoteFloatOp_BITCAST(N, OpNo); break;
2232 case ISD::FCOPYSIGN: R = PromoteFloatOp_FCOPYSIGN(N, OpNo); break;
2233 case ISD::FP_TO_SINT:
2234 case ISD::FP_TO_UINT:
2235 case ISD::LRINT:
2236 case ISD::LLRINT: R = PromoteFloatOp_UnaryOp(N, OpNo); break;
2239 R = PromoteFloatOp_FP_TO_XINT_SAT(N, OpNo); break;
2240 case ISD::FP_EXTEND: R = PromoteFloatOp_FP_EXTEND(N, OpNo); break;
2242 R = PromoteFloatOp_STRICT_FP_EXTEND(N, OpNo);
2243 break;
2244 case ISD::SELECT_CC: R = PromoteFloatOp_SELECT_CC(N, OpNo); break;
2245 case ISD::SETCC: R = PromoteFloatOp_SETCC(N, OpNo); break;
2246 case ISD::STORE: R = PromoteFloatOp_STORE(N, OpNo); break;
2247 }
2248 // clang-format on
2249
2250 if (R.getNode())
2251 ReplaceValueWith(SDValue(N, 0), R);
2252 return false;
2253}
2254
2255SDValue DAGTypeLegalizer::PromoteFloatOp_BITCAST(SDNode *N, unsigned OpNo) {
2256 SDValue Op = N->getOperand(0);
2257 EVT OpVT = Op->getValueType(0);
2258
2259 SDValue Promoted = GetPromotedFloat(N->getOperand(0));
2260 EVT PromotedVT = Promoted->getValueType(0);
2261
2262 // Convert the promoted float value to the desired IVT.
2263 EVT IVT = EVT::getIntegerVT(*DAG.getContext(), OpVT.getSizeInBits());
2264 SDValue Convert = DAG.getNode(GetPromotionOpcode(PromotedVT, OpVT), SDLoc(N),
2265 IVT, Promoted);
2266 // The final result type might not be an scalar so we need a bitcast. The
2267 // bitcast will be further legalized if needed.
2268 return DAG.getBitcast(N->getValueType(0), Convert);
2269}
2270
2271// Promote Operand 1 of FCOPYSIGN. Operand 0 ought to be handled by
2272// PromoteFloatRes_FCOPYSIGN.
2273SDValue DAGTypeLegalizer::PromoteFloatOp_FCOPYSIGN(SDNode *N, unsigned OpNo) {
2274 assert (OpNo == 1 && "Only Operand 1 must need promotion here");
2275 SDValue Op1 = GetPromotedFloat(N->getOperand(1));
2276
2277 return DAG.getNode(N->getOpcode(), SDLoc(N), N->getValueType(0),
2278 N->getOperand(0), Op1);
2279}
2280
2281// Convert the promoted float value to the desired integer type
2282SDValue DAGTypeLegalizer::PromoteFloatOp_UnaryOp(SDNode *N, unsigned OpNo) {
2283 SDValue Op = GetPromotedFloat(N->getOperand(0));
2284 return DAG.getNode(N->getOpcode(), SDLoc(N), N->getValueType(0), Op);
2285}
2286
2287SDValue DAGTypeLegalizer::PromoteFloatOp_FP_TO_XINT_SAT(SDNode *N,
2288 unsigned OpNo) {
2289 SDValue Op = GetPromotedFloat(N->getOperand(0));
2290 return DAG.getNode(N->getOpcode(), SDLoc(N), N->getValueType(0), Op,
2291 N->getOperand(1));
2292}
2293
2294SDValue DAGTypeLegalizer::PromoteFloatOp_FP_EXTEND(SDNode *N, unsigned OpNo) {
2295 SDValue Op = GetPromotedFloat(N->getOperand(0));
2296 EVT VT = N->getValueType(0);
2297
2298 // Desired VT is same as promoted type. Use promoted float directly.
2299 if (VT == Op->getValueType(0))
2300 return Op;
2301
2302 // Else, extend the promoted float value to the desired VT.
2303 return DAG.getNode(ISD::FP_EXTEND, SDLoc(N), VT, Op);
2304}
2305
2306SDValue DAGTypeLegalizer::PromoteFloatOp_STRICT_FP_EXTEND(SDNode *N,
2307 unsigned OpNo) {
2308 assert(OpNo == 1 && "Promoting unpromotable operand");
2309
2310 SDValue Op = GetPromotedFloat(N->getOperand(1));
2311 EVT VT = N->getValueType(0);
2312
2313 // Desired VT is same as promoted type. Use promoted float directly.
2314 if (VT == Op->getValueType(0)) {
2315 ReplaceValueWith(SDValue(N, 1), N->getOperand(0));
2316 return Op;
2317 }
2318
2319 // Else, extend the promoted float value to the desired VT.
2320 SDValue Res = DAG.getNode(ISD::STRICT_FP_EXTEND, SDLoc(N), N->getVTList(),
2321 N->getOperand(0), Op);
2322 ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
2323 return Res;
2324}
2325
2326// Promote the float operands used for comparison. The true- and false-
2327// operands have the same type as the result and are promoted, if needed, by
2328// PromoteFloatRes_SELECT_CC
2329SDValue DAGTypeLegalizer::PromoteFloatOp_SELECT_CC(SDNode *N, unsigned OpNo) {
2330 SDValue LHS = GetPromotedFloat(N->getOperand(0));
2331 SDValue RHS = GetPromotedFloat(N->getOperand(1));
2332
2333 return DAG.getNode(ISD::SELECT_CC, SDLoc(N), N->getValueType(0),
2334 LHS, RHS, N->getOperand(2), N->getOperand(3),
2335 N->getOperand(4));
2336}
2337
2338// Construct a SETCC that compares the promoted values and sets the conditional
2339// code.
2340SDValue DAGTypeLegalizer::PromoteFloatOp_SETCC(SDNode *N, unsigned OpNo) {
2341 EVT VT = N->getValueType(0);
2342 SDValue Op0 = GetPromotedFloat(N->getOperand(0));
2343 SDValue Op1 = GetPromotedFloat(N->getOperand(1));
2344 ISD::CondCode CCCode = cast<CondCodeSDNode>(N->getOperand(2))->get();
2345
2346 return DAG.getSetCC(SDLoc(N), VT, Op0, Op1, CCCode);
2347
2348}
2349
2350// Lower the promoted Float down to the integer value of same size and construct
2351// a STORE of the integer value.
2352SDValue DAGTypeLegalizer::PromoteFloatOp_STORE(SDNode *N, unsigned OpNo) {
2353 StoreSDNode *ST = cast<StoreSDNode>(N);
2354 SDValue Val = ST->getValue();
2355 SDLoc DL(N);
2356
2357 SDValue Promoted = GetPromotedFloat(Val);
2358 EVT VT = ST->getOperand(1).getValueType();
2359 EVT IVT = EVT::getIntegerVT(*DAG.getContext(), VT.getSizeInBits());
2360
2361 SDValue NewVal;
2362 NewVal = DAG.getNode(GetPromotionOpcode(Promoted.getValueType(), VT), DL,
2363 IVT, Promoted);
2364
2365 return DAG.getStore(ST->getChain(), DL, NewVal, ST->getBasePtr(),
2366 ST->getMemOperand());
2367}
2368
2369//===----------------------------------------------------------------------===//
2370// Float Result Promotion
2371//===----------------------------------------------------------------------===//
2372
2373void DAGTypeLegalizer::PromoteFloatResult(SDNode *N, unsigned ResNo) {
2374 LLVM_DEBUG(dbgs() << "Promote float result " << ResNo << ": "; N->dump(&DAG));
2375 SDValue R = SDValue();
2376
2377 // See if the target wants to custom expand this node.
2378 if (CustomLowerNode(N, N->getValueType(ResNo), true)) {
2379 LLVM_DEBUG(dbgs() << "Node has been custom expanded, done\n");
2380 return;
2381 }
2382
2383 switch (N->getOpcode()) {
2384 // These opcodes cannot appear if promotion of FP16 is done in the backend
2385 // instead of Clang
2386 case ISD::FP16_TO_FP:
2387 case ISD::FP_TO_FP16:
2388 default:
2389#ifndef NDEBUG
2390 dbgs() << "PromoteFloatResult #" << ResNo << ": ";
2391 N->dump(&DAG); dbgs() << "\n";
2392#endif
2393 report_fatal_error("Do not know how to promote this operator's result!");
2394
2395 case ISD::BITCAST: R = PromoteFloatRes_BITCAST(N); break;
2396 case ISD::ConstantFP: R = PromoteFloatRes_ConstantFP(N); break;
2398 R = PromoteFloatRes_EXTRACT_VECTOR_ELT(N); break;
2399 case ISD::FCOPYSIGN: R = PromoteFloatRes_FCOPYSIGN(N); break;
2400
2401 // Unary FP Operations
2402 case ISD::FABS:
2403 case ISD::FCBRT:
2404 case ISD::FCEIL:
2405 case ISD::FCOS:
2406 case ISD::FEXP:
2407 case ISD::FEXP2:
2408 case ISD::FEXP10:
2409 case ISD::FFLOOR:
2410 case ISD::FLOG:
2411 case ISD::FLOG2:
2412 case ISD::FLOG10:
2413 case ISD::FNEARBYINT:
2414 case ISD::FNEG:
2415 case ISD::FRINT:
2416 case ISD::FROUND:
2417 case ISD::FROUNDEVEN:
2418 case ISD::FSIN:
2419 case ISD::FSQRT:
2420 case ISD::FTRUNC:
2421 case ISD::FCANONICALIZE: R = PromoteFloatRes_UnaryOp(N); break;
2422
2423 // Binary FP Operations
2424 case ISD::FADD:
2425 case ISD::FDIV:
2426 case ISD::FMAXIMUM:
2427 case ISD::FMINIMUM:
2428 case ISD::FMAXNUM:
2429 case ISD::FMINNUM:
2430 case ISD::FMUL:
2431 case ISD::FPOW:
2432 case ISD::FREM:
2433 case ISD::FSUB: R = PromoteFloatRes_BinOp(N); break;
2434
2435 case ISD::FMA: // FMA is same as FMAD
2436 case ISD::FMAD: R = PromoteFloatRes_FMAD(N); break;
2437
2438 case ISD::FPOWI:
2439 case ISD::FLDEXP: R = PromoteFloatRes_ExpOp(N); break;
2440 case ISD::FFREXP: R = PromoteFloatRes_FFREXP(N); break;
2441
2442 case ISD::FP_ROUND: R = PromoteFloatRes_FP_ROUND(N); break;
2444 R = PromoteFloatRes_STRICT_FP_ROUND(N);
2445 break;
2446 case ISD::LOAD: R = PromoteFloatRes_LOAD(N); break;
2447 case ISD::SELECT: R = PromoteFloatRes_SELECT(N); break;
2448 case ISD::SELECT_CC: R = PromoteFloatRes_SELECT_CC(N); break;
2449
2450 case ISD::SINT_TO_FP:
2451 case ISD::UINT_TO_FP: R = PromoteFloatRes_XINT_TO_FP(N); break;
2452 case ISD::UNDEF: R = PromoteFloatRes_UNDEF(N); break;
2453 case ISD::ATOMIC_SWAP: R = BitcastToInt_ATOMIC_SWAP(N); break;
2460 R = PromoteFloatRes_VECREDUCE(N);
2461 break;
2464 R = PromoteFloatRes_VECREDUCE_SEQ(N);
2465 break;
2466 }
2467
2468 if (R.getNode())
2469 SetPromotedFloat(SDValue(N, ResNo), R);
2470}
2471
2472// Bitcast from i16 to f16: convert the i16 to a f32 value instead.
2473// At this point, it is not possible to determine if the bitcast value is
2474// eventually stored to memory or promoted to f32 or promoted to a floating
2475// point at a higher precision. Some of these cases are handled by FP_EXTEND,
2476// STORE promotion handlers.
2477SDValue DAGTypeLegalizer::PromoteFloatRes_BITCAST(SDNode *N) {
2478 EVT VT = N->getValueType(0);
2479 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
2480 // Input type isn't guaranteed to be a scalar int so bitcast if not. The
2481 // bitcast will be legalized further if necessary.
2482 EVT IVT = EVT::getIntegerVT(*DAG.getContext(),
2483 N->getOperand(0).getValueType().getSizeInBits());
2484 SDValue Cast = DAG.getBitcast(IVT, N->getOperand(0));
2485 return DAG.getNode(GetPromotionOpcode(VT, NVT), SDLoc(N), NVT, Cast);
2486}
2487
2488SDValue DAGTypeLegalizer::PromoteFloatRes_ConstantFP(SDNode *N) {
2489 ConstantFPSDNode *CFPNode = cast<ConstantFPSDNode>(N);
2490 EVT VT = N->getValueType(0);
2491 SDLoc DL(N);
2492
2493 // Get the (bit-cast) APInt of the APFloat and build an integer constant
2494 EVT IVT = EVT::getIntegerVT(*DAG.getContext(), VT.getSizeInBits());
2495 SDValue C = DAG.getConstant(CFPNode->getValueAPF().bitcastToAPInt(), DL,
2496 IVT);
2497
2498 // Convert the Constant to the desired FP type
2499 // FIXME We might be able to do the conversion during compilation and get rid
2500 // of it from the object code
2501 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
2502 return DAG.getNode(GetPromotionOpcode(VT, NVT), DL, NVT, C);
2503}
2504
2505// If the Index operand is a constant, try to redirect the extract operation to
2506// the correct legalized vector. If not, bit-convert the input vector to
2507// equivalent integer vector. Extract the element as an (bit-cast) integer
2508// value and convert it to the promoted type.
2509SDValue DAGTypeLegalizer::PromoteFloatRes_EXTRACT_VECTOR_ELT(SDNode *N) {
2510 SDLoc DL(N);
2511
2512 // If the index is constant, try to extract the value from the legalized
2513 // vector type.
2514 if (isa<ConstantSDNode>(N->getOperand(1))) {
2515 SDValue Vec = N->getOperand(0);
2516 SDValue Idx = N->getOperand(1);
2517 EVT VecVT = Vec->getValueType(0);
2518 EVT EltVT = VecVT.getVectorElementType();
2519
2520 uint64_t IdxVal = Idx->getAsZExtVal();
2521
2522 switch (getTypeAction(VecVT)) {
2523 default: break;
2525 SDValue Res = GetScalarizedVector(N->getOperand(0));
2526 ReplaceValueWith(SDValue(N, 0), Res);
2527 return SDValue();
2528 }
2530 Vec = GetWidenedVector(Vec);
2531 SDValue Res = DAG.getNode(N->getOpcode(), DL, EltVT, Vec, Idx);
2532 ReplaceValueWith(SDValue(N, 0), Res);
2533 return SDValue();
2534 }
2536 SDValue Lo, Hi;
2537 GetSplitVector(Vec, Lo, Hi);
2538
2539 uint64_t LoElts = Lo.getValueType().getVectorNumElements();
2540 SDValue Res;
2541 if (IdxVal < LoElts)
2542 Res = DAG.getNode(N->getOpcode(), DL, EltVT, Lo, Idx);
2543 else
2544 Res = DAG.getNode(N->getOpcode(), DL, EltVT, Hi,
2545 DAG.getConstant(IdxVal - LoElts, DL,
2546 Idx.getValueType()));
2547 ReplaceValueWith(SDValue(N, 0), Res);
2548 return SDValue();
2549 }
2550
2551 }
2552 }
2553
2554 // Bit-convert the input vector to the equivalent integer vector
2555 SDValue NewOp = BitConvertVectorToIntegerVector(N->getOperand(0));
2556 EVT IVT = NewOp.getValueType().getVectorElementType();
2557
2558 // Extract the element as an (bit-cast) integer value
2559 SDValue NewVal = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, IVT,
2560 NewOp, N->getOperand(1));
2561
2562 // Convert the element to the desired FP type
2563 EVT VT = N->getValueType(0);
2564 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
2565 return DAG.getNode(GetPromotionOpcode(VT, NVT), SDLoc(N), NVT, NewVal);
2566}
2567
2568// FCOPYSIGN(X, Y) returns the value of X with the sign of Y. If the result
2569// needs promotion, so does the argument X. Note that Y, if needed, will be
2570// handled during operand promotion.
2571SDValue DAGTypeLegalizer::PromoteFloatRes_FCOPYSIGN(SDNode *N) {
2572 EVT VT = N->getValueType(0);
2573 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
2574 SDValue Op0 = GetPromotedFloat(N->getOperand(0));
2575
2576 SDValue Op1 = N->getOperand(1);
2577
2578 return DAG.getNode(N->getOpcode(), SDLoc(N), NVT, Op0, Op1);
2579}
2580
2581// Unary operation where the result and the operand have PromoteFloat type
2582// action. Construct a new SDNode with the promoted float value of the old
2583// operand.
2584SDValue DAGTypeLegalizer::PromoteFloatRes_UnaryOp(SDNode *N) {
2585 EVT VT = N->getValueType(0);
2586 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
2587 SDValue Op = GetPromotedFloat(N->getOperand(0));
2588
2589 return DAG.getNode(N->getOpcode(), SDLoc(N), NVT, Op);
2590}
2591
2592// Binary operations where the result and both operands have PromoteFloat type
2593// action. Construct a new SDNode with the promoted float values of the old
2594// operands.
2595SDValue DAGTypeLegalizer::PromoteFloatRes_BinOp(SDNode *N) {
2596 EVT VT = N->getValueType(0);
2597 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
2598 SDValue Op0 = GetPromotedFloat(N->getOperand(0));
2599 SDValue Op1 = GetPromotedFloat(N->getOperand(1));
2600 return DAG.getNode(N->getOpcode(), SDLoc(N), NVT, Op0, Op1, N->getFlags());
2601}
2602
2603SDValue DAGTypeLegalizer::PromoteFloatRes_FMAD(SDNode *N) {
2604 EVT VT = N->getValueType(0);
2605 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
2606 SDValue Op0 = GetPromotedFloat(N->getOperand(0));
2607 SDValue Op1 = GetPromotedFloat(N->getOperand(1));
2608 SDValue Op2 = GetPromotedFloat(N->getOperand(2));
2609
2610 return DAG.getNode(N->getOpcode(), SDLoc(N), NVT, Op0, Op1, Op2);
2611}
2612
2613// Promote the Float (first) operand and retain the Integer (second) operand
2614SDValue DAGTypeLegalizer::PromoteFloatRes_ExpOp(SDNode *N) {
2615 EVT VT = N->getValueType(0);
2616 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
2617 SDValue Op0 = GetPromotedFloat(N->getOperand(0));
2618 SDValue Op1 = N->getOperand(1);
2619
2620 return DAG.getNode(N->getOpcode(), SDLoc(N), NVT, Op0, Op1);
2621}
2622
2623SDValue DAGTypeLegalizer::PromoteFloatRes_FFREXP(SDNode *N) {
2624 EVT VT = N->getValueType(0);
2625 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
2626 SDValue Op = GetPromotedFloat(N->getOperand(0));
2627 SDValue Res =
2628 DAG.getNode(N->getOpcode(), SDLoc(N), {NVT, N->getValueType(1)}, Op);
2629
2630 ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
2631 return Res;
2632}
2633
2634// Explicit operation to reduce precision. Reduce the value to half precision
2635// and promote it back to the legal type.
2636SDValue DAGTypeLegalizer::PromoteFloatRes_FP_ROUND(SDNode *N) {
2637 SDLoc DL(N);
2638
2639 SDValue Op = N->getOperand(0);
2640 EVT VT = N->getValueType(0);
2641 EVT OpVT = Op->getValueType(0);
2642 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
2643 EVT IVT = EVT::getIntegerVT(*DAG.getContext(), VT.getSizeInBits());
2644
2645 // Round promoted float to desired precision
2646 SDValue Round = DAG.getNode(GetPromotionOpcode(OpVT, VT), DL, IVT, Op);
2647 // Promote it back to the legal output type
2648 return DAG.getNode(GetPromotionOpcode(VT, NVT), DL, NVT, Round);
2649}
2650
2651// Explicit operation to reduce precision. Reduce the value to half precision
2652// and promote it back to the legal type.
2653SDValue DAGTypeLegalizer::PromoteFloatRes_STRICT_FP_ROUND(SDNode *N) {
2654 SDLoc DL(N);
2655
2656 SDValue Chain = N->getOperand(0);
2657 SDValue Op = N->getOperand(1);
2658 EVT VT = N->getValueType(0);
2659 EVT OpVT = Op->getValueType(0);
2660 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
2661 EVT IVT = EVT::getIntegerVT(*DAG.getContext(), VT.getSizeInBits());
2662
2663 // Round promoted float to desired precision
2664 SDValue Round = DAG.getNode(GetPromotionOpcodeStrict(OpVT, VT), DL,
2665 DAG.getVTList(IVT, MVT::Other), Chain, Op);
2666 // Promote it back to the legal output type
2667 SDValue Res =
2668 DAG.getNode(GetPromotionOpcodeStrict(VT, NVT), DL,
2669 DAG.getVTList(NVT, MVT::Other), Round.getValue(1), Round);
2670 ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
2671 return Res;
2672}
2673
2674SDValue DAGTypeLegalizer::PromoteFloatRes_LOAD(SDNode *N) {
2675 LoadSDNode *L = cast<LoadSDNode>(N);
2676 EVT VT = N->getValueType(0);
2677
2678 // Load the value as an integer value with the same number of bits.
2679 EVT IVT = EVT::getIntegerVT(*DAG.getContext(), VT.getSizeInBits());
2680 SDValue newL = DAG.getLoad(
2681 L->getAddressingMode(), L->getExtensionType(), IVT, SDLoc(N),
2682 L->getChain(), L->getBasePtr(), L->getOffset(), L->getPointerInfo(), IVT,
2683 L->getOriginalAlign(), L->getMemOperand()->getFlags(), L->getAAInfo());
2684 // Legalize the chain result by replacing uses of the old value chain with the
2685 // new one
2686 ReplaceValueWith(SDValue(N, 1), newL.getValue(1));
2687
2688 // Convert the integer value to the desired FP type
2689 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
2690 return DAG.getNode(GetPromotionOpcode(VT, NVT), SDLoc(N), NVT, newL);
2691}
2692
2693// Construct a new SELECT node with the promoted true- and false- values.
2694SDValue DAGTypeLegalizer::PromoteFloatRes_SELECT(SDNode *N) {
2695 SDValue TrueVal = GetPromotedFloat(N->getOperand(1));
2696 SDValue FalseVal = GetPromotedFloat(N->getOperand(2));
2697
2698 return DAG.getNode(ISD::SELECT, SDLoc(N), TrueVal->getValueType(0),
2699 N->getOperand(0), TrueVal, FalseVal);
2700}
2701
2702// Construct a new SELECT_CC node with the promoted true- and false- values.
2703// The operands used for comparison are promoted by PromoteFloatOp_SELECT_CC.
2704SDValue DAGTypeLegalizer::PromoteFloatRes_SELECT_CC(SDNode *N) {
2705 SDValue TrueVal = GetPromotedFloat(N->getOperand(2));
2706 SDValue FalseVal = GetPromotedFloat(N->getOperand(3));
2707
2708 return DAG.getNode(ISD::SELECT_CC, SDLoc(N),
2709 TrueVal.getNode()->getValueType(0), N->getOperand(0),
2710 N->getOperand(1), TrueVal, FalseVal, N->getOperand(4));
2711}
2712
2713// Construct a SDNode that transforms the SINT or UINT operand to the promoted
2714// float type.
2715SDValue DAGTypeLegalizer::PromoteFloatRes_XINT_TO_FP(SDNode *N) {
2716 SDLoc DL(N);
2717 EVT VT = N->getValueType(0);
2718 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
2719 SDValue NV = DAG.getNode(N->getOpcode(), DL, NVT, N->getOperand(0));
2720 // Round the value to the desired precision (that of the source type).
2721 return DAG.getNode(
2722 ISD::FP_EXTEND, DL, NVT,
2723 DAG.getNode(ISD::FP_ROUND, DL, VT, NV,
2724 DAG.getIntPtrConstant(0, DL, /*isTarget=*/true)));
2725}
2726
2727SDValue DAGTypeLegalizer::PromoteFloatRes_UNDEF(SDNode *N) {
2728 return DAG.getUNDEF(TLI.getTypeToTransformTo(*DAG.getContext(),
2729 N->getValueType(0)));
2730}
2731
2732SDValue DAGTypeLegalizer::PromoteFloatRes_VECREDUCE(SDNode *N) {
2733 // Expand and promote recursively.
2734 // TODO: This is non-optimal, but dealing with the concurrently happening
2735 // vector-legalization is non-trivial. We could do something similar to
2736 // PromoteFloatRes_EXTRACT_VECTOR_ELT here.
2737 ReplaceValueWith(SDValue(N, 0), TLI.expandVecReduce(N, DAG));
2738 return SDValue();
2739}
2740
2741SDValue DAGTypeLegalizer::PromoteFloatRes_VECREDUCE_SEQ(SDNode *N) {
2742 ReplaceValueWith(SDValue(N, 0), TLI.expandVecReduceSeq(N, DAG));
2743 return SDValue();
2744}
2745
2746SDValue DAGTypeLegalizer::BitcastToInt_ATOMIC_SWAP(SDNode *N) {
2747 EVT VT = N->getValueType(0);
2748
2749 AtomicSDNode *AM = cast<AtomicSDNode>(N);
2750 SDLoc SL(N);
2751
2752 SDValue CastVal = BitConvertToInteger(AM->getVal());
2753 EVT CastVT = CastVal.getValueType();
2754
2755 SDValue NewAtomic
2756 = DAG.getAtomic(ISD::ATOMIC_SWAP, SL, CastVT,
2757 DAG.getVTList(CastVT, MVT::Other),
2758 { AM->getChain(), AM->getBasePtr(), CastVal },
2759 AM->getMemOperand());
2760
2761 SDValue Result = NewAtomic;
2762
2763 if (getTypeAction(VT) == TargetLowering::TypePromoteFloat) {
2764 EVT NFPVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
2765 Result = DAG.getNode(GetPromotionOpcode(VT, NFPVT), SL, NFPVT,
2766 NewAtomic);
2767 }
2768
2769 // Legalize the chain result by replacing uses of the old value chain with the
2770 // new one
2771 ReplaceValueWith(SDValue(N, 1), NewAtomic.getValue(1));
2772
2773 return Result;
2774
2775}
2776
2777//===----------------------------------------------------------------------===//
2778// Half Result Soft Promotion
2779//===----------------------------------------------------------------------===//
2780
2781void DAGTypeLegalizer::SoftPromoteHalfResult(SDNode *N, unsigned ResNo) {
2782 LLVM_DEBUG(dbgs() << "Soft promote half result " << ResNo << ": ";
2783 N->dump(&DAG));
2784 SDValue R = SDValue();
2785
2786 // See if the target wants to custom expand this node.
2787 if (CustomLowerNode(N, N->getValueType(ResNo), true)) {
2788 LLVM_DEBUG(dbgs() << "Node has been custom expanded, done\n");
2789 return;
2790 }
2791
2792 switch (N->getOpcode()) {
2793 default:
2794#ifndef NDEBUG
2795 dbgs() << "SoftPromoteHalfResult #" << ResNo << ": ";
2796 N->dump(&DAG); dbgs() << "\n";
2797#endif
2798 report_fatal_error("Do not know how to soft promote this operator's "
2799 "result!");
2800
2801 case ISD::BITCAST: R = SoftPromoteHalfRes_BITCAST(N); break;
2802 case ISD::ConstantFP: R = SoftPromoteHalfRes_ConstantFP(N); break;
2804 R = SoftPromoteHalfRes_EXTRACT_VECTOR_ELT(N); break;
2805 case ISD::FCOPYSIGN: R = SoftPromoteHalfRes_FCOPYSIGN(N); break;
2807 case ISD::FP_ROUND: R = SoftPromoteHalfRes_FP_ROUND(N); break;
2808
2809 // Unary FP Operations
2810 case ISD::FABS:
2811 case ISD::FCBRT:
2812 case ISD::FCEIL:
2813 case ISD::FCOS:
2814 case ISD::FEXP:
2815 case ISD::FEXP2:
2816 case ISD::FEXP10:
2817 case ISD::FFLOOR:
2818 case ISD::FLOG:
2819 case ISD::FLOG2:
2820 case ISD::FLOG10:
2821 case ISD::FNEARBYINT:
2822 case ISD::FNEG:
2823 case ISD::FREEZE:
2824 case ISD::FRINT:
2825 case ISD::FROUND:
2826 case ISD::FROUNDEVEN:
2827 case ISD::FSIN:
2828 case ISD::FSQRT:
2829 case ISD::FTRUNC:
2830 case ISD::FCANONICALIZE: R = SoftPromoteHalfRes_UnaryOp(N); break;
2831
2832 // Binary FP Operations
2833 case ISD::FADD:
2834 case ISD::FDIV:
2835 case ISD::FMAXIMUM:
2836 case ISD::FMINIMUM:
2837 case ISD::FMAXNUM:
2838 case ISD::FMINNUM:
2839 case ISD::FMUL:
2840 case ISD::FPOW:
2841 case ISD::FREM:
2842 case ISD::FSUB: R = SoftPromoteHalfRes_BinOp(N); break;
2843
2844 case ISD::FMA: // FMA is same as FMAD
2845 case ISD::FMAD: R = SoftPromoteHalfRes_FMAD(N); break;
2846
2847 case ISD::FPOWI:
2848 case ISD::FLDEXP: R = SoftPromoteHalfRes_ExpOp(N); break;
2849
2850 case ISD::FFREXP: R = SoftPromoteHalfRes_FFREXP(N); break;
2851
2852 case ISD::LOAD: R = SoftPromoteHalfRes_LOAD(N); break;
2853 case ISD::SELECT: R = SoftPromoteHalfRes_SELECT(N); break;
2854 case ISD::SELECT_CC: R = SoftPromoteHalfRes_SELECT_CC(N); break;
2855 case ISD::SINT_TO_FP:
2856 case ISD::UINT_TO_FP: R = SoftPromoteHalfRes_XINT_TO_FP(N); break;
2857 case ISD::UNDEF: R = SoftPromoteHalfRes_UNDEF(N); break;
2858 case ISD::ATOMIC_SWAP: R = BitcastToInt_ATOMIC_SWAP(N); break;
2865 R = SoftPromoteHalfRes_VECREDUCE(N);
2866 break;
2869 R = SoftPromoteHalfRes_VECREDUCE_SEQ(N);
2870 break;
2871 }
2872
2873 if (R.getNode())
2874 SetSoftPromotedHalf(SDValue(N, ResNo), R);
2875}
2876
2877SDValue DAGTypeLegalizer::SoftPromoteHalfRes_BITCAST(SDNode *N) {
2878 return BitConvertToInteger(N->getOperand(0));
2879}
2880
2881SDValue DAGTypeLegalizer::SoftPromoteHalfRes_ConstantFP(SDNode *N) {
2882 ConstantFPSDNode *CN = cast<ConstantFPSDNode>(N);
2883
2884 // Get the (bit-cast) APInt of the APFloat and build an integer constant
2885 return DAG.getConstant(CN->getValueAPF().bitcastToAPInt(), SDLoc(CN),
2886 MVT::i16);
2887}
2888
2889SDValue DAGTypeLegalizer::SoftPromoteHalfRes_EXTRACT_VECTOR_ELT(SDNode *N) {
2890 SDValue NewOp = BitConvertVectorToIntegerVector(N->getOperand(0));
2892 NewOp.getValueType().getVectorElementType(), NewOp,
2893 N->getOperand(1));
2894}
2895
2896SDValue DAGTypeLegalizer::SoftPromoteHalfRes_FCOPYSIGN(SDNode *N) {
2897 SDValue LHS = GetSoftPromotedHalf(N->getOperand(0));
2898 SDValue RHS = BitConvertToInteger(N->getOperand(1));
2899 SDLoc dl(N);
2900
2901 EVT LVT = LHS.getValueType();
2902 EVT RVT = RHS.getValueType();
2903
2904 unsigned LSize = LVT.getSizeInBits();
2905 unsigned RSize = RVT.getSizeInBits();
2906
2907 // First get the sign bit of second operand.
2908 SDValue SignBit = DAG.getNode(
2909 ISD::SHL, dl, RVT, DAG.getConstant(1, dl, RVT),
2910 DAG.getConstant(RSize - 1, dl,
2911 TLI.getShiftAmountTy(RVT, DAG.getDataLayout())));
2912 SignBit = DAG.getNode(ISD::AND, dl, RVT, RHS, SignBit);
2913
2914 // Shift right or sign-extend it if the two operands have different types.
2915 int SizeDiff = RVT.getSizeInBits() - LVT.getSizeInBits();
2916 if (SizeDiff > 0) {
2917 SignBit =
2918 DAG.getNode(ISD::SRL, dl, RVT, SignBit,
2919 DAG.getConstant(SizeDiff, dl,
2920 TLI.getShiftAmountTy(SignBit.getValueType(),
2921 DAG.getDataLayout())));
2922 SignBit = DAG.getNode(ISD::TRUNCATE, dl, LVT, SignBit);
2923 } else if (SizeDiff < 0) {
2924 SignBit = DAG.getNode(ISD::ANY_EXTEND, dl, LVT, SignBit);
2925 SignBit =
2926 DAG.getNode(ISD::SHL, dl, LVT, SignBit,
2927 DAG.getConstant(-SizeDiff, dl,
2928 TLI.getShiftAmountTy(SignBit.getValueType(),
2929 DAG.getDataLayout())));
2930 }
2931
2932 // Clear the sign bit of the first operand.
2933 SDValue Mask = DAG.getNode(
2934 ISD::SHL, dl, LVT, DAG.getConstant(1, dl, LVT),
2935 DAG.getConstant(LSize - 1, dl,
2936 TLI.getShiftAmountTy(LVT, DAG.getDataLayout())));
2937 Mask = DAG.getNode(ISD::SUB, dl, LVT, Mask, DAG.getConstant(1, dl, LVT));
2938 LHS = DAG.getNode(ISD::AND, dl, LVT, LHS, Mask);
2939
2940 // Or the value with the sign bit.
2941 return DAG.getNode(ISD::OR, dl, LVT, LHS, SignBit);
2942}
2943
2944SDValue DAGTypeLegalizer::SoftPromoteHalfRes_FMAD(SDNode *N) {
2945 EVT OVT = N->getValueType(0);
2946 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), OVT);
2947 SDValue Op0 = GetSoftPromotedHalf(N->getOperand(0));
2948 SDValue Op1 = GetSoftPromotedHalf(N->getOperand(1));
2949 SDValue Op2 = GetSoftPromotedHalf(N->getOperand(2));
2950 SDLoc dl(N);
2951
2952 // Promote to the larger FP type.
2953 auto PromotionOpcode = GetPromotionOpcode(OVT, NVT);
2954 Op0 = DAG.getNode(PromotionOpcode, dl, NVT, Op0);
2955 Op1 = DAG.getNode(PromotionOpcode, dl, NVT, Op1);
2956 Op2 = DAG.getNode(PromotionOpcode, dl, NVT, Op2);
2957
2958 SDValue Res = DAG.getNode(N->getOpcode(), dl, NVT, Op0, Op1, Op2);
2959
2960 // Convert back to FP16 as an integer.
2961 return DAG.getNode(GetPromotionOpcode(NVT, OVT), dl, MVT::i16, Res);
2962}
2963
2964SDValue DAGTypeLegalizer::SoftPromoteHalfRes_ExpOp(SDNode *N) {
2965 EVT OVT = N->getValueType(0);
2966 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), OVT);
2967 SDValue Op0 = GetSoftPromotedHalf(N->getOperand(0));
2968 SDValue Op1 = N->getOperand(1);
2969 SDLoc dl(N);
2970
2971 // Promote to the larger FP type.
2972 Op0 = DAG.getNode(GetPromotionOpcode(OVT, NVT), dl, NVT, Op0);
2973
2974 SDValue Res = DAG.getNode(N->getOpcode(), dl, NVT, Op0, Op1);
2975
2976 // Convert back to FP16 as an integer.
2977 return DAG.getNode(GetPromotionOpcode(NVT, OVT), dl, MVT::i16, Res);
2978}
2979
2980SDValue DAGTypeLegalizer::SoftPromoteHalfRes_FFREXP(SDNode *N) {
2981 EVT OVT = N->getValueType(0);
2982 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), OVT);
2983 SDValue Op = GetSoftPromotedHalf(N->getOperand(0));
2984 SDLoc dl(N);
2985
2986 // Promote to the larger FP type.
2987 Op = DAG.getNode(GetPromotionOpcode(OVT, NVT), dl, NVT, Op);
2988
2989 SDValue Res = DAG.getNode(N->getOpcode(), dl,
2990 DAG.getVTList(NVT, N->getValueType(1)), Op);
2991
2992 ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
2993
2994 // Convert back to FP16 as an integer.
2995 return DAG.getNode(GetPromotionOpcode(NVT, OVT), dl, MVT::i16, Res);
2996}
2997
2998SDValue DAGTypeLegalizer::SoftPromoteHalfRes_FP_ROUND(SDNode *N) {
2999 EVT RVT = N->getValueType(0);
3000 EVT SVT = N->getOperand(0).getValueType();
3001
3002 if (N->isStrictFPOpcode()) {
3003 // FIXME: assume we only have two f16 variants for now.
3004 unsigned Opcode;
3005 if (RVT == MVT::f16)
3006 Opcode = ISD::STRICT_FP_TO_FP16;
3007 else if (RVT == MVT::bf16)
3008 Opcode = ISD::STRICT_FP_TO_BF16;
3009 else
3010 llvm_unreachable("unknown half type");
3011 SDValue Res = DAG.getNode(Opcode, SDLoc(N), {MVT::i16, MVT::Other},
3012 {N->getOperand(0), N->getOperand(1)});
3013 ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
3014 return Res;
3015 }
3016
3017 return DAG.getNode(GetPromotionOpcode(SVT, RVT), SDLoc(N), MVT::i16,
3018 N->getOperand(0));
3019}
3020
3021SDValue DAGTypeLegalizer::SoftPromoteHalfRes_LOAD(SDNode *N) {
3022 LoadSDNode *L = cast<LoadSDNode>(N);
3023
3024 // Load the value as an integer value with the same number of bits.
3025 assert(L->getExtensionType() == ISD::NON_EXTLOAD && "Unexpected extension!");
3026 SDValue NewL =
3027 DAG.getLoad(L->getAddressingMode(), L->getExtensionType(), MVT::i16,
3028 SDLoc(N), L->getChain(), L->getBasePtr(), L->getOffset(),
3029 L->getPointerInfo(), MVT::i16, L->getOriginalAlign(),
3030 L->getMemOperand()->getFlags(), L->getAAInfo());
3031 // Legalize the chain result by replacing uses of the old value chain with the
3032 // new one
3033 ReplaceValueWith(SDValue(N, 1), NewL.getValue(1));
3034 return NewL;
3035}
3036
3037SDValue DAGTypeLegalizer::SoftPromoteHalfRes_SELECT(SDNode *N) {
3038 SDValue Op1 = GetSoftPromotedHalf(N->getOperand(1));
3039 SDValue Op2 = GetSoftPromotedHalf(N->getOperand(2));
3040 return DAG.getSelect(SDLoc(N), Op1.getValueType(), N->getOperand(0), Op1,
3041 Op2);
3042}
3043
3044SDValue DAGTypeLegalizer::SoftPromoteHalfRes_SELECT_CC(SDNode *N) {
3045 SDValue Op2 = GetSoftPromotedHalf(N->getOperand(2));
3046 SDValue Op3 = GetSoftPromotedHalf(N->getOperand(3));
3047 return DAG.getNode(ISD::SELECT_CC, SDLoc(N), Op2.getValueType(),
3048 N->getOperand(0), N->getOperand(1), Op2, Op3,
3049 N->getOperand(4));
3050}
3051
3052SDValue DAGTypeLegalizer::SoftPromoteHalfRes_XINT_TO_FP(SDNode *N) {
3053 EVT OVT = N->getValueType(0);
3054 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), OVT);
3055 SDLoc dl(N);
3056
3057 SDValue Res = DAG.getNode(N->getOpcode(), dl, NVT, N->getOperand(0));
3058
3059 // Round the value to the softened type.
3060 return DAG.getNode(GetPromotionOpcode(NVT, OVT), dl, MVT::i16, Res);
3061}
3062
3063SDValue DAGTypeLegalizer::SoftPromoteHalfRes_UNDEF(SDNode *N) {
3064 return DAG.getUNDEF(MVT::i16);
3065}
3066
3067SDValue DAGTypeLegalizer::SoftPromoteHalfRes_UnaryOp(SDNode *N) {
3068 EVT OVT = N->getValueType(0);
3069 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), OVT);
3070 SDValue Op = GetSoftPromotedHalf(N->getOperand(0));
3071 SDLoc dl(N);
3072
3073 // Promote to the larger FP type.
3074 Op = DAG.getNode(GetPromotionOpcode(OVT, NVT), dl, NVT, Op);
3075
3076 SDValue Res = DAG.getNode(N->getOpcode(), dl, NVT, Op);
3077
3078 // Convert back to FP16 as an integer.
3079 return DAG.getNode(GetPromotionOpcode(NVT, OVT), dl, MVT::i16, Res);
3080}
3081
3082SDValue DAGTypeLegalizer::SoftPromoteHalfRes_BinOp(SDNode *N) {
3083 EVT OVT = N->getValueType(0);
3084 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), OVT);
3085 SDValue Op0 = GetSoftPromotedHalf(N->getOperand(0));
3086 SDValue Op1 = GetSoftPromotedHalf(N->getOperand(1));
3087 SDLoc dl(N);
3088
3089 // Promote to the larger FP type.
3090 auto PromotionOpcode = GetPromotionOpcode(OVT, NVT);
3091 Op0 = DAG.getNode(PromotionOpcode, dl, NVT, Op0);
3092 Op1 = DAG.getNode(PromotionOpcode, dl, NVT, Op1);
3093
3094 SDValue Res = DAG.getNode(N->getOpcode(), dl, NVT, Op0, Op1);
3095
3096 // Convert back to FP16 as an integer.
3097 return DAG.getNode(GetPromotionOpcode(NVT, OVT), dl, MVT::i16, Res);
3098}
3099
3100SDValue DAGTypeLegalizer::SoftPromoteHalfRes_VECREDUCE(SDNode *N) {
3101 // Expand and soften recursively.
3102 ReplaceValueWith(SDValue(N, 0), TLI.expandVecReduce(N, DAG));
3103 return SDValue();
3104}
3105
3106SDValue DAGTypeLegalizer::SoftPromoteHalfRes_VECREDUCE_SEQ(SDNode *N) {
3107 // Expand and soften.
3108 ReplaceValueWith(SDValue(N, 0), TLI.expandVecReduceSeq(N, DAG));
3109 return SDValue();
3110}
3111
3112//===----------------------------------------------------------------------===//
3113// Half Operand Soft Promotion
3114//===----------------------------------------------------------------------===//
3115
3116bool DAGTypeLegalizer::SoftPromoteHalfOperand(SDNode *N, unsigned OpNo) {
3117 LLVM_DEBUG(dbgs() << "Soft promote half operand " << OpNo << ": ";
3118 N->dump(&DAG));
3119 SDValue Res = SDValue();
3120
3121 if (CustomLowerNode(N, N->getOperand(OpNo).getValueType(), false)) {
3122 LLVM_DEBUG(dbgs() << "Node has been custom lowered, done\n");
3123 return false;
3124 }
3125
3126 // Nodes that use a promotion-requiring floating point operand, but doesn't
3127 // produce a soft promotion-requiring floating point result, need to be
3128 // legalized to use the soft promoted float operand. Nodes that produce at
3129 // least one soft promotion-requiring floating point result have their
3130 // operands legalized as a part of PromoteFloatResult.
3131 switch (N->getOpcode()) {
3132 default:
3133 #ifndef NDEBUG
3134 dbgs() << "SoftPromoteHalfOperand Op #" << OpNo << ": ";
3135 N->dump(&DAG); dbgs() << "\n";
3136 #endif
3137 report_fatal_error("Do not know how to soft promote this operator's "
3138 "operand!");
3139
3140 case ISD::BITCAST: Res = SoftPromoteHalfOp_BITCAST(N); break;
3141 case ISD::FCOPYSIGN: Res = SoftPromoteHalfOp_FCOPYSIGN(N, OpNo); break;
3142 case ISD::FP_TO_SINT:
3143 case ISD::FP_TO_UINT: Res = SoftPromoteHalfOp_FP_TO_XINT(N); break;
3146 Res = SoftPromoteHalfOp_FP_TO_XINT_SAT(N); break;
3148 case ISD::FP_EXTEND: Res = SoftPromoteHalfOp_FP_EXTEND(N); break;
3149 case ISD::SELECT_CC: Res = SoftPromoteHalfOp_SELECT_CC(N, OpNo); break;
3150 case ISD::SETCC: Res = SoftPromoteHalfOp_SETCC(N); break;
3151 case ISD::STORE: Res = SoftPromoteHalfOp_STORE(N, OpNo); break;
3152 case ISD::STACKMAP:
3153 Res = SoftPromoteHalfOp_STACKMAP(N, OpNo);
3154 break;
3155 case ISD::PATCHPOINT:
3156 Res = SoftPromoteHalfOp_PATCHPOINT(N, OpNo);
3157 break;
3158 }
3159
3160 if (!Res.getNode())
3161 return false;
3162
3163 assert(Res.getNode() != N && "Expected a new node!");
3164
3165 assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 &&
3166 "Invalid operand expansion");
3167
3168 ReplaceValueWith(SDValue(N, 0), Res);
3169 return false;
3170}
3171
3172SDValue DAGTypeLegalizer::SoftPromoteHalfOp_BITCAST(SDNode *N) {
3173 SDValue Op0 = GetSoftPromotedHalf(N->getOperand(0));
3174
3175 return DAG.getNode(ISD::BITCAST, SDLoc(N), N->getValueType(0), Op0);
3176}
3177
3178SDValue DAGTypeLegalizer::SoftPromoteHalfOp_FCOPYSIGN(SDNode *N,
3179 unsigned OpNo) {
3180 assert(OpNo == 1 && "Only Operand 1 must need promotion here");
3181 SDValue Op1 = N->getOperand(1);
3182 EVT RVT = Op1.getValueType();
3183 SDLoc dl(N);
3184
3185 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), Op1.getValueType());
3186
3187 Op1 = GetSoftPromotedHalf(Op1);
3188 Op1 = DAG.getNode(GetPromotionOpcode(RVT, NVT), dl, NVT, Op1);
3189
3190 return DAG.getNode(N->getOpcode(), dl, N->getValueType(0), N->getOperand(0),
3191 Op1);
3192}
3193
3194SDValue DAGTypeLegalizer::SoftPromoteHalfOp_FP_EXTEND(SDNode *N) {
3195 EVT RVT = N->getValueType(0);
3196 bool IsStrict = N->isStrictFPOpcode();
3197 SDValue Op = N->getOperand(IsStrict ? 1 : 0);
3198 EVT SVT = Op.getValueType();
3199 Op = GetSoftPromotedHalf(N->getOperand(IsStrict ? 1 : 0));
3200
3201 if (IsStrict) {
3202 unsigned Opcode;
3203 if (SVT == MVT::f16)
3204 Opcode = ISD::STRICT_FP16_TO_FP;
3205 else if (SVT == MVT::bf16)
3206 Opcode = ISD::STRICT_BF16_TO_FP;
3207 else
3208 llvm_unreachable("unknown half type");
3209 SDValue Res =
3210 DAG.getNode(Opcode, SDLoc(N), {N->getValueType(0), MVT::Other},
3211 {N->getOperand(0), Op});
3212 ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
3213 ReplaceValueWith(SDValue(N, 0), Res);
3214 return SDValue();
3215 }
3216
3217 return DAG.getNode(GetPromotionOpcode(SVT, RVT), SDLoc(N), RVT, Op);
3218}
3219
3220SDValue DAGTypeLegalizer::SoftPromoteHalfOp_FP_TO_XINT(SDNode *N) {
3221 EVT RVT = N->getValueType(0);
3222 SDValue Op = N->getOperand(0);
3223 EVT SVT = Op.getValueType();
3224 SDLoc dl(N);
3225
3226 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), Op.getValueType());
3227
3228 Op = GetSoftPromotedHalf(Op);
3229
3230 SDValue Res = DAG.getNode(GetPromotionOpcode(SVT, RVT), dl, NVT, Op);
3231
3232 return DAG.getNode(N->getOpcode(), dl, N->getValueType(0), Res);
3233}
3234
3235SDValue DAGTypeLegalizer::SoftPromoteHalfOp_FP_TO_XINT_SAT(SDNode *N) {
3236 EVT RVT = N->getValueType(0);
3237 SDValue Op = N->getOperand(0);
3238 EVT SVT = Op.getValueType();
3239 SDLoc dl(N);
3240
3241 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), Op.getValueType());
3242
3243 Op = GetSoftPromotedHalf(Op);
3244
3245 SDValue Res = DAG.getNode(GetPromotionOpcode(SVT, RVT), dl, NVT, Op);
3246
3247 return DAG.getNode(N->getOpcode(), dl, N->getValueType(0), Res,
3248 N->getOperand(1));
3249}
3250
3251SDValue DAGTypeLegalizer::SoftPromoteHalfOp_SELECT_CC(SDNode *N,
3252 unsigned OpNo) {
3253 assert(OpNo == 0 && "Can only soften the comparison values");
3254 SDValue Op0 = N->getOperand(0);
3255 SDValue Op1 = N->getOperand(1);
3256 SDLoc dl(N);
3257
3258 EVT SVT = Op0.getValueType();
3259 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), SVT);
3260
3261 Op0 = GetSoftPromotedHalf(Op0);
3262 Op1 = GetSoftPromotedHalf(Op1);
3263
3264 // Promote to the larger FP type.
3265 auto PromotionOpcode = GetPromotionOpcode(SVT, NVT);
3266 Op0 = DAG.getNode(PromotionOpcode, dl, NVT, Op0);
3267 Op1 = DAG.getNode(PromotionOpcode, dl, NVT, Op1);
3268
3269 return DAG.getNode(ISD::SELECT_CC, SDLoc(N), N->getValueType(0), Op0, Op1,
3270 N->getOperand(2), N->getOperand(3), N->getOperand(4));
3271}
3272
3273SDValue DAGTypeLegalizer::SoftPromoteHalfOp_SETCC(SDNode *N) {
3274 SDValue Op0 = N->getOperand(0);
3275 SDValue Op1 = N->getOperand(1);
3276 ISD::CondCode CCCode = cast<CondCodeSDNode>(N->getOperand(2))->get();
3277 SDLoc dl(N);
3278
3279 EVT SVT = Op0.getValueType();
3280 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), Op0.getValueType());
3281
3282 Op0 = GetSoftPromotedHalf(Op0);
3283 Op1 = GetSoftPromotedHalf(Op1);
3284
3285 // Promote to the larger FP type.
3286 auto PromotionOpcode = GetPromotionOpcode(SVT, NVT);
3287 Op0 = DAG.getNode(PromotionOpcode, dl, NVT, Op0);
3288 Op1 = DAG.getNode(PromotionOpcode, dl, NVT, Op1);
3289
3290 return DAG.getSetCC(SDLoc(N), N->getValueType(0), Op0, Op1, CCCode);
3291}
3292
3293SDValue DAGTypeLegalizer::SoftPromoteHalfOp_STORE(SDNode *N, unsigned OpNo) {
3294 assert(OpNo == 1 && "Can only soften the stored value!");
3295 StoreSDNode *ST = cast<StoreSDNode>(N);
3296 SDValue Val = ST->getValue();
3297 SDLoc dl(N);
3298
3299 assert(!ST->isTruncatingStore() && "Unexpected truncating store.");
3300 SDValue Promoted = GetSoftPromotedHalf(Val);
3301 return DAG.getStore(ST->getChain(), dl, Promoted, ST->getBasePtr(),
3302 ST->getMemOperand());
3303}
3304
3305SDValue DAGTypeLegalizer::SoftPromoteHalfOp_STACKMAP(SDNode *N, unsigned OpNo) {
3306 assert(OpNo > 1); // Because the first two arguments are guaranteed legal.
3307 SmallVector<SDValue> NewOps(N->ops().begin(), N->ops().end());
3308 SDValue Op = N->getOperand(OpNo);
3309 NewOps[OpNo] = GetSoftPromotedHalf(Op);
3311 DAG.getNode(N->getOpcode(), SDLoc(N), N->getVTList(), NewOps);
3312
3313 for (unsigned ResNum = 0; ResNum < N->getNumValues(); ResNum++)
3314 ReplaceValueWith(SDValue(N, ResNum), NewNode.getValue(ResNum));
3315
3316 return SDValue(); // Signal that we replaced the node ourselves.
3317}
3318
3319SDValue DAGTypeLegalizer::SoftPromoteHalfOp_PATCHPOINT(SDNode *N,
3320 unsigned OpNo) {
3321 assert(OpNo >= 7);
3322 SmallVector<SDValue> NewOps(N->ops().begin(), N->ops().end());
3323 SDValue Op = N->getOperand(OpNo);
3324 NewOps[OpNo] = GetSoftPromotedHalf(Op);
3326 DAG.getNode(N->getOpcode(), SDLoc(N), N->getVTList(), NewOps);
3327
3328 for (unsigned ResNum = 0; ResNum < N->getNumValues(); ResNum++)
3329 ReplaceValueWith(SDValue(N, ResNum), NewNode.getValue(ResNum));
3330
3331 return SDValue(); // Signal that we replaced the node ourselves.
3332}
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
DXIL Intrinsic Expansion
Returns the sub type a function will return at a given Idx Should correspond to the result type of an ExtractValue instruction executed with just that one unsigned Idx
#define LLVM_DEBUG(X)
Definition: Debug.h:101
uint64_t Size
static bool isSigned(unsigned int Opcode)
static RTLIB::Libcall findFPToIntLibcall(EVT SrcVT, EVT RetVT, EVT &Promoted, bool Signed)
static RTLIB::Libcall GetFPLibCall(EVT VT, RTLIB::Libcall Call_F32, RTLIB::Libcall Call_F64, RTLIB::Libcall Call_F80, RTLIB::Libcall Call_F128, RTLIB::Libcall Call_PPCF128)
GetFPLibCall - Return the right libcall for the given floating point type.
static ISD::NodeType GetPromotionOpcode(EVT OpVT, EVT RetVT)
static ISD::NodeType GetPromotionOpcodeStrict(EVT OpVT, EVT RetVT)
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
Value * RHS
Value * LHS
APInt bitcastToAPInt() const
Definition: APFloat.h:1210
Class for arbitrary precision integers.
Definition: APInt.h:76
static APInt getAllOnes(unsigned numBits)
Return an APInt of a specified width with all bits set.
Definition: APInt.h:212
void clearBit(unsigned BitPosition)
Set a given bit to 0.
Definition: APInt.h:1385
static APInt getSignMask(unsigned BitWidth)
Get the SignMask for a specific bit width.
Definition: APInt.h:207
const uint64_t * getRawData() const
This function returns a pointer to the internal storage of the APInt.
Definition: APInt.h:547
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
Definition: ArrayRef.h:41
This is an SDNode representing atomic operations.
const SDValue & getVal() const
const APFloat & getValueAPF() const
@ NewNode
This is a new node, not before seen, that was created in the process of legalizing some other node.
Definition: LegalizeTypes.h:43
This class represents an Operation in the Expression.
bool isBigEndian() const
Definition: DataLayout.h:239
void emitError(uint64_t LocCookie, const Twine &ErrorStr)
emitError - Emit an error message to the currently installed error handler with optional location inf...
This class is used to represent ISD::LOAD nodes.
SimpleValueType SimpleTy
@ MODereferenceable
The memory access is dereferenceable (i.e., doesn't trap).
@ MOInvariant
The memory access always returns the same value (or traps).
MachineMemOperand * getMemOperand() const
Return a MachineMemOperand object describing the memory reference performed by operation.
Wrapper class for IR location info (IR ordering and DebugLoc) to be passed into SDNode creation funct...
Represents one node in the SelectionDAG.
unsigned getNumValues() const
Return the number of values defined/returned by this operator.
EVT getValueType(unsigned ResNo) const
Return the type of a specified result.
Unlike LLVM values, Selection DAG nodes may return multiple values as the result of a computation.
SDNode * getNode() const
get the SDNode which holds the desired result
SDValue getValue(unsigned R) const
EVT getValueType() const
Return the ValueType of the referenced return value.
SDValue getExtLoad(ISD::LoadExtType ExtType, const SDLoc &dl, EVT VT, SDValue Chain, SDValue Ptr, MachinePointerInfo PtrInfo, EVT MemVT, MaybeAlign Alignment=MaybeAlign(), MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes())
SDValue getSelect(const SDLoc &DL, EVT VT, SDValue Cond, SDValue LHS, SDValue RHS)
Helper function to make it easier to build Select's if you just have operands and don't want to check...
SDVTList getVTList(EVT VT)
Return an SDVTList that represents the list of values specified.
SDValue getSetCC(const SDLoc &DL, EVT VT, SDValue LHS, SDValue RHS, ISD::CondCode Cond, SDValue Chain=SDValue(), bool IsSignaling=false)
Helper function to make it easier to build SetCC's if you just have an ISD::CondCode instead of an SD...
SDValue getConstantFP(double Val, const SDLoc &DL, EVT VT, bool isTarget=false)
Create a ConstantFPSDNode wrapping a constant value.
SDValue getLoad(EVT VT, const SDLoc &dl, SDValue Chain, SDValue Ptr, MachinePointerInfo PtrInfo, MaybeAlign Alignment=MaybeAlign(), MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes(), const MDNode *Ranges=nullptr)
Loads are not normal binary operators: their result type is not determined by their operands,...
SDValue getAtomic(unsigned Opcode, const SDLoc &dl, EVT MemVT, SDValue Chain, SDValue Ptr, SDValue Val, MachineMemOperand *MMO)
Gets a node for an atomic op, produces result (if relevant) and chain and takes 2 operands.
SDValue getUNDEF(EVT VT)
Return an UNDEF node. UNDEF does not have a useful SDLoc.
SDValue getBitcast(EVT VT, SDValue V)
Return a bitcast using the SDLoc of the value operand, and casting to the provided type.
const DataLayout & getDataLayout() const
Definition: SelectionDAG.h:472
SDValue getConstant(uint64_t Val, const SDLoc &DL, EVT VT, bool isTarget=false, bool isOpaque=false)
Create a ConstantSDNode wrapping a constant value.
SDValue getVAArg(EVT VT, const SDLoc &dl, SDValue Chain, SDValue Ptr, SDValue SV, unsigned Align)
VAArg produces a result and token chain, and takes a pointer and a source value as input.
SDValue getTruncStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr, MachinePointerInfo PtrInfo, EVT SVT, Align Alignment, MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes())
SDValue getStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr, MachinePointerInfo PtrInfo, Align Alignment, MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes())
Helper function to build ISD::STORE nodes.
static const fltSemantics & EVTToAPFloatSemantics(EVT VT)
Returns an APFloat semantics tag appropriate for the given type.
SDValue getSelectCC(const SDLoc &DL, SDValue LHS, SDValue RHS, SDValue True, SDValue False, ISD::CondCode Cond)
Helper function to make it easier to build SelectCC's if you just have an ISD::CondCode instead of an...
SDValue getIntPtrConstant(uint64_t Val, const SDLoc &DL, bool isTarget=false)
SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT, ArrayRef< SDUse > Ops)
Gets or creates the specified node.
const TargetLibraryInfo & getLibInfo() const
Definition: SelectionDAG.h:479
MachineFunction & getMachineFunction() const
Definition: SelectionDAG.h:469
SDValue getCondCode(ISD::CondCode Cond)
LLVMContext * getContext() const
Definition: SelectionDAG.h:485
SDValue getShiftAmountConstant(uint64_t Val, EVT VT, const SDLoc &DL, bool LegalTypes=true)
SDValue CreateStackTemporary(TypeSize Bytes, Align Alignment)
Create a stack temporary based on the size in bytes and the alignment.
SDNode * UpdateNodeOperands(SDNode *N, SDValue Op)
Mutate the specified node in-place to have the specified operands.
SDValue getEntryNode() const
Return the token chain corresponding to the entry of the function.
Definition: SelectionDAG.h:554
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
Definition: SmallVector.h:1209
This class is used to represent ISD::STORE nodes.
unsigned getIntSize() const
Get size of a C-level int or unsigned int, in bits.
EVT getShiftAmountTy(EVT LHSTy, const DataLayout &DL, bool LegalTypes=true) const
Returns the type for the shift amount of a shift opcode.
virtual EVT getTypeToTransformTo(LLVMContext &Context, EVT VT) const
For types supported by the target, this is an identity function.
const char * getLibcallName(RTLIB::Libcall Call) const
Get the libcall routine name for the specified libcall.
void softenSetCCOperands(SelectionDAG &DAG, EVT VT, SDValue &NewLHS, SDValue &NewRHS, ISD::CondCode &CCCode, const SDLoc &DL, const SDValue OldLHS, const SDValue OldRHS) const
Soften the operands of a comparison.
std::pair< SDValue, SDValue > makeLibCall(SelectionDAG &DAG, RTLIB::Libcall LC, EVT RetVT, ArrayRef< SDValue > Ops, MakeLibCallOptions CallOptions, const SDLoc &dl, SDValue Chain=SDValue()) const
Returns a pair of (return value, chain).
SDValue expandVecReduceSeq(SDNode *Node, SelectionDAG &DAG) const
Expand a VECREDUCE_SEQ_* into an explicit ordered calculation.
SDValue expandFP_TO_INT_SAT(SDNode *N, SelectionDAG &DAG) const
Expand FP_TO_[US]INT_SAT into FP_TO_[US]INT and selects or min/max.
SDValue expandVecReduce(SDNode *Node, SelectionDAG &DAG) const
Expand a VECREDUCE_* into an explicit calculation.
SDValue createSelectForFMINNUM_FMAXNUM(SDNode *Node, SelectionDAG &DAG) const
Try to convert the fminnum/fmaxnum to a compare/select sequence.
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
constexpr std::underlying_type_t< E > Mask()
Get a bitmask with 1s in all places up to the high-order bit of E's largest value.
Definition: BitmaskEnum.h:121
@ C
The default llvm calling convention, compatible with C.
Definition: CallingConv.h:34
NodeType
ISD::NodeType enum - This enum defines the target-independent operators for a SelectionDAG.
Definition: ISDOpcodes.h:40
@ SETCC
SetCC operator - This evaluates to a true value iff the condition is true.
Definition: ISDOpcodes.h:750
@ MERGE_VALUES
MERGE_VALUES - This node takes multiple discrete operands and returns them all as its individual resu...
Definition: ISDOpcodes.h:236
@ STRICT_FSETCC
STRICT_FSETCC/STRICT_FSETCCS - Constrained versions of SETCC, used for floating-point operands only.
Definition: ISDOpcodes.h:476
@ VECREDUCE_SEQ_FADD
Generic reduction nodes.
Definition: ISDOpcodes.h:1339
@ ConstantFP
Definition: ISDOpcodes.h:77
@ STRICT_FCEIL
Definition: ISDOpcodes.h:426
@ FMAD
FMAD - Perform a * b + c, while getting the same result as the separately rounded operations.
Definition: ISDOpcodes.h:487
@ LOAD
LOAD and STORE have token chains as their first operand, then the same operands as an LLVM load/store...
Definition: ISDOpcodes.h:1031
@ ANY_EXTEND
ANY_EXTEND - Used for integer types. The high bits are undefined.
Definition: ISDOpcodes.h:783
@ FMA
FMA - Perform a * b + c with no intermediate rounding step.
Definition: ISDOpcodes.h:483
@ SINT_TO_FP
[SU]INT_TO_FP - These operators convert integers (whose interpreted sign depends on the first letter)...
Definition: ISDOpcodes.h:790
@ VECREDUCE_FMAX
FMIN/FMAX nodes can have flags, for NaN/NoNaN variants.
Definition: ISDOpcodes.h:1355
@ FADD
Simple binary floating point operators.
Definition: ISDOpcodes.h:390
@ VECREDUCE_FMAXIMUM
FMINIMUM/FMAXIMUM nodes propatate NaNs and signed zeroes using the llvm.minimum and llvm....
Definition: ISDOpcodes.h:1359
@ STRICT_FSETCCS
Definition: ISDOpcodes.h:477
@ FP16_TO_FP
FP16_TO_FP, FP_TO_FP16 - These operators are used to perform promotions and truncation for half-preci...
Definition: ISDOpcodes.h:913
@ STRICT_FLOG2
Definition: ISDOpcodes.h:421
@ BITCAST
BITCAST - This operator converts between integer, vector and FP values, as if the value was stored to...
Definition: ISDOpcodes.h:903
@ BUILD_PAIR
BUILD_PAIR - This is the opposite of EXTRACT_ELEMENT in some ways.
Definition: ISDOpcodes.h:229
@ FLDEXP
FLDEXP - ldexp, inspired by libm (op0 * 2**op1).
Definition: ISDOpcodes.h:939
@ STRICT_FSQRT
Constrained versions of libm-equivalent floating point intrinsics.
Definition: ISDOpcodes.h:411
@ SIGN_EXTEND
Conversion operators.
Definition: ISDOpcodes.h:774
@ STRICT_UINT_TO_FP
Definition: ISDOpcodes.h:450
@ VECREDUCE_FADD
These reductions have relaxed evaluation order semantics, and have a single vector operand.
Definition: ISDOpcodes.h:1352
@ VECREDUCE_FMIN
Definition: ISDOpcodes.h:1356
@ STRICT_LROUND
Definition: ISDOpcodes.h:431
@ FNEG
Perform various unary floating-point operations inspired by libm.
Definition: ISDOpcodes.h:930
@ BR_CC
BR_CC - Conditional branch.
Definition: ISDOpcodes.h:1077
@ FCANONICALIZE
Returns platform specific canonical encoding of a floating point number.
Definition: ISDOpcodes.h:500
@ SELECT
Select(COND, TRUEVAL, FALSEVAL).
Definition: ISDOpcodes.h:727
@ STRICT_FPOWI
Definition: ISDOpcodes.h:413
@ UNDEF
UNDEF - An undefined node.
Definition: ISDOpcodes.h:211
@ EXTRACT_ELEMENT
EXTRACT_ELEMENT - This is used to get the lower or upper (determined by a Constant,...
Definition: ISDOpcodes.h:222
@ STRICT_FTRUNC
Definition: ISDOpcodes.h:430
@ ARITH_FENCE
ARITH_FENCE - This corresponds to a arithmetic fence intrinsic.
Definition: ISDOpcodes.h:1225
@ STRICT_FP_TO_FP16
Definition: ISDOpcodes.h:916
@ STRICT_FP16_TO_FP
Definition: ISDOpcodes.h:915
@ SHL
Shift and rotation operations.
Definition: ISDOpcodes.h:705
@ STRICT_FMAXNUM
Definition: ISDOpcodes.h:424
@ EXTRACT_VECTOR_ELT
EXTRACT_VECTOR_ELT(VECTOR, IDX) - Returns a single element from VECTOR identified by the (potentially...
Definition: ISDOpcodes.h:535
@ ZERO_EXTEND
ZERO_EXTEND - Used for integer types, zeroing the new bits.
Definition: ISDOpcodes.h:780
@ FP_TO_UINT_SAT
Definition: ISDOpcodes.h:856
@ STRICT_FMINNUM
Definition: ISDOpcodes.h:425
@ SELECT_CC
Select with condition operator - This selects between a true value and a false value (ops #2 and #3) ...
Definition: ISDOpcodes.h:742
@ FMINNUM
FMINNUM/FMAXNUM - Perform floating-point minimum or maximum on two values.
Definition: ISDOpcodes.h:971
@ STRICT_LRINT
Definition: ISDOpcodes.h:433
@ FP_EXTEND
X = FP_EXTEND(Y) - Extend a smaller FP type into a larger FP type.
Definition: ISDOpcodes.h:888
@ STRICT_FROUND
Definition: ISDOpcodes.h:428
@ STRICT_SINT_TO_FP
STRICT_[US]INT_TO_FP - Convert a signed or unsigned integer to a floating point value.
Definition: ISDOpcodes.h:449
@ STRICT_BF16_TO_FP
Definition: ISDOpcodes.h:924
@ STRICT_FFLOOR
Definition: ISDOpcodes.h:427
@ STRICT_FROUNDEVEN
Definition: ISDOpcodes.h:429
@ BF16_TO_FP
BF16_TO_FP, FP_TO_BF16 - These operators are used to perform promotions and truncation for bfloat16.
Definition: ISDOpcodes.h:922
@ STRICT_FP_TO_UINT
Definition: ISDOpcodes.h:443
@ STRICT_FP_ROUND
X = STRICT_FP_ROUND(Y, TRUNC) - Rounding 'Y' from a larger floating point type down to the precision ...
Definition: ISDOpcodes.h:465
@ STRICT_FP_TO_SINT
STRICT_FP_TO_[US]INT - Convert a floating point value to a signed or unsigned integer.
Definition: ISDOpcodes.h:442
@ FMINIMUM
FMINIMUM/FMAXIMUM - NaN-propagating minimum/maximum that also treat -0.0 as less than 0....
Definition: ISDOpcodes.h:984
@ FP_TO_SINT
FP_TO_[US]INT - Convert a floating point value to a signed or unsigned integer.
Definition: ISDOpcodes.h:836
@ STRICT_FP_EXTEND
X = STRICT_FP_EXTEND(Y) - Extend a smaller FP type into a larger FP type.
Definition: ISDOpcodes.h:470
@ AND
Bitwise operators - logical and, logical or, logical xor.
Definition: ISDOpcodes.h:680
@ STRICT_FP_TO_BF16
Definition: ISDOpcodes.h:925
@ VECREDUCE_FMUL
Definition: ISDOpcodes.h:1353
@ STRICT_FADD
Constrained versions of the binary floating point operators.
Definition: ISDOpcodes.h:400
@ STRICT_FLOG10
Definition: ISDOpcodes.h:420
@ STRICT_LLRINT
Definition: ISDOpcodes.h:434
@ STRICT_FEXP2
Definition: ISDOpcodes.h:418
@ ATOMIC_SWAP
Val, OUTCHAIN = ATOMIC_SWAP(INCHAIN, ptr, amt) Val, OUTCHAIN = ATOMIC_LOAD_[OpName](INCHAIN,...
Definition: ISDOpcodes.h:1262
@ FFREXP
FFREXP - frexp, extract fractional and exponent component of a floating-point value.
Definition: ISDOpcodes.h:944
@ FP_ROUND
X = FP_ROUND(Y, TRUNC) - Rounding 'Y' from a larger floating point type down to the precision of the ...
Definition: ISDOpcodes.h:869
@ STRICT_FLDEXP
Definition: ISDOpcodes.h:414
@ STRICT_LLROUND
Definition: ISDOpcodes.h:432
@ STRICT_FNEARBYINT
Definition: ISDOpcodes.h:423
@ FP_TO_SINT_SAT
FP_TO_[US]INT_SAT - Convert floating point value in operand 0 to a signed or unsigned scalar integer ...
Definition: ISDOpcodes.h:855
@ VECREDUCE_FMINIMUM
Definition: ISDOpcodes.h:1360
@ TRUNCATE
TRUNCATE - Completely drop the high bits.
Definition: ISDOpcodes.h:786
@ VAARG
VAARG - VAARG has four operands: an input chain, a pointer, a SRCVALUE, and the alignment.
Definition: ISDOpcodes.h:1146
@ VECREDUCE_SEQ_FMUL
Definition: ISDOpcodes.h:1340
@ FCOPYSIGN
FCOPYSIGN(X, Y) - Return the value of X with the sign of Y.
Definition: ISDOpcodes.h:493
@ STRICT_FRINT
Definition: ISDOpcodes.h:422
@ BUILD_VECTOR
BUILD_VECTOR(ELT0, ELT1, ELT2, ELT3,...) - Return a fixed-width vector with the specified,...
Definition: ISDOpcodes.h:515
bool isNormalStore(const SDNode *N)
Returns true if the specified node is a non-truncating and unindexed store.
bool isUNINDEXEDLoad(const SDNode *N)
Returns true if the specified node is an unindexed load.
bool isUNINDEXEDStore(const SDNode *N)
Returns true if the specified node is an unindexed store.
CondCode
ISD::CondCode enum - These are ordered carefully to make the bitfields below work out,...
Definition: ISDOpcodes.h:1523
bool isNormalLoad(const SDNode *N)
Returns true if the specified node is a non-extending and unindexed load.
Libcall getPOWI(EVT RetVT)
getPOWI - Return the POWI_* value for the given types, or UNKNOWN_LIBCALL if there is none.
Libcall getSINTTOFP(EVT OpVT, EVT RetVT)
getSINTTOFP - Return the SINTTOFP_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
Libcall getLDEXP(EVT RetVT)
getLDEXP - Return the LDEXP_* value for the given types, or UNKNOWN_LIBCALL if there is none.
Libcall getUINTTOFP(EVT OpVT, EVT RetVT)
getUINTTOFP - Return the UINTTOFP_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
Libcall getFREXP(EVT RetVT)
getFREXP - Return the FREXP_* value for the given types, or UNKNOWN_LIBCALL if there is none.
Libcall
RTLIB::Libcall enum - This enum defines all of the runtime library calls the backend can emit.
Libcall getFPTOUINT(EVT OpVT, EVT RetVT)
getFPTOUINT - Return the FPTOUINT_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
Libcall getFPTOSINT(EVT OpVT, EVT RetVT)
getFPTOSINT - Return the FPTOSINT_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
Libcall getFPEXT(EVT OpVT, EVT RetVT)
getFPEXT - Return the FPEXT_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
Libcall getFPROUND(EVT OpVT, EVT RetVT)
getFPROUND - Return the FPROUND_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
DiagnosticInfoOptimizationBase::Argument NV
@ FalseVal
Definition: TGLexer.h:59
This is an optimization pass for GlobalISel generic memory operations.
Definition: AddressRanges.h:18
@ Offset
Definition: DWP.cpp:456
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
Definition: Debug.cpp:163
void report_fatal_error(Error Err, bool gen_crash_diag=true)
Report a serious error, calling any installed error handler.
Definition: Error.cpp:156
DWARFExpression::Operation Op
#define N
static const fltSemantics & PPCDoubleDouble() LLVM_READNONE
Definition: APFloat.cpp:252
Extended Value Type.
Definition: ValueTypes.h:34
bool isSimple() const
Test if the given EVT is simple (as opposed to being extended).
Definition: ValueTypes.h:136
TypeSize getSizeInBits() const
Return the size of the specified value type in bits.
Definition: ValueTypes.h:358
bool isByteSized() const
Return true if the bit size is a multiple of 8.
Definition: ValueTypes.h:233
MVT getSimpleVT() const
Return the SimpleValueType held in the specified simple EVT.
Definition: ValueTypes.h:306
static EVT getIntegerVT(LLVMContext &Context, unsigned BitWidth)
Returns the EVT that represents an integer with the given number of bits.
Definition: ValueTypes.h:64
bool bitsGE(EVT VT) const
Return true if this has no less bits than VT.
Definition: ValueTypes.h:282
EVT getVectorElementType() const
Given a vector type, return the type of each element.
Definition: ValueTypes.h:318
bool bitsLE(EVT VT) const
Return true if this has no more bits than VT.
Definition: ValueTypes.h:298
static MachinePointerInfo getFixedStack(MachineFunction &MF, int FI, int64_t Offset=0)
Return a MachinePointerInfo record that refers to the specified FrameIndex.
These are IR-level optimization flags that may be propagated to SDNodes.
This structure is used to pass arguments to makeLibCall function.
MakeLibCallOptions & setSExt(bool Value=true)
MakeLibCallOptions & setTypeListBeforeSoften(ArrayRef< EVT > OpsVT, EVT RetVT, bool Value=true)