LLVM 19.0.0git
LegalizeFloatTypes.cpp
Go to the documentation of this file.
1//===-------- LegalizeFloatTypes.cpp - Legalization of float types --------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements float type expansion and softening for LegalizeTypes.
10// Softening is the act of turning a computation in an illegal floating point
11// type into a computation in an integer type of the same size; also known as
12// "soft float". For example, turning f32 arithmetic into operations using i32.
13// The resulting integer value is the same as what you would get by performing
14// the floating point operation and bitcasting the result to the integer type.
15// Expansion is the act of changing a computation in an illegal type to be a
16// computation in two identical registers of a smaller type. For example,
17// implementing ppcf128 arithmetic in two f64 registers.
18//
19//===----------------------------------------------------------------------===//
20
21#include "LegalizeTypes.h"
25using namespace llvm;
26
27#define DEBUG_TYPE "legalize-types"
28
29/// GetFPLibCall - Return the right libcall for the given floating point type.
30/// FIXME: This is a local version of RTLIB::getFPLibCall that should be
31/// refactored away (see RTLIB::getPOWI for an example).
33 RTLIB::Libcall Call_F32,
34 RTLIB::Libcall Call_F64,
35 RTLIB::Libcall Call_F80,
36 RTLIB::Libcall Call_F128,
37 RTLIB::Libcall Call_PPCF128) {
38 return
39 VT == MVT::f32 ? Call_F32 :
40 VT == MVT::f64 ? Call_F64 :
41 VT == MVT::f80 ? Call_F80 :
42 VT == MVT::f128 ? Call_F128 :
43 VT == MVT::ppcf128 ? Call_PPCF128 :
44 RTLIB::UNKNOWN_LIBCALL;
45}
46
47//===----------------------------------------------------------------------===//
48// Convert Float Results to Integer
49//===----------------------------------------------------------------------===//
50
51void DAGTypeLegalizer::SoftenFloatResult(SDNode *N, unsigned ResNo) {
52 LLVM_DEBUG(dbgs() << "Soften float result " << ResNo << ": "; N->dump(&DAG));
53 SDValue R = SDValue();
54
55 switch (N->getOpcode()) {
56 // clang-format off
57 default:
58#ifndef NDEBUG
59 dbgs() << "SoftenFloatResult #" << ResNo << ": ";
60 N->dump(&DAG); dbgs() << "\n";
61#endif
62 report_fatal_error("Do not know how to soften the result of this "
63 "operator!");
64 case ISD::EXTRACT_ELEMENT: R = SoftenFloatRes_EXTRACT_ELEMENT(N); break;
65 case ISD::ARITH_FENCE: R = SoftenFloatRes_ARITH_FENCE(N); break;
66 case ISD::MERGE_VALUES:R = SoftenFloatRes_MERGE_VALUES(N, ResNo); break;
67 case ISD::BITCAST: R = SoftenFloatRes_BITCAST(N); break;
68 case ISD::BUILD_PAIR: R = SoftenFloatRes_BUILD_PAIR(N); break;
69 case ISD::ConstantFP: R = SoftenFloatRes_ConstantFP(N); break;
71 R = SoftenFloatRes_EXTRACT_VECTOR_ELT(N, ResNo); break;
72 case ISD::FABS: R = SoftenFloatRes_FABS(N); break;
74 case ISD::FMINNUM: R = SoftenFloatRes_FMINNUM(N); break;
76 case ISD::FMAXNUM: R = SoftenFloatRes_FMAXNUM(N); break;
78 case ISD::FADD: R = SoftenFloatRes_FADD(N); break;
79 case ISD::FCBRT: R = SoftenFloatRes_FCBRT(N); break;
81 case ISD::FCEIL: R = SoftenFloatRes_FCEIL(N); break;
82 case ISD::FCOPYSIGN: R = SoftenFloatRes_FCOPYSIGN(N); break;
84 case ISD::FCOS: R = SoftenFloatRes_FCOS(N); break;
86 case ISD::FDIV: R = SoftenFloatRes_FDIV(N); break;
88 case ISD::FEXP: R = SoftenFloatRes_FEXP(N); break;
90 case ISD::FEXP2: R = SoftenFloatRes_FEXP2(N); break;
91 case ISD::FEXP10: R = SoftenFloatRes_FEXP10(N); break;
93 case ISD::FFLOOR: R = SoftenFloatRes_FFLOOR(N); break;
95 case ISD::FLOG: R = SoftenFloatRes_FLOG(N); break;
97 case ISD::FLOG2: R = SoftenFloatRes_FLOG2(N); break;
99 case ISD::FLOG10: R = SoftenFloatRes_FLOG10(N); break;
100 case ISD::STRICT_FMA:
101 case ISD::FMA: R = SoftenFloatRes_FMA(N); break;
102 case ISD::STRICT_FMUL:
103 case ISD::FMUL: R = SoftenFloatRes_FMUL(N); break;
105 case ISD::FNEARBYINT: R = SoftenFloatRes_FNEARBYINT(N); break;
106 case ISD::FNEG: R = SoftenFloatRes_FNEG(N); break;
108 case ISD::FP_EXTEND: R = SoftenFloatRes_FP_EXTEND(N); break;
110 case ISD::FP_ROUND: R = SoftenFloatRes_FP_ROUND(N); break;
111 case ISD::FP16_TO_FP: R = SoftenFloatRes_FP16_TO_FP(N); break;
112 case ISD::BF16_TO_FP: R = SoftenFloatRes_BF16_TO_FP(N); break;
113 case ISD::STRICT_FPOW:
114 case ISD::FPOW: R = SoftenFloatRes_FPOW(N); break;
116 case ISD::FPOWI:
117 case ISD::FLDEXP:
118 case ISD::STRICT_FLDEXP: R = SoftenFloatRes_ExpOp(N); break;
119 case ISD::FFREXP: R = SoftenFloatRes_FFREXP(N); break;
120 case ISD::STRICT_FREM:
121 case ISD::FREM: R = SoftenFloatRes_FREM(N); break;
123 case ISD::FRINT: R = SoftenFloatRes_FRINT(N); break;
125 case ISD::FROUND: R = SoftenFloatRes_FROUND(N); break;
127 case ISD::FROUNDEVEN: R = SoftenFloatRes_FROUNDEVEN(N); break;
128 case ISD::STRICT_FSIN:
129 case ISD::FSIN: R = SoftenFloatRes_FSIN(N); break;
131 case ISD::FSQRT: R = SoftenFloatRes_FSQRT(N); break;
132 case ISD::STRICT_FSUB:
133 case ISD::FSUB: R = SoftenFloatRes_FSUB(N); break;
134 case ISD::STRICT_FTAN:
135 case ISD::FTAN: R = SoftenFloatRes_FTAN(N); break;
137 case ISD::FTRUNC: R = SoftenFloatRes_FTRUNC(N); break;
138 case ISD::LOAD: R = SoftenFloatRes_LOAD(N); break;
139 case ISD::ATOMIC_LOAD: R = SoftenFloatRes_ATOMIC_LOAD(N); break;
140 case ISD::ATOMIC_SWAP: R = BitcastToInt_ATOMIC_SWAP(N); break;
141 case ISD::SELECT: R = SoftenFloatRes_SELECT(N); break;
142 case ISD::SELECT_CC: R = SoftenFloatRes_SELECT_CC(N); break;
143 case ISD::FREEZE: R = SoftenFloatRes_FREEZE(N); break;
146 case ISD::SINT_TO_FP:
147 case ISD::UINT_TO_FP: R = SoftenFloatRes_XINT_TO_FP(N); break;
148 case ISD::UNDEF: R = SoftenFloatRes_UNDEF(N); break;
149 case ISD::VAARG: R = SoftenFloatRes_VAARG(N); break;
155 case ISD::VECREDUCE_FMINIMUM: R = SoftenFloatRes_VECREDUCE(N); break;
157 case ISD::VECREDUCE_SEQ_FMUL: R = SoftenFloatRes_VECREDUCE_SEQ(N); break;
158 // clang-format on
159 }
160
161 // If R is null, the sub-method took care of registering the result.
162 if (R.getNode()) {
163 assert(R.getNode() != N);
164 SetSoftenedFloat(SDValue(N, ResNo), R);
165 }
166}
167
168SDValue DAGTypeLegalizer::SoftenFloatRes_Unary(SDNode *N, RTLIB::Libcall LC) {
169 bool IsStrict = N->isStrictFPOpcode();
170 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
171 unsigned Offset = IsStrict ? 1 : 0;
172 assert(N->getNumOperands() == (1 + Offset) &&
173 "Unexpected number of operands!");
174 SDValue Op = GetSoftenedFloat(N->getOperand(0 + Offset));
175 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
177 EVT OpVT = N->getOperand(0 + Offset).getValueType();
178 CallOptions.setTypeListBeforeSoften(OpVT, N->getValueType(0), true);
179 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, NVT, Op,
180 CallOptions, SDLoc(N),
181 Chain);
182 if (IsStrict)
183 ReplaceValueWith(SDValue(N, 1), Tmp.second);
184 return Tmp.first;
185}
186
187SDValue DAGTypeLegalizer::SoftenFloatRes_Binary(SDNode *N, RTLIB::Libcall LC) {
188 bool IsStrict = N->isStrictFPOpcode();
189 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
190 unsigned Offset = IsStrict ? 1 : 0;
191 assert(N->getNumOperands() == (2 + Offset) &&
192 "Unexpected number of operands!");
193 SDValue Ops[2] = { GetSoftenedFloat(N->getOperand(0 + Offset)),
194 GetSoftenedFloat(N->getOperand(1 + Offset)) };
195 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
197 EVT OpsVT[2] = { N->getOperand(0 + Offset).getValueType(),
198 N->getOperand(1 + Offset).getValueType() };
199 CallOptions.setTypeListBeforeSoften(OpsVT, N->getValueType(0), true);
200 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, NVT, Ops,
201 CallOptions, SDLoc(N),
202 Chain);
203 if (IsStrict)
204 ReplaceValueWith(SDValue(N, 1), Tmp.second);
205 return Tmp.first;
206}
207
208SDValue DAGTypeLegalizer::SoftenFloatRes_BITCAST(SDNode *N) {
209 return BitConvertToInteger(N->getOperand(0));
210}
211
212SDValue DAGTypeLegalizer::SoftenFloatRes_FREEZE(SDNode *N) {
213 EVT Ty = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
214 return DAG.getNode(ISD::FREEZE, SDLoc(N), Ty,
215 GetSoftenedFloat(N->getOperand(0)));
216}
217
218SDValue DAGTypeLegalizer::SoftenFloatRes_ARITH_FENCE(SDNode *N) {
219 EVT Ty = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
220 SDValue NewFence = DAG.getNode(ISD::ARITH_FENCE, SDLoc(N), Ty,
221 GetSoftenedFloat(N->getOperand(0)));
222 return NewFence;
223}
224
225SDValue DAGTypeLegalizer::SoftenFloatRes_MERGE_VALUES(SDNode *N,
226 unsigned ResNo) {
227 SDValue Op = DisintegrateMERGE_VALUES(N, ResNo);
228 return BitConvertToInteger(Op);
229}
230
231SDValue DAGTypeLegalizer::SoftenFloatRes_BUILD_PAIR(SDNode *N) {
232 // Convert the inputs to integers, and build a new pair out of them.
233 return DAG.getNode(ISD::BUILD_PAIR, SDLoc(N),
235 N->getValueType(0)),
236 BitConvertToInteger(N->getOperand(0)),
237 BitConvertToInteger(N->getOperand(1)));
238}
239
240SDValue DAGTypeLegalizer::SoftenFloatRes_ConstantFP(SDNode *N) {
241 ConstantFPSDNode *CN = cast<ConstantFPSDNode>(N);
242 // In ppcf128, the high 64 bits are always first in memory regardless
243 // of Endianness. LLVM's APFloat representation is not Endian sensitive,
244 // and so always converts into a 128-bit APInt in a non-Endian-sensitive
245 // way. However, APInt's are serialized in an Endian-sensitive fashion,
246 // so on big-Endian targets, the two doubles are output in the wrong
247 // order. Fix this by manually flipping the order of the high 64 bits
248 // and the low 64 bits here.
249 if (DAG.getDataLayout().isBigEndian() &&
250 CN->getValueType(0).getSimpleVT() == llvm::MVT::ppcf128) {
251 uint64_t words[2] = { CN->getValueAPF().bitcastToAPInt().getRawData()[1],
253 APInt Val(128, words);
254 return DAG.getConstant(Val, SDLoc(CN),
256 CN->getValueType(0)));
257 } else {
258 return DAG.getConstant(CN->getValueAPF().bitcastToAPInt(), SDLoc(CN),
260 CN->getValueType(0)));
261 }
262}
263
264SDValue DAGTypeLegalizer::SoftenFloatRes_EXTRACT_ELEMENT(SDNode *N) {
265 SDValue Src = N->getOperand(0);
266 assert(Src.getValueType() == MVT::ppcf128 &&
267 "In floats only ppcf128 can be extracted by element!");
268 return DAG.getNode(ISD::EXTRACT_ELEMENT, SDLoc(N),
269 N->getValueType(0).changeTypeToInteger(),
270 DAG.getBitcast(MVT::i128, Src), N->getOperand(1));
271}
272
273SDValue DAGTypeLegalizer::SoftenFloatRes_EXTRACT_VECTOR_ELT(SDNode *N, unsigned ResNo) {
274 SDValue NewOp = BitConvertVectorToIntegerVector(N->getOperand(0));
277 NewOp, N->getOperand(1));
278}
279
280SDValue DAGTypeLegalizer::SoftenFloatRes_FABS(SDNode *N) {
281 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
282 unsigned Size = NVT.getSizeInBits();
283
284 // Mask = ~(1 << (Size-1))
286 API.clearBit(Size - 1);
287 SDValue Mask = DAG.getConstant(API, SDLoc(N), NVT);
288 SDValue Op = GetSoftenedFloat(N->getOperand(0));
289 return DAG.getNode(ISD::AND, SDLoc(N), NVT, Op, Mask);
290}
291
292SDValue DAGTypeLegalizer::SoftenFloatRes_FMINNUM(SDNode *N) {
293 if (SDValue SelCC = TLI.createSelectForFMINNUM_FMAXNUM(N, DAG))
294 return SoftenFloatRes_SELECT_CC(SelCC.getNode());
295 return SoftenFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
296 RTLIB::FMIN_F32,
297 RTLIB::FMIN_F64,
298 RTLIB::FMIN_F80,
299 RTLIB::FMIN_F128,
300 RTLIB::FMIN_PPCF128));
301}
302
303SDValue DAGTypeLegalizer::SoftenFloatRes_FMAXNUM(SDNode *N) {
304 if (SDValue SelCC = TLI.createSelectForFMINNUM_FMAXNUM(N, DAG))
305 return SoftenFloatRes_SELECT_CC(SelCC.getNode());
306 return SoftenFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
307 RTLIB::FMAX_F32,
308 RTLIB::FMAX_F64,
309 RTLIB::FMAX_F80,
310 RTLIB::FMAX_F128,
311 RTLIB::FMAX_PPCF128));
312}
313
314SDValue DAGTypeLegalizer::SoftenFloatRes_FADD(SDNode *N) {
315 return SoftenFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
316 RTLIB::ADD_F32,
317 RTLIB::ADD_F64,
318 RTLIB::ADD_F80,
319 RTLIB::ADD_F128,
320 RTLIB::ADD_PPCF128));
321}
322
323SDValue DAGTypeLegalizer::SoftenFloatRes_FCBRT(SDNode *N) {
324 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
325 RTLIB::CBRT_F32,
326 RTLIB::CBRT_F64,
327 RTLIB::CBRT_F80,
328 RTLIB::CBRT_F128,
329 RTLIB::CBRT_PPCF128));
330}
331
332SDValue DAGTypeLegalizer::SoftenFloatRes_FCEIL(SDNode *N) {
333 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
334 RTLIB::CEIL_F32,
335 RTLIB::CEIL_F64,
336 RTLIB::CEIL_F80,
337 RTLIB::CEIL_F128,
338 RTLIB::CEIL_PPCF128));
339}
340
341SDValue DAGTypeLegalizer::SoftenFloatRes_FCOPYSIGN(SDNode *N) {
342 SDValue LHS = GetSoftenedFloat(N->getOperand(0));
343 SDValue RHS = BitConvertToInteger(N->getOperand(1));
344 SDLoc dl(N);
345
346 EVT LVT = LHS.getValueType();
347 EVT RVT = RHS.getValueType();
348
349 unsigned LSize = LVT.getSizeInBits();
350 unsigned RSize = RVT.getSizeInBits();
351
352 // First get the sign bit of second operand.
353 SDValue SignBit = DAG.getNode(
354 ISD::SHL, dl, RVT, DAG.getConstant(1, dl, RVT),
355 DAG.getConstant(RSize - 1, dl,
356 TLI.getShiftAmountTy(RVT, DAG.getDataLayout())));
357 SignBit = DAG.getNode(ISD::AND, dl, RVT, RHS, SignBit);
358
359 // Shift right or sign-extend it if the two operands have different types.
360 int SizeDiff = RVT.getSizeInBits() - LVT.getSizeInBits();
361 if (SizeDiff > 0) {
362 SignBit =
363 DAG.getNode(ISD::SRL, dl, RVT, SignBit,
364 DAG.getConstant(SizeDiff, dl,
365 TLI.getShiftAmountTy(SignBit.getValueType(),
366 DAG.getDataLayout())));
367 SignBit = DAG.getNode(ISD::TRUNCATE, dl, LVT, SignBit);
368 } else if (SizeDiff < 0) {
369 SignBit = DAG.getNode(ISD::ANY_EXTEND, dl, LVT, SignBit);
370 SignBit =
371 DAG.getNode(ISD::SHL, dl, LVT, SignBit,
372 DAG.getConstant(-SizeDiff, dl,
373 TLI.getShiftAmountTy(SignBit.getValueType(),
374 DAG.getDataLayout())));
375 }
376
377 // Clear the sign bit of the first operand.
378 SDValue Mask = DAG.getNode(
379 ISD::SHL, dl, LVT, DAG.getConstant(1, dl, LVT),
380 DAG.getConstant(LSize - 1, dl,
381 TLI.getShiftAmountTy(LVT, DAG.getDataLayout())));
382 Mask = DAG.getNode(ISD::SUB, dl, LVT, Mask, DAG.getConstant(1, dl, LVT));
383 LHS = DAG.getNode(ISD::AND, dl, LVT, LHS, Mask);
384
385 // Or the value with the sign bit.
386 return DAG.getNode(ISD::OR, dl, LVT, LHS, SignBit);
387}
388
389SDValue DAGTypeLegalizer::SoftenFloatRes_FCOS(SDNode *N) {
390 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
391 RTLIB::COS_F32,
392 RTLIB::COS_F64,
393 RTLIB::COS_F80,
394 RTLIB::COS_F128,
395 RTLIB::COS_PPCF128));
396}
397
398SDValue DAGTypeLegalizer::SoftenFloatRes_FDIV(SDNode *N) {
399 return SoftenFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
400 RTLIB::DIV_F32,
401 RTLIB::DIV_F64,
402 RTLIB::DIV_F80,
403 RTLIB::DIV_F128,
404 RTLIB::DIV_PPCF128));
405}
406
407SDValue DAGTypeLegalizer::SoftenFloatRes_FEXP(SDNode *N) {
408 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
409 RTLIB::EXP_F32,
410 RTLIB::EXP_F64,
411 RTLIB::EXP_F80,
412 RTLIB::EXP_F128,
413 RTLIB::EXP_PPCF128));
414}
415
416SDValue DAGTypeLegalizer::SoftenFloatRes_FEXP2(SDNode *N) {
417 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
418 RTLIB::EXP2_F32,
419 RTLIB::EXP2_F64,
420 RTLIB::EXP2_F80,
421 RTLIB::EXP2_F128,
422 RTLIB::EXP2_PPCF128));
423}
424
425SDValue DAGTypeLegalizer::SoftenFloatRes_FEXP10(SDNode *N) {
426 return SoftenFloatRes_Unary(
427 N,
428 GetFPLibCall(N->getValueType(0), RTLIB::EXP10_F32, RTLIB::EXP10_F64,
429 RTLIB::EXP10_F80, RTLIB::EXP10_F128, RTLIB::EXP10_PPCF128));
430}
431
432SDValue DAGTypeLegalizer::SoftenFloatRes_FFLOOR(SDNode *N) {
433 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
434 RTLIB::FLOOR_F32,
435 RTLIB::FLOOR_F64,
436 RTLIB::FLOOR_F80,
437 RTLIB::FLOOR_F128,
438 RTLIB::FLOOR_PPCF128));
439}
440
441SDValue DAGTypeLegalizer::SoftenFloatRes_FLOG(SDNode *N) {
442 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
443 RTLIB::LOG_F32,
444 RTLIB::LOG_F64,
445 RTLIB::LOG_F80,
446 RTLIB::LOG_F128,
447 RTLIB::LOG_PPCF128));
448}
449
450SDValue DAGTypeLegalizer::SoftenFloatRes_FLOG2(SDNode *N) {
451 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
452 RTLIB::LOG2_F32,
453 RTLIB::LOG2_F64,
454 RTLIB::LOG2_F80,
455 RTLIB::LOG2_F128,
456 RTLIB::LOG2_PPCF128));
457}
458
459SDValue DAGTypeLegalizer::SoftenFloatRes_FLOG10(SDNode *N) {
460 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
461 RTLIB::LOG10_F32,
462 RTLIB::LOG10_F64,
463 RTLIB::LOG10_F80,
464 RTLIB::LOG10_F128,
465 RTLIB::LOG10_PPCF128));
466}
467
468SDValue DAGTypeLegalizer::SoftenFloatRes_FMA(SDNode *N) {
469 bool IsStrict = N->isStrictFPOpcode();
470 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
471 unsigned Offset = IsStrict ? 1 : 0;
472 SDValue Ops[3] = { GetSoftenedFloat(N->getOperand(0 + Offset)),
473 GetSoftenedFloat(N->getOperand(1 + Offset)),
474 GetSoftenedFloat(N->getOperand(2 + Offset)) };
475 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
477 EVT OpsVT[3] = { N->getOperand(0 + Offset).getValueType(),
478 N->getOperand(1 + Offset).getValueType(),
479 N->getOperand(2 + Offset).getValueType() };
480 CallOptions.setTypeListBeforeSoften(OpsVT, N->getValueType(0), true);
481 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG,
482 GetFPLibCall(N->getValueType(0),
483 RTLIB::FMA_F32,
484 RTLIB::FMA_F64,
485 RTLIB::FMA_F80,
486 RTLIB::FMA_F128,
487 RTLIB::FMA_PPCF128),
488 NVT, Ops, CallOptions, SDLoc(N), Chain);
489 if (IsStrict)
490 ReplaceValueWith(SDValue(N, 1), Tmp.second);
491 return Tmp.first;
492}
493
494SDValue DAGTypeLegalizer::SoftenFloatRes_FMUL(SDNode *N) {
495 return SoftenFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
496 RTLIB::MUL_F32,
497 RTLIB::MUL_F64,
498 RTLIB::MUL_F80,
499 RTLIB::MUL_F128,
500 RTLIB::MUL_PPCF128));
501}
502
503SDValue DAGTypeLegalizer::SoftenFloatRes_FNEARBYINT(SDNode *N) {
504 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
505 RTLIB::NEARBYINT_F32,
506 RTLIB::NEARBYINT_F64,
507 RTLIB::NEARBYINT_F80,
508 RTLIB::NEARBYINT_F128,
509 RTLIB::NEARBYINT_PPCF128));
510}
511
512SDValue DAGTypeLegalizer::SoftenFloatRes_FNEG(SDNode *N) {
513 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
514 SDLoc dl(N);
515
516 // Expand Y = FNEG(X) -> Y = X ^ sign mask
517 APInt SignMask = APInt::getSignMask(NVT.getSizeInBits());
518 return DAG.getNode(ISD::XOR, dl, NVT, GetSoftenedFloat(N->getOperand(0)),
519 DAG.getConstant(SignMask, dl, NVT));
520}
521
522SDValue DAGTypeLegalizer::SoftenFloatRes_FP_EXTEND(SDNode *N) {
523 bool IsStrict = N->isStrictFPOpcode();
524 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
525 SDValue Op = N->getOperand(IsStrict ? 1 : 0);
526
527 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
528
529 if (getTypeAction(Op.getValueType()) == TargetLowering::TypePromoteFloat) {
530 Op = GetPromotedFloat(Op);
531 // If the promotion did the FP_EXTEND to the destination type for us,
532 // there's nothing left to do here.
533 if (Op.getValueType() == N->getValueType(0)) {
534 if (IsStrict)
535 ReplaceValueWith(SDValue(N, 1), Chain);
536 return BitConvertToInteger(Op);
537 }
538 }
539
540 // There's only a libcall for f16 -> f32 and shifting is only valid for bf16
541 // -> f32, so proceed in two stages. Also, it's entirely possible for both
542 // f16 and f32 to be legal, so use the fully hard-float FP_EXTEND rather
543 // than FP16_TO_FP.
544 if ((Op.getValueType() == MVT::f16 || Op.getValueType() == MVT::bf16) &&
545 N->getValueType(0) != MVT::f32) {
546 if (IsStrict) {
548 { MVT::f32, MVT::Other }, { Chain, Op });
549 Chain = Op.getValue(1);
550 } else {
551 Op = DAG.getNode(ISD::FP_EXTEND, SDLoc(N), MVT::f32, Op);
552 }
553 }
554
555 if (Op.getValueType() == MVT::bf16) {
556 // FIXME: Need ReplaceValueWith on chain in strict case
557 return SoftenFloatRes_BF16_TO_FP(N);
558 }
559
560 RTLIB::Libcall LC = RTLIB::getFPEXT(Op.getValueType(), N->getValueType(0));
561 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported FP_EXTEND!");
563 EVT OpVT = N->getOperand(IsStrict ? 1 : 0).getValueType();
564 CallOptions.setTypeListBeforeSoften(OpVT, N->getValueType(0), true);
565 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, NVT, Op,
566 CallOptions, SDLoc(N),
567 Chain);
568 if (IsStrict)
569 ReplaceValueWith(SDValue(N, 1), Tmp.second);
570 return Tmp.first;
571}
572
573// FIXME: Should we just use 'normal' FP_EXTEND / FP_TRUNC instead of special
574// nodes?
575SDValue DAGTypeLegalizer::SoftenFloatRes_FP16_TO_FP(SDNode *N) {
576 EVT MidVT = TLI.getTypeToTransformTo(*DAG.getContext(), MVT::f32);
577 SDValue Op = N->getOperand(0);
579 EVT OpsVT[1] = { N->getOperand(0).getValueType() };
580 CallOptions.setTypeListBeforeSoften(OpsVT, N->getValueType(0), true);
581 SDValue Res32 = TLI.makeLibCall(DAG, RTLIB::FPEXT_F16_F32, MidVT, Op,
582 CallOptions, SDLoc(N)).first;
583 if (N->getValueType(0) == MVT::f32)
584 return Res32;
585
586 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
587 RTLIB::Libcall LC = RTLIB::getFPEXT(MVT::f32, N->getValueType(0));
588 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported FP_EXTEND!");
589 return TLI.makeLibCall(DAG, LC, NVT, Res32, CallOptions, SDLoc(N)).first;
590}
591
592// FIXME: Should we just use 'normal' FP_EXTEND / FP_TRUNC instead of special
593// nodes?
594SDValue DAGTypeLegalizer::SoftenFloatRes_BF16_TO_FP(SDNode *N) {
595 assert(N->getValueType(0) == MVT::f32 &&
596 "Can only soften BF16_TO_FP with f32 result");
597 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), MVT::f32);
598 SDValue Op = N->getOperand(0);
599 SDLoc DL(N);
600 Op = DAG.getNode(ISD::ANY_EXTEND, DL, NVT,
601 DAG.getNode(ISD::BITCAST, DL, MVT::i16, Op));
602 SDValue Res = DAG.getNode(ISD::SHL, DL, NVT, Op,
603 DAG.getShiftAmountConstant(16, NVT, DL));
604 return Res;
605}
606
607SDValue DAGTypeLegalizer::SoftenFloatRes_FP_ROUND(SDNode *N) {
608 bool IsStrict = N->isStrictFPOpcode();
609 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
610 SDValue Op = N->getOperand(IsStrict ? 1 : 0);
611 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
612 RTLIB::Libcall LC = RTLIB::getFPROUND(Op.getValueType(), N->getValueType(0));
613 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported FP_ROUND!");
615 EVT OpVT = N->getOperand(IsStrict ? 1 : 0).getValueType();
616 CallOptions.setTypeListBeforeSoften(OpVT, N->getValueType(0), true);
617 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, NVT, Op,
618 CallOptions, SDLoc(N),
619 Chain);
620 if (IsStrict)
621 ReplaceValueWith(SDValue(N, 1), Tmp.second);
622 return Tmp.first;
623}
624
625SDValue DAGTypeLegalizer::SoftenFloatRes_FPOW(SDNode *N) {
626 return SoftenFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
627 RTLIB::POW_F32,
628 RTLIB::POW_F64,
629 RTLIB::POW_F80,
630 RTLIB::POW_F128,
631 RTLIB::POW_PPCF128));
632}
633
634SDValue DAGTypeLegalizer::SoftenFloatRes_ExpOp(SDNode *N) {
635 bool IsStrict = N->isStrictFPOpcode();
636 unsigned Offset = IsStrict ? 1 : 0;
637 assert((N->getOperand(1 + Offset).getValueType() == MVT::i16 ||
638 N->getOperand(1 + Offset).getValueType() == MVT::i32) &&
639 "Unsupported power type!");
640 bool IsPowI =
641 N->getOpcode() == ISD::FPOWI || N->getOpcode() == ISD::STRICT_FPOWI;
642
643 RTLIB::Libcall LC = IsPowI ? RTLIB::getPOWI(N->getValueType(0))
644 : RTLIB::getLDEXP(N->getValueType(0));
645 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected fpowi.");
646 if (!TLI.getLibcallName(LC)) {
647 // Some targets don't have a powi libcall; use pow instead.
648 // FIXME: Implement this if some target needs it.
649 DAG.getContext()->emitError("Don't know how to soften fpowi to fpow");
650 return DAG.getUNDEF(N->getValueType(0));
651 }
652
653 if (DAG.getLibInfo().getIntSize() !=
654 N->getOperand(1 + Offset).getValueType().getSizeInBits()) {
655 // If the exponent does not match with sizeof(int) a libcall to RTLIB::POWI
656 // would use the wrong type for the argument.
657 DAG.getContext()->emitError("POWI exponent does not match sizeof(int)");
658 return DAG.getUNDEF(N->getValueType(0));
659 }
660
661 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
662 SDValue Ops[2] = { GetSoftenedFloat(N->getOperand(0 + Offset)),
663 N->getOperand(1 + Offset) };
664 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
666 EVT OpsVT[2] = { N->getOperand(0 + Offset).getValueType(),
667 N->getOperand(1 + Offset).getValueType() };
668 CallOptions.setTypeListBeforeSoften(OpsVT, N->getValueType(0), true);
669 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, NVT, Ops,
670 CallOptions, SDLoc(N),
671 Chain);
672 if (IsStrict)
673 ReplaceValueWith(SDValue(N, 1), Tmp.second);
674 return Tmp.first;
675}
676
677SDValue DAGTypeLegalizer::SoftenFloatRes_FFREXP(SDNode *N) {
678 assert(!N->isStrictFPOpcode() && "strictfp not implemented for frexp");
679 EVT VT0 = N->getValueType(0);
680 EVT VT1 = N->getValueType(1);
682
683 if (DAG.getLibInfo().getIntSize() != VT1.getSizeInBits()) {
684 // If the exponent does not match with sizeof(int) a libcall would use the
685 // wrong type for the argument.
686 // TODO: Should be able to handle mismatches.
687 DAG.getContext()->emitError("ffrexp exponent does not match sizeof(int)");
688 return DAG.getUNDEF(N->getValueType(0));
689 }
690
691 EVT NVT0 = TLI.getTypeToTransformTo(*DAG.getContext(), VT0);
692 SDValue StackSlot = DAG.CreateStackTemporary(VT1);
693
694 SDLoc DL(N);
695
697 SDValue Ops[2] = {GetSoftenedFloat(N->getOperand(0)), StackSlot};
698 EVT OpsVT[2] = {VT0, StackSlot.getValueType()};
699
700 // TODO: setTypeListBeforeSoften can't properly express multiple return types,
701 // but we only really need to handle the 0th one for softening anyway.
702 CallOptions.setTypeListBeforeSoften({OpsVT}, VT0, true);
703
704 auto [ReturnVal, Chain] = TLI.makeLibCall(DAG, LC, NVT0, Ops, CallOptions, DL,
705 /*Chain=*/SDValue());
706 int FrameIdx = cast<FrameIndexSDNode>(StackSlot)->getIndex();
707 auto PtrInfo =
709
710 SDValue LoadExp = DAG.getLoad(VT1, DL, Chain, StackSlot, PtrInfo);
711
712 ReplaceValueWith(SDValue(N, 1), LoadExp);
713 return ReturnVal;
714}
715
716SDValue DAGTypeLegalizer::SoftenFloatRes_FREM(SDNode *N) {
717 return SoftenFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
718 RTLIB::REM_F32,
719 RTLIB::REM_F64,
720 RTLIB::REM_F80,
721 RTLIB::REM_F128,
722 RTLIB::REM_PPCF128));
723}
724
725SDValue DAGTypeLegalizer::SoftenFloatRes_FRINT(SDNode *N) {
726 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
727 RTLIB::RINT_F32,
728 RTLIB::RINT_F64,
729 RTLIB::RINT_F80,
730 RTLIB::RINT_F128,
731 RTLIB::RINT_PPCF128));
732}
733
734SDValue DAGTypeLegalizer::SoftenFloatRes_FROUND(SDNode *N) {
735 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
736 RTLIB::ROUND_F32,
737 RTLIB::ROUND_F64,
738 RTLIB::ROUND_F80,
739 RTLIB::ROUND_F128,
740 RTLIB::ROUND_PPCF128));
741}
742
743SDValue DAGTypeLegalizer::SoftenFloatRes_FROUNDEVEN(SDNode *N) {
744 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
745 RTLIB::ROUNDEVEN_F32,
746 RTLIB::ROUNDEVEN_F64,
747 RTLIB::ROUNDEVEN_F80,
748 RTLIB::ROUNDEVEN_F128,
749 RTLIB::ROUNDEVEN_PPCF128));
750}
751
752SDValue DAGTypeLegalizer::SoftenFloatRes_FSIN(SDNode *N) {
753 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
754 RTLIB::SIN_F32,
755 RTLIB::SIN_F64,
756 RTLIB::SIN_F80,
757 RTLIB::SIN_F128,
758 RTLIB::SIN_PPCF128));
759}
760
761SDValue DAGTypeLegalizer::SoftenFloatRes_FSQRT(SDNode *N) {
762 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
763 RTLIB::SQRT_F32,
764 RTLIB::SQRT_F64,
765 RTLIB::SQRT_F80,
766 RTLIB::SQRT_F128,
767 RTLIB::SQRT_PPCF128));
768}
769
770SDValue DAGTypeLegalizer::SoftenFloatRes_FSUB(SDNode *N) {
771 return SoftenFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
772 RTLIB::SUB_F32,
773 RTLIB::SUB_F64,
774 RTLIB::SUB_F80,
775 RTLIB::SUB_F128,
776 RTLIB::SUB_PPCF128));
777}
778
779SDValue DAGTypeLegalizer::SoftenFloatRes_FTAN(SDNode *N) {
780 return SoftenFloatRes_Unary(
781 N, GetFPLibCall(N->getValueType(0), RTLIB::TAN_F32, RTLIB::TAN_F64,
782 RTLIB::TAN_F80, RTLIB::TAN_F128, RTLIB::TAN_PPCF128));
783}
784
785SDValue DAGTypeLegalizer::SoftenFloatRes_FTRUNC(SDNode *N) {
786 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
787 RTLIB::TRUNC_F32,
788 RTLIB::TRUNC_F64,
789 RTLIB::TRUNC_F80,
790 RTLIB::TRUNC_F128,
791 RTLIB::TRUNC_PPCF128));
792}
793
794SDValue DAGTypeLegalizer::SoftenFloatRes_LOAD(SDNode *N) {
795 LoadSDNode *L = cast<LoadSDNode>(N);
796 EVT VT = N->getValueType(0);
797 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
798 SDLoc dl(N);
799
800 auto MMOFlags =
801 L->getMemOperand()->getFlags() &
803 SDValue NewL;
804 if (L->getExtensionType() == ISD::NON_EXTLOAD) {
805 NewL = DAG.getLoad(L->getAddressingMode(), L->getExtensionType(), NVT, dl,
806 L->getChain(), L->getBasePtr(), L->getOffset(),
807 L->getPointerInfo(), NVT, L->getOriginalAlign(),
808 MMOFlags, L->getAAInfo());
809 // Legalized the chain result - switch anything that used the old chain to
810 // use the new one.
811 ReplaceValueWith(SDValue(N, 1), NewL.getValue(1));
812 return NewL;
813 }
814
815 // Do a non-extending load followed by FP_EXTEND.
816 NewL = DAG.getLoad(L->getAddressingMode(), ISD::NON_EXTLOAD, L->getMemoryVT(),
817 dl, L->getChain(), L->getBasePtr(), L->getOffset(),
818 L->getPointerInfo(), L->getMemoryVT(),
819 L->getOriginalAlign(), MMOFlags, L->getAAInfo());
820 // Legalized the chain result - switch anything that used the old chain to
821 // use the new one.
822 ReplaceValueWith(SDValue(N, 1), NewL.getValue(1));
823 auto ExtendNode = DAG.getNode(ISD::FP_EXTEND, dl, VT, NewL);
824 return BitConvertToInteger(ExtendNode);
825}
826
827SDValue DAGTypeLegalizer::SoftenFloatRes_ATOMIC_LOAD(SDNode *N) {
828 AtomicSDNode *L = cast<AtomicSDNode>(N);
829 EVT VT = N->getValueType(0);
830 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
831 SDLoc dl(N);
832
833 if (L->getExtensionType() == ISD::NON_EXTLOAD) {
834 SDValue NewL =
835 DAG.getAtomic(ISD::ATOMIC_LOAD, dl, NVT, DAG.getVTList(NVT, MVT::Other),
836 {L->getChain(), L->getBasePtr()}, L->getMemOperand());
837
838 // Legalized the chain result - switch anything that used the old chain to
839 // use the new one.
840 ReplaceValueWith(SDValue(N, 1), NewL.getValue(1));
841 return NewL;
842 }
843
844 report_fatal_error("softening fp extending atomic load not handled");
845}
846
847SDValue DAGTypeLegalizer::SoftenFloatRes_SELECT(SDNode *N) {
848 SDValue LHS = GetSoftenedFloat(N->getOperand(1));
849 SDValue RHS = GetSoftenedFloat(N->getOperand(2));
850 return DAG.getSelect(SDLoc(N),
851 LHS.getValueType(), N->getOperand(0), LHS, RHS);
852}
853
854SDValue DAGTypeLegalizer::SoftenFloatRes_SELECT_CC(SDNode *N) {
855 SDValue LHS = GetSoftenedFloat(N->getOperand(2));
856 SDValue RHS = GetSoftenedFloat(N->getOperand(3));
857 return DAG.getNode(ISD::SELECT_CC, SDLoc(N),
858 LHS.getValueType(), N->getOperand(0),
859 N->getOperand(1), LHS, RHS, N->getOperand(4));
860}
861
862SDValue DAGTypeLegalizer::SoftenFloatRes_UNDEF(SDNode *N) {
863 return DAG.getUNDEF(TLI.getTypeToTransformTo(*DAG.getContext(),
864 N->getValueType(0)));
865}
866
867SDValue DAGTypeLegalizer::SoftenFloatRes_VAARG(SDNode *N) {
868 SDValue Chain = N->getOperand(0); // Get the chain.
869 SDValue Ptr = N->getOperand(1); // Get the pointer.
870 EVT VT = N->getValueType(0);
871 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
872 SDLoc dl(N);
873
874 SDValue NewVAARG;
875 NewVAARG = DAG.getVAArg(NVT, dl, Chain, Ptr, N->getOperand(2),
876 N->getConstantOperandVal(3));
877
878 // Legalized the chain result - switch anything that used the old chain to
879 // use the new one.
880 if (N != NewVAARG.getValue(1).getNode())
881 ReplaceValueWith(SDValue(N, 1), NewVAARG.getValue(1));
882 return NewVAARG;
883}
884
885SDValue DAGTypeLegalizer::SoftenFloatRes_XINT_TO_FP(SDNode *N) {
886 bool IsStrict = N->isStrictFPOpcode();
887 bool Signed = N->getOpcode() == ISD::SINT_TO_FP ||
888 N->getOpcode() == ISD::STRICT_SINT_TO_FP;
889 EVT SVT = N->getOperand(IsStrict ? 1 : 0).getValueType();
890 EVT RVT = N->getValueType(0);
891 EVT NVT = EVT();
892 SDLoc dl(N);
893
894 // If the input is not legal, eg: i1 -> fp, then it needs to be promoted to
895 // a larger type, eg: i8 -> fp. Even if it is legal, no libcall may exactly
896 // match. Look for an appropriate libcall.
897 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
898 for (unsigned t = MVT::FIRST_INTEGER_VALUETYPE;
899 t <= MVT::LAST_INTEGER_VALUETYPE && LC == RTLIB::UNKNOWN_LIBCALL; ++t) {
900 NVT = (MVT::SimpleValueType)t;
901 // The source needs to big enough to hold the operand.
902 if (NVT.bitsGE(SVT))
903 LC = Signed ? RTLIB::getSINTTOFP(NVT, RVT):RTLIB::getUINTTOFP (NVT, RVT);
904 }
905 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported XINT_TO_FP!");
906
907 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
908 // Sign/zero extend the argument if the libcall takes a larger type.
910 NVT, N->getOperand(IsStrict ? 1 : 0));
912 CallOptions.setSExt(Signed);
913 CallOptions.setTypeListBeforeSoften(SVT, RVT, true);
914 std::pair<SDValue, SDValue> Tmp =
915 TLI.makeLibCall(DAG, LC, TLI.getTypeToTransformTo(*DAG.getContext(), RVT),
916 Op, CallOptions, dl, Chain);
917
918 if (IsStrict)
919 ReplaceValueWith(SDValue(N, 1), Tmp.second);
920 return Tmp.first;
921}
922
923SDValue DAGTypeLegalizer::SoftenFloatRes_VECREDUCE(SDNode *N) {
924 // Expand and soften recursively.
925 ReplaceValueWith(SDValue(N, 0), TLI.expandVecReduce(N, DAG));
926 return SDValue();
927}
928
929SDValue DAGTypeLegalizer::SoftenFloatRes_VECREDUCE_SEQ(SDNode *N) {
930 ReplaceValueWith(SDValue(N, 0), TLI.expandVecReduceSeq(N, DAG));
931 return SDValue();
932}
933
934//===----------------------------------------------------------------------===//
935// Convert Float Operand to Integer
936//===----------------------------------------------------------------------===//
937
938bool DAGTypeLegalizer::SoftenFloatOperand(SDNode *N, unsigned OpNo) {
939 LLVM_DEBUG(dbgs() << "Soften float operand " << OpNo << ": "; N->dump(&DAG));
940 SDValue Res = SDValue();
941
942 switch (N->getOpcode()) {
943 default:
944#ifndef NDEBUG
945 dbgs() << "SoftenFloatOperand Op #" << OpNo << ": ";
946 N->dump(&DAG); dbgs() << "\n";
947#endif
948 report_fatal_error("Do not know how to soften this operator's operand!");
949
950 case ISD::BITCAST: Res = SoftenFloatOp_BITCAST(N); break;
951 case ISD::BR_CC: Res = SoftenFloatOp_BR_CC(N); break;
953 case ISD::FP_TO_FP16: // Same as FP_ROUND for softening purposes
954 case ISD::FP_TO_BF16:
957 case ISD::FP_ROUND: Res = SoftenFloatOp_FP_ROUND(N); break;
960 case ISD::FP_TO_SINT:
961 case ISD::FP_TO_UINT: Res = SoftenFloatOp_FP_TO_XINT(N); break;
964 Res = SoftenFloatOp_FP_TO_XINT_SAT(N); break;
966 case ISD::LROUND: Res = SoftenFloatOp_LROUND(N); break;
968 case ISD::LLROUND: Res = SoftenFloatOp_LLROUND(N); break;
970 case ISD::LRINT: Res = SoftenFloatOp_LRINT(N); break;
972 case ISD::LLRINT: Res = SoftenFloatOp_LLRINT(N); break;
973 case ISD::SELECT_CC: Res = SoftenFloatOp_SELECT_CC(N); break;
976 case ISD::SETCC: Res = SoftenFloatOp_SETCC(N); break;
977 case ISD::STORE: Res = SoftenFloatOp_STORE(N, OpNo); break;
979 Res = SoftenFloatOp_ATOMIC_STORE(N, OpNo);
980 break;
981 case ISD::FCOPYSIGN: Res = SoftenFloatOp_FCOPYSIGN(N); break;
982 }
983
984 // If the result is null, the sub-method took care of registering results etc.
985 if (!Res.getNode()) return false;
986
987 // If the result is N, the sub-method updated N in place. Tell the legalizer
988 // core about this to re-analyze.
989 if (Res.getNode() == N)
990 return true;
991
992 assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 &&
993 "Invalid operand softening");
994
995 ReplaceValueWith(SDValue(N, 0), Res);
996 return false;
997}
998
999SDValue DAGTypeLegalizer::SoftenFloatOp_BITCAST(SDNode *N) {
1000 SDValue Op0 = GetSoftenedFloat(N->getOperand(0));
1001
1002 return DAG.getNode(ISD::BITCAST, SDLoc(N), N->getValueType(0), Op0);
1003}
1004
1005SDValue DAGTypeLegalizer::SoftenFloatOp_FP_ROUND(SDNode *N) {
1006 // We actually deal with the partially-softened FP_TO_FP16 node too, which
1007 // returns an i16 so doesn't meet the constraints necessary for FP_ROUND.
1008 assert(N->getOpcode() == ISD::FP_ROUND || N->getOpcode() == ISD::FP_TO_FP16 ||
1009 N->getOpcode() == ISD::STRICT_FP_TO_FP16 ||
1010 N->getOpcode() == ISD::FP_TO_BF16 ||
1011 N->getOpcode() == ISD::STRICT_FP_TO_BF16 ||
1012 N->getOpcode() == ISD::STRICT_FP_ROUND);
1013
1014 bool IsStrict = N->isStrictFPOpcode();
1015 SDValue Op = N->getOperand(IsStrict ? 1 : 0);
1016 EVT SVT = Op.getValueType();
1017 EVT RVT = N->getValueType(0);
1018 EVT FloatRVT = RVT;
1019 if (N->getOpcode() == ISD::FP_TO_FP16 ||
1020 N->getOpcode() == ISD::STRICT_FP_TO_FP16)
1021 FloatRVT = MVT::f16;
1022 else if (N->getOpcode() == ISD::FP_TO_BF16 ||
1023 N->getOpcode() == ISD::STRICT_FP_TO_BF16)
1024 FloatRVT = MVT::bf16;
1025
1026 RTLIB::Libcall LC = RTLIB::getFPROUND(SVT, FloatRVT);
1027 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported FP_ROUND libcall");
1028
1029 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
1030 Op = GetSoftenedFloat(Op);
1032 CallOptions.setTypeListBeforeSoften(SVT, RVT, true);
1033 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, RVT, Op,
1034 CallOptions, SDLoc(N),
1035 Chain);
1036 if (IsStrict) {
1037 ReplaceValueWith(SDValue(N, 1), Tmp.second);
1038 ReplaceValueWith(SDValue(N, 0), Tmp.first);
1039 return SDValue();
1040 }
1041 return Tmp.first;
1042}
1043
1044SDValue DAGTypeLegalizer::SoftenFloatOp_BR_CC(SDNode *N) {
1045 SDValue NewLHS = N->getOperand(2), NewRHS = N->getOperand(3);
1046 ISD::CondCode CCCode = cast<CondCodeSDNode>(N->getOperand(1))->get();
1047
1048 EVT VT = NewLHS.getValueType();
1049 NewLHS = GetSoftenedFloat(NewLHS);
1050 NewRHS = GetSoftenedFloat(NewRHS);
1051 TLI.softenSetCCOperands(DAG, VT, NewLHS, NewRHS, CCCode, SDLoc(N),
1052 N->getOperand(2), N->getOperand(3));
1053
1054 // If softenSetCCOperands returned a scalar, we need to compare the result
1055 // against zero to select between true and false values.
1056 if (!NewRHS.getNode()) {
1057 NewRHS = DAG.getConstant(0, SDLoc(N), NewLHS.getValueType());
1058 CCCode = ISD::SETNE;
1059 }
1060
1061 // Update N to have the operands specified.
1062 return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0),
1063 DAG.getCondCode(CCCode), NewLHS, NewRHS,
1064 N->getOperand(4)),
1065 0);
1066}
1067
1068// Even if the result type is legal, no libcall may exactly match. (e.g. We
1069// don't have FP-i8 conversions) This helper method looks for an appropriate
1070// promoted libcall.
1071static RTLIB::Libcall findFPToIntLibcall(EVT SrcVT, EVT RetVT, EVT &Promoted,
1072 bool Signed) {
1073 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
1074 for (unsigned IntVT = MVT::FIRST_INTEGER_VALUETYPE;
1075 IntVT <= MVT::LAST_INTEGER_VALUETYPE && LC == RTLIB::UNKNOWN_LIBCALL;
1076 ++IntVT) {
1077 Promoted = (MVT::SimpleValueType)IntVT;
1078 // The type needs to big enough to hold the result.
1079 if (Promoted.bitsGE(RetVT))
1080 LC = Signed ? RTLIB::getFPTOSINT(SrcVT, Promoted)
1081 : RTLIB::getFPTOUINT(SrcVT, Promoted);
1082 }
1083 return LC;
1084}
1085
1086SDValue DAGTypeLegalizer::SoftenFloatOp_FP_TO_XINT(SDNode *N) {
1087 bool IsStrict = N->isStrictFPOpcode();
1088 bool Signed = N->getOpcode() == ISD::FP_TO_SINT ||
1089 N->getOpcode() == ISD::STRICT_FP_TO_SINT;
1090
1091 SDValue Op = N->getOperand(IsStrict ? 1 : 0);
1092 EVT SVT = Op.getValueType();
1093 EVT RVT = N->getValueType(0);
1094 EVT NVT = EVT();
1095 SDLoc dl(N);
1096
1097 // If the result is not legal, eg: fp -> i1, then it needs to be promoted to
1098 // a larger type, eg: fp -> i32. Even if it is legal, no libcall may exactly
1099 // match, eg. we don't have fp -> i8 conversions.
1100 // Look for an appropriate libcall.
1101 RTLIB::Libcall LC = findFPToIntLibcall(SVT, RVT, NVT, Signed);
1102 assert(LC != RTLIB::UNKNOWN_LIBCALL && NVT.isSimple() &&
1103 "Unsupported FP_TO_XINT!");
1104
1105 Op = GetSoftenedFloat(Op);
1106 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
1108 CallOptions.setTypeListBeforeSoften(SVT, RVT, true);
1109 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, NVT, Op,
1110 CallOptions, dl, Chain);
1111
1112 // Truncate the result if the libcall returns a larger type.
1113 SDValue Res = DAG.getNode(ISD::TRUNCATE, dl, RVT, Tmp.first);
1114
1115 if (!IsStrict)
1116 return Res;
1117
1118 ReplaceValueWith(SDValue(N, 1), Tmp.second);
1119 ReplaceValueWith(SDValue(N, 0), Res);
1120 return SDValue();
1121}
1122
1123SDValue DAGTypeLegalizer::SoftenFloatOp_FP_TO_XINT_SAT(SDNode *N) {
1124 SDValue Res = TLI.expandFP_TO_INT_SAT(N, DAG);
1125 return Res;
1126}
1127
1128SDValue DAGTypeLegalizer::SoftenFloatOp_SELECT_CC(SDNode *N) {
1129 SDValue NewLHS = N->getOperand(0), NewRHS = N->getOperand(1);
1130 ISD::CondCode CCCode = cast<CondCodeSDNode>(N->getOperand(4))->get();
1131
1132 EVT VT = NewLHS.getValueType();
1133 NewLHS = GetSoftenedFloat(NewLHS);
1134 NewRHS = GetSoftenedFloat(NewRHS);
1135 TLI.softenSetCCOperands(DAG, VT, NewLHS, NewRHS, CCCode, SDLoc(N),
1136 N->getOperand(0), N->getOperand(1));
1137
1138 // If softenSetCCOperands returned a scalar, we need to compare the result
1139 // against zero to select between true and false values.
1140 if (!NewRHS.getNode()) {
1141 NewRHS = DAG.getConstant(0, SDLoc(N), NewLHS.getValueType());
1142 CCCode = ISD::SETNE;
1143 }
1144
1145 // Update N to have the operands specified.
1146 return SDValue(DAG.UpdateNodeOperands(N, NewLHS, NewRHS,
1147 N->getOperand(2), N->getOperand(3),
1148 DAG.getCondCode(CCCode)),
1149 0);
1150}
1151
1152SDValue DAGTypeLegalizer::SoftenFloatOp_SETCC(SDNode *N) {
1153 bool IsStrict = N->isStrictFPOpcode();
1154 SDValue Op0 = N->getOperand(IsStrict ? 1 : 0);
1155 SDValue Op1 = N->getOperand(IsStrict ? 2 : 1);
1156 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
1157 ISD::CondCode CCCode =
1158 cast<CondCodeSDNode>(N->getOperand(IsStrict ? 3 : 2))->get();
1159
1160 EVT VT = Op0.getValueType();
1161 SDValue NewLHS = GetSoftenedFloat(Op0);
1162 SDValue NewRHS = GetSoftenedFloat(Op1);
1163 TLI.softenSetCCOperands(DAG, VT, NewLHS, NewRHS, CCCode, SDLoc(N), Op0, Op1,
1164 Chain, N->getOpcode() == ISD::STRICT_FSETCCS);
1165
1166 // Update N to have the operands specified.
1167 if (NewRHS.getNode()) {
1168 if (IsStrict)
1169 NewLHS = DAG.getNode(ISD::SETCC, SDLoc(N), N->getValueType(0), NewLHS,
1170 NewRHS, DAG.getCondCode(CCCode));
1171 else
1172 return SDValue(DAG.UpdateNodeOperands(N, NewLHS, NewRHS,
1173 DAG.getCondCode(CCCode)), 0);
1174 }
1175
1176 // Otherwise, softenSetCCOperands returned a scalar, use it.
1177 assert((NewRHS.getNode() || NewLHS.getValueType() == N->getValueType(0)) &&
1178 "Unexpected setcc expansion!");
1179
1180 if (IsStrict) {
1181 ReplaceValueWith(SDValue(N, 0), NewLHS);
1182 ReplaceValueWith(SDValue(N, 1), Chain);
1183 return SDValue();
1184 }
1185 return NewLHS;
1186}
1187
1188SDValue DAGTypeLegalizer::SoftenFloatOp_STORE(SDNode *N, unsigned OpNo) {
1189 assert(ISD::isUNINDEXEDStore(N) && "Indexed store during type legalization!");
1190 assert(OpNo == 1 && "Can only soften the stored value!");
1191 StoreSDNode *ST = cast<StoreSDNode>(N);
1192 SDValue Val = ST->getValue();
1193 SDLoc dl(N);
1194
1195 if (ST->isTruncatingStore())
1196 // Do an FP_ROUND followed by a non-truncating store.
1197 Val = BitConvertToInteger(
1198 DAG.getNode(ISD::FP_ROUND, dl, ST->getMemoryVT(), Val,
1199 DAG.getIntPtrConstant(0, dl, /*isTarget=*/true)));
1200 else
1201 Val = GetSoftenedFloat(Val);
1202
1203 return DAG.getStore(ST->getChain(), dl, Val, ST->getBasePtr(),
1204 ST->getMemOperand());
1205}
1206
1207SDValue DAGTypeLegalizer::SoftenFloatOp_ATOMIC_STORE(SDNode *N, unsigned OpNo) {
1208 assert(OpNo == 1 && "Can only soften the stored value!");
1209 AtomicSDNode *ST = cast<AtomicSDNode>(N);
1210 SDValue Val = ST->getVal();
1211 EVT VT = Val.getValueType();
1212 SDLoc dl(N);
1213
1214 assert(ST->getMemoryVT() == VT && "truncating atomic store not handled");
1215
1216 SDValue NewVal = GetSoftenedFloat(Val);
1217 return DAG.getAtomic(ISD::ATOMIC_STORE, dl, VT, ST->getChain(), NewVal,
1218 ST->getBasePtr(), ST->getMemOperand());
1219}
1220
1221SDValue DAGTypeLegalizer::SoftenFloatOp_FCOPYSIGN(SDNode *N) {
1222 SDValue LHS = N->getOperand(0);
1223 SDValue RHS = BitConvertToInteger(N->getOperand(1));
1224 SDLoc dl(N);
1225
1226 EVT LVT = LHS.getValueType();
1227 EVT ILVT = EVT::getIntegerVT(*DAG.getContext(), LVT.getSizeInBits());
1228 EVT RVT = RHS.getValueType();
1229
1230 unsigned LSize = LVT.getSizeInBits();
1231 unsigned RSize = RVT.getSizeInBits();
1232
1233 // Shift right or sign-extend it if the two operands have different types.
1234 int SizeDiff = RSize - LSize;
1235 if (SizeDiff > 0) {
1236 RHS =
1237 DAG.getNode(ISD::SRL, dl, RVT, RHS,
1238 DAG.getConstant(SizeDiff, dl,
1239 TLI.getShiftAmountTy(RHS.getValueType(),
1240 DAG.getDataLayout())));
1241 RHS = DAG.getNode(ISD::TRUNCATE, dl, ILVT, RHS);
1242 } else if (SizeDiff < 0) {
1243 RHS = DAG.getNode(ISD::ANY_EXTEND, dl, LVT, RHS);
1244 RHS =
1245 DAG.getNode(ISD::SHL, dl, ILVT, RHS,
1246 DAG.getConstant(-SizeDiff, dl,
1247 TLI.getShiftAmountTy(RHS.getValueType(),
1248 DAG.getDataLayout())));
1249 }
1250
1251 RHS = DAG.getBitcast(LVT, RHS);
1252 return DAG.getNode(ISD::FCOPYSIGN, dl, LVT, LHS, RHS);
1253}
1254
1255SDValue DAGTypeLegalizer::SoftenFloatOp_Unary(SDNode *N, RTLIB::Libcall LC) {
1256 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
1257 bool IsStrict = N->isStrictFPOpcode();
1258 unsigned Offset = IsStrict ? 1 : 0;
1259 SDValue Op = GetSoftenedFloat(N->getOperand(0 + Offset));
1260 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
1262 EVT OpVT = N->getOperand(0 + Offset).getValueType();
1263 CallOptions.setTypeListBeforeSoften(OpVT, N->getValueType(0), true);
1264 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, NVT, Op,
1265 CallOptions, SDLoc(N),
1266 Chain);
1267 if (IsStrict) {
1268 ReplaceValueWith(SDValue(N, 1), Tmp.second);
1269 ReplaceValueWith(SDValue(N, 0), Tmp.first);
1270 return SDValue();
1271 }
1272
1273 return Tmp.first;
1274}
1275
1276SDValue DAGTypeLegalizer::SoftenFloatOp_LROUND(SDNode *N) {
1277 EVT OpVT = N->getOperand(N->isStrictFPOpcode() ? 1 : 0).getValueType();
1278 return SoftenFloatOp_Unary(N, GetFPLibCall(OpVT,
1279 RTLIB::LROUND_F32,
1280 RTLIB::LROUND_F64,
1281 RTLIB::LROUND_F80,
1282 RTLIB::LROUND_F128,
1283 RTLIB::LROUND_PPCF128));
1284}
1285
1286SDValue DAGTypeLegalizer::SoftenFloatOp_LLROUND(SDNode *N) {
1287 EVT OpVT = N->getOperand(N->isStrictFPOpcode() ? 1 : 0).getValueType();
1288 return SoftenFloatOp_Unary(N, GetFPLibCall(OpVT,
1289 RTLIB::LLROUND_F32,
1290 RTLIB::LLROUND_F64,
1291 RTLIB::LLROUND_F80,
1292 RTLIB::LLROUND_F128,
1293 RTLIB::LLROUND_PPCF128));
1294}
1295
1296SDValue DAGTypeLegalizer::SoftenFloatOp_LRINT(SDNode *N) {
1297 EVT OpVT = N->getOperand(N->isStrictFPOpcode() ? 1 : 0).getValueType();
1298 return SoftenFloatOp_Unary(N, GetFPLibCall(OpVT,
1299 RTLIB::LRINT_F32,
1300 RTLIB::LRINT_F64,
1301 RTLIB::LRINT_F80,
1302 RTLIB::LRINT_F128,
1303 RTLIB::LRINT_PPCF128));
1304}
1305
1306SDValue DAGTypeLegalizer::SoftenFloatOp_LLRINT(SDNode *N) {
1307 EVT OpVT = N->getOperand(N->isStrictFPOpcode() ? 1 : 0).getValueType();
1308 return SoftenFloatOp_Unary(N, GetFPLibCall(OpVT,
1309 RTLIB::LLRINT_F32,
1310 RTLIB::LLRINT_F64,
1311 RTLIB::LLRINT_F80,
1312 RTLIB::LLRINT_F128,
1313 RTLIB::LLRINT_PPCF128));
1314}
1315
1316//===----------------------------------------------------------------------===//
1317// Float Result Expansion
1318//===----------------------------------------------------------------------===//
1319
1320/// ExpandFloatResult - This method is called when the specified result of the
1321/// specified node is found to need expansion. At this point, the node may also
1322/// have invalid operands or may have other results that need promotion, we just
1323/// know that (at least) one result needs expansion.
1324void DAGTypeLegalizer::ExpandFloatResult(SDNode *N, unsigned ResNo) {
1325 LLVM_DEBUG(dbgs() << "Expand float result: "; N->dump(&DAG));
1326 SDValue Lo, Hi;
1327 Lo = Hi = SDValue();
1328
1329 // See if the target wants to custom expand this node.
1330 if (CustomLowerNode(N, N->getValueType(ResNo), true))
1331 return;
1332
1333 switch (N->getOpcode()) {
1334 default:
1335#ifndef NDEBUG
1336 dbgs() << "ExpandFloatResult #" << ResNo << ": ";
1337 N->dump(&DAG); dbgs() << "\n";
1338#endif
1339 report_fatal_error("Do not know how to expand the result of this "
1340 "operator!");
1341 // clang-format off
1342 case ISD::UNDEF: SplitRes_UNDEF(N, Lo, Hi); break;
1343 case ISD::SELECT: SplitRes_Select(N, Lo, Hi); break;
1344 case ISD::SELECT_CC: SplitRes_SELECT_CC(N, Lo, Hi); break;
1345
1346 case ISD::MERGE_VALUES: ExpandRes_MERGE_VALUES(N, ResNo, Lo, Hi); break;
1347 case ISD::BITCAST: ExpandRes_BITCAST(N, Lo, Hi); break;
1348 case ISD::BUILD_PAIR: ExpandRes_BUILD_PAIR(N, Lo, Hi); break;
1349 case ISD::EXTRACT_ELEMENT: ExpandRes_EXTRACT_ELEMENT(N, Lo, Hi); break;
1350 case ISD::EXTRACT_VECTOR_ELT: ExpandRes_EXTRACT_VECTOR_ELT(N, Lo, Hi); break;
1351 case ISD::VAARG: ExpandRes_VAARG(N, Lo, Hi); break;
1352
1353 case ISD::ConstantFP: ExpandFloatRes_ConstantFP(N, Lo, Hi); break;
1354 case ISD::FABS: ExpandFloatRes_FABS(N, Lo, Hi); break;
1356 case ISD::FMINNUM: ExpandFloatRes_FMINNUM(N, Lo, Hi); break;
1358 case ISD::FMAXNUM: ExpandFloatRes_FMAXNUM(N, Lo, Hi); break;
1359 case ISD::STRICT_FADD:
1360 case ISD::FADD: ExpandFloatRes_FADD(N, Lo, Hi); break;
1361 case ISD::FCBRT: ExpandFloatRes_FCBRT(N, Lo, Hi); break;
1362 case ISD::STRICT_FCEIL:
1363 case ISD::FCEIL: ExpandFloatRes_FCEIL(N, Lo, Hi); break;
1364 case ISD::FCOPYSIGN: ExpandFloatRes_FCOPYSIGN(N, Lo, Hi); break;
1365 case ISD::STRICT_FCOS:
1366 case ISD::FCOS: ExpandFloatRes_FCOS(N, Lo, Hi); break;
1367 case ISD::STRICT_FDIV:
1368 case ISD::FDIV: ExpandFloatRes_FDIV(N, Lo, Hi); break;
1369 case ISD::STRICT_FEXP:
1370 case ISD::FEXP: ExpandFloatRes_FEXP(N, Lo, Hi); break;
1371 case ISD::STRICT_FEXP2:
1372 case ISD::FEXP2: ExpandFloatRes_FEXP2(N, Lo, Hi); break;
1373 case ISD::FEXP10: ExpandFloatRes_FEXP10(N, Lo, Hi); break;
1374 case ISD::STRICT_FFLOOR:
1375 case ISD::FFLOOR: ExpandFloatRes_FFLOOR(N, Lo, Hi); break;
1376 case ISD::STRICT_FLOG:
1377 case ISD::FLOG: ExpandFloatRes_FLOG(N, Lo, Hi); break;
1378 case ISD::STRICT_FLOG2:
1379 case ISD::FLOG2: ExpandFloatRes_FLOG2(N, Lo, Hi); break;
1380 case ISD::STRICT_FLOG10:
1381 case ISD::FLOG10: ExpandFloatRes_FLOG10(N, Lo, Hi); break;
1382 case ISD::STRICT_FMA:
1383 case ISD::FMA: ExpandFloatRes_FMA(N, Lo, Hi); break;
1384 case ISD::STRICT_FMUL:
1385 case ISD::FMUL: ExpandFloatRes_FMUL(N, Lo, Hi); break;
1387 case ISD::FNEARBYINT: ExpandFloatRes_FNEARBYINT(N, Lo, Hi); break;
1388 case ISD::FNEG: ExpandFloatRes_FNEG(N, Lo, Hi); break;
1390 case ISD::FP_EXTEND: ExpandFloatRes_FP_EXTEND(N, Lo, Hi); break;
1391 case ISD::STRICT_FPOW:
1392 case ISD::FPOW: ExpandFloatRes_FPOW(N, Lo, Hi); break;
1393 case ISD::STRICT_FPOWI:
1394 case ISD::FPOWI: ExpandFloatRes_FPOWI(N, Lo, Hi); break;
1395 case ISD::FLDEXP:
1396 case ISD::STRICT_FLDEXP: ExpandFloatRes_FLDEXP(N, Lo, Hi); break;
1397 case ISD::FREEZE: ExpandFloatRes_FREEZE(N, Lo, Hi); break;
1398 case ISD::STRICT_FRINT:
1399 case ISD::FRINT: ExpandFloatRes_FRINT(N, Lo, Hi); break;
1400 case ISD::STRICT_FROUND:
1401 case ISD::FROUND: ExpandFloatRes_FROUND(N, Lo, Hi); break;
1403 case ISD::FROUNDEVEN: ExpandFloatRes_FROUNDEVEN(N, Lo, Hi); break;
1404 case ISD::STRICT_FSIN:
1405 case ISD::FSIN: ExpandFloatRes_FSIN(N, Lo, Hi); break;
1406 case ISD::STRICT_FSQRT:
1407 case ISD::FSQRT: ExpandFloatRes_FSQRT(N, Lo, Hi); break;
1408 case ISD::STRICT_FSUB:
1409 case ISD::FSUB: ExpandFloatRes_FSUB(N, Lo, Hi); break;
1410 case ISD::STRICT_FTAN:
1411 case ISD::FTAN: ExpandFloatRes_FTAN(N, Lo, Hi); break;
1412 case ISD::STRICT_FTRUNC:
1413 case ISD::FTRUNC: ExpandFloatRes_FTRUNC(N, Lo, Hi); break;
1414 case ISD::LOAD: ExpandFloatRes_LOAD(N, Lo, Hi); break;
1417 case ISD::SINT_TO_FP:
1418 case ISD::UINT_TO_FP: ExpandFloatRes_XINT_TO_FP(N, Lo, Hi); break;
1419 case ISD::STRICT_FREM:
1420 case ISD::FREM: ExpandFloatRes_FREM(N, Lo, Hi); break;
1421 // clang-format on
1422 }
1423
1424 // If Lo/Hi is null, the sub-method took care of registering results etc.
1425 if (Lo.getNode())
1426 SetExpandedFloat(SDValue(N, ResNo), Lo, Hi);
1427}
1428
1429void DAGTypeLegalizer::ExpandFloatRes_ConstantFP(SDNode *N, SDValue &Lo,
1430 SDValue &Hi) {
1431 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
1432 assert(NVT.getSizeInBits() == 64 &&
1433 "Do not know how to expand this float constant!");
1434 APInt C = cast<ConstantFPSDNode>(N)->getValueAPF().bitcastToAPInt();
1435 SDLoc dl(N);
1437 APInt(64, C.getRawData()[1])),
1438 dl, NVT);
1440 APInt(64, C.getRawData()[0])),
1441 dl, NVT);
1442}
1443
1444void DAGTypeLegalizer::ExpandFloatRes_Unary(SDNode *N, RTLIB::Libcall LC,
1445 SDValue &Lo, SDValue &Hi) {
1446 bool IsStrict = N->isStrictFPOpcode();
1447 unsigned Offset = IsStrict ? 1 : 0;
1448 SDValue Op = N->getOperand(0 + Offset);
1449 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
1451 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, N->getValueType(0),
1452 Op, CallOptions, SDLoc(N),
1453 Chain);
1454 if (IsStrict)
1455 ReplaceValueWith(SDValue(N, 1), Tmp.second);
1456 GetPairElements(Tmp.first, Lo, Hi);
1457}
1458
1459void DAGTypeLegalizer::ExpandFloatRes_Binary(SDNode *N, RTLIB::Libcall LC,
1460 SDValue &Lo, SDValue &Hi) {
1461 bool IsStrict = N->isStrictFPOpcode();
1462 unsigned Offset = IsStrict ? 1 : 0;
1463 SDValue Ops[] = { N->getOperand(0 + Offset), N->getOperand(1 + Offset) };
1464 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
1466 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, N->getValueType(0),
1467 Ops, CallOptions, SDLoc(N),
1468 Chain);
1469 if (IsStrict)
1470 ReplaceValueWith(SDValue(N, 1), Tmp.second);
1471 GetPairElements(Tmp.first, Lo, Hi);
1472}
1473
1474void DAGTypeLegalizer::ExpandFloatRes_FABS(SDNode *N, SDValue &Lo,
1475 SDValue &Hi) {
1476 assert(N->getValueType(0) == MVT::ppcf128 &&
1477 "Logic only correct for ppcf128!");
1478 SDLoc dl(N);
1479 SDValue Tmp;
1480 GetExpandedFloat(N->getOperand(0), Lo, Tmp);
1481 Hi = DAG.getNode(ISD::FABS, dl, Tmp.getValueType(), Tmp);
1482 // Lo = Hi==fabs(Hi) ? Lo : -Lo;
1483 Lo = DAG.getSelectCC(dl, Tmp, Hi, Lo,
1484 DAG.getNode(ISD::FNEG, dl, Lo.getValueType(), Lo),
1485 ISD::SETEQ);
1486}
1487
1488void DAGTypeLegalizer::ExpandFloatRes_FMINNUM(SDNode *N, SDValue &Lo,
1489 SDValue &Hi) {
1490 ExpandFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
1491 RTLIB::FMIN_F32, RTLIB::FMIN_F64,
1492 RTLIB::FMIN_F80, RTLIB::FMIN_F128,
1493 RTLIB::FMIN_PPCF128), Lo, Hi);
1494}
1495
1496void DAGTypeLegalizer::ExpandFloatRes_FMAXNUM(SDNode *N, SDValue &Lo,
1497 SDValue &Hi) {
1498 ExpandFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
1499 RTLIB::FMAX_F32, RTLIB::FMAX_F64,
1500 RTLIB::FMAX_F80, RTLIB::FMAX_F128,
1501 RTLIB::FMAX_PPCF128), Lo, Hi);
1502}
1503
1504void DAGTypeLegalizer::ExpandFloatRes_FADD(SDNode *N, SDValue &Lo,
1505 SDValue &Hi) {
1506 ExpandFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
1507 RTLIB::ADD_F32, RTLIB::ADD_F64,
1508 RTLIB::ADD_F80, RTLIB::ADD_F128,
1509 RTLIB::ADD_PPCF128), Lo, Hi);
1510}
1511
1512void DAGTypeLegalizer::ExpandFloatRes_FCBRT(SDNode *N, SDValue &Lo,
1513 SDValue &Hi) {
1514 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0), RTLIB::CBRT_F32,
1515 RTLIB::CBRT_F64, RTLIB::CBRT_F80,
1516 RTLIB::CBRT_F128,
1517 RTLIB::CBRT_PPCF128), Lo, Hi);
1518}
1519
1520void DAGTypeLegalizer::ExpandFloatRes_FCEIL(SDNode *N,
1521 SDValue &Lo, SDValue &Hi) {
1522 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
1523 RTLIB::CEIL_F32, RTLIB::CEIL_F64,
1524 RTLIB::CEIL_F80, RTLIB::CEIL_F128,
1525 RTLIB::CEIL_PPCF128), Lo, Hi);
1526}
1527
1528void DAGTypeLegalizer::ExpandFloatRes_FCOPYSIGN(SDNode *N,
1529 SDValue &Lo, SDValue &Hi) {
1530 ExpandFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
1531 RTLIB::COPYSIGN_F32,
1532 RTLIB::COPYSIGN_F64,
1533 RTLIB::COPYSIGN_F80,
1534 RTLIB::COPYSIGN_F128,
1535 RTLIB::COPYSIGN_PPCF128), Lo, Hi);
1536}
1537
1538void DAGTypeLegalizer::ExpandFloatRes_FCOS(SDNode *N,
1539 SDValue &Lo, SDValue &Hi) {
1540 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
1541 RTLIB::COS_F32, RTLIB::COS_F64,
1542 RTLIB::COS_F80, RTLIB::COS_F128,
1543 RTLIB::COS_PPCF128), Lo, Hi);
1544}
1545
1546void DAGTypeLegalizer::ExpandFloatRes_FDIV(SDNode *N, SDValue &Lo,
1547 SDValue &Hi) {
1548 ExpandFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
1549 RTLIB::DIV_F32,
1550 RTLIB::DIV_F64,
1551 RTLIB::DIV_F80,
1552 RTLIB::DIV_F128,
1553 RTLIB::DIV_PPCF128), Lo, Hi);
1554}
1555
1556void DAGTypeLegalizer::ExpandFloatRes_FEXP(SDNode *N,
1557 SDValue &Lo, SDValue &Hi) {
1558 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
1559 RTLIB::EXP_F32, RTLIB::EXP_F64,
1560 RTLIB::EXP_F80, RTLIB::EXP_F128,
1561 RTLIB::EXP_PPCF128), Lo, Hi);
1562}
1563
1564void DAGTypeLegalizer::ExpandFloatRes_FEXP2(SDNode *N,
1565 SDValue &Lo, SDValue &Hi) {
1566 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
1567 RTLIB::EXP2_F32, RTLIB::EXP2_F64,
1568 RTLIB::EXP2_F80, RTLIB::EXP2_F128,
1569 RTLIB::EXP2_PPCF128), Lo, Hi);
1570}
1571
1572void DAGTypeLegalizer::ExpandFloatRes_FEXP10(SDNode *N, SDValue &Lo,
1573 SDValue &Hi) {
1574 ExpandFloatRes_Unary(N,
1575 GetFPLibCall(N->getValueType(0), RTLIB::EXP10_F32,
1576 RTLIB::EXP10_F64, RTLIB::EXP10_F80,
1577 RTLIB::EXP10_F128, RTLIB::EXP10_PPCF128),
1578 Lo, Hi);
1579}
1580
1581void DAGTypeLegalizer::ExpandFloatRes_FFLOOR(SDNode *N,
1582 SDValue &Lo, SDValue &Hi) {
1583 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
1584 RTLIB::FLOOR_F32, RTLIB::FLOOR_F64,
1585 RTLIB::FLOOR_F80, RTLIB::FLOOR_F128,
1586 RTLIB::FLOOR_PPCF128), Lo, Hi);
1587}
1588
1589void DAGTypeLegalizer::ExpandFloatRes_FLOG(SDNode *N,
1590 SDValue &Lo, SDValue &Hi) {
1591 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
1592 RTLIB::LOG_F32, RTLIB::LOG_F64,
1593 RTLIB::LOG_F80, RTLIB::LOG_F128,
1594 RTLIB::LOG_PPCF128), Lo, Hi);
1595}
1596
1597void DAGTypeLegalizer::ExpandFloatRes_FLOG2(SDNode *N,
1598 SDValue &Lo, SDValue &Hi) {
1599 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
1600 RTLIB::LOG2_F32, RTLIB::LOG2_F64,
1601 RTLIB::LOG2_F80, RTLIB::LOG2_F128,
1602 RTLIB::LOG2_PPCF128), Lo, Hi);
1603}
1604
1605void DAGTypeLegalizer::ExpandFloatRes_FLOG10(SDNode *N,
1606 SDValue &Lo, SDValue &Hi) {
1607 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
1608 RTLIB::LOG10_F32, RTLIB::LOG10_F64,
1609 RTLIB::LOG10_F80, RTLIB::LOG10_F128,
1610 RTLIB::LOG10_PPCF128), Lo, Hi);
1611}
1612
1613void DAGTypeLegalizer::ExpandFloatRes_FMA(SDNode *N, SDValue &Lo,
1614 SDValue &Hi) {
1615 bool IsStrict = N->isStrictFPOpcode();
1616 unsigned Offset = IsStrict ? 1 : 0;
1617 SDValue Ops[3] = { N->getOperand(0 + Offset), N->getOperand(1 + Offset),
1618 N->getOperand(2 + Offset) };
1619 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
1621 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, GetFPLibCall(N->getValueType(0),
1622 RTLIB::FMA_F32,
1623 RTLIB::FMA_F64,
1624 RTLIB::FMA_F80,
1625 RTLIB::FMA_F128,
1626 RTLIB::FMA_PPCF128),
1627 N->getValueType(0), Ops, CallOptions,
1628 SDLoc(N), Chain);
1629 if (IsStrict)
1630 ReplaceValueWith(SDValue(N, 1), Tmp.second);
1631 GetPairElements(Tmp.first, Lo, Hi);
1632}
1633
1634void DAGTypeLegalizer::ExpandFloatRes_FMUL(SDNode *N, SDValue &Lo,
1635 SDValue &Hi) {
1636 ExpandFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
1637 RTLIB::MUL_F32,
1638 RTLIB::MUL_F64,
1639 RTLIB::MUL_F80,
1640 RTLIB::MUL_F128,
1641 RTLIB::MUL_PPCF128), Lo, Hi);
1642}
1643
1644void DAGTypeLegalizer::ExpandFloatRes_FNEARBYINT(SDNode *N,
1645 SDValue &Lo, SDValue &Hi) {
1646 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
1647 RTLIB::NEARBYINT_F32,
1648 RTLIB::NEARBYINT_F64,
1649 RTLIB::NEARBYINT_F80,
1650 RTLIB::NEARBYINT_F128,
1651 RTLIB::NEARBYINT_PPCF128), Lo, Hi);
1652}
1653
1654void DAGTypeLegalizer::ExpandFloatRes_FNEG(SDNode *N, SDValue &Lo,
1655 SDValue &Hi) {
1656 SDLoc dl(N);
1657 GetExpandedFloat(N->getOperand(0), Lo, Hi);
1658 Lo = DAG.getNode(ISD::FNEG, dl, Lo.getValueType(), Lo);
1659 Hi = DAG.getNode(ISD::FNEG, dl, Hi.getValueType(), Hi);
1660}
1661
1662void DAGTypeLegalizer::ExpandFloatRes_FP_EXTEND(SDNode *N, SDValue &Lo,
1663 SDValue &Hi) {
1664 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
1665 SDLoc dl(N);
1666 bool IsStrict = N->isStrictFPOpcode();
1667
1668 SDValue Chain;
1669 if (IsStrict) {
1670 // If the expanded type is the same as the input type, just bypass the node.
1671 if (NVT == N->getOperand(1).getValueType()) {
1672 Hi = N->getOperand(1);
1673 Chain = N->getOperand(0);
1674 } else {
1675 // Other we need to extend.
1676 Hi = DAG.getNode(ISD::STRICT_FP_EXTEND, dl, { NVT, MVT::Other },
1677 { N->getOperand(0), N->getOperand(1) });
1678 Chain = Hi.getValue(1);
1679 }
1680 } else {
1681 Hi = DAG.getNode(ISD::FP_EXTEND, dl, NVT, N->getOperand(0));
1682 }
1683
1685 APInt(NVT.getSizeInBits(), 0)), dl, NVT);
1686
1687 if (IsStrict)
1688 ReplaceValueWith(SDValue(N, 1), Chain);
1689}
1690
1691void DAGTypeLegalizer::ExpandFloatRes_FPOW(SDNode *N,
1692 SDValue &Lo, SDValue &Hi) {
1693 ExpandFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
1694 RTLIB::POW_F32, RTLIB::POW_F64,
1695 RTLIB::POW_F80, RTLIB::POW_F128,
1696 RTLIB::POW_PPCF128), Lo, Hi);
1697}
1698
1699void DAGTypeLegalizer::ExpandFloatRes_FPOWI(SDNode *N,
1700 SDValue &Lo, SDValue &Hi) {
1701 ExpandFloatRes_Binary(N, RTLIB::getPOWI(N->getValueType(0)), Lo, Hi);
1702}
1703
1704void DAGTypeLegalizer::ExpandFloatRes_FLDEXP(SDNode *N, SDValue &Lo,
1705 SDValue &Hi) {
1706 ExpandFloatRes_Binary(N, RTLIB::getLDEXP(N->getValueType(0)), Lo, Hi);
1707}
1708
1709void DAGTypeLegalizer::ExpandFloatRes_FREEZE(SDNode *N,
1710 SDValue &Lo, SDValue &Hi) {
1711 assert(N->getValueType(0) == MVT::ppcf128 &&
1712 "Logic only correct for ppcf128!");
1713
1714 SDLoc dl(N);
1715 GetExpandedFloat(N->getOperand(0), Lo, Hi);
1716 Lo = DAG.getNode(ISD::FREEZE, dl, Lo.getValueType(), Lo);
1717 Hi = DAG.getNode(ISD::FREEZE, dl, Hi.getValueType(), Hi);
1718}
1719
1720void DAGTypeLegalizer::ExpandFloatRes_FREM(SDNode *N,
1721 SDValue &Lo, SDValue &Hi) {
1722 ExpandFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
1723 RTLIB::REM_F32, RTLIB::REM_F64,
1724 RTLIB::REM_F80, RTLIB::REM_F128,
1725 RTLIB::REM_PPCF128), Lo, Hi);
1726}
1727
1728void DAGTypeLegalizer::ExpandFloatRes_FRINT(SDNode *N,
1729 SDValue &Lo, SDValue &Hi) {
1730 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
1731 RTLIB::RINT_F32, RTLIB::RINT_F64,
1732 RTLIB::RINT_F80, RTLIB::RINT_F128,
1733 RTLIB::RINT_PPCF128), Lo, Hi);
1734}
1735
1736void DAGTypeLegalizer::ExpandFloatRes_FROUND(SDNode *N,
1737 SDValue &Lo, SDValue &Hi) {
1738 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
1739 RTLIB::ROUND_F32,
1740 RTLIB::ROUND_F64,
1741 RTLIB::ROUND_F80,
1742 RTLIB::ROUND_F128,
1743 RTLIB::ROUND_PPCF128), Lo, Hi);
1744}
1745
1746void DAGTypeLegalizer::ExpandFloatRes_FROUNDEVEN(SDNode *N,
1747 SDValue &Lo, SDValue &Hi) {
1748 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
1749 RTLIB::ROUNDEVEN_F32,
1750 RTLIB::ROUNDEVEN_F64,
1751 RTLIB::ROUNDEVEN_F80,
1752 RTLIB::ROUNDEVEN_F128,
1753 RTLIB::ROUNDEVEN_PPCF128), Lo, Hi);
1754}
1755
1756void DAGTypeLegalizer::ExpandFloatRes_FSIN(SDNode *N,
1757 SDValue &Lo, SDValue &Hi) {
1758 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
1759 RTLIB::SIN_F32, RTLIB::SIN_F64,
1760 RTLIB::SIN_F80, RTLIB::SIN_F128,
1761 RTLIB::SIN_PPCF128), Lo, Hi);
1762}
1763
1764void DAGTypeLegalizer::ExpandFloatRes_FSQRT(SDNode *N,
1765 SDValue &Lo, SDValue &Hi) {
1766 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
1767 RTLIB::SQRT_F32, RTLIB::SQRT_F64,
1768 RTLIB::SQRT_F80, RTLIB::SQRT_F128,
1769 RTLIB::SQRT_PPCF128), Lo, Hi);
1770}
1771
1772void DAGTypeLegalizer::ExpandFloatRes_FSUB(SDNode *N, SDValue &Lo,
1773 SDValue &Hi) {
1774 ExpandFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
1775 RTLIB::SUB_F32,
1776 RTLIB::SUB_F64,
1777 RTLIB::SUB_F80,
1778 RTLIB::SUB_F128,
1779 RTLIB::SUB_PPCF128), Lo, Hi);
1780}
1781
1782void DAGTypeLegalizer::ExpandFloatRes_FTAN(SDNode *N, SDValue &Lo,
1783 SDValue &Hi) {
1784 ExpandFloatRes_Unary(N,
1785 GetFPLibCall(N->getValueType(0), RTLIB::TAN_F32,
1786 RTLIB::TAN_F64, RTLIB::TAN_F80,
1787 RTLIB::TAN_F128, RTLIB::TAN_PPCF128),
1788 Lo, Hi);
1789}
1790
1791void DAGTypeLegalizer::ExpandFloatRes_FTRUNC(SDNode *N,
1792 SDValue &Lo, SDValue &Hi) {
1793 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
1794 RTLIB::TRUNC_F32, RTLIB::TRUNC_F64,
1795 RTLIB::TRUNC_F80, RTLIB::TRUNC_F128,
1796 RTLIB::TRUNC_PPCF128), Lo, Hi);
1797}
1798
1799void DAGTypeLegalizer::ExpandFloatRes_LOAD(SDNode *N, SDValue &Lo,
1800 SDValue &Hi) {
1801 if (ISD::isNormalLoad(N)) {
1802 ExpandRes_NormalLoad(N, Lo, Hi);
1803 return;
1804 }
1805
1806 assert(ISD::isUNINDEXEDLoad(N) && "Indexed load during type legalization!");
1807 LoadSDNode *LD = cast<LoadSDNode>(N);
1808 SDValue Chain = LD->getChain();
1809 SDValue Ptr = LD->getBasePtr();
1810 SDLoc dl(N);
1811
1812 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), LD->getValueType(0));
1813 assert(NVT.isByteSized() && "Expanded type not byte sized!");
1814 assert(LD->getMemoryVT().bitsLE(NVT) && "Float type not round?");
1815
1816 Hi = DAG.getExtLoad(LD->getExtensionType(), dl, NVT, Chain, Ptr,
1817 LD->getMemoryVT(), LD->getMemOperand());
1818
1819 // Remember the chain.
1820 Chain = Hi.getValue(1);
1821
1822 // The low part is zero.
1824 APInt(NVT.getSizeInBits(), 0)), dl, NVT);
1825
1826 // Modified the chain - switch anything that used the old chain to use the
1827 // new one.
1828 ReplaceValueWith(SDValue(LD, 1), Chain);
1829}
1830
1831void DAGTypeLegalizer::ExpandFloatRes_XINT_TO_FP(SDNode *N, SDValue &Lo,
1832 SDValue &Hi) {
1833 assert(N->getValueType(0) == MVT::ppcf128 && "Unsupported XINT_TO_FP!");
1834 EVT VT = N->getValueType(0);
1835 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
1836 bool Strict = N->isStrictFPOpcode();
1837 SDValue Src = N->getOperand(Strict ? 1 : 0);
1838 EVT SrcVT = Src.getValueType();
1839 bool isSigned = N->getOpcode() == ISD::SINT_TO_FP ||
1840 N->getOpcode() == ISD::STRICT_SINT_TO_FP;
1841 SDLoc dl(N);
1842 SDValue Chain = Strict ? N->getOperand(0) : DAG.getEntryNode();
1843
1844 // TODO: Any other flags to propagate?
1846 Flags.setNoFPExcept(N->getFlags().hasNoFPExcept());
1847
1848 // First do an SINT_TO_FP, whether the original was signed or unsigned.
1849 // When promoting partial word types to i32 we must honor the signedness,
1850 // though.
1851 if (SrcVT.bitsLE(MVT::i32)) {
1852 // The integer can be represented exactly in an f64.
1854 APInt(NVT.getSizeInBits(), 0)), dl, NVT);
1855 if (Strict) {
1856 Hi = DAG.getNode(N->getOpcode(), dl, DAG.getVTList(NVT, MVT::Other),
1857 {Chain, Src}, Flags);
1858 Chain = Hi.getValue(1);
1859 } else
1860 Hi = DAG.getNode(N->getOpcode(), dl, NVT, Src);
1861 } else {
1862 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
1863 if (SrcVT.bitsLE(MVT::i64)) {
1865 MVT::i64, Src);
1866 LC = RTLIB::SINTTOFP_I64_PPCF128;
1867 } else if (SrcVT.bitsLE(MVT::i128)) {
1868 Src = DAG.getNode(ISD::SIGN_EXTEND, dl, MVT::i128, Src);
1869 LC = RTLIB::SINTTOFP_I128_PPCF128;
1870 }
1871 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported XINT_TO_FP!");
1872
1874 CallOptions.setSExt(true);
1875 std::pair<SDValue, SDValue> Tmp =
1876 TLI.makeLibCall(DAG, LC, VT, Src, CallOptions, dl, Chain);
1877 if (Strict)
1878 Chain = Tmp.second;
1879 GetPairElements(Tmp.first, Lo, Hi);
1880 }
1881
1882 // No need to complement for unsigned 32-bit integers
1883 if (isSigned || SrcVT.bitsLE(MVT::i32)) {
1884 if (Strict)
1885 ReplaceValueWith(SDValue(N, 1), Chain);
1886
1887 return;
1888 }
1889
1890 // Unsigned - fix up the SINT_TO_FP value just calculated.
1891 // FIXME: For unsigned i128 to ppc_fp128 conversion, we need to carefully
1892 // keep semantics correctness if the integer is not exactly representable
1893 // here. See ExpandLegalINT_TO_FP.
1894 Hi = DAG.getNode(ISD::BUILD_PAIR, dl, VT, Lo, Hi);
1895 SrcVT = Src.getValueType();
1896
1897 // x>=0 ? (ppcf128)(iN)x : (ppcf128)(iN)x + 2^N; N=32,64,128.
1898 static const uint64_t TwoE32[] = { 0x41f0000000000000LL, 0 };
1899 static const uint64_t TwoE64[] = { 0x43f0000000000000LL, 0 };
1900 static const uint64_t TwoE128[] = { 0x47f0000000000000LL, 0 };
1901 ArrayRef<uint64_t> Parts;
1902
1903 switch (SrcVT.getSimpleVT().SimpleTy) {
1904 default:
1905 llvm_unreachable("Unsupported UINT_TO_FP!");
1906 case MVT::i32:
1907 Parts = TwoE32;
1908 break;
1909 case MVT::i64:
1910 Parts = TwoE64;
1911 break;
1912 case MVT::i128:
1913 Parts = TwoE128;
1914 break;
1915 }
1916
1917 // TODO: Are there other fast-math-flags to propagate to this FADD?
1918 SDValue NewLo = DAG.getConstantFP(
1919 APFloat(APFloat::PPCDoubleDouble(), APInt(128, Parts)), dl, MVT::ppcf128);
1920 if (Strict) {
1921 Lo = DAG.getNode(ISD::STRICT_FADD, dl, DAG.getVTList(VT, MVT::Other),
1922 {Chain, Hi, NewLo}, Flags);
1923 Chain = Lo.getValue(1);
1924 ReplaceValueWith(SDValue(N, 1), Chain);
1925 } else
1926 Lo = DAG.getNode(ISD::FADD, dl, VT, Hi, NewLo);
1927 Lo = DAG.getSelectCC(dl, Src, DAG.getConstant(0, dl, SrcVT),
1928 Lo, Hi, ISD::SETLT);
1929 GetPairElements(Lo, Lo, Hi);
1930}
1931
1932
1933//===----------------------------------------------------------------------===//
1934// Float Operand Expansion
1935//===----------------------------------------------------------------------===//
1936
1937/// ExpandFloatOperand - This method is called when the specified operand of the
1938/// specified node is found to need expansion. At this point, all of the result
1939/// types of the node are known to be legal, but other operands of the node may
1940/// need promotion or expansion as well as the specified one.
1941bool DAGTypeLegalizer::ExpandFloatOperand(SDNode *N, unsigned OpNo) {
1942 LLVM_DEBUG(dbgs() << "Expand float operand: "; N->dump(&DAG));
1943 SDValue Res = SDValue();
1944
1945 // See if the target wants to custom expand this node.
1946 if (CustomLowerNode(N, N->getOperand(OpNo).getValueType(), false))
1947 return false;
1948
1949 switch (N->getOpcode()) {
1950 default:
1951#ifndef NDEBUG
1952 dbgs() << "ExpandFloatOperand Op #" << OpNo << ": ";
1953 N->dump(&DAG); dbgs() << "\n";
1954#endif
1955 report_fatal_error("Do not know how to expand this operator's operand!");
1956
1957 case ISD::BITCAST: Res = ExpandOp_BITCAST(N); break;
1958 case ISD::BUILD_VECTOR: Res = ExpandOp_BUILD_VECTOR(N); break;
1959 case ISD::EXTRACT_ELEMENT: Res = ExpandOp_EXTRACT_ELEMENT(N); break;
1960
1961 case ISD::BR_CC: Res = ExpandFloatOp_BR_CC(N); break;
1962 case ISD::FCOPYSIGN: Res = ExpandFloatOp_FCOPYSIGN(N); break;
1964 case ISD::FP_ROUND: Res = ExpandFloatOp_FP_ROUND(N); break;
1967 case ISD::FP_TO_SINT:
1968 case ISD::FP_TO_UINT: Res = ExpandFloatOp_FP_TO_XINT(N); break;
1969 case ISD::LROUND: Res = ExpandFloatOp_LROUND(N); break;
1970 case ISD::LLROUND: Res = ExpandFloatOp_LLROUND(N); break;
1971 case ISD::LRINT: Res = ExpandFloatOp_LRINT(N); break;
1972 case ISD::LLRINT: Res = ExpandFloatOp_LLRINT(N); break;
1973 case ISD::SELECT_CC: Res = ExpandFloatOp_SELECT_CC(N); break;
1974 case ISD::STRICT_FSETCC:
1976 case ISD::SETCC: Res = ExpandFloatOp_SETCC(N); break;
1977 case ISD::STORE: Res = ExpandFloatOp_STORE(cast<StoreSDNode>(N),
1978 OpNo); break;
1979 }
1980
1981 // If the result is null, the sub-method took care of registering results etc.
1982 if (!Res.getNode()) return false;
1983
1984 // If the result is N, the sub-method updated N in place. Tell the legalizer
1985 // core about this.
1986 if (Res.getNode() == N)
1987 return true;
1988
1989 assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 &&
1990 "Invalid operand expansion");
1991
1992 ReplaceValueWith(SDValue(N, 0), Res);
1993 return false;
1994}
1995
1996/// FloatExpandSetCCOperands - Expand the operands of a comparison. This code
1997/// is shared among BR_CC, SELECT_CC, and SETCC handlers.
1998void DAGTypeLegalizer::FloatExpandSetCCOperands(SDValue &NewLHS,
1999 SDValue &NewRHS,
2000 ISD::CondCode &CCCode,
2001 const SDLoc &dl, SDValue &Chain,
2002 bool IsSignaling) {
2003 SDValue LHSLo, LHSHi, RHSLo, RHSHi;
2004 GetExpandedFloat(NewLHS, LHSLo, LHSHi);
2005 GetExpandedFloat(NewRHS, RHSLo, RHSHi);
2006
2007 assert(NewLHS.getValueType() == MVT::ppcf128 && "Unsupported setcc type!");
2008
2009 // FIXME: This generated code sucks. We want to generate
2010 // FCMPU crN, hi1, hi2
2011 // BNE crN, L:
2012 // FCMPU crN, lo1, lo2
2013 // The following can be improved, but not that much.
2014 SDValue Tmp1, Tmp2, Tmp3, OutputChain;
2015 Tmp1 = DAG.getSetCC(dl, getSetCCResultType(LHSHi.getValueType()), LHSHi,
2016 RHSHi, ISD::SETOEQ, Chain, IsSignaling);
2017 OutputChain = Tmp1->getNumValues() > 1 ? Tmp1.getValue(1) : SDValue();
2018 Tmp2 = DAG.getSetCC(dl, getSetCCResultType(LHSLo.getValueType()), LHSLo,
2019 RHSLo, CCCode, OutputChain, IsSignaling);
2020 OutputChain = Tmp2->getNumValues() > 1 ? Tmp2.getValue(1) : SDValue();
2021 Tmp3 = DAG.getNode(ISD::AND, dl, Tmp1.getValueType(), Tmp1, Tmp2);
2022 Tmp1 =
2023 DAG.getSetCC(dl, getSetCCResultType(LHSHi.getValueType()), LHSHi, RHSHi,
2024 ISD::SETUNE, OutputChain, IsSignaling);
2025 OutputChain = Tmp1->getNumValues() > 1 ? Tmp1.getValue(1) : SDValue();
2026 Tmp2 = DAG.getSetCC(dl, getSetCCResultType(LHSHi.getValueType()), LHSHi,
2027 RHSHi, CCCode, OutputChain, IsSignaling);
2028 OutputChain = Tmp2->getNumValues() > 1 ? Tmp2.getValue(1) : SDValue();
2029 Tmp1 = DAG.getNode(ISD::AND, dl, Tmp1.getValueType(), Tmp1, Tmp2);
2030 NewLHS = DAG.getNode(ISD::OR, dl, Tmp1.getValueType(), Tmp1, Tmp3);
2031 NewRHS = SDValue(); // LHS is the result, not a compare.
2032 Chain = OutputChain;
2033}
2034
2035SDValue DAGTypeLegalizer::ExpandFloatOp_BR_CC(SDNode *N) {
2036 SDValue NewLHS = N->getOperand(2), NewRHS = N->getOperand(3);
2037 ISD::CondCode CCCode = cast<CondCodeSDNode>(N->getOperand(1))->get();
2038 SDValue Chain;
2039 FloatExpandSetCCOperands(NewLHS, NewRHS, CCCode, SDLoc(N), Chain);
2040
2041 // If ExpandSetCCOperands returned a scalar, we need to compare the result
2042 // against zero to select between true and false values.
2043 if (!NewRHS.getNode()) {
2044 NewRHS = DAG.getConstant(0, SDLoc(N), NewLHS.getValueType());
2045 CCCode = ISD::SETNE;
2046 }
2047
2048 // Update N to have the operands specified.
2049 return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0),
2050 DAG.getCondCode(CCCode), NewLHS, NewRHS,
2051 N->getOperand(4)), 0);
2052}
2053
2054SDValue DAGTypeLegalizer::ExpandFloatOp_FCOPYSIGN(SDNode *N) {
2055 assert(N->getOperand(1).getValueType() == MVT::ppcf128 &&
2056 "Logic only correct for ppcf128!");
2057 SDValue Lo, Hi;
2058 GetExpandedFloat(N->getOperand(1), Lo, Hi);
2059 // The ppcf128 value is providing only the sign; take it from the
2060 // higher-order double (which must have the larger magnitude).
2061 return DAG.getNode(ISD::FCOPYSIGN, SDLoc(N),
2062 N->getValueType(0), N->getOperand(0), Hi);
2063}
2064
2065SDValue DAGTypeLegalizer::ExpandFloatOp_FP_ROUND(SDNode *N) {
2066 bool IsStrict = N->isStrictFPOpcode();
2067 assert(N->getOperand(IsStrict ? 1 : 0).getValueType() == MVT::ppcf128 &&
2068 "Logic only correct for ppcf128!");
2069 SDValue Lo, Hi;
2070 GetExpandedFloat(N->getOperand(IsStrict ? 1 : 0), Lo, Hi);
2071
2072 if (!IsStrict)
2073 // Round it the rest of the way (e.g. to f32) if needed.
2074 return DAG.getNode(ISD::FP_ROUND, SDLoc(N),
2075 N->getValueType(0), Hi, N->getOperand(1));
2076
2077 // Eliminate the node if the input float type is the same as the output float
2078 // type.
2079 if (Hi.getValueType() == N->getValueType(0)) {
2080 // Connect the output chain to the input chain, unlinking the node.
2081 ReplaceValueWith(SDValue(N, 1), N->getOperand(0));
2082 ReplaceValueWith(SDValue(N, 0), Hi);
2083 return SDValue();
2084 }
2085
2087 {N->getValueType(0), MVT::Other},
2088 {N->getOperand(0), Hi, N->getOperand(2)});
2089 ReplaceValueWith(SDValue(N, 1), Expansion.getValue(1));
2090 ReplaceValueWith(SDValue(N, 0), Expansion);
2091 return SDValue();
2092}
2093
2094SDValue DAGTypeLegalizer::ExpandFloatOp_FP_TO_XINT(SDNode *N) {
2095 EVT RVT = N->getValueType(0);
2096 SDLoc dl(N);
2097
2098 bool IsStrict = N->isStrictFPOpcode();
2099 bool Signed = N->getOpcode() == ISD::FP_TO_SINT ||
2100 N->getOpcode() == ISD::STRICT_FP_TO_SINT;
2101 SDValue Op = N->getOperand(IsStrict ? 1 : 0);
2102 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
2103
2104 EVT NVT;
2105 RTLIB::Libcall LC = findFPToIntLibcall(Op.getValueType(), RVT, NVT, Signed);
2106 assert(LC != RTLIB::UNKNOWN_LIBCALL && NVT.isSimple() &&
2107 "Unsupported FP_TO_XINT!");
2109 std::pair<SDValue, SDValue> Tmp =
2110 TLI.makeLibCall(DAG, LC, NVT, Op, CallOptions, dl, Chain);
2111 if (!IsStrict)
2112 return Tmp.first;
2113
2114 ReplaceValueWith(SDValue(N, 1), Tmp.second);
2115 ReplaceValueWith(SDValue(N, 0), Tmp.first);
2116 return SDValue();
2117}
2118
2119SDValue DAGTypeLegalizer::ExpandFloatOp_SELECT_CC(SDNode *N) {
2120 SDValue NewLHS = N->getOperand(0), NewRHS = N->getOperand(1);
2121 ISD::CondCode CCCode = cast<CondCodeSDNode>(N->getOperand(4))->get();
2122 SDValue Chain;
2123 FloatExpandSetCCOperands(NewLHS, NewRHS, CCCode, SDLoc(N), Chain);
2124
2125 // If ExpandSetCCOperands returned a scalar, we need to compare the result
2126 // against zero to select between true and false values.
2127 if (!NewRHS.getNode()) {
2128 NewRHS = DAG.getConstant(0, SDLoc(N), NewLHS.getValueType());
2129 CCCode = ISD::SETNE;
2130 }
2131
2132 // Update N to have the operands specified.
2133 return SDValue(DAG.UpdateNodeOperands(N, NewLHS, NewRHS,
2134 N->getOperand(2), N->getOperand(3),
2135 DAG.getCondCode(CCCode)), 0);
2136}
2137
2138SDValue DAGTypeLegalizer::ExpandFloatOp_SETCC(SDNode *N) {
2139 bool IsStrict = N->isStrictFPOpcode();
2140 SDValue NewLHS = N->getOperand(IsStrict ? 1 : 0);
2141 SDValue NewRHS = N->getOperand(IsStrict ? 2 : 1);
2142 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
2143 ISD::CondCode CCCode =
2144 cast<CondCodeSDNode>(N->getOperand(IsStrict ? 3 : 2))->get();
2145 FloatExpandSetCCOperands(NewLHS, NewRHS, CCCode, SDLoc(N), Chain,
2146 N->getOpcode() == ISD::STRICT_FSETCCS);
2147
2148 // FloatExpandSetCCOperands always returned a scalar.
2149 assert(!NewRHS.getNode() && "Expect to return scalar");
2150 assert(NewLHS.getValueType() == N->getValueType(0) &&
2151 "Unexpected setcc expansion!");
2152 if (Chain) {
2153 ReplaceValueWith(SDValue(N, 0), NewLHS);
2154 ReplaceValueWith(SDValue(N, 1), Chain);
2155 return SDValue();
2156 }
2157 return NewLHS;
2158}
2159
2160SDValue DAGTypeLegalizer::ExpandFloatOp_STORE(SDNode *N, unsigned OpNo) {
2161 if (ISD::isNormalStore(N))
2162 return ExpandOp_NormalStore(N, OpNo);
2163
2164 assert(ISD::isUNINDEXEDStore(N) && "Indexed store during type legalization!");
2165 assert(OpNo == 1 && "Can only expand the stored value so far");
2166 StoreSDNode *ST = cast<StoreSDNode>(N);
2167
2168 SDValue Chain = ST->getChain();
2169 SDValue Ptr = ST->getBasePtr();
2170
2171 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(),
2172 ST->getValue().getValueType());
2173 assert(NVT.isByteSized() && "Expanded type not byte sized!");
2174 assert(ST->getMemoryVT().bitsLE(NVT) && "Float type not round?");
2175 (void)NVT;
2176
2177 SDValue Lo, Hi;
2178 GetExpandedOp(ST->getValue(), Lo, Hi);
2179
2180 return DAG.getTruncStore(Chain, SDLoc(N), Hi, Ptr,
2181 ST->getMemoryVT(), ST->getMemOperand());
2182}
2183
2184SDValue DAGTypeLegalizer::ExpandFloatOp_LROUND(SDNode *N) {
2185 EVT RVT = N->getValueType(0);
2186 EVT RetVT = N->getOperand(0).getValueType();
2188 return TLI.makeLibCall(DAG, GetFPLibCall(RetVT,
2189 RTLIB::LROUND_F32,
2190 RTLIB::LROUND_F64,
2191 RTLIB::LROUND_F80,
2192 RTLIB::LROUND_F128,
2193 RTLIB::LROUND_PPCF128),
2194 RVT, N->getOperand(0), CallOptions, SDLoc(N)).first;
2195}
2196
2197SDValue DAGTypeLegalizer::ExpandFloatOp_LLROUND(SDNode *N) {
2198 EVT RVT = N->getValueType(0);
2199 EVT RetVT = N->getOperand(0).getValueType();
2201 return TLI.makeLibCall(DAG, GetFPLibCall(RetVT,
2202 RTLIB::LLROUND_F32,
2203 RTLIB::LLROUND_F64,
2204 RTLIB::LLROUND_F80,
2205 RTLIB::LLROUND_F128,
2206 RTLIB::LLROUND_PPCF128),
2207 RVT, N->getOperand(0), CallOptions, SDLoc(N)).first;
2208}
2209
2210SDValue DAGTypeLegalizer::ExpandFloatOp_LRINT(SDNode *N) {
2211 EVT RVT = N->getValueType(0);
2212 EVT RetVT = N->getOperand(0).getValueType();
2214 return TLI.makeLibCall(DAG, GetFPLibCall(RetVT,
2215 RTLIB::LRINT_F32,
2216 RTLIB::LRINT_F64,
2217 RTLIB::LRINT_F80,
2218 RTLIB::LRINT_F128,
2219 RTLIB::LRINT_PPCF128),
2220 RVT, N->getOperand(0), CallOptions, SDLoc(N)).first;
2221}
2222
2223SDValue DAGTypeLegalizer::ExpandFloatOp_LLRINT(SDNode *N) {
2224 EVT RVT = N->getValueType(0);
2225 EVT RetVT = N->getOperand(0).getValueType();
2227 return TLI.makeLibCall(DAG, GetFPLibCall(RetVT,
2228 RTLIB::LLRINT_F32,
2229 RTLIB::LLRINT_F64,
2230 RTLIB::LLRINT_F80,
2231 RTLIB::LLRINT_F128,
2232 RTLIB::LLRINT_PPCF128),
2233 RVT, N->getOperand(0), CallOptions, SDLoc(N)).first;
2234}
2235
2236//===----------------------------------------------------------------------===//
2237// Float Operand Promotion
2238//===----------------------------------------------------------------------===//
2239//
2240
2242 if (OpVT == MVT::f16) {
2243 return ISD::FP16_TO_FP;
2244 } else if (RetVT == MVT::f16) {
2245 return ISD::FP_TO_FP16;
2246 } else if (OpVT == MVT::bf16) {
2247 return ISD::BF16_TO_FP;
2248 } else if (RetVT == MVT::bf16) {
2249 return ISD::FP_TO_BF16;
2250 }
2251
2252 report_fatal_error("Attempt at an invalid promotion-related conversion");
2253}
2254
2256 if (OpVT == MVT::f16)
2258
2259 if (RetVT == MVT::f16)
2261
2262 if (OpVT == MVT::bf16)
2264
2265 if (RetVT == MVT::bf16)
2267
2268 report_fatal_error("Attempt at an invalid promotion-related conversion");
2269}
2270
2271bool DAGTypeLegalizer::PromoteFloatOperand(SDNode *N, unsigned OpNo) {
2272 LLVM_DEBUG(dbgs() << "Promote float operand " << OpNo << ": "; N->dump(&DAG));
2273 SDValue R = SDValue();
2274
2275 if (CustomLowerNode(N, N->getOperand(OpNo).getValueType(), false)) {
2276 LLVM_DEBUG(dbgs() << "Node has been custom lowered, done\n");
2277 return false;
2278 }
2279
2280 // Nodes that use a promotion-requiring floating point operand, but doesn't
2281 // produce a promotion-requiring floating point result, need to be legalized
2282 // to use the promoted float operand. Nodes that produce at least one
2283 // promotion-requiring floating point result have their operands legalized as
2284 // a part of PromoteFloatResult.
2285 // clang-format off
2286 switch (N->getOpcode()) {
2287 default:
2288 #ifndef NDEBUG
2289 dbgs() << "PromoteFloatOperand Op #" << OpNo << ": ";
2290 N->dump(&DAG); dbgs() << "\n";
2291 #endif
2292 report_fatal_error("Do not know how to promote this operator's operand!");
2293
2294 case ISD::BITCAST: R = PromoteFloatOp_BITCAST(N, OpNo); break;
2295 case ISD::FCOPYSIGN: R = PromoteFloatOp_FCOPYSIGN(N, OpNo); break;
2296 case ISD::FP_TO_SINT:
2297 case ISD::FP_TO_UINT:
2298 case ISD::LRINT:
2299 case ISD::LLRINT: R = PromoteFloatOp_UnaryOp(N, OpNo); break;
2302 R = PromoteFloatOp_FP_TO_XINT_SAT(N, OpNo); break;
2303 case ISD::FP_EXTEND: R = PromoteFloatOp_FP_EXTEND(N, OpNo); break;
2305 R = PromoteFloatOp_STRICT_FP_EXTEND(N, OpNo);
2306 break;
2307 case ISD::SELECT_CC: R = PromoteFloatOp_SELECT_CC(N, OpNo); break;
2308 case ISD::SETCC: R = PromoteFloatOp_SETCC(N, OpNo); break;
2309 case ISD::STORE: R = PromoteFloatOp_STORE(N, OpNo); break;
2310 case ISD::ATOMIC_STORE: R = PromoteFloatOp_ATOMIC_STORE(N, OpNo); break;
2311 }
2312 // clang-format on
2313
2314 if (R.getNode())
2315 ReplaceValueWith(SDValue(N, 0), R);
2316 return false;
2317}
2318
2319SDValue DAGTypeLegalizer::PromoteFloatOp_BITCAST(SDNode *N, unsigned OpNo) {
2320 SDValue Op = N->getOperand(0);
2321 EVT OpVT = Op->getValueType(0);
2322
2323 SDValue Promoted = GetPromotedFloat(N->getOperand(0));
2324 EVT PromotedVT = Promoted->getValueType(0);
2325
2326 // Convert the promoted float value to the desired IVT.
2327 EVT IVT = EVT::getIntegerVT(*DAG.getContext(), OpVT.getSizeInBits());
2328 SDValue Convert = DAG.getNode(GetPromotionOpcode(PromotedVT, OpVT), SDLoc(N),
2329 IVT, Promoted);
2330 // The final result type might not be an scalar so we need a bitcast. The
2331 // bitcast will be further legalized if needed.
2332 return DAG.getBitcast(N->getValueType(0), Convert);
2333}
2334
2335// Promote Operand 1 of FCOPYSIGN. Operand 0 ought to be handled by
2336// PromoteFloatRes_FCOPYSIGN.
2337SDValue DAGTypeLegalizer::PromoteFloatOp_FCOPYSIGN(SDNode *N, unsigned OpNo) {
2338 assert (OpNo == 1 && "Only Operand 1 must need promotion here");
2339 SDValue Op1 = GetPromotedFloat(N->getOperand(1));
2340
2341 return DAG.getNode(N->getOpcode(), SDLoc(N), N->getValueType(0),
2342 N->getOperand(0), Op1);
2343}
2344
2345// Convert the promoted float value to the desired integer type
2346SDValue DAGTypeLegalizer::PromoteFloatOp_UnaryOp(SDNode *N, unsigned OpNo) {
2347 SDValue Op = GetPromotedFloat(N->getOperand(0));
2348 return DAG.getNode(N->getOpcode(), SDLoc(N), N->getValueType(0), Op);
2349}
2350
2351SDValue DAGTypeLegalizer::PromoteFloatOp_FP_TO_XINT_SAT(SDNode *N,
2352 unsigned OpNo) {
2353 SDValue Op = GetPromotedFloat(N->getOperand(0));
2354 return DAG.getNode(N->getOpcode(), SDLoc(N), N->getValueType(0), Op,
2355 N->getOperand(1));
2356}
2357
2358SDValue DAGTypeLegalizer::PromoteFloatOp_FP_EXTEND(SDNode *N, unsigned OpNo) {
2359 SDValue Op = GetPromotedFloat(N->getOperand(0));
2360 EVT VT = N->getValueType(0);
2361
2362 // Desired VT is same as promoted type. Use promoted float directly.
2363 if (VT == Op->getValueType(0))
2364 return Op;
2365
2366 // Else, extend the promoted float value to the desired VT.
2367 return DAG.getNode(ISD::FP_EXTEND, SDLoc(N), VT, Op);
2368}
2369
2370SDValue DAGTypeLegalizer::PromoteFloatOp_STRICT_FP_EXTEND(SDNode *N,
2371 unsigned OpNo) {
2372 assert(OpNo == 1 && "Promoting unpromotable operand");
2373
2374 SDValue Op = GetPromotedFloat(N->getOperand(1));
2375 EVT VT = N->getValueType(0);
2376
2377 // Desired VT is same as promoted type. Use promoted float directly.
2378 if (VT == Op->getValueType(0)) {
2379 ReplaceValueWith(SDValue(N, 1), N->getOperand(0));
2380 return Op;
2381 }
2382
2383 // Else, extend the promoted float value to the desired VT.
2384 SDValue Res = DAG.getNode(ISD::STRICT_FP_EXTEND, SDLoc(N), N->getVTList(),
2385 N->getOperand(0), Op);
2386 ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
2387 return Res;
2388}
2389
2390// Promote the float operands used for comparison. The true- and false-
2391// operands have the same type as the result and are promoted, if needed, by
2392// PromoteFloatRes_SELECT_CC
2393SDValue DAGTypeLegalizer::PromoteFloatOp_SELECT_CC(SDNode *N, unsigned OpNo) {
2394 SDValue LHS = GetPromotedFloat(N->getOperand(0));
2395 SDValue RHS = GetPromotedFloat(N->getOperand(1));
2396
2397 return DAG.getNode(ISD::SELECT_CC, SDLoc(N), N->getValueType(0),
2398 LHS, RHS, N->getOperand(2), N->getOperand(3),
2399 N->getOperand(4));
2400}
2401
2402// Construct a SETCC that compares the promoted values and sets the conditional
2403// code.
2404SDValue DAGTypeLegalizer::PromoteFloatOp_SETCC(SDNode *N, unsigned OpNo) {
2405 EVT VT = N->getValueType(0);
2406 SDValue Op0 = GetPromotedFloat(N->getOperand(0));
2407 SDValue Op1 = GetPromotedFloat(N->getOperand(1));
2408 ISD::CondCode CCCode = cast<CondCodeSDNode>(N->getOperand(2))->get();
2409
2410 return DAG.getSetCC(SDLoc(N), VT, Op0, Op1, CCCode);
2411
2412}
2413
2414// Lower the promoted Float down to the integer value of same size and construct
2415// a STORE of the integer value.
2416SDValue DAGTypeLegalizer::PromoteFloatOp_STORE(SDNode *N, unsigned OpNo) {
2417 StoreSDNode *ST = cast<StoreSDNode>(N);
2418 SDValue Val = ST->getValue();
2419 SDLoc DL(N);
2420
2421 SDValue Promoted = GetPromotedFloat(Val);
2422 EVT VT = ST->getOperand(1).getValueType();
2423 EVT IVT = EVT::getIntegerVT(*DAG.getContext(), VT.getSizeInBits());
2424
2425 SDValue NewVal;
2426 NewVal = DAG.getNode(GetPromotionOpcode(Promoted.getValueType(), VT), DL,
2427 IVT, Promoted);
2428
2429 return DAG.getStore(ST->getChain(), DL, NewVal, ST->getBasePtr(),
2430 ST->getMemOperand());
2431}
2432
2433SDValue DAGTypeLegalizer::PromoteFloatOp_ATOMIC_STORE(SDNode *N,
2434 unsigned OpNo) {
2435 AtomicSDNode *ST = cast<AtomicSDNode>(N);
2436 SDValue Val = ST->getVal();
2437 SDLoc DL(N);
2438
2439 SDValue Promoted = GetPromotedFloat(Val);
2440 EVT VT = ST->getOperand(1).getValueType();
2441 EVT IVT = EVT::getIntegerVT(*DAG.getContext(), VT.getSizeInBits());
2442
2443 SDValue NewVal = DAG.getNode(GetPromotionOpcode(Promoted.getValueType(), VT),
2444 DL, IVT, Promoted);
2445
2446 return DAG.getAtomic(ISD::ATOMIC_STORE, DL, IVT, ST->getChain(), NewVal,
2447 ST->getBasePtr(), ST->getMemOperand());
2448}
2449
2450//===----------------------------------------------------------------------===//
2451// Float Result Promotion
2452//===----------------------------------------------------------------------===//
2453
2454void DAGTypeLegalizer::PromoteFloatResult(SDNode *N, unsigned ResNo) {
2455 LLVM_DEBUG(dbgs() << "Promote float result " << ResNo << ": "; N->dump(&DAG));
2456 SDValue R = SDValue();
2457
2458 // See if the target wants to custom expand this node.
2459 if (CustomLowerNode(N, N->getValueType(ResNo), true)) {
2460 LLVM_DEBUG(dbgs() << "Node has been custom expanded, done\n");
2461 return;
2462 }
2463
2464 switch (N->getOpcode()) {
2465 // These opcodes cannot appear if promotion of FP16 is done in the backend
2466 // instead of Clang
2467 case ISD::FP16_TO_FP:
2468 case ISD::FP_TO_FP16:
2469 default:
2470#ifndef NDEBUG
2471 dbgs() << "PromoteFloatResult #" << ResNo << ": ";
2472 N->dump(&DAG); dbgs() << "\n";
2473#endif
2474 report_fatal_error("Do not know how to promote this operator's result!");
2475
2476 case ISD::BITCAST: R = PromoteFloatRes_BITCAST(N); break;
2477 case ISD::ConstantFP: R = PromoteFloatRes_ConstantFP(N); break;
2479 R = PromoteFloatRes_EXTRACT_VECTOR_ELT(N); break;
2480 case ISD::FCOPYSIGN: R = PromoteFloatRes_FCOPYSIGN(N); break;
2481
2482 // Unary FP Operations
2483 case ISD::FABS:
2484 case ISD::FCBRT:
2485 case ISD::FCEIL:
2486 case ISD::FCOS:
2487 case ISD::FEXP:
2488 case ISD::FEXP2:
2489 case ISD::FEXP10:
2490 case ISD::FFLOOR:
2491 case ISD::FLOG:
2492 case ISD::FLOG2:
2493 case ISD::FLOG10:
2494 case ISD::FNEARBYINT:
2495 case ISD::FNEG:
2496 case ISD::FRINT:
2497 case ISD::FROUND:
2498 case ISD::FROUNDEVEN:
2499 case ISD::FSIN:
2500 case ISD::FSQRT:
2501 case ISD::FTRUNC:
2502 case ISD::FTAN:
2503 case ISD::FCANONICALIZE: R = PromoteFloatRes_UnaryOp(N); break;
2504
2505 // Binary FP Operations
2506 case ISD::FADD:
2507 case ISD::FDIV:
2508 case ISD::FMAXIMUM:
2509 case ISD::FMINIMUM:
2510 case ISD::FMAXNUM:
2511 case ISD::FMINNUM:
2512 case ISD::FMAXNUM_IEEE:
2513 case ISD::FMINNUM_IEEE:
2514 case ISD::FMUL:
2515 case ISD::FPOW:
2516 case ISD::FREM:
2517 case ISD::FSUB: R = PromoteFloatRes_BinOp(N); break;
2518
2519 case ISD::FMA: // FMA is same as FMAD
2520 case ISD::FMAD: R = PromoteFloatRes_FMAD(N); break;
2521
2522 case ISD::FPOWI:
2523 case ISD::FLDEXP: R = PromoteFloatRes_ExpOp(N); break;
2524 case ISD::FFREXP: R = PromoteFloatRes_FFREXP(N); break;
2525
2526 case ISD::FP_ROUND: R = PromoteFloatRes_FP_ROUND(N); break;
2528 R = PromoteFloatRes_STRICT_FP_ROUND(N);
2529 break;
2530 case ISD::LOAD: R = PromoteFloatRes_LOAD(N); break;
2531 case ISD::ATOMIC_LOAD:
2532 R = PromoteFloatRes_ATOMIC_LOAD(N);
2533 break;
2534 case ISD::SELECT: R = PromoteFloatRes_SELECT(N); break;
2535 case ISD::SELECT_CC: R = PromoteFloatRes_SELECT_CC(N); break;
2536
2537 case ISD::SINT_TO_FP:
2538 case ISD::UINT_TO_FP: R = PromoteFloatRes_XINT_TO_FP(N); break;
2539 case ISD::UNDEF: R = PromoteFloatRes_UNDEF(N); break;
2540 case ISD::ATOMIC_SWAP: R = BitcastToInt_ATOMIC_SWAP(N); break;
2547 R = PromoteFloatRes_VECREDUCE(N);
2548 break;
2551 R = PromoteFloatRes_VECREDUCE_SEQ(N);
2552 break;
2553 }
2554
2555 if (R.getNode())
2556 SetPromotedFloat(SDValue(N, ResNo), R);
2557}
2558
2559// Bitcast from i16 to f16: convert the i16 to a f32 value instead.
2560// At this point, it is not possible to determine if the bitcast value is
2561// eventually stored to memory or promoted to f32 or promoted to a floating
2562// point at a higher precision. Some of these cases are handled by FP_EXTEND,
2563// STORE promotion handlers.
2564SDValue DAGTypeLegalizer::PromoteFloatRes_BITCAST(SDNode *N) {
2565 EVT VT = N->getValueType(0);
2566 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
2567 // Input type isn't guaranteed to be a scalar int so bitcast if not. The
2568 // bitcast will be legalized further if necessary.
2569 EVT IVT = EVT::getIntegerVT(*DAG.getContext(),
2570 N->getOperand(0).getValueType().getSizeInBits());
2571 SDValue Cast = DAG.getBitcast(IVT, N->getOperand(0));
2572 return DAG.getNode(GetPromotionOpcode(VT, NVT), SDLoc(N), NVT, Cast);
2573}
2574
2575SDValue DAGTypeLegalizer::PromoteFloatRes_ConstantFP(SDNode *N) {
2576 ConstantFPSDNode *CFPNode = cast<ConstantFPSDNode>(N);
2577 EVT VT = N->getValueType(0);
2578 SDLoc DL(N);
2579
2580 // Get the (bit-cast) APInt of the APFloat and build an integer constant
2581 EVT IVT = EVT::getIntegerVT(*DAG.getContext(), VT.getSizeInBits());
2582 SDValue C = DAG.getConstant(CFPNode->getValueAPF().bitcastToAPInt(), DL,
2583 IVT);
2584
2585 // Convert the Constant to the desired FP type
2586 // FIXME We might be able to do the conversion during compilation and get rid
2587 // of it from the object code
2588 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
2589 return DAG.getNode(GetPromotionOpcode(VT, NVT), DL, NVT, C);
2590}
2591
2592// If the Index operand is a constant, try to redirect the extract operation to
2593// the correct legalized vector. If not, bit-convert the input vector to
2594// equivalent integer vector. Extract the element as an (bit-cast) integer
2595// value and convert it to the promoted type.
2596SDValue DAGTypeLegalizer::PromoteFloatRes_EXTRACT_VECTOR_ELT(SDNode *N) {
2597 SDLoc DL(N);
2598
2599 // If the index is constant, try to extract the value from the legalized
2600 // vector type.
2601 if (isa<ConstantSDNode>(N->getOperand(1))) {
2602 SDValue Vec = N->getOperand(0);
2603 SDValue Idx = N->getOperand(1);
2604 EVT VecVT = Vec->getValueType(0);
2605 EVT EltVT = VecVT.getVectorElementType();
2606
2607 uint64_t IdxVal = Idx->getAsZExtVal();
2608
2609 switch (getTypeAction(VecVT)) {
2610 default: break;
2612 SDValue Res = GetScalarizedVector(N->getOperand(0));
2613 ReplaceValueWith(SDValue(N, 0), Res);
2614 return SDValue();
2615 }
2617 Vec = GetWidenedVector(Vec);
2618 SDValue Res = DAG.getNode(N->getOpcode(), DL, EltVT, Vec, Idx);
2619 ReplaceValueWith(SDValue(N, 0), Res);
2620 return SDValue();
2621 }
2623 SDValue Lo, Hi;
2624 GetSplitVector(Vec, Lo, Hi);
2625
2626 uint64_t LoElts = Lo.getValueType().getVectorNumElements();
2627 SDValue Res;
2628 if (IdxVal < LoElts)
2629 Res = DAG.getNode(N->getOpcode(), DL, EltVT, Lo, Idx);
2630 else
2631 Res = DAG.getNode(N->getOpcode(), DL, EltVT, Hi,
2632 DAG.getConstant(IdxVal - LoElts, DL,
2633 Idx.getValueType()));
2634 ReplaceValueWith(SDValue(N, 0), Res);
2635 return SDValue();
2636 }
2637
2638 }
2639 }
2640
2641 // Bit-convert the input vector to the equivalent integer vector
2642 SDValue NewOp = BitConvertVectorToIntegerVector(N->getOperand(0));
2643 EVT IVT = NewOp.getValueType().getVectorElementType();
2644
2645 // Extract the element as an (bit-cast) integer value
2646 SDValue NewVal = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, IVT,
2647 NewOp, N->getOperand(1));
2648
2649 // Convert the element to the desired FP type
2650 EVT VT = N->getValueType(0);
2651 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
2652 return DAG.getNode(GetPromotionOpcode(VT, NVT), SDLoc(N), NVT, NewVal);
2653}
2654
2655// FCOPYSIGN(X, Y) returns the value of X with the sign of Y. If the result
2656// needs promotion, so does the argument X. Note that Y, if needed, will be
2657// handled during operand promotion.
2658SDValue DAGTypeLegalizer::PromoteFloatRes_FCOPYSIGN(SDNode *N) {
2659 EVT VT = N->getValueType(0);
2660 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
2661 SDValue Op0 = GetPromotedFloat(N->getOperand(0));
2662
2663 SDValue Op1 = N->getOperand(1);
2664
2665 return DAG.getNode(N->getOpcode(), SDLoc(N), NVT, Op0, Op1);
2666}
2667
2668// Unary operation where the result and the operand have PromoteFloat type
2669// action. Construct a new SDNode with the promoted float value of the old
2670// operand.
2671SDValue DAGTypeLegalizer::PromoteFloatRes_UnaryOp(SDNode *N) {
2672 EVT VT = N->getValueType(0);
2673 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
2674 SDValue Op = GetPromotedFloat(N->getOperand(0));
2675
2676 return DAG.getNode(N->getOpcode(), SDLoc(N), NVT, Op);
2677}
2678
2679// Binary operations where the result and both operands have PromoteFloat type
2680// action. Construct a new SDNode with the promoted float values of the old
2681// operands.
2682SDValue DAGTypeLegalizer::PromoteFloatRes_BinOp(SDNode *N) {
2683 EVT VT = N->getValueType(0);
2684 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
2685 SDValue Op0 = GetPromotedFloat(N->getOperand(0));
2686 SDValue Op1 = GetPromotedFloat(N->getOperand(1));
2687 return DAG.getNode(N->getOpcode(), SDLoc(N), NVT, Op0, Op1, N->getFlags());
2688}
2689
2690SDValue DAGTypeLegalizer::PromoteFloatRes_FMAD(SDNode *N) {
2691 EVT VT = N->getValueType(0);
2692 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
2693 SDValue Op0 = GetPromotedFloat(N->getOperand(0));
2694 SDValue Op1 = GetPromotedFloat(N->getOperand(1));
2695 SDValue Op2 = GetPromotedFloat(N->getOperand(2));
2696
2697 return DAG.getNode(N->getOpcode(), SDLoc(N), NVT, Op0, Op1, Op2);
2698}
2699
2700// Promote the Float (first) operand and retain the Integer (second) operand
2701SDValue DAGTypeLegalizer::PromoteFloatRes_ExpOp(SDNode *N) {
2702 EVT VT = N->getValueType(0);
2703 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
2704 SDValue Op0 = GetPromotedFloat(N->getOperand(0));
2705 SDValue Op1 = N->getOperand(1);
2706
2707 return DAG.getNode(N->getOpcode(), SDLoc(N), NVT, Op0, Op1);
2708}
2709
2710SDValue DAGTypeLegalizer::PromoteFloatRes_FFREXP(SDNode *N) {
2711 EVT VT = N->getValueType(0);
2712 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
2713 SDValue Op = GetPromotedFloat(N->getOperand(0));
2714 SDValue Res =
2715 DAG.getNode(N->getOpcode(), SDLoc(N), {NVT, N->getValueType(1)}, Op);
2716
2717 ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
2718 return Res;
2719}
2720
2721// Explicit operation to reduce precision. Reduce the value to half precision
2722// and promote it back to the legal type.
2723SDValue DAGTypeLegalizer::PromoteFloatRes_FP_ROUND(SDNode *N) {
2724 SDLoc DL(N);
2725
2726 SDValue Op = N->getOperand(0);
2727 EVT VT = N->getValueType(0);
2728 EVT OpVT = Op->getValueType(0);
2729 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
2730 EVT IVT = EVT::getIntegerVT(*DAG.getContext(), VT.getSizeInBits());
2731
2732 // Round promoted float to desired precision
2733 SDValue Round = DAG.getNode(GetPromotionOpcode(OpVT, VT), DL, IVT, Op);
2734 // Promote it back to the legal output type
2735 return DAG.getNode(GetPromotionOpcode(VT, NVT), DL, NVT, Round);
2736}
2737
2738// Explicit operation to reduce precision. Reduce the value to half precision
2739// and promote it back to the legal type.
2740SDValue DAGTypeLegalizer::PromoteFloatRes_STRICT_FP_ROUND(SDNode *N) {
2741 SDLoc DL(N);
2742
2743 SDValue Chain = N->getOperand(0);
2744 SDValue Op = N->getOperand(1);
2745 EVT VT = N->getValueType(0);
2746 EVT OpVT = Op->getValueType(0);
2747 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
2748 EVT IVT = EVT::getIntegerVT(*DAG.getContext(), VT.getSizeInBits());
2749
2750 // Round promoted float to desired precision
2751 SDValue Round = DAG.getNode(GetPromotionOpcodeStrict(OpVT, VT), DL,
2752 DAG.getVTList(IVT, MVT::Other), Chain, Op);
2753 // Promote it back to the legal output type
2754 SDValue Res =
2755 DAG.getNode(GetPromotionOpcodeStrict(VT, NVT), DL,
2756 DAG.getVTList(NVT, MVT::Other), Round.getValue(1), Round);
2757 ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
2758 return Res;
2759}
2760
2761SDValue DAGTypeLegalizer::PromoteFloatRes_LOAD(SDNode *N) {
2762 LoadSDNode *L = cast<LoadSDNode>(N);
2763 EVT VT = N->getValueType(0);
2764
2765 // Load the value as an integer value with the same number of bits.
2766 EVT IVT = EVT::getIntegerVT(*DAG.getContext(), VT.getSizeInBits());
2767 SDValue newL = DAG.getLoad(
2768 L->getAddressingMode(), L->getExtensionType(), IVT, SDLoc(N),
2769 L->getChain(), L->getBasePtr(), L->getOffset(), L->getPointerInfo(), IVT,
2770 L->getOriginalAlign(), L->getMemOperand()->getFlags(), L->getAAInfo());
2771 // Legalize the chain result by replacing uses of the old value chain with the
2772 // new one
2773 ReplaceValueWith(SDValue(N, 1), newL.getValue(1));
2774
2775 // Convert the integer value to the desired FP type
2776 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
2777 return DAG.getNode(GetPromotionOpcode(VT, NVT), SDLoc(N), NVT, newL);
2778}
2779
2780SDValue DAGTypeLegalizer::PromoteFloatRes_ATOMIC_LOAD(SDNode *N) {
2781 AtomicSDNode *AM = cast<AtomicSDNode>(N);
2782 EVT VT = AM->getValueType(0);
2783
2784 // Load the value as an integer value with the same number of bits.
2785 EVT IVT = EVT::getIntegerVT(*DAG.getContext(), VT.getSizeInBits());
2786 SDValue newL = DAG.getAtomic(
2787 ISD::ATOMIC_LOAD, SDLoc(N), IVT, DAG.getVTList(IVT, MVT::Other),
2788 {AM->getChain(), AM->getBasePtr()}, AM->getMemOperand());
2789
2790 // Legalize the chain result by replacing uses of the old value chain with the
2791 // new one
2792 ReplaceValueWith(SDValue(N, 1), newL.getValue(1));
2793
2794 // Convert the integer value to the desired FP type
2795 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
2796 return DAG.getNode(GetPromotionOpcode(VT, IVT), SDLoc(N), NVT, newL);
2797}
2798
2799// Construct a new SELECT node with the promoted true- and false- values.
2800SDValue DAGTypeLegalizer::PromoteFloatRes_SELECT(SDNode *N) {
2801 SDValue TrueVal = GetPromotedFloat(N->getOperand(1));
2802 SDValue FalseVal = GetPromotedFloat(N->getOperand(2));
2803
2804 return DAG.getNode(ISD::SELECT, SDLoc(N), TrueVal->getValueType(0),
2805 N->getOperand(0), TrueVal, FalseVal);
2806}
2807
2808// Construct a new SELECT_CC node with the promoted true- and false- values.
2809// The operands used for comparison are promoted by PromoteFloatOp_SELECT_CC.
2810SDValue DAGTypeLegalizer::PromoteFloatRes_SELECT_CC(SDNode *N) {
2811 SDValue TrueVal = GetPromotedFloat(N->getOperand(2));
2812 SDValue FalseVal = GetPromotedFloat(N->getOperand(3));
2813
2814 return DAG.getNode(ISD::SELECT_CC, SDLoc(N),
2815 TrueVal.getNode()->getValueType(0), N->getOperand(0),
2816 N->getOperand(1), TrueVal, FalseVal, N->getOperand(4));
2817}
2818
2819// Construct a SDNode that transforms the SINT or UINT operand to the promoted
2820// float type.
2821SDValue DAGTypeLegalizer::PromoteFloatRes_XINT_TO_FP(SDNode *N) {
2822 SDLoc DL(N);
2823 EVT VT = N->getValueType(0);
2824 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
2825 SDValue NV = DAG.getNode(N->getOpcode(), DL, NVT, N->getOperand(0));
2826 // Round the value to the desired precision (that of the source type).
2827 return DAG.getNode(
2828 ISD::FP_EXTEND, DL, NVT,
2829 DAG.getNode(ISD::FP_ROUND, DL, VT, NV,
2830 DAG.getIntPtrConstant(0, DL, /*isTarget=*/true)));
2831}
2832
2833SDValue DAGTypeLegalizer::PromoteFloatRes_UNDEF(SDNode *N) {
2834 return DAG.getUNDEF(TLI.getTypeToTransformTo(*DAG.getContext(),
2835 N->getValueType(0)));
2836}
2837
2838SDValue DAGTypeLegalizer::PromoteFloatRes_VECREDUCE(SDNode *N) {
2839 // Expand and promote recursively.
2840 // TODO: This is non-optimal, but dealing with the concurrently happening
2841 // vector-legalization is non-trivial. We could do something similar to
2842 // PromoteFloatRes_EXTRACT_VECTOR_ELT here.
2843 ReplaceValueWith(SDValue(N, 0), TLI.expandVecReduce(N, DAG));
2844 return SDValue();
2845}
2846
2847SDValue DAGTypeLegalizer::PromoteFloatRes_VECREDUCE_SEQ(SDNode *N) {
2848 ReplaceValueWith(SDValue(N, 0), TLI.expandVecReduceSeq(N, DAG));
2849 return SDValue();
2850}
2851
2852SDValue DAGTypeLegalizer::BitcastToInt_ATOMIC_SWAP(SDNode *N) {
2853 EVT VT = N->getValueType(0);
2854
2855 AtomicSDNode *AM = cast<AtomicSDNode>(N);
2856 SDLoc SL(N);
2857
2858 SDValue CastVal = BitConvertToInteger(AM->getVal());
2859 EVT CastVT = CastVal.getValueType();
2860
2861 SDValue NewAtomic
2862 = DAG.getAtomic(ISD::ATOMIC_SWAP, SL, CastVT,
2863 DAG.getVTList(CastVT, MVT::Other),
2864 { AM->getChain(), AM->getBasePtr(), CastVal },
2865 AM->getMemOperand());
2866
2867 SDValue Result = NewAtomic;
2868
2869 if (getTypeAction(VT) == TargetLowering::TypePromoteFloat) {
2870 EVT NFPVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
2871 Result = DAG.getNode(GetPromotionOpcode(VT, NFPVT), SL, NFPVT,
2872 NewAtomic);
2873 }
2874
2875 // Legalize the chain result by replacing uses of the old value chain with the
2876 // new one
2877 ReplaceValueWith(SDValue(N, 1), NewAtomic.getValue(1));
2878
2879 return Result;
2880
2881}
2882
2883//===----------------------------------------------------------------------===//
2884// Half Result Soft Promotion
2885//===----------------------------------------------------------------------===//
2886
2887void DAGTypeLegalizer::SoftPromoteHalfResult(SDNode *N, unsigned ResNo) {
2888 LLVM_DEBUG(dbgs() << "Soft promote half result " << ResNo << ": ";
2889 N->dump(&DAG));
2890 SDValue R = SDValue();
2891
2892 // See if the target wants to custom expand this node.
2893 if (CustomLowerNode(N, N->getValueType(ResNo), true)) {
2894 LLVM_DEBUG(dbgs() << "Node has been custom expanded, done\n");
2895 return;
2896 }
2897
2898 switch (N->getOpcode()) {
2899 default:
2900#ifndef NDEBUG
2901 dbgs() << "SoftPromoteHalfResult #" << ResNo << ": ";
2902 N->dump(&DAG); dbgs() << "\n";
2903#endif
2904 report_fatal_error("Do not know how to soft promote this operator's "
2905 "result!");
2906
2907 case ISD::ARITH_FENCE:
2908 R = SoftPromoteHalfRes_ARITH_FENCE(N); break;
2909 case ISD::BITCAST: R = SoftPromoteHalfRes_BITCAST(N); break;
2910 case ISD::ConstantFP: R = SoftPromoteHalfRes_ConstantFP(N); break;
2912 R = SoftPromoteHalfRes_EXTRACT_VECTOR_ELT(N); break;
2913 case ISD::FCOPYSIGN: R = SoftPromoteHalfRes_FCOPYSIGN(N); break;
2915 case ISD::FP_ROUND: R = SoftPromoteHalfRes_FP_ROUND(N); break;
2916
2917 // Unary FP Operations
2918 case ISD::FABS:
2919 case ISD::FCBRT:
2920 case ISD::FCEIL:
2921 case ISD::FCOS:
2922 case ISD::FEXP:
2923 case ISD::FEXP2:
2924 case ISD::FEXP10:
2925 case ISD::FFLOOR:
2926 case ISD::FLOG:
2927 case ISD::FLOG2:
2928 case ISD::FLOG10:
2929 case ISD::FNEARBYINT:
2930 case ISD::FNEG:
2931 case ISD::FREEZE:
2932 case ISD::FRINT:
2933 case ISD::FROUND:
2934 case ISD::FROUNDEVEN:
2935 case ISD::FSIN:
2936 case ISD::FSQRT:
2937 case ISD::FTRUNC:
2938 case ISD::FTAN:
2939 case ISD::FCANONICALIZE: R = SoftPromoteHalfRes_UnaryOp(N); break;
2940
2941 // Binary FP Operations
2942 case ISD::FADD:
2943 case ISD::FDIV:
2944 case ISD::FMAXIMUM:
2945 case ISD::FMINIMUM:
2946 case ISD::FMAXNUM:
2947 case ISD::FMINNUM:
2948 case ISD::FMUL:
2949 case ISD::FPOW:
2950 case ISD::FREM:
2951 case ISD::FSUB: R = SoftPromoteHalfRes_BinOp(N); break;
2952
2953 case ISD::FMA: // FMA is same as FMAD
2954 case ISD::FMAD: R = SoftPromoteHalfRes_FMAD(N); break;
2955
2956 case ISD::FPOWI:
2957 case ISD::FLDEXP: R = SoftPromoteHalfRes_ExpOp(N); break;
2958
2959 case ISD::FFREXP: R = SoftPromoteHalfRes_FFREXP(N); break;
2960
2961 case ISD::LOAD: R = SoftPromoteHalfRes_LOAD(N); break;
2962 case ISD::ATOMIC_LOAD:
2963 R = SoftPromoteHalfRes_ATOMIC_LOAD(N);
2964 break;
2965 case ISD::SELECT: R = SoftPromoteHalfRes_SELECT(N); break;
2966 case ISD::SELECT_CC: R = SoftPromoteHalfRes_SELECT_CC(N); break;
2967 case ISD::SINT_TO_FP:
2968 case ISD::UINT_TO_FP: R = SoftPromoteHalfRes_XINT_TO_FP(N); break;
2969 case ISD::UNDEF: R = SoftPromoteHalfRes_UNDEF(N); break;
2970 case ISD::ATOMIC_SWAP: R = BitcastToInt_ATOMIC_SWAP(N); break;
2977 R = SoftPromoteHalfRes_VECREDUCE(N);
2978 break;
2981 R = SoftPromoteHalfRes_VECREDUCE_SEQ(N);
2982 break;
2983 }
2984
2985 if (R.getNode())
2986 SetSoftPromotedHalf(SDValue(N, ResNo), R);
2987}
2988
2989SDValue DAGTypeLegalizer::SoftPromoteHalfRes_ARITH_FENCE(SDNode *N) {
2990 return DAG.getNode(ISD::ARITH_FENCE, SDLoc(N), MVT::i16,
2991 BitConvertToInteger(N->getOperand(0)));
2992}
2993
2994SDValue DAGTypeLegalizer::SoftPromoteHalfRes_BITCAST(SDNode *N) {
2995 return BitConvertToInteger(N->getOperand(0));
2996}
2997
2998SDValue DAGTypeLegalizer::SoftPromoteHalfRes_ConstantFP(SDNode *N) {
2999 ConstantFPSDNode *CN = cast<ConstantFPSDNode>(N);
3000
3001 // Get the (bit-cast) APInt of the APFloat and build an integer constant
3002 return DAG.getConstant(CN->getValueAPF().bitcastToAPInt(), SDLoc(CN),
3003 MVT::i16);
3004}
3005
3006SDValue DAGTypeLegalizer::SoftPromoteHalfRes_EXTRACT_VECTOR_ELT(SDNode *N) {
3007 SDValue NewOp = BitConvertVectorToIntegerVector(N->getOperand(0));
3009 NewOp.getValueType().getVectorElementType(), NewOp,
3010 N->getOperand(1));
3011}
3012
3013SDValue DAGTypeLegalizer::SoftPromoteHalfRes_FCOPYSIGN(SDNode *N) {
3014 SDValue LHS = GetSoftPromotedHalf(N->getOperand(0));
3015 SDValue RHS = BitConvertToInteger(N->getOperand(1));
3016 SDLoc dl(N);
3017
3018 EVT LVT = LHS.getValueType();
3019 EVT RVT = RHS.getValueType();
3020
3021 unsigned LSize = LVT.getSizeInBits();
3022 unsigned RSize = RVT.getSizeInBits();
3023
3024 // First get the sign bit of second operand.
3025 SDValue SignBit = DAG.getNode(
3026 ISD::SHL, dl, RVT, DAG.getConstant(1, dl, RVT),
3027 DAG.getConstant(RSize - 1, dl,
3028 TLI.getShiftAmountTy(RVT, DAG.getDataLayout())));
3029 SignBit = DAG.getNode(ISD::AND, dl, RVT, RHS, SignBit);
3030
3031 // Shift right or sign-extend it if the two operands have different types.
3032 int SizeDiff = RVT.getSizeInBits() - LVT.getSizeInBits();
3033 if (SizeDiff > 0) {
3034 SignBit =
3035 DAG.getNode(ISD::SRL, dl, RVT, SignBit,
3036 DAG.getConstant(SizeDiff, dl,
3037 TLI.getShiftAmountTy(SignBit.getValueType(),
3038 DAG.getDataLayout())));
3039 SignBit = DAG.getNode(ISD::TRUNCATE, dl, LVT, SignBit);
3040 } else if (SizeDiff < 0) {
3041 SignBit = DAG.getNode(ISD::ANY_EXTEND, dl, LVT, SignBit);
3042 SignBit =
3043 DAG.getNode(ISD::SHL, dl, LVT, SignBit,
3044 DAG.getConstant(-SizeDiff, dl,
3045 TLI.getShiftAmountTy(SignBit.getValueType(),
3046 DAG.getDataLayout())));
3047 }
3048
3049 // Clear the sign bit of the first operand.
3050 SDValue Mask = DAG.getNode(
3051 ISD::SHL, dl, LVT, DAG.getConstant(1, dl, LVT),
3052 DAG.getConstant(LSize - 1, dl,
3053 TLI.getShiftAmountTy(LVT, DAG.getDataLayout())));
3054 Mask = DAG.getNode(ISD::SUB, dl, LVT, Mask, DAG.getConstant(1, dl, LVT));
3055 LHS = DAG.getNode(ISD::AND, dl, LVT, LHS, Mask);
3056
3057 // Or the value with the sign bit.
3058 return DAG.getNode(ISD::OR, dl, LVT, LHS, SignBit);
3059}
3060
3061SDValue DAGTypeLegalizer::SoftPromoteHalfRes_FMAD(SDNode *N) {
3062 EVT OVT = N->getValueType(0);
3063 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), OVT);
3064 SDValue Op0 = GetSoftPromotedHalf(N->getOperand(0));
3065 SDValue Op1 = GetSoftPromotedHalf(N->getOperand(1));
3066 SDValue Op2 = GetSoftPromotedHalf(N->getOperand(2));
3067 SDLoc dl(N);
3068
3069 // Promote to the larger FP type.
3070 auto PromotionOpcode = GetPromotionOpcode(OVT, NVT);
3071 Op0 = DAG.getNode(PromotionOpcode, dl, NVT, Op0);
3072 Op1 = DAG.getNode(PromotionOpcode, dl, NVT, Op1);
3073 Op2 = DAG.getNode(PromotionOpcode, dl, NVT, Op2);
3074
3075 SDValue Res = DAG.getNode(N->getOpcode(), dl, NVT, Op0, Op1, Op2);
3076
3077 // Convert back to FP16 as an integer.
3078 return DAG.getNode(GetPromotionOpcode(NVT, OVT), dl, MVT::i16, Res);
3079}
3080
3081SDValue DAGTypeLegalizer::SoftPromoteHalfRes_ExpOp(SDNode *N) {
3082 EVT OVT = N->getValueType(0);
3083 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), OVT);
3084 SDValue Op0 = GetSoftPromotedHalf(N->getOperand(0));
3085 SDValue Op1 = N->getOperand(1);
3086 SDLoc dl(N);
3087
3088 // Promote to the larger FP type.
3089 Op0 = DAG.getNode(GetPromotionOpcode(OVT, NVT), dl, NVT, Op0);
3090
3091 SDValue Res = DAG.getNode(N->getOpcode(), dl, NVT, Op0, Op1);
3092
3093 // Convert back to FP16 as an integer.
3094 return DAG.getNode(GetPromotionOpcode(NVT, OVT), dl, MVT::i16, Res);
3095}
3096
3097SDValue DAGTypeLegalizer::SoftPromoteHalfRes_FFREXP(SDNode *N) {
3098 EVT OVT = N->getValueType(0);
3099 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), OVT);
3100 SDValue Op = GetSoftPromotedHalf(N->getOperand(0));
3101 SDLoc dl(N);
3102
3103 // Promote to the larger FP type.
3104 Op = DAG.getNode(GetPromotionOpcode(OVT, NVT), dl, NVT, Op);
3105
3106 SDValue Res = DAG.getNode(N->getOpcode(), dl,
3107 DAG.getVTList(NVT, N->getValueType(1)), Op);
3108
3109 ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
3110
3111 // Convert back to FP16 as an integer.
3112 return DAG.getNode(GetPromotionOpcode(NVT, OVT), dl, MVT::i16, Res);
3113}
3114
3115SDValue DAGTypeLegalizer::SoftPromoteHalfRes_FP_ROUND(SDNode *N) {
3116 EVT RVT = N->getValueType(0);
3117 EVT SVT = N->getOperand(0).getValueType();
3118
3119 if (N->isStrictFPOpcode()) {
3120 // FIXME: assume we only have two f16 variants for now.
3121 unsigned Opcode;
3122 if (RVT == MVT::f16)
3123 Opcode = ISD::STRICT_FP_TO_FP16;
3124 else if (RVT == MVT::bf16)
3125 Opcode = ISD::STRICT_FP_TO_BF16;
3126 else
3127 llvm_unreachable("unknown half type");
3128 SDValue Res = DAG.getNode(Opcode, SDLoc(N), {MVT::i16, MVT::Other},
3129 {N->getOperand(0), N->getOperand(1)});
3130 ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
3131 return Res;
3132 }
3133
3134 return DAG.getNode(GetPromotionOpcode(SVT, RVT), SDLoc(N), MVT::i16,
3135 N->getOperand(0));
3136}
3137
3138SDValue DAGTypeLegalizer::SoftPromoteHalfRes_LOAD(SDNode *N) {
3139 LoadSDNode *L = cast<LoadSDNode>(N);
3140
3141 // Load the value as an integer value with the same number of bits.
3142 assert(L->getExtensionType() == ISD::NON_EXTLOAD && "Unexpected extension!");
3143 SDValue NewL =
3144 DAG.getLoad(L->getAddressingMode(), L->getExtensionType(), MVT::i16,
3145 SDLoc(N), L->getChain(), L->getBasePtr(), L->getOffset(),
3146 L->getPointerInfo(), MVT::i16, L->getOriginalAlign(),
3147 L->getMemOperand()->getFlags(), L->getAAInfo());
3148 // Legalize the chain result by replacing uses of the old value chain with the
3149 // new one
3150 ReplaceValueWith(SDValue(N, 1), NewL.getValue(1));
3151 return NewL;
3152}
3153
3154SDValue DAGTypeLegalizer::SoftPromoteHalfRes_ATOMIC_LOAD(SDNode *N) {
3155 AtomicSDNode *AM = cast<AtomicSDNode>(N);
3156
3157 // Load the value as an integer value with the same number of bits.
3158 SDValue NewL = DAG.getAtomic(
3159 ISD::ATOMIC_LOAD, SDLoc(N), MVT::i16, DAG.getVTList(MVT::i16, MVT::Other),
3160 {AM->getChain(), AM->getBasePtr()}, AM->getMemOperand());
3161
3162 // Legalize the chain result by replacing uses of the old value chain with the
3163 // new one
3164 ReplaceValueWith(SDValue(N, 1), NewL.getValue(1));
3165 return NewL;
3166}
3167
3168SDValue DAGTypeLegalizer::SoftPromoteHalfRes_SELECT(SDNode *N) {
3169 SDValue Op1 = GetSoftPromotedHalf(N->getOperand(1));
3170 SDValue Op2 = GetSoftPromotedHalf(N->getOperand(2));
3171 return DAG.getSelect(SDLoc(N), Op1.getValueType(), N->getOperand(0), Op1,
3172 Op2);
3173}
3174
3175SDValue DAGTypeLegalizer::SoftPromoteHalfRes_SELECT_CC(SDNode *N) {
3176 SDValue Op2 = GetSoftPromotedHalf(N->getOperand(2));
3177 SDValue Op3 = GetSoftPromotedHalf(N->getOperand(3));
3178 return DAG.getNode(ISD::SELECT_CC, SDLoc(N), Op2.getValueType(),
3179 N->getOperand(0), N->getOperand(1), Op2, Op3,
3180 N->getOperand(4));
3181}
3182
3183SDValue DAGTypeLegalizer::SoftPromoteHalfRes_XINT_TO_FP(SDNode *N) {
3184 EVT OVT = N->getValueType(0);
3185 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), OVT);
3186 SDLoc dl(N);
3187
3188 SDValue Res = DAG.getNode(N->getOpcode(), dl, NVT, N->getOperand(0));
3189
3190 // Round the value to the softened type.
3191 return DAG.getNode(GetPromotionOpcode(NVT, OVT), dl, MVT::i16, Res);
3192}
3193
3194SDValue DAGTypeLegalizer::SoftPromoteHalfRes_UNDEF(SDNode *N) {
3195 return DAG.getUNDEF(MVT::i16);
3196}
3197
3198SDValue DAGTypeLegalizer::SoftPromoteHalfRes_UnaryOp(SDNode *N) {
3199 EVT OVT = N->getValueType(0);
3200 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), OVT);
3201 SDValue Op = GetSoftPromotedHalf(N->getOperand(0));
3202 SDLoc dl(N);
3203
3204 // Promote to the larger FP type.
3205 Op = DAG.getNode(GetPromotionOpcode(OVT, NVT), dl, NVT, Op);
3206
3207 SDValue Res = DAG.getNode(N->getOpcode(), dl, NVT, Op);
3208
3209 // Convert back to FP16 as an integer.
3210 return DAG.getNode(GetPromotionOpcode(NVT, OVT), dl, MVT::i16, Res);
3211}
3212
3213SDValue DAGTypeLegalizer::SoftPromoteHalfRes_BinOp(SDNode *N) {
3214 EVT OVT = N->getValueType(0);
3215 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), OVT);
3216 SDValue Op0 = GetSoftPromotedHalf(N->getOperand(0));
3217 SDValue Op1 = GetSoftPromotedHalf(N->getOperand(1));
3218 SDLoc dl(N);
3219
3220 // Promote to the larger FP type.
3221 auto PromotionOpcode = GetPromotionOpcode(OVT, NVT);
3222 Op0 = DAG.getNode(PromotionOpcode, dl, NVT, Op0);
3223 Op1 = DAG.getNode(PromotionOpcode, dl, NVT, Op1);
3224
3225 SDValue Res = DAG.getNode(N->getOpcode(), dl, NVT, Op0, Op1);
3226
3227 // Convert back to FP16 as an integer.
3228 return DAG.getNode(GetPromotionOpcode(NVT, OVT), dl, MVT::i16, Res);
3229}
3230
3231SDValue DAGTypeLegalizer::SoftPromoteHalfRes_VECREDUCE(SDNode *N) {
3232 // Expand and soften recursively.
3233 ReplaceValueWith(SDValue(N, 0), TLI.expandVecReduce(N, DAG));
3234 return SDValue();
3235}
3236
3237SDValue DAGTypeLegalizer::SoftPromoteHalfRes_VECREDUCE_SEQ(SDNode *N) {
3238 // Expand and soften.
3239 ReplaceValueWith(SDValue(N, 0), TLI.expandVecReduceSeq(N, DAG));
3240 return SDValue();
3241}
3242
3243//===----------------------------------------------------------------------===//
3244// Half Operand Soft Promotion
3245//===----------------------------------------------------------------------===//
3246
3247bool DAGTypeLegalizer::SoftPromoteHalfOperand(SDNode *N, unsigned OpNo) {
3248 LLVM_DEBUG(dbgs() << "Soft promote half operand " << OpNo << ": ";
3249 N->dump(&DAG));
3250 SDValue Res = SDValue();
3251
3252 if (CustomLowerNode(N, N->getOperand(OpNo).getValueType(), false)) {
3253 LLVM_DEBUG(dbgs() << "Node has been custom lowered, done\n");
3254 return false;
3255 }
3256
3257 // Nodes that use a promotion-requiring floating point operand, but doesn't
3258 // produce a soft promotion-requiring floating point result, need to be
3259 // legalized to use the soft promoted float operand. Nodes that produce at
3260 // least one soft promotion-requiring floating point result have their
3261 // operands legalized as a part of PromoteFloatResult.
3262 switch (N->getOpcode()) {
3263 default:
3264 #ifndef NDEBUG
3265 dbgs() << "SoftPromoteHalfOperand Op #" << OpNo << ": ";
3266 N->dump(&DAG); dbgs() << "\n";
3267 #endif
3268 report_fatal_error("Do not know how to soft promote this operator's "
3269 "operand!");
3270
3271 case ISD::BITCAST: Res = SoftPromoteHalfOp_BITCAST(N); break;
3272 case ISD::FCOPYSIGN: Res = SoftPromoteHalfOp_FCOPYSIGN(N, OpNo); break;
3273 case ISD::FP_TO_SINT:
3274 case ISD::FP_TO_UINT: Res = SoftPromoteHalfOp_FP_TO_XINT(N); break;
3277 Res = SoftPromoteHalfOp_FP_TO_XINT_SAT(N); break;
3279 case ISD::FP_EXTEND: Res = SoftPromoteHalfOp_FP_EXTEND(N); break;
3280 case ISD::SELECT_CC: Res = SoftPromoteHalfOp_SELECT_CC(N, OpNo); break;
3281 case ISD::SETCC: Res = SoftPromoteHalfOp_SETCC(N); break;
3282 case ISD::STORE: Res = SoftPromoteHalfOp_STORE(N, OpNo); break;
3283 case ISD::ATOMIC_STORE:
3284 Res = SoftPromoteHalfOp_ATOMIC_STORE(N, OpNo);
3285 break;
3286 case ISD::STACKMAP:
3287 Res = SoftPromoteHalfOp_STACKMAP(N, OpNo);
3288 break;
3289 case ISD::PATCHPOINT:
3290 Res = SoftPromoteHalfOp_PATCHPOINT(N, OpNo);
3291 break;
3292 }
3293
3294 if (!Res.getNode())
3295 return false;
3296
3297 assert(Res.getNode() != N && "Expected a new node!");
3298
3299 assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 &&
3300 "Invalid operand expansion");
3301
3302 ReplaceValueWith(SDValue(N, 0), Res);
3303 return false;
3304}
3305
3306SDValue DAGTypeLegalizer::SoftPromoteHalfOp_BITCAST(SDNode *N) {
3307 SDValue Op0 = GetSoftPromotedHalf(N->getOperand(0));
3308
3309 return DAG.getNode(ISD::BITCAST, SDLoc(N), N->getValueType(0), Op0);
3310}
3311
3312SDValue DAGTypeLegalizer::SoftPromoteHalfOp_FCOPYSIGN(SDNode *N,
3313 unsigned OpNo) {
3314 assert(OpNo == 1 && "Only Operand 1 must need promotion here");
3315 SDValue Op1 = N->getOperand(1);
3316 EVT RVT = Op1.getValueType();
3317 SDLoc dl(N);
3318
3319 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), Op1.getValueType());
3320
3321 Op1 = GetSoftPromotedHalf(Op1);
3322 Op1 = DAG.getNode(GetPromotionOpcode(RVT, NVT), dl, NVT, Op1);
3323
3324 return DAG.getNode(N->getOpcode(), dl, N->getValueType(0), N->getOperand(0),
3325 Op1);
3326}
3327
3328SDValue DAGTypeLegalizer::SoftPromoteHalfOp_FP_EXTEND(SDNode *N) {
3329 EVT RVT = N->getValueType(0);
3330 bool IsStrict = N->isStrictFPOpcode();
3331 SDValue Op = N->getOperand(IsStrict ? 1 : 0);
3332 EVT SVT = Op.getValueType();
3333 Op = GetSoftPromotedHalf(N->getOperand(IsStrict ? 1 : 0));
3334
3335 if (IsStrict) {
3336 unsigned Opcode;
3337 if (SVT == MVT::f16)
3338 Opcode = ISD::STRICT_FP16_TO_FP;
3339 else if (SVT == MVT::bf16)
3340 Opcode = ISD::STRICT_BF16_TO_FP;
3341 else
3342 llvm_unreachable("unknown half type");
3343 SDValue Res =
3344 DAG.getNode(Opcode, SDLoc(N), {N->getValueType(0), MVT::Other},
3345 {N->getOperand(0), Op});
3346 ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
3347 ReplaceValueWith(SDValue(N, 0), Res);
3348 return SDValue();
3349 }
3350
3351 return DAG.getNode(GetPromotionOpcode(SVT, RVT), SDLoc(N), RVT, Op);
3352}
3353
3354SDValue DAGTypeLegalizer::SoftPromoteHalfOp_FP_TO_XINT(SDNode *N) {
3355 EVT RVT = N->getValueType(0);
3356 SDValue Op = N->getOperand(0);
3357 EVT SVT = Op.getValueType();
3358 SDLoc dl(N);
3359
3360 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), Op.getValueType());
3361
3362 Op = GetSoftPromotedHalf(Op);
3363
3364 SDValue Res = DAG.getNode(GetPromotionOpcode(SVT, RVT), dl, NVT, Op);
3365
3366 return DAG.getNode(N->getOpcode(), dl, N->getValueType(0), Res);
3367}
3368
3369SDValue DAGTypeLegalizer::SoftPromoteHalfOp_FP_TO_XINT_SAT(SDNode *N) {
3370 EVT RVT = N->getValueType(0);
3371 SDValue Op = N->getOperand(0);
3372 EVT SVT = Op.getValueType();
3373 SDLoc dl(N);
3374
3375 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), Op.getValueType());
3376
3377 Op = GetSoftPromotedHalf(Op);
3378
3379 SDValue Res = DAG.getNode(GetPromotionOpcode(SVT, RVT), dl, NVT, Op);
3380
3381 return DAG.getNode(N->getOpcode(), dl, N->getValueType(0), Res,
3382 N->getOperand(1));
3383}
3384
3385SDValue DAGTypeLegalizer::SoftPromoteHalfOp_SELECT_CC(SDNode *N,
3386 unsigned OpNo) {
3387 assert(OpNo == 0 && "Can only soften the comparison values");
3388 SDValue Op0 = N->getOperand(0);
3389 SDValue Op1 = N->getOperand(1);
3390 SDLoc dl(N);
3391
3392 EVT SVT = Op0.getValueType();
3393 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), SVT);
3394
3395 Op0 = GetSoftPromotedHalf(Op0);
3396 Op1 = GetSoftPromotedHalf(Op1);
3397
3398 // Promote to the larger FP type.
3399 auto PromotionOpcode = GetPromotionOpcode(SVT, NVT);
3400 Op0 = DAG.getNode(PromotionOpcode, dl, NVT, Op0);
3401 Op1 = DAG.getNode(PromotionOpcode, dl, NVT, Op1);
3402
3403 return DAG.getNode(ISD::SELECT_CC, SDLoc(N), N->getValueType(0), Op0, Op1,
3404 N->getOperand(2), N->getOperand(3), N->getOperand(4));
3405}
3406
3407SDValue DAGTypeLegalizer::SoftPromoteHalfOp_SETCC(SDNode *N) {
3408 SDValue Op0 = N->getOperand(0);
3409 SDValue Op1 = N->getOperand(1);
3410 ISD::CondCode CCCode = cast<CondCodeSDNode>(N->getOperand(2))->get();
3411 SDLoc dl(N);
3412
3413 EVT SVT = Op0.getValueType();
3414 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), Op0.getValueType());
3415
3416 Op0 = GetSoftPromotedHalf(Op0);
3417 Op1 = GetSoftPromotedHalf(Op1);
3418
3419 // Promote to the larger FP type.
3420 auto PromotionOpcode = GetPromotionOpcode(SVT, NVT);
3421 Op0 = DAG.getNode(PromotionOpcode, dl, NVT, Op0);
3422 Op1 = DAG.getNode(PromotionOpcode, dl, NVT, Op1);
3423
3424 return DAG.getSetCC(SDLoc(N), N->getValueType(0), Op0, Op1, CCCode);
3425}
3426
3427SDValue DAGTypeLegalizer::SoftPromoteHalfOp_STORE(SDNode *N, unsigned OpNo) {
3428 assert(OpNo == 1 && "Can only soften the stored value!");
3429 StoreSDNode *ST = cast<StoreSDNode>(N);
3430 SDValue Val = ST->getValue();
3431 SDLoc dl(N);
3432
3433 assert(!ST->isTruncatingStore() && "Unexpected truncating store.");
3434 SDValue Promoted = GetSoftPromotedHalf(Val);
3435 return DAG.getStore(ST->getChain(), dl, Promoted, ST->getBasePtr(),
3436 ST->getMemOperand());
3437}
3438
3439SDValue DAGTypeLegalizer::SoftPromoteHalfOp_ATOMIC_STORE(SDNode *N,
3440 unsigned OpNo) {
3441 assert(OpNo == 1 && "Can only soften the stored value!");
3442 AtomicSDNode *ST = cast<AtomicSDNode>(N);
3443 SDValue Val = ST->getVal();
3444 SDLoc dl(N);
3445
3446 SDValue Promoted = GetSoftPromotedHalf(Val);
3447 return DAG.getAtomic(ISD::ATOMIC_STORE, dl, Promoted.getValueType(),
3448 ST->getChain(), Promoted, ST->getBasePtr(),
3449 ST->getMemOperand());
3450}
3451
3452SDValue DAGTypeLegalizer::SoftPromoteHalfOp_STACKMAP(SDNode *N, unsigned OpNo) {
3453 assert(OpNo > 1); // Because the first two arguments are guaranteed legal.
3454 SmallVector<SDValue> NewOps(N->ops().begin(), N->ops().end());
3455 SDValue Op = N->getOperand(OpNo);
3456 NewOps[OpNo] = GetSoftPromotedHalf(Op);
3458 DAG.getNode(N->getOpcode(), SDLoc(N), N->getVTList(), NewOps);
3459
3460 for (unsigned ResNum = 0; ResNum < N->getNumValues(); ResNum++)
3461 ReplaceValueWith(SDValue(N, ResNum), NewNode.getValue(ResNum));
3462
3463 return SDValue(); // Signal that we replaced the node ourselves.
3464}
3465
3466SDValue DAGTypeLegalizer::SoftPromoteHalfOp_PATCHPOINT(SDNode *N,
3467 unsigned OpNo) {
3468 assert(OpNo >= 7);
3469 SmallVector<SDValue> NewOps(N->ops().begin(), N->ops().end());
3470 SDValue Op = N->getOperand(OpNo);
3471 NewOps[OpNo] = GetSoftPromotedHalf(Op);
3473 DAG.getNode(N->getOpcode(), SDLoc(N), N->getVTList(), NewOps);
3474
3475 for (unsigned ResNum = 0; ResNum < N->getNumValues(); ResNum++)
3476 ReplaceValueWith(SDValue(N, ResNum), NewNode.getValue(ResNum));
3477
3478 return SDValue(); // Signal that we replaced the node ourselves.
3479}
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
DXIL Intrinsic Expansion
Returns the sub type a function will return at a given Idx Should correspond to the result type of an ExtractValue instruction executed with just that one unsigned Idx
#define LLVM_DEBUG(X)
Definition: Debug.h:101
uint64_t Size
static bool isSigned(unsigned int Opcode)
static RTLIB::Libcall findFPToIntLibcall(EVT SrcVT, EVT RetVT, EVT &Promoted, bool Signed)
static RTLIB::Libcall GetFPLibCall(EVT VT, RTLIB::Libcall Call_F32, RTLIB::Libcall Call_F64, RTLIB::Libcall Call_F80, RTLIB::Libcall Call_F128, RTLIB::Libcall Call_PPCF128)
GetFPLibCall - Return the right libcall for the given floating point type.
static ISD::NodeType GetPromotionOpcode(EVT OpVT, EVT RetVT)
static ISD::NodeType GetPromotionOpcodeStrict(EVT OpVT, EVT RetVT)
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
Value * RHS
Value * LHS
APInt bitcastToAPInt() const
Definition: APFloat.h:1254
Class for arbitrary precision integers.
Definition: APInt.h:77
static APInt getAllOnes(unsigned numBits)
Return an APInt of a specified width with all bits set.
Definition: APInt.h:213
void clearBit(unsigned BitPosition)
Set a given bit to 0.
Definition: APInt.h:1386
static APInt getSignMask(unsigned BitWidth)
Get the SignMask for a specific bit width.
Definition: APInt.h:208
const uint64_t * getRawData() const
This function returns a pointer to the internal storage of the APInt.
Definition: APInt.h:548
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
Definition: ArrayRef.h:41
This is an SDNode representing atomic operations.
const SDValue & getVal() const
const APFloat & getValueAPF() const
@ NewNode
This is a new node, not before seen, that was created in the process of legalizing some other node.
Definition: LegalizeTypes.h:43
This class represents an Operation in the Expression.
bool isBigEndian() const
Definition: DataLayout.h:239
void emitError(uint64_t LocCookie, const Twine &ErrorStr)
emitError - Emit an error message to the currently installed error handler with optional location inf...
This class is used to represent ISD::LOAD nodes.
SimpleValueType SimpleTy
@ MODereferenceable
The memory access is dereferenceable (i.e., doesn't trap).
@ MOInvariant
The memory access always returns the same value (or traps).
MachineMemOperand * getMemOperand() const
Return a MachineMemOperand object describing the memory reference performed by operation.
Wrapper class for IR location info (IR ordering and DebugLoc) to be passed into SDNode creation funct...
Represents one node in the SelectionDAG.
unsigned getNumValues() const
Return the number of values defined/returned by this operator.
EVT getValueType(unsigned ResNo) const
Return the type of a specified result.
Unlike LLVM values, Selection DAG nodes may return multiple values as the result of a computation.
SDNode * getNode() const
get the SDNode which holds the desired result
SDValue getValue(unsigned R) const
EVT getValueType() const
Return the ValueType of the referenced return value.
SDValue getExtLoad(ISD::LoadExtType ExtType, const SDLoc &dl, EVT VT, SDValue Chain, SDValue Ptr, MachinePointerInfo PtrInfo, EVT MemVT, MaybeAlign Alignment=MaybeAlign(), MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes())
SDVTList getVTList(EVT VT)
Return an SDVTList that represents the list of values specified.
SDValue getSetCC(const SDLoc &DL, EVT VT, SDValue LHS, SDValue RHS, ISD::CondCode Cond, SDValue Chain=SDValue(), bool IsSignaling=false)
Helper function to make it easier to build SetCC's if you just have an ISD::CondCode instead of an SD...
SDValue getConstantFP(double Val, const SDLoc &DL, EVT VT, bool isTarget=false)
Create a ConstantFPSDNode wrapping a constant value.
SDValue getLoad(EVT VT, const SDLoc &dl, SDValue Chain, SDValue Ptr, MachinePointerInfo PtrInfo, MaybeAlign Alignment=MaybeAlign(), MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes(), const MDNode *Ranges=nullptr)
Loads are not normal binary operators: their result type is not determined by their operands,...
SDValue getAtomic(unsigned Opcode, const SDLoc &dl, EVT MemVT, SDValue Chain, SDValue Ptr, SDValue Val, MachineMemOperand *MMO)
Gets a node for an atomic op, produces result (if relevant) and chain and takes 2 operands.
SDValue getUNDEF(EVT VT)
Return an UNDEF node. UNDEF does not have a useful SDLoc.
SDValue getBitcast(EVT VT, SDValue V)
Return a bitcast using the SDLoc of the value operand, and casting to the provided type.
SDValue getSelect(const SDLoc &DL, EVT VT, SDValue Cond, SDValue LHS, SDValue RHS, SDNodeFlags Flags=SDNodeFlags())
Helper function to make it easier to build Select's if you just have operands and don't want to check...
const DataLayout & getDataLayout() const
Definition: SelectionDAG.h:486
SDValue getConstant(uint64_t Val, const SDLoc &DL, EVT VT, bool isTarget=false, bool isOpaque=false)
Create a ConstantSDNode wrapping a constant value.
SDValue getVAArg(EVT VT, const SDLoc &dl, SDValue Chain, SDValue Ptr, SDValue SV, unsigned Align)
VAArg produces a result and token chain, and takes a pointer and a source value as input.
SDValue getTruncStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr, MachinePointerInfo PtrInfo, EVT SVT, Align Alignment, MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes())
SDValue getStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr, MachinePointerInfo PtrInfo, Align Alignment, MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes())
Helper function to build ISD::STORE nodes.
static const fltSemantics & EVTToAPFloatSemantics(EVT VT)
Returns an APFloat semantics tag appropriate for the given type.
SDValue getSelectCC(const SDLoc &DL, SDValue LHS, SDValue RHS, SDValue True, SDValue False, ISD::CondCode Cond)
Helper function to make it easier to build SelectCC's if you just have an ISD::CondCode instead of an...
SDValue getIntPtrConstant(uint64_t Val, const SDLoc &DL, bool isTarget=false)
SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT, ArrayRef< SDUse > Ops)
Gets or creates the specified node.
const TargetLibraryInfo & getLibInfo() const
Definition: SelectionDAG.h:493
MachineFunction & getMachineFunction() const
Definition: SelectionDAG.h:481
SDValue getCondCode(ISD::CondCode Cond)
LLVMContext * getContext() const
Definition: SelectionDAG.h:499
SDValue getShiftAmountConstant(uint64_t Val, EVT VT, const SDLoc &DL, bool LegalTypes=true)
SDValue CreateStackTemporary(TypeSize Bytes, Align Alignment)
Create a stack temporary based on the size in bytes and the alignment.
SDNode * UpdateNodeOperands(SDNode *N, SDValue Op)
Mutate the specified node in-place to have the specified operands.
SDValue getEntryNode() const
Return the token chain corresponding to the entry of the function.
Definition: SelectionDAG.h:568
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
Definition: SmallVector.h:1209
This class is used to represent ISD::STORE nodes.
unsigned getIntSize() const
Get size of a C-level int or unsigned int, in bits.
EVT getShiftAmountTy(EVT LHSTy, const DataLayout &DL, bool LegalTypes=true) const
Returns the type for the shift amount of a shift opcode.
virtual EVT getTypeToTransformTo(LLVMContext &Context, EVT VT) const
For types supported by the target, this is an identity function.
const char * getLibcallName(RTLIB::Libcall Call) const
Get the libcall routine name for the specified libcall.
void softenSetCCOperands(SelectionDAG &DAG, EVT VT, SDValue &NewLHS, SDValue &NewRHS, ISD::CondCode &CCCode, const SDLoc &DL, const SDValue OldLHS, const SDValue OldRHS) const
Soften the operands of a comparison.
std::pair< SDValue, SDValue > makeLibCall(SelectionDAG &DAG, RTLIB::Libcall LC, EVT RetVT, ArrayRef< SDValue > Ops, MakeLibCallOptions CallOptions, const SDLoc &dl, SDValue Chain=SDValue()) const
Returns a pair of (return value, chain).
SDValue expandVecReduceSeq(SDNode *Node, SelectionDAG &DAG) const
Expand a VECREDUCE_SEQ_* into an explicit ordered calculation.
SDValue expandFP_TO_INT_SAT(SDNode *N, SelectionDAG &DAG) const
Expand FP_TO_[US]INT_SAT into FP_TO_[US]INT and selects or min/max.
SDValue expandVecReduce(SDNode *Node, SelectionDAG &DAG) const
Expand a VECREDUCE_* into an explicit calculation.
SDValue createSelectForFMINNUM_FMAXNUM(SDNode *Node, SelectionDAG &DAG) const
Try to convert the fminnum/fmaxnum to a compare/select sequence.
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
constexpr std::underlying_type_t< E > Mask()
Get a bitmask with 1s in all places up to the high-order bit of E's largest value.
Definition: BitmaskEnum.h:121
@ C
The default llvm calling convention, compatible with C.
Definition: CallingConv.h:34
NodeType
ISD::NodeType enum - This enum defines the target-independent operators for a SelectionDAG.
Definition: ISDOpcodes.h:40
@ SETCC
SetCC operator - This evaluates to a true value iff the condition is true.
Definition: ISDOpcodes.h:758
@ MERGE_VALUES
MERGE_VALUES - This node takes multiple discrete operands and returns them all as its individual resu...
Definition: ISDOpcodes.h:237
@ STRICT_FSETCC
STRICT_FSETCC/STRICT_FSETCCS - Constrained versions of SETCC, used for floating-point operands only.
Definition: ISDOpcodes.h:478
@ VECREDUCE_SEQ_FADD
Generic reduction nodes.
Definition: ISDOpcodes.h:1354
@ ConstantFP
Definition: ISDOpcodes.h:77
@ ATOMIC_STORE
OUTCHAIN = ATOMIC_STORE(INCHAIN, ptr, val) This corresponds to "store atomic" instruction.
Definition: ISDOpcodes.h:1256
@ STRICT_FCEIL
Definition: ISDOpcodes.h:428
@ FMAD
FMAD - Perform a * b + c, while getting the same result as the separately rounded operations.
Definition: ISDOpcodes.h:489
@ FMAXNUM_IEEE
Definition: ISDOpcodes.h:994
@ LOAD
LOAD and STORE have token chains as their first operand, then the same operands as an LLVM load/store...
Definition: ISDOpcodes.h:1046
@ ANY_EXTEND
ANY_EXTEND - Used for integer types. The high bits are undefined.
Definition: ISDOpcodes.h:791
@ FMA
FMA - Perform a * b + c with no intermediate rounding step.
Definition: ISDOpcodes.h:485
@ SINT_TO_FP
[SU]INT_TO_FP - These operators convert integers (whose interpreted sign depends on the first letter)...
Definition: ISDOpcodes.h:798
@ VECREDUCE_FMAX
FMIN/FMAX nodes can have flags, for NaN/NoNaN variants.
Definition: ISDOpcodes.h:1370
@ FADD
Simple binary floating point operators.
Definition: ISDOpcodes.h:391
@ VECREDUCE_FMAXIMUM
FMINIMUM/FMAXIMUM nodes propatate NaNs and signed zeroes using the llvm.minimum and llvm....
Definition: ISDOpcodes.h:1374
@ STRICT_FSETCCS
Definition: ISDOpcodes.h:479
@ FP16_TO_FP
FP16_TO_FP, FP_TO_FP16 - These operators are used to perform promotions and truncation for half-preci...
Definition: ISDOpcodes.h:921
@ STRICT_FLOG2
Definition: ISDOpcodes.h:423
@ BITCAST
BITCAST - This operator converts between integer, vector and FP values, as if the value was stored to...
Definition: ISDOpcodes.h:911
@ BUILD_PAIR
BUILD_PAIR - This is the opposite of EXTRACT_ELEMENT in some ways.
Definition: ISDOpcodes.h:230
@ FLDEXP
FLDEXP - ldexp, inspired by libm (op0 * 2**op1).
Definition: ISDOpcodes.h:948
@ STRICT_FSQRT
Constrained versions of libm-equivalent floating point intrinsics.
Definition: ISDOpcodes.h:412
@ SIGN_EXTEND
Conversion operators.
Definition: ISDOpcodes.h:782
@ STRICT_UINT_TO_FP
Definition: ISDOpcodes.h:452
@ VECREDUCE_FADD
These reductions have relaxed evaluation order semantics, and have a single vector operand.
Definition: ISDOpcodes.h:1367
@ VECREDUCE_FMIN
Definition: ISDOpcodes.h:1371
@ STRICT_LROUND
Definition: ISDOpcodes.h:433
@ FNEG
Perform various unary floating-point operations inspired by libm.
Definition: ISDOpcodes.h:938
@ BR_CC
BR_CC - Conditional branch.
Definition: ISDOpcodes.h:1092
@ FCANONICALIZE
Returns platform specific canonical encoding of a floating point number.
Definition: ISDOpcodes.h:502
@ SELECT
Select(COND, TRUEVAL, FALSEVAL).
Definition: ISDOpcodes.h:735
@ STRICT_FPOWI
Definition: ISDOpcodes.h:414
@ ATOMIC_LOAD
Val, OUTCHAIN = ATOMIC_LOAD(INCHAIN, ptr) This corresponds to "load atomic" instruction.
Definition: ISDOpcodes.h:1252
@ UNDEF
UNDEF - An undefined node.
Definition: ISDOpcodes.h:212
@ EXTRACT_ELEMENT
EXTRACT_ELEMENT - This is used to get the lower or upper (determined by a Constant,...
Definition: ISDOpcodes.h:223
@ STRICT_FTRUNC
Definition: ISDOpcodes.h:432
@ ARITH_FENCE
ARITH_FENCE - This corresponds to a arithmetic fence intrinsic.
Definition: ISDOpcodes.h:1240
@ STRICT_FP_TO_FP16
Definition: ISDOpcodes.h:924
@ STRICT_FP16_TO_FP
Definition: ISDOpcodes.h:923
@ SHL
Shift and rotation operations.
Definition: ISDOpcodes.h:713
@ FMINNUM_IEEE
FMINNUM_IEEE/FMAXNUM_IEEE - Perform floating-point minimumNumber or maximumNumber on two values,...
Definition: ISDOpcodes.h:993
@ STRICT_FMAXNUM
Definition: ISDOpcodes.h:426
@ EXTRACT_VECTOR_ELT
EXTRACT_VECTOR_ELT(VECTOR, IDX) - Returns a single element from VECTOR identified by the (potentially...
Definition: ISDOpcodes.h:537
@ ZERO_EXTEND
ZERO_EXTEND - Used for integer types, zeroing the new bits.
Definition: ISDOpcodes.h:788
@ FP_TO_UINT_SAT
Definition: ISDOpcodes.h:864
@ STRICT_FMINNUM
Definition: ISDOpcodes.h:427
@ SELECT_CC
Select with condition operator - This selects between a true value and a false value (ops #2 and #3) ...
Definition: ISDOpcodes.h:750
@ FMINNUM
FMINNUM/FMAXNUM - Perform floating-point minimum or maximum on two values.
Definition: ISDOpcodes.h:980
@ STRICT_LRINT
Definition: ISDOpcodes.h:435
@ FP_EXTEND
X = FP_EXTEND(Y) - Extend a smaller FP type into a larger FP type.
Definition: ISDOpcodes.h:896
@ STRICT_FROUND
Definition: ISDOpcodes.h:430
@ STRICT_SINT_TO_FP
STRICT_[US]INT_TO_FP - Convert a signed or unsigned integer to a floating point value.
Definition: ISDOpcodes.h:451
@ STRICT_BF16_TO_FP
Definition: ISDOpcodes.h:932
@ STRICT_FFLOOR
Definition: ISDOpcodes.h:429
@ STRICT_FROUNDEVEN
Definition: ISDOpcodes.h:431
@ BF16_TO_FP
BF16_TO_FP, FP_TO_BF16 - These operators are used to perform promotions and truncation for bfloat16.
Definition: ISDOpcodes.h:930
@ STRICT_FP_TO_UINT
Definition: ISDOpcodes.h:445
@ STRICT_FP_ROUND
X = STRICT_FP_ROUND(Y, TRUNC) - Rounding 'Y' from a larger floating point type down to the precision ...
Definition: ISDOpcodes.h:467
@ STRICT_FP_TO_SINT
STRICT_FP_TO_[US]INT - Convert a floating point value to a signed or unsigned integer.
Definition: ISDOpcodes.h:444
@ FMINIMUM
FMINIMUM/FMAXIMUM - NaN-propagating minimum/maximum that also treat -0.0 as less than 0....
Definition: ISDOpcodes.h:999
@ FP_TO_SINT
FP_TO_[US]INT - Convert a floating point value to a signed or unsigned integer.
Definition: ISDOpcodes.h:844
@ STRICT_FP_EXTEND
X = STRICT_FP_EXTEND(Y) - Extend a smaller FP type into a larger FP type.
Definition: ISDOpcodes.h:472
@ AND
Bitwise operators - logical and, logical or, logical xor.
Definition: ISDOpcodes.h:688
@ STRICT_FP_TO_BF16
Definition: ISDOpcodes.h:933
@ VECREDUCE_FMUL
Definition: ISDOpcodes.h:1368
@ STRICT_FADD
Constrained versions of the binary floating point operators.
Definition: ISDOpcodes.h:401
@ STRICT_FLOG10
Definition: ISDOpcodes.h:422
@ STRICT_LLRINT
Definition: ISDOpcodes.h:436
@ STRICT_FEXP2
Definition: ISDOpcodes.h:420
@ ATOMIC_SWAP
Val, OUTCHAIN = ATOMIC_SWAP(INCHAIN, ptr, amt) Val, OUTCHAIN = ATOMIC_LOAD_[OpName](INCHAIN,...
Definition: ISDOpcodes.h:1277
@ FFREXP
FFREXP - frexp, extract fractional and exponent component of a floating-point value.
Definition: ISDOpcodes.h:953
@ FP_ROUND
X = FP_ROUND(Y, TRUNC) - Rounding 'Y' from a larger floating point type down to the precision of the ...
Definition: ISDOpcodes.h:877
@ STRICT_FLDEXP
Definition: ISDOpcodes.h:415
@ STRICT_LLROUND
Definition: ISDOpcodes.h:434
@ STRICT_FNEARBYINT
Definition: ISDOpcodes.h:425
@ FP_TO_SINT_SAT
FP_TO_[US]INT_SAT - Convert floating point value in operand 0 to a signed or unsigned scalar integer ...
Definition: ISDOpcodes.h:863
@ VECREDUCE_FMINIMUM
Definition: ISDOpcodes.h:1375
@ TRUNCATE
TRUNCATE - Completely drop the high bits.
Definition: ISDOpcodes.h:794
@ VAARG
VAARG - VAARG has four operands: an input chain, a pointer, a SRCVALUE, and the alignment.
Definition: ISDOpcodes.h:1161
@ VECREDUCE_SEQ_FMUL
Definition: ISDOpcodes.h:1355
@ FCOPYSIGN
FCOPYSIGN(X, Y) - Return the value of X with the sign of Y.
Definition: ISDOpcodes.h:495