LLVM 18.0.0git
LegalizeFloatTypes.cpp
Go to the documentation of this file.
1//===-------- LegalizeFloatTypes.cpp - Legalization of float types --------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements float type expansion and softening for LegalizeTypes.
10// Softening is the act of turning a computation in an illegal floating point
11// type into a computation in an integer type of the same size; also known as
12// "soft float". For example, turning f32 arithmetic into operations using i32.
13// The resulting integer value is the same as what you would get by performing
14// the floating point operation and bitcasting the result to the integer type.
15// Expansion is the act of changing a computation in an illegal type to be a
16// computation in two identical registers of a smaller type. For example,
17// implementing ppcf128 arithmetic in two f64 registers.
18//
19//===----------------------------------------------------------------------===//
20
21#include "LegalizeTypes.h"
25using namespace llvm;
26
27#define DEBUG_TYPE "legalize-types"
28
29/// GetFPLibCall - Return the right libcall for the given floating point type.
30/// FIXME: This is a local version of RTLIB::getFPLibCall that should be
31/// refactored away (see RTLIB::getPOWI for an example).
33 RTLIB::Libcall Call_F32,
34 RTLIB::Libcall Call_F64,
35 RTLIB::Libcall Call_F80,
36 RTLIB::Libcall Call_F128,
37 RTLIB::Libcall Call_PPCF128) {
38 return
39 VT == MVT::f32 ? Call_F32 :
40 VT == MVT::f64 ? Call_F64 :
41 VT == MVT::f80 ? Call_F80 :
42 VT == MVT::f128 ? Call_F128 :
43 VT == MVT::ppcf128 ? Call_PPCF128 :
44 RTLIB::UNKNOWN_LIBCALL;
45}
46
47//===----------------------------------------------------------------------===//
48// Convert Float Results to Integer
49//===----------------------------------------------------------------------===//
50
51void DAGTypeLegalizer::SoftenFloatResult(SDNode *N, unsigned ResNo) {
52 LLVM_DEBUG(dbgs() << "Soften float result " << ResNo << ": "; N->dump(&DAG));
53 SDValue R = SDValue();
54
55 switch (N->getOpcode()) {
56 default:
57#ifndef NDEBUG
58 dbgs() << "SoftenFloatResult #" << ResNo << ": ";
59 N->dump(&DAG); dbgs() << "\n";
60#endif
61 report_fatal_error("Do not know how to soften the result of this "
62 "operator!");
63
64 case ISD::ARITH_FENCE: R = SoftenFloatRes_ARITH_FENCE(N); break;
65 case ISD::MERGE_VALUES:R = SoftenFloatRes_MERGE_VALUES(N, ResNo); break;
66 case ISD::BITCAST: R = SoftenFloatRes_BITCAST(N); break;
67 case ISD::BUILD_PAIR: R = SoftenFloatRes_BUILD_PAIR(N); break;
68 case ISD::ConstantFP: R = SoftenFloatRes_ConstantFP(N); break;
70 R = SoftenFloatRes_EXTRACT_VECTOR_ELT(N, ResNo); break;
71 case ISD::FABS: R = SoftenFloatRes_FABS(N); break;
73 case ISD::FMINNUM: R = SoftenFloatRes_FMINNUM(N); break;
75 case ISD::FMAXNUM: R = SoftenFloatRes_FMAXNUM(N); break;
77 case ISD::FADD: R = SoftenFloatRes_FADD(N); break;
78 case ISD::FCBRT: R = SoftenFloatRes_FCBRT(N); break;
80 case ISD::FCEIL: R = SoftenFloatRes_FCEIL(N); break;
81 case ISD::FCOPYSIGN: R = SoftenFloatRes_FCOPYSIGN(N); break;
83 case ISD::FCOS: R = SoftenFloatRes_FCOS(N); break;
85 case ISD::FDIV: R = SoftenFloatRes_FDIV(N); break;
87 case ISD::FEXP: R = SoftenFloatRes_FEXP(N); break;
89 case ISD::FEXP2: R = SoftenFloatRes_FEXP2(N); break;
90 case ISD::FEXP10: R = SoftenFloatRes_FEXP10(N); break;
92 case ISD::FFLOOR: R = SoftenFloatRes_FFLOOR(N); break;
94 case ISD::FLOG: R = SoftenFloatRes_FLOG(N); break;
96 case ISD::FLOG2: R = SoftenFloatRes_FLOG2(N); break;
98 case ISD::FLOG10: R = SoftenFloatRes_FLOG10(N); break;
99 case ISD::STRICT_FMA:
100 case ISD::FMA: R = SoftenFloatRes_FMA(N); break;
101 case ISD::STRICT_FMUL:
102 case ISD::FMUL: R = SoftenFloatRes_FMUL(N); break;
104 case ISD::FNEARBYINT: R = SoftenFloatRes_FNEARBYINT(N); break;
105 case ISD::FNEG: R = SoftenFloatRes_FNEG(N); break;
107 case ISD::FP_EXTEND: R = SoftenFloatRes_FP_EXTEND(N); break;
109 case ISD::FP_ROUND: R = SoftenFloatRes_FP_ROUND(N); break;
110 case ISD::FP16_TO_FP: R = SoftenFloatRes_FP16_TO_FP(N); break;
111 case ISD::BF16_TO_FP: R = SoftenFloatRes_BF16_TO_FP(N); break;
112 case ISD::STRICT_FPOW:
113 case ISD::FPOW: R = SoftenFloatRes_FPOW(N); break;
115 case ISD::FPOWI:
116 case ISD::FLDEXP:
117 case ISD::STRICT_FLDEXP: R = SoftenFloatRes_ExpOp(N); break;
118 case ISD::FFREXP:
119 R = SoftenFloatRes_FFREXP(N);
120 break;
121 case ISD::STRICT_FREM:
122 case ISD::FREM: R = SoftenFloatRes_FREM(N); break;
124 case ISD::FRINT: R = SoftenFloatRes_FRINT(N); break;
126 case ISD::FROUND: R = SoftenFloatRes_FROUND(N); break;
128 case ISD::FROUNDEVEN: R = SoftenFloatRes_FROUNDEVEN(N); break;
129 case ISD::STRICT_FSIN:
130 case ISD::FSIN: R = SoftenFloatRes_FSIN(N); break;
132 case ISD::FSQRT: R = SoftenFloatRes_FSQRT(N); break;
133 case ISD::STRICT_FSUB:
134 case ISD::FSUB: R = SoftenFloatRes_FSUB(N); break;
136 case ISD::FTRUNC: R = SoftenFloatRes_FTRUNC(N); break;
137 case ISD::LOAD: R = SoftenFloatRes_LOAD(N); break;
138 case ISD::ATOMIC_SWAP: R = BitcastToInt_ATOMIC_SWAP(N); break;
139 case ISD::SELECT: R = SoftenFloatRes_SELECT(N); break;
140 case ISD::SELECT_CC: R = SoftenFloatRes_SELECT_CC(N); break;
141 case ISD::FREEZE: R = SoftenFloatRes_FREEZE(N); break;
144 case ISD::SINT_TO_FP:
145 case ISD::UINT_TO_FP: R = SoftenFloatRes_XINT_TO_FP(N); break;
146 case ISD::UNDEF: R = SoftenFloatRes_UNDEF(N); break;
147 case ISD::VAARG: R = SoftenFloatRes_VAARG(N); break;
154 R = SoftenFloatRes_VECREDUCE(N);
155 break;
158 R = SoftenFloatRes_VECREDUCE_SEQ(N);
159 break;
160 }
161
162 // If R is null, the sub-method took care of registering the result.
163 if (R.getNode()) {
164 assert(R.getNode() != N);
165 SetSoftenedFloat(SDValue(N, ResNo), R);
166 }
167}
168
169SDValue DAGTypeLegalizer::SoftenFloatRes_Unary(SDNode *N, RTLIB::Libcall LC) {
170 bool IsStrict = N->isStrictFPOpcode();
171 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
172 unsigned Offset = IsStrict ? 1 : 0;
173 assert(N->getNumOperands() == (1 + Offset) &&
174 "Unexpected number of operands!");
175 SDValue Op = GetSoftenedFloat(N->getOperand(0 + Offset));
176 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
178 EVT OpVT = N->getOperand(0 + Offset).getValueType();
179 CallOptions.setTypeListBeforeSoften(OpVT, N->getValueType(0), true);
180 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, NVT, Op,
181 CallOptions, SDLoc(N),
182 Chain);
183 if (IsStrict)
184 ReplaceValueWith(SDValue(N, 1), Tmp.second);
185 return Tmp.first;
186}
187
188SDValue DAGTypeLegalizer::SoftenFloatRes_Binary(SDNode *N, RTLIB::Libcall LC) {
189 bool IsStrict = N->isStrictFPOpcode();
190 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
191 unsigned Offset = IsStrict ? 1 : 0;
192 assert(N->getNumOperands() == (2 + Offset) &&
193 "Unexpected number of operands!");
194 SDValue Ops[2] = { GetSoftenedFloat(N->getOperand(0 + Offset)),
195 GetSoftenedFloat(N->getOperand(1 + Offset)) };
196 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
198 EVT OpsVT[2] = { N->getOperand(0 + Offset).getValueType(),
199 N->getOperand(1 + Offset).getValueType() };
200 CallOptions.setTypeListBeforeSoften(OpsVT, N->getValueType(0), true);
201 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, NVT, Ops,
202 CallOptions, SDLoc(N),
203 Chain);
204 if (IsStrict)
205 ReplaceValueWith(SDValue(N, 1), Tmp.second);
206 return Tmp.first;
207}
208
209SDValue DAGTypeLegalizer::SoftenFloatRes_BITCAST(SDNode *N) {
210 return BitConvertToInteger(N->getOperand(0));
211}
212
213SDValue DAGTypeLegalizer::SoftenFloatRes_FREEZE(SDNode *N) {
214 EVT Ty = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
215 return DAG.getNode(ISD::FREEZE, SDLoc(N), Ty,
216 GetSoftenedFloat(N->getOperand(0)));
217}
218
219SDValue DAGTypeLegalizer::SoftenFloatRes_ARITH_FENCE(SDNode *N) {
220 EVT Ty = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
221 SDValue NewFence = DAG.getNode(ISD::ARITH_FENCE, SDLoc(N), Ty,
222 GetSoftenedFloat(N->getOperand(0)));
223 return NewFence;
224}
225
226SDValue DAGTypeLegalizer::SoftenFloatRes_MERGE_VALUES(SDNode *N,
227 unsigned ResNo) {
228 SDValue Op = DisintegrateMERGE_VALUES(N, ResNo);
229 return BitConvertToInteger(Op);
230}
231
232SDValue DAGTypeLegalizer::SoftenFloatRes_BUILD_PAIR(SDNode *N) {
233 // Convert the inputs to integers, and build a new pair out of them.
234 return DAG.getNode(ISD::BUILD_PAIR, SDLoc(N),
236 N->getValueType(0)),
237 BitConvertToInteger(N->getOperand(0)),
238 BitConvertToInteger(N->getOperand(1)));
239}
240
241SDValue DAGTypeLegalizer::SoftenFloatRes_ConstantFP(SDNode *N) {
242 ConstantFPSDNode *CN = cast<ConstantFPSDNode>(N);
243 // In ppcf128, the high 64 bits are always first in memory regardless
244 // of Endianness. LLVM's APFloat representation is not Endian sensitive,
245 // and so always converts into a 128-bit APInt in a non-Endian-sensitive
246 // way. However, APInt's are serialized in an Endian-sensitive fashion,
247 // so on big-Endian targets, the two doubles are output in the wrong
248 // order. Fix this by manually flipping the order of the high 64 bits
249 // and the low 64 bits here.
250 if (DAG.getDataLayout().isBigEndian() &&
251 CN->getValueType(0).getSimpleVT() == llvm::MVT::ppcf128) {
252 uint64_t words[2] = { CN->getValueAPF().bitcastToAPInt().getRawData()[1],
254 APInt Val(128, words);
255 return DAG.getConstant(Val, SDLoc(CN),
257 CN->getValueType(0)));
258 } else {
259 return DAG.getConstant(CN->getValueAPF().bitcastToAPInt(), SDLoc(CN),
261 CN->getValueType(0)));
262 }
263}
264
265SDValue DAGTypeLegalizer::SoftenFloatRes_EXTRACT_VECTOR_ELT(SDNode *N, unsigned ResNo) {
266 SDValue NewOp = BitConvertVectorToIntegerVector(N->getOperand(0));
269 NewOp, N->getOperand(1));
270}
271
272SDValue DAGTypeLegalizer::SoftenFloatRes_FABS(SDNode *N) {
273 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
274 unsigned Size = NVT.getSizeInBits();
275
276 // Mask = ~(1 << (Size-1))
278 API.clearBit(Size - 1);
279 SDValue Mask = DAG.getConstant(API, SDLoc(N), NVT);
280 SDValue Op = GetSoftenedFloat(N->getOperand(0));
281 return DAG.getNode(ISD::AND, SDLoc(N), NVT, Op, Mask);
282}
283
284SDValue DAGTypeLegalizer::SoftenFloatRes_FMINNUM(SDNode *N) {
285 if (SDValue SelCC = TLI.createSelectForFMINNUM_FMAXNUM(N, DAG))
286 return SoftenFloatRes_SELECT_CC(SelCC.getNode());
287 return SoftenFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
288 RTLIB::FMIN_F32,
289 RTLIB::FMIN_F64,
290 RTLIB::FMIN_F80,
291 RTLIB::FMIN_F128,
292 RTLIB::FMIN_PPCF128));
293}
294
295SDValue DAGTypeLegalizer::SoftenFloatRes_FMAXNUM(SDNode *N) {
296 if (SDValue SelCC = TLI.createSelectForFMINNUM_FMAXNUM(N, DAG))
297 return SoftenFloatRes_SELECT_CC(SelCC.getNode());
298 return SoftenFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
299 RTLIB::FMAX_F32,
300 RTLIB::FMAX_F64,
301 RTLIB::FMAX_F80,
302 RTLIB::FMAX_F128,
303 RTLIB::FMAX_PPCF128));
304}
305
306SDValue DAGTypeLegalizer::SoftenFloatRes_FADD(SDNode *N) {
307 return SoftenFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
308 RTLIB::ADD_F32,
309 RTLIB::ADD_F64,
310 RTLIB::ADD_F80,
311 RTLIB::ADD_F128,
312 RTLIB::ADD_PPCF128));
313}
314
315SDValue DAGTypeLegalizer::SoftenFloatRes_FCBRT(SDNode *N) {
316 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
317 RTLIB::CBRT_F32,
318 RTLIB::CBRT_F64,
319 RTLIB::CBRT_F80,
320 RTLIB::CBRT_F128,
321 RTLIB::CBRT_PPCF128));
322}
323
324SDValue DAGTypeLegalizer::SoftenFloatRes_FCEIL(SDNode *N) {
325 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
326 RTLIB::CEIL_F32,
327 RTLIB::CEIL_F64,
328 RTLIB::CEIL_F80,
329 RTLIB::CEIL_F128,
330 RTLIB::CEIL_PPCF128));
331}
332
333SDValue DAGTypeLegalizer::SoftenFloatRes_FCOPYSIGN(SDNode *N) {
334 SDValue LHS = GetSoftenedFloat(N->getOperand(0));
335 SDValue RHS = BitConvertToInteger(N->getOperand(1));
336 SDLoc dl(N);
337
338 EVT LVT = LHS.getValueType();
339 EVT RVT = RHS.getValueType();
340
341 unsigned LSize = LVT.getSizeInBits();
342 unsigned RSize = RVT.getSizeInBits();
343
344 // First get the sign bit of second operand.
345 SDValue SignBit = DAG.getNode(
346 ISD::SHL, dl, RVT, DAG.getConstant(1, dl, RVT),
347 DAG.getConstant(RSize - 1, dl,
348 TLI.getShiftAmountTy(RVT, DAG.getDataLayout())));
349 SignBit = DAG.getNode(ISD::AND, dl, RVT, RHS, SignBit);
350
351 // Shift right or sign-extend it if the two operands have different types.
352 int SizeDiff = RVT.getSizeInBits() - LVT.getSizeInBits();
353 if (SizeDiff > 0) {
354 SignBit =
355 DAG.getNode(ISD::SRL, dl, RVT, SignBit,
356 DAG.getConstant(SizeDiff, dl,
357 TLI.getShiftAmountTy(SignBit.getValueType(),
358 DAG.getDataLayout())));
359 SignBit = DAG.getNode(ISD::TRUNCATE, dl, LVT, SignBit);
360 } else if (SizeDiff < 0) {
361 SignBit = DAG.getNode(ISD::ANY_EXTEND, dl, LVT, SignBit);
362 SignBit =
363 DAG.getNode(ISD::SHL, dl, LVT, SignBit,
364 DAG.getConstant(-SizeDiff, dl,
365 TLI.getShiftAmountTy(SignBit.getValueType(),
366 DAG.getDataLayout())));
367 }
368
369 // Clear the sign bit of the first operand.
370 SDValue Mask = DAG.getNode(
371 ISD::SHL, dl, LVT, DAG.getConstant(1, dl, LVT),
372 DAG.getConstant(LSize - 1, dl,
373 TLI.getShiftAmountTy(LVT, DAG.getDataLayout())));
374 Mask = DAG.getNode(ISD::SUB, dl, LVT, Mask, DAG.getConstant(1, dl, LVT));
375 LHS = DAG.getNode(ISD::AND, dl, LVT, LHS, Mask);
376
377 // Or the value with the sign bit.
378 return DAG.getNode(ISD::OR, dl, LVT, LHS, SignBit);
379}
380
381SDValue DAGTypeLegalizer::SoftenFloatRes_FCOS(SDNode *N) {
382 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
383 RTLIB::COS_F32,
384 RTLIB::COS_F64,
385 RTLIB::COS_F80,
386 RTLIB::COS_F128,
387 RTLIB::COS_PPCF128));
388}
389
390SDValue DAGTypeLegalizer::SoftenFloatRes_FDIV(SDNode *N) {
391 return SoftenFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
392 RTLIB::DIV_F32,
393 RTLIB::DIV_F64,
394 RTLIB::DIV_F80,
395 RTLIB::DIV_F128,
396 RTLIB::DIV_PPCF128));
397}
398
399SDValue DAGTypeLegalizer::SoftenFloatRes_FEXP(SDNode *N) {
400 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
401 RTLIB::EXP_F32,
402 RTLIB::EXP_F64,
403 RTLIB::EXP_F80,
404 RTLIB::EXP_F128,
405 RTLIB::EXP_PPCF128));
406}
407
408SDValue DAGTypeLegalizer::SoftenFloatRes_FEXP2(SDNode *N) {
409 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
410 RTLIB::EXP2_F32,
411 RTLIB::EXP2_F64,
412 RTLIB::EXP2_F80,
413 RTLIB::EXP2_F128,
414 RTLIB::EXP2_PPCF128));
415}
416
417SDValue DAGTypeLegalizer::SoftenFloatRes_FEXP10(SDNode *N) {
418 return SoftenFloatRes_Unary(
419 N,
420 GetFPLibCall(N->getValueType(0), RTLIB::EXP10_F32, RTLIB::EXP10_F64,
421 RTLIB::EXP10_F80, RTLIB::EXP10_F128, RTLIB::EXP10_PPCF128));
422}
423
424SDValue DAGTypeLegalizer::SoftenFloatRes_FFLOOR(SDNode *N) {
425 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
426 RTLIB::FLOOR_F32,
427 RTLIB::FLOOR_F64,
428 RTLIB::FLOOR_F80,
429 RTLIB::FLOOR_F128,
430 RTLIB::FLOOR_PPCF128));
431}
432
433SDValue DAGTypeLegalizer::SoftenFloatRes_FLOG(SDNode *N) {
434 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
435 RTLIB::LOG_F32,
436 RTLIB::LOG_F64,
437 RTLIB::LOG_F80,
438 RTLIB::LOG_F128,
439 RTLIB::LOG_PPCF128));
440}
441
442SDValue DAGTypeLegalizer::SoftenFloatRes_FLOG2(SDNode *N) {
443 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
444 RTLIB::LOG2_F32,
445 RTLIB::LOG2_F64,
446 RTLIB::LOG2_F80,
447 RTLIB::LOG2_F128,
448 RTLIB::LOG2_PPCF128));
449}
450
451SDValue DAGTypeLegalizer::SoftenFloatRes_FLOG10(SDNode *N) {
452 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
453 RTLIB::LOG10_F32,
454 RTLIB::LOG10_F64,
455 RTLIB::LOG10_F80,
456 RTLIB::LOG10_F128,
457 RTLIB::LOG10_PPCF128));
458}
459
460SDValue DAGTypeLegalizer::SoftenFloatRes_FMA(SDNode *N) {
461 bool IsStrict = N->isStrictFPOpcode();
462 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
463 unsigned Offset = IsStrict ? 1 : 0;
464 SDValue Ops[3] = { GetSoftenedFloat(N->getOperand(0 + Offset)),
465 GetSoftenedFloat(N->getOperand(1 + Offset)),
466 GetSoftenedFloat(N->getOperand(2 + Offset)) };
467 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
469 EVT OpsVT[3] = { N->getOperand(0 + Offset).getValueType(),
470 N->getOperand(1 + Offset).getValueType(),
471 N->getOperand(2 + Offset).getValueType() };
472 CallOptions.setTypeListBeforeSoften(OpsVT, N->getValueType(0), true);
473 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG,
474 GetFPLibCall(N->getValueType(0),
475 RTLIB::FMA_F32,
476 RTLIB::FMA_F64,
477 RTLIB::FMA_F80,
478 RTLIB::FMA_F128,
479 RTLIB::FMA_PPCF128),
480 NVT, Ops, CallOptions, SDLoc(N), Chain);
481 if (IsStrict)
482 ReplaceValueWith(SDValue(N, 1), Tmp.second);
483 return Tmp.first;
484}
485
486SDValue DAGTypeLegalizer::SoftenFloatRes_FMUL(SDNode *N) {
487 return SoftenFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
488 RTLIB::MUL_F32,
489 RTLIB::MUL_F64,
490 RTLIB::MUL_F80,
491 RTLIB::MUL_F128,
492 RTLIB::MUL_PPCF128));
493}
494
495SDValue DAGTypeLegalizer::SoftenFloatRes_FNEARBYINT(SDNode *N) {
496 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
497 RTLIB::NEARBYINT_F32,
498 RTLIB::NEARBYINT_F64,
499 RTLIB::NEARBYINT_F80,
500 RTLIB::NEARBYINT_F128,
501 RTLIB::NEARBYINT_PPCF128));
502}
503
504SDValue DAGTypeLegalizer::SoftenFloatRes_FNEG(SDNode *N) {
505 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
506 SDLoc dl(N);
507
508 // Expand Y = FNEG(X) -> Y = X ^ sign mask
509 APInt SignMask = APInt::getSignMask(NVT.getSizeInBits());
510 return DAG.getNode(ISD::XOR, dl, NVT, GetSoftenedFloat(N->getOperand(0)),
511 DAG.getConstant(SignMask, dl, NVT));
512}
513
514SDValue DAGTypeLegalizer::SoftenFloatRes_FP_EXTEND(SDNode *N) {
515 bool IsStrict = N->isStrictFPOpcode();
516 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
517 SDValue Op = N->getOperand(IsStrict ? 1 : 0);
518
519 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
520
521 if (getTypeAction(Op.getValueType()) == TargetLowering::TypePromoteFloat) {
522 Op = GetPromotedFloat(Op);
523 // If the promotion did the FP_EXTEND to the destination type for us,
524 // there's nothing left to do here.
525 if (Op.getValueType() == N->getValueType(0))
526 return BitConvertToInteger(Op);
527 }
528
529 // There's only a libcall for f16 -> f32 and shifting is only valid for bf16
530 // -> f32, so proceed in two stages. Also, it's entirely possible for both
531 // f16 and f32 to be legal, so use the fully hard-float FP_EXTEND rather
532 // than FP16_TO_FP.
533 if ((Op.getValueType() == MVT::f16 || Op.getValueType() == MVT::bf16) &&
534 N->getValueType(0) != MVT::f32) {
535 if (IsStrict) {
537 { MVT::f32, MVT::Other }, { Chain, Op });
538 Chain = Op.getValue(1);
539 } else {
540 Op = DAG.getNode(ISD::FP_EXTEND, SDLoc(N), MVT::f32, Op);
541 }
542 }
543
544 if (Op.getValueType() == MVT::bf16)
545 return SoftenFloatRes_BF16_TO_FP(N);
546
547 RTLIB::Libcall LC = RTLIB::getFPEXT(Op.getValueType(), N->getValueType(0));
548 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported FP_EXTEND!");
550 EVT OpVT = N->getOperand(IsStrict ? 1 : 0).getValueType();
551 CallOptions.setTypeListBeforeSoften(OpVT, N->getValueType(0), true);
552 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, NVT, Op,
553 CallOptions, SDLoc(N),
554 Chain);
555 if (IsStrict)
556 ReplaceValueWith(SDValue(N, 1), Tmp.second);
557 return Tmp.first;
558}
559
560// FIXME: Should we just use 'normal' FP_EXTEND / FP_TRUNC instead of special
561// nodes?
562SDValue DAGTypeLegalizer::SoftenFloatRes_FP16_TO_FP(SDNode *N) {
563 EVT MidVT = TLI.getTypeToTransformTo(*DAG.getContext(), MVT::f32);
564 SDValue Op = N->getOperand(0);
566 EVT OpsVT[1] = { N->getOperand(0).getValueType() };
567 CallOptions.setTypeListBeforeSoften(OpsVT, N->getValueType(0), true);
568 SDValue Res32 = TLI.makeLibCall(DAG, RTLIB::FPEXT_F16_F32, MidVT, Op,
569 CallOptions, SDLoc(N)).first;
570 if (N->getValueType(0) == MVT::f32)
571 return Res32;
572
573 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
574 RTLIB::Libcall LC = RTLIB::getFPEXT(MVT::f32, N->getValueType(0));
575 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported FP_EXTEND!");
576 return TLI.makeLibCall(DAG, LC, NVT, Res32, CallOptions, SDLoc(N)).first;
577}
578
579// FIXME: Should we just use 'normal' FP_EXTEND / FP_TRUNC instead of special
580// nodes?
581SDValue DAGTypeLegalizer::SoftenFloatRes_BF16_TO_FP(SDNode *N) {
582 assert(N->getValueType(0) == MVT::f32 &&
583 "Can only soften BF16_TO_FP with f32 result");
584 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), MVT::f32);
585 SDValue Op = N->getOperand(0);
586 SDLoc DL(N);
587 Op = DAG.getNode(ISD::ANY_EXTEND, DL, NVT,
588 DAG.getNode(ISD::BITCAST, DL, MVT::i16, Op));
589 SDValue Res = DAG.getNode(ISD::SHL, DL, NVT, Op,
590 DAG.getShiftAmountConstant(16, NVT, DL));
591 return Res;
592}
593
594SDValue DAGTypeLegalizer::SoftenFloatRes_FP_ROUND(SDNode *N) {
595 bool IsStrict = N->isStrictFPOpcode();
596 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
597 SDValue Op = N->getOperand(IsStrict ? 1 : 0);
598 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
599 RTLIB::Libcall LC = RTLIB::getFPROUND(Op.getValueType(), N->getValueType(0));
600 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported FP_ROUND!");
602 EVT OpVT = N->getOperand(IsStrict ? 1 : 0).getValueType();
603 CallOptions.setTypeListBeforeSoften(OpVT, N->getValueType(0), true);
604 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, NVT, Op,
605 CallOptions, SDLoc(N),
606 Chain);
607 if (IsStrict)
608 ReplaceValueWith(SDValue(N, 1), Tmp.second);
609 return Tmp.first;
610}
611
612SDValue DAGTypeLegalizer::SoftenFloatRes_FPOW(SDNode *N) {
613 return SoftenFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
614 RTLIB::POW_F32,
615 RTLIB::POW_F64,
616 RTLIB::POW_F80,
617 RTLIB::POW_F128,
618 RTLIB::POW_PPCF128));
619}
620
621SDValue DAGTypeLegalizer::SoftenFloatRes_ExpOp(SDNode *N) {
622 bool IsStrict = N->isStrictFPOpcode();
623 unsigned Offset = IsStrict ? 1 : 0;
624 assert((N->getOperand(1 + Offset).getValueType() == MVT::i16 ||
625 N->getOperand(1 + Offset).getValueType() == MVT::i32) &&
626 "Unsupported power type!");
627 bool IsPowI =
628 N->getOpcode() == ISD::FPOWI || N->getOpcode() == ISD::STRICT_FPOWI;
629
630 RTLIB::Libcall LC = IsPowI ? RTLIB::getPOWI(N->getValueType(0))
631 : RTLIB::getLDEXP(N->getValueType(0));
632 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected fpowi.");
633 if (!TLI.getLibcallName(LC)) {
634 // Some targets don't have a powi libcall; use pow instead.
635 // FIXME: Implement this if some target needs it.
636 DAG.getContext()->emitError("Don't know how to soften fpowi to fpow");
637 return DAG.getUNDEF(N->getValueType(0));
638 }
639
640 if (DAG.getLibInfo().getIntSize() !=
641 N->getOperand(1 + Offset).getValueType().getSizeInBits()) {
642 // If the exponent does not match with sizeof(int) a libcall to RTLIB::POWI
643 // would use the wrong type for the argument.
644 DAG.getContext()->emitError("POWI exponent does not match sizeof(int)");
645 return DAG.getUNDEF(N->getValueType(0));
646 }
647
648 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
649 SDValue Ops[2] = { GetSoftenedFloat(N->getOperand(0 + Offset)),
650 N->getOperand(1 + Offset) };
651 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
653 EVT OpsVT[2] = { N->getOperand(0 + Offset).getValueType(),
654 N->getOperand(1 + Offset).getValueType() };
655 CallOptions.setTypeListBeforeSoften(OpsVT, N->getValueType(0), true);
656 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, NVT, Ops,
657 CallOptions, SDLoc(N),
658 Chain);
659 if (IsStrict)
660 ReplaceValueWith(SDValue(N, 1), Tmp.second);
661 return Tmp.first;
662}
663
664SDValue DAGTypeLegalizer::SoftenFloatRes_FFREXP(SDNode *N) {
665 assert(!N->isStrictFPOpcode() && "strictfp not implemented for frexp");
666 EVT VT0 = N->getValueType(0);
667 EVT VT1 = N->getValueType(1);
669
670 if (DAG.getLibInfo().getIntSize() != VT1.getSizeInBits()) {
671 // If the exponent does not match with sizeof(int) a libcall would use the
672 // wrong type for the argument.
673 // TODO: Should be able to handle mismatches.
674 DAG.getContext()->emitError("ffrexp exponent does not match sizeof(int)");
675 return DAG.getUNDEF(N->getValueType(0));
676 }
677
678 EVT NVT0 = TLI.getTypeToTransformTo(*DAG.getContext(), VT0);
679 SDValue StackSlot = DAG.CreateStackTemporary(VT1);
680
681 SDLoc DL(N);
682
684 SDValue Ops[2] = {GetSoftenedFloat(N->getOperand(0)), StackSlot};
685 EVT OpsVT[2] = {VT0, StackSlot.getValueType()};
686
687 // TODO: setTypeListBeforeSoften can't properly express multiple return types,
688 // but we only really need to handle the 0th one for softening anyway.
689 CallOptions.setTypeListBeforeSoften({OpsVT}, VT0, true);
690
691 auto [ReturnVal, Chain] = TLI.makeLibCall(DAG, LC, NVT0, Ops, CallOptions, DL,
692 /*Chain=*/SDValue());
693 int FrameIdx = cast<FrameIndexSDNode>(StackSlot)->getIndex();
694 auto PtrInfo =
696
697 SDValue LoadExp = DAG.getLoad(VT1, DL, Chain, StackSlot, PtrInfo);
698
699 ReplaceValueWith(SDValue(N, 1), LoadExp);
700 return ReturnVal;
701}
702
703SDValue DAGTypeLegalizer::SoftenFloatRes_FREM(SDNode *N) {
704 return SoftenFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
705 RTLIB::REM_F32,
706 RTLIB::REM_F64,
707 RTLIB::REM_F80,
708 RTLIB::REM_F128,
709 RTLIB::REM_PPCF128));
710}
711
712SDValue DAGTypeLegalizer::SoftenFloatRes_FRINT(SDNode *N) {
713 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
714 RTLIB::RINT_F32,
715 RTLIB::RINT_F64,
716 RTLIB::RINT_F80,
717 RTLIB::RINT_F128,
718 RTLIB::RINT_PPCF128));
719}
720
721SDValue DAGTypeLegalizer::SoftenFloatRes_FROUND(SDNode *N) {
722 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
723 RTLIB::ROUND_F32,
724 RTLIB::ROUND_F64,
725 RTLIB::ROUND_F80,
726 RTLIB::ROUND_F128,
727 RTLIB::ROUND_PPCF128));
728}
729
730SDValue DAGTypeLegalizer::SoftenFloatRes_FROUNDEVEN(SDNode *N) {
731 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
732 RTLIB::ROUNDEVEN_F32,
733 RTLIB::ROUNDEVEN_F64,
734 RTLIB::ROUNDEVEN_F80,
735 RTLIB::ROUNDEVEN_F128,
736 RTLIB::ROUNDEVEN_PPCF128));
737}
738
739SDValue DAGTypeLegalizer::SoftenFloatRes_FSIN(SDNode *N) {
740 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
741 RTLIB::SIN_F32,
742 RTLIB::SIN_F64,
743 RTLIB::SIN_F80,
744 RTLIB::SIN_F128,
745 RTLIB::SIN_PPCF128));
746}
747
748SDValue DAGTypeLegalizer::SoftenFloatRes_FSQRT(SDNode *N) {
749 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
750 RTLIB::SQRT_F32,
751 RTLIB::SQRT_F64,
752 RTLIB::SQRT_F80,
753 RTLIB::SQRT_F128,
754 RTLIB::SQRT_PPCF128));
755}
756
757SDValue DAGTypeLegalizer::SoftenFloatRes_FSUB(SDNode *N) {
758 return SoftenFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
759 RTLIB::SUB_F32,
760 RTLIB::SUB_F64,
761 RTLIB::SUB_F80,
762 RTLIB::SUB_F128,
763 RTLIB::SUB_PPCF128));
764}
765
766SDValue DAGTypeLegalizer::SoftenFloatRes_FTRUNC(SDNode *N) {
767 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
768 RTLIB::TRUNC_F32,
769 RTLIB::TRUNC_F64,
770 RTLIB::TRUNC_F80,
771 RTLIB::TRUNC_F128,
772 RTLIB::TRUNC_PPCF128));
773}
774
775SDValue DAGTypeLegalizer::SoftenFloatRes_LOAD(SDNode *N) {
776 LoadSDNode *L = cast<LoadSDNode>(N);
777 EVT VT = N->getValueType(0);
778 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
779 SDLoc dl(N);
780
781 auto MMOFlags =
782 L->getMemOperand()->getFlags() &
784 SDValue NewL;
785 if (L->getExtensionType() == ISD::NON_EXTLOAD) {
786 NewL = DAG.getLoad(L->getAddressingMode(), L->getExtensionType(), NVT, dl,
787 L->getChain(), L->getBasePtr(), L->getOffset(),
788 L->getPointerInfo(), NVT, L->getOriginalAlign(),
789 MMOFlags, L->getAAInfo());
790 // Legalized the chain result - switch anything that used the old chain to
791 // use the new one.
792 ReplaceValueWith(SDValue(N, 1), NewL.getValue(1));
793 return NewL;
794 }
795
796 // Do a non-extending load followed by FP_EXTEND.
797 NewL = DAG.getLoad(L->getAddressingMode(), ISD::NON_EXTLOAD, L->getMemoryVT(),
798 dl, L->getChain(), L->getBasePtr(), L->getOffset(),
799 L->getPointerInfo(), L->getMemoryVT(),
800 L->getOriginalAlign(), MMOFlags, L->getAAInfo());
801 // Legalized the chain result - switch anything that used the old chain to
802 // use the new one.
803 ReplaceValueWith(SDValue(N, 1), NewL.getValue(1));
804 auto ExtendNode = DAG.getNode(ISD::FP_EXTEND, dl, VT, NewL);
805 return BitConvertToInteger(ExtendNode);
806}
807
808SDValue DAGTypeLegalizer::SoftenFloatRes_SELECT(SDNode *N) {
809 SDValue LHS = GetSoftenedFloat(N->getOperand(1));
810 SDValue RHS = GetSoftenedFloat(N->getOperand(2));
811 return DAG.getSelect(SDLoc(N),
812 LHS.getValueType(), N->getOperand(0), LHS, RHS);
813}
814
815SDValue DAGTypeLegalizer::SoftenFloatRes_SELECT_CC(SDNode *N) {
816 SDValue LHS = GetSoftenedFloat(N->getOperand(2));
817 SDValue RHS = GetSoftenedFloat(N->getOperand(3));
818 return DAG.getNode(ISD::SELECT_CC, SDLoc(N),
819 LHS.getValueType(), N->getOperand(0),
820 N->getOperand(1), LHS, RHS, N->getOperand(4));
821}
822
823SDValue DAGTypeLegalizer::SoftenFloatRes_UNDEF(SDNode *N) {
824 return DAG.getUNDEF(TLI.getTypeToTransformTo(*DAG.getContext(),
825 N->getValueType(0)));
826}
827
828SDValue DAGTypeLegalizer::SoftenFloatRes_VAARG(SDNode *N) {
829 SDValue Chain = N->getOperand(0); // Get the chain.
830 SDValue Ptr = N->getOperand(1); // Get the pointer.
831 EVT VT = N->getValueType(0);
832 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
833 SDLoc dl(N);
834
835 SDValue NewVAARG;
836 NewVAARG = DAG.getVAArg(NVT, dl, Chain, Ptr, N->getOperand(2),
837 N->getConstantOperandVal(3));
838
839 // Legalized the chain result - switch anything that used the old chain to
840 // use the new one.
841 if (N != NewVAARG.getValue(1).getNode())
842 ReplaceValueWith(SDValue(N, 1), NewVAARG.getValue(1));
843 return NewVAARG;
844}
845
846SDValue DAGTypeLegalizer::SoftenFloatRes_XINT_TO_FP(SDNode *N) {
847 bool IsStrict = N->isStrictFPOpcode();
848 bool Signed = N->getOpcode() == ISD::SINT_TO_FP ||
849 N->getOpcode() == ISD::STRICT_SINT_TO_FP;
850 EVT SVT = N->getOperand(IsStrict ? 1 : 0).getValueType();
851 EVT RVT = N->getValueType(0);
852 EVT NVT = EVT();
853 SDLoc dl(N);
854
855 // If the input is not legal, eg: i1 -> fp, then it needs to be promoted to
856 // a larger type, eg: i8 -> fp. Even if it is legal, no libcall may exactly
857 // match. Look for an appropriate libcall.
858 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
859 for (unsigned t = MVT::FIRST_INTEGER_VALUETYPE;
860 t <= MVT::LAST_INTEGER_VALUETYPE && LC == RTLIB::UNKNOWN_LIBCALL; ++t) {
861 NVT = (MVT::SimpleValueType)t;
862 // The source needs to big enough to hold the operand.
863 if (NVT.bitsGE(SVT))
864 LC = Signed ? RTLIB::getSINTTOFP(NVT, RVT):RTLIB::getUINTTOFP (NVT, RVT);
865 }
866 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported XINT_TO_FP!");
867
868 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
869 // Sign/zero extend the argument if the libcall takes a larger type.
871 NVT, N->getOperand(IsStrict ? 1 : 0));
873 CallOptions.setSExt(Signed);
874 CallOptions.setTypeListBeforeSoften(SVT, RVT, true);
875 std::pair<SDValue, SDValue> Tmp =
876 TLI.makeLibCall(DAG, LC, TLI.getTypeToTransformTo(*DAG.getContext(), RVT),
877 Op, CallOptions, dl, Chain);
878
879 if (IsStrict)
880 ReplaceValueWith(SDValue(N, 1), Tmp.second);
881 return Tmp.first;
882}
883
884SDValue DAGTypeLegalizer::SoftenFloatRes_VECREDUCE(SDNode *N) {
885 // Expand and soften recursively.
886 ReplaceValueWith(SDValue(N, 0), TLI.expandVecReduce(N, DAG));
887 return SDValue();
888}
889
890SDValue DAGTypeLegalizer::SoftenFloatRes_VECREDUCE_SEQ(SDNode *N) {
891 ReplaceValueWith(SDValue(N, 0), TLI.expandVecReduceSeq(N, DAG));
892 return SDValue();
893}
894
895//===----------------------------------------------------------------------===//
896// Convert Float Operand to Integer
897//===----------------------------------------------------------------------===//
898
899bool DAGTypeLegalizer::SoftenFloatOperand(SDNode *N, unsigned OpNo) {
900 LLVM_DEBUG(dbgs() << "Soften float operand " << OpNo << ": "; N->dump(&DAG));
901 SDValue Res = SDValue();
902
903 switch (N->getOpcode()) {
904 default:
905#ifndef NDEBUG
906 dbgs() << "SoftenFloatOperand Op #" << OpNo << ": ";
907 N->dump(&DAG); dbgs() << "\n";
908#endif
909 report_fatal_error("Do not know how to soften this operator's operand!");
910
911 case ISD::BITCAST: Res = SoftenFloatOp_BITCAST(N); break;
912 case ISD::BR_CC: Res = SoftenFloatOp_BR_CC(N); break;
914 case ISD::FP_TO_FP16: // Same as FP_ROUND for softening purposes
915 case ISD::FP_TO_BF16:
917 case ISD::FP_ROUND: Res = SoftenFloatOp_FP_ROUND(N); break;
920 case ISD::FP_TO_SINT:
921 case ISD::FP_TO_UINT: Res = SoftenFloatOp_FP_TO_XINT(N); break;
924 Res = SoftenFloatOp_FP_TO_XINT_SAT(N); break;
926 case ISD::LROUND: Res = SoftenFloatOp_LROUND(N); break;
928 case ISD::LLROUND: Res = SoftenFloatOp_LLROUND(N); break;
930 case ISD::LRINT: Res = SoftenFloatOp_LRINT(N); break;
932 case ISD::LLRINT: Res = SoftenFloatOp_LLRINT(N); break;
933 case ISD::SELECT_CC: Res = SoftenFloatOp_SELECT_CC(N); break;
936 case ISD::SETCC: Res = SoftenFloatOp_SETCC(N); break;
937 case ISD::STORE: Res = SoftenFloatOp_STORE(N, OpNo); break;
938 case ISD::FCOPYSIGN: Res = SoftenFloatOp_FCOPYSIGN(N); break;
939 }
940
941 // If the result is null, the sub-method took care of registering results etc.
942 if (!Res.getNode()) return false;
943
944 // If the result is N, the sub-method updated N in place. Tell the legalizer
945 // core about this to re-analyze.
946 if (Res.getNode() == N)
947 return true;
948
949 assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 &&
950 "Invalid operand softening");
951
952 ReplaceValueWith(SDValue(N, 0), Res);
953 return false;
954}
955
956SDValue DAGTypeLegalizer::SoftenFloatOp_BITCAST(SDNode *N) {
957 SDValue Op0 = GetSoftenedFloat(N->getOperand(0));
958
959 return DAG.getNode(ISD::BITCAST, SDLoc(N), N->getValueType(0), Op0);
960}
961
962SDValue DAGTypeLegalizer::SoftenFloatOp_FP_ROUND(SDNode *N) {
963 // We actually deal with the partially-softened FP_TO_FP16 node too, which
964 // returns an i16 so doesn't meet the constraints necessary for FP_ROUND.
965 assert(N->getOpcode() == ISD::FP_ROUND || N->getOpcode() == ISD::FP_TO_FP16 ||
966 N->getOpcode() == ISD::STRICT_FP_TO_FP16 ||
967 N->getOpcode() == ISD::FP_TO_BF16 ||
968 N->getOpcode() == ISD::STRICT_FP_ROUND);
969
970 bool IsStrict = N->isStrictFPOpcode();
971 SDValue Op = N->getOperand(IsStrict ? 1 : 0);
972 EVT SVT = Op.getValueType();
973 EVT RVT = N->getValueType(0);
974 EVT FloatRVT = RVT;
975 if (N->getOpcode() == ISD::FP_TO_FP16 ||
976 N->getOpcode() == ISD::STRICT_FP_TO_FP16)
977 FloatRVT = MVT::f16;
978 else if (N->getOpcode() == ISD::FP_TO_BF16)
979 FloatRVT = MVT::bf16;
980
981 RTLIB::Libcall LC = RTLIB::getFPROUND(SVT, FloatRVT);
982 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported FP_ROUND libcall");
983
984 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
985 Op = GetSoftenedFloat(Op);
987 CallOptions.setTypeListBeforeSoften(SVT, RVT, true);
988 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, RVT, Op,
989 CallOptions, SDLoc(N),
990 Chain);
991 if (IsStrict) {
992 ReplaceValueWith(SDValue(N, 1), Tmp.second);
993 ReplaceValueWith(SDValue(N, 0), Tmp.first);
994 return SDValue();
995 }
996 return Tmp.first;
997}
998
999SDValue DAGTypeLegalizer::SoftenFloatOp_BR_CC(SDNode *N) {
1000 SDValue NewLHS = N->getOperand(2), NewRHS = N->getOperand(3);
1001 ISD::CondCode CCCode = cast<CondCodeSDNode>(N->getOperand(1))->get();
1002
1003 EVT VT = NewLHS.getValueType();
1004 NewLHS = GetSoftenedFloat(NewLHS);
1005 NewRHS = GetSoftenedFloat(NewRHS);
1006 TLI.softenSetCCOperands(DAG, VT, NewLHS, NewRHS, CCCode, SDLoc(N),
1007 N->getOperand(2), N->getOperand(3));
1008
1009 // If softenSetCCOperands returned a scalar, we need to compare the result
1010 // against zero to select between true and false values.
1011 if (!NewRHS.getNode()) {
1012 NewRHS = DAG.getConstant(0, SDLoc(N), NewLHS.getValueType());
1013 CCCode = ISD::SETNE;
1014 }
1015
1016 // Update N to have the operands specified.
1017 return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0),
1018 DAG.getCondCode(CCCode), NewLHS, NewRHS,
1019 N->getOperand(4)),
1020 0);
1021}
1022
1023// Even if the result type is legal, no libcall may exactly match. (e.g. We
1024// don't have FP-i8 conversions) This helper method looks for an appropriate
1025// promoted libcall.
1026static RTLIB::Libcall findFPToIntLibcall(EVT SrcVT, EVT RetVT, EVT &Promoted,
1027 bool Signed) {
1028 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
1029 for (unsigned IntVT = MVT::FIRST_INTEGER_VALUETYPE;
1030 IntVT <= MVT::LAST_INTEGER_VALUETYPE && LC == RTLIB::UNKNOWN_LIBCALL;
1031 ++IntVT) {
1032 Promoted = (MVT::SimpleValueType)IntVT;
1033 // The type needs to big enough to hold the result.
1034 if (Promoted.bitsGE(RetVT))
1035 LC = Signed ? RTLIB::getFPTOSINT(SrcVT, Promoted)
1036 : RTLIB::getFPTOUINT(SrcVT, Promoted);
1037 }
1038 return LC;
1039}
1040
1041SDValue DAGTypeLegalizer::SoftenFloatOp_FP_TO_XINT(SDNode *N) {
1042 bool IsStrict = N->isStrictFPOpcode();
1043 bool Signed = N->getOpcode() == ISD::FP_TO_SINT ||
1044 N->getOpcode() == ISD::STRICT_FP_TO_SINT;
1045
1046 SDValue Op = N->getOperand(IsStrict ? 1 : 0);
1047 EVT SVT = Op.getValueType();
1048 EVT RVT = N->getValueType(0);
1049 EVT NVT = EVT();
1050 SDLoc dl(N);
1051
1052 // If the result is not legal, eg: fp -> i1, then it needs to be promoted to
1053 // a larger type, eg: fp -> i32. Even if it is legal, no libcall may exactly
1054 // match, eg. we don't have fp -> i8 conversions.
1055 // Look for an appropriate libcall.
1056 RTLIB::Libcall LC = findFPToIntLibcall(SVT, RVT, NVT, Signed);
1057 assert(LC != RTLIB::UNKNOWN_LIBCALL && NVT.isSimple() &&
1058 "Unsupported FP_TO_XINT!");
1059
1060 Op = GetSoftenedFloat(Op);
1061 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
1063 CallOptions.setTypeListBeforeSoften(SVT, RVT, true);
1064 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, NVT, Op,
1065 CallOptions, dl, Chain);
1066
1067 // Truncate the result if the libcall returns a larger type.
1068 SDValue Res = DAG.getNode(ISD::TRUNCATE, dl, RVT, Tmp.first);
1069
1070 if (!IsStrict)
1071 return Res;
1072
1073 ReplaceValueWith(SDValue(N, 1), Tmp.second);
1074 ReplaceValueWith(SDValue(N, 0), Res);
1075 return SDValue();
1076}
1077
1078SDValue DAGTypeLegalizer::SoftenFloatOp_FP_TO_XINT_SAT(SDNode *N) {
1079 SDValue Res = TLI.expandFP_TO_INT_SAT(N, DAG);
1080 return Res;
1081}
1082
1083SDValue DAGTypeLegalizer::SoftenFloatOp_SELECT_CC(SDNode *N) {
1084 SDValue NewLHS = N->getOperand(0), NewRHS = N->getOperand(1);
1085 ISD::CondCode CCCode = cast<CondCodeSDNode>(N->getOperand(4))->get();
1086
1087 EVT VT = NewLHS.getValueType();
1088 NewLHS = GetSoftenedFloat(NewLHS);
1089 NewRHS = GetSoftenedFloat(NewRHS);
1090 TLI.softenSetCCOperands(DAG, VT, NewLHS, NewRHS, CCCode, SDLoc(N),
1091 N->getOperand(0), N->getOperand(1));
1092
1093 // If softenSetCCOperands returned a scalar, we need to compare the result
1094 // against zero to select between true and false values.
1095 if (!NewRHS.getNode()) {
1096 NewRHS = DAG.getConstant(0, SDLoc(N), NewLHS.getValueType());
1097 CCCode = ISD::SETNE;
1098 }
1099
1100 // Update N to have the operands specified.
1101 return SDValue(DAG.UpdateNodeOperands(N, NewLHS, NewRHS,
1102 N->getOperand(2), N->getOperand(3),
1103 DAG.getCondCode(CCCode)),
1104 0);
1105}
1106
1107SDValue DAGTypeLegalizer::SoftenFloatOp_SETCC(SDNode *N) {
1108 bool IsStrict = N->isStrictFPOpcode();
1109 SDValue Op0 = N->getOperand(IsStrict ? 1 : 0);
1110 SDValue Op1 = N->getOperand(IsStrict ? 2 : 1);
1111 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
1112 ISD::CondCode CCCode =
1113 cast<CondCodeSDNode>(N->getOperand(IsStrict ? 3 : 2))->get();
1114
1115 EVT VT = Op0.getValueType();
1116 SDValue NewLHS = GetSoftenedFloat(Op0);
1117 SDValue NewRHS = GetSoftenedFloat(Op1);
1118 TLI.softenSetCCOperands(DAG, VT, NewLHS, NewRHS, CCCode, SDLoc(N), Op0, Op1,
1119 Chain, N->getOpcode() == ISD::STRICT_FSETCCS);
1120
1121 // Update N to have the operands specified.
1122 if (NewRHS.getNode()) {
1123 if (IsStrict)
1124 NewLHS = DAG.getNode(ISD::SETCC, SDLoc(N), N->getValueType(0), NewLHS,
1125 NewRHS, DAG.getCondCode(CCCode));
1126 else
1127 return SDValue(DAG.UpdateNodeOperands(N, NewLHS, NewRHS,
1128 DAG.getCondCode(CCCode)), 0);
1129 }
1130
1131 // Otherwise, softenSetCCOperands returned a scalar, use it.
1132 assert((NewRHS.getNode() || NewLHS.getValueType() == N->getValueType(0)) &&
1133 "Unexpected setcc expansion!");
1134
1135 if (IsStrict) {
1136 ReplaceValueWith(SDValue(N, 0), NewLHS);
1137 ReplaceValueWith(SDValue(N, 1), Chain);
1138 return SDValue();
1139 }
1140 return NewLHS;
1141}
1142
1143SDValue DAGTypeLegalizer::SoftenFloatOp_STORE(SDNode *N, unsigned OpNo) {
1144 assert(ISD::isUNINDEXEDStore(N) && "Indexed store during type legalization!");
1145 assert(OpNo == 1 && "Can only soften the stored value!");
1146 StoreSDNode *ST = cast<StoreSDNode>(N);
1147 SDValue Val = ST->getValue();
1148 SDLoc dl(N);
1149
1150 if (ST->isTruncatingStore())
1151 // Do an FP_ROUND followed by a non-truncating store.
1152 Val = BitConvertToInteger(
1153 DAG.getNode(ISD::FP_ROUND, dl, ST->getMemoryVT(), Val,
1154 DAG.getIntPtrConstant(0, dl, /*isTarget=*/true)));
1155 else
1156 Val = GetSoftenedFloat(Val);
1157
1158 return DAG.getStore(ST->getChain(), dl, Val, ST->getBasePtr(),
1159 ST->getMemOperand());
1160}
1161
1162SDValue DAGTypeLegalizer::SoftenFloatOp_FCOPYSIGN(SDNode *N) {
1163 SDValue LHS = N->getOperand(0);
1164 SDValue RHS = BitConvertToInteger(N->getOperand(1));
1165 SDLoc dl(N);
1166
1167 EVT LVT = LHS.getValueType();
1168 EVT ILVT = EVT::getIntegerVT(*DAG.getContext(), LVT.getSizeInBits());
1169 EVT RVT = RHS.getValueType();
1170
1171 unsigned LSize = LVT.getSizeInBits();
1172 unsigned RSize = RVT.getSizeInBits();
1173
1174 // Shift right or sign-extend it if the two operands have different types.
1175 int SizeDiff = RSize - LSize;
1176 if (SizeDiff > 0) {
1177 RHS =
1178 DAG.getNode(ISD::SRL, dl, RVT, RHS,
1179 DAG.getConstant(SizeDiff, dl,
1180 TLI.getShiftAmountTy(RHS.getValueType(),
1181 DAG.getDataLayout())));
1182 RHS = DAG.getNode(ISD::TRUNCATE, dl, ILVT, RHS);
1183 } else if (SizeDiff < 0) {
1184 RHS = DAG.getNode(ISD::ANY_EXTEND, dl, LVT, RHS);
1185 RHS =
1186 DAG.getNode(ISD::SHL, dl, ILVT, RHS,
1187 DAG.getConstant(-SizeDiff, dl,
1188 TLI.getShiftAmountTy(RHS.getValueType(),
1189 DAG.getDataLayout())));
1190 }
1191
1192 RHS = DAG.getBitcast(LVT, RHS);
1193 return DAG.getNode(ISD::FCOPYSIGN, dl, LVT, LHS, RHS);
1194}
1195
1196SDValue DAGTypeLegalizer::SoftenFloatOp_Unary(SDNode *N, RTLIB::Libcall LC) {
1197 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
1198 bool IsStrict = N->isStrictFPOpcode();
1199 unsigned Offset = IsStrict ? 1 : 0;
1200 SDValue Op = GetSoftenedFloat(N->getOperand(0 + Offset));
1201 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
1203 EVT OpVT = N->getOperand(0 + Offset).getValueType();
1204 CallOptions.setTypeListBeforeSoften(OpVT, N->getValueType(0), true);
1205 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, NVT, Op,
1206 CallOptions, SDLoc(N),
1207 Chain);
1208 if (IsStrict) {
1209 ReplaceValueWith(SDValue(N, 1), Tmp.second);
1210 ReplaceValueWith(SDValue(N, 0), Tmp.first);
1211 return SDValue();
1212 }
1213
1214 return Tmp.first;
1215}
1216
1217SDValue DAGTypeLegalizer::SoftenFloatOp_LROUND(SDNode *N) {
1218 EVT OpVT = N->getOperand(N->isStrictFPOpcode() ? 1 : 0).getValueType();
1219 return SoftenFloatOp_Unary(N, GetFPLibCall(OpVT,
1220 RTLIB::LROUND_F32,
1221 RTLIB::LROUND_F64,
1222 RTLIB::LROUND_F80,
1223 RTLIB::LROUND_F128,
1224 RTLIB::LROUND_PPCF128));
1225}
1226
1227SDValue DAGTypeLegalizer::SoftenFloatOp_LLROUND(SDNode *N) {
1228 EVT OpVT = N->getOperand(N->isStrictFPOpcode() ? 1 : 0).getValueType();
1229 return SoftenFloatOp_Unary(N, GetFPLibCall(OpVT,
1230 RTLIB::LLROUND_F32,
1231 RTLIB::LLROUND_F64,
1232 RTLIB::LLROUND_F80,
1233 RTLIB::LLROUND_F128,
1234 RTLIB::LLROUND_PPCF128));
1235}
1236
1237SDValue DAGTypeLegalizer::SoftenFloatOp_LRINT(SDNode *N) {
1238 EVT OpVT = N->getOperand(N->isStrictFPOpcode() ? 1 : 0).getValueType();
1239 return SoftenFloatOp_Unary(N, GetFPLibCall(OpVT,
1240 RTLIB::LRINT_F32,
1241 RTLIB::LRINT_F64,
1242 RTLIB::LRINT_F80,
1243 RTLIB::LRINT_F128,
1244 RTLIB::LRINT_PPCF128));
1245}
1246
1247SDValue DAGTypeLegalizer::SoftenFloatOp_LLRINT(SDNode *N) {
1248 EVT OpVT = N->getOperand(N->isStrictFPOpcode() ? 1 : 0).getValueType();
1249 return SoftenFloatOp_Unary(N, GetFPLibCall(OpVT,
1250 RTLIB::LLRINT_F32,
1251 RTLIB::LLRINT_F64,
1252 RTLIB::LLRINT_F80,
1253 RTLIB::LLRINT_F128,
1254 RTLIB::LLRINT_PPCF128));
1255}
1256
1257//===----------------------------------------------------------------------===//
1258// Float Result Expansion
1259//===----------------------------------------------------------------------===//
1260
1261/// ExpandFloatResult - This method is called when the specified result of the
1262/// specified node is found to need expansion. At this point, the node may also
1263/// have invalid operands or may have other results that need promotion, we just
1264/// know that (at least) one result needs expansion.
1265void DAGTypeLegalizer::ExpandFloatResult(SDNode *N, unsigned ResNo) {
1266 LLVM_DEBUG(dbgs() << "Expand float result: "; N->dump(&DAG));
1267 SDValue Lo, Hi;
1268 Lo = Hi = SDValue();
1269
1270 // See if the target wants to custom expand this node.
1271 if (CustomLowerNode(N, N->getValueType(ResNo), true))
1272 return;
1273
1274 switch (N->getOpcode()) {
1275 default:
1276#ifndef NDEBUG
1277 dbgs() << "ExpandFloatResult #" << ResNo << ": ";
1278 N->dump(&DAG); dbgs() << "\n";
1279#endif
1280 report_fatal_error("Do not know how to expand the result of this "
1281 "operator!");
1282
1283 case ISD::UNDEF: SplitRes_UNDEF(N, Lo, Hi); break;
1284 case ISD::SELECT: SplitRes_Select(N, Lo, Hi); break;
1285 case ISD::SELECT_CC: SplitRes_SELECT_CC(N, Lo, Hi); break;
1286
1287 case ISD::MERGE_VALUES: ExpandRes_MERGE_VALUES(N, ResNo, Lo, Hi); break;
1288 case ISD::BITCAST: ExpandRes_BITCAST(N, Lo, Hi); break;
1289 case ISD::BUILD_PAIR: ExpandRes_BUILD_PAIR(N, Lo, Hi); break;
1290 case ISD::EXTRACT_ELEMENT: ExpandRes_EXTRACT_ELEMENT(N, Lo, Hi); break;
1291 case ISD::EXTRACT_VECTOR_ELT: ExpandRes_EXTRACT_VECTOR_ELT(N, Lo, Hi); break;
1292 case ISD::VAARG: ExpandRes_VAARG(N, Lo, Hi); break;
1293
1294 case ISD::ConstantFP: ExpandFloatRes_ConstantFP(N, Lo, Hi); break;
1295 case ISD::FABS: ExpandFloatRes_FABS(N, Lo, Hi); break;
1297 case ISD::FMINNUM: ExpandFloatRes_FMINNUM(N, Lo, Hi); break;
1299 case ISD::FMAXNUM: ExpandFloatRes_FMAXNUM(N, Lo, Hi); break;
1300 case ISD::STRICT_FADD:
1301 case ISD::FADD: ExpandFloatRes_FADD(N, Lo, Hi); break;
1302 case ISD::FCBRT: ExpandFloatRes_FCBRT(N, Lo, Hi); break;
1303 case ISD::STRICT_FCEIL:
1304 case ISD::FCEIL: ExpandFloatRes_FCEIL(N, Lo, Hi); break;
1305 case ISD::FCOPYSIGN: ExpandFloatRes_FCOPYSIGN(N, Lo, Hi); break;
1306 case ISD::STRICT_FCOS:
1307 case ISD::FCOS: ExpandFloatRes_FCOS(N, Lo, Hi); break;
1308 case ISD::STRICT_FDIV:
1309 case ISD::FDIV: ExpandFloatRes_FDIV(N, Lo, Hi); break;
1310 case ISD::STRICT_FEXP:
1311 case ISD::FEXP: ExpandFloatRes_FEXP(N, Lo, Hi); break;
1312 case ISD::STRICT_FEXP2:
1313 case ISD::FEXP2: ExpandFloatRes_FEXP2(N, Lo, Hi); break;
1314 case ISD::FEXP10: ExpandFloatRes_FEXP10(N, Lo, Hi); break;
1315 case ISD::STRICT_FFLOOR:
1316 case ISD::FFLOOR: ExpandFloatRes_FFLOOR(N, Lo, Hi); break;
1317 case ISD::STRICT_FLOG:
1318 case ISD::FLOG: ExpandFloatRes_FLOG(N, Lo, Hi); break;
1319 case ISD::STRICT_FLOG2:
1320 case ISD::FLOG2: ExpandFloatRes_FLOG2(N, Lo, Hi); break;
1321 case ISD::STRICT_FLOG10:
1322 case ISD::FLOG10: ExpandFloatRes_FLOG10(N, Lo, Hi); break;
1323 case ISD::STRICT_FMA:
1324 case ISD::FMA: ExpandFloatRes_FMA(N, Lo, Hi); break;
1325 case ISD::STRICT_FMUL:
1326 case ISD::FMUL: ExpandFloatRes_FMUL(N, Lo, Hi); break;
1328 case ISD::FNEARBYINT: ExpandFloatRes_FNEARBYINT(N, Lo, Hi); break;
1329 case ISD::FNEG: ExpandFloatRes_FNEG(N, Lo, Hi); break;
1331 case ISD::FP_EXTEND: ExpandFloatRes_FP_EXTEND(N, Lo, Hi); break;
1332 case ISD::STRICT_FPOW:
1333 case ISD::FPOW: ExpandFloatRes_FPOW(N, Lo, Hi); break;
1334 case ISD::STRICT_FPOWI:
1335 case ISD::FPOWI: ExpandFloatRes_FPOWI(N, Lo, Hi); break;
1336 case ISD::FLDEXP:
1337 case ISD::STRICT_FLDEXP: ExpandFloatRes_FLDEXP(N, Lo, Hi); break;
1338 case ISD::FREEZE: ExpandFloatRes_FREEZE(N, Lo, Hi); break;
1339 case ISD::STRICT_FRINT:
1340 case ISD::FRINT: ExpandFloatRes_FRINT(N, Lo, Hi); break;
1341 case ISD::STRICT_FROUND:
1342 case ISD::FROUND: ExpandFloatRes_FROUND(N, Lo, Hi); break;
1344 case ISD::FROUNDEVEN: ExpandFloatRes_FROUNDEVEN(N, Lo, Hi); break;
1345 case ISD::STRICT_FSIN:
1346 case ISD::FSIN: ExpandFloatRes_FSIN(N, Lo, Hi); break;
1347 case ISD::STRICT_FSQRT:
1348 case ISD::FSQRT: ExpandFloatRes_FSQRT(N, Lo, Hi); break;
1349 case ISD::STRICT_FSUB:
1350 case ISD::FSUB: ExpandFloatRes_FSUB(N, Lo, Hi); break;
1351 case ISD::STRICT_FTRUNC:
1352 case ISD::FTRUNC: ExpandFloatRes_FTRUNC(N, Lo, Hi); break;
1353 case ISD::LOAD: ExpandFloatRes_LOAD(N, Lo, Hi); break;
1356 case ISD::SINT_TO_FP:
1357 case ISD::UINT_TO_FP: ExpandFloatRes_XINT_TO_FP(N, Lo, Hi); break;
1358 case ISD::STRICT_FREM:
1359 case ISD::FREM: ExpandFloatRes_FREM(N, Lo, Hi); break;
1360 }
1361
1362 // If Lo/Hi is null, the sub-method took care of registering results etc.
1363 if (Lo.getNode())
1364 SetExpandedFloat(SDValue(N, ResNo), Lo, Hi);
1365}
1366
1367void DAGTypeLegalizer::ExpandFloatRes_ConstantFP(SDNode *N, SDValue &Lo,
1368 SDValue &Hi) {
1369 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
1370 assert(NVT.getSizeInBits() == 64 &&
1371 "Do not know how to expand this float constant!");
1372 APInt C = cast<ConstantFPSDNode>(N)->getValueAPF().bitcastToAPInt();
1373 SDLoc dl(N);
1375 APInt(64, C.getRawData()[1])),
1376 dl, NVT);
1378 APInt(64, C.getRawData()[0])),
1379 dl, NVT);
1380}
1381
1382void DAGTypeLegalizer::ExpandFloatRes_Unary(SDNode *N, RTLIB::Libcall LC,
1383 SDValue &Lo, SDValue &Hi) {
1384 bool IsStrict = N->isStrictFPOpcode();
1385 unsigned Offset = IsStrict ? 1 : 0;
1386 SDValue Op = N->getOperand(0 + Offset);
1387 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
1389 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, N->getValueType(0),
1390 Op, CallOptions, SDLoc(N),
1391 Chain);
1392 if (IsStrict)
1393 ReplaceValueWith(SDValue(N, 1), Tmp.second);
1394 GetPairElements(Tmp.first, Lo, Hi);
1395}
1396
1397void DAGTypeLegalizer::ExpandFloatRes_Binary(SDNode *N, RTLIB::Libcall LC,
1398 SDValue &Lo, SDValue &Hi) {
1399 bool IsStrict = N->isStrictFPOpcode();
1400 unsigned Offset = IsStrict ? 1 : 0;
1401 SDValue Ops[] = { N->getOperand(0 + Offset), N->getOperand(1 + Offset) };
1402 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
1404 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, N->getValueType(0),
1405 Ops, CallOptions, SDLoc(N),
1406 Chain);
1407 if (IsStrict)
1408 ReplaceValueWith(SDValue(N, 1), Tmp.second);
1409 GetPairElements(Tmp.first, Lo, Hi);
1410}
1411
1412void DAGTypeLegalizer::ExpandFloatRes_FABS(SDNode *N, SDValue &Lo,
1413 SDValue &Hi) {
1414 assert(N->getValueType(0) == MVT::ppcf128 &&
1415 "Logic only correct for ppcf128!");
1416 SDLoc dl(N);
1417 SDValue Tmp;
1418 GetExpandedFloat(N->getOperand(0), Lo, Tmp);
1419 Hi = DAG.getNode(ISD::FABS, dl, Tmp.getValueType(), Tmp);
1420 // Lo = Hi==fabs(Hi) ? Lo : -Lo;
1421 Lo = DAG.getSelectCC(dl, Tmp, Hi, Lo,
1422 DAG.getNode(ISD::FNEG, dl, Lo.getValueType(), Lo),
1423 ISD::SETEQ);
1424}
1425
1426void DAGTypeLegalizer::ExpandFloatRes_FMINNUM(SDNode *N, SDValue &Lo,
1427 SDValue &Hi) {
1428 ExpandFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
1429 RTLIB::FMIN_F32, RTLIB::FMIN_F64,
1430 RTLIB::FMIN_F80, RTLIB::FMIN_F128,
1431 RTLIB::FMIN_PPCF128), Lo, Hi);
1432}
1433
1434void DAGTypeLegalizer::ExpandFloatRes_FMAXNUM(SDNode *N, SDValue &Lo,
1435 SDValue &Hi) {
1436 ExpandFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
1437 RTLIB::FMAX_F32, RTLIB::FMAX_F64,
1438 RTLIB::FMAX_F80, RTLIB::FMAX_F128,
1439 RTLIB::FMAX_PPCF128), Lo, Hi);
1440}
1441
1442void DAGTypeLegalizer::ExpandFloatRes_FADD(SDNode *N, SDValue &Lo,
1443 SDValue &Hi) {
1444 ExpandFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
1445 RTLIB::ADD_F32, RTLIB::ADD_F64,
1446 RTLIB::ADD_F80, RTLIB::ADD_F128,
1447 RTLIB::ADD_PPCF128), Lo, Hi);
1448}
1449
1450void DAGTypeLegalizer::ExpandFloatRes_FCBRT(SDNode *N, SDValue &Lo,
1451 SDValue &Hi) {
1452 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0), RTLIB::CBRT_F32,
1453 RTLIB::CBRT_F64, RTLIB::CBRT_F80,
1454 RTLIB::CBRT_F128,
1455 RTLIB::CBRT_PPCF128), Lo, Hi);
1456}
1457
1458void DAGTypeLegalizer::ExpandFloatRes_FCEIL(SDNode *N,
1459 SDValue &Lo, SDValue &Hi) {
1460 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
1461 RTLIB::CEIL_F32, RTLIB::CEIL_F64,
1462 RTLIB::CEIL_F80, RTLIB::CEIL_F128,
1463 RTLIB::CEIL_PPCF128), Lo, Hi);
1464}
1465
1466void DAGTypeLegalizer::ExpandFloatRes_FCOPYSIGN(SDNode *N,
1467 SDValue &Lo, SDValue &Hi) {
1468 ExpandFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
1469 RTLIB::COPYSIGN_F32,
1470 RTLIB::COPYSIGN_F64,
1471 RTLIB::COPYSIGN_F80,
1472 RTLIB::COPYSIGN_F128,
1473 RTLIB::COPYSIGN_PPCF128), Lo, Hi);
1474}
1475
1476void DAGTypeLegalizer::ExpandFloatRes_FCOS(SDNode *N,
1477 SDValue &Lo, SDValue &Hi) {
1478 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
1479 RTLIB::COS_F32, RTLIB::COS_F64,
1480 RTLIB::COS_F80, RTLIB::COS_F128,
1481 RTLIB::COS_PPCF128), Lo, Hi);
1482}
1483
1484void DAGTypeLegalizer::ExpandFloatRes_FDIV(SDNode *N, SDValue &Lo,
1485 SDValue &Hi) {
1486 ExpandFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
1487 RTLIB::DIV_F32,
1488 RTLIB::DIV_F64,
1489 RTLIB::DIV_F80,
1490 RTLIB::DIV_F128,
1491 RTLIB::DIV_PPCF128), Lo, Hi);
1492}
1493
1494void DAGTypeLegalizer::ExpandFloatRes_FEXP(SDNode *N,
1495 SDValue &Lo, SDValue &Hi) {
1496 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
1497 RTLIB::EXP_F32, RTLIB::EXP_F64,
1498 RTLIB::EXP_F80, RTLIB::EXP_F128,
1499 RTLIB::EXP_PPCF128), Lo, Hi);
1500}
1501
1502void DAGTypeLegalizer::ExpandFloatRes_FEXP2(SDNode *N,
1503 SDValue &Lo, SDValue &Hi) {
1504 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
1505 RTLIB::EXP2_F32, RTLIB::EXP2_F64,
1506 RTLIB::EXP2_F80, RTLIB::EXP2_F128,
1507 RTLIB::EXP2_PPCF128), Lo, Hi);
1508}
1509
1510void DAGTypeLegalizer::ExpandFloatRes_FEXP10(SDNode *N, SDValue &Lo,
1511 SDValue &Hi) {
1512 ExpandFloatRes_Unary(N,
1513 GetFPLibCall(N->getValueType(0), RTLIB::EXP10_F32,
1514 RTLIB::EXP10_F64, RTLIB::EXP10_F80,
1515 RTLIB::EXP10_F128, RTLIB::EXP10_PPCF128),
1516 Lo, Hi);
1517}
1518
1519void DAGTypeLegalizer::ExpandFloatRes_FFLOOR(SDNode *N,
1520 SDValue &Lo, SDValue &Hi) {
1521 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
1522 RTLIB::FLOOR_F32, RTLIB::FLOOR_F64,
1523 RTLIB::FLOOR_F80, RTLIB::FLOOR_F128,
1524 RTLIB::FLOOR_PPCF128), Lo, Hi);
1525}
1526
1527void DAGTypeLegalizer::ExpandFloatRes_FLOG(SDNode *N,
1528 SDValue &Lo, SDValue &Hi) {
1529 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
1530 RTLIB::LOG_F32, RTLIB::LOG_F64,
1531 RTLIB::LOG_F80, RTLIB::LOG_F128,
1532 RTLIB::LOG_PPCF128), Lo, Hi);
1533}
1534
1535void DAGTypeLegalizer::ExpandFloatRes_FLOG2(SDNode *N,
1536 SDValue &Lo, SDValue &Hi) {
1537 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
1538 RTLIB::LOG2_F32, RTLIB::LOG2_F64,
1539 RTLIB::LOG2_F80, RTLIB::LOG2_F128,
1540 RTLIB::LOG2_PPCF128), Lo, Hi);
1541}
1542
1543void DAGTypeLegalizer::ExpandFloatRes_FLOG10(SDNode *N,
1544 SDValue &Lo, SDValue &Hi) {
1545 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
1546 RTLIB::LOG10_F32, RTLIB::LOG10_F64,
1547 RTLIB::LOG10_F80, RTLIB::LOG10_F128,
1548 RTLIB::LOG10_PPCF128), Lo, Hi);
1549}
1550
1551void DAGTypeLegalizer::ExpandFloatRes_FMA(SDNode *N, SDValue &Lo,
1552 SDValue &Hi) {
1553 bool IsStrict = N->isStrictFPOpcode();
1554 unsigned Offset = IsStrict ? 1 : 0;
1555 SDValue Ops[3] = { N->getOperand(0 + Offset), N->getOperand(1 + Offset),
1556 N->getOperand(2 + Offset) };
1557 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
1559 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, GetFPLibCall(N->getValueType(0),
1560 RTLIB::FMA_F32,
1561 RTLIB::FMA_F64,
1562 RTLIB::FMA_F80,
1563 RTLIB::FMA_F128,
1564 RTLIB::FMA_PPCF128),
1565 N->getValueType(0), Ops, CallOptions,
1566 SDLoc(N), Chain);
1567 if (IsStrict)
1568 ReplaceValueWith(SDValue(N, 1), Tmp.second);
1569 GetPairElements(Tmp.first, Lo, Hi);
1570}
1571
1572void DAGTypeLegalizer::ExpandFloatRes_FMUL(SDNode *N, SDValue &Lo,
1573 SDValue &Hi) {
1574 ExpandFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
1575 RTLIB::MUL_F32,
1576 RTLIB::MUL_F64,
1577 RTLIB::MUL_F80,
1578 RTLIB::MUL_F128,
1579 RTLIB::MUL_PPCF128), Lo, Hi);
1580}
1581
1582void DAGTypeLegalizer::ExpandFloatRes_FNEARBYINT(SDNode *N,
1583 SDValue &Lo, SDValue &Hi) {
1584 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
1585 RTLIB::NEARBYINT_F32,
1586 RTLIB::NEARBYINT_F64,
1587 RTLIB::NEARBYINT_F80,
1588 RTLIB::NEARBYINT_F128,
1589 RTLIB::NEARBYINT_PPCF128), Lo, Hi);
1590}
1591
1592void DAGTypeLegalizer::ExpandFloatRes_FNEG(SDNode *N, SDValue &Lo,
1593 SDValue &Hi) {
1594 SDLoc dl(N);
1595 GetExpandedFloat(N->getOperand(0), Lo, Hi);
1596 Lo = DAG.getNode(ISD::FNEG, dl, Lo.getValueType(), Lo);
1597 Hi = DAG.getNode(ISD::FNEG, dl, Hi.getValueType(), Hi);
1598}
1599
1600void DAGTypeLegalizer::ExpandFloatRes_FP_EXTEND(SDNode *N, SDValue &Lo,
1601 SDValue &Hi) {
1602 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
1603 SDLoc dl(N);
1604 bool IsStrict = N->isStrictFPOpcode();
1605
1606 SDValue Chain;
1607 if (IsStrict) {
1608 // If the expanded type is the same as the input type, just bypass the node.
1609 if (NVT == N->getOperand(1).getValueType()) {
1610 Hi = N->getOperand(1);
1611 Chain = N->getOperand(0);
1612 } else {
1613 // Other we need to extend.
1614 Hi = DAG.getNode(ISD::STRICT_FP_EXTEND, dl, { NVT, MVT::Other },
1615 { N->getOperand(0), N->getOperand(1) });
1616 Chain = Hi.getValue(1);
1617 }
1618 } else {
1619 Hi = DAG.getNode(ISD::FP_EXTEND, dl, NVT, N->getOperand(0));
1620 }
1621
1623 APInt(NVT.getSizeInBits(), 0)), dl, NVT);
1624
1625 if (IsStrict)
1626 ReplaceValueWith(SDValue(N, 1), Chain);
1627}
1628
1629void DAGTypeLegalizer::ExpandFloatRes_FPOW(SDNode *N,
1630 SDValue &Lo, SDValue &Hi) {
1631 ExpandFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
1632 RTLIB::POW_F32, RTLIB::POW_F64,
1633 RTLIB::POW_F80, RTLIB::POW_F128,
1634 RTLIB::POW_PPCF128), Lo, Hi);
1635}
1636
1637void DAGTypeLegalizer::ExpandFloatRes_FPOWI(SDNode *N,
1638 SDValue &Lo, SDValue &Hi) {
1639 ExpandFloatRes_Binary(N, RTLIB::getPOWI(N->getValueType(0)), Lo, Hi);
1640}
1641
1642void DAGTypeLegalizer::ExpandFloatRes_FLDEXP(SDNode *N, SDValue &Lo,
1643 SDValue &Hi) {
1644 ExpandFloatRes_Binary(N, RTLIB::getLDEXP(N->getValueType(0)), Lo, Hi);
1645}
1646
1647void DAGTypeLegalizer::ExpandFloatRes_FREEZE(SDNode *N,
1648 SDValue &Lo, SDValue &Hi) {
1649 assert(N->getValueType(0) == MVT::ppcf128 &&
1650 "Logic only correct for ppcf128!");
1651
1652 SDLoc dl(N);
1653 GetExpandedFloat(N->getOperand(0), Lo, Hi);
1654 Lo = DAG.getNode(ISD::FREEZE, dl, Lo.getValueType(), Lo);
1655 Hi = DAG.getNode(ISD::FREEZE, dl, Hi.getValueType(), Hi);
1656}
1657
1658void DAGTypeLegalizer::ExpandFloatRes_FREM(SDNode *N,
1659 SDValue &Lo, SDValue &Hi) {
1660 ExpandFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
1661 RTLIB::REM_F32, RTLIB::REM_F64,
1662 RTLIB::REM_F80, RTLIB::REM_F128,
1663 RTLIB::REM_PPCF128), Lo, Hi);
1664}
1665
1666void DAGTypeLegalizer::ExpandFloatRes_FRINT(SDNode *N,
1667 SDValue &Lo, SDValue &Hi) {
1668 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
1669 RTLIB::RINT_F32, RTLIB::RINT_F64,
1670 RTLIB::RINT_F80, RTLIB::RINT_F128,
1671 RTLIB::RINT_PPCF128), Lo, Hi);
1672}
1673
1674void DAGTypeLegalizer::ExpandFloatRes_FROUND(SDNode *N,
1675 SDValue &Lo, SDValue &Hi) {
1676 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
1677 RTLIB::ROUND_F32,
1678 RTLIB::ROUND_F64,
1679 RTLIB::ROUND_F80,
1680 RTLIB::ROUND_F128,
1681 RTLIB::ROUND_PPCF128), Lo, Hi);
1682}
1683
1684void DAGTypeLegalizer::ExpandFloatRes_FROUNDEVEN(SDNode *N,
1685 SDValue &Lo, SDValue &Hi) {
1686 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
1687 RTLIB::ROUNDEVEN_F32,
1688 RTLIB::ROUNDEVEN_F64,
1689 RTLIB::ROUNDEVEN_F80,
1690 RTLIB::ROUNDEVEN_F128,
1691 RTLIB::ROUNDEVEN_PPCF128), Lo, Hi);
1692}
1693
1694void DAGTypeLegalizer::ExpandFloatRes_FSIN(SDNode *N,
1695 SDValue &Lo, SDValue &Hi) {
1696 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
1697 RTLIB::SIN_F32, RTLIB::SIN_F64,
1698 RTLIB::SIN_F80, RTLIB::SIN_F128,
1699 RTLIB::SIN_PPCF128), Lo, Hi);
1700}
1701
1702void DAGTypeLegalizer::ExpandFloatRes_FSQRT(SDNode *N,
1703 SDValue &Lo, SDValue &Hi) {
1704 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
1705 RTLIB::SQRT_F32, RTLIB::SQRT_F64,
1706 RTLIB::SQRT_F80, RTLIB::SQRT_F128,
1707 RTLIB::SQRT_PPCF128), Lo, Hi);
1708}
1709
1710void DAGTypeLegalizer::ExpandFloatRes_FSUB(SDNode *N, SDValue &Lo,
1711 SDValue &Hi) {
1712 ExpandFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
1713 RTLIB::SUB_F32,
1714 RTLIB::SUB_F64,
1715 RTLIB::SUB_F80,
1716 RTLIB::SUB_F128,
1717 RTLIB::SUB_PPCF128), Lo, Hi);
1718}
1719
1720void DAGTypeLegalizer::ExpandFloatRes_FTRUNC(SDNode *N,
1721 SDValue &Lo, SDValue &Hi) {
1722 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
1723 RTLIB::TRUNC_F32, RTLIB::TRUNC_F64,
1724 RTLIB::TRUNC_F80, RTLIB::TRUNC_F128,
1725 RTLIB::TRUNC_PPCF128), Lo, Hi);
1726}
1727
1728void DAGTypeLegalizer::ExpandFloatRes_LOAD(SDNode *N, SDValue &Lo,
1729 SDValue &Hi) {
1730 if (ISD::isNormalLoad(N)) {
1731 ExpandRes_NormalLoad(N, Lo, Hi);
1732 return;
1733 }
1734
1735 assert(ISD::isUNINDEXEDLoad(N) && "Indexed load during type legalization!");
1736 LoadSDNode *LD = cast<LoadSDNode>(N);
1737 SDValue Chain = LD->getChain();
1738 SDValue Ptr = LD->getBasePtr();
1739 SDLoc dl(N);
1740
1741 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), LD->getValueType(0));
1742 assert(NVT.isByteSized() && "Expanded type not byte sized!");
1743 assert(LD->getMemoryVT().bitsLE(NVT) && "Float type not round?");
1744
1745 Hi = DAG.getExtLoad(LD->getExtensionType(), dl, NVT, Chain, Ptr,
1746 LD->getMemoryVT(), LD->getMemOperand());
1747
1748 // Remember the chain.
1749 Chain = Hi.getValue(1);
1750
1751 // The low part is zero.
1753 APInt(NVT.getSizeInBits(), 0)), dl, NVT);
1754
1755 // Modified the chain - switch anything that used the old chain to use the
1756 // new one.
1757 ReplaceValueWith(SDValue(LD, 1), Chain);
1758}
1759
1760void DAGTypeLegalizer::ExpandFloatRes_XINT_TO_FP(SDNode *N, SDValue &Lo,
1761 SDValue &Hi) {
1762 assert(N->getValueType(0) == MVT::ppcf128 && "Unsupported XINT_TO_FP!");
1763 EVT VT = N->getValueType(0);
1764 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
1765 bool Strict = N->isStrictFPOpcode();
1766 SDValue Src = N->getOperand(Strict ? 1 : 0);
1767 EVT SrcVT = Src.getValueType();
1768 bool isSigned = N->getOpcode() == ISD::SINT_TO_FP ||
1769 N->getOpcode() == ISD::STRICT_SINT_TO_FP;
1770 SDLoc dl(N);
1771 SDValue Chain = Strict ? N->getOperand(0) : DAG.getEntryNode();
1772
1773 // TODO: Any other flags to propagate?
1775 Flags.setNoFPExcept(N->getFlags().hasNoFPExcept());
1776
1777 // First do an SINT_TO_FP, whether the original was signed or unsigned.
1778 // When promoting partial word types to i32 we must honor the signedness,
1779 // though.
1780 if (SrcVT.bitsLE(MVT::i32)) {
1781 // The integer can be represented exactly in an f64.
1783 APInt(NVT.getSizeInBits(), 0)), dl, NVT);
1784 if (Strict) {
1785 Hi = DAG.getNode(N->getOpcode(), dl, DAG.getVTList(NVT, MVT::Other),
1786 {Chain, Src}, Flags);
1787 Chain = Hi.getValue(1);
1788 } else
1789 Hi = DAG.getNode(N->getOpcode(), dl, NVT, Src);
1790 } else {
1791 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
1792 if (SrcVT.bitsLE(MVT::i64)) {
1794 MVT::i64, Src);
1795 LC = RTLIB::SINTTOFP_I64_PPCF128;
1796 } else if (SrcVT.bitsLE(MVT::i128)) {
1797 Src = DAG.getNode(ISD::SIGN_EXTEND, dl, MVT::i128, Src);
1798 LC = RTLIB::SINTTOFP_I128_PPCF128;
1799 }
1800 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported XINT_TO_FP!");
1801
1803 CallOptions.setSExt(true);
1804 std::pair<SDValue, SDValue> Tmp =
1805 TLI.makeLibCall(DAG, LC, VT, Src, CallOptions, dl, Chain);
1806 if (Strict)
1807 Chain = Tmp.second;
1808 GetPairElements(Tmp.first, Lo, Hi);
1809 }
1810
1811 // No need to complement for unsigned 32-bit integers
1812 if (isSigned || SrcVT.bitsLE(MVT::i32)) {
1813 if (Strict)
1814 ReplaceValueWith(SDValue(N, 1), Chain);
1815
1816 return;
1817 }
1818
1819 // Unsigned - fix up the SINT_TO_FP value just calculated.
1820 // FIXME: For unsigned i128 to ppc_fp128 conversion, we need to carefully
1821 // keep semantics correctness if the integer is not exactly representable
1822 // here. See ExpandLegalINT_TO_FP.
1823 Hi = DAG.getNode(ISD::BUILD_PAIR, dl, VT, Lo, Hi);
1824 SrcVT = Src.getValueType();
1825
1826 // x>=0 ? (ppcf128)(iN)x : (ppcf128)(iN)x + 2^N; N=32,64,128.
1827 static const uint64_t TwoE32[] = { 0x41f0000000000000LL, 0 };
1828 static const uint64_t TwoE64[] = { 0x43f0000000000000LL, 0 };
1829 static const uint64_t TwoE128[] = { 0x47f0000000000000LL, 0 };
1830 ArrayRef<uint64_t> Parts;
1831
1832 switch (SrcVT.getSimpleVT().SimpleTy) {
1833 default:
1834 llvm_unreachable("Unsupported UINT_TO_FP!");
1835 case MVT::i32:
1836 Parts = TwoE32;
1837 break;
1838 case MVT::i64:
1839 Parts = TwoE64;
1840 break;
1841 case MVT::i128:
1842 Parts = TwoE128;
1843 break;
1844 }
1845
1846 // TODO: Are there other fast-math-flags to propagate to this FADD?
1847 SDValue NewLo = DAG.getConstantFP(
1848 APFloat(APFloat::PPCDoubleDouble(), APInt(128, Parts)), dl, MVT::ppcf128);
1849 if (Strict) {
1850 Lo = DAG.getNode(ISD::STRICT_FADD, dl, DAG.getVTList(VT, MVT::Other),
1851 {Chain, Hi, NewLo}, Flags);
1852 Chain = Lo.getValue(1);
1853 ReplaceValueWith(SDValue(N, 1), Chain);
1854 } else
1855 Lo = DAG.getNode(ISD::FADD, dl, VT, Hi, NewLo);
1856 Lo = DAG.getSelectCC(dl, Src, DAG.getConstant(0, dl, SrcVT),
1857 Lo, Hi, ISD::SETLT);
1858 GetPairElements(Lo, Lo, Hi);
1859}
1860
1861
1862//===----------------------------------------------------------------------===//
1863// Float Operand Expansion
1864//===----------------------------------------------------------------------===//
1865
1866/// ExpandFloatOperand - This method is called when the specified operand of the
1867/// specified node is found to need expansion. At this point, all of the result
1868/// types of the node are known to be legal, but other operands of the node may
1869/// need promotion or expansion as well as the specified one.
1870bool DAGTypeLegalizer::ExpandFloatOperand(SDNode *N, unsigned OpNo) {
1871 LLVM_DEBUG(dbgs() << "Expand float operand: "; N->dump(&DAG));
1872 SDValue Res = SDValue();
1873
1874 // See if the target wants to custom expand this node.
1875 if (CustomLowerNode(N, N->getOperand(OpNo).getValueType(), false))
1876 return false;
1877
1878 switch (N->getOpcode()) {
1879 default:
1880#ifndef NDEBUG
1881 dbgs() << "ExpandFloatOperand Op #" << OpNo << ": ";
1882 N->dump(&DAG); dbgs() << "\n";
1883#endif
1884 report_fatal_error("Do not know how to expand this operator's operand!");
1885
1886 case ISD::BITCAST: Res = ExpandOp_BITCAST(N); break;
1887 case ISD::BUILD_VECTOR: Res = ExpandOp_BUILD_VECTOR(N); break;
1888 case ISD::EXTRACT_ELEMENT: Res = ExpandOp_EXTRACT_ELEMENT(N); break;
1889
1890 case ISD::BR_CC: Res = ExpandFloatOp_BR_CC(N); break;
1891 case ISD::FCOPYSIGN: Res = ExpandFloatOp_FCOPYSIGN(N); break;
1893 case ISD::FP_ROUND: Res = ExpandFloatOp_FP_ROUND(N); break;
1896 case ISD::FP_TO_SINT:
1897 case ISD::FP_TO_UINT: Res = ExpandFloatOp_FP_TO_XINT(N); break;
1898 case ISD::LROUND: Res = ExpandFloatOp_LROUND(N); break;
1899 case ISD::LLROUND: Res = ExpandFloatOp_LLROUND(N); break;
1900 case ISD::LRINT: Res = ExpandFloatOp_LRINT(N); break;
1901 case ISD::LLRINT: Res = ExpandFloatOp_LLRINT(N); break;
1902 case ISD::SELECT_CC: Res = ExpandFloatOp_SELECT_CC(N); break;
1903 case ISD::STRICT_FSETCC:
1905 case ISD::SETCC: Res = ExpandFloatOp_SETCC(N); break;
1906 case ISD::STORE: Res = ExpandFloatOp_STORE(cast<StoreSDNode>(N),
1907 OpNo); break;
1908 }
1909
1910 // If the result is null, the sub-method took care of registering results etc.
1911 if (!Res.getNode()) return false;
1912
1913 // If the result is N, the sub-method updated N in place. Tell the legalizer
1914 // core about this.
1915 if (Res.getNode() == N)
1916 return true;
1917
1918 assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 &&
1919 "Invalid operand expansion");
1920
1921 ReplaceValueWith(SDValue(N, 0), Res);
1922 return false;
1923}
1924
1925/// FloatExpandSetCCOperands - Expand the operands of a comparison. This code
1926/// is shared among BR_CC, SELECT_CC, and SETCC handlers.
1927void DAGTypeLegalizer::FloatExpandSetCCOperands(SDValue &NewLHS,
1928 SDValue &NewRHS,
1929 ISD::CondCode &CCCode,
1930 const SDLoc &dl, SDValue &Chain,
1931 bool IsSignaling) {
1932 SDValue LHSLo, LHSHi, RHSLo, RHSHi;
1933 GetExpandedFloat(NewLHS, LHSLo, LHSHi);
1934 GetExpandedFloat(NewRHS, RHSLo, RHSHi);
1935
1936 assert(NewLHS.getValueType() == MVT::ppcf128 && "Unsupported setcc type!");
1937
1938 // FIXME: This generated code sucks. We want to generate
1939 // FCMPU crN, hi1, hi2
1940 // BNE crN, L:
1941 // FCMPU crN, lo1, lo2
1942 // The following can be improved, but not that much.
1943 SDValue Tmp1, Tmp2, Tmp3, OutputChain;
1944 Tmp1 = DAG.getSetCC(dl, getSetCCResultType(LHSHi.getValueType()), LHSHi,
1945 RHSHi, ISD::SETOEQ, Chain, IsSignaling);
1946 OutputChain = Tmp1->getNumValues() > 1 ? Tmp1.getValue(1) : SDValue();
1947 Tmp2 = DAG.getSetCC(dl, getSetCCResultType(LHSLo.getValueType()), LHSLo,
1948 RHSLo, CCCode, OutputChain, IsSignaling);
1949 OutputChain = Tmp2->getNumValues() > 1 ? Tmp2.getValue(1) : SDValue();
1950 Tmp3 = DAG.getNode(ISD::AND, dl, Tmp1.getValueType(), Tmp1, Tmp2);
1951 Tmp1 =
1952 DAG.getSetCC(dl, getSetCCResultType(LHSHi.getValueType()), LHSHi, RHSHi,
1953 ISD::SETUNE, OutputChain, IsSignaling);
1954 OutputChain = Tmp1->getNumValues() > 1 ? Tmp1.getValue(1) : SDValue();
1955 Tmp2 = DAG.getSetCC(dl, getSetCCResultType(LHSHi.getValueType()), LHSHi,
1956 RHSHi, CCCode, OutputChain, IsSignaling);
1957 OutputChain = Tmp2->getNumValues() > 1 ? Tmp2.getValue(1) : SDValue();
1958 Tmp1 = DAG.getNode(ISD::AND, dl, Tmp1.getValueType(), Tmp1, Tmp2);
1959 NewLHS = DAG.getNode(ISD::OR, dl, Tmp1.getValueType(), Tmp1, Tmp3);
1960 NewRHS = SDValue(); // LHS is the result, not a compare.
1961 Chain = OutputChain;
1962}
1963
1964SDValue DAGTypeLegalizer::ExpandFloatOp_BR_CC(SDNode *N) {
1965 SDValue NewLHS = N->getOperand(2), NewRHS = N->getOperand(3);
1966 ISD::CondCode CCCode = cast<CondCodeSDNode>(N->getOperand(1))->get();
1967 SDValue Chain;
1968 FloatExpandSetCCOperands(NewLHS, NewRHS, CCCode, SDLoc(N), Chain);
1969
1970 // If ExpandSetCCOperands returned a scalar, we need to compare the result
1971 // against zero to select between true and false values.
1972 if (!NewRHS.getNode()) {
1973 NewRHS = DAG.getConstant(0, SDLoc(N), NewLHS.getValueType());
1974 CCCode = ISD::SETNE;
1975 }
1976
1977 // Update N to have the operands specified.
1978 return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0),
1979 DAG.getCondCode(CCCode), NewLHS, NewRHS,
1980 N->getOperand(4)), 0);
1981}
1982
1983SDValue DAGTypeLegalizer::ExpandFloatOp_FCOPYSIGN(SDNode *N) {
1984 assert(N->getOperand(1).getValueType() == MVT::ppcf128 &&
1985 "Logic only correct for ppcf128!");
1986 SDValue Lo, Hi;
1987 GetExpandedFloat(N->getOperand(1), Lo, Hi);
1988 // The ppcf128 value is providing only the sign; take it from the
1989 // higher-order double (which must have the larger magnitude).
1990 return DAG.getNode(ISD::FCOPYSIGN, SDLoc(N),
1991 N->getValueType(0), N->getOperand(0), Hi);
1992}
1993
1994SDValue DAGTypeLegalizer::ExpandFloatOp_FP_ROUND(SDNode *N) {
1995 bool IsStrict = N->isStrictFPOpcode();
1996 assert(N->getOperand(IsStrict ? 1 : 0).getValueType() == MVT::ppcf128 &&
1997 "Logic only correct for ppcf128!");
1998 SDValue Lo, Hi;
1999 GetExpandedFloat(N->getOperand(IsStrict ? 1 : 0), Lo, Hi);
2000
2001 if (!IsStrict)
2002 // Round it the rest of the way (e.g. to f32) if needed.
2003 return DAG.getNode(ISD::FP_ROUND, SDLoc(N),
2004 N->getValueType(0), Hi, N->getOperand(1));
2005
2006 // Eliminate the node if the input float type is the same as the output float
2007 // type.
2008 if (Hi.getValueType() == N->getValueType(0)) {
2009 // Connect the output chain to the input chain, unlinking the node.
2010 ReplaceValueWith(SDValue(N, 1), N->getOperand(0));
2011 ReplaceValueWith(SDValue(N, 0), Hi);
2012 return SDValue();
2013 }
2014
2015 SDValue Expansion = DAG.getNode(ISD::STRICT_FP_ROUND, SDLoc(N),
2016 {N->getValueType(0), MVT::Other},
2017 {N->getOperand(0), Hi, N->getOperand(2)});
2018 ReplaceValueWith(SDValue(N, 1), Expansion.getValue(1));
2019 ReplaceValueWith(SDValue(N, 0), Expansion);
2020 return SDValue();
2021}
2022
2023SDValue DAGTypeLegalizer::ExpandFloatOp_FP_TO_XINT(SDNode *N) {
2024 EVT RVT = N->getValueType(0);
2025 SDLoc dl(N);
2026
2027 bool IsStrict = N->isStrictFPOpcode();
2028 bool Signed = N->getOpcode() == ISD::FP_TO_SINT ||
2029 N->getOpcode() == ISD::STRICT_FP_TO_SINT;
2030 SDValue Op = N->getOperand(IsStrict ? 1 : 0);
2031 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
2032
2033 EVT NVT;
2034 RTLIB::Libcall LC = findFPToIntLibcall(Op.getValueType(), RVT, NVT, Signed);
2035 assert(LC != RTLIB::UNKNOWN_LIBCALL && NVT.isSimple() &&
2036 "Unsupported FP_TO_XINT!");
2038 std::pair<SDValue, SDValue> Tmp =
2039 TLI.makeLibCall(DAG, LC, NVT, Op, CallOptions, dl, Chain);
2040 if (!IsStrict)
2041 return Tmp.first;
2042
2043 ReplaceValueWith(SDValue(N, 1), Tmp.second);
2044 ReplaceValueWith(SDValue(N, 0), Tmp.first);
2045 return SDValue();
2046}
2047
2048SDValue DAGTypeLegalizer::ExpandFloatOp_SELECT_CC(SDNode *N) {
2049 SDValue NewLHS = N->getOperand(0), NewRHS = N->getOperand(1);
2050 ISD::CondCode CCCode = cast<CondCodeSDNode>(N->getOperand(4))->get();
2051 SDValue Chain;
2052 FloatExpandSetCCOperands(NewLHS, NewRHS, CCCode, SDLoc(N), Chain);
2053
2054 // If ExpandSetCCOperands returned a scalar, we need to compare the result
2055 // against zero to select between true and false values.
2056 if (!NewRHS.getNode()) {
2057 NewRHS = DAG.getConstant(0, SDLoc(N), NewLHS.getValueType());
2058 CCCode = ISD::SETNE;
2059 }
2060
2061 // Update N to have the operands specified.
2062 return SDValue(DAG.UpdateNodeOperands(N, NewLHS, NewRHS,
2063 N->getOperand(2), N->getOperand(3),
2064 DAG.getCondCode(CCCode)), 0);
2065}
2066
2067SDValue DAGTypeLegalizer::ExpandFloatOp_SETCC(SDNode *N) {
2068 bool IsStrict = N->isStrictFPOpcode();
2069 SDValue NewLHS = N->getOperand(IsStrict ? 1 : 0);
2070 SDValue NewRHS = N->getOperand(IsStrict ? 2 : 1);
2071 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
2072 ISD::CondCode CCCode =
2073 cast<CondCodeSDNode>(N->getOperand(IsStrict ? 3 : 2))->get();
2074 FloatExpandSetCCOperands(NewLHS, NewRHS, CCCode, SDLoc(N), Chain,
2075 N->getOpcode() == ISD::STRICT_FSETCCS);
2076
2077 // FloatExpandSetCCOperands always returned a scalar.
2078 assert(!NewRHS.getNode() && "Expect to return scalar");
2079 assert(NewLHS.getValueType() == N->getValueType(0) &&
2080 "Unexpected setcc expansion!");
2081 if (Chain) {
2082 ReplaceValueWith(SDValue(N, 0), NewLHS);
2083 ReplaceValueWith(SDValue(N, 1), Chain);
2084 return SDValue();
2085 }
2086 return NewLHS;
2087}
2088
2089SDValue DAGTypeLegalizer::ExpandFloatOp_STORE(SDNode *N, unsigned OpNo) {
2090 if (ISD::isNormalStore(N))
2091 return ExpandOp_NormalStore(N, OpNo);
2092
2093 assert(ISD::isUNINDEXEDStore(N) && "Indexed store during type legalization!");
2094 assert(OpNo == 1 && "Can only expand the stored value so far");
2095 StoreSDNode *ST = cast<StoreSDNode>(N);
2096
2097 SDValue Chain = ST->getChain();
2098 SDValue Ptr = ST->getBasePtr();
2099
2100 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(),
2101 ST->getValue().getValueType());
2102 assert(NVT.isByteSized() && "Expanded type not byte sized!");
2103 assert(ST->getMemoryVT().bitsLE(NVT) && "Float type not round?");
2104 (void)NVT;
2105
2106 SDValue Lo, Hi;
2107 GetExpandedOp(ST->getValue(), Lo, Hi);
2108
2109 return DAG.getTruncStore(Chain, SDLoc(N), Hi, Ptr,
2110 ST->getMemoryVT(), ST->getMemOperand());
2111}
2112
2113SDValue DAGTypeLegalizer::ExpandFloatOp_LROUND(SDNode *N) {
2114 EVT RVT = N->getValueType(0);
2115 EVT RetVT = N->getOperand(0).getValueType();
2117 return TLI.makeLibCall(DAG, GetFPLibCall(RetVT,
2118 RTLIB::LROUND_F32,
2119 RTLIB::LROUND_F64,
2120 RTLIB::LROUND_F80,
2121 RTLIB::LROUND_F128,
2122 RTLIB::LROUND_PPCF128),
2123 RVT, N->getOperand(0), CallOptions, SDLoc(N)).first;
2124}
2125
2126SDValue DAGTypeLegalizer::ExpandFloatOp_LLROUND(SDNode *N) {
2127 EVT RVT = N->getValueType(0);
2128 EVT RetVT = N->getOperand(0).getValueType();
2130 return TLI.makeLibCall(DAG, GetFPLibCall(RetVT,
2131 RTLIB::LLROUND_F32,
2132 RTLIB::LLROUND_F64,
2133 RTLIB::LLROUND_F80,
2134 RTLIB::LLROUND_F128,
2135 RTLIB::LLROUND_PPCF128),
2136 RVT, N->getOperand(0), CallOptions, SDLoc(N)).first;
2137}
2138
2139SDValue DAGTypeLegalizer::ExpandFloatOp_LRINT(SDNode *N) {
2140 EVT RVT = N->getValueType(0);
2141 EVT RetVT = N->getOperand(0).getValueType();
2143 return TLI.makeLibCall(DAG, GetFPLibCall(RetVT,
2144 RTLIB::LRINT_F32,
2145 RTLIB::LRINT_F64,
2146 RTLIB::LRINT_F80,
2147 RTLIB::LRINT_F128,
2148 RTLIB::LRINT_PPCF128),
2149 RVT, N->getOperand(0), CallOptions, SDLoc(N)).first;
2150}
2151
2152SDValue DAGTypeLegalizer::ExpandFloatOp_LLRINT(SDNode *N) {
2153 EVT RVT = N->getValueType(0);
2154 EVT RetVT = N->getOperand(0).getValueType();
2156 return TLI.makeLibCall(DAG, GetFPLibCall(RetVT,
2157 RTLIB::LLRINT_F32,
2158 RTLIB::LLRINT_F64,
2159 RTLIB::LLRINT_F80,
2160 RTLIB::LLRINT_F128,
2161 RTLIB::LLRINT_PPCF128),
2162 RVT, N->getOperand(0), CallOptions, SDLoc(N)).first;
2163}
2164
2165//===----------------------------------------------------------------------===//
2166// Float Operand Promotion
2167//===----------------------------------------------------------------------===//
2168//
2169
2171 if (OpVT == MVT::f16) {
2172 return ISD::FP16_TO_FP;
2173 } else if (RetVT == MVT::f16) {
2174 return ISD::FP_TO_FP16;
2175 } else if (OpVT == MVT::bf16) {
2176 return ISD::BF16_TO_FP;
2177 } else if (RetVT == MVT::bf16) {
2178 return ISD::FP_TO_BF16;
2179 }
2180
2181 report_fatal_error("Attempt at an invalid promotion-related conversion");
2182}
2183
2184bool DAGTypeLegalizer::PromoteFloatOperand(SDNode *N, unsigned OpNo) {
2185 LLVM_DEBUG(dbgs() << "Promote float operand " << OpNo << ": "; N->dump(&DAG));
2186 SDValue R = SDValue();
2187
2188 if (CustomLowerNode(N, N->getOperand(OpNo).getValueType(), false)) {
2189 LLVM_DEBUG(dbgs() << "Node has been custom lowered, done\n");
2190 return false;
2191 }
2192
2193 // Nodes that use a promotion-requiring floating point operand, but doesn't
2194 // produce a promotion-requiring floating point result, need to be legalized
2195 // to use the promoted float operand. Nodes that produce at least one
2196 // promotion-requiring floating point result have their operands legalized as
2197 // a part of PromoteFloatResult.
2198 // clang-format off
2199 switch (N->getOpcode()) {
2200 default:
2201 #ifndef NDEBUG
2202 dbgs() << "PromoteFloatOperand Op #" << OpNo << ": ";
2203 N->dump(&DAG); dbgs() << "\n";
2204 #endif
2205 report_fatal_error("Do not know how to promote this operator's operand!");
2206
2207 case ISD::BITCAST: R = PromoteFloatOp_BITCAST(N, OpNo); break;
2208 case ISD::FCOPYSIGN: R = PromoteFloatOp_FCOPYSIGN(N, OpNo); break;
2209 case ISD::FP_TO_SINT:
2210 case ISD::FP_TO_UINT:
2211 case ISD::LRINT:
2212 case ISD::LLRINT: R = PromoteFloatOp_UnaryOp(N, OpNo); break;
2215 R = PromoteFloatOp_FP_TO_XINT_SAT(N, OpNo); break;
2216 case ISD::FP_EXTEND: R = PromoteFloatOp_FP_EXTEND(N, OpNo); break;
2217 case ISD::SELECT_CC: R = PromoteFloatOp_SELECT_CC(N, OpNo); break;
2218 case ISD::SETCC: R = PromoteFloatOp_SETCC(N, OpNo); break;
2219 case ISD::STORE: R = PromoteFloatOp_STORE(N, OpNo); break;
2220 }
2221 // clang-format on
2222
2223 if (R.getNode())
2224 ReplaceValueWith(SDValue(N, 0), R);
2225 return false;
2226}
2227
2228SDValue DAGTypeLegalizer::PromoteFloatOp_BITCAST(SDNode *N, unsigned OpNo) {
2229 SDValue Op = N->getOperand(0);
2230 EVT OpVT = Op->getValueType(0);
2231
2232 SDValue Promoted = GetPromotedFloat(N->getOperand(0));
2233 EVT PromotedVT = Promoted->getValueType(0);
2234
2235 // Convert the promoted float value to the desired IVT.
2236 EVT IVT = EVT::getIntegerVT(*DAG.getContext(), OpVT.getSizeInBits());
2237 SDValue Convert = DAG.getNode(GetPromotionOpcode(PromotedVT, OpVT), SDLoc(N),
2238 IVT, Promoted);
2239 // The final result type might not be an scalar so we need a bitcast. The
2240 // bitcast will be further legalized if needed.
2241 return DAG.getBitcast(N->getValueType(0), Convert);
2242}
2243
2244// Promote Operand 1 of FCOPYSIGN. Operand 0 ought to be handled by
2245// PromoteFloatRes_FCOPYSIGN.
2246SDValue DAGTypeLegalizer::PromoteFloatOp_FCOPYSIGN(SDNode *N, unsigned OpNo) {
2247 assert (OpNo == 1 && "Only Operand 1 must need promotion here");
2248 SDValue Op1 = GetPromotedFloat(N->getOperand(1));
2249
2250 return DAG.getNode(N->getOpcode(), SDLoc(N), N->getValueType(0),
2251 N->getOperand(0), Op1);
2252}
2253
2254// Convert the promoted float value to the desired integer type
2255SDValue DAGTypeLegalizer::PromoteFloatOp_UnaryOp(SDNode *N, unsigned OpNo) {
2256 SDValue Op = GetPromotedFloat(N->getOperand(0));
2257 return DAG.getNode(N->getOpcode(), SDLoc(N), N->getValueType(0), Op);
2258}
2259
2260SDValue DAGTypeLegalizer::PromoteFloatOp_FP_TO_XINT_SAT(SDNode *N,
2261 unsigned OpNo) {
2262 SDValue Op = GetPromotedFloat(N->getOperand(0));
2263 return DAG.getNode(N->getOpcode(), SDLoc(N), N->getValueType(0), Op,
2264 N->getOperand(1));
2265}
2266
2267SDValue DAGTypeLegalizer::PromoteFloatOp_FP_EXTEND(SDNode *N, unsigned OpNo) {
2268 SDValue Op = GetPromotedFloat(N->getOperand(0));
2269 EVT VT = N->getValueType(0);
2270
2271 // Desired VT is same as promoted type. Use promoted float directly.
2272 if (VT == Op->getValueType(0))
2273 return Op;
2274
2275 // Else, extend the promoted float value to the desired VT.
2276 return DAG.getNode(ISD::FP_EXTEND, SDLoc(N), VT, Op);
2277}
2278
2279// Promote the float operands used for comparison. The true- and false-
2280// operands have the same type as the result and are promoted, if needed, by
2281// PromoteFloatRes_SELECT_CC
2282SDValue DAGTypeLegalizer::PromoteFloatOp_SELECT_CC(SDNode *N, unsigned OpNo) {
2283 SDValue LHS = GetPromotedFloat(N->getOperand(0));
2284 SDValue RHS = GetPromotedFloat(N->getOperand(1));
2285
2286 return DAG.getNode(ISD::SELECT_CC, SDLoc(N), N->getValueType(0),
2287 LHS, RHS, N->getOperand(2), N->getOperand(3),
2288 N->getOperand(4));
2289}
2290
2291// Construct a SETCC that compares the promoted values and sets the conditional
2292// code.
2293SDValue DAGTypeLegalizer::PromoteFloatOp_SETCC(SDNode *N, unsigned OpNo) {
2294 EVT VT = N->getValueType(0);
2295 SDValue Op0 = GetPromotedFloat(N->getOperand(0));
2296 SDValue Op1 = GetPromotedFloat(N->getOperand(1));
2297 ISD::CondCode CCCode = cast<CondCodeSDNode>(N->getOperand(2))->get();
2298
2299 return DAG.getSetCC(SDLoc(N), VT, Op0, Op1, CCCode);
2300
2301}
2302
2303// Lower the promoted Float down to the integer value of same size and construct
2304// a STORE of the integer value.
2305SDValue DAGTypeLegalizer::PromoteFloatOp_STORE(SDNode *N, unsigned OpNo) {
2306 StoreSDNode *ST = cast<StoreSDNode>(N);
2307 SDValue Val = ST->getValue();
2308 SDLoc DL(N);
2309
2310 SDValue Promoted = GetPromotedFloat(Val);
2311 EVT VT = ST->getOperand(1).getValueType();
2312 EVT IVT = EVT::getIntegerVT(*DAG.getContext(), VT.getSizeInBits());
2313
2314 SDValue NewVal;
2315 NewVal = DAG.getNode(GetPromotionOpcode(Promoted.getValueType(), VT), DL,
2316 IVT, Promoted);
2317
2318 return DAG.getStore(ST->getChain(), DL, NewVal, ST->getBasePtr(),
2319 ST->getMemOperand());
2320}
2321
2322//===----------------------------------------------------------------------===//
2323// Float Result Promotion
2324//===----------------------------------------------------------------------===//
2325
2326void DAGTypeLegalizer::PromoteFloatResult(SDNode *N, unsigned ResNo) {
2327 LLVM_DEBUG(dbgs() << "Promote float result " << ResNo << ": "; N->dump(&DAG));
2328 SDValue R = SDValue();
2329
2330 // See if the target wants to custom expand this node.
2331 if (CustomLowerNode(N, N->getValueType(ResNo), true)) {
2332 LLVM_DEBUG(dbgs() << "Node has been custom expanded, done\n");
2333 return;
2334 }
2335
2336 switch (N->getOpcode()) {
2337 // These opcodes cannot appear if promotion of FP16 is done in the backend
2338 // instead of Clang
2339 case ISD::FP16_TO_FP:
2340 case ISD::FP_TO_FP16:
2341 default:
2342#ifndef NDEBUG
2343 dbgs() << "PromoteFloatResult #" << ResNo << ": ";
2344 N->dump(&DAG); dbgs() << "\n";
2345#endif
2346 report_fatal_error("Do not know how to promote this operator's result!");
2347
2348 case ISD::BITCAST: R = PromoteFloatRes_BITCAST(N); break;
2349 case ISD::ConstantFP: R = PromoteFloatRes_ConstantFP(N); break;
2351 R = PromoteFloatRes_EXTRACT_VECTOR_ELT(N); break;
2352 case ISD::FCOPYSIGN: R = PromoteFloatRes_FCOPYSIGN(N); break;
2353
2354 // Unary FP Operations
2355 case ISD::FABS:
2356 case ISD::FCBRT:
2357 case ISD::FCEIL:
2358 case ISD::FCOS:
2359 case ISD::FEXP:
2360 case ISD::FEXP2:
2361 case ISD::FEXP10:
2362 case ISD::FFLOOR:
2363 case ISD::FLOG:
2364 case ISD::FLOG2:
2365 case ISD::FLOG10:
2366 case ISD::FNEARBYINT:
2367 case ISD::FNEG:
2368 case ISD::FRINT:
2369 case ISD::FROUND:
2370 case ISD::FROUNDEVEN:
2371 case ISD::FSIN:
2372 case ISD::FSQRT:
2373 case ISD::FTRUNC:
2374 case ISD::FCANONICALIZE: R = PromoteFloatRes_UnaryOp(N); break;
2375
2376 // Binary FP Operations
2377 case ISD::FADD:
2378 case ISD::FDIV:
2379 case ISD::FMAXIMUM:
2380 case ISD::FMINIMUM:
2381 case ISD::FMAXNUM:
2382 case ISD::FMINNUM:
2383 case ISD::FMUL:
2384 case ISD::FPOW:
2385 case ISD::FREM:
2386 case ISD::FSUB: R = PromoteFloatRes_BinOp(N); break;
2387
2388 case ISD::FMA: // FMA is same as FMAD
2389 case ISD::FMAD: R = PromoteFloatRes_FMAD(N); break;
2390
2391 case ISD::FPOWI:
2392 case ISD::FLDEXP: R = PromoteFloatRes_ExpOp(N); break;
2393 case ISD::FFREXP: R = PromoteFloatRes_FFREXP(N); break;
2394
2395 case ISD::FP_ROUND: R = PromoteFloatRes_FP_ROUND(N); break;
2396 case ISD::LOAD: R = PromoteFloatRes_LOAD(N); break;
2397 case ISD::SELECT: R = PromoteFloatRes_SELECT(N); break;
2398 case ISD::SELECT_CC: R = PromoteFloatRes_SELECT_CC(N); break;
2399
2400 case ISD::SINT_TO_FP:
2401 case ISD::UINT_TO_FP: R = PromoteFloatRes_XINT_TO_FP(N); break;
2402 case ISD::UNDEF: R = PromoteFloatRes_UNDEF(N); break;
2403 case ISD::ATOMIC_SWAP: R = BitcastToInt_ATOMIC_SWAP(N); break;
2410 R = PromoteFloatRes_VECREDUCE(N);
2411 break;
2414 R = PromoteFloatRes_VECREDUCE_SEQ(N);
2415 break;
2416 }
2417
2418 if (R.getNode())
2419 SetPromotedFloat(SDValue(N, ResNo), R);
2420}
2421
2422// Bitcast from i16 to f16: convert the i16 to a f32 value instead.
2423// At this point, it is not possible to determine if the bitcast value is
2424// eventually stored to memory or promoted to f32 or promoted to a floating
2425// point at a higher precision. Some of these cases are handled by FP_EXTEND,
2426// STORE promotion handlers.
2427SDValue DAGTypeLegalizer::PromoteFloatRes_BITCAST(SDNode *N) {
2428 EVT VT = N->getValueType(0);
2429 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
2430 // Input type isn't guaranteed to be a scalar int so bitcast if not. The
2431 // bitcast will be legalized further if necessary.
2432 EVT IVT = EVT::getIntegerVT(*DAG.getContext(),
2433 N->getOperand(0).getValueType().getSizeInBits());
2434 SDValue Cast = DAG.getBitcast(IVT, N->getOperand(0));
2435 return DAG.getNode(GetPromotionOpcode(VT, NVT), SDLoc(N), NVT, Cast);
2436}
2437
2438SDValue DAGTypeLegalizer::PromoteFloatRes_ConstantFP(SDNode *N) {
2439 ConstantFPSDNode *CFPNode = cast<ConstantFPSDNode>(N);
2440 EVT VT = N->getValueType(0);
2441 SDLoc DL(N);
2442
2443 // Get the (bit-cast) APInt of the APFloat and build an integer constant
2444 EVT IVT = EVT::getIntegerVT(*DAG.getContext(), VT.getSizeInBits());
2445 SDValue C = DAG.getConstant(CFPNode->getValueAPF().bitcastToAPInt(), DL,
2446 IVT);
2447
2448 // Convert the Constant to the desired FP type
2449 // FIXME We might be able to do the conversion during compilation and get rid
2450 // of it from the object code
2451 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
2452 return DAG.getNode(GetPromotionOpcode(VT, NVT), DL, NVT, C);
2453}
2454
2455// If the Index operand is a constant, try to redirect the extract operation to
2456// the correct legalized vector. If not, bit-convert the input vector to
2457// equivalent integer vector. Extract the element as an (bit-cast) integer
2458// value and convert it to the promoted type.
2459SDValue DAGTypeLegalizer::PromoteFloatRes_EXTRACT_VECTOR_ELT(SDNode *N) {
2460 SDLoc DL(N);
2461
2462 // If the index is constant, try to extract the value from the legalized
2463 // vector type.
2464 if (isa<ConstantSDNode>(N->getOperand(1))) {
2465 SDValue Vec = N->getOperand(0);
2466 SDValue Idx = N->getOperand(1);
2467 EVT VecVT = Vec->getValueType(0);
2468 EVT EltVT = VecVT.getVectorElementType();
2469
2470 uint64_t IdxVal = cast<ConstantSDNode>(Idx)->getZExtValue();
2471
2472 switch (getTypeAction(VecVT)) {
2473 default: break;
2475 SDValue Res = GetScalarizedVector(N->getOperand(0));
2476 ReplaceValueWith(SDValue(N, 0), Res);
2477 return SDValue();
2478 }
2480 Vec = GetWidenedVector(Vec);
2481 SDValue Res = DAG.getNode(N->getOpcode(), DL, EltVT, Vec, Idx);
2482 ReplaceValueWith(SDValue(N, 0), Res);
2483 return SDValue();
2484 }
2486 SDValue Lo, Hi;
2487 GetSplitVector(Vec, Lo, Hi);
2488
2489 uint64_t LoElts = Lo.getValueType().getVectorNumElements();
2490 SDValue Res;
2491 if (IdxVal < LoElts)
2492 Res = DAG.getNode(N->getOpcode(), DL, EltVT, Lo, Idx);
2493 else
2494 Res = DAG.getNode(N->getOpcode(), DL, EltVT, Hi,
2495 DAG.getConstant(IdxVal - LoElts, DL,
2496 Idx.getValueType()));
2497 ReplaceValueWith(SDValue(N, 0), Res);
2498 return SDValue();
2499 }
2500
2501 }
2502 }
2503
2504 // Bit-convert the input vector to the equivalent integer vector
2505 SDValue NewOp = BitConvertVectorToIntegerVector(N->getOperand(0));
2506 EVT IVT = NewOp.getValueType().getVectorElementType();
2507
2508 // Extract the element as an (bit-cast) integer value
2509 SDValue NewVal = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, IVT,
2510 NewOp, N->getOperand(1));
2511
2512 // Convert the element to the desired FP type
2513 EVT VT = N->getValueType(0);
2514 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
2515 return DAG.getNode(GetPromotionOpcode(VT, NVT), SDLoc(N), NVT, NewVal);
2516}
2517
2518// FCOPYSIGN(X, Y) returns the value of X with the sign of Y. If the result
2519// needs promotion, so does the argument X. Note that Y, if needed, will be
2520// handled during operand promotion.
2521SDValue DAGTypeLegalizer::PromoteFloatRes_FCOPYSIGN(SDNode *N) {
2522 EVT VT = N->getValueType(0);
2523 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
2524 SDValue Op0 = GetPromotedFloat(N->getOperand(0));
2525
2526 SDValue Op1 = N->getOperand(1);
2527
2528 return DAG.getNode(N->getOpcode(), SDLoc(N), NVT, Op0, Op1);
2529}
2530
2531// Unary operation where the result and the operand have PromoteFloat type
2532// action. Construct a new SDNode with the promoted float value of the old
2533// operand.
2534SDValue DAGTypeLegalizer::PromoteFloatRes_UnaryOp(SDNode *N) {
2535 EVT VT = N->getValueType(0);
2536 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
2537 SDValue Op = GetPromotedFloat(N->getOperand(0));
2538
2539 return DAG.getNode(N->getOpcode(), SDLoc(N), NVT, Op);
2540}
2541
2542// Binary operations where the result and both operands have PromoteFloat type
2543// action. Construct a new SDNode with the promoted float values of the old
2544// operands.
2545SDValue DAGTypeLegalizer::PromoteFloatRes_BinOp(SDNode *N) {
2546 EVT VT = N->getValueType(0);
2547 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
2548 SDValue Op0 = GetPromotedFloat(N->getOperand(0));
2549 SDValue Op1 = GetPromotedFloat(N->getOperand(1));
2550 return DAG.getNode(N->getOpcode(), SDLoc(N), NVT, Op0, Op1, N->getFlags());
2551}
2552
2553SDValue DAGTypeLegalizer::PromoteFloatRes_FMAD(SDNode *N) {
2554 EVT VT = N->getValueType(0);
2555 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
2556 SDValue Op0 = GetPromotedFloat(N->getOperand(0));
2557 SDValue Op1 = GetPromotedFloat(N->getOperand(1));
2558 SDValue Op2 = GetPromotedFloat(N->getOperand(2));
2559
2560 return DAG.getNode(N->getOpcode(), SDLoc(N), NVT, Op0, Op1, Op2);
2561}
2562
2563// Promote the Float (first) operand and retain the Integer (second) operand
2564SDValue DAGTypeLegalizer::PromoteFloatRes_ExpOp(SDNode *N) {
2565 EVT VT = N->getValueType(0);
2566 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
2567 SDValue Op0 = GetPromotedFloat(N->getOperand(0));
2568 SDValue Op1 = N->getOperand(1);
2569
2570 return DAG.getNode(N->getOpcode(), SDLoc(N), NVT, Op0, Op1);
2571}
2572
2573SDValue DAGTypeLegalizer::PromoteFloatRes_FFREXP(SDNode *N) {
2574 EVT VT = N->getValueType(0);
2575 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
2576 SDValue Op = GetPromotedFloat(N->getOperand(0));
2577 SDValue Res =
2578 DAG.getNode(N->getOpcode(), SDLoc(N), {NVT, N->getValueType(1)}, Op);
2579
2580 ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
2581 return Res;
2582}
2583
2584// Explicit operation to reduce precision. Reduce the value to half precision
2585// and promote it back to the legal type.
2586SDValue DAGTypeLegalizer::PromoteFloatRes_FP_ROUND(SDNode *N) {
2587 SDLoc DL(N);
2588
2589 SDValue Op = N->getOperand(0);
2590 EVT VT = N->getValueType(0);
2591 EVT OpVT = Op->getValueType(0);
2592 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
2593 EVT IVT = EVT::getIntegerVT(*DAG.getContext(), VT.getSizeInBits());
2594
2595 // Round promoted float to desired precision
2596 SDValue Round = DAG.getNode(GetPromotionOpcode(OpVT, VT), DL, IVT, Op);
2597 // Promote it back to the legal output type
2598 return DAG.getNode(GetPromotionOpcode(VT, NVT), DL, NVT, Round);
2599}
2600
2601SDValue DAGTypeLegalizer::PromoteFloatRes_LOAD(SDNode *N) {
2602 LoadSDNode *L = cast<LoadSDNode>(N);
2603 EVT VT = N->getValueType(0);
2604
2605 // Load the value as an integer value with the same number of bits.
2606 EVT IVT = EVT::getIntegerVT(*DAG.getContext(), VT.getSizeInBits());
2607 SDValue newL = DAG.getLoad(
2608 L->getAddressingMode(), L->getExtensionType(), IVT, SDLoc(N),
2609 L->getChain(), L->getBasePtr(), L->getOffset(), L->getPointerInfo(), IVT,
2610 L->getOriginalAlign(), L->getMemOperand()->getFlags(), L->getAAInfo());
2611 // Legalize the chain result by replacing uses of the old value chain with the
2612 // new one
2613 ReplaceValueWith(SDValue(N, 1), newL.getValue(1));
2614
2615 // Convert the integer value to the desired FP type
2616 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
2617 return DAG.getNode(GetPromotionOpcode(VT, NVT), SDLoc(N), NVT, newL);
2618}
2619
2620// Construct a new SELECT node with the promoted true- and false- values.
2621SDValue DAGTypeLegalizer::PromoteFloatRes_SELECT(SDNode *N) {
2622 SDValue TrueVal = GetPromotedFloat(N->getOperand(1));
2623 SDValue FalseVal = GetPromotedFloat(N->getOperand(2));
2624
2625 return DAG.getNode(ISD::SELECT, SDLoc(N), TrueVal->getValueType(0),
2626 N->getOperand(0), TrueVal, FalseVal);
2627}
2628
2629// Construct a new SELECT_CC node with the promoted true- and false- values.
2630// The operands used for comparison are promoted by PromoteFloatOp_SELECT_CC.
2631SDValue DAGTypeLegalizer::PromoteFloatRes_SELECT_CC(SDNode *N) {
2632 SDValue TrueVal = GetPromotedFloat(N->getOperand(2));
2633 SDValue FalseVal = GetPromotedFloat(N->getOperand(3));
2634
2635 return DAG.getNode(ISD::SELECT_CC, SDLoc(N),
2636 TrueVal.getNode()->getValueType(0), N->getOperand(0),
2637 N->getOperand(1), TrueVal, FalseVal, N->getOperand(4));
2638}
2639
2640// Construct a SDNode that transforms the SINT or UINT operand to the promoted
2641// float type.
2642SDValue DAGTypeLegalizer::PromoteFloatRes_XINT_TO_FP(SDNode *N) {
2643 SDLoc DL(N);
2644 EVT VT = N->getValueType(0);
2645 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
2646 SDValue NV = DAG.getNode(N->getOpcode(), DL, NVT, N->getOperand(0));
2647 // Round the value to the desired precision (that of the source type).
2648 return DAG.getNode(
2649 ISD::FP_EXTEND, DL, NVT,
2650 DAG.getNode(ISD::FP_ROUND, DL, VT, NV,
2651 DAG.getIntPtrConstant(0, DL, /*isTarget=*/true)));
2652}
2653
2654SDValue DAGTypeLegalizer::PromoteFloatRes_UNDEF(SDNode *N) {
2655 return DAG.getUNDEF(TLI.getTypeToTransformTo(*DAG.getContext(),
2656 N->getValueType(0)));
2657}
2658
2659SDValue DAGTypeLegalizer::PromoteFloatRes_VECREDUCE(SDNode *N) {
2660 // Expand and promote recursively.
2661 // TODO: This is non-optimal, but dealing with the concurrently happening
2662 // vector-legalization is non-trivial. We could do something similar to
2663 // PromoteFloatRes_EXTRACT_VECTOR_ELT here.
2664 ReplaceValueWith(SDValue(N, 0), TLI.expandVecReduce(N, DAG));
2665 return SDValue();
2666}
2667
2668SDValue DAGTypeLegalizer::PromoteFloatRes_VECREDUCE_SEQ(SDNode *N) {
2669 ReplaceValueWith(SDValue(N, 0), TLI.expandVecReduceSeq(N, DAG));
2670 return SDValue();
2671}
2672
2673SDValue DAGTypeLegalizer::BitcastToInt_ATOMIC_SWAP(SDNode *N) {
2674 EVT VT = N->getValueType(0);
2675
2676 AtomicSDNode *AM = cast<AtomicSDNode>(N);
2677 SDLoc SL(N);
2678
2679 SDValue CastVal = BitConvertToInteger(AM->getVal());
2680 EVT CastVT = CastVal.getValueType();
2681
2682 SDValue NewAtomic
2683 = DAG.getAtomic(ISD::ATOMIC_SWAP, SL, CastVT,
2684 DAG.getVTList(CastVT, MVT::Other),
2685 { AM->getChain(), AM->getBasePtr(), CastVal },
2686 AM->getMemOperand());
2687
2688 SDValue Result = NewAtomic;
2689
2690 if (getTypeAction(VT) == TargetLowering::TypePromoteFloat) {
2691 EVT NFPVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
2692 Result = DAG.getNode(GetPromotionOpcode(VT, NFPVT), SL, NFPVT,
2693 NewAtomic);
2694 }
2695
2696 // Legalize the chain result by replacing uses of the old value chain with the
2697 // new one
2698 ReplaceValueWith(SDValue(N, 1), NewAtomic.getValue(1));
2699
2700 return Result;
2701
2702}
2703
2704//===----------------------------------------------------------------------===//
2705// Half Result Soft Promotion
2706//===----------------------------------------------------------------------===//
2707
2708void DAGTypeLegalizer::SoftPromoteHalfResult(SDNode *N, unsigned ResNo) {
2709 LLVM_DEBUG(dbgs() << "Soft promote half result " << ResNo << ": ";
2710 N->dump(&DAG));
2711 SDValue R = SDValue();
2712
2713 // See if the target wants to custom expand this node.
2714 if (CustomLowerNode(N, N->getValueType(ResNo), true)) {
2715 LLVM_DEBUG(dbgs() << "Node has been custom expanded, done\n");
2716 return;
2717 }
2718
2719 switch (N->getOpcode()) {
2720 default:
2721#ifndef NDEBUG
2722 dbgs() << "SoftPromoteHalfResult #" << ResNo << ": ";
2723 N->dump(&DAG); dbgs() << "\n";
2724#endif
2725 report_fatal_error("Do not know how to soft promote this operator's "
2726 "result!");
2727
2728 case ISD::BITCAST: R = SoftPromoteHalfRes_BITCAST(N); break;
2729 case ISD::ConstantFP: R = SoftPromoteHalfRes_ConstantFP(N); break;
2731 R = SoftPromoteHalfRes_EXTRACT_VECTOR_ELT(N); break;
2732 case ISD::FCOPYSIGN: R = SoftPromoteHalfRes_FCOPYSIGN(N); break;
2734 case ISD::FP_ROUND: R = SoftPromoteHalfRes_FP_ROUND(N); break;
2735
2736 // Unary FP Operations
2737 case ISD::FABS:
2738 case ISD::FCBRT:
2739 case ISD::FCEIL:
2740 case ISD::FCOS:
2741 case ISD::FEXP:
2742 case ISD::FEXP2:
2743 case ISD::FEXP10:
2744 case ISD::FFLOOR:
2745 case ISD::FLOG:
2746 case ISD::FLOG2:
2747 case ISD::FLOG10:
2748 case ISD::FNEARBYINT:
2749 case ISD::FNEG:
2750 case ISD::FREEZE:
2751 case ISD::FRINT:
2752 case ISD::FROUND:
2753 case ISD::FROUNDEVEN:
2754 case ISD::FSIN:
2755 case ISD::FSQRT:
2756 case ISD::FTRUNC:
2757 case ISD::FCANONICALIZE: R = SoftPromoteHalfRes_UnaryOp(N); break;
2758
2759 // Binary FP Operations
2760 case ISD::FADD:
2761 case ISD::FDIV:
2762 case ISD::FMAXIMUM:
2763 case ISD::FMINIMUM:
2764 case ISD::FMAXNUM:
2765 case ISD::FMINNUM:
2766 case ISD::FMUL:
2767 case ISD::FPOW:
2768 case ISD::FREM:
2769 case ISD::FSUB: R = SoftPromoteHalfRes_BinOp(N); break;
2770
2771 case ISD::FMA: // FMA is same as FMAD
2772 case ISD::FMAD: R = SoftPromoteHalfRes_FMAD(N); break;
2773
2774 case ISD::FPOWI:
2775 case ISD::FLDEXP: R = SoftPromoteHalfRes_ExpOp(N); break;
2776
2777 case ISD::LOAD: R = SoftPromoteHalfRes_LOAD(N); break;
2778 case ISD::SELECT: R = SoftPromoteHalfRes_SELECT(N); break;
2779 case ISD::SELECT_CC: R = SoftPromoteHalfRes_SELECT_CC(N); break;
2780 case ISD::SINT_TO_FP:
2781 case ISD::UINT_TO_FP: R = SoftPromoteHalfRes_XINT_TO_FP(N); break;
2782 case ISD::UNDEF: R = SoftPromoteHalfRes_UNDEF(N); break;
2783 case ISD::ATOMIC_SWAP: R = BitcastToInt_ATOMIC_SWAP(N); break;
2790 R = SoftPromoteHalfRes_VECREDUCE(N);
2791 break;
2794 R = SoftPromoteHalfRes_VECREDUCE_SEQ(N);
2795 break;
2796 }
2797
2798 if (R.getNode())
2799 SetSoftPromotedHalf(SDValue(N, ResNo), R);
2800}
2801
2802SDValue DAGTypeLegalizer::SoftPromoteHalfRes_BITCAST(SDNode *N) {
2803 return BitConvertToInteger(N->getOperand(0));
2804}
2805
2806SDValue DAGTypeLegalizer::SoftPromoteHalfRes_ConstantFP(SDNode *N) {
2807 ConstantFPSDNode *CN = cast<ConstantFPSDNode>(N);
2808
2809 // Get the (bit-cast) APInt of the APFloat and build an integer constant
2810 return DAG.getConstant(CN->getValueAPF().bitcastToAPInt(), SDLoc(CN),
2811 MVT::i16);
2812}
2813
2814SDValue DAGTypeLegalizer::SoftPromoteHalfRes_EXTRACT_VECTOR_ELT(SDNode *N) {
2815 SDValue NewOp = BitConvertVectorToIntegerVector(N->getOperand(0));
2817 NewOp.getValueType().getVectorElementType(), NewOp,
2818 N->getOperand(1));
2819}
2820
2821SDValue DAGTypeLegalizer::SoftPromoteHalfRes_FCOPYSIGN(SDNode *N) {
2822 SDValue LHS = GetSoftPromotedHalf(N->getOperand(0));
2823 SDValue RHS = BitConvertToInteger(N->getOperand(1));
2824 SDLoc dl(N);
2825
2826 EVT LVT = LHS.getValueType();
2827 EVT RVT = RHS.getValueType();
2828
2829 unsigned LSize = LVT.getSizeInBits();
2830 unsigned RSize = RVT.getSizeInBits();
2831
2832 // First get the sign bit of second operand.
2833 SDValue SignBit = DAG.getNode(
2834 ISD::SHL, dl, RVT, DAG.getConstant(1, dl, RVT),
2835 DAG.getConstant(RSize - 1, dl,
2836 TLI.getShiftAmountTy(RVT, DAG.getDataLayout())));
2837 SignBit = DAG.getNode(ISD::AND, dl, RVT, RHS, SignBit);
2838
2839 // Shift right or sign-extend it if the two operands have different types.
2840 int SizeDiff = RVT.getSizeInBits() - LVT.getSizeInBits();
2841 if (SizeDiff > 0) {
2842 SignBit =
2843 DAG.getNode(ISD::SRL, dl, RVT, SignBit,
2844 DAG.getConstant(SizeDiff, dl,
2845 TLI.getShiftAmountTy(SignBit.getValueType(),
2846 DAG.getDataLayout())));
2847 SignBit = DAG.getNode(ISD::TRUNCATE, dl, LVT, SignBit);
2848 } else if (SizeDiff < 0) {
2849 SignBit = DAG.getNode(ISD::ANY_EXTEND, dl, LVT, SignBit);
2850 SignBit =
2851 DAG.getNode(ISD::SHL, dl, LVT, SignBit,
2852 DAG.getConstant(-SizeDiff, dl,
2853 TLI.getShiftAmountTy(SignBit.getValueType(),
2854 DAG.getDataLayout())));
2855 }
2856
2857 // Clear the sign bit of the first operand.
2858 SDValue Mask = DAG.getNode(
2859 ISD::SHL, dl, LVT, DAG.getConstant(1, dl, LVT),
2860 DAG.getConstant(LSize - 1, dl,
2861 TLI.getShiftAmountTy(LVT, DAG.getDataLayout())));
2862 Mask = DAG.getNode(ISD::SUB, dl, LVT, Mask, DAG.getConstant(1, dl, LVT));
2863 LHS = DAG.getNode(ISD::AND, dl, LVT, LHS, Mask);
2864
2865 // Or the value with the sign bit.
2866 return DAG.getNode(ISD::OR, dl, LVT, LHS, SignBit);
2867}
2868
2869SDValue DAGTypeLegalizer::SoftPromoteHalfRes_FMAD(SDNode *N) {
2870 EVT OVT = N->getValueType(0);
2871 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), OVT);
2872 SDValue Op0 = GetSoftPromotedHalf(N->getOperand(0));
2873 SDValue Op1 = GetSoftPromotedHalf(N->getOperand(1));
2874 SDValue Op2 = GetSoftPromotedHalf(N->getOperand(2));
2875 SDLoc dl(N);
2876
2877 // Promote to the larger FP type.
2878 auto PromotionOpcode = GetPromotionOpcode(OVT, NVT);
2879 Op0 = DAG.getNode(PromotionOpcode, dl, NVT, Op0);
2880 Op1 = DAG.getNode(PromotionOpcode, dl, NVT, Op1);
2881 Op2 = DAG.getNode(PromotionOpcode, dl, NVT, Op2);
2882
2883 SDValue Res = DAG.getNode(N->getOpcode(), dl, NVT, Op0, Op1, Op2);
2884
2885 // Convert back to FP16 as an integer.
2886 return DAG.getNode(GetPromotionOpcode(NVT, OVT), dl, MVT::i16, Res);
2887}
2888
2889SDValue DAGTypeLegalizer::SoftPromoteHalfRes_ExpOp(SDNode *N) {
2890 EVT OVT = N->getValueType(0);
2891 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), OVT);
2892 SDValue Op0 = GetSoftPromotedHalf(N->getOperand(0));
2893 SDValue Op1 = N->getOperand(1);
2894 SDLoc dl(N);
2895
2896 // Promote to the larger FP type.
2897 Op0 = DAG.getNode(GetPromotionOpcode(OVT, NVT), dl, NVT, Op0);
2898
2899 SDValue Res = DAG.getNode(N->getOpcode(), dl, NVT, Op0, Op1);
2900
2901 // Convert back to FP16 as an integer.
2902 return DAG.getNode(GetPromotionOpcode(NVT, OVT), dl, MVT::i16, Res);
2903}
2904
2905SDValue DAGTypeLegalizer::SoftPromoteHalfRes_FP_ROUND(SDNode *N) {
2906 EVT RVT = N->getValueType(0);
2907 EVT SVT = N->getOperand(0).getValueType();
2908
2909 if (N->isStrictFPOpcode()) {
2910 assert(RVT == MVT::f16);
2911 SDValue Res =
2912 DAG.getNode(ISD::STRICT_FP_TO_FP16, SDLoc(N), {MVT::i16, MVT::Other},
2913 {N->getOperand(0), N->getOperand(1)});
2914 ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
2915 return Res;
2916 }
2917
2918 return DAG.getNode(GetPromotionOpcode(SVT, RVT), SDLoc(N), MVT::i16,
2919 N->getOperand(0));
2920}
2921
2922SDValue DAGTypeLegalizer::SoftPromoteHalfRes_LOAD(SDNode *N) {
2923 LoadSDNode *L = cast<LoadSDNode>(N);
2924
2925 // Load the value as an integer value with the same number of bits.
2926 assert(L->getExtensionType() == ISD::NON_EXTLOAD && "Unexpected extension!");
2927 SDValue NewL =
2928 DAG.getLoad(L->getAddressingMode(), L->getExtensionType(), MVT::i16,
2929 SDLoc(N), L->getChain(), L->getBasePtr(), L->getOffset(),
2930 L->getPointerInfo(), MVT::i16, L->getOriginalAlign(),
2931 L->getMemOperand()->getFlags(), L->getAAInfo());
2932 // Legalize the chain result by replacing uses of the old value chain with the
2933 // new one
2934 ReplaceValueWith(SDValue(N, 1), NewL.getValue(1));
2935 return NewL;
2936}
2937
2938SDValue DAGTypeLegalizer::SoftPromoteHalfRes_SELECT(SDNode *N) {
2939 SDValue Op1 = GetSoftPromotedHalf(N->getOperand(1));
2940 SDValue Op2 = GetSoftPromotedHalf(N->getOperand(2));
2941 return DAG.getSelect(SDLoc(N), Op1.getValueType(), N->getOperand(0), Op1,
2942 Op2);
2943}
2944
2945SDValue DAGTypeLegalizer::SoftPromoteHalfRes_SELECT_CC(SDNode *N) {
2946 SDValue Op2 = GetSoftPromotedHalf(N->getOperand(2));
2947 SDValue Op3 = GetSoftPromotedHalf(N->getOperand(3));
2948 return DAG.getNode(ISD::SELECT_CC, SDLoc(N), Op2.getValueType(),
2949 N->getOperand(0), N->getOperand(1), Op2, Op3,
2950 N->getOperand(4));
2951}
2952
2953SDValue DAGTypeLegalizer::SoftPromoteHalfRes_XINT_TO_FP(SDNode *N) {
2954 EVT OVT = N->getValueType(0);
2955 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), OVT);
2956 SDLoc dl(N);
2957
2958 SDValue Res = DAG.getNode(N->getOpcode(), dl, NVT, N->getOperand(0));
2959
2960 // Round the value to the softened type.
2961 return DAG.getNode(GetPromotionOpcode(NVT, OVT), dl, MVT::i16, Res);
2962}
2963
2964SDValue DAGTypeLegalizer::SoftPromoteHalfRes_UNDEF(SDNode *N) {
2965 return DAG.getUNDEF(MVT::i16);
2966}
2967
2968SDValue DAGTypeLegalizer::SoftPromoteHalfRes_UnaryOp(SDNode *N) {
2969 EVT OVT = N->getValueType(0);
2970 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), OVT);
2971 SDValue Op = GetSoftPromotedHalf(N->getOperand(0));
2972 SDLoc dl(N);
2973
2974 // Promote to the larger FP type.
2975 Op = DAG.getNode(GetPromotionOpcode(OVT, NVT), dl, NVT, Op);
2976
2977 SDValue Res = DAG.getNode(N->getOpcode(), dl, NVT, Op);
2978
2979 // Convert back to FP16 as an integer.
2980 return DAG.getNode(GetPromotionOpcode(NVT, OVT), dl, MVT::i16, Res);
2981}
2982
2983SDValue DAGTypeLegalizer::SoftPromoteHalfRes_BinOp(SDNode *N) {
2984 EVT OVT = N->getValueType(0);
2985 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), OVT);
2986 SDValue Op0 = GetSoftPromotedHalf(N->getOperand(0));
2987 SDValue Op1 = GetSoftPromotedHalf(N->getOperand(1));
2988 SDLoc dl(N);
2989
2990 // Promote to the larger FP type.
2991 auto PromotionOpcode = GetPromotionOpcode(OVT, NVT);
2992 Op0 = DAG.getNode(PromotionOpcode, dl, NVT, Op0);
2993 Op1 = DAG.getNode(PromotionOpcode, dl, NVT, Op1);
2994
2995 SDValue Res = DAG.getNode(N->getOpcode(), dl, NVT, Op0, Op1);
2996
2997 // Convert back to FP16 as an integer.
2998 return DAG.getNode(GetPromotionOpcode(NVT, OVT), dl, MVT::i16, Res);
2999}
3000
3001SDValue DAGTypeLegalizer::SoftPromoteHalfRes_VECREDUCE(SDNode *N) {
3002 // Expand and soften recursively.
3003 ReplaceValueWith(SDValue(N, 0), TLI.expandVecReduce(N, DAG));
3004 return SDValue();
3005}
3006
3007SDValue DAGTypeLegalizer::SoftPromoteHalfRes_VECREDUCE_SEQ(SDNode *N) {
3008 // Expand and soften.
3009 ReplaceValueWith(SDValue(N, 0), TLI.expandVecReduceSeq(N, DAG));
3010 return SDValue();
3011}
3012
3013//===----------------------------------------------------------------------===//
3014// Half Operand Soft Promotion
3015//===----------------------------------------------------------------------===//
3016
3017bool DAGTypeLegalizer::SoftPromoteHalfOperand(SDNode *N, unsigned OpNo) {
3018 LLVM_DEBUG(dbgs() << "Soft promote half operand " << OpNo << ": ";
3019 N->dump(&DAG));
3020 SDValue Res = SDValue();
3021
3022 if (CustomLowerNode(N, N->getOperand(OpNo).getValueType(), false)) {
3023 LLVM_DEBUG(dbgs() << "Node has been custom lowered, done\n");
3024 return false;
3025 }
3026
3027 // Nodes that use a promotion-requiring floating point operand, but doesn't
3028 // produce a soft promotion-requiring floating point result, need to be
3029 // legalized to use the soft promoted float operand. Nodes that produce at
3030 // least one soft promotion-requiring floating point result have their
3031 // operands legalized as a part of PromoteFloatResult.
3032 switch (N->getOpcode()) {
3033 default:
3034 #ifndef NDEBUG
3035 dbgs() << "SoftPromoteHalfOperand Op #" << OpNo << ": ";
3036 N->dump(&DAG); dbgs() << "\n";
3037 #endif
3038 report_fatal_error("Do not know how to soft promote this operator's "
3039 "operand!");
3040
3041 case ISD::BITCAST: Res = SoftPromoteHalfOp_BITCAST(N); break;
3042 case ISD::FCOPYSIGN: Res = SoftPromoteHalfOp_FCOPYSIGN(N, OpNo); break;
3043 case ISD::FP_TO_SINT:
3044 case ISD::FP_TO_UINT: Res = SoftPromoteHalfOp_FP_TO_XINT(N); break;
3047 Res = SoftPromoteHalfOp_FP_TO_XINT_SAT(N); break;
3049 case ISD::FP_EXTEND: Res = SoftPromoteHalfOp_FP_EXTEND(N); break;
3050 case ISD::SELECT_CC: Res = SoftPromoteHalfOp_SELECT_CC(N, OpNo); break;
3051 case ISD::SETCC: Res = SoftPromoteHalfOp_SETCC(N); break;
3052 case ISD::STORE: Res = SoftPromoteHalfOp_STORE(N, OpNo); break;
3053 case ISD::STACKMAP:
3054 Res = SoftPromoteHalfOp_STACKMAP(N, OpNo);
3055 break;
3056 case ISD::PATCHPOINT:
3057 Res = SoftPromoteHalfOp_PATCHPOINT(N, OpNo);
3058 break;
3059 }
3060
3061 if (!Res.getNode())
3062 return false;
3063
3064 assert(Res.getNode() != N && "Expected a new node!");
3065
3066 assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 &&
3067 "Invalid operand expansion");
3068
3069 ReplaceValueWith(SDValue(N, 0), Res);
3070 return false;
3071}
3072
3073SDValue DAGTypeLegalizer::SoftPromoteHalfOp_BITCAST(SDNode *N) {
3074 SDValue Op0 = GetSoftPromotedHalf(N->getOperand(0));
3075
3076 return DAG.getNode(ISD::BITCAST, SDLoc(N), N->getValueType(0), Op0);
3077}
3078
3079SDValue DAGTypeLegalizer::SoftPromoteHalfOp_FCOPYSIGN(SDNode *N,
3080 unsigned OpNo) {
3081 assert(OpNo == 1 && "Only Operand 1 must need promotion here");
3082 SDValue Op1 = N->getOperand(1);
3083 EVT RVT = Op1.getValueType();
3084 SDLoc dl(N);
3085
3086 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), Op1.getValueType());
3087
3088 Op1 = GetSoftPromotedHalf(Op1);
3089 Op1 = DAG.getNode(GetPromotionOpcode(RVT, NVT), dl, NVT, Op1);
3090
3091 return DAG.getNode(N->getOpcode(), dl, N->getValueType(0), N->getOperand(0),
3092 Op1);
3093}
3094
3095SDValue DAGTypeLegalizer::SoftPromoteHalfOp_FP_EXTEND(SDNode *N) {
3096 EVT RVT = N->getValueType(0);
3097 bool IsStrict = N->isStrictFPOpcode();
3098 SDValue Op = N->getOperand(IsStrict ? 1 : 0);
3099 EVT SVT = Op.getValueType();
3100 Op = GetSoftPromotedHalf(N->getOperand(IsStrict ? 1 : 0));
3101
3102 if (IsStrict) {
3103 assert(SVT == MVT::f16);
3104 SDValue Res =
3106 {N->getValueType(0), MVT::Other}, {N->getOperand(0), Op});
3107 ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
3108 ReplaceValueWith(SDValue(N, 0), Res);
3109 return SDValue();
3110 }
3111
3112 return DAG.getNode(GetPromotionOpcode(SVT, RVT), SDLoc(N), RVT, Op);
3113}
3114
3115SDValue DAGTypeLegalizer::SoftPromoteHalfOp_FP_TO_XINT(SDNode *N) {
3116 EVT RVT = N->getValueType(0);
3117 SDValue Op = N->getOperand(0);
3118 EVT SVT = Op.getValueType();
3119 SDLoc dl(N);
3120
3121 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), Op.getValueType());
3122
3123 Op = GetSoftPromotedHalf(Op);
3124
3125 SDValue Res = DAG.getNode(GetPromotionOpcode(SVT, RVT), dl, NVT, Op);
3126
3127 return DAG.getNode(N->getOpcode(), dl, N->getValueType(0), Res);
3128}
3129
3130SDValue DAGTypeLegalizer::SoftPromoteHalfOp_FP_TO_XINT_SAT(SDNode *N) {
3131 EVT RVT = N->getValueType(0);
3132 SDValue Op = N->getOperand(0);
3133 EVT SVT = Op.getValueType();
3134 SDLoc dl(N);
3135
3136 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), Op.getValueType());
3137
3138 Op = GetSoftPromotedHalf(Op);
3139
3140 SDValue Res = DAG.getNode(GetPromotionOpcode(SVT, RVT), dl, NVT, Op);
3141
3142 return DAG.getNode(N->getOpcode(), dl, N->getValueType(0), Res,
3143 N->getOperand(1));
3144}
3145
3146SDValue DAGTypeLegalizer::SoftPromoteHalfOp_SELECT_CC(SDNode *N,
3147 unsigned OpNo) {
3148 assert(OpNo == 0 && "Can only soften the comparison values");
3149 SDValue Op0 = N->getOperand(0);
3150 SDValue Op1 = N->getOperand(1);
3151 SDLoc dl(N);
3152
3153 EVT SVT = Op0.getValueType();
3154 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), SVT);
3155
3156 Op0 = GetSoftPromotedHalf(Op0);
3157 Op1 = GetSoftPromotedHalf(Op1);
3158
3159 // Promote to the larger FP type.
3160 auto PromotionOpcode = GetPromotionOpcode(SVT, NVT);
3161 Op0 = DAG.getNode(PromotionOpcode, dl, NVT, Op0);
3162 Op1 = DAG.getNode(PromotionOpcode, dl, NVT, Op1);
3163
3164 return DAG.getNode(ISD::SELECT_CC, SDLoc(N), N->getValueType(0), Op0, Op1,
3165 N->getOperand(2), N->getOperand(3), N->getOperand(4));
3166}
3167
3168SDValue DAGTypeLegalizer::SoftPromoteHalfOp_SETCC(SDNode *N) {
3169 SDValue Op0 = N->getOperand(0);
3170 SDValue Op1 = N->getOperand(1);
3171 ISD::CondCode CCCode = cast<CondCodeSDNode>(N->getOperand(2))->get();
3172 SDLoc dl(N);
3173
3174 EVT SVT = Op0.getValueType();
3175 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), Op0.getValueType());
3176
3177 Op0 = GetSoftPromotedHalf(Op0);
3178 Op1 = GetSoftPromotedHalf(Op1);
3179
3180 // Promote to the larger FP type.
3181 auto PromotionOpcode = GetPromotionOpcode(SVT, NVT);
3182 Op0 = DAG.getNode(PromotionOpcode, dl, NVT, Op0);
3183 Op1 = DAG.getNode(PromotionOpcode, dl, NVT, Op1);
3184
3185 return DAG.getSetCC(SDLoc(N), N->getValueType(0), Op0, Op1, CCCode);
3186}
3187
3188SDValue DAGTypeLegalizer::SoftPromoteHalfOp_STORE(SDNode *N, unsigned OpNo) {
3189 assert(OpNo == 1 && "Can only soften the stored value!");
3190 StoreSDNode *ST = cast<StoreSDNode>(N);
3191 SDValue Val = ST->getValue();
3192 SDLoc dl(N);
3193
3194 assert(!ST->isTruncatingStore() && "Unexpected truncating store.");
3195 SDValue Promoted = GetSoftPromotedHalf(Val);
3196 return DAG.getStore(ST->getChain(), dl, Promoted, ST->getBasePtr(),
3197 ST->getMemOperand());
3198}
3199
3200SDValue DAGTypeLegalizer::SoftPromoteHalfOp_STACKMAP(SDNode *N, unsigned OpNo) {
3201 assert(OpNo > 1); // Because the first two arguments are guaranteed legal.
3202 SmallVector<SDValue> NewOps(N->ops().begin(), N->ops().end());
3203 SDValue Op = N->getOperand(OpNo);
3204 NewOps[OpNo] = GetSoftPromotedHalf(Op);
3206 DAG.getNode(N->getOpcode(), SDLoc(N), N->getVTList(), NewOps);
3207
3208 for (unsigned ResNum = 0; ResNum < N->getNumValues(); ResNum++)
3209 ReplaceValueWith(SDValue(N, ResNum), NewNode.getValue(ResNum));
3210
3211 return SDValue(); // Signal that we replaced the node ourselves.
3212}
3213
3214SDValue DAGTypeLegalizer::SoftPromoteHalfOp_PATCHPOINT(SDNode *N,
3215 unsigned OpNo) {
3216 assert(OpNo >= 7);
3217 SmallVector<SDValue> NewOps(N->ops().begin(), N->ops().end());
3218 SDValue Op = N->getOperand(OpNo);
3219 NewOps[OpNo] = GetSoftPromotedHalf(Op);
3221 DAG.getNode(N->getOpcode(), SDLoc(N), N->getVTList(), NewOps);
3222
3223 for (unsigned ResNum = 0; ResNum < N->getNumValues(); ResNum++)
3224 ReplaceValueWith(SDValue(N, ResNum), NewNode.getValue(ResNum));
3225
3226 return SDValue(); // Signal that we replaced the node ourselves.
3227}
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
Returns the sub type a function will return at a given Idx Should correspond to the result type of an ExtractValue instruction executed with just that one unsigned Idx
#define LLVM_DEBUG(X)
Definition: Debug.h:101
uint64_t Size
static bool isSigned(unsigned int Opcode)
static RTLIB::Libcall findFPToIntLibcall(EVT SrcVT, EVT RetVT, EVT &Promoted, bool Signed)
static RTLIB::Libcall GetFPLibCall(EVT VT, RTLIB::Libcall Call_F32, RTLIB::Libcall Call_F64, RTLIB::Libcall Call_F80, RTLIB::Libcall Call_F128, RTLIB::Libcall Call_PPCF128)
GetFPLibCall - Return the right libcall for the given floating point type.
static ISD::NodeType GetPromotionOpcode(EVT OpVT, EVT RetVT)
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
Value * RHS
Value * LHS
APInt bitcastToAPInt() const
Definition: APFloat.h:1210
Class for arbitrary precision integers.
Definition: APInt.h:76
static APInt getAllOnes(unsigned numBits)
Return an APInt of a specified width with all bits set.
Definition: APInt.h:212
void clearBit(unsigned BitPosition)
Set a given bit to 0.
Definition: APInt.h:1379
static APInt getSignMask(unsigned BitWidth)
Get the SignMask for a specific bit width.
Definition: APInt.h:207
const uint64_t * getRawData() const
This function returns a pointer to the internal storage of the APInt.
Definition: APInt.h:547
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
Definition: ArrayRef.h:41
This is an SDNode representing atomic operations.
const SDValue & getVal() const
const APFloat & getValueAPF() const
@ NewNode
This is a new node, not before seen, that was created in the process of legalizing some other node.
Definition: LegalizeTypes.h:42
This class represents an Operation in the Expression.
bool isBigEndian() const
Definition: DataLayout.h:239
void emitError(uint64_t LocCookie, const Twine &ErrorStr)
emitError - Emit an error message to the currently installed error handler with optional location inf...
This class is used to represent ISD::LOAD nodes.
SimpleValueType SimpleTy
@ MODereferenceable
The memory access is dereferenceable (i.e., doesn't trap).
@ MOInvariant
The memory access always returns the same value (or traps).
MachineMemOperand * getMemOperand() const
Return a MachineMemOperand object describing the memory reference performed by operation.
Wrapper class for IR location info (IR ordering and DebugLoc) to be passed into SDNode creation funct...
Represents one node in the SelectionDAG.
unsigned getNumValues() const
Return the number of values defined/returned by this operator.
EVT getValueType(unsigned ResNo) const
Return the type of a specified result.
Unlike LLVM values, Selection DAG nodes may return multiple values as the result of a computation.
SDNode * getNode() const
get the SDNode which holds the desired result
SDValue getValue(unsigned R) const
EVT getValueType() const
Return the ValueType of the referenced return value.
SDValue getExtLoad(ISD::LoadExtType ExtType, const SDLoc &dl, EVT VT, SDValue Chain, SDValue Ptr, MachinePointerInfo PtrInfo, EVT MemVT, MaybeAlign Alignment=MaybeAlign(), MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes())
SDValue getSelect(const SDLoc &DL, EVT VT, SDValue Cond, SDValue LHS, SDValue RHS)
Helper function to make it easier to build Select's if you just have operands and don't want to check...
SDVTList getVTList(EVT VT)
Return an SDVTList that represents the list of values specified.
SDValue getSetCC(const SDLoc &DL, EVT VT, SDValue LHS, SDValue RHS, ISD::CondCode Cond, SDValue Chain=SDValue(), bool IsSignaling=false)
Helper function to make it easier to build SetCC's if you just have an ISD::CondCode instead of an SD...
SDValue getConstantFP(double Val, const SDLoc &DL, EVT VT, bool isTarget=false)
Create a ConstantFPSDNode wrapping a constant value.
SDValue getLoad(EVT VT, const SDLoc &dl, SDValue Chain, SDValue Ptr, MachinePointerInfo PtrInfo, MaybeAlign Alignment=MaybeAlign(), MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes(), const MDNode *Ranges=nullptr)
Loads are not normal binary operators: their result type is not determined by their operands,...
SDValue getAtomic(unsigned Opcode, const SDLoc &dl, EVT MemVT, SDValue Chain, SDValue Ptr, SDValue Val, MachineMemOperand *MMO)
Gets a node for an atomic op, produces result (if relevant) and chain and takes 2 operands.
SDValue getUNDEF(EVT VT)
Return an UNDEF node. UNDEF does not have a useful SDLoc.
SDValue getBitcast(EVT VT, SDValue V)
Return a bitcast using the SDLoc of the value operand, and casting to the provided type.
const DataLayout & getDataLayout() const
Definition: SelectionDAG.h:472
SDValue getConstant(uint64_t Val, const SDLoc &DL, EVT VT, bool isTarget=false, bool isOpaque=false)
Create a ConstantSDNode wrapping a constant value.
SDValue getVAArg(EVT VT, const SDLoc &dl, SDValue Chain, SDValue Ptr, SDValue SV, unsigned Align)
VAArg produces a result and token chain, and takes a pointer and a source value as input.
SDValue getTruncStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr, MachinePointerInfo PtrInfo, EVT SVT, Align Alignment, MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes())
SDValue getStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr, MachinePointerInfo PtrInfo, Align Alignment, MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes())
Helper function to build ISD::STORE nodes.
static const fltSemantics & EVTToAPFloatSemantics(EVT VT)
Returns an APFloat semantics tag appropriate for the given type.
SDValue getSelectCC(const SDLoc &DL, SDValue LHS, SDValue RHS, SDValue True, SDValue False, ISD::CondCode Cond)
Helper function to make it easier to build SelectCC's if you just have an ISD::CondCode instead of an...
SDValue getIntPtrConstant(uint64_t Val, const SDLoc &DL, bool isTarget=false)
SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT, ArrayRef< SDUse > Ops)
Gets or creates the specified node.
const TargetLibraryInfo & getLibInfo() const
Definition: SelectionDAG.h:479
MachineFunction & getMachineFunction() const
Definition: SelectionDAG.h:469
SDValue getCondCode(ISD::CondCode Cond)
LLVMContext * getContext() const
Definition: SelectionDAG.h:485
SDValue getShiftAmountConstant(uint64_t Val, EVT VT, const SDLoc &DL, bool LegalTypes=true)
SDValue CreateStackTemporary(TypeSize Bytes, Align Alignment)
Create a stack temporary based on the size in bytes and the alignment.
SDNode * UpdateNodeOperands(SDNode *N, SDValue Op)
Mutate the specified node in-place to have the specified operands.
SDValue getEntryNode() const
Return the token chain corresponding to the entry of the function.
Definition: SelectionDAG.h:554
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
Definition: SmallVector.h:1200
This class is used to represent ISD::STORE nodes.
unsigned getIntSize() const
Get size of a C-level int or unsigned int, in bits.
EVT getShiftAmountTy(EVT LHSTy, const DataLayout &DL, bool LegalTypes=true) const
Returns the type for the shift amount of a shift opcode.
virtual EVT getTypeToTransformTo(LLVMContext &Context, EVT VT) const
For types supported by the target, this is an identity function.
const char * getLibcallName(RTLIB::Libcall Call) const
Get the libcall routine name for the specified libcall.
void softenSetCCOperands(SelectionDAG &DAG, EVT VT, SDValue &NewLHS, SDValue &NewRHS, ISD::CondCode &CCCode, const SDLoc &DL, const SDValue OldLHS, const SDValue OldRHS) const
Soften the operands of a comparison.
std::pair< SDValue, SDValue > makeLibCall(SelectionDAG &DAG, RTLIB::Libcall LC, EVT RetVT, ArrayRef< SDValue > Ops, MakeLibCallOptions CallOptions, const SDLoc &dl, SDValue Chain=SDValue()) const
Returns a pair of (return value, chain).
SDValue expandVecReduceSeq(SDNode *Node, SelectionDAG &DAG) const
Expand a VECREDUCE_SEQ_* into an explicit ordered calculation.
SDValue expandFP_TO_INT_SAT(SDNode *N, SelectionDAG &DAG) const
Expand FP_TO_[US]INT_SAT into FP_TO_[US]INT and selects or min/max.
SDValue expandVecReduce(SDNode *Node, SelectionDAG &DAG) const
Expand a VECREDUCE_* into an explicit calculation.
SDValue createSelectForFMINNUM_FMAXNUM(SDNode *Node, SelectionDAG &DAG) const
Try to convert the fminnum/fmaxnum to a compare/select sequence.
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
constexpr std::underlying_type_t< E > Mask()
Get a bitmask with 1s in all places up to the high-order bit of E's largest value.
Definition: BitmaskEnum.h:121
@ C
The default llvm calling convention, compatible with C.
Definition: CallingConv.h:34
NodeType
ISD::NodeType enum - This enum defines the target-independent operators for a SelectionDAG.
Definition: ISDOpcodes.h:40
@ SETCC
SetCC operator - This evaluates to a true value iff the condition is true.
Definition: ISDOpcodes.h:750
@ MERGE_VALUES
MERGE_VALUES - This node takes multiple discrete operands and returns them all as its individual resu...
Definition: ISDOpcodes.h:236
@ STRICT_FSETCC
STRICT_FSETCC/STRICT_FSETCCS - Constrained versions of SETCC, used for floating-point operands only.
Definition: ISDOpcodes.h:476
@ VECREDUCE_SEQ_FADD
Generic reduction nodes.
Definition: ISDOpcodes.h:1331
@ ConstantFP
Definition: ISDOpcodes.h:77
@ STRICT_FCEIL
Definition: ISDOpcodes.h:426
@ FMAD
FMAD - Perform a * b + c, while getting the same result as the separately rounded operations.
Definition: ISDOpcodes.h:487
@ LOAD
LOAD and STORE have token chains as their first operand, then the same operands as an LLVM load/store...
Definition: ISDOpcodes.h:1029
@ ANY_EXTEND
ANY_EXTEND - Used for integer types. The high bits are undefined.
Definition: ISDOpcodes.h:783
@ FMA
FMA - Perform a * b + c with no intermediate rounding step.
Definition: ISDOpcodes.h:483
@ SINT_TO_FP
[SU]INT_TO_FP - These operators convert integers (whose interpreted sign depends on the first letter)...
Definition: ISDOpcodes.h:790
@ VECREDUCE_FMAX
FMIN/FMAX nodes can have flags, for NaN/NoNaN variants.
Definition: ISDOpcodes.h:1347
@ FADD
Simple binary floating point operators.
Definition: ISDOpcodes.h:390
@ VECREDUCE_FMAXIMUM
FMINIMUM/FMAXIMUM nodes propatate NaNs and signed zeroes using the llvm.minimum and llvm....
Definition: ISDOpcodes.h:1351
@ STRICT_FSETCCS
Definition: ISDOpcodes.h:477
@ FP16_TO_FP
FP16_TO_FP, FP_TO_FP16 - These operators are used to perform promotions and truncation for half-preci...
Definition: ISDOpcodes.h:913
@ STRICT_FLOG2
Definition: ISDOpcodes.h:421
@ BITCAST
BITCAST - This operator converts between integer, vector and FP values, as if the value was stored to...
Definition: ISDOpcodes.h:903
@ BUILD_PAIR
BUILD_PAIR - This is the opposite of EXTRACT_ELEMENT in some ways.
Definition: ISDOpcodes.h:229
@ FLDEXP
FLDEXP - ldexp, inspired by libm (op0 * 2**op1).
Definition: ISDOpcodes.h:937
@ STRICT_FSQRT
Constrained versions of libm-equivalent floating point intrinsics.
Definition: ISDOpcodes.h:411
@ SIGN_EXTEND
Conversion operators.
Definition: ISDOpcodes.h:774
@ STRICT_UINT_TO_FP
Definition: ISDOpcodes.h:450
@ VECREDUCE_FADD
These reductions have relaxed evaluation order semantics, and have a single vector operand.
Definition: ISDOpcodes.h:1344
@ VECREDUCE_FMIN
Definition: ISDOpcodes.h:1348
@ STRICT_LROUND
Definition: ISDOpcodes.h:431
@ FNEG
Perform various unary floating-point operations inspired by libm.
Definition: ISDOpcodes.h:928
@ BR_CC
BR_CC - Conditional branch.
Definition: ISDOpcodes.h:1075
@ FCANONICALIZE
Returns platform specific canonical encoding of a floating point number.
Definition: ISDOpcodes.h:500
@ SELECT
Select(COND, TRUEVAL, FALSEVAL).
Definition: ISDOpcodes.h:727
@ STRICT_FPOWI
Definition: ISDOpcodes.h:413
@ UNDEF
UNDEF - An undefined node.
Definition: ISDOpcodes.h:211
@ EXTRACT_ELEMENT
EXTRACT_ELEMENT - This is used to get the lower or upper (determined by a Constant,...
Definition: ISDOpcodes.h:222
@ STRICT_FTRUNC
Definition: ISDOpcodes.h:430
@ ARITH_FENCE
ARITH_FENCE - This corresponds to a arithmetic fence intrinsic.
Definition: ISDOpcodes.h:1217
@ STRICT_FP_TO_FP16
Definition: ISDOpcodes.h:916
@ STRICT_FP16_TO_FP
Definition: ISDOpcodes.h:915
@ SHL
Shift and rotation operations.
Definition: ISDOpcodes.h:705
@ STRICT_FMAXNUM
Definition: ISDOpcodes.h:424
@ EXTRACT_VECTOR_ELT
EXTRACT_VECTOR_ELT(VECTOR, IDX) - Returns a single element from VECTOR identified by the (potentially...
Definition: ISDOpcodes.h:535
@ ZERO_EXTEND
ZERO_EXTEND - Used for integer types, zeroing the new bits.
Definition: ISDOpcodes.h:780
@ FP_TO_UINT_SAT
Definition: ISDOpcodes.h:856
@ STRICT_FMINNUM
Definition: ISDOpcodes.h:425
@ SELECT_CC
Select with condition operator - This selects between a true value and a false value (ops #2 and #3) ...
Definition: ISDOpcodes.h:742
@ FMINNUM
FMINNUM/FMAXNUM - Perform floating-point minimum or maximum on two values.
Definition: ISDOpcodes.h:969
@ STRICT_LRINT
Definition: ISDOpcodes.h:433
@ FP_EXTEND
X = FP_EXTEND(Y) - Extend a smaller FP type into a larger FP type.
Definition: ISDOpcodes.h:888
@ STRICT_FROUND
Definition: ISDOpcodes.h:428
@ STRICT_SINT_TO_FP
STRICT_[US]INT_TO_FP - Convert a signed or unsigned integer to a floating point value.
Definition: ISDOpcodes.h:449
@ STRICT_FFLOOR
Definition: ISDOpcodes.h:427
@ STRICT_FROUNDEVEN
Definition: ISDOpcodes.h:429
@ BF16_TO_FP
BF16_TO_FP, FP_TO_BF16 - These operators are used to perform promotions and truncation for bfloat16.
Definition: ISDOpcodes.h:922
@ STRICT_FP_TO_UINT
Definition: ISDOpcodes.h:443
@ STRICT_FP_ROUND
X = STRICT_FP_ROUND(Y, TRUNC) - Rounding 'Y' from a larger floating point type down to the precision ...
Definition: ISDOpcodes.h:465
@ STRICT_FP_TO_SINT
STRICT_FP_TO_[US]INT - Convert a floating point value to a signed or unsigned integer.
Definition: ISDOpcodes.h:442
@ FMINIMUM
FMINIMUM/FMAXIMUM - NaN-propagating minimum/maximum that also treat -0.0 as less than 0....
Definition: ISDOpcodes.h:982
@ FP_TO_SINT
FP_TO_[US]INT - Convert a floating point value to a signed or unsigned integer.
Definition: ISDOpcodes.h:836
@ STRICT_FP_EXTEND
X = STRICT_FP_EXTEND(Y) - Extend a smaller FP type into a larger FP type.
Definition: ISDOpcodes.h:470
@ AND
Bitwise operators - logical and, logical or, logical xor.
Definition: ISDOpcodes.h:680
@ VECREDUCE_FMUL
Definition: ISDOpcodes.h:1345
@ STRICT_FADD
Constrained versions of the binary floating point operators.
Definition: ISDOpcodes.h:400
@ STRICT_FLOG10
Definition: ISDOpcodes.h:420
@ STRICT_LLRINT
Definition: ISDOpcodes.h:434
@ STRICT_FEXP2
Definition: ISDOpcodes.h:418
@ ATOMIC_SWAP
Val, OUTCHAIN = ATOMIC_SWAP(INCHAIN, ptr, amt) Val, OUTCHAIN = ATOMIC_LOAD_[OpName](INCHAIN,...
Definition: ISDOpcodes.h:1254
@ FFREXP
FFREXP - frexp, extract fractional and exponent component of a floating-point value.
Definition: ISDOpcodes.h:942
@ FP_ROUND
X = FP_ROUND(Y, TRUNC) - Rounding 'Y' from a larger floating point type down to the precision of the ...
Definition: ISDOpcodes.h:869
@ STRICT_FLDEXP
Definition: ISDOpcodes.h:414
@ STRICT_LLROUND
Definition: ISDOpcodes.h:432
@ STRICT_FNEARBYINT
Definition: ISDOpcodes.h:423
@ FP_TO_SINT_SAT
FP_TO_[US]INT_SAT - Convert floating point value in operand 0 to a signed or unsigned scalar integer ...
Definition: ISDOpcodes.h:855
@ VECREDUCE_FMINIMUM
Definition: ISDOpcodes.h:1352
@ TRUNCATE
TRUNCATE - Completely drop the high bits.
Definition: ISDOpcodes.h:786
@ VAARG
VAARG - VAARG has four operands: an input chain, a pointer, a SRCVALUE, and the alignment.
Definition: ISDOpcodes.h:1144
@ VECREDUCE_SEQ_FMUL
Definition: ISDOpcodes.h:1332
@ FCOPYSIGN
FCOPYSIGN(X, Y) - Return the value of X with the sign of Y.
Definition: ISDOpcodes.h:493
@ STRICT_FRINT
Definition: ISDOpcodes.h:422
@ BUILD_VECTOR
BUILD_VECTOR(ELT0, ELT1, ELT2, ELT3,...) - Return a fixed-width vector with the specified,...
Definition: ISDOpcodes.h:515
bool isNormalStore(const SDNode *N)
Returns true if the specified node is a non-truncating and unindexed store.
bool isUNINDEXEDLoad(const SDNode *N)
Returns true if the specified node is an unindexed load.
bool isUNINDEXEDStore(const SDNode *N)
Returns true if the specified node is an unindexed store.
CondCode
ISD::CondCode enum - These are ordered carefully to make the bitfields below work out,...
Definition: ISDOpcodes.h:1506
bool isNormalLoad(const SDNode *N)
Returns true if the specified node is a non-extending and unindexed load.
Libcall getPOWI(EVT RetVT)
getPOWI - Return the POWI_* value for the given types, or UNKNOWN_LIBCALL if there is none.
Libcall getSINTTOFP(EVT OpVT, EVT RetVT)
getSINTTOFP - Return the SINTTOFP_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
Libcall getLDEXP(EVT RetVT)
getLDEXP - Return the LDEXP_* value for the given types, or UNKNOWN_LIBCALL if there is none.
Libcall getUINTTOFP(EVT OpVT, EVT RetVT)
getUINTTOFP - Return the UINTTOFP_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
Libcall getFREXP(EVT RetVT)
getFREXP - Return the FREXP_* value for the given types, or UNKNOWN_LIBCALL if there is none.
Libcall
RTLIB::Libcall enum - This enum defines all of the runtime library calls the backend can emit.
Libcall getFPTOUINT(EVT OpVT, EVT RetVT)
getFPTOUINT - Return the FPTOUINT_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
Libcall getFPTOSINT(EVT OpVT, EVT RetVT)
getFPTOSINT - Return the FPTOSINT_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
Libcall getFPEXT(EVT OpVT, EVT RetVT)
getFPEXT - Return the FPEXT_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
Libcall getFPROUND(EVT OpVT, EVT RetVT)
getFPROUND - Return the FPROUND_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
DiagnosticInfoOptimizationBase::Argument NV
@ FalseVal
Definition: TGLexer.h:59
This is an optimization pass for GlobalISel generic memory operations.
Definition: AddressRanges.h:18
@ Offset
Definition: DWP.cpp:440
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
Definition: Debug.cpp:163
void report_fatal_error(Error Err, bool gen_crash_diag=true)
Report a serious error, calling any installed error handler.
Definition: Error.cpp:156
DWARFExpression::Operation Op
#define N
static const fltSemantics & PPCDoubleDouble() LLVM_READNONE
Definition: APFloat.cpp:252
Extended Value Type.
Definition: ValueTypes.h:34
bool isSimple() const
Test if the given EVT is simple (as opposed to being extended).
Definition: ValueTypes.h:129
TypeSize getSizeInBits() const
Return the size of the specified value type in bits.
Definition: ValueTypes.h:351
bool isByteSized() const
Return true if the bit size is a multiple of 8.
Definition: ValueTypes.h:226
MVT getSimpleVT() const
Return the SimpleValueType held in the specified simple EVT.
Definition: ValueTypes.h:299
static EVT getIntegerVT(LLVMContext &Context, unsigned BitWidth)
Returns the EVT that represents an integer with the given number of bits.
Definition: ValueTypes.h:64
bool bitsGE(EVT VT) const
Return true if this has no less bits than VT.
Definition: ValueTypes.h:275
EVT getVectorElementType() const
Given a vector type, return the type of each element.
Definition: ValueTypes.h:311
bool bitsLE(EVT VT) const
Return true if this has no more bits than VT.
Definition: ValueTypes.h:291
static MachinePointerInfo getFixedStack(MachineFunction &MF, int FI, int64_t Offset=0)
Return a MachinePointerInfo record that refers to the specified FrameIndex.
These are IR-level optimization flags that may be propagated to SDNodes.
This structure is used to pass arguments to makeLibCall function.
MakeLibCallOptions & setSExt(bool Value=true)
MakeLibCallOptions & setTypeListBeforeSoften(ArrayRef< EVT > OpsVT, EVT RetVT, bool Value=true)