LLVM 17.0.0git
LegalizeFloatTypes.cpp
Go to the documentation of this file.
1//===-------- LegalizeFloatTypes.cpp - Legalization of float types --------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements float type expansion and softening for LegalizeTypes.
10// Softening is the act of turning a computation in an illegal floating point
11// type into a computation in an integer type of the same size; also known as
12// "soft float". For example, turning f32 arithmetic into operations using i32.
13// The resulting integer value is the same as what you would get by performing
14// the floating point operation and bitcasting the result to the integer type.
15// Expansion is the act of changing a computation in an illegal type to be a
16// computation in two identical registers of a smaller type. For example,
17// implementing ppcf128 arithmetic in two f64 registers.
18//
19//===----------------------------------------------------------------------===//
20
21#include "LegalizeTypes.h"
25using namespace llvm;
26
27#define DEBUG_TYPE "legalize-types"
28
29/// GetFPLibCall - Return the right libcall for the given floating point type.
30/// FIXME: This is a local version of RTLIB::getFPLibCall that should be
31/// refactored away (see RTLIB::getPOWI for an example).
33 RTLIB::Libcall Call_F32,
34 RTLIB::Libcall Call_F64,
35 RTLIB::Libcall Call_F80,
36 RTLIB::Libcall Call_F128,
37 RTLIB::Libcall Call_PPCF128) {
38 return
39 VT == MVT::f32 ? Call_F32 :
40 VT == MVT::f64 ? Call_F64 :
41 VT == MVT::f80 ? Call_F80 :
42 VT == MVT::f128 ? Call_F128 :
43 VT == MVT::ppcf128 ? Call_PPCF128 :
44 RTLIB::UNKNOWN_LIBCALL;
45}
46
47//===----------------------------------------------------------------------===//
48// Convert Float Results to Integer
49//===----------------------------------------------------------------------===//
50
51void DAGTypeLegalizer::SoftenFloatResult(SDNode *N, unsigned ResNo) {
52 LLVM_DEBUG(dbgs() << "Soften float result " << ResNo << ": "; N->dump(&DAG);
53 dbgs() << "\n");
54 SDValue R = SDValue();
55
56 switch (N->getOpcode()) {
57 default:
58#ifndef NDEBUG
59 dbgs() << "SoftenFloatResult #" << ResNo << ": ";
60 N->dump(&DAG); dbgs() << "\n";
61#endif
62 llvm_unreachable("Do not know how to soften the result of this operator!");
63
64 case ISD::ARITH_FENCE: R = SoftenFloatRes_ARITH_FENCE(N); break;
65 case ISD::MERGE_VALUES:R = SoftenFloatRes_MERGE_VALUES(N, ResNo); break;
66 case ISD::BITCAST: R = SoftenFloatRes_BITCAST(N); break;
67 case ISD::BUILD_PAIR: R = SoftenFloatRes_BUILD_PAIR(N); break;
68 case ISD::ConstantFP: R = SoftenFloatRes_ConstantFP(N); break;
70 R = SoftenFloatRes_EXTRACT_VECTOR_ELT(N, ResNo); break;
71 case ISD::FABS: R = SoftenFloatRes_FABS(N); break;
73 case ISD::FMINNUM: R = SoftenFloatRes_FMINNUM(N); break;
75 case ISD::FMAXNUM: R = SoftenFloatRes_FMAXNUM(N); break;
77 case ISD::FADD: R = SoftenFloatRes_FADD(N); break;
78 case ISD::FCBRT: R = SoftenFloatRes_FCBRT(N); break;
80 case ISD::FCEIL: R = SoftenFloatRes_FCEIL(N); break;
81 case ISD::FCOPYSIGN: R = SoftenFloatRes_FCOPYSIGN(N); break;
83 case ISD::FCOS: R = SoftenFloatRes_FCOS(N); break;
85 case ISD::FDIV: R = SoftenFloatRes_FDIV(N); break;
87 case ISD::FEXP: R = SoftenFloatRes_FEXP(N); break;
89 case ISD::FEXP2: R = SoftenFloatRes_FEXP2(N); break;
91 case ISD::FFLOOR: R = SoftenFloatRes_FFLOOR(N); break;
93 case ISD::FLOG: R = SoftenFloatRes_FLOG(N); break;
95 case ISD::FLOG2: R = SoftenFloatRes_FLOG2(N); break;
97 case ISD::FLOG10: R = SoftenFloatRes_FLOG10(N); break;
98 case ISD::STRICT_FMA:
99 case ISD::FMA: R = SoftenFloatRes_FMA(N); break;
100 case ISD::STRICT_FMUL:
101 case ISD::FMUL: R = SoftenFloatRes_FMUL(N); break;
103 case ISD::FNEARBYINT: R = SoftenFloatRes_FNEARBYINT(N); break;
104 case ISD::FNEG: R = SoftenFloatRes_FNEG(N); break;
106 case ISD::FP_EXTEND: R = SoftenFloatRes_FP_EXTEND(N); break;
108 case ISD::FP_ROUND: R = SoftenFloatRes_FP_ROUND(N); break;
109 case ISD::FP16_TO_FP: R = SoftenFloatRes_FP16_TO_FP(N); break;
110 case ISD::BF16_TO_FP: R = SoftenFloatRes_BF16_TO_FP(N); break;
111 case ISD::STRICT_FPOW:
112 case ISD::FPOW: R = SoftenFloatRes_FPOW(N); break;
114 case ISD::FPOWI: R = SoftenFloatRes_FPOWI(N); break;
115 case ISD::STRICT_FREM:
116 case ISD::FREM: R = SoftenFloatRes_FREM(N); break;
118 case ISD::FRINT: R = SoftenFloatRes_FRINT(N); break;
120 case ISD::FROUND: R = SoftenFloatRes_FROUND(N); break;
122 case ISD::FROUNDEVEN: R = SoftenFloatRes_FROUNDEVEN(N); break;
123 case ISD::STRICT_FSIN:
124 case ISD::FSIN: R = SoftenFloatRes_FSIN(N); break;
126 case ISD::FSQRT: R = SoftenFloatRes_FSQRT(N); break;
127 case ISD::STRICT_FSUB:
128 case ISD::FSUB: R = SoftenFloatRes_FSUB(N); break;
130 case ISD::FTRUNC: R = SoftenFloatRes_FTRUNC(N); break;
131 case ISD::LOAD: R = SoftenFloatRes_LOAD(N); break;
132 case ISD::ATOMIC_SWAP: R = BitcastToInt_ATOMIC_SWAP(N); break;
133 case ISD::SELECT: R = SoftenFloatRes_SELECT(N); break;
134 case ISD::SELECT_CC: R = SoftenFloatRes_SELECT_CC(N); break;
135 case ISD::FREEZE: R = SoftenFloatRes_FREEZE(N); break;
138 case ISD::SINT_TO_FP:
139 case ISD::UINT_TO_FP: R = SoftenFloatRes_XINT_TO_FP(N); break;
140 case ISD::UNDEF: R = SoftenFloatRes_UNDEF(N); break;
141 case ISD::VAARG: R = SoftenFloatRes_VAARG(N); break;
146 R = SoftenFloatRes_VECREDUCE(N);
147 break;
150 R = SoftenFloatRes_VECREDUCE_SEQ(N);
151 break;
152 }
153
154 // If R is null, the sub-method took care of registering the result.
155 if (R.getNode()) {
156 assert(R.getNode() != N);
157 SetSoftenedFloat(SDValue(N, ResNo), R);
158 }
159}
160
161SDValue DAGTypeLegalizer::SoftenFloatRes_Unary(SDNode *N, RTLIB::Libcall LC) {
162 bool IsStrict = N->isStrictFPOpcode();
163 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
164 unsigned Offset = IsStrict ? 1 : 0;
165 assert(N->getNumOperands() == (1 + Offset) &&
166 "Unexpected number of operands!");
167 SDValue Op = GetSoftenedFloat(N->getOperand(0 + Offset));
168 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
170 EVT OpVT = N->getOperand(0 + Offset).getValueType();
171 CallOptions.setTypeListBeforeSoften(OpVT, N->getValueType(0), true);
172 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, NVT, Op,
173 CallOptions, SDLoc(N),
174 Chain);
175 if (IsStrict)
176 ReplaceValueWith(SDValue(N, 1), Tmp.second);
177 return Tmp.first;
178}
179
180SDValue DAGTypeLegalizer::SoftenFloatRes_Binary(SDNode *N, RTLIB::Libcall LC) {
181 bool IsStrict = N->isStrictFPOpcode();
182 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
183 unsigned Offset = IsStrict ? 1 : 0;
184 assert(N->getNumOperands() == (2 + Offset) &&
185 "Unexpected number of operands!");
186 SDValue Ops[2] = { GetSoftenedFloat(N->getOperand(0 + Offset)),
187 GetSoftenedFloat(N->getOperand(1 + Offset)) };
188 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
190 EVT OpsVT[2] = { N->getOperand(0 + Offset).getValueType(),
191 N->getOperand(1 + Offset).getValueType() };
192 CallOptions.setTypeListBeforeSoften(OpsVT, N->getValueType(0), true);
193 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, NVT, Ops,
194 CallOptions, SDLoc(N),
195 Chain);
196 if (IsStrict)
197 ReplaceValueWith(SDValue(N, 1), Tmp.second);
198 return Tmp.first;
199}
200
201SDValue DAGTypeLegalizer::SoftenFloatRes_BITCAST(SDNode *N) {
202 return BitConvertToInteger(N->getOperand(0));
203}
204
205SDValue DAGTypeLegalizer::SoftenFloatRes_FREEZE(SDNode *N) {
206 EVT Ty = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
207 return DAG.getNode(ISD::FREEZE, SDLoc(N), Ty,
208 GetSoftenedFloat(N->getOperand(0)));
209}
210
211SDValue DAGTypeLegalizer::SoftenFloatRes_ARITH_FENCE(SDNode *N) {
212 EVT Ty = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
213 SDValue NewFence = DAG.getNode(ISD::ARITH_FENCE, SDLoc(N), Ty,
214 GetSoftenedFloat(N->getOperand(0)));
215 return NewFence;
216}
217
218SDValue DAGTypeLegalizer::SoftenFloatRes_MERGE_VALUES(SDNode *N,
219 unsigned ResNo) {
220 SDValue Op = DisintegrateMERGE_VALUES(N, ResNo);
221 return BitConvertToInteger(Op);
222}
223
224SDValue DAGTypeLegalizer::SoftenFloatRes_BUILD_PAIR(SDNode *N) {
225 // Convert the inputs to integers, and build a new pair out of them.
226 return DAG.getNode(ISD::BUILD_PAIR, SDLoc(N),
228 N->getValueType(0)),
229 BitConvertToInteger(N->getOperand(0)),
230 BitConvertToInteger(N->getOperand(1)));
231}
232
233SDValue DAGTypeLegalizer::SoftenFloatRes_ConstantFP(SDNode *N) {
234 ConstantFPSDNode *CN = cast<ConstantFPSDNode>(N);
235 // In ppcf128, the high 64 bits are always first in memory regardless
236 // of Endianness. LLVM's APFloat representation is not Endian sensitive,
237 // and so always converts into a 128-bit APInt in a non-Endian-sensitive
238 // way. However, APInt's are serialized in an Endian-sensitive fashion,
239 // so on big-Endian targets, the two doubles are output in the wrong
240 // order. Fix this by manually flipping the order of the high 64 bits
241 // and the low 64 bits here.
242 if (DAG.getDataLayout().isBigEndian() &&
243 CN->getValueType(0).getSimpleVT() == llvm::MVT::ppcf128) {
244 uint64_t words[2] = { CN->getValueAPF().bitcastToAPInt().getRawData()[1],
246 APInt Val(128, words);
247 return DAG.getConstant(Val, SDLoc(CN),
249 CN->getValueType(0)));
250 } else {
251 return DAG.getConstant(CN->getValueAPF().bitcastToAPInt(), SDLoc(CN),
253 CN->getValueType(0)));
254 }
255}
256
257SDValue DAGTypeLegalizer::SoftenFloatRes_EXTRACT_VECTOR_ELT(SDNode *N, unsigned ResNo) {
258 SDValue NewOp = BitConvertVectorToIntegerVector(N->getOperand(0));
261 NewOp, N->getOperand(1));
262}
263
264SDValue DAGTypeLegalizer::SoftenFloatRes_FABS(SDNode *N) {
265 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
266 unsigned Size = NVT.getSizeInBits();
267
268 // Mask = ~(1 << (Size-1))
270 API.clearBit(Size - 1);
271 SDValue Mask = DAG.getConstant(API, SDLoc(N), NVT);
272 SDValue Op = GetSoftenedFloat(N->getOperand(0));
273 return DAG.getNode(ISD::AND, SDLoc(N), NVT, Op, Mask);
274}
275
276SDValue DAGTypeLegalizer::SoftenFloatRes_FMINNUM(SDNode *N) {
277 if (SDValue SelCC = TLI.createSelectForFMINNUM_FMAXNUM(N, DAG))
278 return SoftenFloatRes_SELECT_CC(SelCC.getNode());
279 return SoftenFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
280 RTLIB::FMIN_F32,
281 RTLIB::FMIN_F64,
282 RTLIB::FMIN_F80,
283 RTLIB::FMIN_F128,
284 RTLIB::FMIN_PPCF128));
285}
286
287SDValue DAGTypeLegalizer::SoftenFloatRes_FMAXNUM(SDNode *N) {
288 if (SDValue SelCC = TLI.createSelectForFMINNUM_FMAXNUM(N, DAG))
289 return SoftenFloatRes_SELECT_CC(SelCC.getNode());
290 return SoftenFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
291 RTLIB::FMAX_F32,
292 RTLIB::FMAX_F64,
293 RTLIB::FMAX_F80,
294 RTLIB::FMAX_F128,
295 RTLIB::FMAX_PPCF128));
296}
297
298SDValue DAGTypeLegalizer::SoftenFloatRes_FADD(SDNode *N) {
299 return SoftenFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
300 RTLIB::ADD_F32,
301 RTLIB::ADD_F64,
302 RTLIB::ADD_F80,
303 RTLIB::ADD_F128,
304 RTLIB::ADD_PPCF128));
305}
306
307SDValue DAGTypeLegalizer::SoftenFloatRes_FCBRT(SDNode *N) {
308 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
309 RTLIB::CBRT_F32,
310 RTLIB::CBRT_F64,
311 RTLIB::CBRT_F80,
312 RTLIB::CBRT_F128,
313 RTLIB::CBRT_PPCF128));
314}
315
316SDValue DAGTypeLegalizer::SoftenFloatRes_FCEIL(SDNode *N) {
317 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
318 RTLIB::CEIL_F32,
319 RTLIB::CEIL_F64,
320 RTLIB::CEIL_F80,
321 RTLIB::CEIL_F128,
322 RTLIB::CEIL_PPCF128));
323}
324
325SDValue DAGTypeLegalizer::SoftenFloatRes_FCOPYSIGN(SDNode *N) {
326 SDValue LHS = GetSoftenedFloat(N->getOperand(0));
327 SDValue RHS = BitConvertToInteger(N->getOperand(1));
328 SDLoc dl(N);
329
330 EVT LVT = LHS.getValueType();
331 EVT RVT = RHS.getValueType();
332
333 unsigned LSize = LVT.getSizeInBits();
334 unsigned RSize = RVT.getSizeInBits();
335
336 // First get the sign bit of second operand.
337 SDValue SignBit = DAG.getNode(
338 ISD::SHL, dl, RVT, DAG.getConstant(1, dl, RVT),
339 DAG.getConstant(RSize - 1, dl,
340 TLI.getShiftAmountTy(RVT, DAG.getDataLayout())));
341 SignBit = DAG.getNode(ISD::AND, dl, RVT, RHS, SignBit);
342
343 // Shift right or sign-extend it if the two operands have different types.
344 int SizeDiff = RVT.getSizeInBits() - LVT.getSizeInBits();
345 if (SizeDiff > 0) {
346 SignBit =
347 DAG.getNode(ISD::SRL, dl, RVT, SignBit,
348 DAG.getConstant(SizeDiff, dl,
349 TLI.getShiftAmountTy(SignBit.getValueType(),
350 DAG.getDataLayout())));
351 SignBit = DAG.getNode(ISD::TRUNCATE, dl, LVT, SignBit);
352 } else if (SizeDiff < 0) {
353 SignBit = DAG.getNode(ISD::ANY_EXTEND, dl, LVT, SignBit);
354 SignBit =
355 DAG.getNode(ISD::SHL, dl, LVT, SignBit,
356 DAG.getConstant(-SizeDiff, dl,
357 TLI.getShiftAmountTy(SignBit.getValueType(),
358 DAG.getDataLayout())));
359 }
360
361 // Clear the sign bit of the first operand.
362 SDValue Mask = DAG.getNode(
363 ISD::SHL, dl, LVT, DAG.getConstant(1, dl, LVT),
364 DAG.getConstant(LSize - 1, dl,
365 TLI.getShiftAmountTy(LVT, DAG.getDataLayout())));
366 Mask = DAG.getNode(ISD::SUB, dl, LVT, Mask, DAG.getConstant(1, dl, LVT));
367 LHS = DAG.getNode(ISD::AND, dl, LVT, LHS, Mask);
368
369 // Or the value with the sign bit.
370 return DAG.getNode(ISD::OR, dl, LVT, LHS, SignBit);
371}
372
373SDValue DAGTypeLegalizer::SoftenFloatRes_FCOS(SDNode *N) {
374 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
375 RTLIB::COS_F32,
376 RTLIB::COS_F64,
377 RTLIB::COS_F80,
378 RTLIB::COS_F128,
379 RTLIB::COS_PPCF128));
380}
381
382SDValue DAGTypeLegalizer::SoftenFloatRes_FDIV(SDNode *N) {
383 return SoftenFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
384 RTLIB::DIV_F32,
385 RTLIB::DIV_F64,
386 RTLIB::DIV_F80,
387 RTLIB::DIV_F128,
388 RTLIB::DIV_PPCF128));
389}
390
391SDValue DAGTypeLegalizer::SoftenFloatRes_FEXP(SDNode *N) {
392 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
393 RTLIB::EXP_F32,
394 RTLIB::EXP_F64,
395 RTLIB::EXP_F80,
396 RTLIB::EXP_F128,
397 RTLIB::EXP_PPCF128));
398}
399
400SDValue DAGTypeLegalizer::SoftenFloatRes_FEXP2(SDNode *N) {
401 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
402 RTLIB::EXP2_F32,
403 RTLIB::EXP2_F64,
404 RTLIB::EXP2_F80,
405 RTLIB::EXP2_F128,
406 RTLIB::EXP2_PPCF128));
407}
408
409SDValue DAGTypeLegalizer::SoftenFloatRes_FFLOOR(SDNode *N) {
410 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
411 RTLIB::FLOOR_F32,
412 RTLIB::FLOOR_F64,
413 RTLIB::FLOOR_F80,
414 RTLIB::FLOOR_F128,
415 RTLIB::FLOOR_PPCF128));
416}
417
418SDValue DAGTypeLegalizer::SoftenFloatRes_FLOG(SDNode *N) {
419 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
420 RTLIB::LOG_F32,
421 RTLIB::LOG_F64,
422 RTLIB::LOG_F80,
423 RTLIB::LOG_F128,
424 RTLIB::LOG_PPCF128));
425}
426
427SDValue DAGTypeLegalizer::SoftenFloatRes_FLOG2(SDNode *N) {
428 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
429 RTLIB::LOG2_F32,
430 RTLIB::LOG2_F64,
431 RTLIB::LOG2_F80,
432 RTLIB::LOG2_F128,
433 RTLIB::LOG2_PPCF128));
434}
435
436SDValue DAGTypeLegalizer::SoftenFloatRes_FLOG10(SDNode *N) {
437 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
438 RTLIB::LOG10_F32,
439 RTLIB::LOG10_F64,
440 RTLIB::LOG10_F80,
441 RTLIB::LOG10_F128,
442 RTLIB::LOG10_PPCF128));
443}
444
445SDValue DAGTypeLegalizer::SoftenFloatRes_FMA(SDNode *N) {
446 bool IsStrict = N->isStrictFPOpcode();
447 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
448 unsigned Offset = IsStrict ? 1 : 0;
449 SDValue Ops[3] = { GetSoftenedFloat(N->getOperand(0 + Offset)),
450 GetSoftenedFloat(N->getOperand(1 + Offset)),
451 GetSoftenedFloat(N->getOperand(2 + Offset)) };
452 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
454 EVT OpsVT[3] = { N->getOperand(0 + Offset).getValueType(),
455 N->getOperand(1 + Offset).getValueType(),
456 N->getOperand(2 + Offset).getValueType() };
457 CallOptions.setTypeListBeforeSoften(OpsVT, N->getValueType(0), true);
458 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG,
459 GetFPLibCall(N->getValueType(0),
460 RTLIB::FMA_F32,
461 RTLIB::FMA_F64,
462 RTLIB::FMA_F80,
463 RTLIB::FMA_F128,
464 RTLIB::FMA_PPCF128),
465 NVT, Ops, CallOptions, SDLoc(N), Chain);
466 if (IsStrict)
467 ReplaceValueWith(SDValue(N, 1), Tmp.second);
468 return Tmp.first;
469}
470
471SDValue DAGTypeLegalizer::SoftenFloatRes_FMUL(SDNode *N) {
472 return SoftenFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
473 RTLIB::MUL_F32,
474 RTLIB::MUL_F64,
475 RTLIB::MUL_F80,
476 RTLIB::MUL_F128,
477 RTLIB::MUL_PPCF128));
478}
479
480SDValue DAGTypeLegalizer::SoftenFloatRes_FNEARBYINT(SDNode *N) {
481 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
482 RTLIB::NEARBYINT_F32,
483 RTLIB::NEARBYINT_F64,
484 RTLIB::NEARBYINT_F80,
485 RTLIB::NEARBYINT_F128,
486 RTLIB::NEARBYINT_PPCF128));
487}
488
489SDValue DAGTypeLegalizer::SoftenFloatRes_FNEG(SDNode *N) {
490 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
491 SDLoc dl(N);
492
493 // Expand Y = FNEG(X) -> Y = X ^ sign mask
494 APInt SignMask = APInt::getSignMask(NVT.getSizeInBits());
495 return DAG.getNode(ISD::XOR, dl, NVT, GetSoftenedFloat(N->getOperand(0)),
496 DAG.getConstant(SignMask, dl, NVT));
497}
498
499SDValue DAGTypeLegalizer::SoftenFloatRes_FP_EXTEND(SDNode *N) {
500 bool IsStrict = N->isStrictFPOpcode();
501 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
502 SDValue Op = N->getOperand(IsStrict ? 1 : 0);
503
504 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
505
506 if (getTypeAction(Op.getValueType()) == TargetLowering::TypePromoteFloat) {
507 Op = GetPromotedFloat(Op);
508 // If the promotion did the FP_EXTEND to the destination type for us,
509 // there's nothing left to do here.
510 if (Op.getValueType() == N->getValueType(0))
511 return BitConvertToInteger(Op);
512 }
513
514 // There's only a libcall for f16 -> f32 and shifting is only valid for bf16
515 // -> f32, so proceed in two stages. Also, it's entirely possible for both
516 // f16 and f32 to be legal, so use the fully hard-float FP_EXTEND rather
517 // than FP16_TO_FP.
518 if ((Op.getValueType() == MVT::f16 || Op.getValueType() == MVT::bf16) &&
519 N->getValueType(0) != MVT::f32) {
520 if (IsStrict) {
522 { MVT::f32, MVT::Other }, { Chain, Op });
523 Chain = Op.getValue(1);
524 } else {
525 Op = DAG.getNode(ISD::FP_EXTEND, SDLoc(N), MVT::f32, Op);
526 }
527 }
528
529 if (Op.getValueType() == MVT::bf16)
530 return SoftenFloatRes_BF16_TO_FP(N);
531
532 RTLIB::Libcall LC = RTLIB::getFPEXT(Op.getValueType(), N->getValueType(0));
533 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported FP_EXTEND!");
535 EVT OpVT = N->getOperand(IsStrict ? 1 : 0).getValueType();
536 CallOptions.setTypeListBeforeSoften(OpVT, N->getValueType(0), true);
537 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, NVT, Op,
538 CallOptions, SDLoc(N),
539 Chain);
540 if (IsStrict)
541 ReplaceValueWith(SDValue(N, 1), Tmp.second);
542 return Tmp.first;
543}
544
545// FIXME: Should we just use 'normal' FP_EXTEND / FP_TRUNC instead of special
546// nodes?
547SDValue DAGTypeLegalizer::SoftenFloatRes_FP16_TO_FP(SDNode *N) {
548 EVT MidVT = TLI.getTypeToTransformTo(*DAG.getContext(), MVT::f32);
549 SDValue Op = N->getOperand(0);
551 EVT OpsVT[1] = { N->getOperand(0).getValueType() };
552 CallOptions.setTypeListBeforeSoften(OpsVT, N->getValueType(0), true);
553 SDValue Res32 = TLI.makeLibCall(DAG, RTLIB::FPEXT_F16_F32, MidVT, Op,
554 CallOptions, SDLoc(N)).first;
555 if (N->getValueType(0) == MVT::f32)
556 return Res32;
557
558 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
559 RTLIB::Libcall LC = RTLIB::getFPEXT(MVT::f32, N->getValueType(0));
560 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported FP_EXTEND!");
561 return TLI.makeLibCall(DAG, LC, NVT, Res32, CallOptions, SDLoc(N)).first;
562}
563
564// FIXME: Should we just use 'normal' FP_EXTEND / FP_TRUNC instead of special
565// nodes?
566SDValue DAGTypeLegalizer::SoftenFloatRes_BF16_TO_FP(SDNode *N) {
567 assert(N->getValueType(0) == MVT::f32 &&
568 "Can only soften BF16_TO_FP with f32 result");
569 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), MVT::f32);
570 SDValue Op = N->getOperand(0);
571 SDLoc DL(N);
572 Op = DAG.getNode(ISD::ANY_EXTEND, DL, NVT,
573 DAG.getNode(ISD::BITCAST, DL, MVT::i16, Op));
574 SDValue Res = DAG.getNode(ISD::SHL, DL, NVT, Op,
575 DAG.getShiftAmountConstant(16, NVT, DL));
576 return Res;
577}
578
579SDValue DAGTypeLegalizer::SoftenFloatRes_FP_ROUND(SDNode *N) {
580 bool IsStrict = N->isStrictFPOpcode();
581 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
582 SDValue Op = N->getOperand(IsStrict ? 1 : 0);
583 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
584 RTLIB::Libcall LC = RTLIB::getFPROUND(Op.getValueType(), N->getValueType(0));
585 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported FP_ROUND!");
587 EVT OpVT = N->getOperand(IsStrict ? 1 : 0).getValueType();
588 CallOptions.setTypeListBeforeSoften(OpVT, N->getValueType(0), true);
589 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, NVT, Op,
590 CallOptions, SDLoc(N),
591 Chain);
592 if (IsStrict)
593 ReplaceValueWith(SDValue(N, 1), Tmp.second);
594 return Tmp.first;
595}
596
597SDValue DAGTypeLegalizer::SoftenFloatRes_FPOW(SDNode *N) {
598 return SoftenFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
599 RTLIB::POW_F32,
600 RTLIB::POW_F64,
601 RTLIB::POW_F80,
602 RTLIB::POW_F128,
603 RTLIB::POW_PPCF128));
604}
605
606SDValue DAGTypeLegalizer::SoftenFloatRes_FPOWI(SDNode *N) {
607 bool IsStrict = N->isStrictFPOpcode();
608 unsigned Offset = IsStrict ? 1 : 0;
609 assert((N->getOperand(1 + Offset).getValueType() == MVT::i16 ||
610 N->getOperand(1 + Offset).getValueType() == MVT::i32) &&
611 "Unsupported power type!");
612 RTLIB::Libcall LC = RTLIB::getPOWI(N->getValueType(0));
613 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected fpowi.");
614 if (!TLI.getLibcallName(LC)) {
615 // Some targets don't have a powi libcall; use pow instead.
616 // FIXME: Implement this if some target needs it.
617 DAG.getContext()->emitError("Don't know how to soften fpowi to fpow");
618 return DAG.getUNDEF(N->getValueType(0));
619 }
620
621 if (DAG.getLibInfo().getIntSize() !=
622 N->getOperand(1 + Offset).getValueType().getSizeInBits()) {
623 // If the exponent does not match with sizeof(int) a libcall to RTLIB::POWI
624 // would use the wrong type for the argument.
625 DAG.getContext()->emitError("POWI exponent does not match sizeof(int)");
626 return DAG.getUNDEF(N->getValueType(0));
627 }
628
629 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
630 SDValue Ops[2] = { GetSoftenedFloat(N->getOperand(0 + Offset)),
631 N->getOperand(1 + Offset) };
632 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
634 EVT OpsVT[2] = { N->getOperand(0 + Offset).getValueType(),
635 N->getOperand(1 + Offset).getValueType() };
636 CallOptions.setTypeListBeforeSoften(OpsVT, N->getValueType(0), true);
637 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, NVT, Ops,
638 CallOptions, SDLoc(N),
639 Chain);
640 if (IsStrict)
641 ReplaceValueWith(SDValue(N, 1), Tmp.second);
642 return Tmp.first;
643}
644
645SDValue DAGTypeLegalizer::SoftenFloatRes_FREM(SDNode *N) {
646 return SoftenFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
647 RTLIB::REM_F32,
648 RTLIB::REM_F64,
649 RTLIB::REM_F80,
650 RTLIB::REM_F128,
651 RTLIB::REM_PPCF128));
652}
653
654SDValue DAGTypeLegalizer::SoftenFloatRes_FRINT(SDNode *N) {
655 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
656 RTLIB::RINT_F32,
657 RTLIB::RINT_F64,
658 RTLIB::RINT_F80,
659 RTLIB::RINT_F128,
660 RTLIB::RINT_PPCF128));
661}
662
663SDValue DAGTypeLegalizer::SoftenFloatRes_FROUND(SDNode *N) {
664 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
665 RTLIB::ROUND_F32,
666 RTLIB::ROUND_F64,
667 RTLIB::ROUND_F80,
668 RTLIB::ROUND_F128,
669 RTLIB::ROUND_PPCF128));
670}
671
672SDValue DAGTypeLegalizer::SoftenFloatRes_FROUNDEVEN(SDNode *N) {
673 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
674 RTLIB::ROUNDEVEN_F32,
675 RTLIB::ROUNDEVEN_F64,
676 RTLIB::ROUNDEVEN_F80,
677 RTLIB::ROUNDEVEN_F128,
678 RTLIB::ROUNDEVEN_PPCF128));
679}
680
681SDValue DAGTypeLegalizer::SoftenFloatRes_FSIN(SDNode *N) {
682 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
683 RTLIB::SIN_F32,
684 RTLIB::SIN_F64,
685 RTLIB::SIN_F80,
686 RTLIB::SIN_F128,
687 RTLIB::SIN_PPCF128));
688}
689
690SDValue DAGTypeLegalizer::SoftenFloatRes_FSQRT(SDNode *N) {
691 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
692 RTLIB::SQRT_F32,
693 RTLIB::SQRT_F64,
694 RTLIB::SQRT_F80,
695 RTLIB::SQRT_F128,
696 RTLIB::SQRT_PPCF128));
697}
698
699SDValue DAGTypeLegalizer::SoftenFloatRes_FSUB(SDNode *N) {
700 return SoftenFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
701 RTLIB::SUB_F32,
702 RTLIB::SUB_F64,
703 RTLIB::SUB_F80,
704 RTLIB::SUB_F128,
705 RTLIB::SUB_PPCF128));
706}
707
708SDValue DAGTypeLegalizer::SoftenFloatRes_FTRUNC(SDNode *N) {
709 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
710 RTLIB::TRUNC_F32,
711 RTLIB::TRUNC_F64,
712 RTLIB::TRUNC_F80,
713 RTLIB::TRUNC_F128,
714 RTLIB::TRUNC_PPCF128));
715}
716
717SDValue DAGTypeLegalizer::SoftenFloatRes_LOAD(SDNode *N) {
718 LoadSDNode *L = cast<LoadSDNode>(N);
719 EVT VT = N->getValueType(0);
720 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
721 SDLoc dl(N);
722
723 auto MMOFlags =
724 L->getMemOperand()->getFlags() &
726 SDValue NewL;
727 if (L->getExtensionType() == ISD::NON_EXTLOAD) {
728 NewL = DAG.getLoad(L->getAddressingMode(), L->getExtensionType(), NVT, dl,
729 L->getChain(), L->getBasePtr(), L->getOffset(),
730 L->getPointerInfo(), NVT, L->getOriginalAlign(),
731 MMOFlags, L->getAAInfo());
732 // Legalized the chain result - switch anything that used the old chain to
733 // use the new one.
734 ReplaceValueWith(SDValue(N, 1), NewL.getValue(1));
735 return NewL;
736 }
737
738 // Do a non-extending load followed by FP_EXTEND.
739 NewL = DAG.getLoad(L->getAddressingMode(), ISD::NON_EXTLOAD, L->getMemoryVT(),
740 dl, L->getChain(), L->getBasePtr(), L->getOffset(),
741 L->getPointerInfo(), L->getMemoryVT(),
742 L->getOriginalAlign(), MMOFlags, L->getAAInfo());
743 // Legalized the chain result - switch anything that used the old chain to
744 // use the new one.
745 ReplaceValueWith(SDValue(N, 1), NewL.getValue(1));
746 auto ExtendNode = DAG.getNode(ISD::FP_EXTEND, dl, VT, NewL);
747 return BitConvertToInteger(ExtendNode);
748}
749
750SDValue DAGTypeLegalizer::SoftenFloatRes_SELECT(SDNode *N) {
751 SDValue LHS = GetSoftenedFloat(N->getOperand(1));
752 SDValue RHS = GetSoftenedFloat(N->getOperand(2));
753 return DAG.getSelect(SDLoc(N),
754 LHS.getValueType(), N->getOperand(0), LHS, RHS);
755}
756
757SDValue DAGTypeLegalizer::SoftenFloatRes_SELECT_CC(SDNode *N) {
758 SDValue LHS = GetSoftenedFloat(N->getOperand(2));
759 SDValue RHS = GetSoftenedFloat(N->getOperand(3));
760 return DAG.getNode(ISD::SELECT_CC, SDLoc(N),
761 LHS.getValueType(), N->getOperand(0),
762 N->getOperand(1), LHS, RHS, N->getOperand(4));
763}
764
765SDValue DAGTypeLegalizer::SoftenFloatRes_UNDEF(SDNode *N) {
766 return DAG.getUNDEF(TLI.getTypeToTransformTo(*DAG.getContext(),
767 N->getValueType(0)));
768}
769
770SDValue DAGTypeLegalizer::SoftenFloatRes_VAARG(SDNode *N) {
771 SDValue Chain = N->getOperand(0); // Get the chain.
772 SDValue Ptr = N->getOperand(1); // Get the pointer.
773 EVT VT = N->getValueType(0);
774 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
775 SDLoc dl(N);
776
777 SDValue NewVAARG;
778 NewVAARG = DAG.getVAArg(NVT, dl, Chain, Ptr, N->getOperand(2),
779 N->getConstantOperandVal(3));
780
781 // Legalized the chain result - switch anything that used the old chain to
782 // use the new one.
783 if (N != NewVAARG.getValue(1).getNode())
784 ReplaceValueWith(SDValue(N, 1), NewVAARG.getValue(1));
785 return NewVAARG;
786}
787
788SDValue DAGTypeLegalizer::SoftenFloatRes_XINT_TO_FP(SDNode *N) {
789 bool IsStrict = N->isStrictFPOpcode();
790 bool Signed = N->getOpcode() == ISD::SINT_TO_FP ||
791 N->getOpcode() == ISD::STRICT_SINT_TO_FP;
792 EVT SVT = N->getOperand(IsStrict ? 1 : 0).getValueType();
793 EVT RVT = N->getValueType(0);
794 EVT NVT = EVT();
795 SDLoc dl(N);
796
797 // If the input is not legal, eg: i1 -> fp, then it needs to be promoted to
798 // a larger type, eg: i8 -> fp. Even if it is legal, no libcall may exactly
799 // match. Look for an appropriate libcall.
800 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
801 for (unsigned t = MVT::FIRST_INTEGER_VALUETYPE;
802 t <= MVT::LAST_INTEGER_VALUETYPE && LC == RTLIB::UNKNOWN_LIBCALL; ++t) {
803 NVT = (MVT::SimpleValueType)t;
804 // The source needs to big enough to hold the operand.
805 if (NVT.bitsGE(SVT))
806 LC = Signed ? RTLIB::getSINTTOFP(NVT, RVT):RTLIB::getUINTTOFP (NVT, RVT);
807 }
808 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported XINT_TO_FP!");
809
810 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
811 // Sign/zero extend the argument if the libcall takes a larger type.
813 NVT, N->getOperand(IsStrict ? 1 : 0));
815 CallOptions.setSExt(Signed);
816 CallOptions.setTypeListBeforeSoften(SVT, RVT, true);
817 std::pair<SDValue, SDValue> Tmp =
818 TLI.makeLibCall(DAG, LC, TLI.getTypeToTransformTo(*DAG.getContext(), RVT),
819 Op, CallOptions, dl, Chain);
820
821 if (IsStrict)
822 ReplaceValueWith(SDValue(N, 1), Tmp.second);
823 return Tmp.first;
824}
825
826SDValue DAGTypeLegalizer::SoftenFloatRes_VECREDUCE(SDNode *N) {
827 // Expand and soften recursively.
828 ReplaceValueWith(SDValue(N, 0), TLI.expandVecReduce(N, DAG));
829 return SDValue();
830}
831
832SDValue DAGTypeLegalizer::SoftenFloatRes_VECREDUCE_SEQ(SDNode *N) {
833 ReplaceValueWith(SDValue(N, 0), TLI.expandVecReduceSeq(N, DAG));
834 return SDValue();
835}
836
837//===----------------------------------------------------------------------===//
838// Convert Float Operand to Integer
839//===----------------------------------------------------------------------===//
840
841bool DAGTypeLegalizer::SoftenFloatOperand(SDNode *N, unsigned OpNo) {
842 LLVM_DEBUG(dbgs() << "Soften float operand " << OpNo << ": "; N->dump(&DAG);
843 dbgs() << "\n");
844 SDValue Res = SDValue();
845
846 switch (N->getOpcode()) {
847 default:
848#ifndef NDEBUG
849 dbgs() << "SoftenFloatOperand Op #" << OpNo << ": ";
850 N->dump(&DAG); dbgs() << "\n";
851#endif
852 llvm_unreachable("Do not know how to soften this operator's operand!");
853
854 case ISD::BITCAST: Res = SoftenFloatOp_BITCAST(N); break;
855 case ISD::BR_CC: Res = SoftenFloatOp_BR_CC(N); break;
857 case ISD::FP_TO_FP16: // Same as FP_ROUND for softening purposes
858 case ISD::FP_TO_BF16:
860 case ISD::FP_ROUND: Res = SoftenFloatOp_FP_ROUND(N); break;
863 case ISD::FP_TO_SINT:
864 case ISD::FP_TO_UINT: Res = SoftenFloatOp_FP_TO_XINT(N); break;
867 Res = SoftenFloatOp_FP_TO_XINT_SAT(N); break;
869 case ISD::LROUND: Res = SoftenFloatOp_LROUND(N); break;
871 case ISD::LLROUND: Res = SoftenFloatOp_LLROUND(N); break;
873 case ISD::LRINT: Res = SoftenFloatOp_LRINT(N); break;
875 case ISD::LLRINT: Res = SoftenFloatOp_LLRINT(N); break;
876 case ISD::SELECT_CC: Res = SoftenFloatOp_SELECT_CC(N); break;
879 case ISD::SETCC: Res = SoftenFloatOp_SETCC(N); break;
880 case ISD::STORE: Res = SoftenFloatOp_STORE(N, OpNo); break;
881 case ISD::FCOPYSIGN: Res = SoftenFloatOp_FCOPYSIGN(N); break;
882 }
883
884 // If the result is null, the sub-method took care of registering results etc.
885 if (!Res.getNode()) return false;
886
887 // If the result is N, the sub-method updated N in place. Tell the legalizer
888 // core about this to re-analyze.
889 if (Res.getNode() == N)
890 return true;
891
892 assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 &&
893 "Invalid operand softening");
894
895 ReplaceValueWith(SDValue(N, 0), Res);
896 return false;
897}
898
899SDValue DAGTypeLegalizer::SoftenFloatOp_BITCAST(SDNode *N) {
900 SDValue Op0 = GetSoftenedFloat(N->getOperand(0));
901
902 return DAG.getNode(ISD::BITCAST, SDLoc(N), N->getValueType(0), Op0);
903}
904
905SDValue DAGTypeLegalizer::SoftenFloatOp_FP_ROUND(SDNode *N) {
906 // We actually deal with the partially-softened FP_TO_FP16 node too, which
907 // returns an i16 so doesn't meet the constraints necessary for FP_ROUND.
908 assert(N->getOpcode() == ISD::FP_ROUND || N->getOpcode() == ISD::FP_TO_FP16 ||
909 N->getOpcode() == ISD::STRICT_FP_TO_FP16 ||
910 N->getOpcode() == ISD::FP_TO_BF16 ||
911 N->getOpcode() == ISD::STRICT_FP_ROUND);
912
913 bool IsStrict = N->isStrictFPOpcode();
914 SDValue Op = N->getOperand(IsStrict ? 1 : 0);
915 EVT SVT = Op.getValueType();
916 EVT RVT = N->getValueType(0);
917 EVT FloatRVT = RVT;
918 if (N->getOpcode() == ISD::FP_TO_FP16 ||
919 N->getOpcode() == ISD::STRICT_FP_TO_FP16)
920 FloatRVT = MVT::f16;
921 else if (N->getOpcode() == ISD::FP_TO_BF16)
922 FloatRVT = MVT::bf16;
923
924 RTLIB::Libcall LC = RTLIB::getFPROUND(SVT, FloatRVT);
925 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported FP_ROUND libcall");
926
927 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
928 Op = GetSoftenedFloat(Op);
930 CallOptions.setTypeListBeforeSoften(SVT, RVT, true);
931 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, RVT, Op,
932 CallOptions, SDLoc(N),
933 Chain);
934 if (IsStrict) {
935 ReplaceValueWith(SDValue(N, 1), Tmp.second);
936 ReplaceValueWith(SDValue(N, 0), Tmp.first);
937 return SDValue();
938 }
939 return Tmp.first;
940}
941
942SDValue DAGTypeLegalizer::SoftenFloatOp_BR_CC(SDNode *N) {
943 SDValue NewLHS = N->getOperand(2), NewRHS = N->getOperand(3);
944 ISD::CondCode CCCode = cast<CondCodeSDNode>(N->getOperand(1))->get();
945
946 EVT VT = NewLHS.getValueType();
947 NewLHS = GetSoftenedFloat(NewLHS);
948 NewRHS = GetSoftenedFloat(NewRHS);
949 TLI.softenSetCCOperands(DAG, VT, NewLHS, NewRHS, CCCode, SDLoc(N),
950 N->getOperand(2), N->getOperand(3));
951
952 // If softenSetCCOperands returned a scalar, we need to compare the result
953 // against zero to select between true and false values.
954 if (!NewRHS.getNode()) {
955 NewRHS = DAG.getConstant(0, SDLoc(N), NewLHS.getValueType());
956 CCCode = ISD::SETNE;
957 }
958
959 // Update N to have the operands specified.
960 return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0),
961 DAG.getCondCode(CCCode), NewLHS, NewRHS,
962 N->getOperand(4)),
963 0);
964}
965
966// Even if the result type is legal, no libcall may exactly match. (e.g. We
967// don't have FP-i8 conversions) This helper method looks for an appropriate
968// promoted libcall.
969static RTLIB::Libcall findFPToIntLibcall(EVT SrcVT, EVT RetVT, EVT &Promoted,
970 bool Signed) {
971 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
972 for (unsigned IntVT = MVT::FIRST_INTEGER_VALUETYPE;
973 IntVT <= MVT::LAST_INTEGER_VALUETYPE && LC == RTLIB::UNKNOWN_LIBCALL;
974 ++IntVT) {
975 Promoted = (MVT::SimpleValueType)IntVT;
976 // The type needs to big enough to hold the result.
977 if (Promoted.bitsGE(RetVT))
978 LC = Signed ? RTLIB::getFPTOSINT(SrcVT, Promoted)
979 : RTLIB::getFPTOUINT(SrcVT, Promoted);
980 }
981 return LC;
982}
983
984SDValue DAGTypeLegalizer::SoftenFloatOp_FP_TO_XINT(SDNode *N) {
985 bool IsStrict = N->isStrictFPOpcode();
986 bool Signed = N->getOpcode() == ISD::FP_TO_SINT ||
987 N->getOpcode() == ISD::STRICT_FP_TO_SINT;
988
989 SDValue Op = N->getOperand(IsStrict ? 1 : 0);
990 EVT SVT = Op.getValueType();
991 EVT RVT = N->getValueType(0);
992 EVT NVT = EVT();
993 SDLoc dl(N);
994
995 // If the result is not legal, eg: fp -> i1, then it needs to be promoted to
996 // a larger type, eg: fp -> i32. Even if it is legal, no libcall may exactly
997 // match, eg. we don't have fp -> i8 conversions.
998 // Look for an appropriate libcall.
999 RTLIB::Libcall LC = findFPToIntLibcall(SVT, RVT, NVT, Signed);
1000 assert(LC != RTLIB::UNKNOWN_LIBCALL && NVT.isSimple() &&
1001 "Unsupported FP_TO_XINT!");
1002
1003 Op = GetSoftenedFloat(Op);
1004 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
1006 CallOptions.setTypeListBeforeSoften(SVT, RVT, true);
1007 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, NVT, Op,
1008 CallOptions, dl, Chain);
1009
1010 // Truncate the result if the libcall returns a larger type.
1011 SDValue Res = DAG.getNode(ISD::TRUNCATE, dl, RVT, Tmp.first);
1012
1013 if (!IsStrict)
1014 return Res;
1015
1016 ReplaceValueWith(SDValue(N, 1), Tmp.second);
1017 ReplaceValueWith(SDValue(N, 0), Res);
1018 return SDValue();
1019}
1020
1021SDValue DAGTypeLegalizer::SoftenFloatOp_FP_TO_XINT_SAT(SDNode *N) {
1022 SDValue Res = TLI.expandFP_TO_INT_SAT(N, DAG);
1023 return Res;
1024}
1025
1026SDValue DAGTypeLegalizer::SoftenFloatOp_SELECT_CC(SDNode *N) {
1027 SDValue NewLHS = N->getOperand(0), NewRHS = N->getOperand(1);
1028 ISD::CondCode CCCode = cast<CondCodeSDNode>(N->getOperand(4))->get();
1029
1030 EVT VT = NewLHS.getValueType();
1031 NewLHS = GetSoftenedFloat(NewLHS);
1032 NewRHS = GetSoftenedFloat(NewRHS);
1033 TLI.softenSetCCOperands(DAG, VT, NewLHS, NewRHS, CCCode, SDLoc(N),
1034 N->getOperand(0), N->getOperand(1));
1035
1036 // If softenSetCCOperands returned a scalar, we need to compare the result
1037 // against zero to select between true and false values.
1038 if (!NewRHS.getNode()) {
1039 NewRHS = DAG.getConstant(0, SDLoc(N), NewLHS.getValueType());
1040 CCCode = ISD::SETNE;
1041 }
1042
1043 // Update N to have the operands specified.
1044 return SDValue(DAG.UpdateNodeOperands(N, NewLHS, NewRHS,
1045 N->getOperand(2), N->getOperand(3),
1046 DAG.getCondCode(CCCode)),
1047 0);
1048}
1049
1050SDValue DAGTypeLegalizer::SoftenFloatOp_SETCC(SDNode *N) {
1051 bool IsStrict = N->isStrictFPOpcode();
1052 SDValue Op0 = N->getOperand(IsStrict ? 1 : 0);
1053 SDValue Op1 = N->getOperand(IsStrict ? 2 : 1);
1054 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
1055 ISD::CondCode CCCode =
1056 cast<CondCodeSDNode>(N->getOperand(IsStrict ? 3 : 2))->get();
1057
1058 EVT VT = Op0.getValueType();
1059 SDValue NewLHS = GetSoftenedFloat(Op0);
1060 SDValue NewRHS = GetSoftenedFloat(Op1);
1061 TLI.softenSetCCOperands(DAG, VT, NewLHS, NewRHS, CCCode, SDLoc(N), Op0, Op1,
1062 Chain, N->getOpcode() == ISD::STRICT_FSETCCS);
1063
1064 // Update N to have the operands specified.
1065 if (NewRHS.getNode()) {
1066 if (IsStrict)
1067 NewLHS = DAG.getNode(ISD::SETCC, SDLoc(N), N->getValueType(0), NewLHS,
1068 NewRHS, DAG.getCondCode(CCCode));
1069 else
1070 return SDValue(DAG.UpdateNodeOperands(N, NewLHS, NewRHS,
1071 DAG.getCondCode(CCCode)), 0);
1072 }
1073
1074 // Otherwise, softenSetCCOperands returned a scalar, use it.
1075 assert((NewRHS.getNode() || NewLHS.getValueType() == N->getValueType(0)) &&
1076 "Unexpected setcc expansion!");
1077
1078 if (IsStrict) {
1079 ReplaceValueWith(SDValue(N, 0), NewLHS);
1080 ReplaceValueWith(SDValue(N, 1), Chain);
1081 return SDValue();
1082 }
1083 return NewLHS;
1084}
1085
1086SDValue DAGTypeLegalizer::SoftenFloatOp_STORE(SDNode *N, unsigned OpNo) {
1087 assert(ISD::isUNINDEXEDStore(N) && "Indexed store during type legalization!");
1088 assert(OpNo == 1 && "Can only soften the stored value!");
1089 StoreSDNode *ST = cast<StoreSDNode>(N);
1090 SDValue Val = ST->getValue();
1091 SDLoc dl(N);
1092
1093 if (ST->isTruncatingStore())
1094 // Do an FP_ROUND followed by a non-truncating store.
1095 Val = BitConvertToInteger(
1096 DAG.getNode(ISD::FP_ROUND, dl, ST->getMemoryVT(), Val,
1097 DAG.getIntPtrConstant(0, dl, /*isTarget=*/true)));
1098 else
1099 Val = GetSoftenedFloat(Val);
1100
1101 return DAG.getStore(ST->getChain(), dl, Val, ST->getBasePtr(),
1102 ST->getMemOperand());
1103}
1104
1105SDValue DAGTypeLegalizer::SoftenFloatOp_FCOPYSIGN(SDNode *N) {
1106 SDValue LHS = N->getOperand(0);
1107 SDValue RHS = BitConvertToInteger(N->getOperand(1));
1108 SDLoc dl(N);
1109
1110 EVT LVT = LHS.getValueType();
1111 EVT ILVT = EVT::getIntegerVT(*DAG.getContext(), LVT.getSizeInBits());
1112 EVT RVT = RHS.getValueType();
1113
1114 unsigned LSize = LVT.getSizeInBits();
1115 unsigned RSize = RVT.getSizeInBits();
1116
1117 // Shift right or sign-extend it if the two operands have different types.
1118 int SizeDiff = RSize - LSize;
1119 if (SizeDiff > 0) {
1120 RHS =
1121 DAG.getNode(ISD::SRL, dl, RVT, RHS,
1122 DAG.getConstant(SizeDiff, dl,
1123 TLI.getShiftAmountTy(RHS.getValueType(),
1124 DAG.getDataLayout())));
1125 RHS = DAG.getNode(ISD::TRUNCATE, dl, ILVT, RHS);
1126 } else if (SizeDiff < 0) {
1127 RHS = DAG.getNode(ISD::ANY_EXTEND, dl, LVT, RHS);
1128 RHS =
1129 DAG.getNode(ISD::SHL, dl, ILVT, RHS,
1130 DAG.getConstant(-SizeDiff, dl,
1131 TLI.getShiftAmountTy(RHS.getValueType(),
1132 DAG.getDataLayout())));
1133 }
1134
1135 RHS = DAG.getBitcast(LVT, RHS);
1136 return DAG.getNode(ISD::FCOPYSIGN, dl, LVT, LHS, RHS);
1137}
1138
1139SDValue DAGTypeLegalizer::SoftenFloatOp_Unary(SDNode *N, RTLIB::Libcall LC) {
1140 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
1141 bool IsStrict = N->isStrictFPOpcode();
1142 unsigned Offset = IsStrict ? 1 : 0;
1143 SDValue Op = GetSoftenedFloat(N->getOperand(0 + Offset));
1144 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
1146 EVT OpVT = N->getOperand(0 + Offset).getValueType();
1147 CallOptions.setTypeListBeforeSoften(OpVT, N->getValueType(0), true);
1148 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, NVT, Op,
1149 CallOptions, SDLoc(N),
1150 Chain);
1151 if (IsStrict) {
1152 ReplaceValueWith(SDValue(N, 1), Tmp.second);
1153 ReplaceValueWith(SDValue(N, 0), Tmp.first);
1154 return SDValue();
1155 }
1156
1157 return Tmp.first;
1158}
1159
1160SDValue DAGTypeLegalizer::SoftenFloatOp_LROUND(SDNode *N) {
1161 EVT OpVT = N->getOperand(N->isStrictFPOpcode() ? 1 : 0).getValueType();
1162 return SoftenFloatOp_Unary(N, GetFPLibCall(OpVT,
1163 RTLIB::LROUND_F32,
1164 RTLIB::LROUND_F64,
1165 RTLIB::LROUND_F80,
1166 RTLIB::LROUND_F128,
1167 RTLIB::LROUND_PPCF128));
1168}
1169
1170SDValue DAGTypeLegalizer::SoftenFloatOp_LLROUND(SDNode *N) {
1171 EVT OpVT = N->getOperand(N->isStrictFPOpcode() ? 1 : 0).getValueType();
1172 return SoftenFloatOp_Unary(N, GetFPLibCall(OpVT,
1173 RTLIB::LLROUND_F32,
1174 RTLIB::LLROUND_F64,
1175 RTLIB::LLROUND_F80,
1176 RTLIB::LLROUND_F128,
1177 RTLIB::LLROUND_PPCF128));
1178}
1179
1180SDValue DAGTypeLegalizer::SoftenFloatOp_LRINT(SDNode *N) {
1181 EVT OpVT = N->getOperand(N->isStrictFPOpcode() ? 1 : 0).getValueType();
1182 return SoftenFloatOp_Unary(N, GetFPLibCall(OpVT,
1183 RTLIB::LRINT_F32,
1184 RTLIB::LRINT_F64,
1185 RTLIB::LRINT_F80,
1186 RTLIB::LRINT_F128,
1187 RTLIB::LRINT_PPCF128));
1188}
1189
1190SDValue DAGTypeLegalizer::SoftenFloatOp_LLRINT(SDNode *N) {
1191 EVT OpVT = N->getOperand(N->isStrictFPOpcode() ? 1 : 0).getValueType();
1192 return SoftenFloatOp_Unary(N, GetFPLibCall(OpVT,
1193 RTLIB::LLRINT_F32,
1194 RTLIB::LLRINT_F64,
1195 RTLIB::LLRINT_F80,
1196 RTLIB::LLRINT_F128,
1197 RTLIB::LLRINT_PPCF128));
1198}
1199
1200//===----------------------------------------------------------------------===//
1201// Float Result Expansion
1202//===----------------------------------------------------------------------===//
1203
1204/// ExpandFloatResult - This method is called when the specified result of the
1205/// specified node is found to need expansion. At this point, the node may also
1206/// have invalid operands or may have other results that need promotion, we just
1207/// know that (at least) one result needs expansion.
1208void DAGTypeLegalizer::ExpandFloatResult(SDNode *N, unsigned ResNo) {
1209 LLVM_DEBUG(dbgs() << "Expand float result: "; N->dump(&DAG); dbgs() << "\n");
1210 SDValue Lo, Hi;
1211 Lo = Hi = SDValue();
1212
1213 // See if the target wants to custom expand this node.
1214 if (CustomLowerNode(N, N->getValueType(ResNo), true))
1215 return;
1216
1217 switch (N->getOpcode()) {
1218 default:
1219#ifndef NDEBUG
1220 dbgs() << "ExpandFloatResult #" << ResNo << ": ";
1221 N->dump(&DAG); dbgs() << "\n";
1222#endif
1223 llvm_unreachable("Do not know how to expand the result of this operator!");
1224
1225 case ISD::UNDEF: SplitRes_UNDEF(N, Lo, Hi); break;
1226 case ISD::SELECT: SplitRes_Select(N, Lo, Hi); break;
1227 case ISD::SELECT_CC: SplitRes_SELECT_CC(N, Lo, Hi); break;
1228
1229 case ISD::MERGE_VALUES: ExpandRes_MERGE_VALUES(N, ResNo, Lo, Hi); break;
1230 case ISD::BITCAST: ExpandRes_BITCAST(N, Lo, Hi); break;
1231 case ISD::BUILD_PAIR: ExpandRes_BUILD_PAIR(N, Lo, Hi); break;
1232 case ISD::EXTRACT_ELEMENT: ExpandRes_EXTRACT_ELEMENT(N, Lo, Hi); break;
1233 case ISD::EXTRACT_VECTOR_ELT: ExpandRes_EXTRACT_VECTOR_ELT(N, Lo, Hi); break;
1234 case ISD::VAARG: ExpandRes_VAARG(N, Lo, Hi); break;
1235
1236 case ISD::ConstantFP: ExpandFloatRes_ConstantFP(N, Lo, Hi); break;
1237 case ISD::FABS: ExpandFloatRes_FABS(N, Lo, Hi); break;
1239 case ISD::FMINNUM: ExpandFloatRes_FMINNUM(N, Lo, Hi); break;
1241 case ISD::FMAXNUM: ExpandFloatRes_FMAXNUM(N, Lo, Hi); break;
1242 case ISD::STRICT_FADD:
1243 case ISD::FADD: ExpandFloatRes_FADD(N, Lo, Hi); break;
1244 case ISD::FCBRT: ExpandFloatRes_FCBRT(N, Lo, Hi); break;
1245 case ISD::STRICT_FCEIL:
1246 case ISD::FCEIL: ExpandFloatRes_FCEIL(N, Lo, Hi); break;
1247 case ISD::FCOPYSIGN: ExpandFloatRes_FCOPYSIGN(N, Lo, Hi); break;
1248 case ISD::STRICT_FCOS:
1249 case ISD::FCOS: ExpandFloatRes_FCOS(N, Lo, Hi); break;
1250 case ISD::STRICT_FDIV:
1251 case ISD::FDIV: ExpandFloatRes_FDIV(N, Lo, Hi); break;
1252 case ISD::STRICT_FEXP:
1253 case ISD::FEXP: ExpandFloatRes_FEXP(N, Lo, Hi); break;
1254 case ISD::STRICT_FEXP2:
1255 case ISD::FEXP2: ExpandFloatRes_FEXP2(N, Lo, Hi); break;
1256 case ISD::STRICT_FFLOOR:
1257 case ISD::FFLOOR: ExpandFloatRes_FFLOOR(N, Lo, Hi); break;
1258 case ISD::STRICT_FLOG:
1259 case ISD::FLOG: ExpandFloatRes_FLOG(N, Lo, Hi); break;
1260 case ISD::STRICT_FLOG2:
1261 case ISD::FLOG2: ExpandFloatRes_FLOG2(N, Lo, Hi); break;
1262 case ISD::STRICT_FLOG10:
1263 case ISD::FLOG10: ExpandFloatRes_FLOG10(N, Lo, Hi); break;
1264 case ISD::STRICT_FMA:
1265 case ISD::FMA: ExpandFloatRes_FMA(N, Lo, Hi); break;
1266 case ISD::STRICT_FMUL:
1267 case ISD::FMUL: ExpandFloatRes_FMUL(N, Lo, Hi); break;
1269 case ISD::FNEARBYINT: ExpandFloatRes_FNEARBYINT(N, Lo, Hi); break;
1270 case ISD::FNEG: ExpandFloatRes_FNEG(N, Lo, Hi); break;
1272 case ISD::FP_EXTEND: ExpandFloatRes_FP_EXTEND(N, Lo, Hi); break;
1273 case ISD::STRICT_FPOW:
1274 case ISD::FPOW: ExpandFloatRes_FPOW(N, Lo, Hi); break;
1275 case ISD::STRICT_FPOWI:
1276 case ISD::FPOWI: ExpandFloatRes_FPOWI(N, Lo, Hi); break;
1277 case ISD::FREEZE: ExpandFloatRes_FREEZE(N, Lo, Hi); break;
1278 case ISD::STRICT_FRINT:
1279 case ISD::FRINT: ExpandFloatRes_FRINT(N, Lo, Hi); break;
1280 case ISD::STRICT_FROUND:
1281 case ISD::FROUND: ExpandFloatRes_FROUND(N, Lo, Hi); break;
1283 case ISD::FROUNDEVEN: ExpandFloatRes_FROUNDEVEN(N, Lo, Hi); break;
1284 case ISD::STRICT_FSIN:
1285 case ISD::FSIN: ExpandFloatRes_FSIN(N, Lo, Hi); break;
1286 case ISD::STRICT_FSQRT:
1287 case ISD::FSQRT: ExpandFloatRes_FSQRT(N, Lo, Hi); break;
1288 case ISD::STRICT_FSUB:
1289 case ISD::FSUB: ExpandFloatRes_FSUB(N, Lo, Hi); break;
1290 case ISD::STRICT_FTRUNC:
1291 case ISD::FTRUNC: ExpandFloatRes_FTRUNC(N, Lo, Hi); break;
1292 case ISD::LOAD: ExpandFloatRes_LOAD(N, Lo, Hi); break;
1295 case ISD::SINT_TO_FP:
1296 case ISD::UINT_TO_FP: ExpandFloatRes_XINT_TO_FP(N, Lo, Hi); break;
1297 case ISD::STRICT_FREM:
1298 case ISD::FREM: ExpandFloatRes_FREM(N, Lo, Hi); break;
1299 }
1300
1301 // If Lo/Hi is null, the sub-method took care of registering results etc.
1302 if (Lo.getNode())
1303 SetExpandedFloat(SDValue(N, ResNo), Lo, Hi);
1304}
1305
1306void DAGTypeLegalizer::ExpandFloatRes_ConstantFP(SDNode *N, SDValue &Lo,
1307 SDValue &Hi) {
1308 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
1309 assert(NVT.getSizeInBits() == 64 &&
1310 "Do not know how to expand this float constant!");
1311 APInt C = cast<ConstantFPSDNode>(N)->getValueAPF().bitcastToAPInt();
1312 SDLoc dl(N);
1314 APInt(64, C.getRawData()[1])),
1315 dl, NVT);
1317 APInt(64, C.getRawData()[0])),
1318 dl, NVT);
1319}
1320
1321void DAGTypeLegalizer::ExpandFloatRes_Unary(SDNode *N, RTLIB::Libcall LC,
1322 SDValue &Lo, SDValue &Hi) {
1323 bool IsStrict = N->isStrictFPOpcode();
1324 unsigned Offset = IsStrict ? 1 : 0;
1325 SDValue Op = N->getOperand(0 + Offset);
1326 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
1328 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, N->getValueType(0),
1329 Op, CallOptions, SDLoc(N),
1330 Chain);
1331 if (IsStrict)
1332 ReplaceValueWith(SDValue(N, 1), Tmp.second);
1333 GetPairElements(Tmp.first, Lo, Hi);
1334}
1335
1336void DAGTypeLegalizer::ExpandFloatRes_Binary(SDNode *N, RTLIB::Libcall LC,
1337 SDValue &Lo, SDValue &Hi) {
1338 bool IsStrict = N->isStrictFPOpcode();
1339 unsigned Offset = IsStrict ? 1 : 0;
1340 SDValue Ops[] = { N->getOperand(0 + Offset), N->getOperand(1 + Offset) };
1341 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
1343 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, N->getValueType(0),
1344 Ops, CallOptions, SDLoc(N),
1345 Chain);
1346 if (IsStrict)
1347 ReplaceValueWith(SDValue(N, 1), Tmp.second);
1348 GetPairElements(Tmp.first, Lo, Hi);
1349}
1350
1351void DAGTypeLegalizer::ExpandFloatRes_FABS(SDNode *N, SDValue &Lo,
1352 SDValue &Hi) {
1353 assert(N->getValueType(0) == MVT::ppcf128 &&
1354 "Logic only correct for ppcf128!");
1355 SDLoc dl(N);
1356 SDValue Tmp;
1357 GetExpandedFloat(N->getOperand(0), Lo, Tmp);
1358 Hi = DAG.getNode(ISD::FABS, dl, Tmp.getValueType(), Tmp);
1359 // Lo = Hi==fabs(Hi) ? Lo : -Lo;
1360 Lo = DAG.getSelectCC(dl, Tmp, Hi, Lo,
1361 DAG.getNode(ISD::FNEG, dl, Lo.getValueType(), Lo),
1362 ISD::SETEQ);
1363}
1364
1365void DAGTypeLegalizer::ExpandFloatRes_FMINNUM(SDNode *N, SDValue &Lo,
1366 SDValue &Hi) {
1367 ExpandFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
1368 RTLIB::FMIN_F32, RTLIB::FMIN_F64,
1369 RTLIB::FMIN_F80, RTLIB::FMIN_F128,
1370 RTLIB::FMIN_PPCF128), Lo, Hi);
1371}
1372
1373void DAGTypeLegalizer::ExpandFloatRes_FMAXNUM(SDNode *N, SDValue &Lo,
1374 SDValue &Hi) {
1375 ExpandFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
1376 RTLIB::FMAX_F32, RTLIB::FMAX_F64,
1377 RTLIB::FMAX_F80, RTLIB::FMAX_F128,
1378 RTLIB::FMAX_PPCF128), Lo, Hi);
1379}
1380
1381void DAGTypeLegalizer::ExpandFloatRes_FADD(SDNode *N, SDValue &Lo,
1382 SDValue &Hi) {
1383 ExpandFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
1384 RTLIB::ADD_F32, RTLIB::ADD_F64,
1385 RTLIB::ADD_F80, RTLIB::ADD_F128,
1386 RTLIB::ADD_PPCF128), Lo, Hi);
1387}
1388
1389void DAGTypeLegalizer::ExpandFloatRes_FCBRT(SDNode *N, SDValue &Lo,
1390 SDValue &Hi) {
1391 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0), RTLIB::CBRT_F32,
1392 RTLIB::CBRT_F64, RTLIB::CBRT_F80,
1393 RTLIB::CBRT_F128,
1394 RTLIB::CBRT_PPCF128), Lo, Hi);
1395}
1396
1397void DAGTypeLegalizer::ExpandFloatRes_FCEIL(SDNode *N,
1398 SDValue &Lo, SDValue &Hi) {
1399 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
1400 RTLIB::CEIL_F32, RTLIB::CEIL_F64,
1401 RTLIB::CEIL_F80, RTLIB::CEIL_F128,
1402 RTLIB::CEIL_PPCF128), Lo, Hi);
1403}
1404
1405void DAGTypeLegalizer::ExpandFloatRes_FCOPYSIGN(SDNode *N,
1406 SDValue &Lo, SDValue &Hi) {
1407 ExpandFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
1408 RTLIB::COPYSIGN_F32,
1409 RTLIB::COPYSIGN_F64,
1410 RTLIB::COPYSIGN_F80,
1411 RTLIB::COPYSIGN_F128,
1412 RTLIB::COPYSIGN_PPCF128), Lo, Hi);
1413}
1414
1415void DAGTypeLegalizer::ExpandFloatRes_FCOS(SDNode *N,
1416 SDValue &Lo, SDValue &Hi) {
1417 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
1418 RTLIB::COS_F32, RTLIB::COS_F64,
1419 RTLIB::COS_F80, RTLIB::COS_F128,
1420 RTLIB::COS_PPCF128), Lo, Hi);
1421}
1422
1423void DAGTypeLegalizer::ExpandFloatRes_FDIV(SDNode *N, SDValue &Lo,
1424 SDValue &Hi) {
1425 ExpandFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
1426 RTLIB::DIV_F32,
1427 RTLIB::DIV_F64,
1428 RTLIB::DIV_F80,
1429 RTLIB::DIV_F128,
1430 RTLIB::DIV_PPCF128), Lo, Hi);
1431}
1432
1433void DAGTypeLegalizer::ExpandFloatRes_FEXP(SDNode *N,
1434 SDValue &Lo, SDValue &Hi) {
1435 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
1436 RTLIB::EXP_F32, RTLIB::EXP_F64,
1437 RTLIB::EXP_F80, RTLIB::EXP_F128,
1438 RTLIB::EXP_PPCF128), Lo, Hi);
1439}
1440
1441void DAGTypeLegalizer::ExpandFloatRes_FEXP2(SDNode *N,
1442 SDValue &Lo, SDValue &Hi) {
1443 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
1444 RTLIB::EXP2_F32, RTLIB::EXP2_F64,
1445 RTLIB::EXP2_F80, RTLIB::EXP2_F128,
1446 RTLIB::EXP2_PPCF128), Lo, Hi);
1447}
1448
1449void DAGTypeLegalizer::ExpandFloatRes_FFLOOR(SDNode *N,
1450 SDValue &Lo, SDValue &Hi) {
1451 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
1452 RTLIB::FLOOR_F32, RTLIB::FLOOR_F64,
1453 RTLIB::FLOOR_F80, RTLIB::FLOOR_F128,
1454 RTLIB::FLOOR_PPCF128), Lo, Hi);
1455}
1456
1457void DAGTypeLegalizer::ExpandFloatRes_FLOG(SDNode *N,
1458 SDValue &Lo, SDValue &Hi) {
1459 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
1460 RTLIB::LOG_F32, RTLIB::LOG_F64,
1461 RTLIB::LOG_F80, RTLIB::LOG_F128,
1462 RTLIB::LOG_PPCF128), Lo, Hi);
1463}
1464
1465void DAGTypeLegalizer::ExpandFloatRes_FLOG2(SDNode *N,
1466 SDValue &Lo, SDValue &Hi) {
1467 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
1468 RTLIB::LOG2_F32, RTLIB::LOG2_F64,
1469 RTLIB::LOG2_F80, RTLIB::LOG2_F128,
1470 RTLIB::LOG2_PPCF128), Lo, Hi);
1471}
1472
1473void DAGTypeLegalizer::ExpandFloatRes_FLOG10(SDNode *N,
1474 SDValue &Lo, SDValue &Hi) {
1475 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
1476 RTLIB::LOG10_F32, RTLIB::LOG10_F64,
1477 RTLIB::LOG10_F80, RTLIB::LOG10_F128,
1478 RTLIB::LOG10_PPCF128), Lo, Hi);
1479}
1480
1481void DAGTypeLegalizer::ExpandFloatRes_FMA(SDNode *N, SDValue &Lo,
1482 SDValue &Hi) {
1483 bool IsStrict = N->isStrictFPOpcode();
1484 unsigned Offset = IsStrict ? 1 : 0;
1485 SDValue Ops[3] = { N->getOperand(0 + Offset), N->getOperand(1 + Offset),
1486 N->getOperand(2 + Offset) };
1487 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
1489 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, GetFPLibCall(N->getValueType(0),
1490 RTLIB::FMA_F32,
1491 RTLIB::FMA_F64,
1492 RTLIB::FMA_F80,
1493 RTLIB::FMA_F128,
1494 RTLIB::FMA_PPCF128),
1495 N->getValueType(0), Ops, CallOptions,
1496 SDLoc(N), Chain);
1497 if (IsStrict)
1498 ReplaceValueWith(SDValue(N, 1), Tmp.second);
1499 GetPairElements(Tmp.first, Lo, Hi);
1500}
1501
1502void DAGTypeLegalizer::ExpandFloatRes_FMUL(SDNode *N, SDValue &Lo,
1503 SDValue &Hi) {
1504 ExpandFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
1505 RTLIB::MUL_F32,
1506 RTLIB::MUL_F64,
1507 RTLIB::MUL_F80,
1508 RTLIB::MUL_F128,
1509 RTLIB::MUL_PPCF128), Lo, Hi);
1510}
1511
1512void DAGTypeLegalizer::ExpandFloatRes_FNEARBYINT(SDNode *N,
1513 SDValue &Lo, SDValue &Hi) {
1514 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
1515 RTLIB::NEARBYINT_F32,
1516 RTLIB::NEARBYINT_F64,
1517 RTLIB::NEARBYINT_F80,
1518 RTLIB::NEARBYINT_F128,
1519 RTLIB::NEARBYINT_PPCF128), Lo, Hi);
1520}
1521
1522void DAGTypeLegalizer::ExpandFloatRes_FNEG(SDNode *N, SDValue &Lo,
1523 SDValue &Hi) {
1524 SDLoc dl(N);
1525 GetExpandedFloat(N->getOperand(0), Lo, Hi);
1526 Lo = DAG.getNode(ISD::FNEG, dl, Lo.getValueType(), Lo);
1527 Hi = DAG.getNode(ISD::FNEG, dl, Hi.getValueType(), Hi);
1528}
1529
1530void DAGTypeLegalizer::ExpandFloatRes_FP_EXTEND(SDNode *N, SDValue &Lo,
1531 SDValue &Hi) {
1532 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
1533 SDLoc dl(N);
1534 bool IsStrict = N->isStrictFPOpcode();
1535
1536 SDValue Chain;
1537 if (IsStrict) {
1538 // If the expanded type is the same as the input type, just bypass the node.
1539 if (NVT == N->getOperand(1).getValueType()) {
1540 Hi = N->getOperand(1);
1541 Chain = N->getOperand(0);
1542 } else {
1543 // Other we need to extend.
1544 Hi = DAG.getNode(ISD::STRICT_FP_EXTEND, dl, { NVT, MVT::Other },
1545 { N->getOperand(0), N->getOperand(1) });
1546 Chain = Hi.getValue(1);
1547 }
1548 } else {
1549 Hi = DAG.getNode(ISD::FP_EXTEND, dl, NVT, N->getOperand(0));
1550 }
1551
1553 APInt(NVT.getSizeInBits(), 0)), dl, NVT);
1554
1555 if (IsStrict)
1556 ReplaceValueWith(SDValue(N, 1), Chain);
1557}
1558
1559void DAGTypeLegalizer::ExpandFloatRes_FPOW(SDNode *N,
1560 SDValue &Lo, SDValue &Hi) {
1561 ExpandFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
1562 RTLIB::POW_F32, RTLIB::POW_F64,
1563 RTLIB::POW_F80, RTLIB::POW_F128,
1564 RTLIB::POW_PPCF128), Lo, Hi);
1565}
1566
1567void DAGTypeLegalizer::ExpandFloatRes_FPOWI(SDNode *N,
1568 SDValue &Lo, SDValue &Hi) {
1569 ExpandFloatRes_Binary(N, RTLIB::getPOWI(N->getValueType(0)), Lo, Hi);
1570}
1571
1572void DAGTypeLegalizer::ExpandFloatRes_FREEZE(SDNode *N,
1573 SDValue &Lo, SDValue &Hi) {
1574 assert(N->getValueType(0) == MVT::ppcf128 &&
1575 "Logic only correct for ppcf128!");
1576
1577 SDLoc dl(N);
1578 GetExpandedFloat(N->getOperand(0), Lo, Hi);
1579 Lo = DAG.getNode(ISD::FREEZE, dl, Lo.getValueType(), Lo);
1580 Hi = DAG.getNode(ISD::FREEZE, dl, Hi.getValueType(), Hi);
1581}
1582
1583void DAGTypeLegalizer::ExpandFloatRes_FREM(SDNode *N,
1584 SDValue &Lo, SDValue &Hi) {
1585 ExpandFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
1586 RTLIB::REM_F32, RTLIB::REM_F64,
1587 RTLIB::REM_F80, RTLIB::REM_F128,
1588 RTLIB::REM_PPCF128), Lo, Hi);
1589}
1590
1591void DAGTypeLegalizer::ExpandFloatRes_FRINT(SDNode *N,
1592 SDValue &Lo, SDValue &Hi) {
1593 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
1594 RTLIB::RINT_F32, RTLIB::RINT_F64,
1595 RTLIB::RINT_F80, RTLIB::RINT_F128,
1596 RTLIB::RINT_PPCF128), Lo, Hi);
1597}
1598
1599void DAGTypeLegalizer::ExpandFloatRes_FROUND(SDNode *N,
1600 SDValue &Lo, SDValue &Hi) {
1601 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
1602 RTLIB::ROUND_F32,
1603 RTLIB::ROUND_F64,
1604 RTLIB::ROUND_F80,
1605 RTLIB::ROUND_F128,
1606 RTLIB::ROUND_PPCF128), Lo, Hi);
1607}
1608
1609void DAGTypeLegalizer::ExpandFloatRes_FROUNDEVEN(SDNode *N,
1610 SDValue &Lo, SDValue &Hi) {
1611 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
1612 RTLIB::ROUNDEVEN_F32,
1613 RTLIB::ROUNDEVEN_F64,
1614 RTLIB::ROUNDEVEN_F80,
1615 RTLIB::ROUNDEVEN_F128,
1616 RTLIB::ROUNDEVEN_PPCF128), Lo, Hi);
1617}
1618
1619void DAGTypeLegalizer::ExpandFloatRes_FSIN(SDNode *N,
1620 SDValue &Lo, SDValue &Hi) {
1621 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
1622 RTLIB::SIN_F32, RTLIB::SIN_F64,
1623 RTLIB::SIN_F80, RTLIB::SIN_F128,
1624 RTLIB::SIN_PPCF128), Lo, Hi);
1625}
1626
1627void DAGTypeLegalizer::ExpandFloatRes_FSQRT(SDNode *N,
1628 SDValue &Lo, SDValue &Hi) {
1629 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
1630 RTLIB::SQRT_F32, RTLIB::SQRT_F64,
1631 RTLIB::SQRT_F80, RTLIB::SQRT_F128,
1632 RTLIB::SQRT_PPCF128), Lo, Hi);
1633}
1634
1635void DAGTypeLegalizer::ExpandFloatRes_FSUB(SDNode *N, SDValue &Lo,
1636 SDValue &Hi) {
1637 ExpandFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
1638 RTLIB::SUB_F32,
1639 RTLIB::SUB_F64,
1640 RTLIB::SUB_F80,
1641 RTLIB::SUB_F128,
1642 RTLIB::SUB_PPCF128), Lo, Hi);
1643}
1644
1645void DAGTypeLegalizer::ExpandFloatRes_FTRUNC(SDNode *N,
1646 SDValue &Lo, SDValue &Hi) {
1647 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
1648 RTLIB::TRUNC_F32, RTLIB::TRUNC_F64,
1649 RTLIB::TRUNC_F80, RTLIB::TRUNC_F128,
1650 RTLIB::TRUNC_PPCF128), Lo, Hi);
1651}
1652
1653void DAGTypeLegalizer::ExpandFloatRes_LOAD(SDNode *N, SDValue &Lo,
1654 SDValue &Hi) {
1655 if (ISD::isNormalLoad(N)) {
1656 ExpandRes_NormalLoad(N, Lo, Hi);
1657 return;
1658 }
1659
1660 assert(ISD::isUNINDEXEDLoad(N) && "Indexed load during type legalization!");
1661 LoadSDNode *LD = cast<LoadSDNode>(N);
1662 SDValue Chain = LD->getChain();
1663 SDValue Ptr = LD->getBasePtr();
1664 SDLoc dl(N);
1665
1666 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), LD->getValueType(0));
1667 assert(NVT.isByteSized() && "Expanded type not byte sized!");
1668 assert(LD->getMemoryVT().bitsLE(NVT) && "Float type not round?");
1669
1670 Hi = DAG.getExtLoad(LD->getExtensionType(), dl, NVT, Chain, Ptr,
1671 LD->getMemoryVT(), LD->getMemOperand());
1672
1673 // Remember the chain.
1674 Chain = Hi.getValue(1);
1675
1676 // The low part is zero.
1678 APInt(NVT.getSizeInBits(), 0)), dl, NVT);
1679
1680 // Modified the chain - switch anything that used the old chain to use the
1681 // new one.
1682 ReplaceValueWith(SDValue(LD, 1), Chain);
1683}
1684
1685void DAGTypeLegalizer::ExpandFloatRes_XINT_TO_FP(SDNode *N, SDValue &Lo,
1686 SDValue &Hi) {
1687 assert(N->getValueType(0) == MVT::ppcf128 && "Unsupported XINT_TO_FP!");
1688 EVT VT = N->getValueType(0);
1689 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
1690 bool Strict = N->isStrictFPOpcode();
1691 SDValue Src = N->getOperand(Strict ? 1 : 0);
1692 EVT SrcVT = Src.getValueType();
1693 bool isSigned = N->getOpcode() == ISD::SINT_TO_FP ||
1694 N->getOpcode() == ISD::STRICT_SINT_TO_FP;
1695 SDLoc dl(N);
1696 SDValue Chain = Strict ? N->getOperand(0) : DAG.getEntryNode();
1697
1698 // TODO: Any other flags to propagate?
1700 Flags.setNoFPExcept(N->getFlags().hasNoFPExcept());
1701
1702 // First do an SINT_TO_FP, whether the original was signed or unsigned.
1703 // When promoting partial word types to i32 we must honor the signedness,
1704 // though.
1705 if (SrcVT.bitsLE(MVT::i32)) {
1706 // The integer can be represented exactly in an f64.
1708 APInt(NVT.getSizeInBits(), 0)), dl, NVT);
1709 if (Strict) {
1710 Hi = DAG.getNode(N->getOpcode(), dl, DAG.getVTList(NVT, MVT::Other),
1711 {Chain, Src}, Flags);
1712 Chain = Hi.getValue(1);
1713 } else
1714 Hi = DAG.getNode(N->getOpcode(), dl, NVT, Src);
1715 } else {
1716 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
1717 if (SrcVT.bitsLE(MVT::i64)) {
1719 MVT::i64, Src);
1720 LC = RTLIB::SINTTOFP_I64_PPCF128;
1721 } else if (SrcVT.bitsLE(MVT::i128)) {
1722 Src = DAG.getNode(ISD::SIGN_EXTEND, dl, MVT::i128, Src);
1723 LC = RTLIB::SINTTOFP_I128_PPCF128;
1724 }
1725 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported XINT_TO_FP!");
1726
1728 CallOptions.setSExt(true);
1729 std::pair<SDValue, SDValue> Tmp =
1730 TLI.makeLibCall(DAG, LC, VT, Src, CallOptions, dl, Chain);
1731 if (Strict)
1732 Chain = Tmp.second;
1733 GetPairElements(Tmp.first, Lo, Hi);
1734 }
1735
1736 // No need to complement for unsigned 32-bit integers
1737 if (isSigned || SrcVT.bitsLE(MVT::i32)) {
1738 if (Strict)
1739 ReplaceValueWith(SDValue(N, 1), Chain);
1740
1741 return;
1742 }
1743
1744 // Unsigned - fix up the SINT_TO_FP value just calculated.
1745 // FIXME: For unsigned i128 to ppc_fp128 conversion, we need to carefully
1746 // keep semantics correctness if the integer is not exactly representable
1747 // here. See ExpandLegalINT_TO_FP.
1748 Hi = DAG.getNode(ISD::BUILD_PAIR, dl, VT, Lo, Hi);
1749 SrcVT = Src.getValueType();
1750
1751 // x>=0 ? (ppcf128)(iN)x : (ppcf128)(iN)x + 2^N; N=32,64,128.
1752 static const uint64_t TwoE32[] = { 0x41f0000000000000LL, 0 };
1753 static const uint64_t TwoE64[] = { 0x43f0000000000000LL, 0 };
1754 static const uint64_t TwoE128[] = { 0x47f0000000000000LL, 0 };
1755 ArrayRef<uint64_t> Parts;
1756
1757 switch (SrcVT.getSimpleVT().SimpleTy) {
1758 default:
1759 llvm_unreachable("Unsupported UINT_TO_FP!");
1760 case MVT::i32:
1761 Parts = TwoE32;
1762 break;
1763 case MVT::i64:
1764 Parts = TwoE64;
1765 break;
1766 case MVT::i128:
1767 Parts = TwoE128;
1768 break;
1769 }
1770
1771 // TODO: Are there other fast-math-flags to propagate to this FADD?
1772 SDValue NewLo = DAG.getConstantFP(
1773 APFloat(APFloat::PPCDoubleDouble(), APInt(128, Parts)), dl, MVT::ppcf128);
1774 if (Strict) {
1775 Lo = DAG.getNode(ISD::STRICT_FADD, dl, DAG.getVTList(VT, MVT::Other),
1776 {Chain, Hi, NewLo}, Flags);
1777 Chain = Lo.getValue(1);
1778 ReplaceValueWith(SDValue(N, 1), Chain);
1779 } else
1780 Lo = DAG.getNode(ISD::FADD, dl, VT, Hi, NewLo);
1781 Lo = DAG.getSelectCC(dl, Src, DAG.getConstant(0, dl, SrcVT),
1782 Lo, Hi, ISD::SETLT);
1783 GetPairElements(Lo, Lo, Hi);
1784}
1785
1786
1787//===----------------------------------------------------------------------===//
1788// Float Operand Expansion
1789//===----------------------------------------------------------------------===//
1790
1791/// ExpandFloatOperand - This method is called when the specified operand of the
1792/// specified node is found to need expansion. At this point, all of the result
1793/// types of the node are known to be legal, but other operands of the node may
1794/// need promotion or expansion as well as the specified one.
1795bool DAGTypeLegalizer::ExpandFloatOperand(SDNode *N, unsigned OpNo) {
1796 LLVM_DEBUG(dbgs() << "Expand float operand: "; N->dump(&DAG); dbgs() << "\n");
1797 SDValue Res = SDValue();
1798
1799 // See if the target wants to custom expand this node.
1800 if (CustomLowerNode(N, N->getOperand(OpNo).getValueType(), false))
1801 return false;
1802
1803 switch (N->getOpcode()) {
1804 default:
1805#ifndef NDEBUG
1806 dbgs() << "ExpandFloatOperand Op #" << OpNo << ": ";
1807 N->dump(&DAG); dbgs() << "\n";
1808#endif
1809 llvm_unreachable("Do not know how to expand this operator's operand!");
1810
1811 case ISD::BITCAST: Res = ExpandOp_BITCAST(N); break;
1812 case ISD::BUILD_VECTOR: Res = ExpandOp_BUILD_VECTOR(N); break;
1813 case ISD::EXTRACT_ELEMENT: Res = ExpandOp_EXTRACT_ELEMENT(N); break;
1814
1815 case ISD::BR_CC: Res = ExpandFloatOp_BR_CC(N); break;
1816 case ISD::FCOPYSIGN: Res = ExpandFloatOp_FCOPYSIGN(N); break;
1818 case ISD::FP_ROUND: Res = ExpandFloatOp_FP_ROUND(N); break;
1821 case ISD::FP_TO_SINT:
1822 case ISD::FP_TO_UINT: Res = ExpandFloatOp_FP_TO_XINT(N); break;
1823 case ISD::LROUND: Res = ExpandFloatOp_LROUND(N); break;
1824 case ISD::LLROUND: Res = ExpandFloatOp_LLROUND(N); break;
1825 case ISD::LRINT: Res = ExpandFloatOp_LRINT(N); break;
1826 case ISD::LLRINT: Res = ExpandFloatOp_LLRINT(N); break;
1827 case ISD::SELECT_CC: Res = ExpandFloatOp_SELECT_CC(N); break;
1828 case ISD::STRICT_FSETCC:
1830 case ISD::SETCC: Res = ExpandFloatOp_SETCC(N); break;
1831 case ISD::STORE: Res = ExpandFloatOp_STORE(cast<StoreSDNode>(N),
1832 OpNo); break;
1833 }
1834
1835 // If the result is null, the sub-method took care of registering results etc.
1836 if (!Res.getNode()) return false;
1837
1838 // If the result is N, the sub-method updated N in place. Tell the legalizer
1839 // core about this.
1840 if (Res.getNode() == N)
1841 return true;
1842
1843 assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 &&
1844 "Invalid operand expansion");
1845
1846 ReplaceValueWith(SDValue(N, 0), Res);
1847 return false;
1848}
1849
1850/// FloatExpandSetCCOperands - Expand the operands of a comparison. This code
1851/// is shared among BR_CC, SELECT_CC, and SETCC handlers.
1852void DAGTypeLegalizer::FloatExpandSetCCOperands(SDValue &NewLHS,
1853 SDValue &NewRHS,
1854 ISD::CondCode &CCCode,
1855 const SDLoc &dl, SDValue &Chain,
1856 bool IsSignaling) {
1857 SDValue LHSLo, LHSHi, RHSLo, RHSHi;
1858 GetExpandedFloat(NewLHS, LHSLo, LHSHi);
1859 GetExpandedFloat(NewRHS, RHSLo, RHSHi);
1860
1861 assert(NewLHS.getValueType() == MVT::ppcf128 && "Unsupported setcc type!");
1862
1863 // FIXME: This generated code sucks. We want to generate
1864 // FCMPU crN, hi1, hi2
1865 // BNE crN, L:
1866 // FCMPU crN, lo1, lo2
1867 // The following can be improved, but not that much.
1868 SDValue Tmp1, Tmp2, Tmp3, OutputChain;
1869 Tmp1 = DAG.getSetCC(dl, getSetCCResultType(LHSHi.getValueType()), LHSHi,
1870 RHSHi, ISD::SETOEQ, Chain, IsSignaling);
1871 OutputChain = Tmp1->getNumValues() > 1 ? Tmp1.getValue(1) : SDValue();
1872 Tmp2 = DAG.getSetCC(dl, getSetCCResultType(LHSLo.getValueType()), LHSLo,
1873 RHSLo, CCCode, OutputChain, IsSignaling);
1874 OutputChain = Tmp2->getNumValues() > 1 ? Tmp2.getValue(1) : SDValue();
1875 Tmp3 = DAG.getNode(ISD::AND, dl, Tmp1.getValueType(), Tmp1, Tmp2);
1876 Tmp1 =
1877 DAG.getSetCC(dl, getSetCCResultType(LHSHi.getValueType()), LHSHi, RHSHi,
1878 ISD::SETUNE, OutputChain, IsSignaling);
1879 OutputChain = Tmp1->getNumValues() > 1 ? Tmp1.getValue(1) : SDValue();
1880 Tmp2 = DAG.getSetCC(dl, getSetCCResultType(LHSHi.getValueType()), LHSHi,
1881 RHSHi, CCCode, OutputChain, IsSignaling);
1882 OutputChain = Tmp2->getNumValues() > 1 ? Tmp2.getValue(1) : SDValue();
1883 Tmp1 = DAG.getNode(ISD::AND, dl, Tmp1.getValueType(), Tmp1, Tmp2);
1884 NewLHS = DAG.getNode(ISD::OR, dl, Tmp1.getValueType(), Tmp1, Tmp3);
1885 NewRHS = SDValue(); // LHS is the result, not a compare.
1886 Chain = OutputChain;
1887}
1888
1889SDValue DAGTypeLegalizer::ExpandFloatOp_BR_CC(SDNode *N) {
1890 SDValue NewLHS = N->getOperand(2), NewRHS = N->getOperand(3);
1891 ISD::CondCode CCCode = cast<CondCodeSDNode>(N->getOperand(1))->get();
1892 SDValue Chain;
1893 FloatExpandSetCCOperands(NewLHS, NewRHS, CCCode, SDLoc(N), Chain);
1894
1895 // If ExpandSetCCOperands returned a scalar, we need to compare the result
1896 // against zero to select between true and false values.
1897 if (!NewRHS.getNode()) {
1898 NewRHS = DAG.getConstant(0, SDLoc(N), NewLHS.getValueType());
1899 CCCode = ISD::SETNE;
1900 }
1901
1902 // Update N to have the operands specified.
1903 return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0),
1904 DAG.getCondCode(CCCode), NewLHS, NewRHS,
1905 N->getOperand(4)), 0);
1906}
1907
1908SDValue DAGTypeLegalizer::ExpandFloatOp_FCOPYSIGN(SDNode *N) {
1909 assert(N->getOperand(1).getValueType() == MVT::ppcf128 &&
1910 "Logic only correct for ppcf128!");
1911 SDValue Lo, Hi;
1912 GetExpandedFloat(N->getOperand(1), Lo, Hi);
1913 // The ppcf128 value is providing only the sign; take it from the
1914 // higher-order double (which must have the larger magnitude).
1915 return DAG.getNode(ISD::FCOPYSIGN, SDLoc(N),
1916 N->getValueType(0), N->getOperand(0), Hi);
1917}
1918
1919SDValue DAGTypeLegalizer::ExpandFloatOp_FP_ROUND(SDNode *N) {
1920 bool IsStrict = N->isStrictFPOpcode();
1921 assert(N->getOperand(IsStrict ? 1 : 0).getValueType() == MVT::ppcf128 &&
1922 "Logic only correct for ppcf128!");
1923 SDValue Lo, Hi;
1924 GetExpandedFloat(N->getOperand(IsStrict ? 1 : 0), Lo, Hi);
1925
1926 if (!IsStrict)
1927 // Round it the rest of the way (e.g. to f32) if needed.
1928 return DAG.getNode(ISD::FP_ROUND, SDLoc(N),
1929 N->getValueType(0), Hi, N->getOperand(1));
1930
1931 // Eliminate the node if the input float type is the same as the output float
1932 // type.
1933 if (Hi.getValueType() == N->getValueType(0)) {
1934 // Connect the output chain to the input chain, unlinking the node.
1935 ReplaceValueWith(SDValue(N, 1), N->getOperand(0));
1936 ReplaceValueWith(SDValue(N, 0), Hi);
1937 return SDValue();
1938 }
1939
1940 SDValue Expansion = DAG.getNode(ISD::STRICT_FP_ROUND, SDLoc(N),
1941 {N->getValueType(0), MVT::Other},
1942 {N->getOperand(0), Hi, N->getOperand(2)});
1943 ReplaceValueWith(SDValue(N, 1), Expansion.getValue(1));
1944 ReplaceValueWith(SDValue(N, 0), Expansion);
1945 return SDValue();
1946}
1947
1948SDValue DAGTypeLegalizer::ExpandFloatOp_FP_TO_XINT(SDNode *N) {
1949 EVT RVT = N->getValueType(0);
1950 SDLoc dl(N);
1951
1952 bool IsStrict = N->isStrictFPOpcode();
1953 bool Signed = N->getOpcode() == ISD::FP_TO_SINT ||
1954 N->getOpcode() == ISD::STRICT_FP_TO_SINT;
1955 SDValue Op = N->getOperand(IsStrict ? 1 : 0);
1956 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
1957
1958 EVT NVT;
1959 RTLIB::Libcall LC = findFPToIntLibcall(Op.getValueType(), RVT, NVT, Signed);
1960 assert(LC != RTLIB::UNKNOWN_LIBCALL && NVT.isSimple() &&
1961 "Unsupported FP_TO_XINT!");
1963 std::pair<SDValue, SDValue> Tmp =
1964 TLI.makeLibCall(DAG, LC, NVT, Op, CallOptions, dl, Chain);
1965 if (!IsStrict)
1966 return Tmp.first;
1967
1968 ReplaceValueWith(SDValue(N, 1), Tmp.second);
1969 ReplaceValueWith(SDValue(N, 0), Tmp.first);
1970 return SDValue();
1971}
1972
1973SDValue DAGTypeLegalizer::ExpandFloatOp_SELECT_CC(SDNode *N) {
1974 SDValue NewLHS = N->getOperand(0), NewRHS = N->getOperand(1);
1975 ISD::CondCode CCCode = cast<CondCodeSDNode>(N->getOperand(4))->get();
1976 SDValue Chain;
1977 FloatExpandSetCCOperands(NewLHS, NewRHS, CCCode, SDLoc(N), Chain);
1978
1979 // If ExpandSetCCOperands returned a scalar, we need to compare the result
1980 // against zero to select between true and false values.
1981 if (!NewRHS.getNode()) {
1982 NewRHS = DAG.getConstant(0, SDLoc(N), NewLHS.getValueType());
1983 CCCode = ISD::SETNE;
1984 }
1985
1986 // Update N to have the operands specified.
1987 return SDValue(DAG.UpdateNodeOperands(N, NewLHS, NewRHS,
1988 N->getOperand(2), N->getOperand(3),
1989 DAG.getCondCode(CCCode)), 0);
1990}
1991
1992SDValue DAGTypeLegalizer::ExpandFloatOp_SETCC(SDNode *N) {
1993 bool IsStrict = N->isStrictFPOpcode();
1994 SDValue NewLHS = N->getOperand(IsStrict ? 1 : 0);
1995 SDValue NewRHS = N->getOperand(IsStrict ? 2 : 1);
1996 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
1997 ISD::CondCode CCCode =
1998 cast<CondCodeSDNode>(N->getOperand(IsStrict ? 3 : 2))->get();
1999 FloatExpandSetCCOperands(NewLHS, NewRHS, CCCode, SDLoc(N), Chain,
2000 N->getOpcode() == ISD::STRICT_FSETCCS);
2001
2002 // FloatExpandSetCCOperands always returned a scalar.
2003 assert(!NewRHS.getNode() && "Expect to return scalar");
2004 assert(NewLHS.getValueType() == N->getValueType(0) &&
2005 "Unexpected setcc expansion!");
2006 if (Chain) {
2007 ReplaceValueWith(SDValue(N, 0), NewLHS);
2008 ReplaceValueWith(SDValue(N, 1), Chain);
2009 return SDValue();
2010 }
2011 return NewLHS;
2012}
2013
2014SDValue DAGTypeLegalizer::ExpandFloatOp_STORE(SDNode *N, unsigned OpNo) {
2015 if (ISD::isNormalStore(N))
2016 return ExpandOp_NormalStore(N, OpNo);
2017
2018 assert(ISD::isUNINDEXEDStore(N) && "Indexed store during type legalization!");
2019 assert(OpNo == 1 && "Can only expand the stored value so far");
2020 StoreSDNode *ST = cast<StoreSDNode>(N);
2021
2022 SDValue Chain = ST->getChain();
2023 SDValue Ptr = ST->getBasePtr();
2024
2025 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(),
2026 ST->getValue().getValueType());
2027 assert(NVT.isByteSized() && "Expanded type not byte sized!");
2028 assert(ST->getMemoryVT().bitsLE(NVT) && "Float type not round?");
2029 (void)NVT;
2030
2031 SDValue Lo, Hi;
2032 GetExpandedOp(ST->getValue(), Lo, Hi);
2033
2034 return DAG.getTruncStore(Chain, SDLoc(N), Hi, Ptr,
2035 ST->getMemoryVT(), ST->getMemOperand());
2036}
2037
2038SDValue DAGTypeLegalizer::ExpandFloatOp_LROUND(SDNode *N) {
2039 EVT RVT = N->getValueType(0);
2040 EVT RetVT = N->getOperand(0).getValueType();
2042 return TLI.makeLibCall(DAG, GetFPLibCall(RetVT,
2043 RTLIB::LROUND_F32,
2044 RTLIB::LROUND_F64,
2045 RTLIB::LROUND_F80,
2046 RTLIB::LROUND_F128,
2047 RTLIB::LROUND_PPCF128),
2048 RVT, N->getOperand(0), CallOptions, SDLoc(N)).first;
2049}
2050
2051SDValue DAGTypeLegalizer::ExpandFloatOp_LLROUND(SDNode *N) {
2052 EVT RVT = N->getValueType(0);
2053 EVT RetVT = N->getOperand(0).getValueType();
2055 return TLI.makeLibCall(DAG, GetFPLibCall(RetVT,
2056 RTLIB::LLROUND_F32,
2057 RTLIB::LLROUND_F64,
2058 RTLIB::LLROUND_F80,
2059 RTLIB::LLROUND_F128,
2060 RTLIB::LLROUND_PPCF128),
2061 RVT, N->getOperand(0), CallOptions, SDLoc(N)).first;
2062}
2063
2064SDValue DAGTypeLegalizer::ExpandFloatOp_LRINT(SDNode *N) {
2065 EVT RVT = N->getValueType(0);
2066 EVT RetVT = N->getOperand(0).getValueType();
2068 return TLI.makeLibCall(DAG, GetFPLibCall(RetVT,
2069 RTLIB::LRINT_F32,
2070 RTLIB::LRINT_F64,
2071 RTLIB::LRINT_F80,
2072 RTLIB::LRINT_F128,
2073 RTLIB::LRINT_PPCF128),
2074 RVT, N->getOperand(0), CallOptions, SDLoc(N)).first;
2075}
2076
2077SDValue DAGTypeLegalizer::ExpandFloatOp_LLRINT(SDNode *N) {
2078 EVT RVT = N->getValueType(0);
2079 EVT RetVT = N->getOperand(0).getValueType();
2081 return TLI.makeLibCall(DAG, GetFPLibCall(RetVT,
2082 RTLIB::LLRINT_F32,
2083 RTLIB::LLRINT_F64,
2084 RTLIB::LLRINT_F80,
2085 RTLIB::LLRINT_F128,
2086 RTLIB::LLRINT_PPCF128),
2087 RVT, N->getOperand(0), CallOptions, SDLoc(N)).first;
2088}
2089
2090//===----------------------------------------------------------------------===//
2091// Float Operand Promotion
2092//===----------------------------------------------------------------------===//
2093//
2094
2096 if (OpVT == MVT::f16) {
2097 return ISD::FP16_TO_FP;
2098 } else if (RetVT == MVT::f16) {
2099 return ISD::FP_TO_FP16;
2100 } else if (OpVT == MVT::bf16) {
2101 return ISD::BF16_TO_FP;
2102 } else if (RetVT == MVT::bf16) {
2103 return ISD::FP_TO_BF16;
2104 }
2105
2106 report_fatal_error("Attempt at an invalid promotion-related conversion");
2107}
2108
2109bool DAGTypeLegalizer::PromoteFloatOperand(SDNode *N, unsigned OpNo) {
2110 LLVM_DEBUG(dbgs() << "Promote float operand " << OpNo << ": "; N->dump(&DAG);
2111 dbgs() << "\n");
2112 SDValue R = SDValue();
2113
2114 if (CustomLowerNode(N, N->getOperand(OpNo).getValueType(), false)) {
2115 LLVM_DEBUG(dbgs() << "Node has been custom lowered, done\n");
2116 return false;
2117 }
2118
2119 // Nodes that use a promotion-requiring floating point operand, but doesn't
2120 // produce a promotion-requiring floating point result, need to be legalized
2121 // to use the promoted float operand. Nodes that produce at least one
2122 // promotion-requiring floating point result have their operands legalized as
2123 // a part of PromoteFloatResult.
2124 switch (N->getOpcode()) {
2125 default:
2126 #ifndef NDEBUG
2127 dbgs() << "PromoteFloatOperand Op #" << OpNo << ": ";
2128 N->dump(&DAG); dbgs() << "\n";
2129 #endif
2130 llvm_unreachable("Do not know how to promote this operator's operand!");
2131
2132 case ISD::BITCAST: R = PromoteFloatOp_BITCAST(N, OpNo); break;
2133 case ISD::FCOPYSIGN: R = PromoteFloatOp_FCOPYSIGN(N, OpNo); break;
2134 case ISD::FP_TO_SINT:
2135 case ISD::FP_TO_UINT: R = PromoteFloatOp_FP_TO_XINT(N, OpNo); break;
2138 R = PromoteFloatOp_FP_TO_XINT_SAT(N, OpNo); break;
2139 case ISD::FP_EXTEND: R = PromoteFloatOp_FP_EXTEND(N, OpNo); break;
2140 case ISD::SELECT_CC: R = PromoteFloatOp_SELECT_CC(N, OpNo); break;
2141 case ISD::SETCC: R = PromoteFloatOp_SETCC(N, OpNo); break;
2142 case ISD::STORE: R = PromoteFloatOp_STORE(N, OpNo); break;
2143 }
2144
2145 if (R.getNode())
2146 ReplaceValueWith(SDValue(N, 0), R);
2147 return false;
2148}
2149
2150SDValue DAGTypeLegalizer::PromoteFloatOp_BITCAST(SDNode *N, unsigned OpNo) {
2151 SDValue Op = N->getOperand(0);
2152 EVT OpVT = Op->getValueType(0);
2153
2154 SDValue Promoted = GetPromotedFloat(N->getOperand(0));
2155 EVT PromotedVT = Promoted->getValueType(0);
2156
2157 // Convert the promoted float value to the desired IVT.
2158 EVT IVT = EVT::getIntegerVT(*DAG.getContext(), OpVT.getSizeInBits());
2159 SDValue Convert = DAG.getNode(GetPromotionOpcode(PromotedVT, OpVT), SDLoc(N),
2160 IVT, Promoted);
2161 // The final result type might not be an scalar so we need a bitcast. The
2162 // bitcast will be further legalized if needed.
2163 return DAG.getBitcast(N->getValueType(0), Convert);
2164}
2165
2166// Promote Operand 1 of FCOPYSIGN. Operand 0 ought to be handled by
2167// PromoteFloatRes_FCOPYSIGN.
2168SDValue DAGTypeLegalizer::PromoteFloatOp_FCOPYSIGN(SDNode *N, unsigned OpNo) {
2169 assert (OpNo == 1 && "Only Operand 1 must need promotion here");
2170 SDValue Op1 = GetPromotedFloat(N->getOperand(1));
2171
2172 return DAG.getNode(N->getOpcode(), SDLoc(N), N->getValueType(0),
2173 N->getOperand(0), Op1);
2174}
2175
2176// Convert the promoted float value to the desired integer type
2177SDValue DAGTypeLegalizer::PromoteFloatOp_FP_TO_XINT(SDNode *N, unsigned OpNo) {
2178 SDValue Op = GetPromotedFloat(N->getOperand(0));
2179 return DAG.getNode(N->getOpcode(), SDLoc(N), N->getValueType(0), Op);
2180}
2181
2182SDValue DAGTypeLegalizer::PromoteFloatOp_FP_TO_XINT_SAT(SDNode *N,
2183 unsigned OpNo) {
2184 SDValue Op = GetPromotedFloat(N->getOperand(0));
2185 return DAG.getNode(N->getOpcode(), SDLoc(N), N->getValueType(0), Op,
2186 N->getOperand(1));
2187}
2188
2189SDValue DAGTypeLegalizer::PromoteFloatOp_FP_EXTEND(SDNode *N, unsigned OpNo) {
2190 SDValue Op = GetPromotedFloat(N->getOperand(0));
2191 EVT VT = N->getValueType(0);
2192
2193 // Desired VT is same as promoted type. Use promoted float directly.
2194 if (VT == Op->getValueType(0))
2195 return Op;
2196
2197 // Else, extend the promoted float value to the desired VT.
2198 return DAG.getNode(ISD::FP_EXTEND, SDLoc(N), VT, Op);
2199}
2200
2201// Promote the float operands used for comparison. The true- and false-
2202// operands have the same type as the result and are promoted, if needed, by
2203// PromoteFloatRes_SELECT_CC
2204SDValue DAGTypeLegalizer::PromoteFloatOp_SELECT_CC(SDNode *N, unsigned OpNo) {
2205 SDValue LHS = GetPromotedFloat(N->getOperand(0));
2206 SDValue RHS = GetPromotedFloat(N->getOperand(1));
2207
2208 return DAG.getNode(ISD::SELECT_CC, SDLoc(N), N->getValueType(0),
2209 LHS, RHS, N->getOperand(2), N->getOperand(3),
2210 N->getOperand(4));
2211}
2212
2213// Construct a SETCC that compares the promoted values and sets the conditional
2214// code.
2215SDValue DAGTypeLegalizer::PromoteFloatOp_SETCC(SDNode *N, unsigned OpNo) {
2216 EVT VT = N->getValueType(0);
2217 SDValue Op0 = GetPromotedFloat(N->getOperand(0));
2218 SDValue Op1 = GetPromotedFloat(N->getOperand(1));
2219 ISD::CondCode CCCode = cast<CondCodeSDNode>(N->getOperand(2))->get();
2220
2221 return DAG.getSetCC(SDLoc(N), VT, Op0, Op1, CCCode);
2222
2223}
2224
2225// Lower the promoted Float down to the integer value of same size and construct
2226// a STORE of the integer value.
2227SDValue DAGTypeLegalizer::PromoteFloatOp_STORE(SDNode *N, unsigned OpNo) {
2228 StoreSDNode *ST = cast<StoreSDNode>(N);
2229 SDValue Val = ST->getValue();
2230 SDLoc DL(N);
2231
2232 SDValue Promoted = GetPromotedFloat(Val);
2233 EVT VT = ST->getOperand(1).getValueType();
2234 EVT IVT = EVT::getIntegerVT(*DAG.getContext(), VT.getSizeInBits());
2235
2236 SDValue NewVal;
2237 NewVal = DAG.getNode(GetPromotionOpcode(Promoted.getValueType(), VT), DL,
2238 IVT, Promoted);
2239
2240 return DAG.getStore(ST->getChain(), DL, NewVal, ST->getBasePtr(),
2241 ST->getMemOperand());
2242}
2243
2244//===----------------------------------------------------------------------===//
2245// Float Result Promotion
2246//===----------------------------------------------------------------------===//
2247
2248void DAGTypeLegalizer::PromoteFloatResult(SDNode *N, unsigned ResNo) {
2249 LLVM_DEBUG(dbgs() << "Promote float result " << ResNo << ": "; N->dump(&DAG);
2250 dbgs() << "\n");
2251 SDValue R = SDValue();
2252
2253 // See if the target wants to custom expand this node.
2254 if (CustomLowerNode(N, N->getValueType(ResNo), true)) {
2255 LLVM_DEBUG(dbgs() << "Node has been custom expanded, done\n");
2256 return;
2257 }
2258
2259 switch (N->getOpcode()) {
2260 // These opcodes cannot appear if promotion of FP16 is done in the backend
2261 // instead of Clang
2262 case ISD::FP16_TO_FP:
2263 case ISD::FP_TO_FP16:
2264 default:
2265#ifndef NDEBUG
2266 dbgs() << "PromoteFloatResult #" << ResNo << ": ";
2267 N->dump(&DAG); dbgs() << "\n";
2268#endif
2269 llvm_unreachable("Do not know how to promote this operator's result!");
2270
2271 case ISD::BITCAST: R = PromoteFloatRes_BITCAST(N); break;
2272 case ISD::ConstantFP: R = PromoteFloatRes_ConstantFP(N); break;
2274 R = PromoteFloatRes_EXTRACT_VECTOR_ELT(N); break;
2275 case ISD::FCOPYSIGN: R = PromoteFloatRes_FCOPYSIGN(N); break;
2276
2277 // Unary FP Operations
2278 case ISD::FABS:
2279 case ISD::FCBRT:
2280 case ISD::FCEIL:
2281 case ISD::FCOS:
2282 case ISD::FEXP:
2283 case ISD::FEXP2:
2284 case ISD::FFLOOR:
2285 case ISD::FLOG:
2286 case ISD::FLOG2:
2287 case ISD::FLOG10:
2288 case ISD::FNEARBYINT:
2289 case ISD::FNEG:
2290 case ISD::FRINT:
2291 case ISD::FROUND:
2292 case ISD::FROUNDEVEN:
2293 case ISD::FSIN:
2294 case ISD::FSQRT:
2295 case ISD::FTRUNC:
2296 case ISD::FCANONICALIZE: R = PromoteFloatRes_UnaryOp(N); break;
2297
2298 // Binary FP Operations
2299 case ISD::FADD:
2300 case ISD::FDIV:
2301 case ISD::FMAXIMUM:
2302 case ISD::FMINIMUM:
2303 case ISD::FMAXNUM:
2304 case ISD::FMINNUM:
2305 case ISD::FMUL:
2306 case ISD::FPOW:
2307 case ISD::FREM:
2308 case ISD::FSUB: R = PromoteFloatRes_BinOp(N); break;
2309
2310 case ISD::FMA: // FMA is same as FMAD
2311 case ISD::FMAD: R = PromoteFloatRes_FMAD(N); break;
2312
2313 case ISD::FPOWI: R = PromoteFloatRes_FPOWI(N); break;
2314
2315 case ISD::FP_ROUND: R = PromoteFloatRes_FP_ROUND(N); break;
2316 case ISD::LOAD: R = PromoteFloatRes_LOAD(N); break;
2317 case ISD::SELECT: R = PromoteFloatRes_SELECT(N); break;
2318 case ISD::SELECT_CC: R = PromoteFloatRes_SELECT_CC(N); break;
2319
2320 case ISD::SINT_TO_FP:
2321 case ISD::UINT_TO_FP: R = PromoteFloatRes_XINT_TO_FP(N); break;
2322 case ISD::UNDEF: R = PromoteFloatRes_UNDEF(N); break;
2323 case ISD::ATOMIC_SWAP: R = BitcastToInt_ATOMIC_SWAP(N); break;
2328 R = PromoteFloatRes_VECREDUCE(N);
2329 break;
2332 R = PromoteFloatRes_VECREDUCE_SEQ(N);
2333 break;
2334 }
2335
2336 if (R.getNode())
2337 SetPromotedFloat(SDValue(N, ResNo), R);
2338}
2339
2340// Bitcast from i16 to f16: convert the i16 to a f32 value instead.
2341// At this point, it is not possible to determine if the bitcast value is
2342// eventually stored to memory or promoted to f32 or promoted to a floating
2343// point at a higher precision. Some of these cases are handled by FP_EXTEND,
2344// STORE promotion handlers.
2345SDValue DAGTypeLegalizer::PromoteFloatRes_BITCAST(SDNode *N) {
2346 EVT VT = N->getValueType(0);
2347 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
2348 // Input type isn't guaranteed to be a scalar int so bitcast if not. The
2349 // bitcast will be legalized further if necessary.
2350 EVT IVT = EVT::getIntegerVT(*DAG.getContext(),
2351 N->getOperand(0).getValueType().getSizeInBits());
2352 SDValue Cast = DAG.getBitcast(IVT, N->getOperand(0));
2353 return DAG.getNode(GetPromotionOpcode(VT, NVT), SDLoc(N), NVT, Cast);
2354}
2355
2356SDValue DAGTypeLegalizer::PromoteFloatRes_ConstantFP(SDNode *N) {
2357 ConstantFPSDNode *CFPNode = cast<ConstantFPSDNode>(N);
2358 EVT VT = N->getValueType(0);
2359 SDLoc DL(N);
2360
2361 // Get the (bit-cast) APInt of the APFloat and build an integer constant
2362 EVT IVT = EVT::getIntegerVT(*DAG.getContext(), VT.getSizeInBits());
2363 SDValue C = DAG.getConstant(CFPNode->getValueAPF().bitcastToAPInt(), DL,
2364 IVT);
2365
2366 // Convert the Constant to the desired FP type
2367 // FIXME We might be able to do the conversion during compilation and get rid
2368 // of it from the object code
2369 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
2370 return DAG.getNode(GetPromotionOpcode(VT, NVT), DL, NVT, C);
2371}
2372
2373// If the Index operand is a constant, try to redirect the extract operation to
2374// the correct legalized vector. If not, bit-convert the input vector to
2375// equivalent integer vector. Extract the element as an (bit-cast) integer
2376// value and convert it to the promoted type.
2377SDValue DAGTypeLegalizer::PromoteFloatRes_EXTRACT_VECTOR_ELT(SDNode *N) {
2378 SDLoc DL(N);
2379
2380 // If the index is constant, try to extract the value from the legalized
2381 // vector type.
2382 if (isa<ConstantSDNode>(N->getOperand(1))) {
2383 SDValue Vec = N->getOperand(0);
2384 SDValue Idx = N->getOperand(1);
2385 EVT VecVT = Vec->getValueType(0);
2386 EVT EltVT = VecVT.getVectorElementType();
2387
2388 uint64_t IdxVal = cast<ConstantSDNode>(Idx)->getZExtValue();
2389
2390 switch (getTypeAction(VecVT)) {
2391 default: break;
2393 SDValue Res = GetScalarizedVector(N->getOperand(0));
2394 ReplaceValueWith(SDValue(N, 0), Res);
2395 return SDValue();
2396 }
2398 Vec = GetWidenedVector(Vec);
2399 SDValue Res = DAG.getNode(N->getOpcode(), DL, EltVT, Vec, Idx);
2400 ReplaceValueWith(SDValue(N, 0), Res);
2401 return SDValue();
2402 }
2404 SDValue Lo, Hi;
2405 GetSplitVector(Vec, Lo, Hi);
2406
2407 uint64_t LoElts = Lo.getValueType().getVectorNumElements();
2408 SDValue Res;
2409 if (IdxVal < LoElts)
2410 Res = DAG.getNode(N->getOpcode(), DL, EltVT, Lo, Idx);
2411 else
2412 Res = DAG.getNode(N->getOpcode(), DL, EltVT, Hi,
2413 DAG.getConstant(IdxVal - LoElts, DL,
2414 Idx.getValueType()));
2415 ReplaceValueWith(SDValue(N, 0), Res);
2416 return SDValue();
2417 }
2418
2419 }
2420 }
2421
2422 // Bit-convert the input vector to the equivalent integer vector
2423 SDValue NewOp = BitConvertVectorToIntegerVector(N->getOperand(0));
2424 EVT IVT = NewOp.getValueType().getVectorElementType();
2425
2426 // Extract the element as an (bit-cast) integer value
2427 SDValue NewVal = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, IVT,
2428 NewOp, N->getOperand(1));
2429
2430 // Convert the element to the desired FP type
2431 EVT VT = N->getValueType(0);
2432 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
2433 return DAG.getNode(GetPromotionOpcode(VT, NVT), SDLoc(N), NVT, NewVal);
2434}
2435
2436// FCOPYSIGN(X, Y) returns the value of X with the sign of Y. If the result
2437// needs promotion, so does the argument X. Note that Y, if needed, will be
2438// handled during operand promotion.
2439SDValue DAGTypeLegalizer::PromoteFloatRes_FCOPYSIGN(SDNode *N) {
2440 EVT VT = N->getValueType(0);
2441 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
2442 SDValue Op0 = GetPromotedFloat(N->getOperand(0));
2443
2444 SDValue Op1 = N->getOperand(1);
2445
2446 return DAG.getNode(N->getOpcode(), SDLoc(N), NVT, Op0, Op1);
2447}
2448
2449// Unary operation where the result and the operand have PromoteFloat type
2450// action. Construct a new SDNode with the promoted float value of the old
2451// operand.
2452SDValue DAGTypeLegalizer::PromoteFloatRes_UnaryOp(SDNode *N) {
2453 EVT VT = N->getValueType(0);
2454 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
2455 SDValue Op = GetPromotedFloat(N->getOperand(0));
2456
2457 return DAG.getNode(N->getOpcode(), SDLoc(N), NVT, Op);
2458}
2459
2460// Binary operations where the result and both operands have PromoteFloat type
2461// action. Construct a new SDNode with the promoted float values of the old
2462// operands.
2463SDValue DAGTypeLegalizer::PromoteFloatRes_BinOp(SDNode *N) {
2464 EVT VT = N->getValueType(0);
2465 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
2466 SDValue Op0 = GetPromotedFloat(N->getOperand(0));
2467 SDValue Op1 = GetPromotedFloat(N->getOperand(1));
2468 return DAG.getNode(N->getOpcode(), SDLoc(N), NVT, Op0, Op1, N->getFlags());
2469}
2470
2471SDValue DAGTypeLegalizer::PromoteFloatRes_FMAD(SDNode *N) {
2472 EVT VT = N->getValueType(0);
2473 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
2474 SDValue Op0 = GetPromotedFloat(N->getOperand(0));
2475 SDValue Op1 = GetPromotedFloat(N->getOperand(1));
2476 SDValue Op2 = GetPromotedFloat(N->getOperand(2));
2477
2478 return DAG.getNode(N->getOpcode(), SDLoc(N), NVT, Op0, Op1, Op2);
2479}
2480
2481// Promote the Float (first) operand and retain the Integer (second) operand
2482SDValue DAGTypeLegalizer::PromoteFloatRes_FPOWI(SDNode *N) {
2483 EVT VT = N->getValueType(0);
2484 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
2485 SDValue Op0 = GetPromotedFloat(N->getOperand(0));
2486 SDValue Op1 = N->getOperand(1);
2487
2488 return DAG.getNode(N->getOpcode(), SDLoc(N), NVT, Op0, Op1);
2489}
2490
2491// Explicit operation to reduce precision. Reduce the value to half precision
2492// and promote it back to the legal type.
2493SDValue DAGTypeLegalizer::PromoteFloatRes_FP_ROUND(SDNode *N) {
2494 SDLoc DL(N);
2495
2496 SDValue Op = N->getOperand(0);
2497 EVT VT = N->getValueType(0);
2498 EVT OpVT = Op->getValueType(0);
2499 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
2500 EVT IVT = EVT::getIntegerVT(*DAG.getContext(), VT.getSizeInBits());
2501
2502 // Round promoted float to desired precision
2503 SDValue Round = DAG.getNode(GetPromotionOpcode(OpVT, VT), DL, IVT, Op);
2504 // Promote it back to the legal output type
2505 return DAG.getNode(GetPromotionOpcode(VT, NVT), DL, NVT, Round);
2506}
2507
2508SDValue DAGTypeLegalizer::PromoteFloatRes_LOAD(SDNode *N) {
2509 LoadSDNode *L = cast<LoadSDNode>(N);
2510 EVT VT = N->getValueType(0);
2511
2512 // Load the value as an integer value with the same number of bits.
2513 EVT IVT = EVT::getIntegerVT(*DAG.getContext(), VT.getSizeInBits());
2514 SDValue newL = DAG.getLoad(
2515 L->getAddressingMode(), L->getExtensionType(), IVT, SDLoc(N),
2516 L->getChain(), L->getBasePtr(), L->getOffset(), L->getPointerInfo(), IVT,
2517 L->getOriginalAlign(), L->getMemOperand()->getFlags(), L->getAAInfo());
2518 // Legalize the chain result by replacing uses of the old value chain with the
2519 // new one
2520 ReplaceValueWith(SDValue(N, 1), newL.getValue(1));
2521
2522 // Convert the integer value to the desired FP type
2523 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
2524 return DAG.getNode(GetPromotionOpcode(VT, NVT), SDLoc(N), NVT, newL);
2525}
2526
2527// Construct a new SELECT node with the promoted true- and false- values.
2528SDValue DAGTypeLegalizer::PromoteFloatRes_SELECT(SDNode *N) {
2529 SDValue TrueVal = GetPromotedFloat(N->getOperand(1));
2530 SDValue FalseVal = GetPromotedFloat(N->getOperand(2));
2531
2532 return DAG.getNode(ISD::SELECT, SDLoc(N), TrueVal->getValueType(0),
2533 N->getOperand(0), TrueVal, FalseVal);
2534}
2535
2536// Construct a new SELECT_CC node with the promoted true- and false- values.
2537// The operands used for comparison are promoted by PromoteFloatOp_SELECT_CC.
2538SDValue DAGTypeLegalizer::PromoteFloatRes_SELECT_CC(SDNode *N) {
2539 SDValue TrueVal = GetPromotedFloat(N->getOperand(2));
2540 SDValue FalseVal = GetPromotedFloat(N->getOperand(3));
2541
2542 return DAG.getNode(ISD::SELECT_CC, SDLoc(N),
2543 TrueVal.getNode()->getValueType(0), N->getOperand(0),
2544 N->getOperand(1), TrueVal, FalseVal, N->getOperand(4));
2545}
2546
2547// Construct a SDNode that transforms the SINT or UINT operand to the promoted
2548// float type.
2549SDValue DAGTypeLegalizer::PromoteFloatRes_XINT_TO_FP(SDNode *N) {
2550 SDLoc DL(N);
2551 EVT VT = N->getValueType(0);
2552 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
2553 SDValue NV = DAG.getNode(N->getOpcode(), DL, NVT, N->getOperand(0));
2554 // Round the value to the desired precision (that of the source type).
2555 return DAG.getNode(
2556 ISD::FP_EXTEND, DL, NVT,
2557 DAG.getNode(ISD::FP_ROUND, DL, VT, NV,
2558 DAG.getIntPtrConstant(0, DL, /*isTarget=*/true)));
2559}
2560
2561SDValue DAGTypeLegalizer::PromoteFloatRes_UNDEF(SDNode *N) {
2562 return DAG.getUNDEF(TLI.getTypeToTransformTo(*DAG.getContext(),
2563 N->getValueType(0)));
2564}
2565
2566SDValue DAGTypeLegalizer::PromoteFloatRes_VECREDUCE(SDNode *N) {
2567 // Expand and promote recursively.
2568 // TODO: This is non-optimal, but dealing with the concurrently happening
2569 // vector-legalization is non-trivial. We could do something similar to
2570 // PromoteFloatRes_EXTRACT_VECTOR_ELT here.
2571 ReplaceValueWith(SDValue(N, 0), TLI.expandVecReduce(N, DAG));
2572 return SDValue();
2573}
2574
2575SDValue DAGTypeLegalizer::PromoteFloatRes_VECREDUCE_SEQ(SDNode *N) {
2576 ReplaceValueWith(SDValue(N, 0), TLI.expandVecReduceSeq(N, DAG));
2577 return SDValue();
2578}
2579
2580SDValue DAGTypeLegalizer::BitcastToInt_ATOMIC_SWAP(SDNode *N) {
2581 EVT VT = N->getValueType(0);
2582
2583 AtomicSDNode *AM = cast<AtomicSDNode>(N);
2584 SDLoc SL(N);
2585
2586 SDValue CastVal = BitConvertToInteger(AM->getVal());
2587 EVT CastVT = CastVal.getValueType();
2588
2589 SDValue NewAtomic
2590 = DAG.getAtomic(ISD::ATOMIC_SWAP, SL, CastVT,
2591 DAG.getVTList(CastVT, MVT::Other),
2592 { AM->getChain(), AM->getBasePtr(), CastVal },
2593 AM->getMemOperand());
2594
2595 SDValue Result = NewAtomic;
2596
2597 if (getTypeAction(VT) == TargetLowering::TypePromoteFloat) {
2598 EVT NFPVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
2599 Result = DAG.getNode(GetPromotionOpcode(VT, NFPVT), SL, NFPVT,
2600 NewAtomic);
2601 }
2602
2603 // Legalize the chain result by replacing uses of the old value chain with the
2604 // new one
2605 ReplaceValueWith(SDValue(N, 1), NewAtomic.getValue(1));
2606
2607 return Result;
2608
2609}
2610
2611//===----------------------------------------------------------------------===//
2612// Half Result Soft Promotion
2613//===----------------------------------------------------------------------===//
2614
2615void DAGTypeLegalizer::SoftPromoteHalfResult(SDNode *N, unsigned ResNo) {
2616 LLVM_DEBUG(dbgs() << "Soft promote half result " << ResNo << ": ";
2617 N->dump(&DAG); dbgs() << "\n");
2618 SDValue R = SDValue();
2619
2620 // See if the target wants to custom expand this node.
2621 if (CustomLowerNode(N, N->getValueType(ResNo), true)) {
2622 LLVM_DEBUG(dbgs() << "Node has been custom expanded, done\n");
2623 return;
2624 }
2625
2626 switch (N->getOpcode()) {
2627 default:
2628#ifndef NDEBUG
2629 dbgs() << "SoftPromoteHalfResult #" << ResNo << ": ";
2630 N->dump(&DAG); dbgs() << "\n";
2631#endif
2632 llvm_unreachable("Do not know how to soft promote this operator's result!");
2633
2634 case ISD::BITCAST: R = SoftPromoteHalfRes_BITCAST(N); break;
2635 case ISD::ConstantFP: R = SoftPromoteHalfRes_ConstantFP(N); break;
2637 R = SoftPromoteHalfRes_EXTRACT_VECTOR_ELT(N); break;
2638 case ISD::FCOPYSIGN: R = SoftPromoteHalfRes_FCOPYSIGN(N); break;
2640 case ISD::FP_ROUND: R = SoftPromoteHalfRes_FP_ROUND(N); break;
2641
2642 // Unary FP Operations
2643 case ISD::FABS:
2644 case ISD::FCBRT:
2645 case ISD::FCEIL:
2646 case ISD::FCOS:
2647 case ISD::FEXP:
2648 case ISD::FEXP2:
2649 case ISD::FFLOOR:
2650 case ISD::FLOG:
2651 case ISD::FLOG2:
2652 case ISD::FLOG10:
2653 case ISD::FNEARBYINT:
2654 case ISD::FNEG:
2655 case ISD::FREEZE:
2656 case ISD::FRINT:
2657 case ISD::FROUND:
2658 case ISD::FROUNDEVEN:
2659 case ISD::FSIN:
2660 case ISD::FSQRT:
2661 case ISD::FTRUNC:
2662 case ISD::FCANONICALIZE: R = SoftPromoteHalfRes_UnaryOp(N); break;
2663
2664 // Binary FP Operations
2665 case ISD::FADD:
2666 case ISD::FDIV:
2667 case ISD::FMAXIMUM:
2668 case ISD::FMINIMUM:
2669 case ISD::FMAXNUM:
2670 case ISD::FMINNUM:
2671 case ISD::FMUL:
2672 case ISD::FPOW:
2673 case ISD::FREM:
2674 case ISD::FSUB: R = SoftPromoteHalfRes_BinOp(N); break;
2675
2676 case ISD::FMA: // FMA is same as FMAD
2677 case ISD::FMAD: R = SoftPromoteHalfRes_FMAD(N); break;
2678
2679 case ISD::FPOWI: R = SoftPromoteHalfRes_FPOWI(N); break;
2680
2681 case ISD::LOAD: R = SoftPromoteHalfRes_LOAD(N); break;
2682 case ISD::SELECT: R = SoftPromoteHalfRes_SELECT(N); break;
2683 case ISD::SELECT_CC: R = SoftPromoteHalfRes_SELECT_CC(N); break;
2684 case ISD::SINT_TO_FP:
2685 case ISD::UINT_TO_FP: R = SoftPromoteHalfRes_XINT_TO_FP(N); break;
2686 case ISD::UNDEF: R = SoftPromoteHalfRes_UNDEF(N); break;
2687 case ISD::ATOMIC_SWAP: R = BitcastToInt_ATOMIC_SWAP(N); break;
2692 R = SoftPromoteHalfRes_VECREDUCE(N);
2693 break;
2696 R = SoftPromoteHalfRes_VECREDUCE_SEQ(N);
2697 break;
2698 }
2699
2700 if (R.getNode())
2701 SetSoftPromotedHalf(SDValue(N, ResNo), R);
2702}
2703
2704SDValue DAGTypeLegalizer::SoftPromoteHalfRes_BITCAST(SDNode *N) {
2705 return BitConvertToInteger(N->getOperand(0));
2706}
2707
2708SDValue DAGTypeLegalizer::SoftPromoteHalfRes_ConstantFP(SDNode *N) {
2709 ConstantFPSDNode *CN = cast<ConstantFPSDNode>(N);
2710
2711 // Get the (bit-cast) APInt of the APFloat and build an integer constant
2712 return DAG.getConstant(CN->getValueAPF().bitcastToAPInt(), SDLoc(CN),
2713 MVT::i16);
2714}
2715
2716SDValue DAGTypeLegalizer::SoftPromoteHalfRes_EXTRACT_VECTOR_ELT(SDNode *N) {
2717 SDValue NewOp = BitConvertVectorToIntegerVector(N->getOperand(0));
2719 NewOp.getValueType().getVectorElementType(), NewOp,
2720 N->getOperand(1));
2721}
2722
2723SDValue DAGTypeLegalizer::SoftPromoteHalfRes_FCOPYSIGN(SDNode *N) {
2724 SDValue LHS = GetSoftPromotedHalf(N->getOperand(0));
2725 SDValue RHS = BitConvertToInteger(N->getOperand(1));
2726 SDLoc dl(N);
2727
2728 EVT LVT = LHS.getValueType();
2729 EVT RVT = RHS.getValueType();
2730
2731 unsigned LSize = LVT.getSizeInBits();
2732 unsigned RSize = RVT.getSizeInBits();
2733
2734 // First get the sign bit of second operand.
2735 SDValue SignBit = DAG.getNode(
2736 ISD::SHL, dl, RVT, DAG.getConstant(1, dl, RVT),
2737 DAG.getConstant(RSize - 1, dl,
2738 TLI.getShiftAmountTy(RVT, DAG.getDataLayout())));
2739 SignBit = DAG.getNode(ISD::AND, dl, RVT, RHS, SignBit);
2740
2741 // Shift right or sign-extend it if the two operands have different types.
2742 int SizeDiff = RVT.getSizeInBits() - LVT.getSizeInBits();
2743 if (SizeDiff > 0) {
2744 SignBit =
2745 DAG.getNode(ISD::SRL, dl, RVT, SignBit,
2746 DAG.getConstant(SizeDiff, dl,
2747 TLI.getShiftAmountTy(SignBit.getValueType(),
2748 DAG.getDataLayout())));
2749 SignBit = DAG.getNode(ISD::TRUNCATE, dl, LVT, SignBit);
2750 } else if (SizeDiff < 0) {
2751 SignBit = DAG.getNode(ISD::ANY_EXTEND, dl, LVT, SignBit);
2752 SignBit =
2753 DAG.getNode(ISD::SHL, dl, LVT, SignBit,
2754 DAG.getConstant(-SizeDiff, dl,
2755 TLI.getShiftAmountTy(SignBit.getValueType(),
2756 DAG.getDataLayout())));
2757 }
2758
2759 // Clear the sign bit of the first operand.
2760 SDValue Mask = DAG.getNode(
2761 ISD::SHL, dl, LVT, DAG.getConstant(1, dl, LVT),
2762 DAG.getConstant(LSize - 1, dl,
2763 TLI.getShiftAmountTy(LVT, DAG.getDataLayout())));
2764 Mask = DAG.getNode(ISD::SUB, dl, LVT, Mask, DAG.getConstant(1, dl, LVT));
2765 LHS = DAG.getNode(ISD::AND, dl, LVT, LHS, Mask);
2766
2767 // Or the value with the sign bit.
2768 return DAG.getNode(ISD::OR, dl, LVT, LHS, SignBit);
2769}
2770
2771SDValue DAGTypeLegalizer::SoftPromoteHalfRes_FMAD(SDNode *N) {
2772 EVT OVT = N->getValueType(0);
2773 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), OVT);
2774 SDValue Op0 = GetSoftPromotedHalf(N->getOperand(0));
2775 SDValue Op1 = GetSoftPromotedHalf(N->getOperand(1));
2776 SDValue Op2 = GetSoftPromotedHalf(N->getOperand(2));
2777 SDLoc dl(N);
2778
2779 // Promote to the larger FP type.
2780 auto PromotionOpcode = GetPromotionOpcode(OVT, NVT);
2781 Op0 = DAG.getNode(PromotionOpcode, dl, NVT, Op0);
2782 Op1 = DAG.getNode(PromotionOpcode, dl, NVT, Op1);
2783 Op2 = DAG.getNode(PromotionOpcode, dl, NVT, Op2);
2784
2785 SDValue Res = DAG.getNode(N->getOpcode(), dl, NVT, Op0, Op1, Op2);
2786
2787 // Convert back to FP16 as an integer.
2788 return DAG.getNode(GetPromotionOpcode(NVT, OVT), dl, MVT::i16, Res);
2789}
2790
2791SDValue DAGTypeLegalizer::SoftPromoteHalfRes_FPOWI(SDNode *N) {
2792 EVT OVT = N->getValueType(0);
2793 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), OVT);
2794 SDValue Op0 = GetSoftPromotedHalf(N->getOperand(0));
2795 SDValue Op1 = N->getOperand(1);
2796 SDLoc dl(N);
2797
2798 // Promote to the larger FP type.
2799 Op0 = DAG.getNode(GetPromotionOpcode(OVT, NVT), dl, NVT, Op0);
2800
2801 SDValue Res = DAG.getNode(N->getOpcode(), dl, NVT, Op0, Op1);
2802
2803 // Convert back to FP16 as an integer.
2804 return DAG.getNode(GetPromotionOpcode(NVT, OVT), dl, MVT::i16, Res);
2805}
2806
2807SDValue DAGTypeLegalizer::SoftPromoteHalfRes_FP_ROUND(SDNode *N) {
2808 EVT RVT = N->getValueType(0);
2809 EVT SVT = N->getOperand(0).getValueType();
2810
2811 if (N->isStrictFPOpcode()) {
2812 assert(RVT == MVT::f16);
2813 SDValue Res =
2814 DAG.getNode(ISD::STRICT_FP_TO_FP16, SDLoc(N), {MVT::i16, MVT::Other},
2815 {N->getOperand(0), N->getOperand(1)});
2816 ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
2817 return Res;
2818 }
2819
2820 return DAG.getNode(GetPromotionOpcode(SVT, RVT), SDLoc(N), MVT::i16,
2821 N->getOperand(0));
2822}
2823
2824SDValue DAGTypeLegalizer::SoftPromoteHalfRes_LOAD(SDNode *N) {
2825 LoadSDNode *L = cast<LoadSDNode>(N);
2826
2827 // Load the value as an integer value with the same number of bits.
2828 assert(L->getExtensionType() == ISD::NON_EXTLOAD && "Unexpected extension!");
2829 SDValue NewL =
2830 DAG.getLoad(L->getAddressingMode(), L->getExtensionType(), MVT::i16,
2831 SDLoc(N), L->getChain(), L->getBasePtr(), L->getOffset(),
2832 L->getPointerInfo(), MVT::i16, L->getOriginalAlign(),
2833 L->getMemOperand()->getFlags(), L->getAAInfo());
2834 // Legalize the chain result by replacing uses of the old value chain with the
2835 // new one
2836 ReplaceValueWith(SDValue(N, 1), NewL.getValue(1));
2837 return NewL;
2838}
2839
2840SDValue DAGTypeLegalizer::SoftPromoteHalfRes_SELECT(SDNode *N) {
2841 SDValue Op1 = GetSoftPromotedHalf(N->getOperand(1));
2842 SDValue Op2 = GetSoftPromotedHalf(N->getOperand(2));
2843 return DAG.getSelect(SDLoc(N), Op1.getValueType(), N->getOperand(0), Op1,
2844 Op2);
2845}
2846
2847SDValue DAGTypeLegalizer::SoftPromoteHalfRes_SELECT_CC(SDNode *N) {
2848 SDValue Op2 = GetSoftPromotedHalf(N->getOperand(2));
2849 SDValue Op3 = GetSoftPromotedHalf(N->getOperand(3));
2850 return DAG.getNode(ISD::SELECT_CC, SDLoc(N), Op2.getValueType(),
2851 N->getOperand(0), N->getOperand(1), Op2, Op3,
2852 N->getOperand(4));
2853}
2854
2855SDValue DAGTypeLegalizer::SoftPromoteHalfRes_XINT_TO_FP(SDNode *N) {
2856 EVT OVT = N->getValueType(0);
2857 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), OVT);
2858 SDLoc dl(N);
2859
2860 SDValue Res = DAG.getNode(N->getOpcode(), dl, NVT, N->getOperand(0));
2861
2862 // Round the value to the softened type.
2863 return DAG.getNode(GetPromotionOpcode(NVT, OVT), dl, MVT::i16, Res);
2864}
2865
2866SDValue DAGTypeLegalizer::SoftPromoteHalfRes_UNDEF(SDNode *N) {
2867 return DAG.getUNDEF(MVT::i16);
2868}
2869
2870SDValue DAGTypeLegalizer::SoftPromoteHalfRes_UnaryOp(SDNode *N) {
2871 EVT OVT = N->getValueType(0);
2872 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), OVT);
2873 SDValue Op = GetSoftPromotedHalf(N->getOperand(0));
2874 SDLoc dl(N);
2875
2876 // Promote to the larger FP type.
2877 Op = DAG.getNode(GetPromotionOpcode(OVT, NVT), dl, NVT, Op);
2878
2879 SDValue Res = DAG.getNode(N->getOpcode(), dl, NVT, Op);
2880
2881 // Convert back to FP16 as an integer.
2882 return DAG.getNode(GetPromotionOpcode(NVT, OVT), dl, MVT::i16, Res);
2883}
2884
2885SDValue DAGTypeLegalizer::SoftPromoteHalfRes_BinOp(SDNode *N) {
2886 EVT OVT = N->getValueType(0);
2887 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), OVT);
2888 SDValue Op0 = GetSoftPromotedHalf(N->getOperand(0));
2889 SDValue Op1 = GetSoftPromotedHalf(N->getOperand(1));
2890 SDLoc dl(N);
2891
2892 // Promote to the larger FP type.
2893 auto PromotionOpcode = GetPromotionOpcode(OVT, NVT);
2894 Op0 = DAG.getNode(PromotionOpcode, dl, NVT, Op0);
2895 Op1 = DAG.getNode(PromotionOpcode, dl, NVT, Op1);
2896
2897 SDValue Res = DAG.getNode(N->getOpcode(), dl, NVT, Op0, Op1);
2898
2899 // Convert back to FP16 as an integer.
2900 return DAG.getNode(GetPromotionOpcode(NVT, OVT), dl, MVT::i16, Res);
2901}
2902
2903SDValue DAGTypeLegalizer::SoftPromoteHalfRes_VECREDUCE(SDNode *N) {
2904 // Expand and soften recursively.
2905 ReplaceValueWith(SDValue(N, 0), TLI.expandVecReduce(N, DAG));
2906 return SDValue();
2907}
2908
2909SDValue DAGTypeLegalizer::SoftPromoteHalfRes_VECREDUCE_SEQ(SDNode *N) {
2910 // Expand and soften.
2911 ReplaceValueWith(SDValue(N, 0), TLI.expandVecReduceSeq(N, DAG));
2912 return SDValue();
2913}
2914
2915//===----------------------------------------------------------------------===//
2916// Half Operand Soft Promotion
2917//===----------------------------------------------------------------------===//
2918
2919bool DAGTypeLegalizer::SoftPromoteHalfOperand(SDNode *N, unsigned OpNo) {
2920 LLVM_DEBUG(dbgs() << "Soft promote half operand " << OpNo << ": ";
2921 N->dump(&DAG); dbgs() << "\n");
2922 SDValue Res = SDValue();
2923
2924 if (CustomLowerNode(N, N->getOperand(OpNo).getValueType(), false)) {
2925 LLVM_DEBUG(dbgs() << "Node has been custom lowered, done\n");
2926 return false;
2927 }
2928
2929 // Nodes that use a promotion-requiring floating point operand, but doesn't
2930 // produce a soft promotion-requiring floating point result, need to be
2931 // legalized to use the soft promoted float operand. Nodes that produce at
2932 // least one soft promotion-requiring floating point result have their
2933 // operands legalized as a part of PromoteFloatResult.
2934 switch (N->getOpcode()) {
2935 default:
2936 #ifndef NDEBUG
2937 dbgs() << "SoftPromoteHalfOperand Op #" << OpNo << ": ";
2938 N->dump(&DAG); dbgs() << "\n";
2939 #endif
2940 llvm_unreachable("Do not know how to soft promote this operator's operand!");
2941
2942 case ISD::BITCAST: Res = SoftPromoteHalfOp_BITCAST(N); break;
2943 case ISD::FCOPYSIGN: Res = SoftPromoteHalfOp_FCOPYSIGN(N, OpNo); break;
2944 case ISD::FP_TO_SINT:
2945 case ISD::FP_TO_UINT: Res = SoftPromoteHalfOp_FP_TO_XINT(N); break;
2948 Res = SoftPromoteHalfOp_FP_TO_XINT_SAT(N); break;
2950 case ISD::FP_EXTEND: Res = SoftPromoteHalfOp_FP_EXTEND(N); break;
2951 case ISD::SELECT_CC: Res = SoftPromoteHalfOp_SELECT_CC(N, OpNo); break;
2952 case ISD::SETCC: Res = SoftPromoteHalfOp_SETCC(N); break;
2953 case ISD::STORE: Res = SoftPromoteHalfOp_STORE(N, OpNo); break;
2954 case ISD::STACKMAP:
2955 Res = SoftPromoteHalfOp_STACKMAP(N, OpNo);
2956 break;
2957 case ISD::PATCHPOINT:
2958 Res = SoftPromoteHalfOp_PATCHPOINT(N, OpNo);
2959 break;
2960 }
2961
2962 if (!Res.getNode())
2963 return false;
2964
2965 assert(Res.getNode() != N && "Expected a new node!");
2966
2967 assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 &&
2968 "Invalid operand expansion");
2969
2970 ReplaceValueWith(SDValue(N, 0), Res);
2971 return false;
2972}
2973
2974SDValue DAGTypeLegalizer::SoftPromoteHalfOp_BITCAST(SDNode *N) {
2975 SDValue Op0 = GetSoftPromotedHalf(N->getOperand(0));
2976
2977 return DAG.getNode(ISD::BITCAST, SDLoc(N), N->getValueType(0), Op0);
2978}
2979
2980SDValue DAGTypeLegalizer::SoftPromoteHalfOp_FCOPYSIGN(SDNode *N,
2981 unsigned OpNo) {
2982 assert(OpNo == 1 && "Only Operand 1 must need promotion here");
2983 SDValue Op1 = N->getOperand(1);
2984 EVT RVT = Op1.getValueType();
2985 SDLoc dl(N);
2986
2987 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), Op1.getValueType());
2988
2989 Op1 = GetSoftPromotedHalf(Op1);
2990 Op1 = DAG.getNode(GetPromotionOpcode(RVT, NVT), dl, NVT, Op1);
2991
2992 return DAG.getNode(N->getOpcode(), dl, N->getValueType(0), N->getOperand(0),
2993 Op1);
2994}
2995
2996SDValue DAGTypeLegalizer::SoftPromoteHalfOp_FP_EXTEND(SDNode *N) {
2997 EVT RVT = N->getValueType(0);
2998 bool IsStrict = N->isStrictFPOpcode();
2999 SDValue Op = N->getOperand(IsStrict ? 1 : 0);
3000 EVT SVT = Op.getValueType();
3001 Op = GetSoftPromotedHalf(N->getOperand(IsStrict ? 1 : 0));
3002
3003 if (IsStrict) {
3004 assert(SVT == MVT::f16);
3005 SDValue Res =
3007 {N->getValueType(0), MVT::Other}, {N->getOperand(0), Op});
3008 ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
3009 ReplaceValueWith(SDValue(N, 0), Res);
3010 return SDValue();
3011 }
3012
3013 return DAG.getNode(GetPromotionOpcode(SVT, RVT), SDLoc(N), RVT, Op);
3014}
3015
3016SDValue DAGTypeLegalizer::SoftPromoteHalfOp_FP_TO_XINT(SDNode *N) {
3017 EVT RVT = N->getValueType(0);
3018 SDValue Op = N->getOperand(0);
3019 EVT SVT = Op.getValueType();
3020 SDLoc dl(N);
3021
3022 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), Op.getValueType());
3023
3024 Op = GetSoftPromotedHalf(Op);
3025
3026 SDValue Res = DAG.getNode(GetPromotionOpcode(SVT, RVT), dl, NVT, Op);
3027
3028 return DAG.getNode(N->getOpcode(), dl, N->getValueType(0), Res);
3029}
3030
3031SDValue DAGTypeLegalizer::SoftPromoteHalfOp_FP_TO_XINT_SAT(SDNode *N) {
3032 EVT RVT = N->getValueType(0);
3033 SDValue Op = N->getOperand(0);
3034 EVT SVT = Op.getValueType();
3035 SDLoc dl(N);
3036
3037 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), Op.getValueType());
3038
3039 Op = GetSoftPromotedHalf(Op);
3040
3041 SDValue Res = DAG.getNode(GetPromotionOpcode(SVT, RVT), dl, NVT, Op);
3042
3043 return DAG.getNode(N->getOpcode(), dl, N->getValueType(0), Res,
3044 N->getOperand(1));
3045}
3046
3047SDValue DAGTypeLegalizer::SoftPromoteHalfOp_SELECT_CC(SDNode *N,
3048 unsigned OpNo) {
3049 assert(OpNo == 0 && "Can only soften the comparison values");
3050 SDValue Op0 = N->getOperand(0);
3051 SDValue Op1 = N->getOperand(1);
3052 SDLoc dl(N);
3053
3054 EVT SVT = Op0.getValueType();
3055 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), SVT);
3056
3057 Op0 = GetSoftPromotedHalf(Op0);
3058 Op1 = GetSoftPromotedHalf(Op1);
3059
3060 // Promote to the larger FP type.
3061 auto PromotionOpcode = GetPromotionOpcode(SVT, NVT);
3062 Op0 = DAG.getNode(PromotionOpcode, dl, NVT, Op0);
3063 Op1 = DAG.getNode(PromotionOpcode, dl, NVT, Op1);
3064
3065 return DAG.getNode(ISD::SELECT_CC, SDLoc(N), N->getValueType(0), Op0, Op1,
3066 N->getOperand(2), N->getOperand(3), N->getOperand(4));
3067}
3068
3069SDValue DAGTypeLegalizer::SoftPromoteHalfOp_SETCC(SDNode *N) {
3070 SDValue Op0 = N->getOperand(0);
3071 SDValue Op1 = N->getOperand(1);
3072 ISD::CondCode CCCode = cast<CondCodeSDNode>(N->getOperand(2))->get();
3073 SDLoc dl(N);
3074
3075 EVT SVT = Op0.getValueType();
3076 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), Op0.getValueType());
3077
3078 Op0 = GetSoftPromotedHalf(Op0);
3079 Op1 = GetSoftPromotedHalf(Op1);
3080
3081 // Promote to the larger FP type.
3082 auto PromotionOpcode = GetPromotionOpcode(SVT, NVT);
3083 Op0 = DAG.getNode(PromotionOpcode, dl, NVT, Op0);
3084 Op1 = DAG.getNode(PromotionOpcode, dl, NVT, Op1);
3085
3086 return DAG.getSetCC(SDLoc(N), N->getValueType(0), Op0, Op1, CCCode);
3087}
3088
3089SDValue DAGTypeLegalizer::SoftPromoteHalfOp_STORE(SDNode *N, unsigned OpNo) {
3090 assert(OpNo == 1 && "Can only soften the stored value!");
3091 StoreSDNode *ST = cast<StoreSDNode>(N);
3092 SDValue Val = ST->getValue();
3093 SDLoc dl(N);
3094
3095 assert(!ST->isTruncatingStore() && "Unexpected truncating store.");
3096 SDValue Promoted = GetSoftPromotedHalf(Val);
3097 return DAG.getStore(ST->getChain(), dl, Promoted, ST->getBasePtr(),
3098 ST->getMemOperand());
3099}
3100
3101SDValue DAGTypeLegalizer::SoftPromoteHalfOp_STACKMAP(SDNode *N, unsigned OpNo) {
3102 assert(OpNo > 1); // Because the first two arguments are guaranteed legal.
3103 SmallVector<SDValue> NewOps(N->ops().begin(), N->ops().end());
3104 SDValue Op = N->getOperand(OpNo);
3105 NewOps[OpNo] = GetSoftPromotedHalf(Op);
3107 DAG.getNode(N->getOpcode(), SDLoc(N), N->getVTList(), NewOps);
3108
3109 for (unsigned ResNum = 0; ResNum < N->getNumValues(); ResNum++)
3110 ReplaceValueWith(SDValue(N, ResNum), NewNode.getValue(ResNum));
3111
3112 return SDValue(); // Signal that we replaced the node ourselves.
3113}
3114
3115SDValue DAGTypeLegalizer::SoftPromoteHalfOp_PATCHPOINT(SDNode *N,
3116 unsigned OpNo) {
3117 assert(OpNo >= 7);
3118 SmallVector<SDValue> NewOps(N->ops().begin(), N->ops().end());
3119 SDValue Op = N->getOperand(OpNo);
3120 NewOps[OpNo] = GetSoftPromotedHalf(Op);
3122 DAG.getNode(N->getOpcode(), SDLoc(N), N->getVTList(), NewOps);
3123
3124 for (unsigned ResNum = 0; ResNum < N->getNumValues(); ResNum++)
3125 ReplaceValueWith(SDValue(N, ResNum), NewNode.getValue(ResNum));
3126
3127 return SDValue(); // Signal that we replaced the node ourselves.
3128}
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
Returns the sub type a function will return at a given Idx Should correspond to the result type of an ExtractValue instruction executed with just that one unsigned Idx
#define LLVM_DEBUG(X)
Definition: Debug.h:101
uint64_t Size
static bool isSigned(unsigned int Opcode)
static RTLIB::Libcall findFPToIntLibcall(EVT SrcVT, EVT RetVT, EVT &Promoted, bool Signed)
static RTLIB::Libcall GetFPLibCall(EVT VT, RTLIB::Libcall Call_F32, RTLIB::Libcall Call_F64, RTLIB::Libcall Call_F80, RTLIB::Libcall Call_F128, RTLIB::Libcall Call_PPCF128)
GetFPLibCall - Return the right libcall for the given floating point type.
static ISD::NodeType GetPromotionOpcode(EVT OpVT, EVT RetVT)
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
@ Flags
Definition: TextStubV5.cpp:93
Value * RHS
Value * LHS
APInt bitcastToAPInt() const
Definition: APFloat.h:1184
Class for arbitrary precision integers.
Definition: APInt.h:75
static APInt getAllOnes(unsigned numBits)
Return an APInt of a specified width with all bits set.
Definition: APInt.h:214
void clearBit(unsigned BitPosition)
Set a given bit to 0.
Definition: APInt.h:1389
static APInt getSignMask(unsigned BitWidth)
Get the SignMask for a specific bit width.
Definition: APInt.h:209
const uint64_t * getRawData() const
This function returns a pointer to the internal storage of the APInt.
Definition: APInt.h:557
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
Definition: ArrayRef.h:41
This is an SDNode representing atomic operations.
const SDValue & getVal() const
const APFloat & getValueAPF() const
@ NewNode
This is a new node, not before seen, that was created in the process of legalizing some other node.
Definition: LegalizeTypes.h:42
bool isBigEndian() const
Definition: DataLayout.h:239
void emitError(uint64_t LocCookie, const Twine &ErrorStr)
emitError - Emit an error message to the currently installed error handler with optional location inf...
This class is used to represent ISD::LOAD nodes.
SimpleValueType SimpleTy
@ MODereferenceable
The memory access is dereferenceable (i.e., doesn't trap).
@ MOInvariant
The memory access always returns the same value (or traps).
MachineMemOperand * getMemOperand() const
Return a MachineMemOperand object describing the memory reference performed by operation.
Wrapper class for IR location info (IR ordering and DebugLoc) to be passed into SDNode creation funct...
Represents one node in the SelectionDAG.
unsigned getNumValues() const
Return the number of values defined/returned by this operator.
EVT getValueType(unsigned ResNo) const
Return the type of a specified result.
Unlike LLVM values, Selection DAG nodes may return multiple values as the result of a computation.
SDNode * getNode() const
get the SDNode which holds the desired result
SDValue getValue(unsigned R) const
EVT getValueType() const
Return the ValueType of the referenced return value.
SDValue getExtLoad(ISD::LoadExtType ExtType, const SDLoc &dl, EVT VT, SDValue Chain, SDValue Ptr, MachinePointerInfo PtrInfo, EVT MemVT, MaybeAlign Alignment=MaybeAlign(), MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes())
SDValue getSelect(const SDLoc &DL, EVT VT, SDValue Cond, SDValue LHS, SDValue RHS)
Helper function to make it easier to build Select's if you just have operands and don't want to check...
SDVTList getVTList(EVT VT)
Return an SDVTList that represents the list of values specified.
SDValue getSetCC(const SDLoc &DL, EVT VT, SDValue LHS, SDValue RHS, ISD::CondCode Cond, SDValue Chain=SDValue(), bool IsSignaling=false)
Helper function to make it easier to build SetCC's if you just have an ISD::CondCode instead of an SD...
SDValue getConstantFP(double Val, const SDLoc &DL, EVT VT, bool isTarget=false)
Create a ConstantFPSDNode wrapping a constant value.
SDValue getLoad(EVT VT, const SDLoc &dl, SDValue Chain, SDValue Ptr, MachinePointerInfo PtrInfo, MaybeAlign Alignment=MaybeAlign(), MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes(), const MDNode *Ranges=nullptr)
Loads are not normal binary operators: their result type is not determined by their operands,...
SDValue getAtomic(unsigned Opcode, const SDLoc &dl, EVT MemVT, SDValue Chain, SDValue Ptr, SDValue Val, MachineMemOperand *MMO)
Gets a node for an atomic op, produces result (if relevant) and chain and takes 2 operands.
SDValue getUNDEF(EVT VT)
Return an UNDEF node. UNDEF does not have a useful SDLoc.
SDValue getBitcast(EVT VT, SDValue V)
Return a bitcast using the SDLoc of the value operand, and casting to the provided type.
const DataLayout & getDataLayout() const
Definition: SelectionDAG.h:472
SDValue getConstant(uint64_t Val, const SDLoc &DL, EVT VT, bool isTarget=false, bool isOpaque=false)
Create a ConstantSDNode wrapping a constant value.
SDValue getVAArg(EVT VT, const SDLoc &dl, SDValue Chain, SDValue Ptr, SDValue SV, unsigned Align)
VAArg produces a result and token chain, and takes a pointer and a source value as input.
SDValue getTruncStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr, MachinePointerInfo PtrInfo, EVT SVT, Align Alignment, MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes())
SDValue getStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr, MachinePointerInfo PtrInfo, Align Alignment, MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes())
Helper function to build ISD::STORE nodes.
static const fltSemantics & EVTToAPFloatSemantics(EVT VT)
Returns an APFloat semantics tag appropriate for the given type.
SDValue getSelectCC(const SDLoc &DL, SDValue LHS, SDValue RHS, SDValue True, SDValue False, ISD::CondCode Cond)
Helper function to make it easier to build SelectCC's if you just have an ISD::CondCode instead of an...
SDValue getIntPtrConstant(uint64_t Val, const SDLoc &DL, bool isTarget=false)
SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT, ArrayRef< SDUse > Ops)
Gets or creates the specified node.
const TargetLibraryInfo & getLibInfo() const
Definition: SelectionDAG.h:479
SDValue getCondCode(ISD::CondCode Cond)
LLVMContext * getContext() const
Definition: SelectionDAG.h:485
SDValue getShiftAmountConstant(uint64_t Val, EVT VT, const SDLoc &DL, bool LegalTypes=true)
SDNode * UpdateNodeOperands(SDNode *N, SDValue Op)
Mutate the specified node in-place to have the specified operands.
SDValue getEntryNode() const
Return the token chain corresponding to the entry of the function.
Definition: SelectionDAG.h:554
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
Definition: SmallVector.h:1200
This class is used to represent ISD::STORE nodes.
unsigned getIntSize() const
Get size of a C-level int or unsigned int, in bits.
EVT getShiftAmountTy(EVT LHSTy, const DataLayout &DL, bool LegalTypes=true) const
Returns the type for the shift amount of a shift opcode.
virtual EVT getTypeToTransformTo(LLVMContext &Context, EVT VT) const
For types supported by the target, this is an identity function.
const char * getLibcallName(RTLIB::Libcall Call) const
Get the libcall routine name for the specified libcall.
void softenSetCCOperands(SelectionDAG &DAG, EVT VT, SDValue &NewLHS, SDValue &NewRHS, ISD::CondCode &CCCode, const SDLoc &DL, const SDValue OldLHS, const SDValue OldRHS) const
Soften the operands of a comparison.
std::pair< SDValue, SDValue > makeLibCall(SelectionDAG &DAG, RTLIB::Libcall LC, EVT RetVT, ArrayRef< SDValue > Ops, MakeLibCallOptions CallOptions, const SDLoc &dl, SDValue Chain=SDValue()) const
Returns a pair of (return value, chain).
SDValue expandVecReduceSeq(SDNode *Node, SelectionDAG &DAG) const
Expand a VECREDUCE_SEQ_* into an explicit ordered calculation.
SDValue expandFP_TO_INT_SAT(SDNode *N, SelectionDAG &DAG) const
Expand FP_TO_[US]INT_SAT into FP_TO_[US]INT and selects or min/max.
SDValue expandVecReduce(SDNode *Node, SelectionDAG &DAG) const
Expand a VECREDUCE_* into an explicit calculation.
SDValue createSelectForFMINNUM_FMAXNUM(SDNode *Node, SelectionDAG &DAG) const
Try to convert the fminnum/fmaxnum to a compare/select sequence.
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
constexpr std::underlying_type_t< E > Mask()
Get a bitmask with 1s in all places up to the high-order bit of E's largest value.
Definition: BitmaskEnum.h:119
@ C
The default llvm calling convention, compatible with C.
Definition: CallingConv.h:34
NodeType
ISD::NodeType enum - This enum defines the target-independent operators for a SelectionDAG.
Definition: ISDOpcodes.h:40
@ SETCC
SetCC operator - This evaluates to a true value iff the condition is true.
Definition: ISDOpcodes.h:749
@ MERGE_VALUES
MERGE_VALUES - This node takes multiple discrete operands and returns them all as its individual resu...
Definition: ISDOpcodes.h:236
@ STRICT_FSETCC
STRICT_FSETCC/STRICT_FSETCCS - Constrained versions of SETCC, used for floating-point operands only.
Definition: ISDOpcodes.h:475
@ VECREDUCE_SEQ_FADD
Generic reduction nodes.
Definition: ISDOpcodes.h:1276
@ ConstantFP
Definition: ISDOpcodes.h:77
@ STRICT_FCEIL
Definition: ISDOpcodes.h:425
@ FMAD
FMAD - Perform a * b + c, while getting the same result as the separately rounded operations.
Definition: ISDOpcodes.h:486
@ LOAD
LOAD and STORE have token chains as their first operand, then the same operands as an LLVM load/store...
Definition: ISDOpcodes.h:978
@ ANY_EXTEND
ANY_EXTEND - Used for integer types. The high bits are undefined.
Definition: ISDOpcodes.h:779
@ FMA
FMA - Perform a * b + c with no intermediate rounding step.
Definition: ISDOpcodes.h:482
@ SINT_TO_FP
[SU]INT_TO_FP - These operators convert integers (whose interpreted sign depends on the first letter)...
Definition: ISDOpcodes.h:786
@ VECREDUCE_FMAX
FMIN/FMAX nodes can have flags, for NaN/NoNaN variants.
Definition: ISDOpcodes.h:1292
@ FADD
Simple binary floating point operators.
Definition: ISDOpcodes.h:390
@ STRICT_FSETCCS
Definition: ISDOpcodes.h:476
@ FP16_TO_FP
FP16_TO_FP, FP_TO_FP16 - These operators are used to perform promotions and truncation for half-preci...
Definition: ISDOpcodes.h:908
@ STRICT_FLOG2
Definition: ISDOpcodes.h:420
@ BITCAST
BITCAST - This operator converts between integer, vector and FP values, as if the value was stored to...
Definition: ISDOpcodes.h:898
@ BUILD_PAIR
BUILD_PAIR - This is the opposite of EXTRACT_ELEMENT in some ways.
Definition: ISDOpcodes.h:229
@ STRICT_FSQRT
Constrained versions of libm-equivalent floating point intrinsics.
Definition: ISDOpcodes.h:411
@ SIGN_EXTEND
Conversion operators.
Definition: ISDOpcodes.h:773
@ STRICT_UINT_TO_FP
Definition: ISDOpcodes.h:449
@ VECREDUCE_FADD
These reductions have relaxed evaluation order semantics, and have a single vector operand.
Definition: ISDOpcodes.h:1289
@ VECREDUCE_FMIN
Definition: ISDOpcodes.h:1293
@ STRICT_LROUND
Definition: ISDOpcodes.h:430
@ FNEG
Perform various unary floating-point operations inspired by libm.
Definition: ISDOpcodes.h:923
@ BR_CC
BR_CC - Conditional branch.
Definition: ISDOpcodes.h:1020
@ FCANONICALIZE
Returns platform specific canonical encoding of a floating point number.
Definition: ISDOpcodes.h:499
@ SELECT
Select(COND, TRUEVAL, FALSEVAL).
Definition: ISDOpcodes.h:726
@ STRICT_FPOWI
Definition: ISDOpcodes.h:413
@ UNDEF
UNDEF - An undefined node.
Definition: ISDOpcodes.h:211
@ EXTRACT_ELEMENT
EXTRACT_ELEMENT - This is used to get the lower or upper (determined by a Constant,...
Definition: ISDOpcodes.h:222
@ STRICT_FTRUNC
Definition: ISDOpcodes.h:429
@ ARITH_FENCE
ARITH_FENCE - This corresponds to a arithmetic fence intrinsic.
Definition: ISDOpcodes.h:1162
@ STRICT_FP_TO_FP16
Definition: ISDOpcodes.h:911
@ STRICT_FP16_TO_FP
Definition: ISDOpcodes.h:910
@ SHL
Shift and rotation operations.
Definition: ISDOpcodes.h:704
@ STRICT_FMAXNUM
Definition: ISDOpcodes.h:423
@ EXTRACT_VECTOR_ELT
EXTRACT_VECTOR_ELT(VECTOR, IDX) - Returns a single element from VECTOR identified by the (potentially...
Definition: ISDOpcodes.h:534
@ ZERO_EXTEND
ZERO_EXTEND - Used for integer types, zeroing the new bits.
Definition: ISDOpcodes.h:776
@ FP_TO_UINT_SAT
Definition: ISDOpcodes.h:852
@ STRICT_FMINNUM
Definition: ISDOpcodes.h:424
@ SELECT_CC
Select with condition operator - This selects between a true value and a false value (ops #2 and #3) ...
Definition: ISDOpcodes.h:741
@ FMINNUM
FMINNUM/FMAXNUM - Perform floating-point minimum or maximum on two values.
Definition: ISDOpcodes.h:955
@ STRICT_LRINT
Definition: ISDOpcodes.h:432
@ FP_EXTEND
X = FP_EXTEND(Y) - Extend a smaller FP type into a larger FP type.
Definition: ISDOpcodes.h:883
@ STRICT_FROUND
Definition: ISDOpcodes.h:427
@ STRICT_SINT_TO_FP
STRICT_[US]INT_TO_FP - Convert a signed or unsigned integer to a floating point value.
Definition: ISDOpcodes.h:448
@ STRICT_FFLOOR
Definition: ISDOpcodes.h:426
@ STRICT_FROUNDEVEN
Definition: ISDOpcodes.h:428
@ BF16_TO_FP
BF16_TO_FP, FP_TO_BF16 - These operators are used to perform promotions and truncation for bfloat16.
Definition: ISDOpcodes.h:917
@ STRICT_FP_TO_UINT
Definition: ISDOpcodes.h:442
@ STRICT_FP_ROUND
X = STRICT_FP_ROUND(Y, TRUNC) - Rounding 'Y' from a larger floating point type down to the precision ...
Definition: ISDOpcodes.h:464
@ STRICT_FP_TO_SINT
STRICT_FP_TO_[US]INT - Convert a floating point value to a signed or unsigned integer.
Definition: ISDOpcodes.h:441
@ FMINIMUM
FMINIMUM/FMAXIMUM - NaN-propagating minimum/maximum that also treat -0.0 as less than 0....
Definition: ISDOpcodes.h:968
@ FP_TO_SINT
FP_TO_[US]INT - Convert a floating point value to a signed or unsigned integer.
Definition: ISDOpcodes.h:832
@ STRICT_FP_EXTEND
X = STRICT_FP_EXTEND(Y) - Extend a smaller FP type into a larger FP type.
Definition: ISDOpcodes.h:469
@ AND
Bitwise operators - logical and, logical or, logical xor.
Definition: ISDOpcodes.h:679
@ VECREDUCE_FMUL
Definition: ISDOpcodes.h:1290
@ STRICT_FADD
Constrained versions of the binary floating point operators.
Definition: ISDOpcodes.h:400
@ STRICT_FLOG10
Definition: ISDOpcodes.h:419
@ STRICT_LLRINT
Definition: ISDOpcodes.h:433
@ STRICT_FEXP2
Definition: ISDOpcodes.h:417
@ ATOMIC_SWAP
Val, OUTCHAIN = ATOMIC_SWAP(INCHAIN, ptr, amt) Val, OUTCHAIN = ATOMIC_LOAD_[OpName](INCHAIN,...
Definition: ISDOpcodes.h:1199
@ FP_ROUND
X = FP_ROUND(Y, TRUNC) - Rounding 'Y' from a larger floating point type down to the precision of the ...
Definition: ISDOpcodes.h:865
@ STRICT_LLROUND
Definition: ISDOpcodes.h:431
@ STRICT_FNEARBYINT
Definition: ISDOpcodes.h:422
@ FP_TO_SINT_SAT
FP_TO_[US]INT_SAT - Convert floating point value in operand 0 to a signed or unsigned scalar integer ...
Definition: ISDOpcodes.h:851
@ TRUNCATE
TRUNCATE - Completely drop the high bits.
Definition: ISDOpcodes.h:782
@ VAARG
VAARG - VAARG has four operands: an input chain, a pointer, a SRCVALUE, and the alignment.
Definition: ISDOpcodes.h:1089
@ VECREDUCE_SEQ_FMUL
Definition: ISDOpcodes.h:1277
@ FCOPYSIGN
FCOPYSIGN(X, Y) - Return the value of X with the sign of Y.
Definition: ISDOpcodes.h:492
@ STRICT_FRINT
Definition: ISDOpcodes.h:421
@ BUILD_VECTOR
BUILD_VECTOR(ELT0, ELT1, ELT2, ELT3,...) - Return a fixed-width vector with the specified,...
Definition: ISDOpcodes.h:514
bool isNormalStore(const SDNode *N)
Returns true if the specified node is a non-truncating and unindexed store.
bool isUNINDEXEDLoad(const SDNode *N)
Returns true if the specified node is an unindexed load.
bool isUNINDEXEDStore(const SDNode *N)
Returns true if the specified node is an unindexed store.
CondCode
ISD::CondCode enum - These are ordered carefully to make the bitfields below work out,...
Definition: ISDOpcodes.h:1447
bool isNormalLoad(const SDNode *N)
Returns true if the specified node is a non-extending and unindexed load.
Libcall getPOWI(EVT RetVT)
getPOWI - Return the POWI_* value for the given types, or UNKNOWN_LIBCALL if there is none.
Libcall getSINTTOFP(EVT OpVT, EVT RetVT)
getSINTTOFP - Return the SINTTOFP_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
Libcall getUINTTOFP(EVT OpVT, EVT RetVT)
getUINTTOFP - Return the UINTTOFP_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
Libcall
RTLIB::Libcall enum - This enum defines all of the runtime library calls the backend can emit.
Libcall getFPTOUINT(EVT OpVT, EVT RetVT)
getFPTOUINT - Return the FPTOUINT_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
Libcall getFPTOSINT(EVT OpVT, EVT RetVT)
getFPTOSINT - Return the FPTOSINT_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
Libcall getFPEXT(EVT OpVT, EVT RetVT)
getFPEXT - Return the FPEXT_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
Libcall getFPROUND(EVT OpVT, EVT RetVT)
getFPROUND - Return the FPROUND_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
DiagnosticInfoOptimizationBase::Argument NV
@ FalseVal
Definition: TGLexer.h:62
This is an optimization pass for GlobalISel generic memory operations.
Definition: AddressRanges.h:18
@ Offset
Definition: DWP.cpp:406
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
Definition: Debug.cpp:163
void report_fatal_error(Error Err, bool gen_crash_diag=true)
Report a serious error, calling any installed error handler.
Definition: Error.cpp:145
#define N
static const fltSemantics & PPCDoubleDouble() LLVM_READNONE
Definition: APFloat.cpp:247
Extended Value Type.
Definition: ValueTypes.h:34
bool isSimple() const
Test if the given EVT is simple (as opposed to being extended).
Definition: ValueTypes.h:129
TypeSize getSizeInBits() const
Return the size of the specified value type in bits.
Definition: ValueTypes.h:351
bool isByteSized() const
Return true if the bit size is a multiple of 8.
Definition: ValueTypes.h:226
MVT getSimpleVT() const
Return the SimpleValueType held in the specified simple EVT.
Definition: ValueTypes.h:299
static EVT getIntegerVT(LLVMContext &Context, unsigned BitWidth)
Returns the EVT that represents an integer with the given number of bits.
Definition: ValueTypes.h:64
bool bitsGE(EVT VT) const
Return true if this has no less bits than VT.
Definition: ValueTypes.h:275
EVT getVectorElementType() const
Given a vector type, return the type of each element.
Definition: ValueTypes.h:311
bool bitsLE(EVT VT) const
Return true if this has no more bits than VT.
Definition: ValueTypes.h:291
These are IR-level optimization flags that may be propagated to SDNodes.
This structure is used to pass arguments to makeLibCall function.
MakeLibCallOptions & setSExt(bool Value=true)
MakeLibCallOptions & setTypeListBeforeSoften(ArrayRef< EVT > OpsVT, EVT RetVT, bool Value=true)