LLVM 18.0.0git
LegalizeFloatTypes.cpp
Go to the documentation of this file.
1//===-------- LegalizeFloatTypes.cpp - Legalization of float types --------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements float type expansion and softening for LegalizeTypes.
10// Softening is the act of turning a computation in an illegal floating point
11// type into a computation in an integer type of the same size; also known as
12// "soft float". For example, turning f32 arithmetic into operations using i32.
13// The resulting integer value is the same as what you would get by performing
14// the floating point operation and bitcasting the result to the integer type.
15// Expansion is the act of changing a computation in an illegal type to be a
16// computation in two identical registers of a smaller type. For example,
17// implementing ppcf128 arithmetic in two f64 registers.
18//
19//===----------------------------------------------------------------------===//
20
21#include "LegalizeTypes.h"
22#include "llvm/Analysis/TargetLibraryInfo.h"
23#include "llvm/Support/ErrorHandling.h"
24#include "llvm/Support/raw_ostream.h"
25using namespace llvm;
26
27#define DEBUG_TYPE "legalize-types"
28
29/// GetFPLibCall - Return the right libcall for the given floating point type.
30/// FIXME: This is a local version of RTLIB::getFPLibCall that should be
31/// refactored away (see RTLIB::getPOWI for an example).
32static RTLIB::Libcall GetFPLibCall(EVT VT,
33 RTLIB::Libcall Call_F32,
34 RTLIB::Libcall Call_F64,
35 RTLIB::Libcall Call_F80,
36 RTLIB::Libcall Call_F128,
37 RTLIB::Libcall Call_PPCF128) {
38 return
39 VT == MVT::f32 ? Call_F32 :
40 VT == MVT::f64 ? Call_F64 :
41 VT == MVT::f80 ? Call_F80 :
42 VT == MVT::f128 ? Call_F128 :
43 VT == MVT::ppcf128 ? Call_PPCF128 :
44 RTLIB::UNKNOWN_LIBCALL;
45}
46
47//===----------------------------------------------------------------------===//
48// Convert Float Results to Integer
49//===----------------------------------------------------------------------===//
50
51void DAGTypeLegalizer::SoftenFloatResult(SDNode *N, unsigned ResNo) {
52 LLVM_DEBUG(dbgs() << "Soften float result " << ResNo << ": "; N->dump(&DAG));
53 SDValue R = SDValue();
54
55 switch (N->getOpcode()) {
56 default:
57#ifndef NDEBUG
58 dbgs() << "SoftenFloatResult #" << ResNo << ": ";
59 N->dump(&DAG); dbgs() << "\n";
60#endif
61 report_fatal_error("Do not know how to soften the result of this "
62 "operator!");
63
64 case ISD::ARITH_FENCE: R = SoftenFloatRes_ARITH_FENCE(N); break;
65 case ISD::MERGE_VALUES:R = SoftenFloatRes_MERGE_VALUES(N, ResNo); break;
66 case ISD::BITCAST: R = SoftenFloatRes_BITCAST(N); break;
67 case ISD::BUILD_PAIR: R = SoftenFloatRes_BUILD_PAIR(N); break;
68 case ISD::ConstantFP: R = SoftenFloatRes_ConstantFP(N); break;
69 case ISD::EXTRACT_VECTOR_ELT:
70 R = SoftenFloatRes_EXTRACT_VECTOR_ELT(N, ResNo); break;
71 case ISD::FABS: R = SoftenFloatRes_FABS(N); break;
72 case ISD::STRICT_FMINNUM:
73 case ISD::FMINNUM: R = SoftenFloatRes_FMINNUM(N); break;
74 case ISD::STRICT_FMAXNUM:
75 case ISD::FMAXNUM: R = SoftenFloatRes_FMAXNUM(N); break;
76 case ISD::STRICT_FADD:
77 case ISD::FADD: R = SoftenFloatRes_FADD(N); break;
78 case ISD::FCBRT: R = SoftenFloatRes_FCBRT(N); break;
79 case ISD::STRICT_FCEIL:
80 case ISD::FCEIL: R = SoftenFloatRes_FCEIL(N); break;
81 case ISD::FCOPYSIGN: R = SoftenFloatRes_FCOPYSIGN(N); break;
82 case ISD::STRICT_FCOS:
83 case ISD::FCOS: R = SoftenFloatRes_FCOS(N); break;
84 case ISD::STRICT_FDIV:
85 case ISD::FDIV: R = SoftenFloatRes_FDIV(N); break;
86 case ISD::STRICT_FEXP:
87 case ISD::FEXP: R = SoftenFloatRes_FEXP(N); break;
88 case ISD::STRICT_FEXP2:
89 case ISD::FEXP2: R = SoftenFloatRes_FEXP2(N); break;
90 case ISD::FEXP10: R = SoftenFloatRes_FEXP10(N); break;
91 case ISD::STRICT_FFLOOR:
92 case ISD::FFLOOR: R = SoftenFloatRes_FFLOOR(N); break;
93 case ISD::STRICT_FLOG:
94 case ISD::FLOG: R = SoftenFloatRes_FLOG(N); break;
95 case ISD::STRICT_FLOG2:
96 case ISD::FLOG2: R = SoftenFloatRes_FLOG2(N); break;
97 case ISD::STRICT_FLOG10:
98 case ISD::FLOG10: R = SoftenFloatRes_FLOG10(N); break;
99 case ISD::STRICT_FMA:
100 case ISD::FMA: R = SoftenFloatRes_FMA(N); break;
101 case ISD::STRICT_FMUL:
102 case ISD::FMUL: R = SoftenFloatRes_FMUL(N); break;
103 case ISD::STRICT_FNEARBYINT:
104 case ISD::FNEARBYINT: R = SoftenFloatRes_FNEARBYINT(N); break;
105 case ISD::FNEG: R = SoftenFloatRes_FNEG(N); break;
106 case ISD::STRICT_FP_EXTEND:
107 case ISD::FP_EXTEND: R = SoftenFloatRes_FP_EXTEND(N); break;
108 case ISD::STRICT_FP_ROUND:
109 case ISD::FP_ROUND: R = SoftenFloatRes_FP_ROUND(N); break;
110 case ISD::FP16_TO_FP: R = SoftenFloatRes_FP16_TO_FP(N); break;
111 case ISD::BF16_TO_FP: R = SoftenFloatRes_BF16_TO_FP(N); break;
112 case ISD::STRICT_FPOW:
113 case ISD::FPOW: R = SoftenFloatRes_FPOW(N); break;
114 case ISD::STRICT_FPOWI:
115 case ISD::FPOWI:
116 case ISD::FLDEXP:
117 case ISD::STRICT_FLDEXP: R = SoftenFloatRes_ExpOp(N); break;
118 case ISD::FFREXP:
119 R = SoftenFloatRes_FFREXP(N);
120 break;
121 case ISD::STRICT_FREM:
122 case ISD::FREM: R = SoftenFloatRes_FREM(N); break;
123 case ISD::STRICT_FRINT:
124 case ISD::FRINT: R = SoftenFloatRes_FRINT(N); break;
125 case ISD::STRICT_FROUND:
126 case ISD::FROUND: R = SoftenFloatRes_FROUND(N); break;
127 case ISD::STRICT_FROUNDEVEN:
128 case ISD::FROUNDEVEN: R = SoftenFloatRes_FROUNDEVEN(N); break;
129 case ISD::STRICT_FSIN:
130 case ISD::FSIN: R = SoftenFloatRes_FSIN(N); break;
131 case ISD::STRICT_FSQRT:
132 case ISD::FSQRT: R = SoftenFloatRes_FSQRT(N); break;
133 case ISD::STRICT_FSUB:
134 case ISD::FSUB: R = SoftenFloatRes_FSUB(N); break;
135 case ISD::STRICT_FTRUNC:
136 case ISD::FTRUNC: R = SoftenFloatRes_FTRUNC(N); break;
137 case ISD::LOAD: R = SoftenFloatRes_LOAD(N); break;
138 case ISD::ATOMIC_SWAP: R = BitcastToInt_ATOMIC_SWAP(N); break;
139 case ISD::SELECT: R = SoftenFloatRes_SELECT(N); break;
140 case ISD::SELECT_CC: R = SoftenFloatRes_SELECT_CC(N); break;
141 case ISD::FREEZE: R = SoftenFloatRes_FREEZE(N); break;
142 case ISD::STRICT_SINT_TO_FP:
143 case ISD::STRICT_UINT_TO_FP:
144 case ISD::SINT_TO_FP:
145 case ISD::UINT_TO_FP: R = SoftenFloatRes_XINT_TO_FP(N); break;
146 case ISD::UNDEF: R = SoftenFloatRes_UNDEF(N); break;
147 case ISD::VAARG: R = SoftenFloatRes_VAARG(N); break;
148 case ISD::VECREDUCE_FADD:
149 case ISD::VECREDUCE_FMUL:
150 case ISD::VECREDUCE_FMIN:
151 case ISD::VECREDUCE_FMAX:
152 case ISD::VECREDUCE_FMAXIMUM:
153 case ISD::VECREDUCE_FMINIMUM:
154 R = SoftenFloatRes_VECREDUCE(N);
155 break;
156 case ISD::VECREDUCE_SEQ_FADD:
157 case ISD::VECREDUCE_SEQ_FMUL:
158 R = SoftenFloatRes_VECREDUCE_SEQ(N);
159 break;
160 }
161
162 // If R is null, the sub-method took care of registering the result.
163 if (R.getNode()) {
164 assert(R.getNode() != N);
165 SetSoftenedFloat(SDValue(N, ResNo), R);
166 }
167}
168
169SDValue DAGTypeLegalizer::SoftenFloatRes_Unary(SDNode *N, RTLIB::Libcall LC) {
170 bool IsStrict = N->isStrictFPOpcode();
171 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
172 unsigned Offset = IsStrict ? 1 : 0;
173 assert(N->getNumOperands() == (1 + Offset) &&
174 "Unexpected number of operands!");
175 SDValue Op = GetSoftenedFloat(N->getOperand(0 + Offset));
176 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
177 TargetLowering::MakeLibCallOptions CallOptions;
178 EVT OpVT = N->getOperand(0 + Offset).getValueType();
179 CallOptions.setTypeListBeforeSoften(OpVT, N->getValueType(0), true);
180 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, NVT, Op,
181 CallOptions, SDLoc(N),
182 Chain);
183 if (IsStrict)
184 ReplaceValueWith(SDValue(N, 1), Tmp.second);
185 return Tmp.first;
186}
187
188SDValue DAGTypeLegalizer::SoftenFloatRes_Binary(SDNode *N, RTLIB::Libcall LC) {
189 bool IsStrict = N->isStrictFPOpcode();
190 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
191 unsigned Offset = IsStrict ? 1 : 0;
192 assert(N->getNumOperands() == (2 + Offset) &&
193 "Unexpected number of operands!");
194 SDValue Ops[2] = { GetSoftenedFloat(N->getOperand(0 + Offset)),
195 GetSoftenedFloat(N->getOperand(1 + Offset)) };
196 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
197 TargetLowering::MakeLibCallOptions CallOptions;
198 EVT OpsVT[2] = { N->getOperand(0 + Offset).getValueType(),
199 N->getOperand(1 + Offset).getValueType() };
200 CallOptions.setTypeListBeforeSoften(OpsVT, N->getValueType(0), true);
201 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, NVT, Ops,
202 CallOptions, SDLoc(N),
203 Chain);
204 if (IsStrict)
205 ReplaceValueWith(SDValue(N, 1), Tmp.second);
206 return Tmp.first;
207}
208
209SDValue DAGTypeLegalizer::SoftenFloatRes_BITCAST(SDNode *N) {
210 return BitConvertToInteger(N->getOperand(0));
211}
212
213SDValue DAGTypeLegalizer::SoftenFloatRes_FREEZE(SDNode *N) {
214 EVT Ty = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
215 return DAG.getNode(ISD::FREEZE, SDLoc(N), Ty,
216 GetSoftenedFloat(N->getOperand(0)));
217}
218
219SDValue DAGTypeLegalizer::SoftenFloatRes_ARITH_FENCE(SDNode *N) {
220 EVT Ty = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
221 SDValue NewFence = DAG.getNode(ISD::ARITH_FENCE, SDLoc(N), Ty,
222 GetSoftenedFloat(N->getOperand(0)));
223 return NewFence;
224}
225
226SDValue DAGTypeLegalizer::SoftenFloatRes_MERGE_VALUES(SDNode *N,
227 unsigned ResNo) {
228 SDValue Op = DisintegrateMERGE_VALUES(N, ResNo);
229 return BitConvertToInteger(Op);
230}
231
232SDValue DAGTypeLegalizer::SoftenFloatRes_BUILD_PAIR(SDNode *N) {
233 // Convert the inputs to integers, and build a new pair out of them.
234 return DAG.getNode(ISD::BUILD_PAIR, SDLoc(N),
235 TLI.getTypeToTransformTo(*DAG.getContext(),
236 N->getValueType(0)),
237 BitConvertToInteger(N->getOperand(0)),
238 BitConvertToInteger(N->getOperand(1)));
239}
240
241SDValue DAGTypeLegalizer::SoftenFloatRes_ConstantFP(SDNode *N) {
242 ConstantFPSDNode *CN = cast<ConstantFPSDNode>(N);
243 // In ppcf128, the high 64 bits are always first in memory regardless
244 // of Endianness. LLVM's APFloat representation is not Endian sensitive,
245 // and so always converts into a 128-bit APInt in a non-Endian-sensitive
246 // way. However, APInt's are serialized in an Endian-sensitive fashion,
247 // so on big-Endian targets, the two doubles are output in the wrong
248 // order. Fix this by manually flipping the order of the high 64 bits
249 // and the low 64 bits here.
250 if (DAG.getDataLayout().isBigEndian() &&
251 CN->getValueType(0).getSimpleVT() == llvm::MVT::ppcf128) {
252 uint64_t words[2] = { CN->getValueAPF().bitcastToAPInt().getRawData()[1],
253 CN->getValueAPF().bitcastToAPInt().getRawData()[0] };
254 APInt Val(128, words);
255 return DAG.getConstant(Val, SDLoc(CN),
256 TLI.getTypeToTransformTo(*DAG.getContext(),
257 CN->getValueType(0)));
258 } else {
259 return DAG.getConstant(CN->getValueAPF().bitcastToAPInt(), SDLoc(CN),
260 TLI.getTypeToTransformTo(*DAG.getContext(),
261 CN->getValueType(0)));
262 }
263}
264
265SDValue DAGTypeLegalizer::SoftenFloatRes_EXTRACT_VECTOR_ELT(SDNode *N, unsigned ResNo) {
266 SDValue NewOp = BitConvertVectorToIntegerVector(N->getOperand(0));
267 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SDLoc(N),
268 NewOp.getValueType().getVectorElementType(),
269 NewOp, N->getOperand(1));
270}
271
272SDValue DAGTypeLegalizer::SoftenFloatRes_FABS(SDNode *N) {
273 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
274 unsigned Size = NVT.getSizeInBits();
275
276 // Mask = ~(1 << (Size-1))
277 APInt API = APInt::getAllOnes(Size);
278 API.clearBit(Size - 1);
279 SDValue Mask = DAG.getConstant(API, SDLoc(N), NVT);
280 SDValue Op = GetSoftenedFloat(N->getOperand(0));
281 return DAG.getNode(ISD::AND, SDLoc(N), NVT, Op, Mask);
282}
283
284SDValue DAGTypeLegalizer::SoftenFloatRes_FMINNUM(SDNode *N) {
285 if (SDValue SelCC = TLI.createSelectForFMINNUM_FMAXNUM(N, DAG))
286 return SoftenFloatRes_SELECT_CC(SelCC.getNode());
287 return SoftenFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
288 RTLIB::FMIN_F32,
289 RTLIB::FMIN_F64,
290 RTLIB::FMIN_F80,
291 RTLIB::FMIN_F128,
292 RTLIB::FMIN_PPCF128));
293}
294
295SDValue DAGTypeLegalizer::SoftenFloatRes_FMAXNUM(SDNode *N) {
296 if (SDValue SelCC = TLI.createSelectForFMINNUM_FMAXNUM(N, DAG))
297 return SoftenFloatRes_SELECT_CC(SelCC.getNode());
298 return SoftenFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
299 RTLIB::FMAX_F32,
300 RTLIB::FMAX_F64,
301 RTLIB::FMAX_F80,
302 RTLIB::FMAX_F128,
303 RTLIB::FMAX_PPCF128));
304}
305
306SDValue DAGTypeLegalizer::SoftenFloatRes_FADD(SDNode *N) {
307 return SoftenFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
308 RTLIB::ADD_F32,
309 RTLIB::ADD_F64,
310 RTLIB::ADD_F80,
311 RTLIB::ADD_F128,
312 RTLIB::ADD_PPCF128));
313}
314
315SDValue DAGTypeLegalizer::SoftenFloatRes_FCBRT(SDNode *N) {
316 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
317 RTLIB::CBRT_F32,
318 RTLIB::CBRT_F64,
319 RTLIB::CBRT_F80,
320 RTLIB::CBRT_F128,
321 RTLIB::CBRT_PPCF128));
322}
323
324SDValue DAGTypeLegalizer::SoftenFloatRes_FCEIL(SDNode *N) {
325 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
326 RTLIB::CEIL_F32,
327 RTLIB::CEIL_F64,
328 RTLIB::CEIL_F80,
329 RTLIB::CEIL_F128,
330 RTLIB::CEIL_PPCF128));
331}
332
333SDValue DAGTypeLegalizer::SoftenFloatRes_FCOPYSIGN(SDNode *N) {
334 SDValue LHS = GetSoftenedFloat(N->getOperand(0));
335 SDValue RHS = BitConvertToInteger(N->getOperand(1));
336 SDLoc dl(N);
337
338 EVT LVT = LHS.getValueType();
339 EVT RVT = RHS.getValueType();
340
341 unsigned LSize = LVT.getSizeInBits();
342 unsigned RSize = RVT.getSizeInBits();
343
344 // First get the sign bit of second operand.
345 SDValue SignBit = DAG.getNode(
346 ISD::SHL, dl, RVT, DAG.getConstant(1, dl, RVT),
347 DAG.getConstant(RSize - 1, dl,
348 TLI.getShiftAmountTy(RVT, DAG.getDataLayout())));
349 SignBit = DAG.getNode(ISD::AND, dl, RVT, RHS, SignBit);
350
351 // Shift right or sign-extend it if the two operands have different types.
352 int SizeDiff = RVT.getSizeInBits() - LVT.getSizeInBits();
353 if (SizeDiff > 0) {
354 SignBit =
355 DAG.getNode(ISD::SRL, dl, RVT, SignBit,
356 DAG.getConstant(SizeDiff, dl,
357 TLI.getShiftAmountTy(SignBit.getValueType(),
358 DAG.getDataLayout())));
359 SignBit = DAG.getNode(ISD::TRUNCATE, dl, LVT, SignBit);
360 } else if (SizeDiff < 0) {
361 SignBit = DAG.getNode(ISD::ANY_EXTEND, dl, LVT, SignBit);
362 SignBit =
363 DAG.getNode(ISD::SHL, dl, LVT, SignBit,
364 DAG.getConstant(-SizeDiff, dl,
365 TLI.getShiftAmountTy(SignBit.getValueType(),
366 DAG.getDataLayout())));
367 }
368
369 // Clear the sign bit of the first operand.
370 SDValue Mask = DAG.getNode(
371 ISD::SHL, dl, LVT, DAG.getConstant(1, dl, LVT),
372 DAG.getConstant(LSize - 1, dl,
373 TLI.getShiftAmountTy(LVT, DAG.getDataLayout())));
374 Mask = DAG.getNode(ISD::SUB, dl, LVT, Mask, DAG.getConstant(1, dl, LVT));
375 LHS = DAG.getNode(ISD::AND, dl, LVT, LHS, Mask);
376
377 // Or the value with the sign bit.
378 return DAG.getNode(ISD::OR, dl, LVT, LHS, SignBit);
379}
380
381SDValue DAGTypeLegalizer::SoftenFloatRes_FCOS(SDNode *N) {
382 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
383 RTLIB::COS_F32,
384 RTLIB::COS_F64,
385 RTLIB::COS_F80,
386 RTLIB::COS_F128,
387 RTLIB::COS_PPCF128));
388}
389
390SDValue DAGTypeLegalizer::SoftenFloatRes_FDIV(SDNode *N) {
391 return SoftenFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
392 RTLIB::DIV_F32,
393 RTLIB::DIV_F64,
394 RTLIB::DIV_F80,
395 RTLIB::DIV_F128,
396 RTLIB::DIV_PPCF128));
397}
398
399SDValue DAGTypeLegalizer::SoftenFloatRes_FEXP(SDNode *N) {
400 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
401 RTLIB::EXP_F32,
402 RTLIB::EXP_F64,
403 RTLIB::EXP_F80,
404 RTLIB::EXP_F128,
405 RTLIB::EXP_PPCF128));
406}
407
408SDValue DAGTypeLegalizer::SoftenFloatRes_FEXP2(SDNode *N) {
409 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
410 RTLIB::EXP2_F32,
411 RTLIB::EXP2_F64,
412 RTLIB::EXP2_F80,
413 RTLIB::EXP2_F128,
414 RTLIB::EXP2_PPCF128));
415}
416
417SDValue DAGTypeLegalizer::SoftenFloatRes_FEXP10(SDNode *N) {
418 return SoftenFloatRes_Unary(
419 N,
420 GetFPLibCall(N->getValueType(0), RTLIB::EXP10_F32, RTLIB::EXP10_F64,
421 RTLIB::EXP10_F80, RTLIB::EXP10_F128, RTLIB::EXP10_PPCF128));
422}
423
424SDValue DAGTypeLegalizer::SoftenFloatRes_FFLOOR(SDNode *N) {
425 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
426 RTLIB::FLOOR_F32,
427 RTLIB::FLOOR_F64,
428 RTLIB::FLOOR_F80,
429 RTLIB::FLOOR_F128,
430 RTLIB::FLOOR_PPCF128));
431}
432
433SDValue DAGTypeLegalizer::SoftenFloatRes_FLOG(SDNode *N) {
434 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
435 RTLIB::LOG_F32,
436 RTLIB::LOG_F64,
437 RTLIB::LOG_F80,
438 RTLIB::LOG_F128,
439 RTLIB::LOG_PPCF128));
440}
441
442SDValue DAGTypeLegalizer::SoftenFloatRes_FLOG2(SDNode *N) {
443 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
444 RTLIB::LOG2_F32,
445 RTLIB::LOG2_F64,
446 RTLIB::LOG2_F80,
447 RTLIB::LOG2_F128,
448 RTLIB::LOG2_PPCF128));
449}
450
451SDValue DAGTypeLegalizer::SoftenFloatRes_FLOG10(SDNode *N) {
452 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
453 RTLIB::LOG10_F32,
454 RTLIB::LOG10_F64,
455 RTLIB::LOG10_F80,
456 RTLIB::LOG10_F128,
457 RTLIB::LOG10_PPCF128));
458}
459
460SDValue DAGTypeLegalizer::SoftenFloatRes_FMA(SDNode *N) {
461 bool IsStrict = N->isStrictFPOpcode();
462 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
463 unsigned Offset = IsStrict ? 1 : 0;
464 SDValue Ops[3] = { GetSoftenedFloat(N->getOperand(0 + Offset)),
465 GetSoftenedFloat(N->getOperand(1 + Offset)),
466 GetSoftenedFloat(N->getOperand(2 + Offset)) };
467 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
468 TargetLowering::MakeLibCallOptions CallOptions;
469 EVT OpsVT[3] = { N->getOperand(0 + Offset).getValueType(),
470 N->getOperand(1 + Offset).getValueType(),
471 N->getOperand(2 + Offset).getValueType() };
472 CallOptions.setTypeListBeforeSoften(OpsVT, N->getValueType(0), true);
473 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG,
474 GetFPLibCall(N->getValueType(0),
475 RTLIB::FMA_F32,
476 RTLIB::FMA_F64,
477 RTLIB::FMA_F80,
478 RTLIB::FMA_F128,
479 RTLIB::FMA_PPCF128),
480 NVT, Ops, CallOptions, SDLoc(N), Chain);
481 if (IsStrict)
482 ReplaceValueWith(SDValue(N, 1), Tmp.second);
483 return Tmp.first;
484}
485
486SDValue DAGTypeLegalizer::SoftenFloatRes_FMUL(SDNode *N) {
487 return SoftenFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
488 RTLIB::MUL_F32,
489 RTLIB::MUL_F64,
490 RTLIB::MUL_F80,
491 RTLIB::MUL_F128,
492 RTLIB::MUL_PPCF128));
493}
494
495SDValue DAGTypeLegalizer::SoftenFloatRes_FNEARBYINT(SDNode *N) {
496 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
497 RTLIB::NEARBYINT_F32,
498 RTLIB::NEARBYINT_F64,
499 RTLIB::NEARBYINT_F80,
500 RTLIB::NEARBYINT_F128,
501 RTLIB::NEARBYINT_PPCF128));
502}
503
504SDValue DAGTypeLegalizer::SoftenFloatRes_FNEG(SDNode *N) {
505 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
506 SDLoc dl(N);
507
508 // Expand Y = FNEG(X) -> Y = X ^ sign mask
509 APInt SignMask = APInt::getSignMask(NVT.getSizeInBits());
510 return DAG.getNode(ISD::XOR, dl, NVT, GetSoftenedFloat(N->getOperand(0)),
511 DAG.getConstant(SignMask, dl, NVT));
512}
513
514SDValue DAGTypeLegalizer::SoftenFloatRes_FP_EXTEND(SDNode *N) {
515 bool IsStrict = N->isStrictFPOpcode();
516 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
517 SDValue Op = N->getOperand(IsStrict ? 1 : 0);
518
519 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
520
521 if (getTypeAction(Op.getValueType()) == TargetLowering::TypePromoteFloat) {
522 Op = GetPromotedFloat(Op);
523 // If the promotion did the FP_EXTEND to the destination type for us,
524 // there's nothing left to do here.
525 if (Op.getValueType() == N->getValueType(0))
526 return BitConvertToInteger(Op);
527 }
528
529 // There's only a libcall for f16 -> f32 and shifting is only valid for bf16
530 // -> f32, so proceed in two stages. Also, it's entirely possible for both
531 // f16 and f32 to be legal, so use the fully hard-float FP_EXTEND rather
532 // than FP16_TO_FP.
533 if ((Op.getValueType() == MVT::f16 || Op.getValueType() == MVT::bf16) &&
534 N->getValueType(0) != MVT::f32) {
535 if (IsStrict) {
536 Op = DAG.getNode(ISD::STRICT_FP_EXTEND, SDLoc(N),
537 { MVT::f32, MVT::Other }, { Chain, Op });
538 Chain = Op.getValue(1);
539 } else {
540 Op = DAG.getNode(ISD::FP_EXTEND, SDLoc(N), MVT::f32, Op);
541 }
542 }
543
544 if (Op.getValueType() == MVT::bf16)
545 return SoftenFloatRes_BF16_TO_FP(N);
546
547 RTLIB::Libcall LC = RTLIB::getFPEXT(Op.getValueType(), N->getValueType(0));
548 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported FP_EXTEND!");
549 TargetLowering::MakeLibCallOptions CallOptions;
550 EVT OpVT = N->getOperand(IsStrict ? 1 : 0).getValueType();
551 CallOptions.setTypeListBeforeSoften(OpVT, N->getValueType(0), true);
552 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, NVT, Op,
553 CallOptions, SDLoc(N),
554 Chain);
555 if (IsStrict)
556 ReplaceValueWith(SDValue(N, 1), Tmp.second);
557 return Tmp.first;
558}
559
560// FIXME: Should we just use 'normal' FP_EXTEND / FP_TRUNC instead of special
561// nodes?
562SDValue DAGTypeLegalizer::SoftenFloatRes_FP16_TO_FP(SDNode *N) {
563 EVT MidVT = TLI.getTypeToTransformTo(*DAG.getContext(), MVT::f32);
564 SDValue Op = N->getOperand(0);
565 TargetLowering::MakeLibCallOptions CallOptions;
566 EVT OpsVT[1] = { N->getOperand(0).getValueType() };
567 CallOptions.setTypeListBeforeSoften(OpsVT, N->getValueType(0), true);
568 SDValue Res32 = TLI.makeLibCall(DAG, RTLIB::FPEXT_F16_F32, MidVT, Op,
569 CallOptions, SDLoc(N)).first;
570 if (N->getValueType(0) == MVT::f32)
571 return Res32;
572
573 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
574 RTLIB::Libcall LC = RTLIB::getFPEXT(MVT::f32, N->getValueType(0));
575 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported FP_EXTEND!");
576 return TLI.makeLibCall(DAG, LC, NVT, Res32, CallOptions, SDLoc(N)).first;
577}
578
579// FIXME: Should we just use 'normal' FP_EXTEND / FP_TRUNC instead of special
580// nodes?
581SDValue DAGTypeLegalizer::SoftenFloatRes_BF16_TO_FP(SDNode *N) {
582 assert(N->getValueType(0) == MVT::f32 &&
583 "Can only soften BF16_TO_FP with f32 result");
584 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), MVT::f32);
585 SDValue Op = N->getOperand(0);
586 SDLoc DL(N);
587 Op = DAG.getNode(ISD::ANY_EXTEND, DL, NVT,
588 DAG.getNode(ISD::BITCAST, DL, MVT::i16, Op));
589 SDValue Res = DAG.getNode(ISD::SHL, DL, NVT, Op,
590 DAG.getShiftAmountConstant(16, NVT, DL));
591 return Res;
592}
593
594SDValue DAGTypeLegalizer::SoftenFloatRes_FP_ROUND(SDNode *N) {
595 bool IsStrict = N->isStrictFPOpcode();
596 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
597 SDValue Op = N->getOperand(IsStrict ? 1 : 0);
598 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
599 RTLIB::Libcall LC = RTLIB::getFPROUND(Op.getValueType(), N->getValueType(0));
600 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported FP_ROUND!");
601 TargetLowering::MakeLibCallOptions CallOptions;
602 EVT OpVT = N->getOperand(IsStrict ? 1 : 0).getValueType();
603 CallOptions.setTypeListBeforeSoften(OpVT, N->getValueType(0), true);
604 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, NVT, Op,
605 CallOptions, SDLoc(N),
606 Chain);
607 if (IsStrict)
608 ReplaceValueWith(SDValue(N, 1), Tmp.second);
609 return Tmp.first;
610}
611
612SDValue DAGTypeLegalizer::SoftenFloatRes_FPOW(SDNode *N) {
613 return SoftenFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
614 RTLIB::POW_F32,
615 RTLIB::POW_F64,
616 RTLIB::POW_F80,
617 RTLIB::POW_F128,
618 RTLIB::POW_PPCF128));
619}
620
621SDValue DAGTypeLegalizer::SoftenFloatRes_ExpOp(SDNode *N) {
622 bool IsStrict = N->isStrictFPOpcode();
623 unsigned Offset = IsStrict ? 1 : 0;
624 assert((N->getOperand(1 + Offset).getValueType() == MVT::i16 ||
625 N->getOperand(1 + Offset).getValueType() == MVT::i32) &&
626 "Unsupported power type!");
627 bool IsPowI =
628 N->getOpcode() == ISD::FPOWI || N->getOpcode() == ISD::STRICT_FPOWI;
629
630 RTLIB::Libcall LC = IsPowI ? RTLIB::getPOWI(N->getValueType(0))
631 : RTLIB::getLDEXP(N->getValueType(0));
632 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected fpowi.");
633 if (!TLI.getLibcallName(LC)) {
634 // Some targets don't have a powi libcall; use pow instead.
635 // FIXME: Implement this if some target needs it.
636 DAG.getContext()->emitError("Don't know how to soften fpowi to fpow");
637 return DAG.getUNDEF(N->getValueType(0));
638 }
639
640 if (DAG.getLibInfo().getIntSize() !=
641 N->getOperand(1 + Offset).getValueType().getSizeInBits()) {
642 // If the exponent does not match with sizeof(int) a libcall to RTLIB::POWI
643 // would use the wrong type for the argument.
644 DAG.getContext()->emitError("POWI exponent does not match sizeof(int)");
645 return DAG.getUNDEF(N->getValueType(0));
646 }
647
648 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
649 SDValue Ops[2] = { GetSoftenedFloat(N->getOperand(0 + Offset)),
650 N->getOperand(1 + Offset) };
651 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
652 TargetLowering::MakeLibCallOptions CallOptions;
653 EVT OpsVT[2] = { N->getOperand(0 + Offset).getValueType(),
654 N->getOperand(1 + Offset).getValueType() };
655 CallOptions.setTypeListBeforeSoften(OpsVT, N->getValueType(0), true);
656 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, NVT, Ops,
657 CallOptions, SDLoc(N),
658 Chain);
659 if (IsStrict)
660 ReplaceValueWith(SDValue(N, 1), Tmp.second);
661 return Tmp.first;
662}
663
664SDValue DAGTypeLegalizer::SoftenFloatRes_FFREXP(SDNode *N) {
665 assert(!N->isStrictFPOpcode() && "strictfp not implemented for frexp");
666 EVT VT0 = N->getValueType(0);
667 EVT VT1 = N->getValueType(1);
668 RTLIB::Libcall LC = RTLIB::getFREXP(VT0);
669
670 if (DAG.getLibInfo().getIntSize() != VT1.getSizeInBits()) {
671 // If the exponent does not match with sizeof(int) a libcall would use the
672 // wrong type for the argument.
673 // TODO: Should be able to handle mismatches.
674 DAG.getContext()->emitError("ffrexp exponent does not match sizeof(int)");
675 return DAG.getUNDEF(N->getValueType(0));
676 }
677
678 EVT NVT0 = TLI.getTypeToTransformTo(*DAG.getContext(), VT0);
679 SDValue StackSlot = DAG.CreateStackTemporary(VT1);
680
681 SDLoc DL(N);
682
683 TargetLowering::MakeLibCallOptions CallOptions;
684 SDValue Ops[2] = {GetSoftenedFloat(N->getOperand(0)), StackSlot};
685 EVT OpsVT[2] = {VT0, StackSlot.getValueType()};
686
687 // TODO: setTypeListBeforeSoften can't properly express multiple return types,
688 // but we only really need to handle the 0th one for softening anyway.
689 CallOptions.setTypeListBeforeSoften({OpsVT}, VT0, true);
690
691 auto [ReturnVal, Chain] = TLI.makeLibCall(DAG, LC, NVT0, Ops, CallOptions, DL,
692 /*Chain=*/SDValue());
693 int FrameIdx = cast<FrameIndexSDNode>(StackSlot)->getIndex();
694 auto PtrInfo =
695 MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), FrameIdx);
696
697 SDValue LoadExp = DAG.getLoad(VT1, DL, Chain, StackSlot, PtrInfo);
698
699 ReplaceValueWith(SDValue(N, 1), LoadExp);
700 return ReturnVal;
701}
702
703SDValue DAGTypeLegalizer::SoftenFloatRes_FREM(SDNode *N) {
704 return SoftenFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
705 RTLIB::REM_F32,
706 RTLIB::REM_F64,
707 RTLIB::REM_F80,
708 RTLIB::REM_F128,
709 RTLIB::REM_PPCF128));
710}
711
712SDValue DAGTypeLegalizer::SoftenFloatRes_FRINT(SDNode *N) {
713 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
714 RTLIB::RINT_F32,
715 RTLIB::RINT_F64,
716 RTLIB::RINT_F80,
717 RTLIB::RINT_F128,
718 RTLIB::RINT_PPCF128));
719}
720
721SDValue DAGTypeLegalizer::SoftenFloatRes_FROUND(SDNode *N) {
722 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
723 RTLIB::ROUND_F32,
724 RTLIB::ROUND_F64,
725 RTLIB::ROUND_F80,
726 RTLIB::ROUND_F128,
727 RTLIB::ROUND_PPCF128));
728}
729
730SDValue DAGTypeLegalizer::SoftenFloatRes_FROUNDEVEN(SDNode *N) {
731 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
732 RTLIB::ROUNDEVEN_F32,
733 RTLIB::ROUNDEVEN_F64,
734 RTLIB::ROUNDEVEN_F80,
735 RTLIB::ROUNDEVEN_F128,
736 RTLIB::ROUNDEVEN_PPCF128));
737}
738
739SDValue DAGTypeLegalizer::SoftenFloatRes_FSIN(SDNode *N) {
740 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
741 RTLIB::SIN_F32,
742 RTLIB::SIN_F64,
743 RTLIB::SIN_F80,
744 RTLIB::SIN_F128,
745 RTLIB::SIN_PPCF128));
746}
747
748SDValue DAGTypeLegalizer::SoftenFloatRes_FSQRT(SDNode *N) {
749 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
750 RTLIB::SQRT_F32,
751 RTLIB::SQRT_F64,
752 RTLIB::SQRT_F80,
753 RTLIB::SQRT_F128,
754 RTLIB::SQRT_PPCF128));
755}
756
757SDValue DAGTypeLegalizer::SoftenFloatRes_FSUB(SDNode *N) {
758 return SoftenFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
759 RTLIB::SUB_F32,
760 RTLIB::SUB_F64,
761 RTLIB::SUB_F80,
762 RTLIB::SUB_F128,
763 RTLIB::SUB_PPCF128));
764}
765
766SDValue DAGTypeLegalizer::SoftenFloatRes_FTRUNC(SDNode *N) {
767 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
768 RTLIB::TRUNC_F32,
769 RTLIB::TRUNC_F64,
770 RTLIB::TRUNC_F80,
771 RTLIB::TRUNC_F128,
772 RTLIB::TRUNC_PPCF128));
773}
774
775SDValue DAGTypeLegalizer::SoftenFloatRes_LOAD(SDNode *N) {
776 LoadSDNode *L = cast<LoadSDNode>(N);
777 EVT VT = N->getValueType(0);
778 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
779 SDLoc dl(N);
780
781 auto MMOFlags =
782 L->getMemOperand()->getFlags() &
783 ~(MachineMemOperand::MOInvariant | MachineMemOperand::MODereferenceable);
784 SDValue NewL;
785 if (L->getExtensionType() == ISD::NON_EXTLOAD) {
786 NewL = DAG.getLoad(L->getAddressingMode(), L->getExtensionType(), NVT, dl,
787 L->getChain(), L->getBasePtr(), L->getOffset(),
788 L->getPointerInfo(), NVT, L->getOriginalAlign(),
789 MMOFlags, L->getAAInfo());
790 // Legalized the chain result - switch anything that used the old chain to
791 // use the new one.
792 ReplaceValueWith(SDValue(N, 1), NewL.getValue(1));
793 return NewL;
794 }
795
796 // Do a non-extending load followed by FP_EXTEND.
797 NewL = DAG.getLoad(L->getAddressingMode(), ISD::NON_EXTLOAD, L->getMemoryVT(),
798 dl, L->getChain(), L->getBasePtr(), L->getOffset(),
799 L->getPointerInfo(), L->getMemoryVT(),
800 L->getOriginalAlign(), MMOFlags, L->getAAInfo());
801 // Legalized the chain result - switch anything that used the old chain to
802 // use the new one.
803 ReplaceValueWith(SDValue(N, 1), NewL.getValue(1));
804 auto ExtendNode = DAG.getNode(ISD::FP_EXTEND, dl, VT, NewL);
805 return BitConvertToInteger(ExtendNode);
806}
807
808SDValue DAGTypeLegalizer::SoftenFloatRes_SELECT(SDNode *N) {
809 SDValue LHS = GetSoftenedFloat(N->getOperand(1));
810 SDValue RHS = GetSoftenedFloat(N->getOperand(2));
811 return DAG.getSelect(SDLoc(N),
812 LHS.getValueType(), N->getOperand(0), LHS, RHS);
813}
814
815SDValue DAGTypeLegalizer::SoftenFloatRes_SELECT_CC(SDNode *N) {
816 SDValue LHS = GetSoftenedFloat(N->getOperand(2));
817 SDValue RHS = GetSoftenedFloat(N->getOperand(3));
818 return DAG.getNode(ISD::SELECT_CC, SDLoc(N),
819 LHS.getValueType(), N->getOperand(0),
820 N->getOperand(1), LHS, RHS, N->getOperand(4));
821}
822
823SDValue DAGTypeLegalizer::SoftenFloatRes_UNDEF(SDNode *N) {
824 return DAG.getUNDEF(TLI.getTypeToTransformTo(*DAG.getContext(),
825 N->getValueType(0)));
826}
827
828SDValue DAGTypeLegalizer::SoftenFloatRes_VAARG(SDNode *N) {
829 SDValue Chain = N->getOperand(0); // Get the chain.
830 SDValue Ptr = N->getOperand(1); // Get the pointer.
831 EVT VT = N->getValueType(0);
832 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
833 SDLoc dl(N);
834
835 SDValue NewVAARG;
836 NewVAARG = DAG.getVAArg(NVT, dl, Chain, Ptr, N->getOperand(2),
837 N->getConstantOperandVal(3));
838
839 // Legalized the chain result - switch anything that used the old chain to
840 // use the new one.
841 if (N != NewVAARG.getValue(1).getNode())
842 ReplaceValueWith(SDValue(N, 1), NewVAARG.getValue(1));
843 return NewVAARG;
844}
845
846SDValue DAGTypeLegalizer::SoftenFloatRes_XINT_TO_FP(SDNode *N) {
847 bool IsStrict = N->isStrictFPOpcode();
848 bool Signed = N->getOpcode() == ISD::SINT_TO_FP ||
849 N->getOpcode() == ISD::STRICT_SINT_TO_FP;
850 EVT SVT = N->getOperand(IsStrict ? 1 : 0).getValueType();
851 EVT RVT = N->getValueType(0);
852 EVT NVT = EVT();
853 SDLoc dl(N);
854
855 // If the input is not legal, eg: i1 -> fp, then it needs to be promoted to
856 // a larger type, eg: i8 -> fp. Even if it is legal, no libcall may exactly
857 // match. Look for an appropriate libcall.
858 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
859 for (unsigned t = MVT::FIRST_INTEGER_VALUETYPE;
860 t <= MVT::LAST_INTEGER_VALUETYPE && LC == RTLIB::UNKNOWN_LIBCALL; ++t) {
861 NVT = (MVT::SimpleValueType)t;
862 // The source needs to big enough to hold the operand.
863 if (NVT.bitsGE(SVT))
864 LC = Signed ? RTLIB::getSINTTOFP(NVT, RVT):RTLIB::getUINTTOFP (NVT, RVT);
865 }
866 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported XINT_TO_FP!");
867
868 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
869 // Sign/zero extend the argument if the libcall takes a larger type.
870 SDValue Op = DAG.getNode(Signed ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND, dl,
871 NVT, N->getOperand(IsStrict ? 1 : 0));
872 TargetLowering::MakeLibCallOptions CallOptions;
873 CallOptions.setSExt(Signed);
874 CallOptions.setTypeListBeforeSoften(SVT, RVT, true);
875 std::pair<SDValue, SDValue> Tmp =
876 TLI.makeLibCall(DAG, LC, TLI.getTypeToTransformTo(*DAG.getContext(), RVT),
877 Op, CallOptions, dl, Chain);
878
879 if (IsStrict)
880 ReplaceValueWith(SDValue(N, 1), Tmp.second);
881 return Tmp.first;
882}
883
884SDValue DAGTypeLegalizer::SoftenFloatRes_VECREDUCE(SDNode *N) {
885 // Expand and soften recursively.
886 ReplaceValueWith(SDValue(N, 0), TLI.expandVecReduce(N, DAG));
887 return SDValue();
888}
889
890SDValue DAGTypeLegalizer::SoftenFloatRes_VECREDUCE_SEQ(SDNode *N) {
891 ReplaceValueWith(SDValue(N, 0), TLI.expandVecReduceSeq(N, DAG));
892 return SDValue();
893}
894
895//===----------------------------------------------------------------------===//
896// Convert Float Operand to Integer
897//===----------------------------------------------------------------------===//
898
899bool DAGTypeLegalizer::SoftenFloatOperand(SDNode *N, unsigned OpNo) {
900 LLVM_DEBUG(dbgs() << "Soften float operand " << OpNo << ": "; N->dump(&DAG));
901 SDValue Res = SDValue();
902
903 switch (N->getOpcode()) {
904 default:
905#ifndef NDEBUG
906 dbgs() << "SoftenFloatOperand Op #" << OpNo << ": ";
907 N->dump(&DAG); dbgs() << "\n";
908#endif
909 report_fatal_error("Do not know how to soften this operator's operand!");
910
911 case ISD::BITCAST: Res = SoftenFloatOp_BITCAST(N); break;
912 case ISD::BR_CC: Res = SoftenFloatOp_BR_CC(N); break;
913 case ISD::STRICT_FP_TO_FP16:
914 case ISD::FP_TO_FP16: // Same as FP_ROUND for softening purposes
915 case ISD::FP_TO_BF16:
916 case ISD::STRICT_FP_ROUND:
917 case ISD::FP_ROUND: Res = SoftenFloatOp_FP_ROUND(N); break;
918 case ISD::STRICT_FP_TO_SINT:
919 case ISD::STRICT_FP_TO_UINT:
920 case ISD::FP_TO_SINT:
921 case ISD::FP_TO_UINT: Res = SoftenFloatOp_FP_TO_XINT(N); break;
922 case ISD::FP_TO_SINT_SAT:
923 case ISD::FP_TO_UINT_SAT:
924 Res = SoftenFloatOp_FP_TO_XINT_SAT(N); break;
925 case ISD::STRICT_LROUND:
926 case ISD::LROUND: Res = SoftenFloatOp_LROUND(N); break;
927 case ISD::STRICT_LLROUND:
928 case ISD::LLROUND: Res = SoftenFloatOp_LLROUND(N); break;
929 case ISD::STRICT_LRINT:
930 case ISD::LRINT: Res = SoftenFloatOp_LRINT(N); break;
931 case ISD::STRICT_LLRINT:
932 case ISD::LLRINT: Res = SoftenFloatOp_LLRINT(N); break;
933 case ISD::SELECT_CC: Res = SoftenFloatOp_SELECT_CC(N); break;
934 case ISD::STRICT_FSETCC:
935 case ISD::STRICT_FSETCCS:
936 case ISD::SETCC: Res = SoftenFloatOp_SETCC(N); break;
937 case ISD::STORE: Res = SoftenFloatOp_STORE(N, OpNo); break;
938 case ISD::FCOPYSIGN: Res = SoftenFloatOp_FCOPYSIGN(N); break;
939 }
940
941 // If the result is null, the sub-method took care of registering results etc.
942 if (!Res.getNode()) return false;
943
944 // If the result is N, the sub-method updated N in place. Tell the legalizer
945 // core about this to re-analyze.
946 if (Res.getNode() == N)
947 return true;
948
949 assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 &&
950 "Invalid operand softening");
951
952 ReplaceValueWith(SDValue(N, 0), Res);
953 return false;
954}
955
956SDValue DAGTypeLegalizer::SoftenFloatOp_BITCAST(SDNode *N) {
957 SDValue Op0 = GetSoftenedFloat(N->getOperand(0));
958
959 return DAG.getNode(ISD::BITCAST, SDLoc(N), N->getValueType(0), Op0);
960}
961
962SDValue DAGTypeLegalizer::SoftenFloatOp_FP_ROUND(SDNode *N) {
963 // We actually deal with the partially-softened FP_TO_FP16 node too, which
964 // returns an i16 so doesn't meet the constraints necessary for FP_ROUND.
965 assert(N->getOpcode() == ISD::FP_ROUND || N->getOpcode() == ISD::FP_TO_FP16 ||
966 N->getOpcode() == ISD::STRICT_FP_TO_FP16 ||
967 N->getOpcode() == ISD::FP_TO_BF16 ||
968 N->getOpcode() == ISD::STRICT_FP_ROUND);
969
970 bool IsStrict = N->isStrictFPOpcode();
971 SDValue Op = N->getOperand(IsStrict ? 1 : 0);
972 EVT SVT = Op.getValueType();
973 EVT RVT = N->getValueType(0);
974 EVT FloatRVT = RVT;
975 if (N->getOpcode() == ISD::FP_TO_FP16 ||
976 N->getOpcode() == ISD::STRICT_FP_TO_FP16)
977 FloatRVT = MVT::f16;
978 else if (N->getOpcode() == ISD::FP_TO_BF16)
979 FloatRVT = MVT::bf16;
980
981 RTLIB::Libcall LC = RTLIB::getFPROUND(SVT, FloatRVT);
982 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported FP_ROUND libcall");
983
984 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
985 Op = GetSoftenedFloat(Op);
986 TargetLowering::MakeLibCallOptions CallOptions;
987 CallOptions.setTypeListBeforeSoften(SVT, RVT, true);
988 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, RVT, Op,
989 CallOptions, SDLoc(N),
990 Chain);
991 if (IsStrict) {
992 ReplaceValueWith(SDValue(N, 1), Tmp.second);
993 ReplaceValueWith(SDValue(N, 0), Tmp.first);
994 return SDValue();
995 }
996 return Tmp.first;
997}
998
999SDValue DAGTypeLegalizer::SoftenFloatOp_BR_CC(SDNode *N) {
1000 SDValue NewLHS = N->getOperand(2), NewRHS = N->getOperand(3);
1001 ISD::CondCode CCCode = cast<CondCodeSDNode>(N->getOperand(1))->get();
1002
1003 EVT VT = NewLHS.getValueType();
1004 NewLHS = GetSoftenedFloat(NewLHS);
1005 NewRHS = GetSoftenedFloat(NewRHS);
1006 TLI.softenSetCCOperands(DAG, VT, NewLHS, NewRHS, CCCode, SDLoc(N),
1007 N->getOperand(2), N->getOperand(3));
1008
1009 // If softenSetCCOperands returned a scalar, we need to compare the result
1010 // against zero to select between true and false values.
1011 if (!NewRHS.getNode()) {
1012 NewRHS = DAG.getConstant(0, SDLoc(N), NewLHS.getValueType());
1013 CCCode = ISD::SETNE;
1014 }
1015
1016 // Update N to have the operands specified.
1017 return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0),
1018 DAG.getCondCode(CCCode), NewLHS, NewRHS,
1019 N->getOperand(4)),
1020 0);
1021}
1022
1023// Even if the result type is legal, no libcall may exactly match. (e.g. We
1024// don't have FP-i8 conversions) This helper method looks for an appropriate
1025// promoted libcall.
1026static RTLIB::Libcall findFPToIntLibcall(EVT SrcVT, EVT RetVT, EVT &Promoted,
1027 bool Signed) {
1028 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
1029 for (unsigned IntVT = MVT::FIRST_INTEGER_VALUETYPE;
1030 IntVT <= MVT::LAST_INTEGER_VALUETYPE && LC == RTLIB::UNKNOWN_LIBCALL;
1031 ++IntVT) {
1032 Promoted = (MVT::SimpleValueType)IntVT;
1033 // The type needs to big enough to hold the result.
1034 if (Promoted.bitsGE(RetVT))
1035 LC = Signed ? RTLIB::getFPTOSINT(SrcVT, Promoted)
1036 : RTLIB::getFPTOUINT(SrcVT, Promoted);
1037 }
1038 return LC;
1039}
1040
1041SDValue DAGTypeLegalizer::SoftenFloatOp_FP_TO_XINT(SDNode *N) {
1042 bool IsStrict = N->isStrictFPOpcode();
1043 bool Signed = N->getOpcode() == ISD::FP_TO_SINT ||
1044 N->getOpcode() == ISD::STRICT_FP_TO_SINT;
1045
1046 SDValue Op = N->getOperand(IsStrict ? 1 : 0);
1047 EVT SVT = Op.getValueType();
1048 EVT RVT = N->getValueType(0);
1049 EVT NVT = EVT();
1050 SDLoc dl(N);
1051
1052 // If the result is not legal, eg: fp -> i1, then it needs to be promoted to
1053 // a larger type, eg: fp -> i32. Even if it is legal, no libcall may exactly
1054 // match, eg. we don't have fp -> i8 conversions.
1055 // Look for an appropriate libcall.
1056 RTLIB::Libcall LC = findFPToIntLibcall(SVT, RVT, NVT, Signed);
1057 assert(LC != RTLIB::UNKNOWN_LIBCALL && NVT.isSimple() &&
1058 "Unsupported FP_TO_XINT!");
1059
1060 Op = GetSoftenedFloat(Op);
1061 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
1062 TargetLowering::MakeLibCallOptions CallOptions;
1063 CallOptions.setTypeListBeforeSoften(SVT, RVT, true);
1064 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, NVT, Op,
1065 CallOptions, dl, Chain);
1066
1067 // Truncate the result if the libcall returns a larger type.
1068 SDValue Res = DAG.getNode(ISD::TRUNCATE, dl, RVT, Tmp.first);
1069
1070 if (!IsStrict)
1071 return Res;
1072
1073 ReplaceValueWith(SDValue(N, 1), Tmp.second);
1074 ReplaceValueWith(SDValue(N, 0), Res);
1075 return SDValue();
1076}
1077
1078SDValue DAGTypeLegalizer::SoftenFloatOp_FP_TO_XINT_SAT(SDNode *N) {
1079 SDValue Res = TLI.expandFP_TO_INT_SAT(N, DAG);
1080 return Res;
1081}
1082
1083SDValue DAGTypeLegalizer::SoftenFloatOp_SELECT_CC(SDNode *N) {
1084 SDValue NewLHS = N->getOperand(0), NewRHS = N->getOperand(1);
1085 ISD::CondCode CCCode = cast<CondCodeSDNode>(N->getOperand(4))->get();
1086
1087 EVT VT = NewLHS.getValueType();
1088 NewLHS = GetSoftenedFloat(NewLHS);
1089 NewRHS = GetSoftenedFloat(NewRHS);
1090 TLI.softenSetCCOperands(DAG, VT, NewLHS, NewRHS, CCCode, SDLoc(N),
1091 N->getOperand(0), N->getOperand(1));
1092
1093 // If softenSetCCOperands returned a scalar, we need to compare the result
1094 // against zero to select between true and false values.
1095 if (!NewRHS.getNode()) {
1096 NewRHS = DAG.getConstant(0, SDLoc(N), NewLHS.getValueType());
1097 CCCode = ISD::SETNE;
1098 }
1099
1100 // Update N to have the operands specified.
1101 return SDValue(DAG.UpdateNodeOperands(N, NewLHS, NewRHS,
1102 N->getOperand(2), N->getOperand(3),
1103 DAG.getCondCode(CCCode)),
1104 0);
1105}
1106
1107SDValue DAGTypeLegalizer::SoftenFloatOp_SETCC(SDNode *N) {
1108 bool IsStrict = N->isStrictFPOpcode();
1109 SDValue Op0 = N->getOperand(IsStrict ? 1 : 0);
1110 SDValue Op1 = N->getOperand(IsStrict ? 2 : 1);
1111 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
1112 ISD::CondCode CCCode =
1113 cast<CondCodeSDNode>(N->getOperand(IsStrict ? 3 : 2))->get();
1114
1115 EVT VT = Op0.getValueType();
1116 SDValue NewLHS = GetSoftenedFloat(Op0);
1117 SDValue NewRHS = GetSoftenedFloat(Op1);
1118 TLI.softenSetCCOperands(DAG, VT, NewLHS, NewRHS, CCCode, SDLoc(N), Op0, Op1,
1119 Chain, N->getOpcode() == ISD::STRICT_FSETCCS);
1120
1121 // Update N to have the operands specified.
1122 if (NewRHS.getNode()) {
1123 if (IsStrict)
1124 NewLHS = DAG.getNode(ISD::SETCC, SDLoc(N), N->getValueType(0), NewLHS,
1125 NewRHS, DAG.getCondCode(CCCode));
1126 else
1127 return SDValue(DAG.UpdateNodeOperands(N, NewLHS, NewRHS,
1128 DAG.getCondCode(CCCode)), 0);
1129 }
1130
1131 // Otherwise, softenSetCCOperands returned a scalar, use it.
1132 assert((NewRHS.getNode() || NewLHS.getValueType() == N->getValueType(0)) &&
1133 "Unexpected setcc expansion!");
1134
1135 if (IsStrict) {
1136 ReplaceValueWith(SDValue(N, 0), NewLHS);
1137 ReplaceValueWith(SDValue(N, 1), Chain);
1138 return SDValue();
1139 }
1140 return NewLHS;
1141}
1142
1143SDValue DAGTypeLegalizer::SoftenFloatOp_STORE(SDNode *N, unsigned OpNo) {
1144 assert(ISD::isUNINDEXEDStore(N) && "Indexed store during type legalization!");
1145 assert(OpNo == 1 && "Can only soften the stored value!");
1146 StoreSDNode *ST = cast<StoreSDNode>(N);
1147 SDValue Val = ST->getValue();
1148 SDLoc dl(N);
1149
1150 if (ST->isTruncatingStore())
1151 // Do an FP_ROUND followed by a non-truncating store.
1152 Val = BitConvertToInteger(
1153 DAG.getNode(ISD::FP_ROUND, dl, ST->getMemoryVT(), Val,
1154 DAG.getIntPtrConstant(0, dl, /*isTarget=*/true)));
1155 else
1156 Val = GetSoftenedFloat(Val);
1157
1158 return DAG.getStore(ST->getChain(), dl, Val, ST->getBasePtr(),
1159 ST->getMemOperand());
1160}
1161
1162SDValue DAGTypeLegalizer::SoftenFloatOp_FCOPYSIGN(SDNode *N) {
1163 SDValue LHS = N->getOperand(0);
1164 SDValue RHS = BitConvertToInteger(N->getOperand(1));
1165 SDLoc dl(N);
1166
1167 EVT LVT = LHS.getValueType();
1168 EVT ILVT = EVT::getIntegerVT(*DAG.getContext(), LVT.getSizeInBits());
1169 EVT RVT = RHS.getValueType();
1170
1171 unsigned LSize = LVT.getSizeInBits();
1172 unsigned RSize = RVT.getSizeInBits();
1173
1174 // Shift right or sign-extend it if the two operands have different types.
1175 int SizeDiff = RSize - LSize;
1176 if (SizeDiff > 0) {
1177 RHS =
1178 DAG.getNode(ISD::SRL, dl, RVT, RHS,
1179 DAG.getConstant(SizeDiff, dl,
1180 TLI.getShiftAmountTy(RHS.getValueType(),
1181 DAG.getDataLayout())));
1182 RHS = DAG.getNode(ISD::TRUNCATE, dl, ILVT, RHS);
1183 } else if (SizeDiff < 0) {
1184 RHS = DAG.getNode(ISD::ANY_EXTEND, dl, LVT, RHS);
1185 RHS =
1186 DAG.getNode(ISD::SHL, dl, ILVT, RHS,
1187 DAG.getConstant(-SizeDiff, dl,
1188 TLI.getShiftAmountTy(RHS.getValueType(),
1189 DAG.getDataLayout())));
1190 }
1191
1192 RHS = DAG.getBitcast(LVT, RHS);
1193 return DAG.getNode(ISD::FCOPYSIGN, dl, LVT, LHS, RHS);
1194}
1195
1196SDValue DAGTypeLegalizer::SoftenFloatOp_Unary(SDNode *N, RTLIB::Libcall LC) {
1197 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
1198 bool IsStrict = N->isStrictFPOpcode();
1199 unsigned Offset = IsStrict ? 1 : 0;
1200 SDValue Op = GetSoftenedFloat(N->getOperand(0 + Offset));
1201 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
1202 TargetLowering::MakeLibCallOptions CallOptions;
1203 EVT OpVT = N->getOperand(0 + Offset).getValueType();
1204 CallOptions.setTypeListBeforeSoften(OpVT, N->getValueType(0), true);
1205 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, NVT, Op,
1206 CallOptions, SDLoc(N),
1207 Chain);
1208 if (IsStrict) {
1209 ReplaceValueWith(SDValue(N, 1), Tmp.second);
1210 ReplaceValueWith(SDValue(N, 0), Tmp.first);
1211 return SDValue();
1212 }
1213
1214 return Tmp.first;
1215}
1216
1217SDValue DAGTypeLegalizer::SoftenFloatOp_LROUND(SDNode *N) {
1218 EVT OpVT = N->getOperand(N->isStrictFPOpcode() ? 1 : 0).getValueType();
1219 return SoftenFloatOp_Unary(N, GetFPLibCall(OpVT,
1220 RTLIB::LROUND_F32,
1221 RTLIB::LROUND_F64,
1222 RTLIB::LROUND_F80,
1223 RTLIB::LROUND_F128,
1224 RTLIB::LROUND_PPCF128));
1225}
1226
1227SDValue DAGTypeLegalizer::SoftenFloatOp_LLROUND(SDNode *N) {
1228 EVT OpVT = N->getOperand(N->isStrictFPOpcode() ? 1 : 0).getValueType();
1229 return SoftenFloatOp_Unary(N, GetFPLibCall(OpVT,
1230 RTLIB::LLROUND_F32,
1231 RTLIB::LLROUND_F64,
1232 RTLIB::LLROUND_F80,
1233 RTLIB::LLROUND_F128,
1234 RTLIB::LLROUND_PPCF128));
1235}
1236
1237SDValue DAGTypeLegalizer::SoftenFloatOp_LRINT(SDNode *N) {
1238 EVT OpVT = N->getOperand(N->isStrictFPOpcode() ? 1 : 0).getValueType();
1239 return SoftenFloatOp_Unary(N, GetFPLibCall(OpVT,
1240 RTLIB::LRINT_F32,
1241 RTLIB::LRINT_F64,
1242 RTLIB::LRINT_F80,
1243 RTLIB::LRINT_F128,
1244 RTLIB::LRINT_PPCF128));
1245}
1246
1247SDValue DAGTypeLegalizer::SoftenFloatOp_LLRINT(SDNode *N) {
1248 EVT OpVT = N->getOperand(N->isStrictFPOpcode() ? 1 : 0).getValueType();
1249 return SoftenFloatOp_Unary(N, GetFPLibCall(OpVT,
1250 RTLIB::LLRINT_F32,
1251 RTLIB::LLRINT_F64,
1252 RTLIB::LLRINT_F80,
1253 RTLIB::LLRINT_F128,
1254 RTLIB::LLRINT_PPCF128));
1255}
1256
1257//===----------------------------------------------------------------------===//
1258// Float Result Expansion
1259//===----------------------------------------------------------------------===//
1260
1261/// ExpandFloatResult - This method is called when the specified result of the
1262/// specified node is found to need expansion. At this point, the node may also
1263/// have invalid operands or may have other results that need promotion, we just
1264/// know that (at least) one result needs expansion.
1265void DAGTypeLegalizer::ExpandFloatResult(SDNode *N, unsigned ResNo) {
1266 LLVM_DEBUG(dbgs() << "Expand float result: "; N->dump(&DAG));
1267 SDValue Lo, Hi;
1268 Lo = Hi = SDValue();
1269
1270 // See if the target wants to custom expand this node.
1271 if (CustomLowerNode(N, N->getValueType(ResNo), true))
1272 return;
1273
1274 switch (N->getOpcode()) {
1275 default:
1276#ifndef NDEBUG
1277 dbgs() << "ExpandFloatResult #" << ResNo << ": ";
1278 N->dump(&DAG); dbgs() << "\n";
1279#endif
1280 report_fatal_error("Do not know how to expand the result of this "
1281 "operator!");
1282
1283 case ISD::UNDEF: SplitRes_UNDEF(N, Lo, Hi); break;
1284 case ISD::SELECT: SplitRes_Select(N, Lo, Hi); break;
1285 case ISD::SELECT_CC: SplitRes_SELECT_CC(N, Lo, Hi); break;
1286
1287 case ISD::MERGE_VALUES: ExpandRes_MERGE_VALUES(N, ResNo, Lo, Hi); break;
1288 case ISD::BITCAST: ExpandRes_BITCAST(N, Lo, Hi); break;
1289 case ISD::BUILD_PAIR: ExpandRes_BUILD_PAIR(N, Lo, Hi); break;
1290 case ISD::EXTRACT_ELEMENT: ExpandRes_EXTRACT_ELEMENT(N, Lo, Hi); break;
1291 case ISD::EXTRACT_VECTOR_ELT: ExpandRes_EXTRACT_VECTOR_ELT(N, Lo, Hi); break;
1292 case ISD::VAARG: ExpandRes_VAARG(N, Lo, Hi); break;
1293
1294 case ISD::ConstantFP: ExpandFloatRes_ConstantFP(N, Lo, Hi); break;
1295 case ISD::FABS: ExpandFloatRes_FABS(N, Lo, Hi); break;
1296 case ISD::STRICT_FMINNUM:
1297 case ISD::FMINNUM: ExpandFloatRes_FMINNUM(N, Lo, Hi); break;
1298 case ISD::STRICT_FMAXNUM:
1299 case ISD::FMAXNUM: ExpandFloatRes_FMAXNUM(N, Lo, Hi); break;
1300 case ISD::STRICT_FADD:
1301 case ISD::FADD: ExpandFloatRes_FADD(N, Lo, Hi); break;
1302 case ISD::FCBRT: ExpandFloatRes_FCBRT(N, Lo, Hi); break;
1303 case ISD::STRICT_FCEIL:
1304 case ISD::FCEIL: ExpandFloatRes_FCEIL(N, Lo, Hi); break;
1305 case ISD::FCOPYSIGN: ExpandFloatRes_FCOPYSIGN(N, Lo, Hi); break;
1306 case ISD::STRICT_FCOS:
1307 case ISD::FCOS: ExpandFloatRes_FCOS(N, Lo, Hi); break;
1308 case ISD::STRICT_FDIV:
1309 case ISD::FDIV: ExpandFloatRes_FDIV(N, Lo, Hi); break;
1310 case ISD::STRICT_FEXP:
1311 case ISD::FEXP: ExpandFloatRes_FEXP(N, Lo, Hi); break;
1312 case ISD::STRICT_FEXP2:
1313 case ISD::FEXP2: ExpandFloatRes_FEXP2(N, Lo, Hi); break;
1314 case ISD::FEXP10: ExpandFloatRes_FEXP10(N, Lo, Hi); break;
1315 case ISD::STRICT_FFLOOR:
1316 case ISD::FFLOOR: ExpandFloatRes_FFLOOR(N, Lo, Hi); break;
1317 case ISD::STRICT_FLOG:
1318 case ISD::FLOG: ExpandFloatRes_FLOG(N, Lo, Hi); break;
1319 case ISD::STRICT_FLOG2:
1320 case ISD::FLOG2: ExpandFloatRes_FLOG2(N, Lo, Hi); break;
1321 case ISD::STRICT_FLOG10:
1322 case ISD::FLOG10: ExpandFloatRes_FLOG10(N, Lo, Hi); break;
1323 case ISD::STRICT_FMA:
1324 case ISD::FMA: ExpandFloatRes_FMA(N, Lo, Hi); break;
1325 case ISD::STRICT_FMUL:
1326 case ISD::FMUL: ExpandFloatRes_FMUL(N, Lo, Hi); break;
1327 case ISD::STRICT_FNEARBYINT:
1328 case ISD::FNEARBYINT: ExpandFloatRes_FNEARBYINT(N, Lo, Hi); break;
1329 case ISD::FNEG: ExpandFloatRes_FNEG(N, Lo, Hi); break;
1330 case ISD::STRICT_FP_EXTEND:
1331 case ISD::FP_EXTEND: ExpandFloatRes_FP_EXTEND(N, Lo, Hi); break;
1332 case ISD::STRICT_FPOW:
1333 case ISD::FPOW: ExpandFloatRes_FPOW(N, Lo, Hi); break;
1334 case ISD::STRICT_FPOWI:
1335 case ISD::FPOWI: ExpandFloatRes_FPOWI(N, Lo, Hi); break;
1336 case ISD::FLDEXP:
1337 case ISD::STRICT_FLDEXP: ExpandFloatRes_FLDEXP(N, Lo, Hi); break;
1338 case ISD::FREEZE: ExpandFloatRes_FREEZE(N, Lo, Hi); break;
1339 case ISD::STRICT_FRINT:
1340 case ISD::FRINT: ExpandFloatRes_FRINT(N, Lo, Hi); break;
1341 case ISD::STRICT_FROUND:
1342 case ISD::FROUND: ExpandFloatRes_FROUND(N, Lo, Hi); break;
1343 case ISD::STRICT_FROUNDEVEN:
1344 case ISD::FROUNDEVEN: ExpandFloatRes_FROUNDEVEN(N, Lo, Hi); break;
1345 case ISD::STRICT_FSIN:
1346 case ISD::FSIN: ExpandFloatRes_FSIN(N, Lo, Hi); break;
1347 case ISD::STRICT_FSQRT:
1348 case ISD::FSQRT: ExpandFloatRes_FSQRT(N, Lo, Hi); break;
1349 case ISD::STRICT_FSUB:
1350 case ISD::FSUB: ExpandFloatRes_FSUB(N, Lo, Hi); break;
1351 case ISD::STRICT_FTRUNC:
1352 case ISD::FTRUNC: ExpandFloatRes_FTRUNC(N, Lo, Hi); break;
1353 case ISD::LOAD: ExpandFloatRes_LOAD(N, Lo, Hi); break;
1354 case ISD::STRICT_SINT_TO_FP:
1355 case ISD::STRICT_UINT_TO_FP:
1356 case ISD::SINT_TO_FP:
1357 case ISD::UINT_TO_FP: ExpandFloatRes_XINT_TO_FP(N, Lo, Hi); break;
1358 case ISD::STRICT_FREM:
1359 case ISD::FREM: ExpandFloatRes_FREM(N, Lo, Hi); break;
1360 }
1361
1362 // If Lo/Hi is null, the sub-method took care of registering results etc.
1363 if (Lo.getNode())
1364 SetExpandedFloat(SDValue(N, ResNo), Lo, Hi);
1365}
1366
1367void DAGTypeLegalizer::ExpandFloatRes_ConstantFP(SDNode *N, SDValue &Lo,
1368 SDValue &Hi) {
1369 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
1370 assert(NVT.getSizeInBits() == 64 &&
1371 "Do not know how to expand this float constant!");
1372 APInt C = cast<ConstantFPSDNode>(N)->getValueAPF().bitcastToAPInt();
1373 SDLoc dl(N);
1374 Lo = DAG.getConstantFP(APFloat(DAG.EVTToAPFloatSemantics(NVT),
1375 APInt(64, C.getRawData()[1])),
1376 dl, NVT);
1377 Hi = DAG.getConstantFP(APFloat(DAG.EVTToAPFloatSemantics(NVT),
1378 APInt(64, C.getRawData()[0])),
1379 dl, NVT);
1380}
1381
1382void DAGTypeLegalizer::ExpandFloatRes_Unary(SDNode *N, RTLIB::Libcall LC,
1383 SDValue &Lo, SDValue &Hi) {
1384 bool IsStrict = N->isStrictFPOpcode();
1385 unsigned Offset = IsStrict ? 1 : 0;
1386 SDValue Op = N->getOperand(0 + Offset);
1387 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
1388 TargetLowering::MakeLibCallOptions CallOptions;
1389 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, N->getValueType(0),
1390 Op, CallOptions, SDLoc(N),
1391 Chain);
1392 if (IsStrict)
1393 ReplaceValueWith(SDValue(N, 1), Tmp.second);
1394 GetPairElements(Tmp.first, Lo, Hi);
1395}
1396
1397void DAGTypeLegalizer::ExpandFloatRes_Binary(SDNode *N, RTLIB::Libcall LC,
1398 SDValue &Lo, SDValue &Hi) {
1399 bool IsStrict = N->isStrictFPOpcode();
1400 unsigned Offset = IsStrict ? 1 : 0;
1401 SDValue Ops[] = { N->getOperand(0 + Offset), N->getOperand(1 + Offset) };
1402 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
1403 TargetLowering::MakeLibCallOptions CallOptions;
1404 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, N->getValueType(0),
1405 Ops, CallOptions, SDLoc(N),
1406 Chain);
1407 if (IsStrict)
1408 ReplaceValueWith(SDValue(N, 1), Tmp.second);
1409 GetPairElements(Tmp.first, Lo, Hi);
1410}
1411
1412void DAGTypeLegalizer::ExpandFloatRes_FABS(SDNode *N, SDValue &Lo,
1413 SDValue &Hi) {
1414 assert(N->getValueType(0) == MVT::ppcf128 &&
1415 "Logic only correct for ppcf128!");
1416 SDLoc dl(N);
1417 SDValue Tmp;
1418 GetExpandedFloat(N->getOperand(0), Lo, Tmp);
1419 Hi = DAG.getNode(ISD::FABS, dl, Tmp.getValueType(), Tmp);
1420 // Lo = Hi==fabs(Hi) ? Lo : -Lo;
1421 Lo = DAG.getSelectCC(dl, Tmp, Hi, Lo,
1422 DAG.getNode(ISD::FNEG, dl, Lo.getValueType(), Lo),
1423 ISD::SETEQ);
1424}
1425
1426void DAGTypeLegalizer::ExpandFloatRes_FMINNUM(SDNode *N, SDValue &Lo,
1427 SDValue &Hi) {
1428 ExpandFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
1429 RTLIB::FMIN_F32, RTLIB::FMIN_F64,
1430 RTLIB::FMIN_F80, RTLIB::FMIN_F128,
1431 RTLIB::FMIN_PPCF128), Lo, Hi);
1432}
1433
1434void DAGTypeLegalizer::ExpandFloatRes_FMAXNUM(SDNode *N, SDValue &Lo,
1435 SDValue &Hi) {
1436 ExpandFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
1437 RTLIB::FMAX_F32, RTLIB::FMAX_F64,
1438 RTLIB::FMAX_F80, RTLIB::FMAX_F128,
1439 RTLIB::FMAX_PPCF128), Lo, Hi);
1440}
1441
1442void DAGTypeLegalizer::ExpandFloatRes_FADD(SDNode *N, SDValue &Lo,
1443 SDValue &Hi) {
1444 ExpandFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
1445 RTLIB::ADD_F32, RTLIB::ADD_F64,
1446 RTLIB::ADD_F80, RTLIB::ADD_F128,
1447 RTLIB::ADD_PPCF128), Lo, Hi);
1448}
1449
1450void DAGTypeLegalizer::ExpandFloatRes_FCBRT(SDNode *N, SDValue &Lo,
1451 SDValue &Hi) {
1452 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0), RTLIB::CBRT_F32,
1453 RTLIB::CBRT_F64, RTLIB::CBRT_F80,
1454 RTLIB::CBRT_F128,
1455 RTLIB::CBRT_PPCF128), Lo, Hi);
1456}
1457
1458void DAGTypeLegalizer::ExpandFloatRes_FCEIL(SDNode *N,
1459 SDValue &Lo, SDValue &Hi) {
1460 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
1461 RTLIB::CEIL_F32, RTLIB::CEIL_F64,
1462 RTLIB::CEIL_F80, RTLIB::CEIL_F128,
1463 RTLIB::CEIL_PPCF128), Lo, Hi);
1464}
1465
1466void DAGTypeLegalizer::ExpandFloatRes_FCOPYSIGN(SDNode *N,
1467 SDValue &Lo, SDValue &Hi) {
1468 ExpandFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
1469 RTLIB::COPYSIGN_F32,
1470 RTLIB::COPYSIGN_F64,
1471 RTLIB::COPYSIGN_F80,
1472 RTLIB::COPYSIGN_F128,
1473 RTLIB::COPYSIGN_PPCF128), Lo, Hi);
1474}
1475
1476void DAGTypeLegalizer::ExpandFloatRes_FCOS(SDNode *N,
1477 SDValue &Lo, SDValue &Hi) {
1478 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
1479 RTLIB::COS_F32, RTLIB::COS_F64,
1480 RTLIB::COS_F80, RTLIB::COS_F128,
1481 RTLIB::COS_PPCF128), Lo, Hi);
1482}
1483
1484void DAGTypeLegalizer::ExpandFloatRes_FDIV(SDNode *N, SDValue &Lo,
1485 SDValue &Hi) {
1486 ExpandFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
1487 RTLIB::DIV_F32,
1488 RTLIB::DIV_F64,
1489 RTLIB::DIV_F80,
1490 RTLIB::DIV_F128,
1491 RTLIB::DIV_PPCF128), Lo, Hi);
1492}
1493
1494void DAGTypeLegalizer::ExpandFloatRes_FEXP(SDNode *N,
1495 SDValue &Lo, SDValue &Hi) {
1496 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
1497 RTLIB::EXP_F32, RTLIB::EXP_F64,
1498 RTLIB::EXP_F80, RTLIB::EXP_F128,
1499 RTLIB::EXP_PPCF128), Lo, Hi);
1500}
1501
1502void DAGTypeLegalizer::ExpandFloatRes_FEXP2(SDNode *N,
1503 SDValue &Lo, SDValue &Hi) {
1504 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
1505 RTLIB::EXP2_F32, RTLIB::EXP2_F64,
1506 RTLIB::EXP2_F80, RTLIB::EXP2_F128,
1507 RTLIB::EXP2_PPCF128), Lo, Hi);
1508}
1509
1510void DAGTypeLegalizer::ExpandFloatRes_FEXP10(SDNode *N, SDValue &Lo,
1511 SDValue &Hi) {
1512 ExpandFloatRes_Unary(N,
1513 GetFPLibCall(N->getValueType(0), RTLIB::EXP10_F32,
1514 RTLIB::EXP10_F64, RTLIB::EXP10_F80,
1515 RTLIB::EXP10_F128, RTLIB::EXP10_PPCF128),
1516 Lo, Hi);
1517}
1518
1519void DAGTypeLegalizer::ExpandFloatRes_FFLOOR(SDNode *N,
1520 SDValue &Lo, SDValue &Hi) {
1521 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
1522 RTLIB::FLOOR_F32, RTLIB::FLOOR_F64,
1523 RTLIB::FLOOR_F80, RTLIB::FLOOR_F128,
1524 RTLIB::FLOOR_PPCF128), Lo, Hi);
1525}
1526
1527void DAGTypeLegalizer::ExpandFloatRes_FLOG(SDNode *N,
1528 SDValue &Lo, SDValue &Hi) {
1529 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
1530 RTLIB::LOG_F32, RTLIB::LOG_F64,
1531 RTLIB::LOG_F80, RTLIB::LOG_F128,
1532 RTLIB::LOG_PPCF128), Lo, Hi);
1533}
1534
1535void DAGTypeLegalizer::ExpandFloatRes_FLOG2(SDNode *N,
1536 SDValue &Lo, SDValue &Hi) {
1537 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
1538 RTLIB::LOG2_F32, RTLIB::LOG2_F64,
1539 RTLIB::LOG2_F80, RTLIB::LOG2_F128,
1540 RTLIB::LOG2_PPCF128), Lo, Hi);
1541}
1542
1543void DAGTypeLegalizer::ExpandFloatRes_FLOG10(SDNode *N,
1544 SDValue &Lo, SDValue &Hi) {
1545 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
1546 RTLIB::LOG10_F32, RTLIB::LOG10_F64,
1547 RTLIB::LOG10_F80, RTLIB::LOG10_F128,
1548 RTLIB::LOG10_PPCF128), Lo, Hi);
1549}
1550
1551void DAGTypeLegalizer::ExpandFloatRes_FMA(SDNode *N, SDValue &Lo,
1552 SDValue &Hi) {
1553 bool IsStrict = N->isStrictFPOpcode();
1554 unsigned Offset = IsStrict ? 1 : 0;
1555 SDValue Ops[3] = { N->getOperand(0 + Offset), N->getOperand(1 + Offset),
1556 N->getOperand(2 + Offset) };
1557 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
1558 TargetLowering::MakeLibCallOptions CallOptions;
1559 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, GetFPLibCall(N->getValueType(0),
1560 RTLIB::FMA_F32,
1561 RTLIB::FMA_F64,
1562 RTLIB::FMA_F80,
1563 RTLIB::FMA_F128,
1564 RTLIB::FMA_PPCF128),
1565 N->getValueType(0), Ops, CallOptions,
1566 SDLoc(N), Chain);
1567 if (IsStrict)
1568 ReplaceValueWith(SDValue(N, 1), Tmp.second);
1569 GetPairElements(Tmp.first, Lo, Hi);
1570}
1571
1572void DAGTypeLegalizer::ExpandFloatRes_FMUL(SDNode *N, SDValue &Lo,
1573 SDValue &Hi) {
1574 ExpandFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
1575 RTLIB::MUL_F32,
1576 RTLIB::MUL_F64,
1577 RTLIB::MUL_F80,
1578 RTLIB::MUL_F128,
1579 RTLIB::MUL_PPCF128), Lo, Hi);
1580}
1581
1582void DAGTypeLegalizer::ExpandFloatRes_FNEARBYINT(SDNode *N,
1583 SDValue &Lo, SDValue &Hi) {
1584 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
1585 RTLIB::NEARBYINT_F32,
1586 RTLIB::NEARBYINT_F64,
1587 RTLIB::NEARBYINT_F80,
1588 RTLIB::NEARBYINT_F128,
1589 RTLIB::NEARBYINT_PPCF128), Lo, Hi);
1590}
1591
1592void DAGTypeLegalizer::ExpandFloatRes_FNEG(SDNode *N, SDValue &Lo,
1593 SDValue &Hi) {
1594 SDLoc dl(N);
1595 GetExpandedFloat(N->getOperand(0), Lo, Hi);
1596 Lo = DAG.getNode(ISD::FNEG, dl, Lo.getValueType(), Lo);
1597 Hi = DAG.getNode(ISD::FNEG, dl, Hi.getValueType(), Hi);
1598}
1599
1600void DAGTypeLegalizer::ExpandFloatRes_FP_EXTEND(SDNode *N, SDValue &Lo,
1601 SDValue &Hi) {
1602 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
1603 SDLoc dl(N);
1604 bool IsStrict = N->isStrictFPOpcode();
1605
1606 SDValue Chain;
1607 if (IsStrict) {
1608 // If the expanded type is the same as the input type, just bypass the node.
1609 if (NVT == N->getOperand(1).getValueType()) {
1610 Hi = N->getOperand(1);
1611 Chain = N->getOperand(0);
1612 } else {
1613 // Other we need to extend.
1614 Hi = DAG.getNode(ISD::STRICT_FP_EXTEND, dl, { NVT, MVT::Other },
1615 { N->getOperand(0), N->getOperand(1) });
1616 Chain = Hi.getValue(1);
1617 }
1618 } else {
1619 Hi = DAG.getNode(ISD::FP_EXTEND, dl, NVT, N->getOperand(0));
1620 }
1621
1622 Lo = DAG.getConstantFP(APFloat(DAG.EVTToAPFloatSemantics(NVT),
1623 APInt(NVT.getSizeInBits(), 0)), dl, NVT);
1624
1625 if (IsStrict)
1626 ReplaceValueWith(SDValue(N, 1), Chain);
1627}
1628
1629void DAGTypeLegalizer::ExpandFloatRes_FPOW(SDNode *N,
1630 SDValue &Lo, SDValue &Hi) {
1631 ExpandFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
1632 RTLIB::POW_F32, RTLIB::POW_F64,
1633 RTLIB::POW_F80, RTLIB::POW_F128,
1634 RTLIB::POW_PPCF128), Lo, Hi);
1635}
1636
1637void DAGTypeLegalizer::ExpandFloatRes_FPOWI(SDNode *N,
1638 SDValue &Lo, SDValue &Hi) {
1639 ExpandFloatRes_Binary(N, RTLIB::getPOWI(N->getValueType(0)), Lo, Hi);
1640}
1641
1642void DAGTypeLegalizer::ExpandFloatRes_FLDEXP(SDNode *N, SDValue &Lo,
1643 SDValue &Hi) {
1644 ExpandFloatRes_Binary(N, RTLIB::getLDEXP(N->getValueType(0)), Lo, Hi);
1645}
1646
1647void DAGTypeLegalizer::ExpandFloatRes_FREEZE(SDNode *N,
1648 SDValue &Lo, SDValue &Hi) {
1649 assert(N->getValueType(0) == MVT::ppcf128 &&
1650 "Logic only correct for ppcf128!");
1651
1652 SDLoc dl(N);
1653 GetExpandedFloat(N->getOperand(0), Lo, Hi);
1654 Lo = DAG.getNode(ISD::FREEZE, dl, Lo.getValueType(), Lo);
1655 Hi = DAG.getNode(ISD::FREEZE, dl, Hi.getValueType(), Hi);
1656}
1657
1658void DAGTypeLegalizer::ExpandFloatRes_FREM(SDNode *N,
1659 SDValue &Lo, SDValue &Hi) {
1660 ExpandFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
1661 RTLIB::REM_F32, RTLIB::REM_F64,
1662 RTLIB::REM_F80, RTLIB::REM_F128,
1663 RTLIB::REM_PPCF128), Lo, Hi);
1664}
1665
1666void DAGTypeLegalizer::ExpandFloatRes_FRINT(SDNode *N,
1667 SDValue &Lo, SDValue &Hi) {
1668 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
1669 RTLIB::RINT_F32, RTLIB::RINT_F64,
1670 RTLIB::RINT_F80, RTLIB::RINT_F128,
1671 RTLIB::RINT_PPCF128), Lo, Hi);
1672}
1673
1674void DAGTypeLegalizer::ExpandFloatRes_FROUND(SDNode *N,
1675 SDValue &Lo, SDValue &Hi) {
1676 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
1677 RTLIB::ROUND_F32,
1678 RTLIB::ROUND_F64,
1679 RTLIB::ROUND_F80,
1680 RTLIB::ROUND_F128,
1681 RTLIB::ROUND_PPCF128), Lo, Hi);
1682}
1683
1684void DAGTypeLegalizer::ExpandFloatRes_FROUNDEVEN(SDNode *N,
1685 SDValue &Lo, SDValue &Hi) {
1686 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
1687 RTLIB::ROUNDEVEN_F32,
1688 RTLIB::ROUNDEVEN_F64,
1689 RTLIB::ROUNDEVEN_F80,
1690 RTLIB::ROUNDEVEN_F128,
1691 RTLIB::ROUNDEVEN_PPCF128), Lo, Hi);
1692}
1693
1694void DAGTypeLegalizer::ExpandFloatRes_FSIN(SDNode *N,
1695 SDValue &Lo, SDValue &Hi) {
1696 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
1697 RTLIB::SIN_F32, RTLIB::SIN_F64,
1698 RTLIB::SIN_F80, RTLIB::SIN_F128,
1699 RTLIB::SIN_PPCF128), Lo, Hi);
1700}
1701
1702void DAGTypeLegalizer::ExpandFloatRes_FSQRT(SDNode *N,
1703 SDValue &Lo, SDValue &Hi) {
1704 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
1705 RTLIB::SQRT_F32, RTLIB::SQRT_F64,
1706 RTLIB::SQRT_F80, RTLIB::SQRT_F128,
1707 RTLIB::SQRT_PPCF128), Lo, Hi);
1708}
1709
1710void DAGTypeLegalizer::ExpandFloatRes_FSUB(SDNode *N, SDValue &Lo,
1711 SDValue &Hi) {
1712 ExpandFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
1713 RTLIB::SUB_F32,
1714 RTLIB::SUB_F64,
1715 RTLIB::SUB_F80,
1716 RTLIB::SUB_F128,
1717 RTLIB::SUB_PPCF128), Lo, Hi);
1718}
1719
1720void DAGTypeLegalizer::ExpandFloatRes_FTRUNC(SDNode *N,
1721 SDValue &Lo, SDValue &Hi) {
1722 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
1723 RTLIB::TRUNC_F32, RTLIB::TRUNC_F64,
1724 RTLIB::TRUNC_F80, RTLIB::TRUNC_F128,
1725 RTLIB::TRUNC_PPCF128), Lo, Hi);
1726}
1727
1728void DAGTypeLegalizer::ExpandFloatRes_LOAD(SDNode *N, SDValue &Lo,
1729 SDValue &Hi) {
1730 if (ISD::isNormalLoad(N)) {
1731 ExpandRes_NormalLoad(N, Lo, Hi);
1732 return;
1733 }
1734
1735 assert(ISD::isUNINDEXEDLoad(N) && "Indexed load during type legalization!");
1736 LoadSDNode *LD = cast<LoadSDNode>(N);
1737 SDValue Chain = LD->getChain();
1738 SDValue Ptr = LD->getBasePtr();
1739 SDLoc dl(N);
1740
1741 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), LD->getValueType(0));
1742 assert(NVT.isByteSized() && "Expanded type not byte sized!");
1743 assert(LD->getMemoryVT().bitsLE(NVT) && "Float type not round?");
1744
1745 Hi = DAG.getExtLoad(LD->getExtensionType(), dl, NVT, Chain, Ptr,
1746 LD->getMemoryVT(), LD->getMemOperand());
1747
1748 // Remember the chain.
1749 Chain = Hi.getValue(1);
1750
1751 // The low part is zero.
1752 Lo = DAG.getConstantFP(APFloat(DAG.EVTToAPFloatSemantics(NVT),
1753 APInt(NVT.getSizeInBits(), 0)), dl, NVT);
1754
1755 // Modified the chain - switch anything that used the old chain to use the
1756 // new one.
1757 ReplaceValueWith(SDValue(LD, 1), Chain);
1758}
1759
1760void DAGTypeLegalizer::ExpandFloatRes_XINT_TO_FP(SDNode *N, SDValue &Lo,
1761 SDValue &Hi) {
1762 assert(N->getValueType(0) == MVT::ppcf128 && "Unsupported XINT_TO_FP!");
1763 EVT VT = N->getValueType(0);
1764 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
1765 bool Strict = N->isStrictFPOpcode();
1766 SDValue Src = N->getOperand(Strict ? 1 : 0);
1767 EVT SrcVT = Src.getValueType();
1768 bool isSigned = N->getOpcode() == ISD::SINT_TO_FP ||
1769 N->getOpcode() == ISD::STRICT_SINT_TO_FP;
1770 SDLoc dl(N);
1771 SDValue Chain = Strict ? N->getOperand(0) : DAG.getEntryNode();
1772
1773 // TODO: Any other flags to propagate?
1774 SDNodeFlags Flags;
1775 Flags.setNoFPExcept(N->getFlags().hasNoFPExcept());
1776
1777 // First do an SINT_TO_FP, whether the original was signed or unsigned.
1778 // When promoting partial word types to i32 we must honor the signedness,
1779 // though.
1780 if (SrcVT.bitsLE(MVT::i32)) {
1781 // The integer can be represented exactly in an f64.
1782 Lo = DAG.getConstantFP(APFloat(DAG.EVTToAPFloatSemantics(NVT),
1783 APInt(NVT.getSizeInBits(), 0)), dl, NVT);
1784 if (Strict) {
1785 Hi = DAG.getNode(N->getOpcode(), dl, DAG.getVTList(NVT, MVT::Other),
1786 {Chain, Src}, Flags);
1787 Chain = Hi.getValue(1);
1788 } else
1789 Hi = DAG.getNode(N->getOpcode(), dl, NVT, Src);
1790 } else {
1791 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
1792 if (SrcVT.bitsLE(MVT::i64)) {
1793 Src = DAG.getNode(isSigned ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND, dl,
1794 MVT::i64, Src);
1795 LC = RTLIB::SINTTOFP_I64_PPCF128;
1796 } else if (SrcVT.bitsLE(MVT::i128)) {
1797 Src = DAG.getNode(ISD::SIGN_EXTEND, dl, MVT::i128, Src);
1798 LC = RTLIB::SINTTOFP_I128_PPCF128;
1799 }
1800 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported XINT_TO_FP!");
1801
1802 TargetLowering::MakeLibCallOptions CallOptions;
1803 CallOptions.setSExt(true);
1804 std::pair<SDValue, SDValue> Tmp =
1805 TLI.makeLibCall(DAG, LC, VT, Src, CallOptions, dl, Chain);
1806 if (Strict)
1807 Chain = Tmp.second;
1808 GetPairElements(Tmp.first, Lo, Hi);
1809 }
1810
1811 // No need to complement for unsigned 32-bit integers
1812 if (isSigned || SrcVT.bitsLE(MVT::i32)) {
1813 if (Strict)
1814 ReplaceValueWith(SDValue(N, 1), Chain);
1815
1816 return;
1817 }
1818
1819 // Unsigned - fix up the SINT_TO_FP value just calculated.
1820 // FIXME: For unsigned i128 to ppc_fp128 conversion, we need to carefully
1821 // keep semantics correctness if the integer is not exactly representable
1822 // here. See ExpandLegalINT_TO_FP.
1823 Hi = DAG.getNode(ISD::BUILD_PAIR, dl, VT, Lo, Hi);
1824 SrcVT = Src.getValueType();
1825
1826 // x>=0 ? (ppcf128)(iN)x : (ppcf128)(iN)x + 2^N; N=32,64,128.
1827 static const uint64_t TwoE32[] = { 0x41f0000000000000LL, 0 };
1828 static const uint64_t TwoE64[] = { 0x43f0000000000000LL, 0 };
1829 static const uint64_t TwoE128[] = { 0x47f0000000000000LL, 0 };
1830 ArrayRef<uint64_t> Parts;
1831
1832 switch (SrcVT.getSimpleVT().SimpleTy) {
1833 default:
1834 llvm_unreachable("Unsupported UINT_TO_FP!");
1835 case MVT::i32:
1836 Parts = TwoE32;
1837 break;
1838 case MVT::i64:
1839 Parts = TwoE64;
1840 break;
1841 case MVT::i128:
1842 Parts = TwoE128;
1843 break;
1844 }
1845
1846 // TODO: Are there other fast-math-flags to propagate to this FADD?
1847 SDValue NewLo = DAG.getConstantFP(
1848 APFloat(APFloat::PPCDoubleDouble(), APInt(128, Parts)), dl, MVT::ppcf128);
1849 if (Strict) {
1850 Lo = DAG.getNode(ISD::STRICT_FADD, dl, DAG.getVTList(VT, MVT::Other),
1851 {Chain, Hi, NewLo}, Flags);
1852 Chain = Lo.getValue(1);
1853 ReplaceValueWith(SDValue(N, 1), Chain);
1854 } else
1855 Lo = DAG.getNode(ISD::FADD, dl, VT, Hi, NewLo);
1856 Lo = DAG.getSelectCC(dl, Src, DAG.getConstant(0, dl, SrcVT),
1857 Lo, Hi, ISD::SETLT);
1858 GetPairElements(Lo, Lo, Hi);
1859}
1860
1861
1862//===----------------------------------------------------------------------===//
1863// Float Operand Expansion
1864//===----------------------------------------------------------------------===//
1865
1866/// ExpandFloatOperand - This method is called when the specified operand of the
1867/// specified node is found to need expansion. At this point, all of the result
1868/// types of the node are known to be legal, but other operands of the node may
1869/// need promotion or expansion as well as the specified one.
1870bool DAGTypeLegalizer::ExpandFloatOperand(SDNode *N, unsigned OpNo) {
1871 LLVM_DEBUG(dbgs() << "Expand float operand: "; N->dump(&DAG));
1872 SDValue Res = SDValue();
1873
1874 // See if the target wants to custom expand this node.
1875 if (CustomLowerNode(N, N->getOperand(OpNo).getValueType(), false))
1876 return false;
1877
1878 switch (N->getOpcode()) {
1879 default:
1880#ifndef NDEBUG
1881 dbgs() << "ExpandFloatOperand Op #" << OpNo << ": ";
1882 N->dump(&DAG); dbgs() << "\n";
1883#endif
1884 report_fatal_error("Do not know how to expand this operator's operand!");
1885
1886 case ISD::BITCAST: Res = ExpandOp_BITCAST(N); break;
1887 case ISD::BUILD_VECTOR: Res = ExpandOp_BUILD_VECTOR(N); break;
1888 case ISD::EXTRACT_ELEMENT: Res = ExpandOp_EXTRACT_ELEMENT(N); break;
1889
1890 case ISD::BR_CC: Res = ExpandFloatOp_BR_CC(N); break;
1891 case ISD::FCOPYSIGN: Res = ExpandFloatOp_FCOPYSIGN(N); break;
1892 case ISD::STRICT_FP_ROUND:
1893 case ISD::FP_ROUND: Res = ExpandFloatOp_FP_ROUND(N); break;
1894 case ISD::STRICT_FP_TO_SINT:
1895 case ISD::STRICT_FP_TO_UINT:
1896 case ISD::FP_TO_SINT:
1897 case ISD::FP_TO_UINT: Res = ExpandFloatOp_FP_TO_XINT(N); break;
1898 case ISD::LROUND: Res = ExpandFloatOp_LROUND(N); break;
1899 case ISD::LLROUND: Res = ExpandFloatOp_LLROUND(N); break;
1900 case ISD::LRINT: Res = ExpandFloatOp_LRINT(N); break;
1901 case ISD::LLRINT: Res = ExpandFloatOp_LLRINT(N); break;
1902 case ISD::SELECT_CC: Res = ExpandFloatOp_SELECT_CC(N); break;
1903 case ISD::STRICT_FSETCC:
1904 case ISD::STRICT_FSETCCS:
1905 case ISD::SETCC: Res = ExpandFloatOp_SETCC(N); break;
1906 case ISD::STORE: Res = ExpandFloatOp_STORE(cast<StoreSDNode>(N),
1907 OpNo); break;
1908 }
1909
1910 // If the result is null, the sub-method took care of registering results etc.
1911 if (!Res.getNode()) return false;
1912
1913 // If the result is N, the sub-method updated N in place. Tell the legalizer
1914 // core about this.
1915 if (Res.getNode() == N)
1916 return true;
1917
1918 assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 &&
1919 "Invalid operand expansion");
1920
1921 ReplaceValueWith(SDValue(N, 0), Res);
1922 return false;
1923}
1924
1925/// FloatExpandSetCCOperands - Expand the operands of a comparison. This code
1926/// is shared among BR_CC, SELECT_CC, and SETCC handlers.
1927void DAGTypeLegalizer::FloatExpandSetCCOperands(SDValue &NewLHS,
1928 SDValue &NewRHS,
1929 ISD::CondCode &CCCode,
1930 const SDLoc &dl, SDValue &Chain,
1931 bool IsSignaling) {
1932 SDValue LHSLo, LHSHi, RHSLo, RHSHi;
1933 GetExpandedFloat(NewLHS, LHSLo, LHSHi);
1934 GetExpandedFloat(NewRHS, RHSLo, RHSHi);
1935
1936 assert(NewLHS.getValueType() == MVT::ppcf128 && "Unsupported setcc type!");
1937
1938 // FIXME: This generated code sucks. We want to generate
1939 // FCMPU crN, hi1, hi2
1940 // BNE crN, L:
1941 // FCMPU crN, lo1, lo2
1942 // The following can be improved, but not that much.
1943 SDValue Tmp1, Tmp2, Tmp3, OutputChain;
1944 Tmp1 = DAG.getSetCC(dl, getSetCCResultType(LHSHi.getValueType()), LHSHi,
1945 RHSHi, ISD::SETOEQ, Chain, IsSignaling);
1946 OutputChain = Tmp1->getNumValues() > 1 ? Tmp1.getValue(1) : SDValue();
1947 Tmp2 = DAG.getSetCC(dl, getSetCCResultType(LHSLo.getValueType()), LHSLo,
1948 RHSLo, CCCode, OutputChain, IsSignaling);
1949 OutputChain = Tmp2->getNumValues() > 1 ? Tmp2.getValue(1) : SDValue();
1950 Tmp3 = DAG.getNode(ISD::AND, dl, Tmp1.getValueType(), Tmp1, Tmp2);
1951 Tmp1 =
1952 DAG.getSetCC(dl, getSetCCResultType(LHSHi.getValueType()), LHSHi, RHSHi,
1953 ISD::SETUNE, OutputChain, IsSignaling);
1954 OutputChain = Tmp1->getNumValues() > 1 ? Tmp1.getValue(1) : SDValue();
1955 Tmp2 = DAG.getSetCC(dl, getSetCCResultType(LHSHi.getValueType()), LHSHi,
1956 RHSHi, CCCode, OutputChain, IsSignaling);
1957 OutputChain = Tmp2->getNumValues() > 1 ? Tmp2.getValue(1) : SDValue();
1958 Tmp1 = DAG.getNode(ISD::AND, dl, Tmp1.getValueType(), Tmp1, Tmp2);
1959 NewLHS = DAG.getNode(ISD::OR, dl, Tmp1.getValueType(), Tmp1, Tmp3);
1960 NewRHS = SDValue(); // LHS is the result, not a compare.
1961 Chain = OutputChain;
1962}
1963
1964SDValue DAGTypeLegalizer::ExpandFloatOp_BR_CC(SDNode *N) {
1965 SDValue NewLHS = N->getOperand(2), NewRHS = N->getOperand(3);
1966 ISD::CondCode CCCode = cast<CondCodeSDNode>(N->getOperand(1))->get();
1967 SDValue Chain;
1968 FloatExpandSetCCOperands(NewLHS, NewRHS, CCCode, SDLoc(N), Chain);
1969
1970 // If ExpandSetCCOperands returned a scalar, we need to compare the result
1971 // against zero to select between true and false values.
1972 if (!NewRHS.getNode()) {
1973 NewRHS = DAG.getConstant(0, SDLoc(N), NewLHS.getValueType());
1974 CCCode = ISD::SETNE;
1975 }
1976
1977 // Update N to have the operands specified.
1978 return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0),
1979 DAG.getCondCode(CCCode), NewLHS, NewRHS,
1980 N->getOperand(4)), 0);
1981}
1982
1983SDValue DAGTypeLegalizer::ExpandFloatOp_FCOPYSIGN(SDNode *N) {
1984 assert(N->getOperand(1).getValueType() == MVT::ppcf128 &&
1985 "Logic only correct for ppcf128!");
1986 SDValue Lo, Hi;
1987 GetExpandedFloat(N->getOperand(1), Lo, Hi);
1988 // The ppcf128 value is providing only the sign; take it from the
1989 // higher-order double (which must have the larger magnitude).
1990 return DAG.getNode(ISD::FCOPYSIGN, SDLoc(N),
1991 N->getValueType(0), N->getOperand(0), Hi);
1992}
1993
1994SDValue DAGTypeLegalizer::ExpandFloatOp_FP_ROUND(SDNode *N) {
1995 bool IsStrict = N->isStrictFPOpcode();
1996 assert(N->getOperand(IsStrict ? 1 : 0).getValueType() == MVT::ppcf128 &&
1997 "Logic only correct for ppcf128!");
1998 SDValue Lo, Hi;
1999 GetExpandedFloat(N->getOperand(IsStrict ? 1 : 0), Lo, Hi);
2000
2001 if (!IsStrict)
2002 // Round it the rest of the way (e.g. to f32) if needed.
2003 return DAG.getNode(ISD::FP_ROUND, SDLoc(N),
2004 N->getValueType(0), Hi, N->getOperand(1));
2005
2006 // Eliminate the node if the input float type is the same as the output float
2007 // type.
2008 if (Hi.getValueType() == N->getValueType(0)) {
2009 // Connect the output chain to the input chain, unlinking the node.
2010 ReplaceValueWith(SDValue(N, 1), N->getOperand(0));
2011 ReplaceValueWith(SDValue(N, 0), Hi);
2012 return SDValue();
2013 }
2014
2015 SDValue Expansion = DAG.getNode(ISD::STRICT_FP_ROUND, SDLoc(N),
2016 {N->getValueType(0), MVT::Other},
2017 {N->getOperand(0), Hi, N->getOperand(2)});
2018 ReplaceValueWith(SDValue(N, 1), Expansion.getValue(1));
2019 ReplaceValueWith(SDValue(N, 0), Expansion);
2020 return SDValue();
2021}
2022
2023SDValue DAGTypeLegalizer::ExpandFloatOp_FP_TO_XINT(SDNode *N) {
2024 EVT RVT = N->getValueType(0);
2025 SDLoc dl(N);
2026
2027 bool IsStrict = N->isStrictFPOpcode();
2028 bool Signed = N->getOpcode() == ISD::FP_TO_SINT ||
2029 N->getOpcode() == ISD::STRICT_FP_TO_SINT;
2030 SDValue Op = N->getOperand(IsStrict ? 1 : 0);
2031 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
2032
2033 EVT NVT;
2034 RTLIB::Libcall LC = findFPToIntLibcall(Op.getValueType(), RVT, NVT, Signed);
2035 assert(LC != RTLIB::UNKNOWN_LIBCALL && NVT.isSimple() &&
2036 "Unsupported FP_TO_XINT!");
2037 TargetLowering::MakeLibCallOptions CallOptions;
2038 std::pair<SDValue, SDValue> Tmp =
2039 TLI.makeLibCall(DAG, LC, NVT, Op, CallOptions, dl, Chain);
2040 if (!IsStrict)
2041 return Tmp.first;
2042
2043 ReplaceValueWith(SDValue(N, 1), Tmp.second);
2044 ReplaceValueWith(SDValue(N, 0), Tmp.first);
2045 return SDValue();
2046}
2047
2048SDValue DAGTypeLegalizer::ExpandFloatOp_SELECT_CC(SDNode *N) {
2049 SDValue NewLHS = N->getOperand(0), NewRHS = N->getOperand(1);
2050 ISD::CondCode CCCode = cast<CondCodeSDNode>(N->getOperand(4))->get();
2051 SDValue Chain;
2052 FloatExpandSetCCOperands(NewLHS, NewRHS, CCCode, SDLoc(N), Chain);
2053
2054 // If ExpandSetCCOperands returned a scalar, we need to compare the result
2055 // against zero to select between true and false values.
2056 if (!NewRHS.getNode()) {
2057 NewRHS = DAG.getConstant(0, SDLoc(N), NewLHS.getValueType());
2058 CCCode = ISD::SETNE;
2059 }
2060
2061 // Update N to have the operands specified.
2062 return SDValue(DAG.UpdateNodeOperands(N, NewLHS, NewRHS,
2063 N->getOperand(2), N->getOperand(3),
2064 DAG.getCondCode(CCCode)), 0);
2065}
2066
2067SDValue DAGTypeLegalizer::ExpandFloatOp_SETCC(SDNode *N) {
2068 bool IsStrict = N->isStrictFPOpcode();
2069 SDValue NewLHS = N->getOperand(IsStrict ? 1 : 0);
2070 SDValue NewRHS = N->getOperand(IsStrict ? 2 : 1);
2071 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
2072 ISD::CondCode CCCode =
2073 cast<CondCodeSDNode>(N->getOperand(IsStrict ? 3 : 2))->get();
2074 FloatExpandSetCCOperands(NewLHS, NewRHS, CCCode, SDLoc(N), Chain,
2075 N->getOpcode() == ISD::STRICT_FSETCCS);
2076
2077 // FloatExpandSetCCOperands always returned a scalar.
2078 assert(!NewRHS.getNode() && "Expect to return scalar");
2079 assert(NewLHS.getValueType() == N->getValueType(0) &&
2080 "Unexpected setcc expansion!");
2081 if (Chain) {
2082 ReplaceValueWith(SDValue(N, 0), NewLHS);
2083 ReplaceValueWith(SDValue(N, 1), Chain);
2084 return SDValue();
2085 }
2086 return NewLHS;
2087}
2088
2089SDValue DAGTypeLegalizer::ExpandFloatOp_STORE(SDNode *N, unsigned OpNo) {
2090 if (ISD::isNormalStore(N))
2091 return ExpandOp_NormalStore(N, OpNo);
2092
2093 assert(ISD::isUNINDEXEDStore(N) && "Indexed store during type legalization!");
2094 assert(OpNo == 1 && "Can only expand the stored value so far");
2095 StoreSDNode *ST = cast<StoreSDNode>(N);
2096
2097 SDValue Chain = ST->getChain();
2098 SDValue Ptr = ST->getBasePtr();
2099
2100 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(),
2101 ST->getValue().getValueType());
2102 assert(NVT.isByteSized() && "Expanded type not byte sized!");
2103 assert(ST->getMemoryVT().bitsLE(NVT) && "Float type not round?");
2104 (void)NVT;
2105
2106 SDValue Lo, Hi;
2107 GetExpandedOp(ST->getValue(), Lo, Hi);
2108
2109 return DAG.getTruncStore(Chain, SDLoc(N), Hi, Ptr,
2110 ST->getMemoryVT(), ST->getMemOperand());
2111}
2112
2113SDValue DAGTypeLegalizer::ExpandFloatOp_LROUND(SDNode *N) {
2114 EVT RVT = N->getValueType(0);
2115 EVT RetVT = N->getOperand(0).getValueType();
2116 TargetLowering::MakeLibCallOptions CallOptions;
2117 return TLI.makeLibCall(DAG, GetFPLibCall(RetVT,
2118 RTLIB::LROUND_F32,
2119 RTLIB::LROUND_F64,
2120 RTLIB::LROUND_F80,
2121 RTLIB::LROUND_F128,
2122 RTLIB::LROUND_PPCF128),
2123 RVT, N->getOperand(0), CallOptions, SDLoc(N)).first;
2124}
2125
2126SDValue DAGTypeLegalizer::ExpandFloatOp_LLROUND(SDNode *N) {
2127 EVT RVT = N->getValueType(0);
2128 EVT RetVT = N->getOperand(0).getValueType();
2129 TargetLowering::MakeLibCallOptions CallOptions;
2130 return TLI.makeLibCall(DAG, GetFPLibCall(RetVT,
2131 RTLIB::LLROUND_F32,
2132 RTLIB::LLROUND_F64,
2133 RTLIB::LLROUND_F80,
2134 RTLIB::LLROUND_F128,
2135 RTLIB::LLROUND_PPCF128),
2136 RVT, N->getOperand(0), CallOptions, SDLoc(N)).first;
2137}
2138
2139SDValue DAGTypeLegalizer::ExpandFloatOp_LRINT(SDNode *N) {
2140 EVT RVT = N->getValueType(0);
2141 EVT RetVT = N->getOperand(0).getValueType();
2142 TargetLowering::MakeLibCallOptions CallOptions;
2143 return TLI.makeLibCall(DAG, GetFPLibCall(RetVT,
2144 RTLIB::LRINT_F32,
2145 RTLIB::LRINT_F64,
2146 RTLIB::LRINT_F80,
2147 RTLIB::LRINT_F128,
2148 RTLIB::LRINT_PPCF128),
2149 RVT, N->getOperand(0), CallOptions, SDLoc(N)).first;
2150}
2151
2152SDValue DAGTypeLegalizer::ExpandFloatOp_LLRINT(SDNode *N) {
2153 EVT RVT = N->getValueType(0);
2154 EVT RetVT = N->getOperand(0).getValueType();
2155 TargetLowering::MakeLibCallOptions CallOptions;
2156 return TLI.makeLibCall(DAG, GetFPLibCall(RetVT,
2157 RTLIB::LLRINT_F32,
2158 RTLIB::LLRINT_F64,
2159 RTLIB::LLRINT_F80,
2160 RTLIB::LLRINT_F128,
2161 RTLIB::LLRINT_PPCF128),
2162 RVT, N->getOperand(0), CallOptions, SDLoc(N)).first;
2163}
2164
2165//===----------------------------------------------------------------------===//
2166// Float Operand Promotion
2167//===----------------------------------------------------------------------===//
2168//
2169
2170static ISD::NodeType GetPromotionOpcode(EVT OpVT, EVT RetVT) {
2171 if (OpVT == MVT::f16) {
2172 return ISD::FP16_TO_FP;
2173 } else if (RetVT == MVT::f16) {
2174 return ISD::FP_TO_FP16;
2175 } else if (OpVT == MVT::bf16) {
2176 return ISD::BF16_TO_FP;
2177 } else if (RetVT == MVT::bf16) {
2178 return ISD::FP_TO_BF16;
2179 }
2180
2181 report_fatal_error("Attempt at an invalid promotion-related conversion");
2182}
2183
2184bool DAGTypeLegalizer::PromoteFloatOperand(SDNode *N, unsigned OpNo) {
2185 LLVM_DEBUG(dbgs() << "Promote float operand " << OpNo << ": "; N->dump(&DAG));
2186 SDValue R = SDValue();
2187
2188 if (CustomLowerNode(N, N->getOperand(OpNo).getValueType(), false)) {
2189 LLVM_DEBUG(dbgs() << "Node has been custom lowered, done\n");
2190 return false;
2191 }
2192
2193 // Nodes that use a promotion-requiring floating point operand, but doesn't
2194 // produce a promotion-requiring floating point result, need to be legalized
2195 // to use the promoted float operand. Nodes that produce at least one
2196 // promotion-requiring floating point result have their operands legalized as
2197 // a part of PromoteFloatResult.
2198 // clang-format off
2199 switch (N->getOpcode()) {
2200 default:
2201 #ifndef NDEBUG
2202 dbgs() << "PromoteFloatOperand Op #" << OpNo << ": ";
2203 N->dump(&DAG); dbgs() << "\n";
2204 #endif
2205 report_fatal_error("Do not know how to promote this operator's operand!");
2206
2207 case ISD::BITCAST: R = PromoteFloatOp_BITCAST(N, OpNo); break;
2208 case ISD::FCOPYSIGN: R = PromoteFloatOp_FCOPYSIGN(N, OpNo); break;
2209 case ISD::FP_TO_SINT:
2210 case ISD::FP_TO_UINT:
2211 case ISD::LRINT:
2212 case ISD::LLRINT: R = PromoteFloatOp_UnaryOp(N, OpNo); break;
2213 case ISD::FP_TO_SINT_SAT:
2214 case ISD::FP_TO_UINT_SAT:
2215 R = PromoteFloatOp_FP_TO_XINT_SAT(N, OpNo); break;
2216 case ISD::FP_EXTEND: R = PromoteFloatOp_FP_EXTEND(N, OpNo); break;
2217 case ISD::SELECT_CC: R = PromoteFloatOp_SELECT_CC(N, OpNo); break;
2218 case ISD::SETCC: R = PromoteFloatOp_SETCC(N, OpNo); break;
2219 case ISD::STORE: R = PromoteFloatOp_STORE(N, OpNo); break;
2220 }
2221 // clang-format on
2222
2223 if (R.getNode())
2224 ReplaceValueWith(SDValue(N, 0), R);
2225 return false;
2226}
2227
2228SDValue DAGTypeLegalizer::PromoteFloatOp_BITCAST(SDNode *N, unsigned OpNo) {
2229 SDValue Op = N->getOperand(0);
2230 EVT OpVT = Op->getValueType(0);
2231
2232 SDValue Promoted = GetPromotedFloat(N->getOperand(0));
2233 EVT PromotedVT = Promoted->getValueType(0);
2234
2235 // Convert the promoted float value to the desired IVT.
2236 EVT IVT = EVT::getIntegerVT(*DAG.getContext(), OpVT.getSizeInBits());
2237 SDValue Convert = DAG.getNode(GetPromotionOpcode(PromotedVT, OpVT), SDLoc(N),
2238 IVT, Promoted);
2239 // The final result type might not be an scalar so we need a bitcast. The
2240 // bitcast will be further legalized if needed.
2241 return DAG.getBitcast(N->getValueType(0), Convert);
2242}
2243
2244// Promote Operand 1 of FCOPYSIGN. Operand 0 ought to be handled by
2245// PromoteFloatRes_FCOPYSIGN.
2246SDValue DAGTypeLegalizer::PromoteFloatOp_FCOPYSIGN(SDNode *N, unsigned OpNo) {
2247 assert (OpNo == 1 && "Only Operand 1 must need promotion here");
2248 SDValue Op1 = GetPromotedFloat(N->getOperand(1));
2249
2250 return DAG.getNode(N->getOpcode(), SDLoc(N), N->getValueType(0),
2251 N->getOperand(0), Op1);
2252}
2253
2254// Convert the promoted float value to the desired integer type
2255SDValue DAGTypeLegalizer::PromoteFloatOp_UnaryOp(SDNode *N, unsigned OpNo) {
2256 SDValue Op = GetPromotedFloat(N->getOperand(0));
2257 return DAG.getNode(N->getOpcode(), SDLoc(N), N->getValueType(0), Op);
2258}
2259
2260SDValue DAGTypeLegalizer::PromoteFloatOp_FP_TO_XINT_SAT(SDNode *N,
2261 unsigned OpNo) {
2262 SDValue Op = GetPromotedFloat(N->getOperand(0));
2263 return DAG.getNode(N->getOpcode(), SDLoc(N), N->getValueType(0), Op,
2264 N->getOperand(1));
2265}
2266
2267SDValue DAGTypeLegalizer::PromoteFloatOp_FP_EXTEND(SDNode *N, unsigned OpNo) {
2268 SDValue Op = GetPromotedFloat(N->getOperand(0));
2269 EVT VT = N->getValueType(0);
2270
2271 // Desired VT is same as promoted type. Use promoted float directly.
2272 if (VT == Op->getValueType(0))
2273 return Op;
2274
2275 // Else, extend the promoted float value to the desired VT.
2276 return DAG.getNode(ISD::FP_EXTEND, SDLoc(N), VT, Op);
2277}
2278
2279// Promote the float operands used for comparison. The true- and false-
2280// operands have the same type as the result and are promoted, if needed, by
2281// PromoteFloatRes_SELECT_CC
2282SDValue DAGTypeLegalizer::PromoteFloatOp_SELECT_CC(SDNode *N, unsigned OpNo) {
2283 SDValue LHS = GetPromotedFloat(N->getOperand(0));
2284 SDValue RHS = GetPromotedFloat(N->getOperand(1));
2285
2286 return DAG.getNode(ISD::SELECT_CC, SDLoc(N), N->getValueType(0),
2287 LHS, RHS, N->getOperand(2), N->getOperand(3),
2288 N->getOperand(4));
2289}
2290
2291// Construct a SETCC that compares the promoted values and sets the conditional
2292// code.
2293SDValue DAGTypeLegalizer::PromoteFloatOp_SETCC(SDNode *N, unsigned OpNo) {
2294 EVT VT = N->getValueType(0);
2295 SDValue Op0 = GetPromotedFloat(N->getOperand(0));
2296 SDValue Op1 = GetPromotedFloat(N->getOperand(1));
2297 ISD::CondCode CCCode = cast<CondCodeSDNode>(N->getOperand(2))->get();
2298
2299 return DAG.getSetCC(SDLoc(N), VT, Op0, Op1, CCCode);
2300
2301}
2302
2303// Lower the promoted Float down to the integer value of same size and construct
2304// a STORE of the integer value.
2305SDValue DAGTypeLegalizer::PromoteFloatOp_STORE(SDNode *N, unsigned OpNo) {
2306 StoreSDNode *ST = cast<StoreSDNode>(N);
2307 SDValue Val = ST->getValue();
2308 SDLoc DL(N);
2309
2310 SDValue Promoted = GetPromotedFloat(Val);
2311 EVT VT = ST->getOperand(1).getValueType();
2312 EVT IVT = EVT::getIntegerVT(*DAG.getContext(), VT.getSizeInBits());
2313
2314 SDValue NewVal;
2315 NewVal = DAG.getNode(GetPromotionOpcode(Promoted.getValueType(), VT), DL,
2316 IVT, Promoted);
2317
2318 return DAG.getStore(ST->getChain(), DL, NewVal, ST->getBasePtr(),
2319 ST->getMemOperand());
2320}
2321
2322//===----------------------------------------------------------------------===//
2323// Float Result Promotion
2324//===----------------------------------------------------------------------===//
2325
2326void DAGTypeLegalizer::PromoteFloatResult(SDNode *N, unsigned ResNo) {
2327 LLVM_DEBUG(dbgs() << "Promote float result " << ResNo << ": "; N->dump(&DAG));
2328 SDValue R = SDValue();
2329
2330 // See if the target wants to custom expand this node.
2331 if (CustomLowerNode(N, N->getValueType(ResNo), true)) {
2332 LLVM_DEBUG(dbgs() << "Node has been custom expanded, done\n");
2333 return;
2334 }
2335
2336 switch (N->getOpcode()) {
2337 // These opcodes cannot appear if promotion of FP16 is done in the backend
2338 // instead of Clang
2339 case ISD::FP16_TO_FP:
2340 case ISD::FP_TO_FP16:
2341 default:
2342#ifndef NDEBUG
2343 dbgs() << "PromoteFloatResult #" << ResNo << ": ";
2344 N->dump(&DAG); dbgs() << "\n";
2345#endif
2346 report_fatal_error("Do not know how to promote this operator's result!");
2347
2348 case ISD::BITCAST: R = PromoteFloatRes_BITCAST(N); break;
2349 case ISD::ConstantFP: R = PromoteFloatRes_ConstantFP(N); break;
2350 case ISD::EXTRACT_VECTOR_ELT:
2351 R = PromoteFloatRes_EXTRACT_VECTOR_ELT(N); break;
2352 case ISD::FCOPYSIGN: R = PromoteFloatRes_FCOPYSIGN(N); break;
2353
2354 // Unary FP Operations
2355 case ISD::FABS:
2356 case ISD::FCBRT:
2357 case ISD::FCEIL:
2358 case ISD::FCOS:
2359 case ISD::FEXP:
2360 case ISD::FEXP2:
2361 case ISD::FEXP10:
2362 case ISD::FFLOOR:
2363 case ISD::FLOG:
2364 case ISD::FLOG2:
2365 case ISD::FLOG10:
2366 case ISD::FNEARBYINT:
2367 case ISD::FNEG:
2368 case ISD::FRINT:
2369 case ISD::FROUND:
2370 case ISD::FROUNDEVEN:
2371 case ISD::FSIN:
2372 case ISD::FSQRT:
2373 case ISD::FTRUNC:
2374 case ISD::FCANONICALIZE: R = PromoteFloatRes_UnaryOp(N); break;
2375
2376 // Binary FP Operations
2377 case ISD::FADD:
2378 case ISD::FDIV:
2379 case ISD::FMAXIMUM:
2380 case ISD::FMINIMUM:
2381 case ISD::FMAXNUM:
2382 case ISD::FMINNUM:
2383 case ISD::FMUL:
2384 case ISD::FPOW:
2385 case ISD::FREM:
2386 case ISD::FSUB: R = PromoteFloatRes_BinOp(N); break;
2387
2388 case ISD::FMA: // FMA is same as FMAD
2389 case ISD::FMAD: R = PromoteFloatRes_FMAD(N); break;
2390
2391 case ISD::FPOWI:
2392 case ISD::FLDEXP: R = PromoteFloatRes_ExpOp(N); break;
2393 case ISD::FFREXP: R = PromoteFloatRes_FFREXP(N); break;
2394
2395 case ISD::FP_ROUND: R = PromoteFloatRes_FP_ROUND(N); break;
2396 case ISD::LOAD: R = PromoteFloatRes_LOAD(N); break;
2397 case ISD::SELECT: R = PromoteFloatRes_SELECT(N); break;
2398 case ISD::SELECT_CC: R = PromoteFloatRes_SELECT_CC(N); break;
2399
2400 case ISD::SINT_TO_FP:
2401 case ISD::UINT_TO_FP: R = PromoteFloatRes_XINT_TO_FP(N); break;
2402 case ISD::UNDEF: R = PromoteFloatRes_UNDEF(N); break;
2403 case ISD::ATOMIC_SWAP: R = BitcastToInt_ATOMIC_SWAP(N); break;
2404 case ISD::VECREDUCE_FADD:
2405 case ISD::VECREDUCE_FMUL:
2406 case ISD::VECREDUCE_FMIN:
2407 case ISD::VECREDUCE_FMAX:
2408 case ISD::VECREDUCE_FMAXIMUM:
2409 case ISD::VECREDUCE_FMINIMUM:
2410 R = PromoteFloatRes_VECREDUCE(N);
2411 break;
2412 case ISD::VECREDUCE_SEQ_FADD:
2413 case ISD::VECREDUCE_SEQ_FMUL:
2414 R = PromoteFloatRes_VECREDUCE_SEQ(N);
2415 break;
2416 }
2417
2418 if (R.getNode())
2419 SetPromotedFloat(SDValue(N, ResNo), R);
2420}
2421
2422// Bitcast from i16 to f16: convert the i16 to a f32 value instead.
2423// At this point, it is not possible to determine if the bitcast value is
2424// eventually stored to memory or promoted to f32 or promoted to a floating
2425// point at a higher precision. Some of these cases are handled by FP_EXTEND,
2426// STORE promotion handlers.
2427SDValue DAGTypeLegalizer::PromoteFloatRes_BITCAST(SDNode *N) {
2428 EVT VT = N->getValueType(0);
2429 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
2430 // Input type isn't guaranteed to be a scalar int so bitcast if not. The
2431 // bitcast will be legalized further if necessary.
2432 EVT IVT = EVT::getIntegerVT(*DAG.getContext(),
2433 N->getOperand(0).getValueType().getSizeInBits());
2434 SDValue Cast = DAG.getBitcast(IVT, N->getOperand(0));
2435 return DAG.getNode(GetPromotionOpcode(VT, NVT), SDLoc(N), NVT, Cast);
2436}
2437
2438SDValue DAGTypeLegalizer::PromoteFloatRes_ConstantFP(SDNode *N) {
2439 ConstantFPSDNode *CFPNode = cast<ConstantFPSDNode>(N);
2440 EVT VT = N->getValueType(0);
2441 SDLoc DL(N);
2442
2443 // Get the (bit-cast) APInt of the APFloat and build an integer constant
2444 EVT IVT = EVT::getIntegerVT(*DAG.getContext(), VT.getSizeInBits());
2445 SDValue C = DAG.getConstant(CFPNode->getValueAPF().bitcastToAPInt(), DL,
2446 IVT);
2447
2448 // Convert the Constant to the desired FP type
2449 // FIXME We might be able to do the conversion during compilation and get rid
2450 // of it from the object code
2451 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
2452 return DAG.getNode(GetPromotionOpcode(VT, NVT), DL, NVT, C);
2453}
2454
2455// If the Index operand is a constant, try to redirect the extract operation to
2456// the correct legalized vector. If not, bit-convert the input vector to
2457// equivalent integer vector. Extract the element as an (bit-cast) integer
2458// value and convert it to the promoted type.
2459SDValue DAGTypeLegalizer::PromoteFloatRes_EXTRACT_VECTOR_ELT(SDNode *N) {
2460 SDLoc DL(N);
2461
2462 // If the index is constant, try to extract the value from the legalized
2463 // vector type.
2464 if (isa<ConstantSDNode>(N->getOperand(1))) {
2465 SDValue Vec = N->getOperand(0);
2466 SDValue Idx = N->getOperand(1);
2467 EVT VecVT = Vec->getValueType(0);
2468 EVT EltVT = VecVT.getVectorElementType();
2469
2470 uint64_t IdxVal = cast<ConstantSDNode>(Idx)->getZExtValue();
2471
2472 switch (getTypeAction(VecVT)) {
2473 default: break;
2474 case TargetLowering::TypeScalarizeVector: {
2475 SDValue Res = GetScalarizedVector(N->getOperand(0));
2476 ReplaceValueWith(SDValue(N, 0), Res);
2477 return SDValue();
2478 }
2479 case TargetLowering::TypeWidenVector: {
2480 Vec = GetWidenedVector(Vec);
2481 SDValue Res = DAG.getNode(N->getOpcode(), DL, EltVT, Vec, Idx);
2482 ReplaceValueWith(SDValue(N, 0), Res);
2483 return SDValue();
2484 }
2485 case TargetLowering::TypeSplitVector: {
2486 SDValue Lo, Hi;
2487 GetSplitVector(Vec, Lo, Hi);
2488
2489 uint64_t LoElts = Lo.getValueType().getVectorNumElements();
2490 SDValue Res;
2491 if (IdxVal < LoElts)
2492 Res = DAG.getNode(N->getOpcode(), DL, EltVT, Lo, Idx);
2493 else
2494 Res = DAG.getNode(N->getOpcode(), DL, EltVT, Hi,
2495 DAG.getConstant(IdxVal - LoElts, DL,
2496 Idx.getValueType()));
2497 ReplaceValueWith(SDValue(N, 0), Res);
2498 return SDValue();
2499 }
2500
2501 }
2502 }
2503
2504 // Bit-convert the input vector to the equivalent integer vector
2505 SDValue NewOp = BitConvertVectorToIntegerVector(N->getOperand(0));
2506 EVT IVT = NewOp.getValueType().getVectorElementType();
2507
2508 // Extract the element as an (bit-cast) integer value
2509 SDValue NewVal = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, IVT,
2510 NewOp, N->getOperand(1));
2511
2512 // Convert the element to the desired FP type
2513 EVT VT = N->getValueType(0);
2514 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
2515 return DAG.getNode(GetPromotionOpcode(VT, NVT), SDLoc(N), NVT, NewVal);
2516}
2517
2518// FCOPYSIGN(X, Y) returns the value of X with the sign of Y. If the result
2519// needs promotion, so does the argument X. Note that Y, if needed, will be
2520// handled during operand promotion.
2521SDValue DAGTypeLegalizer::PromoteFloatRes_FCOPYSIGN(SDNode *N) {
2522 EVT VT = N->getValueType(0);
2523 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
2524 SDValue Op0 = GetPromotedFloat(N->getOperand(0));
2525
2526 SDValue Op1 = N->getOperand(1);
2527
2528 return DAG.getNode(N->getOpcode(), SDLoc(N), NVT, Op0, Op1);
2529}
2530
2531// Unary operation where the result and the operand have PromoteFloat type
2532// action. Construct a new SDNode with the promoted float value of the old
2533// operand.
2534SDValue DAGTypeLegalizer::PromoteFloatRes_UnaryOp(SDNode *N) {
2535 EVT VT = N->getValueType(0);
2536 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
2537 SDValue Op = GetPromotedFloat(N->getOperand(0));
2538
2539 return DAG.getNode(N->getOpcode(), SDLoc(N), NVT, Op);
2540}
2541
2542// Binary operations where the result and both operands have PromoteFloat type
2543// action. Construct a new SDNode with the promoted float values of the old
2544// operands.
2545SDValue DAGTypeLegalizer::PromoteFloatRes_BinOp(SDNode *N) {
2546 EVT VT = N->getValueType(0);
2547 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
2548 SDValue Op0 = GetPromotedFloat(N->getOperand(0));
2549 SDValue Op1 = GetPromotedFloat(N->getOperand(1));
2550 return DAG.getNode(N->getOpcode(), SDLoc(N), NVT, Op0, Op1, N->getFlags());
2551}
2552
2553SDValue DAGTypeLegalizer::PromoteFloatRes_FMAD(SDNode *N) {
2554 EVT VT = N->getValueType(0);
2555 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
2556 SDValue Op0 = GetPromotedFloat(N->getOperand(0));
2557 SDValue Op1 = GetPromotedFloat(N->getOperand(1));
2558 SDValue Op2 = GetPromotedFloat(N->getOperand(2));
2559
2560 return DAG.getNode(N->getOpcode(), SDLoc(N), NVT, Op0, Op1, Op2);
2561}
2562
2563// Promote the Float (first) operand and retain the Integer (second) operand
2564SDValue DAGTypeLegalizer::PromoteFloatRes_ExpOp(SDNode *N) {
2565 EVT VT = N->getValueType(0);
2566 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
2567 SDValue Op0 = GetPromotedFloat(N->getOperand(0));
2568 SDValue Op1 = N->getOperand(1);
2569
2570 return DAG.getNode(N->getOpcode(), SDLoc(N), NVT, Op0, Op1);
2571}
2572
2573SDValue DAGTypeLegalizer::PromoteFloatRes_FFREXP(SDNode *N) {
2574 EVT VT = N->getValueType(0);
2575 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
2576 SDValue Op = GetPromotedFloat(N->getOperand(0));
2577 SDValue Res =
2578 DAG.getNode(N->getOpcode(), SDLoc(N), {NVT, N->getValueType(1)}, Op);
2579
2580 ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
2581 return Res;
2582}
2583
2584// Explicit operation to reduce precision. Reduce the value to half precision
2585// and promote it back to the legal type.
2586SDValue DAGTypeLegalizer::PromoteFloatRes_FP_ROUND(SDNode *N) {
2587 SDLoc DL(N);
2588
2589 SDValue Op = N->getOperand(0);
2590 EVT VT = N->getValueType(0);
2591 EVT OpVT = Op->getValueType(0);
2592 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
2593 EVT IVT = EVT::getIntegerVT(*DAG.getContext(), VT.getSizeInBits());
2594
2595 // Round promoted float to desired precision
2596 SDValue Round = DAG.getNode(GetPromotionOpcode(OpVT, VT), DL, IVT, Op);
2597 // Promote it back to the legal output type
2598 return DAG.getNode(GetPromotionOpcode(VT, NVT), DL, NVT, Round);
2599}
2600
2601SDValue DAGTypeLegalizer::PromoteFloatRes_LOAD(SDNode *N) {
2602 LoadSDNode *L = cast<LoadSDNode>(N);
2603 EVT VT = N->getValueType(0);
2604
2605 // Load the value as an integer value with the same number of bits.
2606 EVT IVT = EVT::getIntegerVT(*DAG.getContext(), VT.getSizeInBits());
2607 SDValue newL = DAG.getLoad(
2608 L->getAddressingMode(), L->getExtensionType(), IVT, SDLoc(N),
2609 L->getChain(), L->getBasePtr(), L->getOffset(), L->getPointerInfo(), IVT,
2610 L->getOriginalAlign(), L->getMemOperand()->getFlags(), L->getAAInfo());
2611 // Legalize the chain result by replacing uses of the old value chain with the
2612 // new one
2613 ReplaceValueWith(SDValue(N, 1), newL.getValue(1));
2614
2615 // Convert the integer value to the desired FP type
2616 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
2617 return DAG.getNode(GetPromotionOpcode(VT, NVT), SDLoc(N), NVT, newL);
2618}
2619
2620// Construct a new SELECT node with the promoted true- and false- values.
2621SDValue DAGTypeLegalizer::PromoteFloatRes_SELECT(SDNode *N) {
2622 SDValue TrueVal = GetPromotedFloat(N->getOperand(1));
2623 SDValue FalseVal = GetPromotedFloat(N->getOperand(2));
2624
2625 return DAG.getNode(ISD::SELECT, SDLoc(N), TrueVal->getValueType(0),
2626 N->getOperand(0), TrueVal, FalseVal);
2627}
2628
2629// Construct a new SELECT_CC node with the promoted true- and false- values.
2630// The operands used for comparison are promoted by PromoteFloatOp_SELECT_CC.
2631SDValue DAGTypeLegalizer::PromoteFloatRes_SELECT_CC(SDNode *N) {
2632 SDValue TrueVal = GetPromotedFloat(N->getOperand(2));
2633 SDValue FalseVal = GetPromotedFloat(N->getOperand(3));
2634
2635 return DAG.getNode(ISD::SELECT_CC, SDLoc(N),
2636 TrueVal.getNode()->getValueType(0), N->getOperand(0),
2637 N->getOperand(1), TrueVal, FalseVal, N->getOperand(4));
2638}
2639
2640// Construct a SDNode that transforms the SINT or UINT operand to the promoted
2641// float type.
2642SDValue DAGTypeLegalizer::PromoteFloatRes_XINT_TO_FP(SDNode *N) {
2643 SDLoc DL(N);
2644 EVT VT = N->getValueType(0);
2645 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
2646 SDValue NV = DAG.getNode(N->getOpcode(), DL, NVT, N->getOperand(0));
2647 // Round the value to the desired precision (that of the source type).
2648 return DAG.getNode(
2649 ISD::FP_EXTEND, DL, NVT,
2650 DAG.getNode(ISD::FP_ROUND, DL, VT, NV,
2651 DAG.getIntPtrConstant(0, DL, /*isTarget=*/true)));
2652}
2653
2654SDValue DAGTypeLegalizer::PromoteFloatRes_UNDEF(SDNode *N) {
2655 return DAG.getUNDEF(TLI.getTypeToTransformTo(*DAG.getContext(),
2656 N->getValueType(0)));
2657}
2658
2659SDValue DAGTypeLegalizer::PromoteFloatRes_VECREDUCE(SDNode *N) {
2660 // Expand and promote recursively.
2661 // TODO: This is non-optimal, but dealing with the concurrently happening
2662 // vector-legalization is non-trivial. We could do something similar to
2663 // PromoteFloatRes_EXTRACT_VECTOR_ELT here.
2664 ReplaceValueWith(SDValue(N, 0), TLI.expandVecReduce(N, DAG));
2665 return SDValue();
2666}
2667
2668SDValue DAGTypeLegalizer::PromoteFloatRes_VECREDUCE_SEQ(SDNode *N) {
2669 ReplaceValueWith(SDValue(N, 0), TLI.expandVecReduceSeq(N, DAG));
2670 return SDValue();
2671}
2672
2673SDValue DAGTypeLegalizer::BitcastToInt_ATOMIC_SWAP(SDNode *N) {
2674 EVT VT = N->getValueType(0);
2675
2676 AtomicSDNode *AM = cast<AtomicSDNode>(N);
2677 SDLoc SL(N);
2678
2679 SDValue CastVal = BitConvertToInteger(AM->getVal());
2680 EVT CastVT = CastVal.getValueType();
2681
2682 SDValue NewAtomic
2683 = DAG.getAtomic(ISD::ATOMIC_SWAP, SL, CastVT,
2684 DAG.getVTList(CastVT, MVT::Other),
2685 { AM->getChain(), AM->getBasePtr(), CastVal },
2686 AM->getMemOperand());
2687
2688 SDValue Result = NewAtomic;
2689
2690 if (getTypeAction(VT) == TargetLowering::TypePromoteFloat) {
2691 EVT NFPVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
2692 Result = DAG.getNode(GetPromotionOpcode(VT, NFPVT), SL, NFPVT,
2693 NewAtomic);
2694 }
2695
2696 // Legalize the chain result by replacing uses of the old value chain with the
2697 // new one
2698 ReplaceValueWith(SDValue(N, 1), NewAtomic.getValue(1));
2699
2700 return Result;
2701
2702}
2703
2704//===----------------------------------------------------------------------===//
2705// Half Result Soft Promotion
2706//===----------------------------------------------------------------------===//
2707
2708void DAGTypeLegalizer::SoftPromoteHalfResult(SDNode *N, unsigned ResNo) {
2709 LLVM_DEBUG(dbgs() << "Soft promote half result " << ResNo << ": ";
2710 N->dump(&DAG));
2711 SDValue R = SDValue();
2712
2713 // See if the target wants to custom expand this node.
2714 if (CustomLowerNode(N, N->getValueType(ResNo), true)) {
2715 LLVM_DEBUG(dbgs() << "Node has been custom expanded, done\n");
2716 return;
2717 }
2718
2719 switch (N->getOpcode()) {
2720 default:
2721#ifndef NDEBUG
2722 dbgs() << "SoftPromoteHalfResult #" << ResNo << ": ";
2723 N->dump(&DAG); dbgs() << "\n";
2724#endif
2725 report_fatal_error("Do not know how to soft promote this operator's "
2726 "result!");
2727
2728 case ISD::BITCAST: R = SoftPromoteHalfRes_BITCAST(N); break;
2729 case ISD::ConstantFP: R = SoftPromoteHalfRes_ConstantFP(N); break;
2730 case ISD::EXTRACT_VECTOR_ELT:
2731 R = SoftPromoteHalfRes_EXTRACT_VECTOR_ELT(N); break;
2732 case ISD::FCOPYSIGN: R = SoftPromoteHalfRes_FCOPYSIGN(N); break;
2733 case ISD::STRICT_FP_ROUND:
2734 case ISD::FP_ROUND: R = SoftPromoteHalfRes_FP_ROUND(N); break;
2735
2736 // Unary FP Operations
2737 case ISD::FABS:
2738 case ISD::FCBRT:
2739 case ISD::FCEIL:
2740 case ISD::FCOS:
2741 case ISD::FEXP:
2742 case ISD::FEXP2:
2743 case ISD::FEXP10:
2744 case ISD::FFLOOR:
2745 case ISD::FLOG:
2746 case ISD::FLOG2:
2747 case ISD::FLOG10:
2748 case ISD::FNEARBYINT:
2749 case ISD::FNEG:
2750 case ISD::FREEZE:
2751 case ISD::FRINT:
2752 case ISD::FROUND:
2753 case ISD::FROUNDEVEN:
2754 case ISD::FSIN:
2755 case ISD::FSQRT:
2756 case ISD::FTRUNC:
2757 case ISD::FCANONICALIZE: R = SoftPromoteHalfRes_UnaryOp(N); break;
2758
2759 // Binary FP Operations
2760 case ISD::FADD:
2761 case ISD::FDIV:
2762 case ISD::FMAXIMUM:
2763 case ISD::FMINIMUM:
2764 case ISD::FMAXNUM:
2765 case ISD::FMINNUM:
2766 case ISD::FMUL:
2767 case ISD::FPOW:
2768 case ISD::FREM:
2769 case ISD::FSUB: R = SoftPromoteHalfRes_BinOp(N); break;
2770
2771 case ISD::FMA: // FMA is same as FMAD
2772 case ISD::FMAD: R = SoftPromoteHalfRes_FMAD(N); break;
2773
2774 case ISD::FPOWI:
2775 case ISD::FLDEXP: R = SoftPromoteHalfRes_ExpOp(N); break;
2776
2777 case ISD::LOAD: R = SoftPromoteHalfRes_LOAD(N); break;
2778 case ISD::SELECT: R = SoftPromoteHalfRes_SELECT(N); break;
2779 case ISD::SELECT_CC: R = SoftPromoteHalfRes_SELECT_CC(N); break;
2780 case ISD::SINT_TO_FP:
2781 case ISD::UINT_TO_FP: R = SoftPromoteHalfRes_XINT_TO_FP(N); break;
2782 case ISD::UNDEF: R = SoftPromoteHalfRes_UNDEF(N); break;
2783 case ISD::ATOMIC_SWAP: R = BitcastToInt_ATOMIC_SWAP(N); break;
2784 case ISD::VECREDUCE_FADD:
2785 case ISD::VECREDUCE_FMUL:
2786 case ISD::VECREDUCE_FMIN:
2787 case ISD::VECREDUCE_FMAX:
2788 case ISD::VECREDUCE_FMAXIMUM:
2789 case ISD::VECREDUCE_FMINIMUM:
2790 R = SoftPromoteHalfRes_VECREDUCE(N);
2791 break;
2792 case ISD::VECREDUCE_SEQ_FADD:
2793 case ISD::VECREDUCE_SEQ_FMUL:
2794 R = SoftPromoteHalfRes_VECREDUCE_SEQ(N);
2795 break;
2796 }
2797
2798 if (R.getNode())
2799 SetSoftPromotedHalf(SDValue(N, ResNo), R);
2800}
2801
2802SDValue DAGTypeLegalizer::SoftPromoteHalfRes_BITCAST(SDNode *N) {
2803 return BitConvertToInteger(N->getOperand(0));
2804}
2805
2806SDValue DAGTypeLegalizer::SoftPromoteHalfRes_ConstantFP(SDNode *N) {
2807 ConstantFPSDNode *CN = cast<ConstantFPSDNode>(N);
2808
2809 // Get the (bit-cast) APInt of the APFloat and build an integer constant
2810 return DAG.getConstant(CN->getValueAPF().bitcastToAPInt(), SDLoc(CN),
2811 MVT::i16);
2812}
2813
2814SDValue DAGTypeLegalizer::SoftPromoteHalfRes_EXTRACT_VECTOR_ELT(SDNode *N) {
2815 SDValue NewOp = BitConvertVectorToIntegerVector(N->getOperand(0));
2816 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SDLoc(N),
2817 NewOp.getValueType().getVectorElementType(), NewOp,
2818 N->getOperand(1));
2819}
2820
2821SDValue DAGTypeLegalizer::SoftPromoteHalfRes_FCOPYSIGN(SDNode *N) {
2822 SDValue LHS = GetSoftPromotedHalf(N->getOperand(0));
2823 SDValue RHS = BitConvertToInteger(N->getOperand(1));
2824 SDLoc dl(N);
2825
2826 EVT LVT = LHS.getValueType();
2827 EVT RVT = RHS.getValueType();
2828
2829 unsigned LSize = LVT.getSizeInBits();
2830 unsigned RSize = RVT.getSizeInBits();
2831
2832 // First get the sign bit of second operand.
2833 SDValue SignBit = DAG.getNode(
2834 ISD::SHL, dl, RVT, DAG.getConstant(1, dl, RVT),
2835 DAG.getConstant(RSize - 1, dl,
2836 TLI.getShiftAmountTy(RVT, DAG.getDataLayout())));
2837 SignBit = DAG.getNode(ISD::AND, dl, RVT, RHS, SignBit);
2838
2839 // Shift right or sign-extend it if the two operands have different types.
2840 int SizeDiff = RVT.getSizeInBits() - LVT.getSizeInBits();
2841 if (SizeDiff > 0) {
2842 SignBit =
2843 DAG.getNode(ISD::SRL, dl, RVT, SignBit,
2844 DAG.getConstant(SizeDiff, dl,
2845 TLI.getShiftAmountTy(SignBit.getValueType(),
2846 DAG.getDataLayout())));
2847 SignBit = DAG.getNode(ISD::TRUNCATE, dl, LVT, SignBit);
2848 } else if (SizeDiff < 0) {
2849 SignBit = DAG.getNode(ISD::ANY_EXTEND, dl, LVT, SignBit);
2850 SignBit =
2851 DAG.getNode(ISD::SHL, dl, LVT, SignBit,
2852 DAG.getConstant(-SizeDiff, dl,
2853 TLI.getShiftAmountTy(SignBit.getValueType(),
2854 DAG.getDataLayout())));
2855 }
2856
2857 // Clear the sign bit of the first operand.
2858 SDValue Mask = DAG.getNode(
2859 ISD::SHL, dl, LVT, DAG.getConstant(1, dl, LVT),
2860 DAG.getConstant(LSize - 1, dl,
2861 TLI.getShiftAmountTy(LVT, DAG.getDataLayout())));
2862 Mask = DAG.getNode(ISD::SUB, dl, LVT, Mask, DAG.getConstant(1, dl, LVT));
2863 LHS = DAG.getNode(ISD::AND, dl, LVT, LHS, Mask);
2864
2865 // Or the value with the sign bit.
2866 return DAG.getNode(ISD::OR, dl, LVT, LHS, SignBit);
2867}
2868
2869SDValue DAGTypeLegalizer::SoftPromoteHalfRes_FMAD(SDNode *N) {
2870 EVT OVT = N->getValueType(0);
2871 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), OVT);
2872 SDValue Op0 = GetSoftPromotedHalf(N->getOperand(0));
2873 SDValue Op1 = GetSoftPromotedHalf(N->getOperand(1));
2874 SDValue Op2 = GetSoftPromotedHalf(N->getOperand(2));
2875 SDLoc dl(N);
2876
2877 // Promote to the larger FP type.
2878 auto PromotionOpcode = GetPromotionOpcode(OVT, NVT);
2879 Op0 = DAG.getNode(PromotionOpcode, dl, NVT, Op0);
2880 Op1 = DAG.getNode(PromotionOpcode, dl, NVT, Op1);
2881 Op2 = DAG.getNode(PromotionOpcode, dl, NVT, Op2);
2882
2883 SDValue Res = DAG.getNode(N->getOpcode(), dl, NVT, Op0, Op1, Op2);
2884
2885 // Convert back to FP16 as an integer.
2886 return DAG.getNode(GetPromotionOpcode(NVT, OVT), dl, MVT::i16, Res);
2887}
2888
2889SDValue DAGTypeLegalizer::SoftPromoteHalfRes_ExpOp(SDNode *N) {
2890 EVT OVT = N->getValueType(0);
2891 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), OVT);
2892 SDValue Op0 = GetSoftPromotedHalf(N->getOperand(0));
2893 SDValue Op1 = N->getOperand(1);
2894 SDLoc dl(N);
2895
2896 // Promote to the larger FP type.
2897 Op0 = DAG.getNode(GetPromotionOpcode(OVT, NVT), dl, NVT, Op0);
2898
2899 SDValue Res = DAG.getNode(N->getOpcode(), dl, NVT, Op0, Op1);
2900
2901 // Convert back to FP16 as an integer.
2902 return DAG.getNode(GetPromotionOpcode(NVT, OVT), dl, MVT::i16, Res);
2903}
2904
2905SDValue DAGTypeLegalizer::SoftPromoteHalfRes_FP_ROUND(SDNode *N) {
2906 EVT RVT = N->getValueType(0);
2907 EVT SVT = N->getOperand(0).getValueType();
2908
2909 if (N->isStrictFPOpcode()) {
2910 assert(RVT == MVT::f16);
2911 SDValue Res =
2912 DAG.getNode(ISD::STRICT_FP_TO_FP16, SDLoc(N), {MVT::i16, MVT::Other},
2913 {N->getOperand(0), N->getOperand(1)});
2914 ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
2915 return Res;
2916 }
2917
2918 return DAG.getNode(GetPromotionOpcode(SVT, RVT), SDLoc(N), MVT::i16,
2919 N->getOperand(0));
2920}
2921
2922SDValue DAGTypeLegalizer::SoftPromoteHalfRes_LOAD(SDNode *N) {
2923 LoadSDNode *L = cast<LoadSDNode>(N);
2924
2925 // Load the value as an integer value with the same number of bits.
2926 assert(L->getExtensionType() == ISD::NON_EXTLOAD && "Unexpected extension!");
2927 SDValue NewL =
2928 DAG.getLoad(L->getAddressingMode(), L->getExtensionType(), MVT::i16,
2929 SDLoc(N), L->getChain(), L->getBasePtr(), L->getOffset(),
2930 L->getPointerInfo(), MVT::i16, L->getOriginalAlign(),
2931 L->getMemOperand()->getFlags(), L->getAAInfo());
2932 // Legalize the chain result by replacing uses of the old value chain with the
2933 // new one
2934 ReplaceValueWith(SDValue(N, 1), NewL.getValue(1));
2935 return NewL;
2936}
2937
2938SDValue DAGTypeLegalizer::SoftPromoteHalfRes_SELECT(SDNode *N) {
2939 SDValue Op1 = GetSoftPromotedHalf(N->getOperand(1));
2940 SDValue Op2 = GetSoftPromotedHalf(N->getOperand(2));
2941 return DAG.getSelect(SDLoc(N), Op1.getValueType(), N->getOperand(0), Op1,
2942 Op2);
2943}
2944
2945SDValue DAGTypeLegalizer::SoftPromoteHalfRes_SELECT_CC(SDNode *N) {
2946 SDValue Op2 = GetSoftPromotedHalf(N->getOperand(2));
2947 SDValue Op3 = GetSoftPromotedHalf(N->getOperand(3));
2948 return DAG.getNode(ISD::SELECT_CC, SDLoc(N), Op2.getValueType(),
2949 N->getOperand(0), N->getOperand(1), Op2, Op3,
2950 N->getOperand(4));
2951}
2952
2953SDValue DAGTypeLegalizer::SoftPromoteHalfRes_XINT_TO_FP(SDNode *N) {
2954 EVT OVT = N->getValueType(0);
2955 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), OVT);
2956 SDLoc dl(N);
2957
2958 SDValue Res = DAG.getNode(N->getOpcode(), dl, NVT, N->getOperand(0));
2959
2960 // Round the value to the softened type.
2961 return DAG.getNode(GetPromotionOpcode(NVT, OVT), dl, MVT::i16, Res);
2962}
2963
2964SDValue DAGTypeLegalizer::SoftPromoteHalfRes_UNDEF(SDNode *N) {
2965 return DAG.getUNDEF(MVT::i16);
2966}
2967
2968SDValue DAGTypeLegalizer::SoftPromoteHalfRes_UnaryOp(SDNode *N) {
2969 EVT OVT = N->getValueType(0);
2970 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), OVT);
2971 SDValue Op = GetSoftPromotedHalf(N->getOperand(0));
2972 SDLoc dl(N);
2973
2974 // Promote to the larger FP type.
2975 Op = DAG.getNode(GetPromotionOpcode(OVT, NVT), dl, NVT, Op);
2976
2977 SDValue Res = DAG.getNode(N->getOpcode(), dl, NVT, Op);
2978
2979 // Convert back to FP16 as an integer.
2980 return DAG.getNode(GetPromotionOpcode(NVT, OVT), dl, MVT::i16, Res);
2981}
2982
2983SDValue DAGTypeLegalizer::SoftPromoteHalfRes_BinOp(SDNode *N) {
2984 EVT OVT = N->getValueType(0);
2985 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), OVT);
2986 SDValue Op0 = GetSoftPromotedHalf(N->getOperand(0));
2987 SDValue Op1 = GetSoftPromotedHalf(N->getOperand(1));
2988 SDLoc dl(N);
2989
2990 // Promote to the larger FP type.
2991 auto PromotionOpcode = GetPromotionOpcode(OVT, NVT);
2992 Op0 = DAG.getNode(PromotionOpcode, dl, NVT, Op0);
2993 Op1 = DAG.getNode(PromotionOpcode, dl, NVT, Op1);
2994
2995 SDValue Res = DAG.getNode(N->getOpcode(), dl, NVT, Op0, Op1);
2996
2997 // Convert back to FP16 as an integer.
2998 return DAG.getNode(GetPromotionOpcode(NVT, OVT), dl, MVT::i16, Res);
2999}
3000
3001SDValue DAGTypeLegalizer::SoftPromoteHalfRes_VECREDUCE(SDNode *N) {
3002 // Expand and soften recursively.
3003 ReplaceValueWith(SDValue(N, 0), TLI.expandVecReduce(N, DAG));
3004 return SDValue();
3005}
3006
3007SDValue DAGTypeLegalizer::SoftPromoteHalfRes_VECREDUCE_SEQ(SDNode *N) {
3008 // Expand and soften.
3009 ReplaceValueWith(SDValue(N, 0), TLI.expandVecReduceSeq(N, DAG));
3010 return SDValue();
3011}
3012
3013//===----------------------------------------------------------------------===//
3014// Half Operand Soft Promotion
3015//===----------------------------------------------------------------------===//
3016
3017bool DAGTypeLegalizer::SoftPromoteHalfOperand(SDNode *N, unsigned OpNo) {
3018 LLVM_DEBUG(dbgs() << "Soft promote half operand " << OpNo << ": ";
3019 N->dump(&DAG));
3020 SDValue Res = SDValue();
3021
3022 if (CustomLowerNode(N, N->getOperand(OpNo).getValueType(), false)) {
3023 LLVM_DEBUG(dbgs() << "Node has been custom lowered, done\n");
3024 return false;
3025 }
3026
3027 // Nodes that use a promotion-requiring floating point operand, but doesn't
3028 // produce a soft promotion-requiring floating point result, need to be
3029 // legalized to use the soft promoted float operand. Nodes that produce at
3030 // least one soft promotion-requiring floating point result have their
3031 // operands legalized as a part of PromoteFloatResult.
3032 switch (N->getOpcode()) {
3033 default:
3034 #ifndef NDEBUG
3035 dbgs() << "SoftPromoteHalfOperand Op #" << OpNo << ": ";
3036 N->dump(&DAG); dbgs() << "\n";
3037 #endif
3038 report_fatal_error("Do not know how to soft promote this operator's "
3039 "operand!");
3040
3041 case ISD::BITCAST: Res = SoftPromoteHalfOp_BITCAST(N); break;
3042 case ISD::FCOPYSIGN: Res = SoftPromoteHalfOp_FCOPYSIGN(N, OpNo); break;
3043 case ISD::FP_TO_SINT:
3044 case ISD::FP_TO_UINT: Res = SoftPromoteHalfOp_FP_TO_XINT(N); break;
3045 case ISD::FP_TO_SINT_SAT:
3046 case ISD::FP_TO_UINT_SAT:
3047 Res = SoftPromoteHalfOp_FP_TO_XINT_SAT(N); break;
3048 case ISD::STRICT_FP_EXTEND:
3049 case ISD::FP_EXTEND: Res = SoftPromoteHalfOp_FP_EXTEND(N); break;
3050 case ISD::SELECT_CC: Res = SoftPromoteHalfOp_SELECT_CC(N, OpNo); break;
3051 case ISD::SETCC: Res = SoftPromoteHalfOp_SETCC(N); break;
3052 case ISD::STORE: Res = SoftPromoteHalfOp_STORE(N, OpNo); break;
3053 case ISD::STACKMAP:
3054 Res = SoftPromoteHalfOp_STACKMAP(N, OpNo);
3055 break;
3056 case ISD::PATCHPOINT:
3057 Res = SoftPromoteHalfOp_PATCHPOINT(N, OpNo);
3058 break;
3059 }
3060
3061 if (!Res.getNode())
3062 return false;
3063
3064 assert(Res.getNode() != N && "Expected a new node!");
3065
3066 assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 &&
3067 "Invalid operand expansion");
3068
3069 ReplaceValueWith(SDValue(N, 0), Res);
3070 return false;
3071}
3072
3073SDValue DAGTypeLegalizer::SoftPromoteHalfOp_BITCAST(SDNode *N) {
3074 SDValue Op0 = GetSoftPromotedHalf(N->getOperand(0));
3075
3076 return DAG.getNode(ISD::BITCAST, SDLoc(N), N->getValueType(0), Op0);
3077}
3078
3079SDValue DAGTypeLegalizer::SoftPromoteHalfOp_FCOPYSIGN(SDNode *N,
3080 unsigned OpNo) {
3081 assert(OpNo == 1 && "Only Operand 1 must need promotion here");
3082 SDValue Op1 = N->getOperand(1);
3083 EVT RVT = Op1.getValueType();
3084 SDLoc dl(N);
3085
3086 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), Op1.getValueType());
3087
3088 Op1 = GetSoftPromotedHalf(Op1);
3089 Op1 = DAG.getNode(GetPromotionOpcode(RVT, NVT), dl, NVT, Op1);
3090
3091 return DAG.getNode(N->getOpcode(), dl, N->getValueType(0), N->getOperand(0),
3092 Op1);
3093}
3094
3095SDValue DAGTypeLegalizer::SoftPromoteHalfOp_FP_EXTEND(SDNode *N) {
3096 EVT RVT = N->getValueType(0);
3097 bool IsStrict = N->isStrictFPOpcode();
3098 SDValue Op = N->getOperand(IsStrict ? 1 : 0);
3099 EVT SVT = Op.getValueType();
3100 Op = GetSoftPromotedHalf(N->getOperand(IsStrict ? 1 : 0));
3101
3102 if (IsStrict) {
3103 assert(SVT == MVT::f16);
3104 SDValue Res =
3105 DAG.getNode(ISD::STRICT_FP16_TO_FP, SDLoc(N),
3106 {N->getValueType(0), MVT::Other}, {N->getOperand(0), Op});
3107 ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
3108 ReplaceValueWith(SDValue(N, 0), Res);
3109 return SDValue();
3110 }
3111
3112 return DAG.getNode(GetPromotionOpcode(SVT, RVT), SDLoc(N), RVT, Op);
3113}
3114
3115SDValue DAGTypeLegalizer::SoftPromoteHalfOp_FP_TO_XINT(SDNode *N) {
3116 EVT RVT = N->getValueType(0);
3117 SDValue Op = N->getOperand(0);
3118 EVT SVT = Op.getValueType();
3119 SDLoc dl(N);
3120
3121 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), Op.getValueType());
3122
3123 Op = GetSoftPromotedHalf(Op);
3124
3125 SDValue Res = DAG.getNode(GetPromotionOpcode(SVT, RVT), dl, NVT, Op);
3126
3127 return DAG.getNode(N->getOpcode(), dl, N->getValueType(0), Res);
3128}
3129
3130SDValue DAGTypeLegalizer::SoftPromoteHalfOp_FP_TO_XINT_SAT(SDNode *N) {
3131 EVT RVT = N->getValueType(0);
3132 SDValue Op = N->getOperand(0);
3133 EVT SVT = Op.getValueType();
3134 SDLoc dl(N);
3135
3136 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), Op.getValueType());
3137
3138 Op = GetSoftPromotedHalf(Op);
3139
3140 SDValue Res = DAG.getNode(GetPromotionOpcode(SVT, RVT), dl, NVT, Op);
3141
3142 return DAG.getNode(N->getOpcode(), dl, N->getValueType(0), Res,
3143 N->getOperand(1));
3144}
3145
3146SDValue DAGTypeLegalizer::SoftPromoteHalfOp_SELECT_CC(SDNode *N,
3147 unsigned OpNo) {
3148 assert(OpNo == 0 && "Can only soften the comparison values");
3149 SDValue Op0 = N->getOperand(0);
3150 SDValue Op1 = N->getOperand(1);
3151 SDLoc dl(N);
3152
3153 EVT SVT = Op0.getValueType();
3154 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), SVT);
3155
3156 Op0 = GetSoftPromotedHalf(Op0);
3157 Op1 = GetSoftPromotedHalf(Op1);
3158
3159 // Promote to the larger FP type.
3160 auto PromotionOpcode = GetPromotionOpcode(SVT, NVT);
3161 Op0 = DAG.getNode(PromotionOpcode, dl, NVT, Op0);
3162 Op1 = DAG.getNode(PromotionOpcode, dl, NVT, Op1);
3163
3164 return DAG.getNode(ISD::SELECT_CC, SDLoc(N), N->getValueType(0), Op0, Op1,
3165 N->getOperand(2), N->getOperand(3), N->getOperand(4));
3166}
3167
3168SDValue DAGTypeLegalizer::SoftPromoteHalfOp_SETCC(SDNode *N) {
3169 SDValue Op0 = N->getOperand(0);
3170 SDValue Op1 = N->getOperand(1);
3171 ISD::CondCode CCCode = cast<CondCodeSDNode>(N->getOperand(2))->get();
3172 SDLoc dl(N);
3173
3174 EVT SVT = Op0.getValueType();
3175 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), Op0.getValueType());
3176
3177 Op0 = GetSoftPromotedHalf(Op0);
3178 Op1 = GetSoftPromotedHalf(Op1);
3179
3180 // Promote to the larger FP type.
3181 auto PromotionOpcode = GetPromotionOpcode(SVT, NVT);
3182 Op0 = DAG.getNode(PromotionOpcode, dl, NVT, Op0);
3183 Op1 = DAG.getNode(PromotionOpcode, dl, NVT, Op1);
3184
3185 return DAG.getSetCC(SDLoc(N), N->getValueType(0), Op0, Op1, CCCode);
3186}
3187
3188SDValue DAGTypeLegalizer::SoftPromoteHalfOp_STORE(SDNode *N, unsigned OpNo) {
3189 assert(OpNo == 1 && "Can only soften the stored value!");
3190 StoreSDNode *ST = cast<StoreSDNode>(N);
3191 SDValue Val = ST->getValue();
3192 SDLoc dl(N);
3193
3194 assert(!ST->isTruncatingStore() && "Unexpected truncating store.");
3195 SDValue Promoted = GetSoftPromotedHalf(Val);
3196 return DAG.getStore(ST->getChain(), dl, Promoted, ST->getBasePtr(),
3197 ST->getMemOperand());
3198}
3199
3200SDValue DAGTypeLegalizer::SoftPromoteHalfOp_STACKMAP(SDNode *N, unsigned OpNo) {
3201 assert(OpNo > 1); // Because the first two arguments are guaranteed legal.
3202 SmallVector<SDValue> NewOps(N->ops().begin(), N->ops().end());
3203 SDValue Op = N->getOperand(OpNo);
3204 NewOps[OpNo] = GetSoftPromotedHalf(Op);
3205 SDValue NewNode =
3206 DAG.getNode(N->getOpcode(), SDLoc(N), N->getVTList(), NewOps);
3207
3208 for (unsigned ResNum = 0; ResNum < N->getNumValues(); ResNum++)
3209 ReplaceValueWith(SDValue(N, ResNum), NewNode.getValue(ResNum));
3210
3211 return SDValue(); // Signal that we replaced the node ourselves.
3212}
3213
3214SDValue DAGTypeLegalizer::SoftPromoteHalfOp_PATCHPOINT(SDNode *N,
3215 unsigned OpNo) {
3216 assert(OpNo >= 7);
3217 SmallVector<SDValue> NewOps(N->ops().begin(), N->ops().end());
3218 SDValue Op = N->getOperand(OpNo);
3219 NewOps[OpNo] = GetSoftPromotedHalf(Op);
3220 SDValue NewNode =
3221 DAG.getNode(N->getOpcode(), SDLoc(N), N->getVTList(), NewOps);
3222
3223 for (unsigned ResNum = 0; ResNum < N->getNumValues(); ResNum++)
3224 ReplaceValueWith(SDValue(N, ResNum), NewNode.getValue(ResNum));
3225
3226 return SDValue(); // Signal that we replaced the node ourselves.
3227}
DL
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
Definition: AArch64SLSHardening.cpp:75
Idx
Returns the sub type a function will return at a given Idx Should correspond to the result type of an ExtractValue instruction executed with just that one unsigned Idx
Definition: DeadArgumentElimination.cpp:354
LLVM_DEBUG
#define LLVM_DEBUG(X)
Definition: Debug.h:101
Size
uint64_t Size
Definition: ELFObjHandler.cpp:81
isSigned
static bool isSigned(unsigned int Opcode)
Definition: ExpandLargeDivRem.cpp:53
findFPToIntLibcall
static RTLIB::Libcall findFPToIntLibcall(EVT SrcVT, EVT RetVT, EVT &Promoted, bool Signed)
Definition: LegalizeFloatTypes.cpp:1026
GetFPLibCall
static RTLIB::Libcall GetFPLibCall(EVT VT, RTLIB::Libcall Call_F32, RTLIB::Libcall Call_F64, RTLIB::Libcall Call_F80, RTLIB::Libcall Call_F128, RTLIB::Libcall Call_PPCF128)
GetFPLibCall - Return the right libcall for the given floating point type.
Definition: LegalizeFloatTypes.cpp:32
GetPromotionOpcode
static ISD::NodeType GetPromotionOpcode(EVT OpVT, EVT RetVT)
Definition: LegalizeFloatTypes.cpp:2170
assert
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
RHS
Value * RHS
Definition: X86PartialReduction.cpp:76
LHS
Value * LHS
Definition: X86PartialReduction.cpp:75
llvm::APFloat::bitcastToAPInt
APInt bitcastToAPInt() const
Definition: APFloat.h:1210
llvm::APInt
Class for arbitrary precision integers.
Definition: APInt.h:76
llvm::APInt::getAllOnes
static APInt getAllOnes(unsigned numBits)
Return an APInt of a specified width with all bits set.
Definition: APInt.h:212
llvm::APInt::clearBit
void clearBit(unsigned BitPosition)
Set a given bit to 0.
Definition: APInt.h:1379
llvm::APInt::getSignMask
static APInt getSignMask(unsigned BitWidth)
Get the SignMask for a specific bit width.
Definition: APInt.h:207
llvm::APInt::getRawData
const uint64_t * getRawData() const
This function returns a pointer to the internal storage of the APInt.
Definition: APInt.h:547
llvm::ArrayRef
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
Definition: ArrayRef.h:41
llvm::AtomicSDNode
This is an SDNode representing atomic operations.
Definition: SelectionDAGNodes.h:1456
llvm::AtomicSDNode::getVal
const SDValue & getVal() const
Definition: SelectionDAGNodes.h:1468
llvm::ConstantFPSDNode::getValueAPF
const APFloat & getValueAPF() const
Definition: SelectionDAGNodes.h:1658
llvm::DAGTypeLegalizer::NewNode
@ NewNode
This is a new node, not before seen, that was created in the process of legalizing some other node.
Definition: LegalizeTypes.h:42
llvm::DWARFExpression::Operation
This class represents an Operation in the Expression.
Definition: DWARFExpression.h:32
llvm::DataLayout::isBigEndian
bool isBigEndian() const
Definition: DataLayout.h:239
llvm::LLVMContext::emitError
void emitError(uint64_t LocCookie, const Twine &ErrorStr)
emitError - Emit an error message to the currently installed error handler with optional location inf...
Definition: LLVMContext.cpp:276
llvm::LoadSDNode
This class is used to represent ISD::LOAD nodes.
Definition: SelectionDAGNodes.h:2364
llvm::MVT::SimpleTy
SimpleValueType SimpleTy
Definition: MachineValueType.h:58
llvm::MachineMemOperand::MODereferenceable
@ MODereferenceable
The memory access is dereferenceable (i.e., doesn't trap).
Definition: MachineMemOperand.h:143
llvm::MachineMemOperand::MOInvariant
@ MOInvariant
The memory access always returns the same value (or traps).
Definition: MachineMemOperand.h:145
llvm::MemSDNode::getMemOperand
MachineMemOperand * getMemOperand() const
Return a MachineMemOperand object describing the memory reference performed by operation.
Definition: SelectionDAGNodes.h:1368
llvm::SDLoc
Wrapper class for IR location info (IR ordering and DebugLoc) to be passed into SDNode creation funct...
Definition: SelectionDAGNodes.h:1115
llvm::SDNode
Represents one node in the SelectionDAG.
Definition: SelectionDAGNodes.h:472
llvm::SDNode::getNumValues
unsigned getNumValues() const
Return the number of values defined/returned by this operator.
Definition: SelectionDAGNodes.h:992
llvm::SDNode::getValueType
EVT getValueType(unsigned ResNo) const
Return the type of a specified result.
Definition: SelectionDAGNodes.h:995
llvm::SDValue
Unlike LLVM values, Selection DAG nodes may return multiple values as the result of a computation.
Definition: SelectionDAGNodes.h:145
llvm::SDValue::getNode
SDNode * getNode() const
get the SDNode which holds the desired result
Definition: SelectionDAGNodes.h:159
llvm::SDValue::getValue
SDValue getValue(unsigned R) const
Definition: SelectionDAGNodes.h:179
llvm::SDValue::getValueType
EVT getValueType() const
Return the ValueType of the referenced return value.
Definition: SelectionDAGNodes.h:1150
llvm::SelectionDAG::getExtLoad
SDValue getExtLoad(ISD::LoadExtType ExtType, const SDLoc &dl, EVT VT, SDValue Chain, SDValue Ptr, MachinePointerInfo PtrInfo, EVT MemVT, MaybeAlign Alignment=MaybeAlign(), MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes())
Definition: SelectionDAG.cpp:8578
llvm::SelectionDAG::getSelect
SDValue getSelect(const SDLoc &DL, EVT VT, SDValue Cond, SDValue LHS, SDValue RHS)
Helper function to make it easier to build Select's if you just have operands and don't want to check...
Definition: SelectionDAG.h:1235
llvm::SelectionDAG::getVTList
SDVTList getVTList(EVT VT)
Return an SDVTList that represents the list of values specified.
Definition: SelectionDAG.cpp:10065
llvm::SelectionDAG::getSetCC
SDValue getSetCC(const SDLoc &DL, EVT VT, SDValue LHS, SDValue RHS, ISD::CondCode Cond, SDValue Chain=SDValue(), bool IsSignaling=false)
Helper function to make it easier to build SetCC's if you just have an ISD::CondCode instead of an SD...
Definition: SelectionDAG.h:1206
llvm::SelectionDAG::getConstantFP
SDValue getConstantFP(double Val, const SDLoc &DL, EVT VT, bool isTarget=false)
Create a ConstantFPSDNode wrapping a constant value.
Definition: SelectionDAG.cpp:1780
llvm::SelectionDAG::getLoad
SDValue getLoad(EVT VT, const SDLoc &dl, SDValue Chain, SDValue Ptr, MachinePointerInfo PtrInfo, MaybeAlign Alignment=MaybeAlign(), MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes(), const MDNode *Ranges=nullptr)
Loads are not normal binary operators: their result type is not determined by their operands,...
Definition: SelectionDAG.cpp:8561
llvm::SelectionDAG::getAtomic
SDValue getAtomic(unsigned Opcode, const SDLoc &dl, EVT MemVT, SDValue Chain, SDValue Ptr, SDValue Val, MachineMemOperand *MMO)
Gets a node for an atomic op, produces result (if relevant) and chain and takes 2 operands.
Definition: SelectionDAG.cpp:8281
llvm::SelectionDAG::getUNDEF
SDValue getUNDEF(EVT VT)
Return an UNDEF node. UNDEF does not have a useful SDLoc.
Definition: SelectionDAG.h:1097
llvm::SelectionDAG::getBitcast
SDValue getBitcast(EVT VT, SDValue V)
Return a bitcast using the SDLoc of the value operand, and casting to the provided type.
Definition: SelectionDAG.cpp:2331
llvm::SelectionDAG::getDataLayout
const DataLayout & getDataLayout() const
Definition: SelectionDAG.h:472
llvm::SelectionDAG::getConstant
SDValue getConstant(uint64_t Val, const SDLoc &DL, EVT VT, bool isTarget=false, bool isOpaque=false)
Create a ConstantSDNode wrapping a constant value.
Definition: SelectionDAG.cpp:1601
llvm::SelectionDAG::getVAArg
SDValue getVAArg(EVT VT, const SDLoc &dl, SDValue Chain, SDValue Ptr, SDValue SV, unsigned Align)
VAArg produces a result and token chain, and takes a pointer and a source value as input.
Definition: SelectionDAG.cpp:9693
llvm::SelectionDAG::getTruncStore
SDValue getTruncStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr, MachinePointerInfo PtrInfo, EVT SVT, Align Alignment, MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes())
Definition: SelectionDAG.cpp:8663
llvm::SelectionDAG::getStore
SDValue getStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr, MachinePointerInfo PtrInfo, Align Alignment, MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes())
Helper function to build ISD::STORE nodes.
Definition: SelectionDAG.cpp:8611
llvm::SelectionDAG::EVTToAPFloatSemantics
static const fltSemantics & EVTToAPFloatSemantics(EVT VT)
Returns an APFloat semantics tag appropriate for the given type.
Definition: SelectionDAG.h:1845
llvm::SelectionDAG::getSelectCC
SDValue getSelectCC(const SDLoc &DL, SDValue LHS, SDValue RHS, SDValue True, SDValue False, ISD::CondCode Cond)
Helper function to make it easier to build SelectCC's if you just have an ISD::CondCode instead of an...
Definition: SelectionDAG.h:1245
llvm::SelectionDAG::getIntPtrConstant
SDValue getIntPtrConstant(uint64_t Val, const SDLoc &DL, bool isTarget=false)
Definition: SelectionDAG.cpp:1726
llvm::SelectionDAG::getNode
SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT, ArrayRef< SDUse > Ops)
Gets or creates the specified node.
Definition: SelectionDAG.cpp:9699
llvm::SelectionDAG::getLibInfo
const TargetLibraryInfo & getLibInfo() const
Definition: SelectionDAG.h:479
llvm::SelectionDAG::getMachineFunction
MachineFunction & getMachineFunction() const
Definition: SelectionDAG.h:469
llvm::SelectionDAG::getCondCode
SDValue getCondCode(ISD::CondCode Cond)
Definition: SelectionDAG.cpp:1981
llvm::SelectionDAG::getContext
LLVMContext * getContext() const
Definition: SelectionDAG.h:485
llvm::SelectionDAG::getShiftAmountConstant
SDValue getShiftAmountConstant(uint64_t Val, EVT VT, const SDLoc &DL, bool LegalTypes=true)
Definition: SelectionDAG.cpp:1731
llvm::SelectionDAG::CreateStackTemporary
SDValue CreateStackTemporary(TypeSize Bytes, Align Alignment)
Create a stack temporary based on the size in bytes and the alignment.
Definition: SelectionDAG.cpp:2449
llvm::SelectionDAG::UpdateNodeOperands
SDNode * UpdateNodeOperands(SDNode *N, SDValue Op)
Mutate the specified node in-place to have the specified operands.
Definition: SelectionDAG.cpp:10155
llvm::SelectionDAG::getEntryNode
SDValue getEntryNode() const
Return the token chain corresponding to the entry of the function.
Definition: SelectionDAG.h:554
llvm::SmallVector
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
Definition: SmallVector.h:1200
llvm::StoreSDNode
This class is used to represent ISD::STORE nodes.
Definition: SelectionDAGNodes.h:2392
llvm::TargetLibraryInfo::getIntSize
unsigned getIntSize() const
Get size of a C-level int or unsigned int, in bits.
Definition: TargetLibraryInfo.h:555
llvm::TargetLoweringBase::getShiftAmountTy
EVT getShiftAmountTy(EVT LHSTy, const DataLayout &DL, bool LegalTypes=true) const
Returns the type for the shift amount of a shift opcode.
Definition: TargetLoweringBase.cpp:957
llvm::TargetLoweringBase::getTypeToTransformTo
virtual EVT getTypeToTransformTo(LLVMContext &Context, EVT VT) const
For types supported by the target, this is an identity function.
Definition: TargetLowering.h:1094
llvm::TargetLoweringBase::getLibcallName
const char * getLibcallName(RTLIB::Libcall Call) const
Get the libcall routine name for the specified libcall.
Definition: TargetLowering.h:3315
llvm::TargetLowering::softenSetCCOperands
void softenSetCCOperands(SelectionDAG &DAG, EVT VT, SDValue &NewLHS, SDValue &NewRHS, ISD::CondCode &CCCode, const SDLoc &DL, const SDValue OldLHS, const SDValue OldRHS) const
Soften the operands of a comparison.
Definition: TargetLowering.cpp:290
llvm::TargetLowering::makeLibCall
std::pair< SDValue, SDValue > makeLibCall(SelectionDAG &DAG, RTLIB::Libcall LC, EVT RetVT, ArrayRef< SDValue > Ops, MakeLibCallOptions CallOptions, const SDLoc &dl, SDValue Chain=SDValue()) const
Returns a pair of (return value, chain).
Definition: TargetLowering.cpp:145
llvm::TargetLowering::expandVecReduceSeq
SDValue expandVecReduceSeq(SDNode *Node, SelectionDAG &DAG) const
Expand a VECREDUCE_SEQ_* into an explicit ordered calculation.
Definition: TargetLowering.cpp:10635
llvm::TargetLowering::expandFP_TO_INT_SAT
SDValue expandFP_TO_INT_SAT(SDNode *N, SelectionDAG &DAG) const
Expand FP_TO_[US]INT_SAT into FP_TO_[US]INT and selects or min/max.
Definition: TargetLowering.cpp:10686
llvm::TargetLowering::expandVecReduce
SDValue expandVecReduce(SDNode *Node, SelectionDAG &DAG) const
Expand a VECREDUCE_* into an explicit calculation.
Definition: TargetLowering.cpp:10595
llvm::TargetLowering::createSelectForFMINNUM_FMAXNUM
SDValue createSelectForFMINNUM_FMAXNUM(SDNode *Node, SelectionDAG &DAG) const
Try to convert the fminnum/fmaxnum to a compare/select sequence.
Definition: TargetLowering.cpp:8237
uint64_t
llvm_unreachable
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
Definition: ErrorHandling.h:143
llvm::ARM_MB::LD
@ LD
Definition: ARMBaseInfo.h:72
llvm::ARM_MB::ST
@ ST
Definition: ARMBaseInfo.h:73
llvm::BitmaskEnumDetail::Mask
constexpr std::underlying_type_t< E > Mask()
Get a bitmask with 1s in all places up to the high-order bit of E's largest value.
Definition: BitmaskEnum.h:121
llvm::CallingConv::C
@ C
The default llvm calling convention, compatible with C.
Definition: CallingConv.h:34
llvm::ISD::NodeType
NodeType
ISD::NodeType enum - This enum defines the target-independent operators for a SelectionDAG.
Definition: ISDOpcodes.h:40
llvm::ISD::SETCC
@ SETCC
SetCC operator - This evaluates to a true value iff the condition is true.
Definition: ISDOpcodes.h:750
llvm::ISD::MERGE_VALUES
@ MERGE_VALUES
MERGE_VALUES - This node takes multiple discrete operands and returns them all as its individual resu...
Definition: ISDOpcodes.h:236
llvm::ISD::STRICT_FSETCC
@ STRICT_FSETCC
STRICT_FSETCC/STRICT_FSETCCS - Constrained versions of SETCC, used for floating-point operands only.
Definition: ISDOpcodes.h:476
llvm::ISD::VECREDUCE_SEQ_FADD
@ VECREDUCE_SEQ_FADD
Generic reduction nodes.
Definition: ISDOpcodes.h:1331
llvm::ISD::ConstantFP
@ ConstantFP
Definition: ISDOpcodes.h:77
llvm::ISD::STRICT_FCEIL
@ STRICT_FCEIL
Definition: ISDOpcodes.h:426
llvm::ISD::FMAD
@ FMAD
FMAD - Perform a * b + c, while getting the same result as the separately rounded operations.
Definition: ISDOpcodes.h:487
llvm::ISD::LOAD
@ LOAD
LOAD and STORE have token chains as their first operand, then the same operands as an LLVM load/store...
Definition: ISDOpcodes.h:1029
llvm::ISD::ANY_EXTEND
@ ANY_EXTEND
ANY_EXTEND - Used for integer types. The high bits are undefined.
Definition: ISDOpcodes.h:783
llvm::ISD::FMA
@ FMA
FMA - Perform a * b + c with no intermediate rounding step.
Definition: ISDOpcodes.h:483
llvm::ISD::SINT_TO_FP
@ SINT_TO_FP
[SU]INT_TO_FP - These operators convert integers (whose interpreted sign depends on the first letter)...
Definition: ISDOpcodes.h:790
llvm::ISD::VECREDUCE_FMAX
@ VECREDUCE_FMAX
FMIN/FMAX nodes can have flags, for NaN/NoNaN variants.
Definition: ISDOpcodes.h:1347
llvm::ISD::FADD
@ FADD
Simple binary floating point operators.
Definition: ISDOpcodes.h:390
llvm::ISD::VECREDUCE_FMAXIMUM
@ VECREDUCE_FMAXIMUM
FMINIMUM/FMAXIMUM nodes propatate NaNs and signed zeroes using the llvm.minimum and llvm....
Definition: ISDOpcodes.h:1351
llvm::ISD::STRICT_FSETCCS
@ STRICT_FSETCCS
Definition: ISDOpcodes.h:477
llvm::ISD::FP16_TO_FP
@ FP16_TO_FP
FP16_TO_FP, FP_TO_FP16 - These operators are used to perform promotions and truncation for half-preci...
Definition: ISDOpcodes.h:913
llvm::ISD::STRICT_FLOG2
@ STRICT_FLOG2
Definition: ISDOpcodes.h:421
llvm::ISD::BITCAST
@ BITCAST
BITCAST - This operator converts between integer, vector and FP values, as if the value was stored to...
Definition: ISDOpcodes.h:903
llvm::ISD::BUILD_PAIR
@ BUILD_PAIR
BUILD_PAIR - This is the opposite of EXTRACT_ELEMENT in some ways.
Definition: ISDOpcodes.h:229
llvm::ISD::FLDEXP
@ FLDEXP
FLDEXP - ldexp, inspired by libm (op0 * 2**op1).
Definition: ISDOpcodes.h:937
llvm::ISD::STRICT_FSQRT
@ STRICT_FSQRT
Constrained versions of libm-equivalent floating point intrinsics.
Definition: ISDOpcodes.h:411
llvm::ISD::SIGN_EXTEND
@ SIGN_EXTEND
Conversion operators.
Definition: ISDOpcodes.h:774
llvm::ISD::STRICT_UINT_TO_FP
@ STRICT_UINT_TO_FP
Definition: ISDOpcodes.h:450
llvm::ISD::VECREDUCE_FADD
@ VECREDUCE_FADD
These reductions have relaxed evaluation order semantics, and have a single vector operand.
Definition: ISDOpcodes.h:1344
llvm::ISD::VECREDUCE_FMIN
@ VECREDUCE_FMIN
Definition: ISDOpcodes.h:1348
llvm::ISD::STRICT_LROUND
@ STRICT_LROUND
Definition: ISDOpcodes.h:431
llvm::ISD::FNEG
@ FNEG
Perform various unary floating-point operations inspired by libm.
Definition: ISDOpcodes.h:928
llvm::ISD::BR_CC
@ BR_CC
BR_CC - Conditional branch.
Definition: ISDOpcodes.h:1075
llvm::ISD::FCANONICALIZE
@ FCANONICALIZE
Returns platform specific canonical encoding of a floating point number.
Definition: ISDOpcodes.h:500
llvm::ISD::SELECT
@ SELECT
Select(COND, TRUEVAL, FALSEVAL).
Definition: ISDOpcodes.h:727
llvm::ISD::STRICT_FPOWI
@ STRICT_FPOWI
Definition: ISDOpcodes.h:413
llvm::ISD::UNDEF
@ UNDEF
UNDEF - An undefined node.
Definition: ISDOpcodes.h:211
llvm::ISD::EXTRACT_ELEMENT
@ EXTRACT_ELEMENT
EXTRACT_ELEMENT - This is used to get the lower or upper (determined by a Constant,...
Definition: ISDOpcodes.h:222
llvm::ISD::STRICT_FTRUNC
@ STRICT_FTRUNC
Definition: ISDOpcodes.h:430
llvm::ISD::ARITH_FENCE
@ ARITH_FENCE
ARITH_FENCE - This corresponds to a arithmetic fence intrinsic.
Definition: ISDOpcodes.h:1217
llvm::ISD::STRICT_FP_TO_FP16
@ STRICT_FP_TO_FP16
Definition: ISDOpcodes.h:916
llvm::ISD::STRICT_FP16_TO_FP
@ STRICT_FP16_TO_FP
Definition: ISDOpcodes.h:915
llvm::ISD::SHL
@ SHL
Shift and rotation operations.
Definition: ISDOpcodes.h:705
llvm::ISD::STRICT_FMAXNUM
@ STRICT_FMAXNUM
Definition: ISDOpcodes.h:424
llvm::ISD::EXTRACT_VECTOR_ELT
@ EXTRACT_VECTOR_ELT
EXTRACT_VECTOR_ELT(VECTOR, IDX) - Returns a single element from VECTOR identified by the (potentially...
Definition: ISDOpcodes.h:535
llvm::ISD::ZERO_EXTEND
@ ZERO_EXTEND
ZERO_EXTEND - Used for integer types, zeroing the new bits.
Definition: ISDOpcodes.h:780
llvm::ISD::FP_TO_UINT_SAT
@ FP_TO_UINT_SAT
Definition: ISDOpcodes.h:856
llvm::ISD::STRICT_FMINNUM
@ STRICT_FMINNUM
Definition: ISDOpcodes.h:425
llvm::ISD::SELECT_CC
@ SELECT_CC
Select with condition operator - This selects between a true value and a false value (ops #2 and #3) ...
Definition: ISDOpcodes.h:742
llvm::ISD::FMINNUM
@ FMINNUM
FMINNUM/FMAXNUM - Perform floating-point minimum or maximum on two values.
Definition: ISDOpcodes.h:969
llvm::ISD::STRICT_LRINT
@ STRICT_LRINT
Definition: ISDOpcodes.h:433
llvm::ISD::FP_EXTEND
@ FP_EXTEND
X = FP_EXTEND(Y) - Extend a smaller FP type into a larger FP type.
Definition: ISDOpcodes.h:888
llvm::ISD::STRICT_FROUND
@ STRICT_FROUND
Definition: ISDOpcodes.h:428
llvm::ISD::STRICT_SINT_TO_FP
@ STRICT_SINT_TO_FP
STRICT_[US]INT_TO_FP - Convert a signed or unsigned integer to a floating point value.
Definition: ISDOpcodes.h:449
llvm::ISD::STRICT_FFLOOR
@ STRICT_FFLOOR
Definition: ISDOpcodes.h:427
llvm::ISD::STRICT_FROUNDEVEN
@ STRICT_FROUNDEVEN
Definition: ISDOpcodes.h:429
llvm::ISD::BF16_TO_FP
@ BF16_TO_FP
BF16_TO_FP, FP_TO_BF16 - These operators are used to perform promotions and truncation for bfloat16.
Definition: ISDOpcodes.h:922
llvm::ISD::STRICT_FP_TO_UINT
@ STRICT_FP_TO_UINT
Definition: ISDOpcodes.h:443
llvm::ISD::STRICT_FP_ROUND
@ STRICT_FP_ROUND
X = STRICT_FP_ROUND(Y, TRUNC) - Rounding 'Y' from a larger floating point type down to the precision ...
Definition: ISDOpcodes.h:465
llvm::ISD::STRICT_FP_TO_SINT
@ STRICT_FP_TO_SINT
STRICT_FP_TO_[US]INT - Convert a floating point value to a signed or unsigned integer.
Definition: ISDOpcodes.h:442
llvm::ISD::FMINIMUM
@ FMINIMUM
FMINIMUM/FMAXIMUM - NaN-propagating minimum/maximum that also treat -0.0 as less than 0....
Definition: ISDOpcodes.h:982
llvm::ISD::FP_TO_SINT
@ FP_TO_SINT
FP_TO_[US]INT - Convert a floating point value to a signed or unsigned integer.
Definition: ISDOpcodes.h:836
llvm::ISD::STRICT_FP_EXTEND
@ STRICT_FP_EXTEND
X = STRICT_FP_EXTEND(Y) - Extend a smaller FP type into a larger FP type.
Definition: ISDOpcodes.h:470
llvm::ISD::AND
@ AND
Bitwise operators - logical and, logical or, logical xor.
Definition: ISDOpcodes.h:680
llvm::ISD::VECREDUCE_FMUL
@ VECREDUCE_FMUL
Definition: ISDOpcodes.h:1345
llvm::ISD::STRICT_FADD
@ STRICT_FADD
Constrained versions of the binary floating point operators.
Definition: ISDOpcodes.h:400
llvm::ISD::STRICT_FLOG10
@ STRICT_FLOG10
Definition: ISDOpcodes.h:420
llvm::ISD::STRICT_LLRINT
@ STRICT_LLRINT
Definition: ISDOpcodes.h:434
llvm::ISD::STRICT_FEXP2
@ STRICT_FEXP2
Definition: ISDOpcodes.h:418
llvm::ISD::ATOMIC_SWAP
@ ATOMIC_SWAP
Val, OUTCHAIN = ATOMIC_SWAP(INCHAIN, ptr, amt) Val, OUTCHAIN = ATOMIC_LOAD_[OpName](INCHAIN,...
Definition: ISDOpcodes.h:1254
llvm::ISD::FFREXP
@ FFREXP
FFREXP - frexp, extract fractional and exponent component of a floating-point value.
Definition: ISDOpcodes.h:942
llvm::ISD::FP_ROUND
@ FP_ROUND
X = FP_ROUND(Y, TRUNC) - Rounding 'Y' from a larger floating point type down to the precision of the ...
Definition: ISDOpcodes.h:869
llvm::ISD::STRICT_FLDEXP
@ STRICT_FLDEXP
Definition: ISDOpcodes.h:414
llvm::ISD::STRICT_LLROUND
@ STRICT_LLROUND
Definition: ISDOpcodes.h:432
llvm::ISD::STRICT_FNEARBYINT
@ STRICT_FNEARBYINT
Definition: ISDOpcodes.h:423
llvm::ISD::FP_TO_SINT_SAT
@ FP_TO_SINT_SAT
FP_TO_[US]INT_SAT - Convert floating point value in operand 0 to a signed or unsigned scalar integer ...
Definition: ISDOpcodes.h:855
llvm::ISD::VECREDUCE_FMINIMUM
@ VECREDUCE_FMINIMUM
Definition: ISDOpcodes.h:1352
llvm::ISD::TRUNCATE
@ TRUNCATE
TRUNCATE - Completely drop the high bits.
Definition: ISDOpcodes.h:786
llvm::ISD::VAARG
@ VAARG
VAARG - VAARG has four operands: an input chain, a pointer, a SRCVALUE, and the alignment.
Definition: ISDOpcodes.h:1144
llvm::ISD::VECREDUCE_SEQ_FMUL
@ VECREDUCE_SEQ_FMUL
Definition: ISDOpcodes.h:1332
llvm::ISD::FCOPYSIGN
@ FCOPYSIGN
FCOPYSIGN(X, Y) - Return the value of X with the sign of Y.
Definition: ISDOpcodes.h:493
llvm::ISD::STRICT_FRINT
@ STRICT_FRINT
Definition: ISDOpcodes.h:422
llvm::ISD::BUILD_VECTOR
@ BUILD_VECTOR
BUILD_VECTOR(ELT0, ELT1, ELT2, ELT3,...) - Return a fixed-width vector with the specified,...
Definition: ISDOpcodes.h:515
llvm::ISD::isNormalStore
bool isNormalStore(const SDNode *N)
Returns true if the specified node is a non-truncating and unindexed store.
Definition: SelectionDAGNodes.h:3120
llvm::ISD::isUNINDEXEDLoad
bool isUNINDEXEDLoad(const SDNode *N)
Returns true if the specified node is an unindexed load.
Definition: SelectionDAGNodes.h:3113
llvm::ISD::isUNINDEXEDStore
bool isUNINDEXEDStore(const SDNode *N)
Returns true if the specified node is an unindexed store.
Definition: SelectionDAGNodes.h:3127
llvm::ISD::CondCode
CondCode
ISD::CondCode enum - These are ordered carefully to make the bitfields below work out,...
Definition: ISDOpcodes.h:1506
llvm::ISD::isNormalLoad
bool isNormalLoad(const SDNode *N)
Returns true if the specified node is a non-extending and unindexed load.
Definition: SelectionDAGNodes.h:3082
llvm::RTLIB::getPOWI
Libcall getPOWI(EVT RetVT)
getPOWI - Return the POWI_* value for the given types, or UNKNOWN_LIBCALL if there is none.
Definition: TargetLoweringBase.cpp:508
llvm::RTLIB::getSINTTOFP
Libcall getSINTTOFP(EVT OpVT, EVT RetVT)
getSINTTOFP - Return the SINTTOFP_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
Definition: TargetLoweringBase.cpp:418
llvm::RTLIB::getLDEXP
Libcall getLDEXP(EVT RetVT)
getLDEXP - Return the LDEXP_* value for the given types, or UNKNOWN_LIBCALL if there is none.
Definition: TargetLoweringBase.cpp:513
llvm::RTLIB::getUINTTOFP
Libcall getUINTTOFP(EVT OpVT, EVT RetVT)
getUINTTOFP - Return the UINTTOFP_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
Definition: TargetLoweringBase.cpp:464
llvm::RTLIB::getFREXP
Libcall getFREXP(EVT RetVT)
getFREXP - Return the FREXP_* value for the given types, or UNKNOWN_LIBCALL if there is none.
Definition: TargetLoweringBase.cpp:518
llvm::RTLIB::Libcall
Libcall
RTLIB::Libcall enum - This enum defines all of the runtime library calls the backend can emit.
Definition: RuntimeLibcalls.h:30
llvm::RTLIB::getFPTOUINT
Libcall getFPTOUINT(EVT OpVT, EVT RetVT)
getFPTOUINT - Return the FPTOUINT_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
Definition: TargetLoweringBase.cpp:369
llvm::RTLIB::getFPTOSINT
Libcall getFPTOSINT(EVT OpVT, EVT RetVT)
getFPTOSINT - Return the FPTOSINT_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
Definition: TargetLoweringBase.cpp:320
llvm::RTLIB::getFPEXT
Libcall getFPEXT(EVT OpVT, EVT RetVT)
getFPEXT - Return the FPEXT_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
Definition: TargetLoweringBase.cpp:245
llvm::RTLIB::getFPROUND
Libcall getFPROUND(EVT OpVT, EVT RetVT)
getFPROUND - Return the FPROUND_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
Definition: TargetLoweringBase.cpp:277
llvm::ms_demangle::QualifierMangleMode::Result
@ Result
llvm::ore::NV
DiagnosticInfoOptimizationBase::Argument NV
Definition: OptimizationRemarkEmitter.h:136
llvm::tgtok::FalseVal
@ FalseVal
Definition: TGLexer.h:59
llvm
This is an optimization pass for GlobalISel generic memory operations.
Definition: AddressRanges.h:18
llvm::Offset
@ Offset
Definition: DWP.cpp:440
llvm::dbgs
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
Definition: Debug.cpp:163
llvm::report_fatal_error
void report_fatal_error(Error Err, bool gen_crash_diag=true)
Report a serious error, calling any installed error handler.
Definition: Error.cpp:156
llvm::Op
DWARFExpression::Operation Op
Definition: DWARFExpression.cpp:22
raw_ostream.h
N
#define N
llvm::APFloatBase::PPCDoubleDouble
static const fltSemantics & PPCDoubleDouble() LLVM_READNONE
Definition: APFloat.cpp:252
llvm::EVT
Extended Value Type.
Definition: ValueTypes.h:34
llvm::EVT::isSimple
bool isSimple() const
Test if the given EVT is simple (as opposed to being extended).
Definition: ValueTypes.h:129
llvm::EVT::getSizeInBits
TypeSize getSizeInBits() const
Return the size of the specified value type in bits.
Definition: ValueTypes.h:351
llvm::EVT::isByteSized
bool isByteSized() const
Return true if the bit size is a multiple of 8.
Definition: ValueTypes.h:226
llvm::EVT::getSimpleVT
MVT getSimpleVT() const
Return the SimpleValueType held in the specified simple EVT.
Definition: ValueTypes.h:299
llvm::EVT::getIntegerVT
static EVT getIntegerVT(LLVMContext &Context, unsigned BitWidth)
Returns the EVT that represents an integer with the given number of bits.
Definition: ValueTypes.h:64
llvm::EVT::bitsGE
bool bitsGE(EVT VT) const
Return true if this has no less bits than VT.
Definition: ValueTypes.h:275
llvm::EVT::getVectorElementType
EVT getVectorElementType() const
Given a vector type, return the type of each element.
Definition: ValueTypes.h:311
llvm::EVT::bitsLE
bool bitsLE(EVT VT) const
Return true if this has no more bits than VT.
Definition: ValueTypes.h:291
llvm::MachinePointerInfo::getFixedStack
static MachinePointerInfo getFixedStack(MachineFunction &MF, int FI, int64_t Offset=0)
Return a MachinePointerInfo record that refers to the specified FrameIndex.
Definition: MachineOperand.cpp:1048
llvm::SDNodeFlags
These are IR-level optimization flags that may be propagated to SDNodes.
Definition: SelectionDAGNodes.h:379
llvm::TargetLowering::MakeLibCallOptions
This structure is used to pass arguments to makeLibCall function.
Definition: TargetLowering.h:4539
llvm::TargetLowering::MakeLibCallOptions::setSExt
MakeLibCallOptions & setSExt(bool Value=true)
Definition: TargetLowering.h:4554
llvm::TargetLowering::MakeLibCallOptions::setTypeListBeforeSoften
MakeLibCallOptions & setTypeListBeforeSoften(ArrayRef< EVT > OpsVT, EVT RetVT, bool Value=true)
Definition: TargetLowering.h:4574
Generated on Tue Nov 28 2023 18:37:12 for LLVM by doxygen 1.9.6