LLVM 17.0.0git
LegalizeFloatTypes.cpp
Go to the documentation of this file.
1//===-------- LegalizeFloatTypes.cpp - Legalization of float types --------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements float type expansion and softening for LegalizeTypes.
10// Softening is the act of turning a computation in an illegal floating point
11// type into a computation in an integer type of the same size; also known as
12// "soft float". For example, turning f32 arithmetic into operations using i32.
13// The resulting integer value is the same as what you would get by performing
14// the floating point operation and bitcasting the result to the integer type.
15// Expansion is the act of changing a computation in an illegal type to be a
16// computation in two identical registers of a smaller type. For example,
17// implementing ppcf128 arithmetic in two f64 registers.
18//
19//===----------------------------------------------------------------------===//
20
21#include "LegalizeTypes.h"
22#include "llvm/Analysis/TargetLibraryInfo.h"
23#include "llvm/Support/ErrorHandling.h"
24#include "llvm/Support/raw_ostream.h"
25using namespace llvm;
26
27#define DEBUG_TYPE "legalize-types"
28
29/// GetFPLibCall - Return the right libcall for the given floating point type.
30/// FIXME: This is a local version of RTLIB::getFPLibCall that should be
31/// refactored away (see RTLIB::getPOWI for an example).
32static RTLIB::Libcall GetFPLibCall(EVT VT,
33 RTLIB::Libcall Call_F32,
34 RTLIB::Libcall Call_F64,
35 RTLIB::Libcall Call_F80,
36 RTLIB::Libcall Call_F128,
37 RTLIB::Libcall Call_PPCF128) {
38 return
39 VT == MVT::f32 ? Call_F32 :
40 VT == MVT::f64 ? Call_F64 :
41 VT == MVT::f80 ? Call_F80 :
42 VT == MVT::f128 ? Call_F128 :
43 VT == MVT::ppcf128 ? Call_PPCF128 :
44 RTLIB::UNKNOWN_LIBCALL;
45}
46
47//===----------------------------------------------------------------------===//
48// Convert Float Results to Integer
49//===----------------------------------------------------------------------===//
50
51void DAGTypeLegalizer::SoftenFloatResult(SDNode *N, unsigned ResNo) {
52 LLVM_DEBUG(dbgs() << "Soften float result " << ResNo << ": "; N->dump(&DAG);
53 dbgs() << "\n");
54 SDValue R = SDValue();
55
56 switch (N->getOpcode()) {
57 default:
58#ifndef NDEBUG
59 dbgs() << "SoftenFloatResult #" << ResNo << ": ";
60 N->dump(&DAG); dbgs() << "\n";
61#endif
62 llvm_unreachable("Do not know how to soften the result of this operator!");
63
64 case ISD::ARITH_FENCE: R = SoftenFloatRes_ARITH_FENCE(N); break;
65 case ISD::MERGE_VALUES:R = SoftenFloatRes_MERGE_VALUES(N, ResNo); break;
66 case ISD::BITCAST: R = SoftenFloatRes_BITCAST(N); break;
67 case ISD::BUILD_PAIR: R = SoftenFloatRes_BUILD_PAIR(N); break;
68 case ISD::ConstantFP: R = SoftenFloatRes_ConstantFP(N); break;
69 case ISD::EXTRACT_VECTOR_ELT:
70 R = SoftenFloatRes_EXTRACT_VECTOR_ELT(N, ResNo); break;
71 case ISD::FABS: R = SoftenFloatRes_FABS(N); break;
72 case ISD::STRICT_FMINNUM:
73 case ISD::FMINNUM: R = SoftenFloatRes_FMINNUM(N); break;
74 case ISD::STRICT_FMAXNUM:
75 case ISD::FMAXNUM: R = SoftenFloatRes_FMAXNUM(N); break;
76 case ISD::STRICT_FADD:
77 case ISD::FADD: R = SoftenFloatRes_FADD(N); break;
78 case ISD::FCBRT: R = SoftenFloatRes_FCBRT(N); break;
79 case ISD::STRICT_FCEIL:
80 case ISD::FCEIL: R = SoftenFloatRes_FCEIL(N); break;
81 case ISD::FCOPYSIGN: R = SoftenFloatRes_FCOPYSIGN(N); break;
82 case ISD::STRICT_FCOS:
83 case ISD::FCOS: R = SoftenFloatRes_FCOS(N); break;
84 case ISD::STRICT_FDIV:
85 case ISD::FDIV: R = SoftenFloatRes_FDIV(N); break;
86 case ISD::STRICT_FEXP:
87 case ISD::FEXP: R = SoftenFloatRes_FEXP(N); break;
88 case ISD::STRICT_FEXP2:
89 case ISD::FEXP2: R = SoftenFloatRes_FEXP2(N); break;
90 case ISD::STRICT_FFLOOR:
91 case ISD::FFLOOR: R = SoftenFloatRes_FFLOOR(N); break;
92 case ISD::STRICT_FLOG:
93 case ISD::FLOG: R = SoftenFloatRes_FLOG(N); break;
94 case ISD::STRICT_FLOG2:
95 case ISD::FLOG2: R = SoftenFloatRes_FLOG2(N); break;
96 case ISD::STRICT_FLOG10:
97 case ISD::FLOG10: R = SoftenFloatRes_FLOG10(N); break;
98 case ISD::STRICT_FMA:
99 case ISD::FMA: R = SoftenFloatRes_FMA(N); break;
100 case ISD::STRICT_FMUL:
101 case ISD::FMUL: R = SoftenFloatRes_FMUL(N); break;
102 case ISD::STRICT_FNEARBYINT:
103 case ISD::FNEARBYINT: R = SoftenFloatRes_FNEARBYINT(N); break;
104 case ISD::FNEG: R = SoftenFloatRes_FNEG(N); break;
105 case ISD::STRICT_FP_EXTEND:
106 case ISD::FP_EXTEND: R = SoftenFloatRes_FP_EXTEND(N); break;
107 case ISD::STRICT_FP_ROUND:
108 case ISD::FP_ROUND: R = SoftenFloatRes_FP_ROUND(N); break;
109 case ISD::FP16_TO_FP: R = SoftenFloatRes_FP16_TO_FP(N); break;
110 case ISD::BF16_TO_FP: R = SoftenFloatRes_BF16_TO_FP(N); break;
111 case ISD::STRICT_FPOW:
112 case ISD::FPOW: R = SoftenFloatRes_FPOW(N); break;
113 case ISD::STRICT_FPOWI:
114 case ISD::FPOWI: R = SoftenFloatRes_FPOWI(N); break;
115 case ISD::STRICT_FREM:
116 case ISD::FREM: R = SoftenFloatRes_FREM(N); break;
117 case ISD::STRICT_FRINT:
118 case ISD::FRINT: R = SoftenFloatRes_FRINT(N); break;
119 case ISD::STRICT_FROUND:
120 case ISD::FROUND: R = SoftenFloatRes_FROUND(N); break;
121 case ISD::STRICT_FROUNDEVEN:
122 case ISD::FROUNDEVEN: R = SoftenFloatRes_FROUNDEVEN(N); break;
123 case ISD::STRICT_FSIN:
124 case ISD::FSIN: R = SoftenFloatRes_FSIN(N); break;
125 case ISD::STRICT_FSQRT:
126 case ISD::FSQRT: R = SoftenFloatRes_FSQRT(N); break;
127 case ISD::STRICT_FSUB:
128 case ISD::FSUB: R = SoftenFloatRes_FSUB(N); break;
129 case ISD::STRICT_FTRUNC:
130 case ISD::FTRUNC: R = SoftenFloatRes_FTRUNC(N); break;
131 case ISD::LOAD: R = SoftenFloatRes_LOAD(N); break;
132 case ISD::ATOMIC_SWAP: R = BitcastToInt_ATOMIC_SWAP(N); break;
133 case ISD::SELECT: R = SoftenFloatRes_SELECT(N); break;
134 case ISD::SELECT_CC: R = SoftenFloatRes_SELECT_CC(N); break;
135 case ISD::FREEZE: R = SoftenFloatRes_FREEZE(N); break;
136 case ISD::STRICT_SINT_TO_FP:
137 case ISD::STRICT_UINT_TO_FP:
138 case ISD::SINT_TO_FP:
139 case ISD::UINT_TO_FP: R = SoftenFloatRes_XINT_TO_FP(N); break;
140 case ISD::UNDEF: R = SoftenFloatRes_UNDEF(N); break;
141 case ISD::VAARG: R = SoftenFloatRes_VAARG(N); break;
142 case ISD::VECREDUCE_FADD:
143 case ISD::VECREDUCE_FMUL:
144 case ISD::VECREDUCE_FMIN:
145 case ISD::VECREDUCE_FMAX:
146 R = SoftenFloatRes_VECREDUCE(N);
147 break;
148 case ISD::VECREDUCE_SEQ_FADD:
149 case ISD::VECREDUCE_SEQ_FMUL:
150 R = SoftenFloatRes_VECREDUCE_SEQ(N);
151 break;
152 }
153
154 // If R is null, the sub-method took care of registering the result.
155 if (R.getNode()) {
156 assert(R.getNode() != N);
157 SetSoftenedFloat(SDValue(N, ResNo), R);
158 }
159}
160
161SDValue DAGTypeLegalizer::SoftenFloatRes_Unary(SDNode *N, RTLIB::Libcall LC) {
162 bool IsStrict = N->isStrictFPOpcode();
163 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
164 unsigned Offset = IsStrict ? 1 : 0;
165 assert(N->getNumOperands() == (1 + Offset) &&
166 "Unexpected number of operands!");
167 SDValue Op = GetSoftenedFloat(N->getOperand(0 + Offset));
168 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
169 TargetLowering::MakeLibCallOptions CallOptions;
170 EVT OpVT = N->getOperand(0 + Offset).getValueType();
171 CallOptions.setTypeListBeforeSoften(OpVT, N->getValueType(0), true);
172 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, NVT, Op,
173 CallOptions, SDLoc(N),
174 Chain);
175 if (IsStrict)
176 ReplaceValueWith(SDValue(N, 1), Tmp.second);
177 return Tmp.first;
178}
179
180SDValue DAGTypeLegalizer::SoftenFloatRes_Binary(SDNode *N, RTLIB::Libcall LC) {
181 bool IsStrict = N->isStrictFPOpcode();
182 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
183 unsigned Offset = IsStrict ? 1 : 0;
184 assert(N->getNumOperands() == (2 + Offset) &&
185 "Unexpected number of operands!");
186 SDValue Ops[2] = { GetSoftenedFloat(N->getOperand(0 + Offset)),
187 GetSoftenedFloat(N->getOperand(1 + Offset)) };
188 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
189 TargetLowering::MakeLibCallOptions CallOptions;
190 EVT OpsVT[2] = { N->getOperand(0 + Offset).getValueType(),
191 N->getOperand(1 + Offset).getValueType() };
192 CallOptions.setTypeListBeforeSoften(OpsVT, N->getValueType(0), true);
193 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, NVT, Ops,
194 CallOptions, SDLoc(N),
195 Chain);
196 if (IsStrict)
197 ReplaceValueWith(SDValue(N, 1), Tmp.second);
198 return Tmp.first;
199}
200
201SDValue DAGTypeLegalizer::SoftenFloatRes_BITCAST(SDNode *N) {
202 return BitConvertToInteger(N->getOperand(0));
203}
204
205SDValue DAGTypeLegalizer::SoftenFloatRes_FREEZE(SDNode *N) {
206 EVT Ty = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
207 return DAG.getNode(ISD::FREEZE, SDLoc(N), Ty,
208 GetSoftenedFloat(N->getOperand(0)));
209}
210
211SDValue DAGTypeLegalizer::SoftenFloatRes_ARITH_FENCE(SDNode *N) {
212 EVT Ty = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
213 SDValue NewFence = DAG.getNode(ISD::ARITH_FENCE, SDLoc(N), Ty,
214 GetSoftenedFloat(N->getOperand(0)));
215 return NewFence;
216}
217
218SDValue DAGTypeLegalizer::SoftenFloatRes_MERGE_VALUES(SDNode *N,
219 unsigned ResNo) {
220 SDValue Op = DisintegrateMERGE_VALUES(N, ResNo);
221 return BitConvertToInteger(Op);
222}
223
224SDValue DAGTypeLegalizer::SoftenFloatRes_BUILD_PAIR(SDNode *N) {
225 // Convert the inputs to integers, and build a new pair out of them.
226 return DAG.getNode(ISD::BUILD_PAIR, SDLoc(N),
227 TLI.getTypeToTransformTo(*DAG.getContext(),
228 N->getValueType(0)),
229 BitConvertToInteger(N->getOperand(0)),
230 BitConvertToInteger(N->getOperand(1)));
231}
232
233SDValue DAGTypeLegalizer::SoftenFloatRes_ConstantFP(SDNode *N) {
234 ConstantFPSDNode *CN = cast<ConstantFPSDNode>(N);
235 // In ppcf128, the high 64 bits are always first in memory regardless
236 // of Endianness. LLVM's APFloat representation is not Endian sensitive,
237 // and so always converts into a 128-bit APInt in a non-Endian-sensitive
238 // way. However, APInt's are serialized in an Endian-sensitive fashion,
239 // so on big-Endian targets, the two doubles are output in the wrong
240 // order. Fix this by manually flipping the order of the high 64 bits
241 // and the low 64 bits here.
242 if (DAG.getDataLayout().isBigEndian() &&
243 CN->getValueType(0).getSimpleVT() == llvm::MVT::ppcf128) {
244 uint64_t words[2] = { CN->getValueAPF().bitcastToAPInt().getRawData()[1],
245 CN->getValueAPF().bitcastToAPInt().getRawData()[0] };
246 APInt Val(128, words);
247 return DAG.getConstant(Val, SDLoc(CN),
248 TLI.getTypeToTransformTo(*DAG.getContext(),
249 CN->getValueType(0)));
250 } else {
251 return DAG.getConstant(CN->getValueAPF().bitcastToAPInt(), SDLoc(CN),
252 TLI.getTypeToTransformTo(*DAG.getContext(),
253 CN->getValueType(0)));
254 }
255}
256
257SDValue DAGTypeLegalizer::SoftenFloatRes_EXTRACT_VECTOR_ELT(SDNode *N, unsigned ResNo) {
258 SDValue NewOp = BitConvertVectorToIntegerVector(N->getOperand(0));
259 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SDLoc(N),
260 NewOp.getValueType().getVectorElementType(),
261 NewOp, N->getOperand(1));
262}
263
264SDValue DAGTypeLegalizer::SoftenFloatRes_FABS(SDNode *N) {
265 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
266 unsigned Size = NVT.getSizeInBits();
267
268 // Mask = ~(1 << (Size-1))
269 APInt API = APInt::getAllOnes(Size);
270 API.clearBit(Size - 1);
271 SDValue Mask = DAG.getConstant(API, SDLoc(N), NVT);
272 SDValue Op = GetSoftenedFloat(N->getOperand(0));
273 return DAG.getNode(ISD::AND, SDLoc(N), NVT, Op, Mask);
274}
275
276SDValue DAGTypeLegalizer::SoftenFloatRes_FMINNUM(SDNode *N) {
277 if (SDValue SelCC = TLI.createSelectForFMINNUM_FMAXNUM(N, DAG))
278 return SoftenFloatRes_SELECT_CC(SelCC.getNode());
279 return SoftenFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
280 RTLIB::FMIN_F32,
281 RTLIB::FMIN_F64,
282 RTLIB::FMIN_F80,
283 RTLIB::FMIN_F128,
284 RTLIB::FMIN_PPCF128));
285}
286
287SDValue DAGTypeLegalizer::SoftenFloatRes_FMAXNUM(SDNode *N) {
288 if (SDValue SelCC = TLI.createSelectForFMINNUM_FMAXNUM(N, DAG))
289 return SoftenFloatRes_SELECT_CC(SelCC.getNode());
290 return SoftenFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
291 RTLIB::FMAX_F32,
292 RTLIB::FMAX_F64,
293 RTLIB::FMAX_F80,
294 RTLIB::FMAX_F128,
295 RTLIB::FMAX_PPCF128));
296}
297
298SDValue DAGTypeLegalizer::SoftenFloatRes_FADD(SDNode *N) {
299 return SoftenFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
300 RTLIB::ADD_F32,
301 RTLIB::ADD_F64,
302 RTLIB::ADD_F80,
303 RTLIB::ADD_F128,
304 RTLIB::ADD_PPCF128));
305}
306
307SDValue DAGTypeLegalizer::SoftenFloatRes_FCBRT(SDNode *N) {
308 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
309 RTLIB::CBRT_F32,
310 RTLIB::CBRT_F64,
311 RTLIB::CBRT_F80,
312 RTLIB::CBRT_F128,
313 RTLIB::CBRT_PPCF128));
314}
315
316SDValue DAGTypeLegalizer::SoftenFloatRes_FCEIL(SDNode *N) {
317 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
318 RTLIB::CEIL_F32,
319 RTLIB::CEIL_F64,
320 RTLIB::CEIL_F80,
321 RTLIB::CEIL_F128,
322 RTLIB::CEIL_PPCF128));
323}
324
325SDValue DAGTypeLegalizer::SoftenFloatRes_FCOPYSIGN(SDNode *N) {
326 SDValue LHS = GetSoftenedFloat(N->getOperand(0));
327 SDValue RHS = BitConvertToInteger(N->getOperand(1));
328 SDLoc dl(N);
329
330 EVT LVT = LHS.getValueType();
331 EVT RVT = RHS.getValueType();
332
333 unsigned LSize = LVT.getSizeInBits();
334 unsigned RSize = RVT.getSizeInBits();
335
336 // First get the sign bit of second operand.
337 SDValue SignBit = DAG.getNode(
338 ISD::SHL, dl, RVT, DAG.getConstant(1, dl, RVT),
339 DAG.getConstant(RSize - 1, dl,
340 TLI.getShiftAmountTy(RVT, DAG.getDataLayout())));
341 SignBit = DAG.getNode(ISD::AND, dl, RVT, RHS, SignBit);
342
343 // Shift right or sign-extend it if the two operands have different types.
344 int SizeDiff = RVT.getSizeInBits() - LVT.getSizeInBits();
345 if (SizeDiff > 0) {
346 SignBit =
347 DAG.getNode(ISD::SRL, dl, RVT, SignBit,
348 DAG.getConstant(SizeDiff, dl,
349 TLI.getShiftAmountTy(SignBit.getValueType(),
350 DAG.getDataLayout())));
351 SignBit = DAG.getNode(ISD::TRUNCATE, dl, LVT, SignBit);
352 } else if (SizeDiff < 0) {
353 SignBit = DAG.getNode(ISD::ANY_EXTEND, dl, LVT, SignBit);
354 SignBit =
355 DAG.getNode(ISD::SHL, dl, LVT, SignBit,
356 DAG.getConstant(-SizeDiff, dl,
357 TLI.getShiftAmountTy(SignBit.getValueType(),
358 DAG.getDataLayout())));
359 }
360
361 // Clear the sign bit of the first operand.
362 SDValue Mask = DAG.getNode(
363 ISD::SHL, dl, LVT, DAG.getConstant(1, dl, LVT),
364 DAG.getConstant(LSize - 1, dl,
365 TLI.getShiftAmountTy(LVT, DAG.getDataLayout())));
366 Mask = DAG.getNode(ISD::SUB, dl, LVT, Mask, DAG.getConstant(1, dl, LVT));
367 LHS = DAG.getNode(ISD::AND, dl, LVT, LHS, Mask);
368
369 // Or the value with the sign bit.
370 return DAG.getNode(ISD::OR, dl, LVT, LHS, SignBit);
371}
372
373SDValue DAGTypeLegalizer::SoftenFloatRes_FCOS(SDNode *N) {
374 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
375 RTLIB::COS_F32,
376 RTLIB::COS_F64,
377 RTLIB::COS_F80,
378 RTLIB::COS_F128,
379 RTLIB::COS_PPCF128));
380}
381
382SDValue DAGTypeLegalizer::SoftenFloatRes_FDIV(SDNode *N) {
383 return SoftenFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
384 RTLIB::DIV_F32,
385 RTLIB::DIV_F64,
386 RTLIB::DIV_F80,
387 RTLIB::DIV_F128,
388 RTLIB::DIV_PPCF128));
389}
390
391SDValue DAGTypeLegalizer::SoftenFloatRes_FEXP(SDNode *N) {
392 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
393 RTLIB::EXP_F32,
394 RTLIB::EXP_F64,
395 RTLIB::EXP_F80,
396 RTLIB::EXP_F128,
397 RTLIB::EXP_PPCF128));
398}
399
400SDValue DAGTypeLegalizer::SoftenFloatRes_FEXP2(SDNode *N) {
401 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
402 RTLIB::EXP2_F32,
403 RTLIB::EXP2_F64,
404 RTLIB::EXP2_F80,
405 RTLIB::EXP2_F128,
406 RTLIB::EXP2_PPCF128));
407}
408
409SDValue DAGTypeLegalizer::SoftenFloatRes_FFLOOR(SDNode *N) {
410 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
411 RTLIB::FLOOR_F32,
412 RTLIB::FLOOR_F64,
413 RTLIB::FLOOR_F80,
414 RTLIB::FLOOR_F128,
415 RTLIB::FLOOR_PPCF128));
416}
417
418SDValue DAGTypeLegalizer::SoftenFloatRes_FLOG(SDNode *N) {
419 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
420 RTLIB::LOG_F32,
421 RTLIB::LOG_F64,
422 RTLIB::LOG_F80,
423 RTLIB::LOG_F128,
424 RTLIB::LOG_PPCF128));
425}
426
427SDValue DAGTypeLegalizer::SoftenFloatRes_FLOG2(SDNode *N) {
428 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
429 RTLIB::LOG2_F32,
430 RTLIB::LOG2_F64,
431 RTLIB::LOG2_F80,
432 RTLIB::LOG2_F128,
433 RTLIB::LOG2_PPCF128));
434}
435
436SDValue DAGTypeLegalizer::SoftenFloatRes_FLOG10(SDNode *N) {
437 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
438 RTLIB::LOG10_F32,
439 RTLIB::LOG10_F64,
440 RTLIB::LOG10_F80,
441 RTLIB::LOG10_F128,
442 RTLIB::LOG10_PPCF128));
443}
444
445SDValue DAGTypeLegalizer::SoftenFloatRes_FMA(SDNode *N) {
446 bool IsStrict = N->isStrictFPOpcode();
447 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
448 unsigned Offset = IsStrict ? 1 : 0;
449 SDValue Ops[3] = { GetSoftenedFloat(N->getOperand(0 + Offset)),
450 GetSoftenedFloat(N->getOperand(1 + Offset)),
451 GetSoftenedFloat(N->getOperand(2 + Offset)) };
452 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
453 TargetLowering::MakeLibCallOptions CallOptions;
454 EVT OpsVT[3] = { N->getOperand(0 + Offset).getValueType(),
455 N->getOperand(1 + Offset).getValueType(),
456 N->getOperand(2 + Offset).getValueType() };
457 CallOptions.setTypeListBeforeSoften(OpsVT, N->getValueType(0), true);
458 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG,
459 GetFPLibCall(N->getValueType(0),
460 RTLIB::FMA_F32,
461 RTLIB::FMA_F64,
462 RTLIB::FMA_F80,
463 RTLIB::FMA_F128,
464 RTLIB::FMA_PPCF128),
465 NVT, Ops, CallOptions, SDLoc(N), Chain);
466 if (IsStrict)
467 ReplaceValueWith(SDValue(N, 1), Tmp.second);
468 return Tmp.first;
469}
470
471SDValue DAGTypeLegalizer::SoftenFloatRes_FMUL(SDNode *N) {
472 return SoftenFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
473 RTLIB::MUL_F32,
474 RTLIB::MUL_F64,
475 RTLIB::MUL_F80,
476 RTLIB::MUL_F128,
477 RTLIB::MUL_PPCF128));
478}
479
480SDValue DAGTypeLegalizer::SoftenFloatRes_FNEARBYINT(SDNode *N) {
481 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
482 RTLIB::NEARBYINT_F32,
483 RTLIB::NEARBYINT_F64,
484 RTLIB::NEARBYINT_F80,
485 RTLIB::NEARBYINT_F128,
486 RTLIB::NEARBYINT_PPCF128));
487}
488
489SDValue DAGTypeLegalizer::SoftenFloatRes_FNEG(SDNode *N) {
490 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
491 SDLoc dl(N);
492
493 // Expand Y = FNEG(X) -> Y = X ^ sign mask
494 APInt SignMask = APInt::getSignMask(NVT.getSizeInBits());
495 return DAG.getNode(ISD::XOR, dl, NVT, GetSoftenedFloat(N->getOperand(0)),
496 DAG.getConstant(SignMask, dl, NVT));
497}
498
499SDValue DAGTypeLegalizer::SoftenFloatRes_FP_EXTEND(SDNode *N) {
500 bool IsStrict = N->isStrictFPOpcode();
501 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
502 SDValue Op = N->getOperand(IsStrict ? 1 : 0);
503
504 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
505
506 if (getTypeAction(Op.getValueType()) == TargetLowering::TypePromoteFloat) {
507 Op = GetPromotedFloat(Op);
508 // If the promotion did the FP_EXTEND to the destination type for us,
509 // there's nothing left to do here.
510 if (Op.getValueType() == N->getValueType(0))
511 return BitConvertToInteger(Op);
512 }
513
514 // There's only a libcall for f16 -> f32 and shifting is only valid for bf16
515 // -> f32, so proceed in two stages. Also, it's entirely possible for both
516 // f16 and f32 to be legal, so use the fully hard-float FP_EXTEND rather
517 // than FP16_TO_FP.
518 if ((Op.getValueType() == MVT::f16 || Op.getValueType() == MVT::bf16) &&
519 N->getValueType(0) != MVT::f32) {
520 if (IsStrict) {
521 Op = DAG.getNode(ISD::STRICT_FP_EXTEND, SDLoc(N),
522 { MVT::f32, MVT::Other }, { Chain, Op });
523 Chain = Op.getValue(1);
524 } else {
525 Op = DAG.getNode(ISD::FP_EXTEND, SDLoc(N), MVT::f32, Op);
526 }
527 }
528
529 if (Op.getValueType() == MVT::bf16)
530 return SoftenFloatRes_BF16_TO_FP(N);
531
532 RTLIB::Libcall LC = RTLIB::getFPEXT(Op.getValueType(), N->getValueType(0));
533 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported FP_EXTEND!");
534 TargetLowering::MakeLibCallOptions CallOptions;
535 EVT OpVT = N->getOperand(IsStrict ? 1 : 0).getValueType();
536 CallOptions.setTypeListBeforeSoften(OpVT, N->getValueType(0), true);
537 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, NVT, Op,
538 CallOptions, SDLoc(N),
539 Chain);
540 if (IsStrict)
541 ReplaceValueWith(SDValue(N, 1), Tmp.second);
542 return Tmp.first;
543}
544
545// FIXME: Should we just use 'normal' FP_EXTEND / FP_TRUNC instead of special
546// nodes?
547SDValue DAGTypeLegalizer::SoftenFloatRes_FP16_TO_FP(SDNode *N) {
548 EVT MidVT = TLI.getTypeToTransformTo(*DAG.getContext(), MVT::f32);
549 SDValue Op = N->getOperand(0);
550 TargetLowering::MakeLibCallOptions CallOptions;
551 EVT OpsVT[1] = { N->getOperand(0).getValueType() };
552 CallOptions.setTypeListBeforeSoften(OpsVT, N->getValueType(0), true);
553 SDValue Res32 = TLI.makeLibCall(DAG, RTLIB::FPEXT_F16_F32, MidVT, Op,
554 CallOptions, SDLoc(N)).first;
555 if (N->getValueType(0) == MVT::f32)
556 return Res32;
557
558 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
559 RTLIB::Libcall LC = RTLIB::getFPEXT(MVT::f32, N->getValueType(0));
560 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported FP_EXTEND!");
561 return TLI.makeLibCall(DAG, LC, NVT, Res32, CallOptions, SDLoc(N)).first;
562}
563
564// FIXME: Should we just use 'normal' FP_EXTEND / FP_TRUNC instead of special
565// nodes?
566SDValue DAGTypeLegalizer::SoftenFloatRes_BF16_TO_FP(SDNode *N) {
567 assert(N->getValueType(0) == MVT::f32 &&
568 "Can only soften BF16_TO_FP with f32 result");
569 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), MVT::f32);
570 SDValue Op = N->getOperand(0);
571 SDLoc DL(N);
572 Op = DAG.getNode(ISD::ANY_EXTEND, DL, NVT,
573 DAG.getNode(ISD::BITCAST, DL, MVT::i16, Op));
574 SDValue Res = DAG.getNode(ISD::SHL, DL, NVT, Op,
575 DAG.getShiftAmountConstant(16, NVT, DL));
576 return Res;
577}
578
579SDValue DAGTypeLegalizer::SoftenFloatRes_FP_ROUND(SDNode *N) {
580 bool IsStrict = N->isStrictFPOpcode();
581 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
582 SDValue Op = N->getOperand(IsStrict ? 1 : 0);
583 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
584 RTLIB::Libcall LC = RTLIB::getFPROUND(Op.getValueType(), N->getValueType(0));
585 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported FP_ROUND!");
586 TargetLowering::MakeLibCallOptions CallOptions;
587 EVT OpVT = N->getOperand(IsStrict ? 1 : 0).getValueType();
588 CallOptions.setTypeListBeforeSoften(OpVT, N->getValueType(0), true);
589 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, NVT, Op,
590 CallOptions, SDLoc(N),
591 Chain);
592 if (IsStrict)
593 ReplaceValueWith(SDValue(N, 1), Tmp.second);
594 return Tmp.first;
595}
596
597SDValue DAGTypeLegalizer::SoftenFloatRes_FPOW(SDNode *N) {
598 return SoftenFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
599 RTLIB::POW_F32,
600 RTLIB::POW_F64,
601 RTLIB::POW_F80,
602 RTLIB::POW_F128,
603 RTLIB::POW_PPCF128));
604}
605
606SDValue DAGTypeLegalizer::SoftenFloatRes_FPOWI(SDNode *N) {
607 bool IsStrict = N->isStrictFPOpcode();
608 unsigned Offset = IsStrict ? 1 : 0;
609 assert((N->getOperand(1 + Offset).getValueType() == MVT::i16 ||
610 N->getOperand(1 + Offset).getValueType() == MVT::i32) &&
611 "Unsupported power type!");
612 RTLIB::Libcall LC = RTLIB::getPOWI(N->getValueType(0));
613 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected fpowi.");
614 if (!TLI.getLibcallName(LC)) {
615 // Some targets don't have a powi libcall; use pow instead.
616 // FIXME: Implement this if some target needs it.
617 DAG.getContext()->emitError("Don't know how to soften fpowi to fpow");
618 return DAG.getUNDEF(N->getValueType(0));
619 }
620
621 if (DAG.getLibInfo().getIntSize() !=
622 N->getOperand(1 + Offset).getValueType().getSizeInBits()) {
623 // If the exponent does not match with sizeof(int) a libcall to RTLIB::POWI
624 // would use the wrong type for the argument.
625 DAG.getContext()->emitError("POWI exponent does not match sizeof(int)");
626 return DAG.getUNDEF(N->getValueType(0));
627 }
628
629 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
630 SDValue Ops[2] = { GetSoftenedFloat(N->getOperand(0 + Offset)),
631 N->getOperand(1 + Offset) };
632 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
633 TargetLowering::MakeLibCallOptions CallOptions;
634 EVT OpsVT[2] = { N->getOperand(0 + Offset).getValueType(),
635 N->getOperand(1 + Offset).getValueType() };
636 CallOptions.setTypeListBeforeSoften(OpsVT, N->getValueType(0), true);
637 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, NVT, Ops,
638 CallOptions, SDLoc(N),
639 Chain);
640 if (IsStrict)
641 ReplaceValueWith(SDValue(N, 1), Tmp.second);
642 return Tmp.first;
643}
644
645SDValue DAGTypeLegalizer::SoftenFloatRes_FREM(SDNode *N) {
646 return SoftenFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
647 RTLIB::REM_F32,
648 RTLIB::REM_F64,
649 RTLIB::REM_F80,
650 RTLIB::REM_F128,
651 RTLIB::REM_PPCF128));
652}
653
654SDValue DAGTypeLegalizer::SoftenFloatRes_FRINT(SDNode *N) {
655 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
656 RTLIB::RINT_F32,
657 RTLIB::RINT_F64,
658 RTLIB::RINT_F80,
659 RTLIB::RINT_F128,
660 RTLIB::RINT_PPCF128));
661}
662
663SDValue DAGTypeLegalizer::SoftenFloatRes_FROUND(SDNode *N) {
664 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
665 RTLIB::ROUND_F32,
666 RTLIB::ROUND_F64,
667 RTLIB::ROUND_F80,
668 RTLIB::ROUND_F128,
669 RTLIB::ROUND_PPCF128));
670}
671
672SDValue DAGTypeLegalizer::SoftenFloatRes_FROUNDEVEN(SDNode *N) {
673 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
674 RTLIB::ROUNDEVEN_F32,
675 RTLIB::ROUNDEVEN_F64,
676 RTLIB::ROUNDEVEN_F80,
677 RTLIB::ROUNDEVEN_F128,
678 RTLIB::ROUNDEVEN_PPCF128));
679}
680
681SDValue DAGTypeLegalizer::SoftenFloatRes_FSIN(SDNode *N) {
682 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
683 RTLIB::SIN_F32,
684 RTLIB::SIN_F64,
685 RTLIB::SIN_F80,
686 RTLIB::SIN_F128,
687 RTLIB::SIN_PPCF128));
688}
689
690SDValue DAGTypeLegalizer::SoftenFloatRes_FSQRT(SDNode *N) {
691 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
692 RTLIB::SQRT_F32,
693 RTLIB::SQRT_F64,
694 RTLIB::SQRT_F80,
695 RTLIB::SQRT_F128,
696 RTLIB::SQRT_PPCF128));
697}
698
699SDValue DAGTypeLegalizer::SoftenFloatRes_FSUB(SDNode *N) {
700 return SoftenFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
701 RTLIB::SUB_F32,
702 RTLIB::SUB_F64,
703 RTLIB::SUB_F80,
704 RTLIB::SUB_F128,
705 RTLIB::SUB_PPCF128));
706}
707
708SDValue DAGTypeLegalizer::SoftenFloatRes_FTRUNC(SDNode *N) {
709 return SoftenFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
710 RTLIB::TRUNC_F32,
711 RTLIB::TRUNC_F64,
712 RTLIB::TRUNC_F80,
713 RTLIB::TRUNC_F128,
714 RTLIB::TRUNC_PPCF128));
715}
716
717SDValue DAGTypeLegalizer::SoftenFloatRes_LOAD(SDNode *N) {
718 LoadSDNode *L = cast<LoadSDNode>(N);
719 EVT VT = N->getValueType(0);
720 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
721 SDLoc dl(N);
722
723 auto MMOFlags =
724 L->getMemOperand()->getFlags() &
725 ~(MachineMemOperand::MOInvariant | MachineMemOperand::MODereferenceable);
726 SDValue NewL;
727 if (L->getExtensionType() == ISD::NON_EXTLOAD) {
728 NewL = DAG.getLoad(L->getAddressingMode(), L->getExtensionType(), NVT, dl,
729 L->getChain(), L->getBasePtr(), L->getOffset(),
730 L->getPointerInfo(), NVT, L->getOriginalAlign(),
731 MMOFlags, L->getAAInfo());
732 // Legalized the chain result - switch anything that used the old chain to
733 // use the new one.
734 ReplaceValueWith(SDValue(N, 1), NewL.getValue(1));
735 return NewL;
736 }
737
738 // Do a non-extending load followed by FP_EXTEND.
739 NewL = DAG.getLoad(L->getAddressingMode(), ISD::NON_EXTLOAD, L->getMemoryVT(),
740 dl, L->getChain(), L->getBasePtr(), L->getOffset(),
741 L->getPointerInfo(), L->getMemoryVT(),
742 L->getOriginalAlign(), MMOFlags, L->getAAInfo());
743 // Legalized the chain result - switch anything that used the old chain to
744 // use the new one.
745 ReplaceValueWith(SDValue(N, 1), NewL.getValue(1));
746 auto ExtendNode = DAG.getNode(ISD::FP_EXTEND, dl, VT, NewL);
747 return BitConvertToInteger(ExtendNode);
748}
749
750SDValue DAGTypeLegalizer::SoftenFloatRes_SELECT(SDNode *N) {
751 SDValue LHS = GetSoftenedFloat(N->getOperand(1));
752 SDValue RHS = GetSoftenedFloat(N->getOperand(2));
753 return DAG.getSelect(SDLoc(N),
754 LHS.getValueType(), N->getOperand(0), LHS, RHS);
755}
756
757SDValue DAGTypeLegalizer::SoftenFloatRes_SELECT_CC(SDNode *N) {
758 SDValue LHS = GetSoftenedFloat(N->getOperand(2));
759 SDValue RHS = GetSoftenedFloat(N->getOperand(3));
760 return DAG.getNode(ISD::SELECT_CC, SDLoc(N),
761 LHS.getValueType(), N->getOperand(0),
762 N->getOperand(1), LHS, RHS, N->getOperand(4));
763}
764
765SDValue DAGTypeLegalizer::SoftenFloatRes_UNDEF(SDNode *N) {
766 return DAG.getUNDEF(TLI.getTypeToTransformTo(*DAG.getContext(),
767 N->getValueType(0)));
768}
769
770SDValue DAGTypeLegalizer::SoftenFloatRes_VAARG(SDNode *N) {
771 SDValue Chain = N->getOperand(0); // Get the chain.
772 SDValue Ptr = N->getOperand(1); // Get the pointer.
773 EVT VT = N->getValueType(0);
774 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
775 SDLoc dl(N);
776
777 SDValue NewVAARG;
778 NewVAARG = DAG.getVAArg(NVT, dl, Chain, Ptr, N->getOperand(2),
779 N->getConstantOperandVal(3));
780
781 // Legalized the chain result - switch anything that used the old chain to
782 // use the new one.
783 if (N != NewVAARG.getValue(1).getNode())
784 ReplaceValueWith(SDValue(N, 1), NewVAARG.getValue(1));
785 return NewVAARG;
786}
787
788SDValue DAGTypeLegalizer::SoftenFloatRes_XINT_TO_FP(SDNode *N) {
789 bool IsStrict = N->isStrictFPOpcode();
790 bool Signed = N->getOpcode() == ISD::SINT_TO_FP ||
791 N->getOpcode() == ISD::STRICT_SINT_TO_FP;
792 EVT SVT = N->getOperand(IsStrict ? 1 : 0).getValueType();
793 EVT RVT = N->getValueType(0);
794 EVT NVT = EVT();
795 SDLoc dl(N);
796
797 // If the input is not legal, eg: i1 -> fp, then it needs to be promoted to
798 // a larger type, eg: i8 -> fp. Even if it is legal, no libcall may exactly
799 // match. Look for an appropriate libcall.
800 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
801 for (unsigned t = MVT::FIRST_INTEGER_VALUETYPE;
802 t <= MVT::LAST_INTEGER_VALUETYPE && LC == RTLIB::UNKNOWN_LIBCALL; ++t) {
803 NVT = (MVT::SimpleValueType)t;
804 // The source needs to big enough to hold the operand.
805 if (NVT.bitsGE(SVT))
806 LC = Signed ? RTLIB::getSINTTOFP(NVT, RVT):RTLIB::getUINTTOFP (NVT, RVT);
807 }
808 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported XINT_TO_FP!");
809
810 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
811 // Sign/zero extend the argument if the libcall takes a larger type.
812 SDValue Op = DAG.getNode(Signed ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND, dl,
813 NVT, N->getOperand(IsStrict ? 1 : 0));
814 TargetLowering::MakeLibCallOptions CallOptions;
815 CallOptions.setSExt(Signed);
816 CallOptions.setTypeListBeforeSoften(SVT, RVT, true);
817 std::pair<SDValue, SDValue> Tmp =
818 TLI.makeLibCall(DAG, LC, TLI.getTypeToTransformTo(*DAG.getContext(), RVT),
819 Op, CallOptions, dl, Chain);
820
821 if (IsStrict)
822 ReplaceValueWith(SDValue(N, 1), Tmp.second);
823 return Tmp.first;
824}
825
826SDValue DAGTypeLegalizer::SoftenFloatRes_VECREDUCE(SDNode *N) {
827 // Expand and soften recursively.
828 ReplaceValueWith(SDValue(N, 0), TLI.expandVecReduce(N, DAG));
829 return SDValue();
830}
831
832SDValue DAGTypeLegalizer::SoftenFloatRes_VECREDUCE_SEQ(SDNode *N) {
833 ReplaceValueWith(SDValue(N, 0), TLI.expandVecReduceSeq(N, DAG));
834 return SDValue();
835}
836
837//===----------------------------------------------------------------------===//
838// Convert Float Operand to Integer
839//===----------------------------------------------------------------------===//
840
841bool DAGTypeLegalizer::SoftenFloatOperand(SDNode *N, unsigned OpNo) {
842 LLVM_DEBUG(dbgs() << "Soften float operand " << OpNo << ": "; N->dump(&DAG);
843 dbgs() << "\n");
844 SDValue Res = SDValue();
845
846 switch (N->getOpcode()) {
847 default:
848#ifndef NDEBUG
849 dbgs() << "SoftenFloatOperand Op #" << OpNo << ": ";
850 N->dump(&DAG); dbgs() << "\n";
851#endif
852 llvm_unreachable("Do not know how to soften this operator's operand!");
853
854 case ISD::BITCAST: Res = SoftenFloatOp_BITCAST(N); break;
855 case ISD::BR_CC: Res = SoftenFloatOp_BR_CC(N); break;
856 case ISD::STRICT_FP_TO_FP16:
857 case ISD::FP_TO_FP16: // Same as FP_ROUND for softening purposes
858 case ISD::FP_TO_BF16:
859 case ISD::STRICT_FP_ROUND:
860 case ISD::FP_ROUND: Res = SoftenFloatOp_FP_ROUND(N); break;
861 case ISD::STRICT_FP_TO_SINT:
862 case ISD::STRICT_FP_TO_UINT:
863 case ISD::FP_TO_SINT:
864 case ISD::FP_TO_UINT: Res = SoftenFloatOp_FP_TO_XINT(N); break;
865 case ISD::FP_TO_SINT_SAT:
866 case ISD::FP_TO_UINT_SAT:
867 Res = SoftenFloatOp_FP_TO_XINT_SAT(N); break;
868 case ISD::STRICT_LROUND:
869 case ISD::LROUND: Res = SoftenFloatOp_LROUND(N); break;
870 case ISD::STRICT_LLROUND:
871 case ISD::LLROUND: Res = SoftenFloatOp_LLROUND(N); break;
872 case ISD::STRICT_LRINT:
873 case ISD::LRINT: Res = SoftenFloatOp_LRINT(N); break;
874 case ISD::STRICT_LLRINT:
875 case ISD::LLRINT: Res = SoftenFloatOp_LLRINT(N); break;
876 case ISD::SELECT_CC: Res = SoftenFloatOp_SELECT_CC(N); break;
877 case ISD::STRICT_FSETCC:
878 case ISD::STRICT_FSETCCS:
879 case ISD::SETCC: Res = SoftenFloatOp_SETCC(N); break;
880 case ISD::STORE: Res = SoftenFloatOp_STORE(N, OpNo); break;
881 case ISD::FCOPYSIGN: Res = SoftenFloatOp_FCOPYSIGN(N); break;
882 }
883
884 // If the result is null, the sub-method took care of registering results etc.
885 if (!Res.getNode()) return false;
886
887 // If the result is N, the sub-method updated N in place. Tell the legalizer
888 // core about this to re-analyze.
889 if (Res.getNode() == N)
890 return true;
891
892 assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 &&
893 "Invalid operand softening");
894
895 ReplaceValueWith(SDValue(N, 0), Res);
896 return false;
897}
898
899SDValue DAGTypeLegalizer::SoftenFloatOp_BITCAST(SDNode *N) {
900 SDValue Op0 = GetSoftenedFloat(N->getOperand(0));
901
902 return DAG.getNode(ISD::BITCAST, SDLoc(N), N->getValueType(0), Op0);
903}
904
905SDValue DAGTypeLegalizer::SoftenFloatOp_FP_ROUND(SDNode *N) {
906 // We actually deal with the partially-softened FP_TO_FP16 node too, which
907 // returns an i16 so doesn't meet the constraints necessary for FP_ROUND.
908 assert(N->getOpcode() == ISD::FP_ROUND || N->getOpcode() == ISD::FP_TO_FP16 ||
909 N->getOpcode() == ISD::STRICT_FP_TO_FP16 ||
910 N->getOpcode() == ISD::FP_TO_BF16 ||
911 N->getOpcode() == ISD::STRICT_FP_ROUND);
912
913 bool IsStrict = N->isStrictFPOpcode();
914 SDValue Op = N->getOperand(IsStrict ? 1 : 0);
915 EVT SVT = Op.getValueType();
916 EVT RVT = N->getValueType(0);
917 EVT FloatRVT = RVT;
918 if (N->getOpcode() == ISD::FP_TO_FP16 ||
919 N->getOpcode() == ISD::STRICT_FP_TO_FP16)
920 FloatRVT = MVT::f16;
921 else if (N->getOpcode() == ISD::FP_TO_BF16)
922 FloatRVT = MVT::bf16;
923
924 RTLIB::Libcall LC = RTLIB::getFPROUND(SVT, FloatRVT);
925 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported FP_ROUND libcall");
926
927 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
928 Op = GetSoftenedFloat(Op);
929 TargetLowering::MakeLibCallOptions CallOptions;
930 CallOptions.setTypeListBeforeSoften(SVT, RVT, true);
931 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, RVT, Op,
932 CallOptions, SDLoc(N),
933 Chain);
934 if (IsStrict) {
935 ReplaceValueWith(SDValue(N, 1), Tmp.second);
936 ReplaceValueWith(SDValue(N, 0), Tmp.first);
937 return SDValue();
938 }
939 return Tmp.first;
940}
941
942SDValue DAGTypeLegalizer::SoftenFloatOp_BR_CC(SDNode *N) {
943 SDValue NewLHS = N->getOperand(2), NewRHS = N->getOperand(3);
944 ISD::CondCode CCCode = cast<CondCodeSDNode>(N->getOperand(1))->get();
945
946 EVT VT = NewLHS.getValueType();
947 NewLHS = GetSoftenedFloat(NewLHS);
948 NewRHS = GetSoftenedFloat(NewRHS);
949 TLI.softenSetCCOperands(DAG, VT, NewLHS, NewRHS, CCCode, SDLoc(N),
950 N->getOperand(2), N->getOperand(3));
951
952 // If softenSetCCOperands returned a scalar, we need to compare the result
953 // against zero to select between true and false values.
954 if (!NewRHS.getNode()) {
955 NewRHS = DAG.getConstant(0, SDLoc(N), NewLHS.getValueType());
956 CCCode = ISD::SETNE;
957 }
958
959 // Update N to have the operands specified.
960 return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0),
961 DAG.getCondCode(CCCode), NewLHS, NewRHS,
962 N->getOperand(4)),
963 0);
964}
965
966// Even if the result type is legal, no libcall may exactly match. (e.g. We
967// don't have FP-i8 conversions) This helper method looks for an appropriate
968// promoted libcall.
969static RTLIB::Libcall findFPToIntLibcall(EVT SrcVT, EVT RetVT, EVT &Promoted,
970 bool Signed) {
971 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
972 for (unsigned IntVT = MVT::FIRST_INTEGER_VALUETYPE;
973 IntVT <= MVT::LAST_INTEGER_VALUETYPE && LC == RTLIB::UNKNOWN_LIBCALL;
974 ++IntVT) {
975 Promoted = (MVT::SimpleValueType)IntVT;
976 // The type needs to big enough to hold the result.
977 if (Promoted.bitsGE(RetVT))
978 LC = Signed ? RTLIB::getFPTOSINT(SrcVT, Promoted)
979 : RTLIB::getFPTOUINT(SrcVT, Promoted);
980 }
981 return LC;
982}
983
984SDValue DAGTypeLegalizer::SoftenFloatOp_FP_TO_XINT(SDNode *N) {
985 bool IsStrict = N->isStrictFPOpcode();
986 bool Signed = N->getOpcode() == ISD::FP_TO_SINT ||
987 N->getOpcode() == ISD::STRICT_FP_TO_SINT;
988
989 SDValue Op = N->getOperand(IsStrict ? 1 : 0);
990 EVT SVT = Op.getValueType();
991 EVT RVT = N->getValueType(0);
992 EVT NVT = EVT();
993 SDLoc dl(N);
994
995 // If the result is not legal, eg: fp -> i1, then it needs to be promoted to
996 // a larger type, eg: fp -> i32. Even if it is legal, no libcall may exactly
997 // match, eg. we don't have fp -> i8 conversions.
998 // Look for an appropriate libcall.
999 RTLIB::Libcall LC = findFPToIntLibcall(SVT, RVT, NVT, Signed);
1000 assert(LC != RTLIB::UNKNOWN_LIBCALL && NVT.isSimple() &&
1001 "Unsupported FP_TO_XINT!");
1002
1003 Op = GetSoftenedFloat(Op);
1004 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
1005 TargetLowering::MakeLibCallOptions CallOptions;
1006 CallOptions.setTypeListBeforeSoften(SVT, RVT, true);
1007 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, NVT, Op,
1008 CallOptions, dl, Chain);
1009
1010 // Truncate the result if the libcall returns a larger type.
1011 SDValue Res = DAG.getNode(ISD::TRUNCATE, dl, RVT, Tmp.first);
1012
1013 if (!IsStrict)
1014 return Res;
1015
1016 ReplaceValueWith(SDValue(N, 1), Tmp.second);
1017 ReplaceValueWith(SDValue(N, 0), Res);
1018 return SDValue();
1019}
1020
1021SDValue DAGTypeLegalizer::SoftenFloatOp_FP_TO_XINT_SAT(SDNode *N) {
1022 SDValue Res = TLI.expandFP_TO_INT_SAT(N, DAG);
1023 return Res;
1024}
1025
1026SDValue DAGTypeLegalizer::SoftenFloatOp_SELECT_CC(SDNode *N) {
1027 SDValue NewLHS = N->getOperand(0), NewRHS = N->getOperand(1);
1028 ISD::CondCode CCCode = cast<CondCodeSDNode>(N->getOperand(4))->get();
1029
1030 EVT VT = NewLHS.getValueType();
1031 NewLHS = GetSoftenedFloat(NewLHS);
1032 NewRHS = GetSoftenedFloat(NewRHS);
1033 TLI.softenSetCCOperands(DAG, VT, NewLHS, NewRHS, CCCode, SDLoc(N),
1034 N->getOperand(0), N->getOperand(1));
1035
1036 // If softenSetCCOperands returned a scalar, we need to compare the result
1037 // against zero to select between true and false values.
1038 if (!NewRHS.getNode()) {
1039 NewRHS = DAG.getConstant(0, SDLoc(N), NewLHS.getValueType());
1040 CCCode = ISD::SETNE;
1041 }
1042
1043 // Update N to have the operands specified.
1044 return SDValue(DAG.UpdateNodeOperands(N, NewLHS, NewRHS,
1045 N->getOperand(2), N->getOperand(3),
1046 DAG.getCondCode(CCCode)),
1047 0);
1048}
1049
1050SDValue DAGTypeLegalizer::SoftenFloatOp_SETCC(SDNode *N) {
1051 bool IsStrict = N->isStrictFPOpcode();
1052 SDValue Op0 = N->getOperand(IsStrict ? 1 : 0);
1053 SDValue Op1 = N->getOperand(IsStrict ? 2 : 1);
1054 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
1055 ISD::CondCode CCCode =
1056 cast<CondCodeSDNode>(N->getOperand(IsStrict ? 3 : 2))->get();
1057
1058 EVT VT = Op0.getValueType();
1059 SDValue NewLHS = GetSoftenedFloat(Op0);
1060 SDValue NewRHS = GetSoftenedFloat(Op1);
1061 TLI.softenSetCCOperands(DAG, VT, NewLHS, NewRHS, CCCode, SDLoc(N), Op0, Op1,
1062 Chain, N->getOpcode() == ISD::STRICT_FSETCCS);
1063
1064 // Update N to have the operands specified.
1065 if (NewRHS.getNode()) {
1066 if (IsStrict)
1067 NewLHS = DAG.getNode(ISD::SETCC, SDLoc(N), N->getValueType(0), NewLHS,
1068 NewRHS, DAG.getCondCode(CCCode));
1069 else
1070 return SDValue(DAG.UpdateNodeOperands(N, NewLHS, NewRHS,
1071 DAG.getCondCode(CCCode)), 0);
1072 }
1073
1074 // Otherwise, softenSetCCOperands returned a scalar, use it.
1075 assert((NewRHS.getNode() || NewLHS.getValueType() == N->getValueType(0)) &&
1076 "Unexpected setcc expansion!");
1077
1078 if (IsStrict) {
1079 ReplaceValueWith(SDValue(N, 0), NewLHS);
1080 ReplaceValueWith(SDValue(N, 1), Chain);
1081 return SDValue();
1082 }
1083 return NewLHS;
1084}
1085
1086SDValue DAGTypeLegalizer::SoftenFloatOp_STORE(SDNode *N, unsigned OpNo) {
1087 assert(ISD::isUNINDEXEDStore(N) && "Indexed store during type legalization!");
1088 assert(OpNo == 1 && "Can only soften the stored value!");
1089 StoreSDNode *ST = cast<StoreSDNode>(N);
1090 SDValue Val = ST->getValue();
1091 SDLoc dl(N);
1092
1093 if (ST->isTruncatingStore())
1094 // Do an FP_ROUND followed by a non-truncating store.
1095 Val = BitConvertToInteger(
1096 DAG.getNode(ISD::FP_ROUND, dl, ST->getMemoryVT(), Val,
1097 DAG.getIntPtrConstant(0, dl, /*isTarget=*/true)));
1098 else
1099 Val = GetSoftenedFloat(Val);
1100
1101 return DAG.getStore(ST->getChain(), dl, Val, ST->getBasePtr(),
1102 ST->getMemOperand());
1103}
1104
1105SDValue DAGTypeLegalizer::SoftenFloatOp_FCOPYSIGN(SDNode *N) {
1106 SDValue LHS = N->getOperand(0);
1107 SDValue RHS = BitConvertToInteger(N->getOperand(1));
1108 SDLoc dl(N);
1109
1110 EVT LVT = LHS.getValueType();
1111 EVT ILVT = EVT::getIntegerVT(*DAG.getContext(), LVT.getSizeInBits());
1112 EVT RVT = RHS.getValueType();
1113
1114 unsigned LSize = LVT.getSizeInBits();
1115 unsigned RSize = RVT.getSizeInBits();
1116
1117 // Shift right or sign-extend it if the two operands have different types.
1118 int SizeDiff = RSize - LSize;
1119 if (SizeDiff > 0) {
1120 RHS =
1121 DAG.getNode(ISD::SRL, dl, RVT, RHS,
1122 DAG.getConstant(SizeDiff, dl,
1123 TLI.getShiftAmountTy(RHS.getValueType(),
1124 DAG.getDataLayout())));
1125 RHS = DAG.getNode(ISD::TRUNCATE, dl, ILVT, RHS);
1126 } else if (SizeDiff < 0) {
1127 RHS = DAG.getNode(ISD::ANY_EXTEND, dl, LVT, RHS);
1128 RHS =
1129 DAG.getNode(ISD::SHL, dl, ILVT, RHS,
1130 DAG.getConstant(-SizeDiff, dl,
1131 TLI.getShiftAmountTy(RHS.getValueType(),
1132 DAG.getDataLayout())));
1133 }
1134
1135 RHS = DAG.getBitcast(LVT, RHS);
1136 return DAG.getNode(ISD::FCOPYSIGN, dl, LVT, LHS, RHS);
1137}
1138
1139SDValue DAGTypeLegalizer::SoftenFloatOp_Unary(SDNode *N, RTLIB::Libcall LC) {
1140 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
1141 bool IsStrict = N->isStrictFPOpcode();
1142 unsigned Offset = IsStrict ? 1 : 0;
1143 SDValue Op = GetSoftenedFloat(N->getOperand(0 + Offset));
1144 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
1145 TargetLowering::MakeLibCallOptions CallOptions;
1146 EVT OpVT = N->getOperand(0 + Offset).getValueType();
1147 CallOptions.setTypeListBeforeSoften(OpVT, N->getValueType(0), true);
1148 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, NVT, Op,
1149 CallOptions, SDLoc(N),
1150 Chain);
1151 if (IsStrict) {
1152 ReplaceValueWith(SDValue(N, 1), Tmp.second);
1153 ReplaceValueWith(SDValue(N, 0), Tmp.first);
1154 return SDValue();
1155 }
1156
1157 return Tmp.first;
1158}
1159
1160SDValue DAGTypeLegalizer::SoftenFloatOp_LROUND(SDNode *N) {
1161 EVT OpVT = N->getOperand(N->isStrictFPOpcode() ? 1 : 0).getValueType();
1162 return SoftenFloatOp_Unary(N, GetFPLibCall(OpVT,
1163 RTLIB::LROUND_F32,
1164 RTLIB::LROUND_F64,
1165 RTLIB::LROUND_F80,
1166 RTLIB::LROUND_F128,
1167 RTLIB::LROUND_PPCF128));
1168}
1169
1170SDValue DAGTypeLegalizer::SoftenFloatOp_LLROUND(SDNode *N) {
1171 EVT OpVT = N->getOperand(N->isStrictFPOpcode() ? 1 : 0).getValueType();
1172 return SoftenFloatOp_Unary(N, GetFPLibCall(OpVT,
1173 RTLIB::LLROUND_F32,
1174 RTLIB::LLROUND_F64,
1175 RTLIB::LLROUND_F80,
1176 RTLIB::LLROUND_F128,
1177 RTLIB::LLROUND_PPCF128));
1178}
1179
1180SDValue DAGTypeLegalizer::SoftenFloatOp_LRINT(SDNode *N) {
1181 EVT OpVT = N->getOperand(N->isStrictFPOpcode() ? 1 : 0).getValueType();
1182 return SoftenFloatOp_Unary(N, GetFPLibCall(OpVT,
1183 RTLIB::LRINT_F32,
1184 RTLIB::LRINT_F64,
1185 RTLIB::LRINT_F80,
1186 RTLIB::LRINT_F128,
1187 RTLIB::LRINT_PPCF128));
1188}
1189
1190SDValue DAGTypeLegalizer::SoftenFloatOp_LLRINT(SDNode *N) {
1191 EVT OpVT = N->getOperand(N->isStrictFPOpcode() ? 1 : 0).getValueType();
1192 return SoftenFloatOp_Unary(N, GetFPLibCall(OpVT,
1193 RTLIB::LLRINT_F32,
1194 RTLIB::LLRINT_F64,
1195 RTLIB::LLRINT_F80,
1196 RTLIB::LLRINT_F128,
1197 RTLIB::LLRINT_PPCF128));
1198}
1199
1200//===----------------------------------------------------------------------===//
1201// Float Result Expansion
1202//===----------------------------------------------------------------------===//
1203
1204/// ExpandFloatResult - This method is called when the specified result of the
1205/// specified node is found to need expansion. At this point, the node may also
1206/// have invalid operands or may have other results that need promotion, we just
1207/// know that (at least) one result needs expansion.
1208void DAGTypeLegalizer::ExpandFloatResult(SDNode *N, unsigned ResNo) {
1209 LLVM_DEBUG(dbgs() << "Expand float result: "; N->dump(&DAG); dbgs() << "\n");
1210 SDValue Lo, Hi;
1211 Lo = Hi = SDValue();
1212
1213 // See if the target wants to custom expand this node.
1214 if (CustomLowerNode(N, N->getValueType(ResNo), true))
1215 return;
1216
1217 switch (N->getOpcode()) {
1218 default:
1219#ifndef NDEBUG
1220 dbgs() << "ExpandFloatResult #" << ResNo << ": ";
1221 N->dump(&DAG); dbgs() << "\n";
1222#endif
1223 llvm_unreachable("Do not know how to expand the result of this operator!");
1224
1225 case ISD::UNDEF: SplitRes_UNDEF(N, Lo, Hi); break;
1226 case ISD::SELECT: SplitRes_Select(N, Lo, Hi); break;
1227 case ISD::SELECT_CC: SplitRes_SELECT_CC(N, Lo, Hi); break;
1228
1229 case ISD::MERGE_VALUES: ExpandRes_MERGE_VALUES(N, ResNo, Lo, Hi); break;
1230 case ISD::BITCAST: ExpandRes_BITCAST(N, Lo, Hi); break;
1231 case ISD::BUILD_PAIR: ExpandRes_BUILD_PAIR(N, Lo, Hi); break;
1232 case ISD::EXTRACT_ELEMENT: ExpandRes_EXTRACT_ELEMENT(N, Lo, Hi); break;
1233 case ISD::EXTRACT_VECTOR_ELT: ExpandRes_EXTRACT_VECTOR_ELT(N, Lo, Hi); break;
1234 case ISD::VAARG: ExpandRes_VAARG(N, Lo, Hi); break;
1235
1236 case ISD::ConstantFP: ExpandFloatRes_ConstantFP(N, Lo, Hi); break;
1237 case ISD::FABS: ExpandFloatRes_FABS(N, Lo, Hi); break;
1238 case ISD::STRICT_FMINNUM:
1239 case ISD::FMINNUM: ExpandFloatRes_FMINNUM(N, Lo, Hi); break;
1240 case ISD::STRICT_FMAXNUM:
1241 case ISD::FMAXNUM: ExpandFloatRes_FMAXNUM(N, Lo, Hi); break;
1242 case ISD::STRICT_FADD:
1243 case ISD::FADD: ExpandFloatRes_FADD(N, Lo, Hi); break;
1244 case ISD::FCBRT: ExpandFloatRes_FCBRT(N, Lo, Hi); break;
1245 case ISD::STRICT_FCEIL:
1246 case ISD::FCEIL: ExpandFloatRes_FCEIL(N, Lo, Hi); break;
1247 case ISD::FCOPYSIGN: ExpandFloatRes_FCOPYSIGN(N, Lo, Hi); break;
1248 case ISD::STRICT_FCOS:
1249 case ISD::FCOS: ExpandFloatRes_FCOS(N, Lo, Hi); break;
1250 case ISD::STRICT_FDIV:
1251 case ISD::FDIV: ExpandFloatRes_FDIV(N, Lo, Hi); break;
1252 case ISD::STRICT_FEXP:
1253 case ISD::FEXP: ExpandFloatRes_FEXP(N, Lo, Hi); break;
1254 case ISD::STRICT_FEXP2:
1255 case ISD::FEXP2: ExpandFloatRes_FEXP2(N, Lo, Hi); break;
1256 case ISD::STRICT_FFLOOR:
1257 case ISD::FFLOOR: ExpandFloatRes_FFLOOR(N, Lo, Hi); break;
1258 case ISD::STRICT_FLOG:
1259 case ISD::FLOG: ExpandFloatRes_FLOG(N, Lo, Hi); break;
1260 case ISD::STRICT_FLOG2:
1261 case ISD::FLOG2: ExpandFloatRes_FLOG2(N, Lo, Hi); break;
1262 case ISD::STRICT_FLOG10:
1263 case ISD::FLOG10: ExpandFloatRes_FLOG10(N, Lo, Hi); break;
1264 case ISD::STRICT_FMA:
1265 case ISD::FMA: ExpandFloatRes_FMA(N, Lo, Hi); break;
1266 case ISD::STRICT_FMUL:
1267 case ISD::FMUL: ExpandFloatRes_FMUL(N, Lo, Hi); break;
1268 case ISD::STRICT_FNEARBYINT:
1269 case ISD::FNEARBYINT: ExpandFloatRes_FNEARBYINT(N, Lo, Hi); break;
1270 case ISD::FNEG: ExpandFloatRes_FNEG(N, Lo, Hi); break;
1271 case ISD::STRICT_FP_EXTEND:
1272 case ISD::FP_EXTEND: ExpandFloatRes_FP_EXTEND(N, Lo, Hi); break;
1273 case ISD::STRICT_FPOW:
1274 case ISD::FPOW: ExpandFloatRes_FPOW(N, Lo, Hi); break;
1275 case ISD::STRICT_FPOWI:
1276 case ISD::FPOWI: ExpandFloatRes_FPOWI(N, Lo, Hi); break;
1277 case ISD::FREEZE: ExpandFloatRes_FREEZE(N, Lo, Hi); break;
1278 case ISD::STRICT_FRINT:
1279 case ISD::FRINT: ExpandFloatRes_FRINT(N, Lo, Hi); break;
1280 case ISD::STRICT_FROUND:
1281 case ISD::FROUND: ExpandFloatRes_FROUND(N, Lo, Hi); break;
1282 case ISD::STRICT_FROUNDEVEN:
1283 case ISD::FROUNDEVEN: ExpandFloatRes_FROUNDEVEN(N, Lo, Hi); break;
1284 case ISD::STRICT_FSIN:
1285 case ISD::FSIN: ExpandFloatRes_FSIN(N, Lo, Hi); break;
1286 case ISD::STRICT_FSQRT:
1287 case ISD::FSQRT: ExpandFloatRes_FSQRT(N, Lo, Hi); break;
1288 case ISD::STRICT_FSUB:
1289 case ISD::FSUB: ExpandFloatRes_FSUB(N, Lo, Hi); break;
1290 case ISD::STRICT_FTRUNC:
1291 case ISD::FTRUNC: ExpandFloatRes_FTRUNC(N, Lo, Hi); break;
1292 case ISD::LOAD: ExpandFloatRes_LOAD(N, Lo, Hi); break;
1293 case ISD::STRICT_SINT_TO_FP:
1294 case ISD::STRICT_UINT_TO_FP:
1295 case ISD::SINT_TO_FP:
1296 case ISD::UINT_TO_FP: ExpandFloatRes_XINT_TO_FP(N, Lo, Hi); break;
1297 case ISD::STRICT_FREM:
1298 case ISD::FREM: ExpandFloatRes_FREM(N, Lo, Hi); break;
1299 }
1300
1301 // If Lo/Hi is null, the sub-method took care of registering results etc.
1302 if (Lo.getNode())
1303 SetExpandedFloat(SDValue(N, ResNo), Lo, Hi);
1304}
1305
1306void DAGTypeLegalizer::ExpandFloatRes_ConstantFP(SDNode *N, SDValue &Lo,
1307 SDValue &Hi) {
1308 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
1309 assert(NVT.getSizeInBits() == 64 &&
1310 "Do not know how to expand this float constant!");
1311 APInt C = cast<ConstantFPSDNode>(N)->getValueAPF().bitcastToAPInt();
1312 SDLoc dl(N);
1313 Lo = DAG.getConstantFP(APFloat(DAG.EVTToAPFloatSemantics(NVT),
1314 APInt(64, C.getRawData()[1])),
1315 dl, NVT);
1316 Hi = DAG.getConstantFP(APFloat(DAG.EVTToAPFloatSemantics(NVT),
1317 APInt(64, C.getRawData()[0])),
1318 dl, NVT);
1319}
1320
1321void DAGTypeLegalizer::ExpandFloatRes_Unary(SDNode *N, RTLIB::Libcall LC,
1322 SDValue &Lo, SDValue &Hi) {
1323 bool IsStrict = N->isStrictFPOpcode();
1324 unsigned Offset = IsStrict ? 1 : 0;
1325 SDValue Op = N->getOperand(0 + Offset);
1326 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
1327 TargetLowering::MakeLibCallOptions CallOptions;
1328 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, N->getValueType(0),
1329 Op, CallOptions, SDLoc(N),
1330 Chain);
1331 if (IsStrict)
1332 ReplaceValueWith(SDValue(N, 1), Tmp.second);
1333 GetPairElements(Tmp.first, Lo, Hi);
1334}
1335
1336void DAGTypeLegalizer::ExpandFloatRes_Binary(SDNode *N, RTLIB::Libcall LC,
1337 SDValue &Lo, SDValue &Hi) {
1338 bool IsStrict = N->isStrictFPOpcode();
1339 unsigned Offset = IsStrict ? 1 : 0;
1340 SDValue Ops[] = { N->getOperand(0 + Offset), N->getOperand(1 + Offset) };
1341 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
1342 TargetLowering::MakeLibCallOptions CallOptions;
1343 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, N->getValueType(0),
1344 Ops, CallOptions, SDLoc(N),
1345 Chain);
1346 if (IsStrict)
1347 ReplaceValueWith(SDValue(N, 1), Tmp.second);
1348 GetPairElements(Tmp.first, Lo, Hi);
1349}
1350
1351void DAGTypeLegalizer::ExpandFloatRes_FABS(SDNode *N, SDValue &Lo,
1352 SDValue &Hi) {
1353 assert(N->getValueType(0) == MVT::ppcf128 &&
1354 "Logic only correct for ppcf128!");
1355 SDLoc dl(N);
1356 SDValue Tmp;
1357 GetExpandedFloat(N->getOperand(0), Lo, Tmp);
1358 Hi = DAG.getNode(ISD::FABS, dl, Tmp.getValueType(), Tmp);
1359 // Lo = Hi==fabs(Hi) ? Lo : -Lo;
1360 Lo = DAG.getSelectCC(dl, Tmp, Hi, Lo,
1361 DAG.getNode(ISD::FNEG, dl, Lo.getValueType(), Lo),
1362 ISD::SETEQ);
1363}
1364
1365void DAGTypeLegalizer::ExpandFloatRes_FMINNUM(SDNode *N, SDValue &Lo,
1366 SDValue &Hi) {
1367 ExpandFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
1368 RTLIB::FMIN_F32, RTLIB::FMIN_F64,
1369 RTLIB::FMIN_F80, RTLIB::FMIN_F128,
1370 RTLIB::FMIN_PPCF128), Lo, Hi);
1371}
1372
1373void DAGTypeLegalizer::ExpandFloatRes_FMAXNUM(SDNode *N, SDValue &Lo,
1374 SDValue &Hi) {
1375 ExpandFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
1376 RTLIB::FMAX_F32, RTLIB::FMAX_F64,
1377 RTLIB::FMAX_F80, RTLIB::FMAX_F128,
1378 RTLIB::FMAX_PPCF128), Lo, Hi);
1379}
1380
1381void DAGTypeLegalizer::ExpandFloatRes_FADD(SDNode *N, SDValue &Lo,
1382 SDValue &Hi) {
1383 ExpandFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
1384 RTLIB::ADD_F32, RTLIB::ADD_F64,
1385 RTLIB::ADD_F80, RTLIB::ADD_F128,
1386 RTLIB::ADD_PPCF128), Lo, Hi);
1387}
1388
1389void DAGTypeLegalizer::ExpandFloatRes_FCBRT(SDNode *N, SDValue &Lo,
1390 SDValue &Hi) {
1391 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0), RTLIB::CBRT_F32,
1392 RTLIB::CBRT_F64, RTLIB::CBRT_F80,
1393 RTLIB::CBRT_F128,
1394 RTLIB::CBRT_PPCF128), Lo, Hi);
1395}
1396
1397void DAGTypeLegalizer::ExpandFloatRes_FCEIL(SDNode *N,
1398 SDValue &Lo, SDValue &Hi) {
1399 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
1400 RTLIB::CEIL_F32, RTLIB::CEIL_F64,
1401 RTLIB::CEIL_F80, RTLIB::CEIL_F128,
1402 RTLIB::CEIL_PPCF128), Lo, Hi);
1403}
1404
1405void DAGTypeLegalizer::ExpandFloatRes_FCOPYSIGN(SDNode *N,
1406 SDValue &Lo, SDValue &Hi) {
1407 ExpandFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
1408 RTLIB::COPYSIGN_F32,
1409 RTLIB::COPYSIGN_F64,
1410 RTLIB::COPYSIGN_F80,
1411 RTLIB::COPYSIGN_F128,
1412 RTLIB::COPYSIGN_PPCF128), Lo, Hi);
1413}
1414
1415void DAGTypeLegalizer::ExpandFloatRes_FCOS(SDNode *N,
1416 SDValue &Lo, SDValue &Hi) {
1417 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
1418 RTLIB::COS_F32, RTLIB::COS_F64,
1419 RTLIB::COS_F80, RTLIB::COS_F128,
1420 RTLIB::COS_PPCF128), Lo, Hi);
1421}
1422
1423void DAGTypeLegalizer::ExpandFloatRes_FDIV(SDNode *N, SDValue &Lo,
1424 SDValue &Hi) {
1425 ExpandFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
1426 RTLIB::DIV_F32,
1427 RTLIB::DIV_F64,
1428 RTLIB::DIV_F80,
1429 RTLIB::DIV_F128,
1430 RTLIB::DIV_PPCF128), Lo, Hi);
1431}
1432
1433void DAGTypeLegalizer::ExpandFloatRes_FEXP(SDNode *N,
1434 SDValue &Lo, SDValue &Hi) {
1435 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
1436 RTLIB::EXP_F32, RTLIB::EXP_F64,
1437 RTLIB::EXP_F80, RTLIB::EXP_F128,
1438 RTLIB::EXP_PPCF128), Lo, Hi);
1439}
1440
1441void DAGTypeLegalizer::ExpandFloatRes_FEXP2(SDNode *N,
1442 SDValue &Lo, SDValue &Hi) {
1443 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
1444 RTLIB::EXP2_F32, RTLIB::EXP2_F64,
1445 RTLIB::EXP2_F80, RTLIB::EXP2_F128,
1446 RTLIB::EXP2_PPCF128), Lo, Hi);
1447}
1448
1449void DAGTypeLegalizer::ExpandFloatRes_FFLOOR(SDNode *N,
1450 SDValue &Lo, SDValue &Hi) {
1451 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
1452 RTLIB::FLOOR_F32, RTLIB::FLOOR_F64,
1453 RTLIB::FLOOR_F80, RTLIB::FLOOR_F128,
1454 RTLIB::FLOOR_PPCF128), Lo, Hi);
1455}
1456
1457void DAGTypeLegalizer::ExpandFloatRes_FLOG(SDNode *N,
1458 SDValue &Lo, SDValue &Hi) {
1459 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
1460 RTLIB::LOG_F32, RTLIB::LOG_F64,
1461 RTLIB::LOG_F80, RTLIB::LOG_F128,
1462 RTLIB::LOG_PPCF128), Lo, Hi);
1463}
1464
1465void DAGTypeLegalizer::ExpandFloatRes_FLOG2(SDNode *N,
1466 SDValue &Lo, SDValue &Hi) {
1467 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
1468 RTLIB::LOG2_F32, RTLIB::LOG2_F64,
1469 RTLIB::LOG2_F80, RTLIB::LOG2_F128,
1470 RTLIB::LOG2_PPCF128), Lo, Hi);
1471}
1472
1473void DAGTypeLegalizer::ExpandFloatRes_FLOG10(SDNode *N,
1474 SDValue &Lo, SDValue &Hi) {
1475 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
1476 RTLIB::LOG10_F32, RTLIB::LOG10_F64,
1477 RTLIB::LOG10_F80, RTLIB::LOG10_F128,
1478 RTLIB::LOG10_PPCF128), Lo, Hi);
1479}
1480
1481void DAGTypeLegalizer::ExpandFloatRes_FMA(SDNode *N, SDValue &Lo,
1482 SDValue &Hi) {
1483 bool IsStrict = N->isStrictFPOpcode();
1484 unsigned Offset = IsStrict ? 1 : 0;
1485 SDValue Ops[3] = { N->getOperand(0 + Offset), N->getOperand(1 + Offset),
1486 N->getOperand(2 + Offset) };
1487 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
1488 TargetLowering::MakeLibCallOptions CallOptions;
1489 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, GetFPLibCall(N->getValueType(0),
1490 RTLIB::FMA_F32,
1491 RTLIB::FMA_F64,
1492 RTLIB::FMA_F80,
1493 RTLIB::FMA_F128,
1494 RTLIB::FMA_PPCF128),
1495 N->getValueType(0), Ops, CallOptions,
1496 SDLoc(N), Chain);
1497 if (IsStrict)
1498 ReplaceValueWith(SDValue(N, 1), Tmp.second);
1499 GetPairElements(Tmp.first, Lo, Hi);
1500}
1501
1502void DAGTypeLegalizer::ExpandFloatRes_FMUL(SDNode *N, SDValue &Lo,
1503 SDValue &Hi) {
1504 ExpandFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
1505 RTLIB::MUL_F32,
1506 RTLIB::MUL_F64,
1507 RTLIB::MUL_F80,
1508 RTLIB::MUL_F128,
1509 RTLIB::MUL_PPCF128), Lo, Hi);
1510}
1511
1512void DAGTypeLegalizer::ExpandFloatRes_FNEARBYINT(SDNode *N,
1513 SDValue &Lo, SDValue &Hi) {
1514 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
1515 RTLIB::NEARBYINT_F32,
1516 RTLIB::NEARBYINT_F64,
1517 RTLIB::NEARBYINT_F80,
1518 RTLIB::NEARBYINT_F128,
1519 RTLIB::NEARBYINT_PPCF128), Lo, Hi);
1520}
1521
1522void DAGTypeLegalizer::ExpandFloatRes_FNEG(SDNode *N, SDValue &Lo,
1523 SDValue &Hi) {
1524 SDLoc dl(N);
1525 GetExpandedFloat(N->getOperand(0), Lo, Hi);
1526 Lo = DAG.getNode(ISD::FNEG, dl, Lo.getValueType(), Lo);
1527 Hi = DAG.getNode(ISD::FNEG, dl, Hi.getValueType(), Hi);
1528}
1529
1530void DAGTypeLegalizer::ExpandFloatRes_FP_EXTEND(SDNode *N, SDValue &Lo,
1531 SDValue &Hi) {
1532 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
1533 SDLoc dl(N);
1534 bool IsStrict = N->isStrictFPOpcode();
1535
1536 SDValue Chain;
1537 if (IsStrict) {
1538 // If the expanded type is the same as the input type, just bypass the node.
1539 if (NVT == N->getOperand(1).getValueType()) {
1540 Hi = N->getOperand(1);
1541 Chain = N->getOperand(0);
1542 } else {
1543 // Other we need to extend.
1544 Hi = DAG.getNode(ISD::STRICT_FP_EXTEND, dl, { NVT, MVT::Other },
1545 { N->getOperand(0), N->getOperand(1) });
1546 Chain = Hi.getValue(1);
1547 }
1548 } else {
1549 Hi = DAG.getNode(ISD::FP_EXTEND, dl, NVT, N->getOperand(0));
1550 }
1551
1552 Lo = DAG.getConstantFP(APFloat(DAG.EVTToAPFloatSemantics(NVT),
1553 APInt(NVT.getSizeInBits(), 0)), dl, NVT);
1554
1555 if (IsStrict)
1556 ReplaceValueWith(SDValue(N, 1), Chain);
1557}
1558
1559void DAGTypeLegalizer::ExpandFloatRes_FPOW(SDNode *N,
1560 SDValue &Lo, SDValue &Hi) {
1561 ExpandFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
1562 RTLIB::POW_F32, RTLIB::POW_F64,
1563 RTLIB::POW_F80, RTLIB::POW_F128,
1564 RTLIB::POW_PPCF128), Lo, Hi);
1565}
1566
1567void DAGTypeLegalizer::ExpandFloatRes_FPOWI(SDNode *N,
1568 SDValue &Lo, SDValue &Hi) {
1569 ExpandFloatRes_Binary(N, RTLIB::getPOWI(N->getValueType(0)), Lo, Hi);
1570}
1571
1572void DAGTypeLegalizer::ExpandFloatRes_FREEZE(SDNode *N,
1573 SDValue &Lo, SDValue &Hi) {
1574 assert(N->getValueType(0) == MVT::ppcf128 &&
1575 "Logic only correct for ppcf128!");
1576
1577 SDLoc dl(N);
1578 GetExpandedFloat(N->getOperand(0), Lo, Hi);
1579 Lo = DAG.getNode(ISD::FREEZE, dl, Lo.getValueType(), Lo);
1580 Hi = DAG.getNode(ISD::FREEZE, dl, Hi.getValueType(), Hi);
1581}
1582
1583void DAGTypeLegalizer::ExpandFloatRes_FREM(SDNode *N,
1584 SDValue &Lo, SDValue &Hi) {
1585 ExpandFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
1586 RTLIB::REM_F32, RTLIB::REM_F64,
1587 RTLIB::REM_F80, RTLIB::REM_F128,
1588 RTLIB::REM_PPCF128), Lo, Hi);
1589}
1590
1591void DAGTypeLegalizer::ExpandFloatRes_FRINT(SDNode *N,
1592 SDValue &Lo, SDValue &Hi) {
1593 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
1594 RTLIB::RINT_F32, RTLIB::RINT_F64,
1595 RTLIB::RINT_F80, RTLIB::RINT_F128,
1596 RTLIB::RINT_PPCF128), Lo, Hi);
1597}
1598
1599void DAGTypeLegalizer::ExpandFloatRes_FROUND(SDNode *N,
1600 SDValue &Lo, SDValue &Hi) {
1601 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
1602 RTLIB::ROUND_F32,
1603 RTLIB::ROUND_F64,
1604 RTLIB::ROUND_F80,
1605 RTLIB::ROUND_F128,
1606 RTLIB::ROUND_PPCF128), Lo, Hi);
1607}
1608
1609void DAGTypeLegalizer::ExpandFloatRes_FROUNDEVEN(SDNode *N,
1610 SDValue &Lo, SDValue &Hi) {
1611 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
1612 RTLIB::ROUNDEVEN_F32,
1613 RTLIB::ROUNDEVEN_F64,
1614 RTLIB::ROUNDEVEN_F80,
1615 RTLIB::ROUNDEVEN_F128,
1616 RTLIB::ROUNDEVEN_PPCF128), Lo, Hi);
1617}
1618
1619void DAGTypeLegalizer::ExpandFloatRes_FSIN(SDNode *N,
1620 SDValue &Lo, SDValue &Hi) {
1621 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
1622 RTLIB::SIN_F32, RTLIB::SIN_F64,
1623 RTLIB::SIN_F80, RTLIB::SIN_F128,
1624 RTLIB::SIN_PPCF128), Lo, Hi);
1625}
1626
1627void DAGTypeLegalizer::ExpandFloatRes_FSQRT(SDNode *N,
1628 SDValue &Lo, SDValue &Hi) {
1629 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
1630 RTLIB::SQRT_F32, RTLIB::SQRT_F64,
1631 RTLIB::SQRT_F80, RTLIB::SQRT_F128,
1632 RTLIB::SQRT_PPCF128), Lo, Hi);
1633}
1634
1635void DAGTypeLegalizer::ExpandFloatRes_FSUB(SDNode *N, SDValue &Lo,
1636 SDValue &Hi) {
1637 ExpandFloatRes_Binary(N, GetFPLibCall(N->getValueType(0),
1638 RTLIB::SUB_F32,
1639 RTLIB::SUB_F64,
1640 RTLIB::SUB_F80,
1641 RTLIB::SUB_F128,
1642 RTLIB::SUB_PPCF128), Lo, Hi);
1643}
1644
1645void DAGTypeLegalizer::ExpandFloatRes_FTRUNC(SDNode *N,
1646 SDValue &Lo, SDValue &Hi) {
1647 ExpandFloatRes_Unary(N, GetFPLibCall(N->getValueType(0),
1648 RTLIB::TRUNC_F32, RTLIB::TRUNC_F64,
1649 RTLIB::TRUNC_F80, RTLIB::TRUNC_F128,
1650 RTLIB::TRUNC_PPCF128), Lo, Hi);
1651}
1652
1653void DAGTypeLegalizer::ExpandFloatRes_LOAD(SDNode *N, SDValue &Lo,
1654 SDValue &Hi) {
1655 if (ISD::isNormalLoad(N)) {
1656 ExpandRes_NormalLoad(N, Lo, Hi);
1657 return;
1658 }
1659
1660 assert(ISD::isUNINDEXEDLoad(N) && "Indexed load during type legalization!");
1661 LoadSDNode *LD = cast<LoadSDNode>(N);
1662 SDValue Chain = LD->getChain();
1663 SDValue Ptr = LD->getBasePtr();
1664 SDLoc dl(N);
1665
1666 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), LD->getValueType(0));
1667 assert(NVT.isByteSized() && "Expanded type not byte sized!");
1668 assert(LD->getMemoryVT().bitsLE(NVT) && "Float type not round?");
1669
1670 Hi = DAG.getExtLoad(LD->getExtensionType(), dl, NVT, Chain, Ptr,
1671 LD->getMemoryVT(), LD->getMemOperand());
1672
1673 // Remember the chain.
1674 Chain = Hi.getValue(1);
1675
1676 // The low part is zero.
1677 Lo = DAG.getConstantFP(APFloat(DAG.EVTToAPFloatSemantics(NVT),
1678 APInt(NVT.getSizeInBits(), 0)), dl, NVT);
1679
1680 // Modified the chain - switch anything that used the old chain to use the
1681 // new one.
1682 ReplaceValueWith(SDValue(LD, 1), Chain);
1683}
1684
1685void DAGTypeLegalizer::ExpandFloatRes_XINT_TO_FP(SDNode *N, SDValue &Lo,
1686 SDValue &Hi) {
1687 assert(N->getValueType(0) == MVT::ppcf128 && "Unsupported XINT_TO_FP!");
1688 EVT VT = N->getValueType(0);
1689 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
1690 bool Strict = N->isStrictFPOpcode();
1691 SDValue Src = N->getOperand(Strict ? 1 : 0);
1692 EVT SrcVT = Src.getValueType();
1693 bool isSigned = N->getOpcode() == ISD::SINT_TO_FP ||
1694 N->getOpcode() == ISD::STRICT_SINT_TO_FP;
1695 SDLoc dl(N);
1696 SDValue Chain = Strict ? N->getOperand(0) : DAG.getEntryNode();
1697
1698 // TODO: Any other flags to propagate?
1699 SDNodeFlags Flags;
1700 Flags.setNoFPExcept(N->getFlags().hasNoFPExcept());
1701
1702 // First do an SINT_TO_FP, whether the original was signed or unsigned.
1703 // When promoting partial word types to i32 we must honor the signedness,
1704 // though.
1705 if (SrcVT.bitsLE(MVT::i32)) {
1706 // The integer can be represented exactly in an f64.
1707 Lo = DAG.getConstantFP(APFloat(DAG.EVTToAPFloatSemantics(NVT),
1708 APInt(NVT.getSizeInBits(), 0)), dl, NVT);
1709 if (Strict) {
1710 Hi = DAG.getNode(N->getOpcode(), dl, DAG.getVTList(NVT, MVT::Other),
1711 {Chain, Src}, Flags);
1712 Chain = Hi.getValue(1);
1713 } else
1714 Hi = DAG.getNode(N->getOpcode(), dl, NVT, Src);
1715 } else {
1716 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
1717 if (SrcVT.bitsLE(MVT::i64)) {
1718 Src = DAG.getNode(isSigned ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND, dl,
1719 MVT::i64, Src);
1720 LC = RTLIB::SINTTOFP_I64_PPCF128;
1721 } else if (SrcVT.bitsLE(MVT::i128)) {
1722 Src = DAG.getNode(ISD::SIGN_EXTEND, dl, MVT::i128, Src);
1723 LC = RTLIB::SINTTOFP_I128_PPCF128;
1724 }
1725 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unsupported XINT_TO_FP!");
1726
1727 TargetLowering::MakeLibCallOptions CallOptions;
1728 CallOptions.setSExt(true);
1729 std::pair<SDValue, SDValue> Tmp =
1730 TLI.makeLibCall(DAG, LC, VT, Src, CallOptions, dl, Chain);
1731 if (Strict)
1732 Chain = Tmp.second;
1733 GetPairElements(Tmp.first, Lo, Hi);
1734 }
1735
1736 // No need to complement for unsigned 32-bit integers
1737 if (isSigned || SrcVT.bitsLE(MVT::i32)) {
1738 if (Strict)
1739 ReplaceValueWith(SDValue(N, 1), Chain);
1740
1741 return;
1742 }
1743
1744 // Unsigned - fix up the SINT_TO_FP value just calculated.
1745 // FIXME: For unsigned i128 to ppc_fp128 conversion, we need to carefully
1746 // keep semantics correctness if the integer is not exactly representable
1747 // here. See ExpandLegalINT_TO_FP.
1748 Hi = DAG.getNode(ISD::BUILD_PAIR, dl, VT, Lo, Hi);
1749 SrcVT = Src.getValueType();
1750
1751 // x>=0 ? (ppcf128)(iN)x : (ppcf128)(iN)x + 2^N; N=32,64,128.
1752 static const uint64_t TwoE32[] = { 0x41f0000000000000LL, 0 };
1753 static const uint64_t TwoE64[] = { 0x43f0000000000000LL, 0 };
1754 static const uint64_t TwoE128[] = { 0x47f0000000000000LL, 0 };
1755 ArrayRef<uint64_t> Parts;
1756
1757 switch (SrcVT.getSimpleVT().SimpleTy) {
1758 default:
1759 llvm_unreachable("Unsupported UINT_TO_FP!");
1760 case MVT::i32:
1761 Parts = TwoE32;
1762 break;
1763 case MVT::i64:
1764 Parts = TwoE64;
1765 break;
1766 case MVT::i128:
1767 Parts = TwoE128;
1768 break;
1769 }
1770
1771 // TODO: Are there other fast-math-flags to propagate to this FADD?
1772 SDValue NewLo = DAG.getConstantFP(
1773 APFloat(APFloat::PPCDoubleDouble(), APInt(128, Parts)), dl, MVT::ppcf128);
1774 if (Strict) {
1775 Lo = DAG.getNode(ISD::STRICT_FADD, dl, DAG.getVTList(VT, MVT::Other),
1776 {Chain, Hi, NewLo}, Flags);
1777 Chain = Lo.getValue(1);
1778 ReplaceValueWith(SDValue(N, 1), Chain);
1779 } else
1780 Lo = DAG.getNode(ISD::FADD, dl, VT, Hi, NewLo);
1781 Lo = DAG.getSelectCC(dl, Src, DAG.getConstant(0, dl, SrcVT),
1782 Lo, Hi, ISD::SETLT);
1783 GetPairElements(Lo, Lo, Hi);
1784}
1785
1786
1787//===----------------------------------------------------------------------===//
1788// Float Operand Expansion
1789//===----------------------------------------------------------------------===//
1790
1791/// ExpandFloatOperand - This method is called when the specified operand of the
1792/// specified node is found to need expansion. At this point, all of the result
1793/// types of the node are known to be legal, but other operands of the node may
1794/// need promotion or expansion as well as the specified one.
1795bool DAGTypeLegalizer::ExpandFloatOperand(SDNode *N, unsigned OpNo) {
1796 LLVM_DEBUG(dbgs() << "Expand float operand: "; N->dump(&DAG); dbgs() << "\n");
1797 SDValue Res = SDValue();
1798
1799 // See if the target wants to custom expand this node.
1800 if (CustomLowerNode(N, N->getOperand(OpNo).getValueType(), false))
1801 return false;
1802
1803 switch (N->getOpcode()) {
1804 default:
1805#ifndef NDEBUG
1806 dbgs() << "ExpandFloatOperand Op #" << OpNo << ": ";
1807 N->dump(&DAG); dbgs() << "\n";
1808#endif
1809 llvm_unreachable("Do not know how to expand this operator's operand!");
1810
1811 case ISD::BITCAST: Res = ExpandOp_BITCAST(N); break;
1812 case ISD::BUILD_VECTOR: Res = ExpandOp_BUILD_VECTOR(N); break;
1813 case ISD::EXTRACT_ELEMENT: Res = ExpandOp_EXTRACT_ELEMENT(N); break;
1814
1815 case ISD::BR_CC: Res = ExpandFloatOp_BR_CC(N); break;
1816 case ISD::FCOPYSIGN: Res = ExpandFloatOp_FCOPYSIGN(N); break;
1817 case ISD::STRICT_FP_ROUND:
1818 case ISD::FP_ROUND: Res = ExpandFloatOp_FP_ROUND(N); break;
1819 case ISD::STRICT_FP_TO_SINT:
1820 case ISD::STRICT_FP_TO_UINT:
1821 case ISD::FP_TO_SINT:
1822 case ISD::FP_TO_UINT: Res = ExpandFloatOp_FP_TO_XINT(N); break;
1823 case ISD::LROUND: Res = ExpandFloatOp_LROUND(N); break;
1824 case ISD::LLROUND: Res = ExpandFloatOp_LLROUND(N); break;
1825 case ISD::LRINT: Res = ExpandFloatOp_LRINT(N); break;
1826 case ISD::LLRINT: Res = ExpandFloatOp_LLRINT(N); break;
1827 case ISD::SELECT_CC: Res = ExpandFloatOp_SELECT_CC(N); break;
1828 case ISD::STRICT_FSETCC:
1829 case ISD::STRICT_FSETCCS:
1830 case ISD::SETCC: Res = ExpandFloatOp_SETCC(N); break;
1831 case ISD::STORE: Res = ExpandFloatOp_STORE(cast<StoreSDNode>(N),
1832 OpNo); break;
1833 }
1834
1835 // If the result is null, the sub-method took care of registering results etc.
1836 if (!Res.getNode()) return false;
1837
1838 // If the result is N, the sub-method updated N in place. Tell the legalizer
1839 // core about this.
1840 if (Res.getNode() == N)
1841 return true;
1842
1843 assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 &&
1844 "Invalid operand expansion");
1845
1846 ReplaceValueWith(SDValue(N, 0), Res);
1847 return false;
1848}
1849
1850/// FloatExpandSetCCOperands - Expand the operands of a comparison. This code
1851/// is shared among BR_CC, SELECT_CC, and SETCC handlers.
1852void DAGTypeLegalizer::FloatExpandSetCCOperands(SDValue &NewLHS,
1853 SDValue &NewRHS,
1854 ISD::CondCode &CCCode,
1855 const SDLoc &dl, SDValue &Chain,
1856 bool IsSignaling) {
1857 SDValue LHSLo, LHSHi, RHSLo, RHSHi;
1858 GetExpandedFloat(NewLHS, LHSLo, LHSHi);
1859 GetExpandedFloat(NewRHS, RHSLo, RHSHi);
1860
1861 assert(NewLHS.getValueType() == MVT::ppcf128 && "Unsupported setcc type!");
1862
1863 // FIXME: This generated code sucks. We want to generate
1864 // FCMPU crN, hi1, hi2
1865 // BNE crN, L:
1866 // FCMPU crN, lo1, lo2
1867 // The following can be improved, but not that much.
1868 SDValue Tmp1, Tmp2, Tmp3, OutputChain;
1869 Tmp1 = DAG.getSetCC(dl, getSetCCResultType(LHSHi.getValueType()), LHSHi,
1870 RHSHi, ISD::SETOEQ, Chain, IsSignaling);
1871 OutputChain = Tmp1->getNumValues() > 1 ? Tmp1.getValue(1) : SDValue();
1872 Tmp2 = DAG.getSetCC(dl, getSetCCResultType(LHSLo.getValueType()), LHSLo,
1873 RHSLo, CCCode, OutputChain, IsSignaling);
1874 OutputChain = Tmp2->getNumValues() > 1 ? Tmp2.getValue(1) : SDValue();
1875 Tmp3 = DAG.getNode(ISD::AND, dl, Tmp1.getValueType(), Tmp1, Tmp2);
1876 Tmp1 =
1877 DAG.getSetCC(dl, getSetCCResultType(LHSHi.getValueType()), LHSHi, RHSHi,
1878 ISD::SETUNE, OutputChain, IsSignaling);
1879 OutputChain = Tmp1->getNumValues() > 1 ? Tmp1.getValue(1) : SDValue();
1880 Tmp2 = DAG.getSetCC(dl, getSetCCResultType(LHSHi.getValueType()), LHSHi,
1881 RHSHi, CCCode, OutputChain, IsSignaling);
1882 OutputChain = Tmp2->getNumValues() > 1 ? Tmp2.getValue(1) : SDValue();
1883 Tmp1 = DAG.getNode(ISD::AND, dl, Tmp1.getValueType(), Tmp1, Tmp2);
1884 NewLHS = DAG.getNode(ISD::OR, dl, Tmp1.getValueType(), Tmp1, Tmp3);
1885 NewRHS = SDValue(); // LHS is the result, not a compare.
1886 Chain = OutputChain;
1887}
1888
1889SDValue DAGTypeLegalizer::ExpandFloatOp_BR_CC(SDNode *N) {
1890 SDValue NewLHS = N->getOperand(2), NewRHS = N->getOperand(3);
1891 ISD::CondCode CCCode = cast<CondCodeSDNode>(N->getOperand(1))->get();
1892 SDValue Chain;
1893 FloatExpandSetCCOperands(NewLHS, NewRHS, CCCode, SDLoc(N), Chain);
1894
1895 // If ExpandSetCCOperands returned a scalar, we need to compare the result
1896 // against zero to select between true and false values.
1897 if (!NewRHS.getNode()) {
1898 NewRHS = DAG.getConstant(0, SDLoc(N), NewLHS.getValueType());
1899 CCCode = ISD::SETNE;
1900 }
1901
1902 // Update N to have the operands specified.
1903 return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0),
1904 DAG.getCondCode(CCCode), NewLHS, NewRHS,
1905 N->getOperand(4)), 0);
1906}
1907
1908SDValue DAGTypeLegalizer::ExpandFloatOp_FCOPYSIGN(SDNode *N) {
1909 assert(N->getOperand(1).getValueType() == MVT::ppcf128 &&
1910 "Logic only correct for ppcf128!");
1911 SDValue Lo, Hi;
1912 GetExpandedFloat(N->getOperand(1), Lo, Hi);
1913 // The ppcf128 value is providing only the sign; take it from the
1914 // higher-order double (which must have the larger magnitude).
1915 return DAG.getNode(ISD::FCOPYSIGN, SDLoc(N),
1916 N->getValueType(0), N->getOperand(0), Hi);
1917}
1918
1919SDValue DAGTypeLegalizer::ExpandFloatOp_FP_ROUND(SDNode *N) {
1920 bool IsStrict = N->isStrictFPOpcode();
1921 assert(N->getOperand(IsStrict ? 1 : 0).getValueType() == MVT::ppcf128 &&
1922 "Logic only correct for ppcf128!");
1923 SDValue Lo, Hi;
1924 GetExpandedFloat(N->getOperand(IsStrict ? 1 : 0), Lo, Hi);
1925
1926 if (!IsStrict)
1927 // Round it the rest of the way (e.g. to f32) if needed.
1928 return DAG.getNode(ISD::FP_ROUND, SDLoc(N),
1929 N->getValueType(0), Hi, N->getOperand(1));
1930
1931 // Eliminate the node if the input float type is the same as the output float
1932 // type.
1933 if (Hi.getValueType() == N->getValueType(0)) {
1934 // Connect the output chain to the input chain, unlinking the node.
1935 ReplaceValueWith(SDValue(N, 1), N->getOperand(0));
1936 ReplaceValueWith(SDValue(N, 0), Hi);
1937 return SDValue();
1938 }
1939
1940 SDValue Expansion = DAG.getNode(ISD::STRICT_FP_ROUND, SDLoc(N),
1941 {N->getValueType(0), MVT::Other},
1942 {N->getOperand(0), Hi, N->getOperand(2)});
1943 ReplaceValueWith(SDValue(N, 1), Expansion.getValue(1));
1944 ReplaceValueWith(SDValue(N, 0), Expansion);
1945 return SDValue();
1946}
1947
1948SDValue DAGTypeLegalizer::ExpandFloatOp_FP_TO_XINT(SDNode *N) {
1949 EVT RVT = N->getValueType(0);
1950 SDLoc dl(N);
1951
1952 bool IsStrict = N->isStrictFPOpcode();
1953 bool Signed = N->getOpcode() == ISD::FP_TO_SINT ||
1954 N->getOpcode() == ISD::STRICT_FP_TO_SINT;
1955 SDValue Op = N->getOperand(IsStrict ? 1 : 0);
1956 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
1957
1958 EVT NVT;
1959 RTLIB::Libcall LC = findFPToIntLibcall(Op.getValueType(), RVT, NVT, Signed);
1960 assert(LC != RTLIB::UNKNOWN_LIBCALL && NVT.isSimple() &&
1961 "Unsupported FP_TO_XINT!");
1962 TargetLowering::MakeLibCallOptions CallOptions;
1963 std::pair<SDValue, SDValue> Tmp =
1964 TLI.makeLibCall(DAG, LC, NVT, Op, CallOptions, dl, Chain);
1965 if (!IsStrict)
1966 return Tmp.first;
1967
1968 ReplaceValueWith(SDValue(N, 1), Tmp.second);
1969 ReplaceValueWith(SDValue(N, 0), Tmp.first);
1970 return SDValue();
1971}
1972
1973SDValue DAGTypeLegalizer::ExpandFloatOp_SELECT_CC(SDNode *N) {
1974 SDValue NewLHS = N->getOperand(0), NewRHS = N->getOperand(1);
1975 ISD::CondCode CCCode = cast<CondCodeSDNode>(N->getOperand(4))->get();
1976 SDValue Chain;
1977 FloatExpandSetCCOperands(NewLHS, NewRHS, CCCode, SDLoc(N), Chain);
1978
1979 // If ExpandSetCCOperands returned a scalar, we need to compare the result
1980 // against zero to select between true and false values.
1981 if (!NewRHS.getNode()) {
1982 NewRHS = DAG.getConstant(0, SDLoc(N), NewLHS.getValueType());
1983 CCCode = ISD::SETNE;
1984 }
1985
1986 // Update N to have the operands specified.
1987 return SDValue(DAG.UpdateNodeOperands(N, NewLHS, NewRHS,
1988 N->getOperand(2), N->getOperand(3),
1989 DAG.getCondCode(CCCode)), 0);
1990}
1991
1992SDValue DAGTypeLegalizer::ExpandFloatOp_SETCC(SDNode *N) {
1993 bool IsStrict = N->isStrictFPOpcode();
1994 SDValue NewLHS = N->getOperand(IsStrict ? 1 : 0);
1995 SDValue NewRHS = N->getOperand(IsStrict ? 2 : 1);
1996 SDValue Chain = IsStrict ? N->getOperand(0) : SDValue();
1997 ISD::CondCode CCCode =
1998 cast<CondCodeSDNode>(N->getOperand(IsStrict ? 3 : 2))->get();
1999 FloatExpandSetCCOperands(NewLHS, NewRHS, CCCode, SDLoc(N), Chain,
2000 N->getOpcode() == ISD::STRICT_FSETCCS);
2001
2002 // FloatExpandSetCCOperands always returned a scalar.
2003 assert(!NewRHS.getNode() && "Expect to return scalar");
2004 assert(NewLHS.getValueType() == N->getValueType(0) &&
2005 "Unexpected setcc expansion!");
2006 if (Chain) {
2007 ReplaceValueWith(SDValue(N, 0), NewLHS);
2008 ReplaceValueWith(SDValue(N, 1), Chain);
2009 return SDValue();
2010 }
2011 return NewLHS;
2012}
2013
2014SDValue DAGTypeLegalizer::ExpandFloatOp_STORE(SDNode *N, unsigned OpNo) {
2015 if (ISD::isNormalStore(N))
2016 return ExpandOp_NormalStore(N, OpNo);
2017
2018 assert(ISD::isUNINDEXEDStore(N) && "Indexed store during type legalization!");
2019 assert(OpNo == 1 && "Can only expand the stored value so far");
2020 StoreSDNode *ST = cast<StoreSDNode>(N);
2021
2022 SDValue Chain = ST->getChain();
2023 SDValue Ptr = ST->getBasePtr();
2024
2025 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(),
2026 ST->getValue().getValueType());
2027 assert(NVT.isByteSized() && "Expanded type not byte sized!");
2028 assert(ST->getMemoryVT().bitsLE(NVT) && "Float type not round?");
2029 (void)NVT;
2030
2031 SDValue Lo, Hi;
2032 GetExpandedOp(ST->getValue(), Lo, Hi);
2033
2034 return DAG.getTruncStore(Chain, SDLoc(N), Hi, Ptr,
2035 ST->getMemoryVT(), ST->getMemOperand());
2036}
2037
2038SDValue DAGTypeLegalizer::ExpandFloatOp_LROUND(SDNode *N) {
2039 EVT RVT = N->getValueType(0);
2040 EVT RetVT = N->getOperand(0).getValueType();
2041 TargetLowering::MakeLibCallOptions CallOptions;
2042 return TLI.makeLibCall(DAG, GetFPLibCall(RetVT,
2043 RTLIB::LROUND_F32,
2044 RTLIB::LROUND_F64,
2045 RTLIB::LROUND_F80,
2046 RTLIB::LROUND_F128,
2047 RTLIB::LROUND_PPCF128),
2048 RVT, N->getOperand(0), CallOptions, SDLoc(N)).first;
2049}
2050
2051SDValue DAGTypeLegalizer::ExpandFloatOp_LLROUND(SDNode *N) {
2052 EVT RVT = N->getValueType(0);
2053 EVT RetVT = N->getOperand(0).getValueType();
2054 TargetLowering::MakeLibCallOptions CallOptions;
2055 return TLI.makeLibCall(DAG, GetFPLibCall(RetVT,
2056 RTLIB::LLROUND_F32,
2057 RTLIB::LLROUND_F64,
2058 RTLIB::LLROUND_F80,
2059 RTLIB::LLROUND_F128,
2060 RTLIB::LLROUND_PPCF128),
2061 RVT, N->getOperand(0), CallOptions, SDLoc(N)).first;
2062}
2063
2064SDValue DAGTypeLegalizer::ExpandFloatOp_LRINT(SDNode *N) {
2065 EVT RVT = N->getValueType(0);
2066 EVT RetVT = N->getOperand(0).getValueType();
2067 TargetLowering::MakeLibCallOptions CallOptions;
2068 return TLI.makeLibCall(DAG, GetFPLibCall(RetVT,
2069 RTLIB::LRINT_F32,
2070 RTLIB::LRINT_F64,
2071 RTLIB::LRINT_F80,
2072 RTLIB::LRINT_F128,
2073 RTLIB::LRINT_PPCF128),
2074 RVT, N->getOperand(0), CallOptions, SDLoc(N)).first;
2075}
2076
2077SDValue DAGTypeLegalizer::ExpandFloatOp_LLRINT(SDNode *N) {
2078 EVT RVT = N->getValueType(0);
2079 EVT RetVT = N->getOperand(0).getValueType();
2080 TargetLowering::MakeLibCallOptions CallOptions;
2081 return TLI.makeLibCall(DAG, GetFPLibCall(RetVT,
2082 RTLIB::LLRINT_F32,
2083 RTLIB::LLRINT_F64,
2084 RTLIB::LLRINT_F80,
2085 RTLIB::LLRINT_F128,
2086 RTLIB::LLRINT_PPCF128),
2087 RVT, N->getOperand(0), CallOptions, SDLoc(N)).first;
2088}
2089
2090//===----------------------------------------------------------------------===//
2091// Float Operand Promotion
2092//===----------------------------------------------------------------------===//
2093//
2094
2095static ISD::NodeType GetPromotionOpcode(EVT OpVT, EVT RetVT) {
2096 if (OpVT == MVT::f16) {
2097 return ISD::FP16_TO_FP;
2098 } else if (RetVT == MVT::f16) {
2099 return ISD::FP_TO_FP16;
2100 } else if (OpVT == MVT::bf16) {
2101 return ISD::BF16_TO_FP;
2102 } else if (RetVT == MVT::bf16) {
2103 return ISD::FP_TO_BF16;
2104 }
2105
2106 report_fatal_error("Attempt at an invalid promotion-related conversion");
2107}
2108
2109bool DAGTypeLegalizer::PromoteFloatOperand(SDNode *N, unsigned OpNo) {
2110 LLVM_DEBUG(dbgs() << "Promote float operand " << OpNo << ": "; N->dump(&DAG);
2111 dbgs() << "\n");
2112 SDValue R = SDValue();
2113
2114 if (CustomLowerNode(N, N->getOperand(OpNo).getValueType(), false)) {
2115 LLVM_DEBUG(dbgs() << "Node has been custom lowered, done\n");
2116 return false;
2117 }
2118
2119 // Nodes that use a promotion-requiring floating point operand, but doesn't
2120 // produce a promotion-requiring floating point result, need to be legalized
2121 // to use the promoted float operand. Nodes that produce at least one
2122 // promotion-requiring floating point result have their operands legalized as
2123 // a part of PromoteFloatResult.
2124 switch (N->getOpcode()) {
2125 default:
2126 #ifndef NDEBUG
2127 dbgs() << "PromoteFloatOperand Op #" << OpNo << ": ";
2128 N->dump(&DAG); dbgs() << "\n";
2129 #endif
2130 llvm_unreachable("Do not know how to promote this operator's operand!");
2131
2132 case ISD::BITCAST: R = PromoteFloatOp_BITCAST(N, OpNo); break;
2133 case ISD::FCOPYSIGN: R = PromoteFloatOp_FCOPYSIGN(N, OpNo); break;
2134 case ISD::FP_TO_SINT:
2135 case ISD::FP_TO_UINT: R = PromoteFloatOp_FP_TO_XINT(N, OpNo); break;
2136 case ISD::FP_TO_SINT_SAT:
2137 case ISD::FP_TO_UINT_SAT:
2138 R = PromoteFloatOp_FP_TO_XINT_SAT(N, OpNo); break;
2139 case ISD::FP_EXTEND: R = PromoteFloatOp_FP_EXTEND(N, OpNo); break;
2140 case ISD::SELECT_CC: R = PromoteFloatOp_SELECT_CC(N, OpNo); break;
2141 case ISD::SETCC: R = PromoteFloatOp_SETCC(N, OpNo); break;
2142 case ISD::STORE: R = PromoteFloatOp_STORE(N, OpNo); break;
2143 }
2144
2145 if (R.getNode())
2146 ReplaceValueWith(SDValue(N, 0), R);
2147 return false;
2148}
2149
2150SDValue DAGTypeLegalizer::PromoteFloatOp_BITCAST(SDNode *N, unsigned OpNo) {
2151 SDValue Op = N->getOperand(0);
2152 EVT OpVT = Op->getValueType(0);
2153
2154 SDValue Promoted = GetPromotedFloat(N->getOperand(0));
2155 EVT PromotedVT = Promoted->getValueType(0);
2156
2157 // Convert the promoted float value to the desired IVT.
2158 EVT IVT = EVT::getIntegerVT(*DAG.getContext(), OpVT.getSizeInBits());
2159 SDValue Convert = DAG.getNode(GetPromotionOpcode(PromotedVT, OpVT), SDLoc(N),
2160 IVT, Promoted);
2161 // The final result type might not be an scalar so we need a bitcast. The
2162 // bitcast will be further legalized if needed.
2163 return DAG.getBitcast(N->getValueType(0), Convert);
2164}
2165
2166// Promote Operand 1 of FCOPYSIGN. Operand 0 ought to be handled by
2167// PromoteFloatRes_FCOPYSIGN.
2168SDValue DAGTypeLegalizer::PromoteFloatOp_FCOPYSIGN(SDNode *N, unsigned OpNo) {
2169 assert (OpNo == 1 && "Only Operand 1 must need promotion here");
2170 SDValue Op1 = GetPromotedFloat(N->getOperand(1));
2171
2172 return DAG.getNode(N->getOpcode(), SDLoc(N), N->getValueType(0),
2173 N->getOperand(0), Op1);
2174}
2175
2176// Convert the promoted float value to the desired integer type
2177SDValue DAGTypeLegalizer::PromoteFloatOp_FP_TO_XINT(SDNode *N, unsigned OpNo) {
2178 SDValue Op = GetPromotedFloat(N->getOperand(0));
2179 return DAG.getNode(N->getOpcode(), SDLoc(N), N->getValueType(0), Op);
2180}
2181
2182SDValue DAGTypeLegalizer::PromoteFloatOp_FP_TO_XINT_SAT(SDNode *N,
2183 unsigned OpNo) {
2184 SDValue Op = GetPromotedFloat(N->getOperand(0));
2185 return DAG.getNode(N->getOpcode(), SDLoc(N), N->getValueType(0), Op,
2186 N->getOperand(1));
2187}
2188
2189SDValue DAGTypeLegalizer::PromoteFloatOp_FP_EXTEND(SDNode *N, unsigned OpNo) {
2190 SDValue Op = GetPromotedFloat(N->getOperand(0));
2191 EVT VT = N->getValueType(0);
2192
2193 // Desired VT is same as promoted type. Use promoted float directly.
2194 if (VT == Op->getValueType(0))
2195 return Op;
2196
2197 // Else, extend the promoted float value to the desired VT.
2198 return DAG.getNode(ISD::FP_EXTEND, SDLoc(N), VT, Op);
2199}
2200
2201// Promote the float operands used for comparison. The true- and false-
2202// operands have the same type as the result and are promoted, if needed, by
2203// PromoteFloatRes_SELECT_CC
2204SDValue DAGTypeLegalizer::PromoteFloatOp_SELECT_CC(SDNode *N, unsigned OpNo) {
2205 SDValue LHS = GetPromotedFloat(N->getOperand(0));
2206 SDValue RHS = GetPromotedFloat(N->getOperand(1));
2207
2208 return DAG.getNode(ISD::SELECT_CC, SDLoc(N), N->getValueType(0),
2209 LHS, RHS, N->getOperand(2), N->getOperand(3),
2210 N->getOperand(4));
2211}
2212
2213// Construct a SETCC that compares the promoted values and sets the conditional
2214// code.
2215SDValue DAGTypeLegalizer::PromoteFloatOp_SETCC(SDNode *N, unsigned OpNo) {
2216 EVT VT = N->getValueType(0);
2217 SDValue Op0 = GetPromotedFloat(N->getOperand(0));
2218 SDValue Op1 = GetPromotedFloat(N->getOperand(1));
2219 ISD::CondCode CCCode = cast<CondCodeSDNode>(N->getOperand(2))->get();
2220
2221 return DAG.getSetCC(SDLoc(N), VT, Op0, Op1, CCCode);
2222
2223}
2224
2225// Lower the promoted Float down to the integer value of same size and construct
2226// a STORE of the integer value.
2227SDValue DAGTypeLegalizer::PromoteFloatOp_STORE(SDNode *N, unsigned OpNo) {
2228 StoreSDNode *ST = cast<StoreSDNode>(N);
2229 SDValue Val = ST->getValue();
2230 SDLoc DL(N);
2231
2232 SDValue Promoted = GetPromotedFloat(Val);
2233 EVT VT = ST->getOperand(1).getValueType();
2234 EVT IVT = EVT::getIntegerVT(*DAG.getContext(), VT.getSizeInBits());
2235
2236 SDValue NewVal;
2237 NewVal = DAG.getNode(GetPromotionOpcode(Promoted.getValueType(), VT), DL,
2238 IVT, Promoted);
2239
2240 return DAG.getStore(ST->getChain(), DL, NewVal, ST->getBasePtr(),
2241 ST->getMemOperand());
2242}
2243
2244//===----------------------------------------------------------------------===//
2245// Float Result Promotion
2246//===----------------------------------------------------------------------===//
2247
2248void DAGTypeLegalizer::PromoteFloatResult(SDNode *N, unsigned ResNo) {
2249 LLVM_DEBUG(dbgs() << "Promote float result " << ResNo << ": "; N->dump(&DAG);
2250 dbgs() << "\n");
2251 SDValue R = SDValue();
2252
2253 // See if the target wants to custom expand this node.
2254 if (CustomLowerNode(N, N->getValueType(ResNo), true)) {
2255 LLVM_DEBUG(dbgs() << "Node has been custom expanded, done\n");
2256 return;
2257 }
2258
2259 switch (N->getOpcode()) {
2260 // These opcodes cannot appear if promotion of FP16 is done in the backend
2261 // instead of Clang
2262 case ISD::FP16_TO_FP:
2263 case ISD::FP_TO_FP16:
2264 default:
2265#ifndef NDEBUG
2266 dbgs() << "PromoteFloatResult #" << ResNo << ": ";
2267 N->dump(&DAG); dbgs() << "\n";
2268#endif
2269 llvm_unreachable("Do not know how to promote this operator's result!");
2270
2271 case ISD::BITCAST: R = PromoteFloatRes_BITCAST(N); break;
2272 case ISD::ConstantFP: R = PromoteFloatRes_ConstantFP(N); break;
2273 case ISD::EXTRACT_VECTOR_ELT:
2274 R = PromoteFloatRes_EXTRACT_VECTOR_ELT(N); break;
2275 case ISD::FCOPYSIGN: R = PromoteFloatRes_FCOPYSIGN(N); break;
2276
2277 // Unary FP Operations
2278 case ISD::FABS:
2279 case ISD::FCBRT:
2280 case ISD::FCEIL:
2281 case ISD::FCOS:
2282 case ISD::FEXP:
2283 case ISD::FEXP2:
2284 case ISD::FFLOOR:
2285 case ISD::FLOG:
2286 case ISD::FLOG2:
2287 case ISD::FLOG10:
2288 case ISD::FNEARBYINT:
2289 case ISD::FNEG:
2290 case ISD::FRINT:
2291 case ISD::FROUND:
2292 case ISD::FROUNDEVEN:
2293 case ISD::FSIN:
2294 case ISD::FSQRT:
2295 case ISD::FTRUNC:
2296 case ISD::FCANONICALIZE: R = PromoteFloatRes_UnaryOp(N); break;
2297
2298 // Binary FP Operations
2299 case ISD::FADD:
2300 case ISD::FDIV:
2301 case ISD::FMAXIMUM:
2302 case ISD::FMINIMUM:
2303 case ISD::FMAXNUM:
2304 case ISD::FMINNUM:
2305 case ISD::FMUL:
2306 case ISD::FPOW:
2307 case ISD::FREM:
2308 case ISD::FSUB: R = PromoteFloatRes_BinOp(N); break;
2309
2310 case ISD::FMA: // FMA is same as FMAD
2311 case ISD::FMAD: R = PromoteFloatRes_FMAD(N); break;
2312
2313 case ISD::FPOWI: R = PromoteFloatRes_FPOWI(N); break;
2314
2315 case ISD::FP_ROUND: R = PromoteFloatRes_FP_ROUND(N); break;
2316 case ISD::LOAD: R = PromoteFloatRes_LOAD(N); break;
2317 case ISD::SELECT: R = PromoteFloatRes_SELECT(N); break;
2318 case ISD::SELECT_CC: R = PromoteFloatRes_SELECT_CC(N); break;
2319
2320 case ISD::SINT_TO_FP:
2321 case ISD::UINT_TO_FP: R = PromoteFloatRes_XINT_TO_FP(N); break;
2322 case ISD::UNDEF: R = PromoteFloatRes_UNDEF(N); break;
2323 case ISD::ATOMIC_SWAP: R = BitcastToInt_ATOMIC_SWAP(N); break;
2324 case ISD::VECREDUCE_FADD:
2325 case ISD::VECREDUCE_FMUL:
2326 case ISD::VECREDUCE_FMIN:
2327 case ISD::VECREDUCE_FMAX:
2328 R = PromoteFloatRes_VECREDUCE(N);
2329 break;
2330 case ISD::VECREDUCE_SEQ_FADD:
2331 case ISD::VECREDUCE_SEQ_FMUL:
2332 R = PromoteFloatRes_VECREDUCE_SEQ(N);
2333 break;
2334 }
2335
2336 if (R.getNode())
2337 SetPromotedFloat(SDValue(N, ResNo), R);
2338}
2339
2340// Bitcast from i16 to f16: convert the i16 to a f32 value instead.
2341// At this point, it is not possible to determine if the bitcast value is
2342// eventually stored to memory or promoted to f32 or promoted to a floating
2343// point at a higher precision. Some of these cases are handled by FP_EXTEND,
2344// STORE promotion handlers.
2345SDValue DAGTypeLegalizer::PromoteFloatRes_BITCAST(SDNode *N) {
2346 EVT VT = N->getValueType(0);
2347 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
2348 // Input type isn't guaranteed to be a scalar int so bitcast if not. The
2349 // bitcast will be legalized further if necessary.
2350 EVT IVT = EVT::getIntegerVT(*DAG.getContext(),
2351 N->getOperand(0).getValueType().getSizeInBits());
2352 SDValue Cast = DAG.getBitcast(IVT, N->getOperand(0));
2353 return DAG.getNode(GetPromotionOpcode(VT, NVT), SDLoc(N), NVT, Cast);
2354}
2355
2356SDValue DAGTypeLegalizer::PromoteFloatRes_ConstantFP(SDNode *N) {
2357 ConstantFPSDNode *CFPNode = cast<ConstantFPSDNode>(N);
2358 EVT VT = N->getValueType(0);
2359 SDLoc DL(N);
2360
2361 // Get the (bit-cast) APInt of the APFloat and build an integer constant
2362 EVT IVT = EVT::getIntegerVT(*DAG.getContext(), VT.getSizeInBits());
2363 SDValue C = DAG.getConstant(CFPNode->getValueAPF().bitcastToAPInt(), DL,
2364 IVT);
2365
2366 // Convert the Constant to the desired FP type
2367 // FIXME We might be able to do the conversion during compilation and get rid
2368 // of it from the object code
2369 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
2370 return DAG.getNode(GetPromotionOpcode(VT, NVT), DL, NVT, C);
2371}
2372
2373// If the Index operand is a constant, try to redirect the extract operation to
2374// the correct legalized vector. If not, bit-convert the input vector to
2375// equivalent integer vector. Extract the element as an (bit-cast) integer
2376// value and convert it to the promoted type.
2377SDValue DAGTypeLegalizer::PromoteFloatRes_EXTRACT_VECTOR_ELT(SDNode *N) {
2378 SDLoc DL(N);
2379
2380 // If the index is constant, try to extract the value from the legalized
2381 // vector type.
2382 if (isa<ConstantSDNode>(N->getOperand(1))) {
2383 SDValue Vec = N->getOperand(0);
2384 SDValue Idx = N->getOperand(1);
2385 EVT VecVT = Vec->getValueType(0);
2386 EVT EltVT = VecVT.getVectorElementType();
2387
2388 uint64_t IdxVal = cast<ConstantSDNode>(Idx)->getZExtValue();
2389
2390 switch (getTypeAction(VecVT)) {
2391 default: break;
2392 case TargetLowering::TypeScalarizeVector: {
2393 SDValue Res = GetScalarizedVector(N->getOperand(0));
2394 ReplaceValueWith(SDValue(N, 0), Res);
2395 return SDValue();
2396 }
2397 case TargetLowering::TypeWidenVector: {
2398 Vec = GetWidenedVector(Vec);
2399 SDValue Res = DAG.getNode(N->getOpcode(), DL, EltVT, Vec, Idx);
2400 ReplaceValueWith(SDValue(N, 0), Res);
2401 return SDValue();
2402 }
2403 case TargetLowering::TypeSplitVector: {
2404 SDValue Lo, Hi;
2405 GetSplitVector(Vec, Lo, Hi);
2406
2407 uint64_t LoElts = Lo.getValueType().getVectorNumElements();
2408 SDValue Res;
2409 if (IdxVal < LoElts)
2410 Res = DAG.getNode(N->getOpcode(), DL, EltVT, Lo, Idx);
2411 else
2412 Res = DAG.getNode(N->getOpcode(), DL, EltVT, Hi,
2413 DAG.getConstant(IdxVal - LoElts, DL,
2414 Idx.getValueType()));
2415 ReplaceValueWith(SDValue(N, 0), Res);
2416 return SDValue();
2417 }
2418
2419 }
2420 }
2421
2422 // Bit-convert the input vector to the equivalent integer vector
2423 SDValue NewOp = BitConvertVectorToIntegerVector(N->getOperand(0));
2424 EVT IVT = NewOp.getValueType().getVectorElementType();
2425
2426 // Extract the element as an (bit-cast) integer value
2427 SDValue NewVal = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, IVT,
2428 NewOp, N->getOperand(1));
2429
2430 // Convert the element to the desired FP type
2431 EVT VT = N->getValueType(0);
2432 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
2433 return DAG.getNode(GetPromotionOpcode(VT, NVT), SDLoc(N), NVT, NewVal);
2434}
2435
2436// FCOPYSIGN(X, Y) returns the value of X with the sign of Y. If the result
2437// needs promotion, so does the argument X. Note that Y, if needed, will be
2438// handled during operand promotion.
2439SDValue DAGTypeLegalizer::PromoteFloatRes_FCOPYSIGN(SDNode *N) {
2440 EVT VT = N->getValueType(0);
2441 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
2442 SDValue Op0 = GetPromotedFloat(N->getOperand(0));
2443
2444 SDValue Op1 = N->getOperand(1);
2445
2446 return DAG.getNode(N->getOpcode(), SDLoc(N), NVT, Op0, Op1);
2447}
2448
2449// Unary operation where the result and the operand have PromoteFloat type
2450// action. Construct a new SDNode with the promoted float value of the old
2451// operand.
2452SDValue DAGTypeLegalizer::PromoteFloatRes_UnaryOp(SDNode *N) {
2453 EVT VT = N->getValueType(0);
2454 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
2455 SDValue Op = GetPromotedFloat(N->getOperand(0));
2456
2457 return DAG.getNode(N->getOpcode(), SDLoc(N), NVT, Op);
2458}
2459
2460// Binary operations where the result and both operands have PromoteFloat type
2461// action. Construct a new SDNode with the promoted float values of the old
2462// operands.
2463SDValue DAGTypeLegalizer::PromoteFloatRes_BinOp(SDNode *N) {
2464 EVT VT = N->getValueType(0);
2465 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
2466 SDValue Op0 = GetPromotedFloat(N->getOperand(0));
2467 SDValue Op1 = GetPromotedFloat(N->getOperand(1));
2468 return DAG.getNode(N->getOpcode(), SDLoc(N), NVT, Op0, Op1, N->getFlags());
2469}
2470
2471SDValue DAGTypeLegalizer::PromoteFloatRes_FMAD(SDNode *N) {
2472 EVT VT = N->getValueType(0);
2473 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
2474 SDValue Op0 = GetPromotedFloat(N->getOperand(0));
2475 SDValue Op1 = GetPromotedFloat(N->getOperand(1));
2476 SDValue Op2 = GetPromotedFloat(N->getOperand(2));
2477
2478 return DAG.getNode(N->getOpcode(), SDLoc(N), NVT, Op0, Op1, Op2);
2479}
2480
2481// Promote the Float (first) operand and retain the Integer (second) operand
2482SDValue DAGTypeLegalizer::PromoteFloatRes_FPOWI(SDNode *N) {
2483 EVT VT = N->getValueType(0);
2484 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
2485 SDValue Op0 = GetPromotedFloat(N->getOperand(0));
2486 SDValue Op1 = N->getOperand(1);
2487
2488 return DAG.getNode(N->getOpcode(), SDLoc(N), NVT, Op0, Op1);
2489}
2490
2491// Explicit operation to reduce precision. Reduce the value to half precision
2492// and promote it back to the legal type.
2493SDValue DAGTypeLegalizer::PromoteFloatRes_FP_ROUND(SDNode *N) {
2494 SDLoc DL(N);
2495
2496 SDValue Op = N->getOperand(0);
2497 EVT VT = N->getValueType(0);
2498 EVT OpVT = Op->getValueType(0);
2499 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0));
2500 EVT IVT = EVT::getIntegerVT(*DAG.getContext(), VT.getSizeInBits());
2501
2502 // Round promoted float to desired precision
2503 SDValue Round = DAG.getNode(GetPromotionOpcode(OpVT, VT), DL, IVT, Op);
2504 // Promote it back to the legal output type
2505 return DAG.getNode(GetPromotionOpcode(VT, NVT), DL, NVT, Round);
2506}
2507
2508SDValue DAGTypeLegalizer::PromoteFloatRes_LOAD(SDNode *N) {
2509 LoadSDNode *L = cast<LoadSDNode>(N);
2510 EVT VT = N->getValueType(0);
2511
2512 // Load the value as an integer value with the same number of bits.
2513 EVT IVT = EVT::getIntegerVT(*DAG.getContext(), VT.getSizeInBits());
2514 SDValue newL = DAG.getLoad(
2515 L->getAddressingMode(), L->getExtensionType(), IVT, SDLoc(N),
2516 L->getChain(), L->getBasePtr(), L->getOffset(), L->getPointerInfo(), IVT,
2517 L->getOriginalAlign(), L->getMemOperand()->getFlags(), L->getAAInfo());
2518 // Legalize the chain result by replacing uses of the old value chain with the
2519 // new one
2520 ReplaceValueWith(SDValue(N, 1), newL.getValue(1));
2521
2522 // Convert the integer value to the desired FP type
2523 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
2524 return DAG.getNode(GetPromotionOpcode(VT, NVT), SDLoc(N), NVT, newL);
2525}
2526
2527// Construct a new SELECT node with the promoted true- and false- values.
2528SDValue DAGTypeLegalizer::PromoteFloatRes_SELECT(SDNode *N) {
2529 SDValue TrueVal = GetPromotedFloat(N->getOperand(1));
2530 SDValue FalseVal = GetPromotedFloat(N->getOperand(2));
2531
2532 return DAG.getNode(ISD::SELECT, SDLoc(N), TrueVal->getValueType(0),
2533 N->getOperand(0), TrueVal, FalseVal);
2534}
2535
2536// Construct a new SELECT_CC node with the promoted true- and false- values.
2537// The operands used for comparison are promoted by PromoteFloatOp_SELECT_CC.
2538SDValue DAGTypeLegalizer::PromoteFloatRes_SELECT_CC(SDNode *N) {
2539 SDValue TrueVal = GetPromotedFloat(N->getOperand(2));
2540 SDValue FalseVal = GetPromotedFloat(N->getOperand(3));
2541
2542 return DAG.getNode(ISD::SELECT_CC, SDLoc(N),
2543 TrueVal.getNode()->getValueType(0), N->getOperand(0),
2544 N->getOperand(1), TrueVal, FalseVal, N->getOperand(4));
2545}
2546
2547// Construct a SDNode that transforms the SINT or UINT operand to the promoted
2548// float type.
2549SDValue DAGTypeLegalizer::PromoteFloatRes_XINT_TO_FP(SDNode *N) {
2550 SDLoc DL(N);
2551 EVT VT = N->getValueType(0);
2552 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
2553 SDValue NV = DAG.getNode(N->getOpcode(), DL, NVT, N->getOperand(0));
2554 // Round the value to the desired precision (that of the source type).
2555 return DAG.getNode(
2556 ISD::FP_EXTEND, DL, NVT,
2557 DAG.getNode(ISD::FP_ROUND, DL, VT, NV,
2558 DAG.getIntPtrConstant(0, DL, /*isTarget=*/true)));
2559}
2560
2561SDValue DAGTypeLegalizer::PromoteFloatRes_UNDEF(SDNode *N) {
2562 return DAG.getUNDEF(TLI.getTypeToTransformTo(*DAG.getContext(),
2563 N->getValueType(0)));
2564}
2565
2566SDValue DAGTypeLegalizer::PromoteFloatRes_VECREDUCE(SDNode *N) {
2567 // Expand and promote recursively.
2568 // TODO: This is non-optimal, but dealing with the concurrently happening
2569 // vector-legalization is non-trivial. We could do something similar to
2570 // PromoteFloatRes_EXTRACT_VECTOR_ELT here.
2571 ReplaceValueWith(SDValue(N, 0), TLI.expandVecReduce(N, DAG));
2572 return SDValue();
2573}
2574
2575SDValue DAGTypeLegalizer::PromoteFloatRes_VECREDUCE_SEQ(SDNode *N) {
2576 ReplaceValueWith(SDValue(N, 0), TLI.expandVecReduceSeq(N, DAG));
2577 return SDValue();
2578}
2579
2580SDValue DAGTypeLegalizer::BitcastToInt_ATOMIC_SWAP(SDNode *N) {
2581 EVT VT = N->getValueType(0);
2582
2583 AtomicSDNode *AM = cast<AtomicSDNode>(N);
2584 SDLoc SL(N);
2585
2586 SDValue CastVal = BitConvertToInteger(AM->getVal());
2587 EVT CastVT = CastVal.getValueType();
2588
2589 SDValue NewAtomic
2590 = DAG.getAtomic(ISD::ATOMIC_SWAP, SL, CastVT,
2591 DAG.getVTList(CastVT, MVT::Other),
2592 { AM->getChain(), AM->getBasePtr(), CastVal },
2593 AM->getMemOperand());
2594
2595 SDValue Result = NewAtomic;
2596
2597 if (getTypeAction(VT) == TargetLowering::TypePromoteFloat) {
2598 EVT NFPVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
2599 Result = DAG.getNode(GetPromotionOpcode(VT, NFPVT), SL, NFPVT,
2600 NewAtomic);
2601 }
2602
2603 // Legalize the chain result by replacing uses of the old value chain with the
2604 // new one
2605 ReplaceValueWith(SDValue(N, 1), NewAtomic.getValue(1));
2606
2607 return Result;
2608
2609}
2610
2611//===----------------------------------------------------------------------===//
2612// Half Result Soft Promotion
2613//===----------------------------------------------------------------------===//
2614
2615void DAGTypeLegalizer::SoftPromoteHalfResult(SDNode *N, unsigned ResNo) {
2616 LLVM_DEBUG(dbgs() << "Soft promote half result " << ResNo << ": ";
2617 N->dump(&DAG); dbgs() << "\n");
2618 SDValue R = SDValue();
2619
2620 // See if the target wants to custom expand this node.
2621 if (CustomLowerNode(N, N->getValueType(ResNo), true)) {
2622 LLVM_DEBUG(dbgs() << "Node has been custom expanded, done\n");
2623 return;
2624 }
2625
2626 switch (N->getOpcode()) {
2627 default:
2628#ifndef NDEBUG
2629 dbgs() << "SoftPromoteHalfResult #" << ResNo << ": ";
2630 N->dump(&DAG); dbgs() << "\n";
2631#endif
2632 llvm_unreachable("Do not know how to soft promote this operator's result!");
2633
2634 case ISD::BITCAST: R = SoftPromoteHalfRes_BITCAST(N); break;
2635 case ISD::ConstantFP: R = SoftPromoteHalfRes_ConstantFP(N); break;
2636 case ISD::EXTRACT_VECTOR_ELT:
2637 R = SoftPromoteHalfRes_EXTRACT_VECTOR_ELT(N); break;
2638 case ISD::FCOPYSIGN: R = SoftPromoteHalfRes_FCOPYSIGN(N); break;
2639 case ISD::STRICT_FP_ROUND:
2640 case ISD::FP_ROUND: R = SoftPromoteHalfRes_FP_ROUND(N); break;
2641
2642 // Unary FP Operations
2643 case ISD::FABS:
2644 case ISD::FCBRT:
2645 case ISD::FCEIL:
2646 case ISD::FCOS:
2647 case ISD::FEXP:
2648 case ISD::FEXP2:
2649 case ISD::FFLOOR:
2650 case ISD::FLOG:
2651 case ISD::FLOG2:
2652 case ISD::FLOG10:
2653 case ISD::FNEARBYINT:
2654 case ISD::FNEG:
2655 case ISD::FREEZE:
2656 case ISD::FRINT:
2657 case ISD::FROUND:
2658 case ISD::FROUNDEVEN:
2659 case ISD::FSIN:
2660 case ISD::FSQRT:
2661 case ISD::FTRUNC:
2662 case ISD::FCANONICALIZE: R = SoftPromoteHalfRes_UnaryOp(N); break;
2663
2664 // Binary FP Operations
2665 case ISD::FADD:
2666 case ISD::FDIV:
2667 case ISD::FMAXIMUM:
2668 case ISD::FMINIMUM:
2669 case ISD::FMAXNUM:
2670 case ISD::FMINNUM:
2671 case ISD::FMUL:
2672 case ISD::FPOW:
2673 case ISD::FREM:
2674 case ISD::FSUB: R = SoftPromoteHalfRes_BinOp(N); break;
2675
2676 case ISD::FMA: // FMA is same as FMAD
2677 case ISD::FMAD: R = SoftPromoteHalfRes_FMAD(N); break;
2678
2679 case ISD::FPOWI: R = SoftPromoteHalfRes_FPOWI(N); break;
2680
2681 case ISD::LOAD: R = SoftPromoteHalfRes_LOAD(N); break;
2682 case ISD::SELECT: R = SoftPromoteHalfRes_SELECT(N); break;
2683 case ISD::SELECT_CC: R = SoftPromoteHalfRes_SELECT_CC(N); break;
2684 case ISD::SINT_TO_FP:
2685 case ISD::UINT_TO_FP: R = SoftPromoteHalfRes_XINT_TO_FP(N); break;
2686 case ISD::UNDEF: R = SoftPromoteHalfRes_UNDEF(N); break;
2687 case ISD::ATOMIC_SWAP: R = BitcastToInt_ATOMIC_SWAP(N); break;
2688 case ISD::VECREDUCE_FADD:
2689 case ISD::VECREDUCE_FMUL:
2690 case ISD::VECREDUCE_FMIN:
2691 case ISD::VECREDUCE_FMAX:
2692 R = SoftPromoteHalfRes_VECREDUCE(N);
2693 break;
2694 case ISD::VECREDUCE_SEQ_FADD:
2695 case ISD::VECREDUCE_SEQ_FMUL:
2696 R = SoftPromoteHalfRes_VECREDUCE_SEQ(N);
2697 break;
2698 }
2699
2700 if (R.getNode())
2701 SetSoftPromotedHalf(SDValue(N, ResNo), R);
2702}
2703
2704SDValue DAGTypeLegalizer::SoftPromoteHalfRes_BITCAST(SDNode *N) {
2705 return BitConvertToInteger(N->getOperand(0));
2706}
2707
2708SDValue DAGTypeLegalizer::SoftPromoteHalfRes_ConstantFP(SDNode *N) {
2709 ConstantFPSDNode *CN = cast<ConstantFPSDNode>(N);
2710
2711 // Get the (bit-cast) APInt of the APFloat and build an integer constant
2712 return DAG.getConstant(CN->getValueAPF().bitcastToAPInt(), SDLoc(CN),
2713 MVT::i16);
2714}
2715
2716SDValue DAGTypeLegalizer::SoftPromoteHalfRes_EXTRACT_VECTOR_ELT(SDNode *N) {
2717 SDValue NewOp = BitConvertVectorToIntegerVector(N->getOperand(0));
2718 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SDLoc(N),
2719 NewOp.getValueType().getVectorElementType(), NewOp,
2720 N->getOperand(1));
2721}
2722
2723SDValue DAGTypeLegalizer::SoftPromoteHalfRes_FCOPYSIGN(SDNode *N) {
2724 SDValue LHS = GetSoftPromotedHalf(N->getOperand(0));
2725 SDValue RHS = BitConvertToInteger(N->getOperand(1));
2726 SDLoc dl(N);
2727
2728 EVT LVT = LHS.getValueType();
2729 EVT RVT = RHS.getValueType();
2730
2731 unsigned LSize = LVT.getSizeInBits();
2732 unsigned RSize = RVT.getSizeInBits();
2733
2734 // First get the sign bit of second operand.
2735 SDValue SignBit = DAG.getNode(
2736 ISD::SHL, dl, RVT, DAG.getConstant(1, dl, RVT),
2737 DAG.getConstant(RSize - 1, dl,
2738 TLI.getShiftAmountTy(RVT, DAG.getDataLayout())));
2739 SignBit = DAG.getNode(ISD::AND, dl, RVT, RHS, SignBit);
2740
2741 // Shift right or sign-extend it if the two operands have different types.
2742 int SizeDiff = RVT.getSizeInBits() - LVT.getSizeInBits();
2743 if (SizeDiff > 0) {
2744 SignBit =
2745 DAG.getNode(ISD::SRL, dl, RVT, SignBit,
2746 DAG.getConstant(SizeDiff, dl,
2747 TLI.getShiftAmountTy(SignBit.getValueType(),
2748 DAG.getDataLayout())));
2749 SignBit = DAG.getNode(ISD::TRUNCATE, dl, LVT, SignBit);
2750 } else if (SizeDiff < 0) {
2751 SignBit = DAG.getNode(ISD::ANY_EXTEND, dl, LVT, SignBit);
2752 SignBit =
2753 DAG.getNode(ISD::SHL, dl, LVT, SignBit,
2754 DAG.getConstant(-SizeDiff, dl,
2755 TLI.getShiftAmountTy(SignBit.getValueType(),
2756 DAG.getDataLayout())));
2757 }
2758
2759 // Clear the sign bit of the first operand.
2760 SDValue Mask = DAG.getNode(
2761 ISD::SHL, dl, LVT, DAG.getConstant(1, dl, LVT),
2762 DAG.getConstant(LSize - 1, dl,
2763 TLI.getShiftAmountTy(LVT, DAG.getDataLayout())));
2764 Mask = DAG.getNode(ISD::SUB, dl, LVT, Mask, DAG.getConstant(1, dl, LVT));
2765 LHS = DAG.getNode(ISD::AND, dl, LVT, LHS, Mask);
2766
2767 // Or the value with the sign bit.
2768 return DAG.getNode(ISD::OR, dl, LVT, LHS, SignBit);
2769}
2770
2771SDValue DAGTypeLegalizer::SoftPromoteHalfRes_FMAD(SDNode *N) {
2772 EVT OVT = N->getValueType(0);
2773 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), OVT);
2774 SDValue Op0 = GetSoftPromotedHalf(N->getOperand(0));
2775 SDValue Op1 = GetSoftPromotedHalf(N->getOperand(1));
2776 SDValue Op2 = GetSoftPromotedHalf(N->getOperand(2));
2777 SDLoc dl(N);
2778
2779 // Promote to the larger FP type.
2780 auto PromotionOpcode = GetPromotionOpcode(OVT, NVT);
2781 Op0 = DAG.getNode(PromotionOpcode, dl, NVT, Op0);
2782 Op1 = DAG.getNode(PromotionOpcode, dl, NVT, Op1);
2783 Op2 = DAG.getNode(PromotionOpcode, dl, NVT, Op2);
2784
2785 SDValue Res = DAG.getNode(N->getOpcode(), dl, NVT, Op0, Op1, Op2);
2786
2787 // Convert back to FP16 as an integer.
2788 return DAG.getNode(GetPromotionOpcode(NVT, OVT), dl, MVT::i16, Res);
2789}
2790
2791SDValue DAGTypeLegalizer::SoftPromoteHalfRes_FPOWI(SDNode *N) {
2792 EVT OVT = N->getValueType(0);
2793 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), OVT);
2794 SDValue Op0 = GetSoftPromotedHalf(N->getOperand(0));
2795 SDValue Op1 = N->getOperand(1);
2796 SDLoc dl(N);
2797
2798 // Promote to the larger FP type.
2799 Op0 = DAG.getNode(GetPromotionOpcode(OVT, NVT), dl, NVT, Op0);
2800
2801 SDValue Res = DAG.getNode(N->getOpcode(), dl, NVT, Op0, Op1);
2802
2803 // Convert back to FP16 as an integer.
2804 return DAG.getNode(GetPromotionOpcode(NVT, OVT), dl, MVT::i16, Res);
2805}
2806
2807SDValue DAGTypeLegalizer::SoftPromoteHalfRes_FP_ROUND(SDNode *N) {
2808 EVT RVT = N->getValueType(0);
2809 EVT SVT = N->getOperand(0).getValueType();
2810
2811 if (N->isStrictFPOpcode()) {
2812 assert(RVT == MVT::f16);
2813 SDValue Res =
2814 DAG.getNode(ISD::STRICT_FP_TO_FP16, SDLoc(N), {MVT::i16, MVT::Other},
2815 {N->getOperand(0), N->getOperand(1)});
2816 ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
2817 return Res;
2818 }
2819
2820 return DAG.getNode(GetPromotionOpcode(SVT, RVT), SDLoc(N), MVT::i16,
2821 N->getOperand(0));
2822}
2823
2824SDValue DAGTypeLegalizer::SoftPromoteHalfRes_LOAD(SDNode *N) {
2825 LoadSDNode *L = cast<LoadSDNode>(N);
2826
2827 // Load the value as an integer value with the same number of bits.
2828 assert(L->getExtensionType() == ISD::NON_EXTLOAD && "Unexpected extension!");
2829 SDValue NewL =
2830 DAG.getLoad(L->getAddressingMode(), L->getExtensionType(), MVT::i16,
2831 SDLoc(N), L->getChain(), L->getBasePtr(), L->getOffset(),
2832 L->getPointerInfo(), MVT::i16, L->getOriginalAlign(),
2833 L->getMemOperand()->getFlags(), L->getAAInfo());
2834 // Legalize the chain result by replacing uses of the old value chain with the
2835 // new one
2836 ReplaceValueWith(SDValue(N, 1), NewL.getValue(1));
2837 return NewL;
2838}
2839
2840SDValue DAGTypeLegalizer::SoftPromoteHalfRes_SELECT(SDNode *N) {
2841 SDValue Op1 = GetSoftPromotedHalf(N->getOperand(1));
2842 SDValue Op2 = GetSoftPromotedHalf(N->getOperand(2));
2843 return DAG.getSelect(SDLoc(N), Op1.getValueType(), N->getOperand(0), Op1,
2844 Op2);
2845}
2846
2847SDValue DAGTypeLegalizer::SoftPromoteHalfRes_SELECT_CC(SDNode *N) {
2848 SDValue Op2 = GetSoftPromotedHalf(N->getOperand(2));
2849 SDValue Op3 = GetSoftPromotedHalf(N->getOperand(3));
2850 return DAG.getNode(ISD::SELECT_CC, SDLoc(N), Op2.getValueType(),
2851 N->getOperand(0), N->getOperand(1), Op2, Op3,
2852 N->getOperand(4));
2853}
2854
2855SDValue DAGTypeLegalizer::SoftPromoteHalfRes_XINT_TO_FP(SDNode *N) {
2856 EVT OVT = N->getValueType(0);
2857 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), OVT);
2858 SDLoc dl(N);
2859
2860 SDValue Res = DAG.getNode(N->getOpcode(), dl, NVT, N->getOperand(0));
2861
2862 // Round the value to the softened type.
2863 return DAG.getNode(GetPromotionOpcode(NVT, OVT), dl, MVT::i16, Res);
2864}
2865
2866SDValue DAGTypeLegalizer::SoftPromoteHalfRes_UNDEF(SDNode *N) {
2867 return DAG.getUNDEF(MVT::i16);
2868}
2869
2870SDValue DAGTypeLegalizer::SoftPromoteHalfRes_UnaryOp(SDNode *N) {
2871 EVT OVT = N->getValueType(0);
2872 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), OVT);
2873 SDValue Op = GetSoftPromotedHalf(N->getOperand(0));
2874 SDLoc dl(N);
2875
2876 // Promote to the larger FP type.
2877 Op = DAG.getNode(GetPromotionOpcode(OVT, NVT), dl, NVT, Op);
2878
2879 SDValue Res = DAG.getNode(N->getOpcode(), dl, NVT, Op);
2880
2881 // Convert back to FP16 as an integer.
2882 return DAG.getNode(GetPromotionOpcode(NVT, OVT), dl, MVT::i16, Res);
2883}
2884
2885SDValue DAGTypeLegalizer::SoftPromoteHalfRes_BinOp(SDNode *N) {
2886 EVT OVT = N->getValueType(0);
2887 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), OVT);
2888 SDValue Op0 = GetSoftPromotedHalf(N->getOperand(0));
2889 SDValue Op1 = GetSoftPromotedHalf(N->getOperand(1));
2890 SDLoc dl(N);
2891
2892 // Promote to the larger FP type.
2893 auto PromotionOpcode = GetPromotionOpcode(OVT, NVT);
2894 Op0 = DAG.getNode(PromotionOpcode, dl, NVT, Op0);
2895 Op1 = DAG.getNode(PromotionOpcode, dl, NVT, Op1);
2896
2897 SDValue Res = DAG.getNode(N->getOpcode(), dl, NVT, Op0, Op1);
2898
2899 // Convert back to FP16 as an integer.
2900 return DAG.getNode(GetPromotionOpcode(NVT, OVT), dl, MVT::i16, Res);
2901}
2902
2903SDValue DAGTypeLegalizer::SoftPromoteHalfRes_VECREDUCE(SDNode *N) {
2904 // Expand and soften recursively.
2905 ReplaceValueWith(SDValue(N, 0), TLI.expandVecReduce(N, DAG));
2906 return SDValue();
2907}
2908
2909SDValue DAGTypeLegalizer::SoftPromoteHalfRes_VECREDUCE_SEQ(SDNode *N) {
2910 // Expand and soften.
2911 ReplaceValueWith(SDValue(N, 0), TLI.expandVecReduceSeq(N, DAG));
2912 return SDValue();
2913}
2914
2915//===----------------------------------------------------------------------===//
2916// Half Operand Soft Promotion
2917//===----------------------------------------------------------------------===//
2918
2919bool DAGTypeLegalizer::SoftPromoteHalfOperand(SDNode *N, unsigned OpNo) {
2920 LLVM_DEBUG(dbgs() << "Soft promote half operand " << OpNo << ": ";
2921 N->dump(&DAG); dbgs() << "\n");
2922 SDValue Res = SDValue();
2923
2924 if (CustomLowerNode(N, N->getOperand(OpNo).getValueType(), false)) {
2925 LLVM_DEBUG(dbgs() << "Node has been custom lowered, done\n");
2926 return false;
2927 }
2928
2929 // Nodes that use a promotion-requiring floating point operand, but doesn't
2930 // produce a soft promotion-requiring floating point result, need to be
2931 // legalized to use the soft promoted float operand. Nodes that produce at
2932 // least one soft promotion-requiring floating point result have their
2933 // operands legalized as a part of PromoteFloatResult.
2934 switch (N->getOpcode()) {
2935 default:
2936 #ifndef NDEBUG
2937 dbgs() << "SoftPromoteHalfOperand Op #" << OpNo << ": ";
2938 N->dump(&DAG); dbgs() << "\n";
2939 #endif
2940 llvm_unreachable("Do not know how to soft promote this operator's operand!");
2941
2942 case ISD::BITCAST: Res = SoftPromoteHalfOp_BITCAST(N); break;
2943 case ISD::FCOPYSIGN: Res = SoftPromoteHalfOp_FCOPYSIGN(N, OpNo); break;
2944 case ISD::FP_TO_SINT:
2945 case ISD::FP_TO_UINT: Res = SoftPromoteHalfOp_FP_TO_XINT(N); break;
2946 case ISD::FP_TO_SINT_SAT:
2947 case ISD::FP_TO_UINT_SAT:
2948 Res = SoftPromoteHalfOp_FP_TO_XINT_SAT(N); break;
2949 case ISD::STRICT_FP_EXTEND:
2950 case ISD::FP_EXTEND: Res = SoftPromoteHalfOp_FP_EXTEND(N); break;
2951 case ISD::SELECT_CC: Res = SoftPromoteHalfOp_SELECT_CC(N, OpNo); break;
2952 case ISD::SETCC: Res = SoftPromoteHalfOp_SETCC(N); break;
2953 case ISD::STORE: Res = SoftPromoteHalfOp_STORE(N, OpNo); break;
2954 case ISD::STACKMAP:
2955 Res = SoftPromoteHalfOp_STACKMAP(N, OpNo);
2956 break;
2957 case ISD::PATCHPOINT:
2958 Res = SoftPromoteHalfOp_PATCHPOINT(N, OpNo);
2959 break;
2960 }
2961
2962 if (!Res.getNode())
2963 return false;
2964
2965 assert(Res.getNode() != N && "Expected a new node!");
2966
2967 assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 &&
2968 "Invalid operand expansion");
2969
2970 ReplaceValueWith(SDValue(N, 0), Res);
2971 return false;
2972}
2973
2974SDValue DAGTypeLegalizer::SoftPromoteHalfOp_BITCAST(SDNode *N) {
2975 SDValue Op0 = GetSoftPromotedHalf(N->getOperand(0));
2976
2977 return DAG.getNode(ISD::BITCAST, SDLoc(N), N->getValueType(0), Op0);
2978}
2979
2980SDValue DAGTypeLegalizer::SoftPromoteHalfOp_FCOPYSIGN(SDNode *N,
2981 unsigned OpNo) {
2982 assert(OpNo == 1 && "Only Operand 1 must need promotion here");
2983 SDValue Op1 = N->getOperand(1);
2984 EVT RVT = Op1.getValueType();
2985 SDLoc dl(N);
2986
2987 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), Op1.getValueType());
2988
2989 Op1 = GetSoftPromotedHalf(Op1);
2990 Op1 = DAG.getNode(GetPromotionOpcode(RVT, NVT), dl, NVT, Op1);
2991
2992 return DAG.getNode(N->getOpcode(), dl, N->getValueType(0), N->getOperand(0),
2993 Op1);
2994}
2995
2996SDValue DAGTypeLegalizer::SoftPromoteHalfOp_FP_EXTEND(SDNode *N) {
2997 EVT RVT = N->getValueType(0);
2998 bool IsStrict = N->isStrictFPOpcode();
2999 SDValue Op = N->getOperand(IsStrict ? 1 : 0);
3000 EVT SVT = Op.getValueType();
3001 Op = GetSoftPromotedHalf(N->getOperand(IsStrict ? 1 : 0));
3002
3003 if (IsStrict) {
3004 assert(SVT == MVT::f16);
3005 SDValue Res =
3006 DAG.getNode(ISD::STRICT_FP16_TO_FP, SDLoc(N),
3007 {N->getValueType(0), MVT::Other}, {N->getOperand(0), Op});
3008 ReplaceValueWith(SDValue(N, 1), Res.getValue(1));
3009 ReplaceValueWith(SDValue(N, 0), Res);
3010 return SDValue();
3011 }
3012
3013 return DAG.getNode(GetPromotionOpcode(SVT, RVT), SDLoc(N), RVT, Op);
3014}
3015
3016SDValue DAGTypeLegalizer::SoftPromoteHalfOp_FP_TO_XINT(SDNode *N) {
3017 EVT RVT = N->getValueType(0);
3018 SDValue Op = N->getOperand(0);
3019 EVT SVT = Op.getValueType();
3020 SDLoc dl(N);
3021
3022 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), Op.getValueType());
3023
3024 Op = GetSoftPromotedHalf(Op);
3025
3026 SDValue Res = DAG.getNode(GetPromotionOpcode(SVT, RVT), dl, NVT, Op);
3027
3028 return DAG.getNode(N->getOpcode(), dl, N->getValueType(0), Res);
3029}
3030
3031SDValue DAGTypeLegalizer::SoftPromoteHalfOp_FP_TO_XINT_SAT(SDNode *N) {
3032 EVT RVT = N->getValueType(0);
3033 SDValue Op = N->getOperand(0);
3034 EVT SVT = Op.getValueType();
3035 SDLoc dl(N);
3036
3037 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), Op.getValueType());
3038
3039 Op = GetSoftPromotedHalf(Op);
3040
3041 SDValue Res = DAG.getNode(GetPromotionOpcode(SVT, RVT), dl, NVT, Op);
3042
3043 return DAG.getNode(N->getOpcode(), dl, N->getValueType(0), Res,
3044 N->getOperand(1));
3045}
3046
3047SDValue DAGTypeLegalizer::SoftPromoteHalfOp_SELECT_CC(SDNode *N,
3048 unsigned OpNo) {
3049 assert(OpNo == 0 && "Can only soften the comparison values");
3050 SDValue Op0 = N->getOperand(0);
3051 SDValue Op1 = N->getOperand(1);
3052 SDLoc dl(N);
3053
3054 EVT SVT = Op0.getValueType();
3055 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), SVT);
3056
3057 Op0 = GetSoftPromotedHalf(Op0);
3058 Op1 = GetSoftPromotedHalf(Op1);
3059
3060 // Promote to the larger FP type.
3061 auto PromotionOpcode = GetPromotionOpcode(SVT, NVT);
3062 Op0 = DAG.getNode(PromotionOpcode, dl, NVT, Op0);
3063 Op1 = DAG.getNode(PromotionOpcode, dl, NVT, Op1);
3064
3065 return DAG.getNode(ISD::SELECT_CC, SDLoc(N), N->getValueType(0), Op0, Op1,
3066 N->getOperand(2), N->getOperand(3), N->getOperand(4));
3067}
3068
3069SDValue DAGTypeLegalizer::SoftPromoteHalfOp_SETCC(SDNode *N) {
3070 SDValue Op0 = N->getOperand(0);
3071 SDValue Op1 = N->getOperand(1);
3072 ISD::CondCode CCCode = cast<CondCodeSDNode>(N->getOperand(2))->get();
3073 SDLoc dl(N);
3074
3075 EVT SVT = Op0.getValueType();
3076 EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), Op0.getValueType());
3077
3078 Op0 = GetSoftPromotedHalf(Op0);
3079 Op1 = GetSoftPromotedHalf(Op1);
3080
3081 // Promote to the larger FP type.
3082 auto PromotionOpcode = GetPromotionOpcode(SVT, NVT);
3083 Op0 = DAG.getNode(PromotionOpcode, dl, NVT, Op0);
3084 Op1 = DAG.getNode(PromotionOpcode, dl, NVT, Op1);
3085
3086 return DAG.getSetCC(SDLoc(N), N->getValueType(0), Op0, Op1, CCCode);
3087}
3088
3089SDValue DAGTypeLegalizer::SoftPromoteHalfOp_STORE(SDNode *N, unsigned OpNo) {
3090 assert(OpNo == 1 && "Can only soften the stored value!");
3091 StoreSDNode *ST = cast<StoreSDNode>(N);
3092 SDValue Val = ST->getValue();
3093 SDLoc dl(N);
3094
3095 assert(!ST->isTruncatingStore() && "Unexpected truncating store.");
3096 SDValue Promoted = GetSoftPromotedHalf(Val);
3097 return DAG.getStore(ST->getChain(), dl, Promoted, ST->getBasePtr(),
3098 ST->getMemOperand());
3099}
3100
3101SDValue DAGTypeLegalizer::SoftPromoteHalfOp_STACKMAP(SDNode *N, unsigned OpNo) {
3102 assert(OpNo > 1); // Because the first two arguments are guaranteed legal.
3103 SmallVector<SDValue> NewOps(N->ops().begin(), N->ops().end());
3104 SDValue Op = N->getOperand(OpNo);
3105 NewOps[OpNo] = GetSoftPromotedHalf(Op);
3106 SDValue NewNode =
3107 DAG.getNode(N->getOpcode(), SDLoc(N), N->getVTList(), NewOps);
3108
3109 for (unsigned ResNum = 0; ResNum < N->getNumValues(); ResNum++)
3110 ReplaceValueWith(SDValue(N, ResNum), NewNode.getValue(ResNum));
3111
3112 return SDValue(); // Signal that we replaced the node ourselves.
3113}
3114
3115SDValue DAGTypeLegalizer::SoftPromoteHalfOp_PATCHPOINT(SDNode *N,
3116 unsigned OpNo) {
3117 assert(OpNo >= 7);
3118 SmallVector<SDValue> NewOps(N->ops().begin(), N->ops().end());
3119 SDValue Op = N->getOperand(OpNo);
3120 NewOps[OpNo] = GetSoftPromotedHalf(Op);
3121 SDValue NewNode =
3122 DAG.getNode(N->getOpcode(), SDLoc(N), N->getVTList(), NewOps);
3123
3124 for (unsigned ResNum = 0; ResNum < N->getNumValues(); ResNum++)
3125 ReplaceValueWith(SDValue(N, ResNum), NewNode.getValue(ResNum));
3126
3127 return SDValue(); // Signal that we replaced the node ourselves.
3128}
DL
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
Definition: AArch64SLSHardening.cpp:76
Idx
Returns the sub type a function will return at a given Idx Should correspond to the result type of an ExtractValue instruction executed with just that one unsigned Idx
Definition: DeadArgumentElimination.cpp:352
LLVM_DEBUG
#define LLVM_DEBUG(X)
Definition: Debug.h:101
Size
uint64_t Size
Definition: ELFObjHandler.cpp:81
isSigned
static bool isSigned(unsigned int Opcode)
Definition: ExpandLargeDivRem.cpp:52
findFPToIntLibcall
static RTLIB::Libcall findFPToIntLibcall(EVT SrcVT, EVT RetVT, EVT &Promoted, bool Signed)
Definition: LegalizeFloatTypes.cpp:969
GetFPLibCall
static RTLIB::Libcall GetFPLibCall(EVT VT, RTLIB::Libcall Call_F32, RTLIB::Libcall Call_F64, RTLIB::Libcall Call_F80, RTLIB::Libcall Call_F128, RTLIB::Libcall Call_PPCF128)
GetFPLibCall - Return the right libcall for the given floating point type.
Definition: LegalizeFloatTypes.cpp:32
GetPromotionOpcode
static ISD::NodeType GetPromotionOpcode(EVT OpVT, EVT RetVT)
Definition: LegalizeFloatTypes.cpp:2095
assert
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
Flags
@ Flags
Definition: TextStubV5.cpp:93
RHS
Value * RHS
Definition: X86PartialReduction.cpp:76
LHS
Value * LHS
Definition: X86PartialReduction.cpp:75
llvm::APFloat::bitcastToAPInt
APInt bitcastToAPInt() const
Definition: APFloat.h:1184
llvm::APInt
Class for arbitrary precision integers.
Definition: APInt.h:75
llvm::APInt::getAllOnes
static APInt getAllOnes(unsigned numBits)
Return an APInt of a specified width with all bits set.
Definition: APInt.h:214
llvm::APInt::clearBit
void clearBit(unsigned BitPosition)
Set a given bit to 0.
Definition: APInt.h:1389
llvm::APInt::getSignMask
static APInt getSignMask(unsigned BitWidth)
Get the SignMask for a specific bit width.
Definition: APInt.h:209
llvm::APInt::getRawData
const uint64_t * getRawData() const
This function returns a pointer to the internal storage of the APInt.
Definition: APInt.h:557
llvm::ArrayRef
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
Definition: ArrayRef.h:41
llvm::AtomicSDNode
This is an SDNode representing atomic operations.
Definition: SelectionDAGNodes.h:1444
llvm::AtomicSDNode::getVal
const SDValue & getVal() const
Definition: SelectionDAGNodes.h:1454
llvm::ConstantFPSDNode::getValueAPF
const APFloat & getValueAPF() const
Definition: SelectionDAGNodes.h:1646
llvm::DAGTypeLegalizer::NewNode
@ NewNode
This is a new node, not before seen, that was created in the process of legalizing some other node.
Definition: LegalizeTypes.h:42
llvm::DataLayout::isBigEndian
bool isBigEndian() const
Definition: DataLayout.h:239
llvm::LLVMContext::emitError
void emitError(uint64_t LocCookie, const Twine &ErrorStr)
emitError - Emit an error message to the currently installed error handler with optional location inf...
Definition: LLVMContext.cpp:271
llvm::LoadSDNode
This class is used to represent ISD::LOAD nodes.
Definition: SelectionDAGNodes.h:2352
llvm::MVT::SimpleTy
SimpleValueType SimpleTy
Definition: MachineValueType.h:58
llvm::MachineMemOperand::MODereferenceable
@ MODereferenceable
The memory access is dereferenceable (i.e., doesn't trap).
Definition: MachineMemOperand.h:141
llvm::MachineMemOperand::MOInvariant
@ MOInvariant
The memory access always returns the same value (or traps).
Definition: MachineMemOperand.h:143
llvm::MemSDNode::getMemOperand
MachineMemOperand * getMemOperand() const
Return a MachineMemOperand object describing the memory reference performed by operation.
Definition: SelectionDAGNodes.h:1359
llvm::SDLoc
Wrapper class for IR location info (IR ordering and DebugLoc) to be passed into SDNode creation funct...
Definition: SelectionDAGNodes.h:1106
llvm::SDNode
Represents one node in the SelectionDAG.
Definition: SelectionDAGNodes.h:463
llvm::SDNode::getNumValues
unsigned getNumValues() const
Return the number of values defined/returned by this operator.
Definition: SelectionDAGNodes.h:983
llvm::SDNode::getValueType
EVT getValueType(unsigned ResNo) const
Return the type of a specified result.
Definition: SelectionDAGNodes.h:986
llvm::SDValue
Unlike LLVM values, Selection DAG nodes may return multiple values as the result of a computation.
Definition: SelectionDAGNodes.h:145
llvm::SDValue::getNode
SDNode * getNode() const
get the SDNode which holds the desired result
Definition: SelectionDAGNodes.h:159
llvm::SDValue::getValue
SDValue getValue(unsigned R) const
Definition: SelectionDAGNodes.h:179
llvm::SDValue::getValueType
EVT getValueType() const
Return the ValueType of the referenced return value.
Definition: SelectionDAGNodes.h:1141
llvm::SelectionDAG::getExtLoad
SDValue getExtLoad(ISD::LoadExtType ExtType, const SDLoc &dl, EVT VT, SDValue Chain, SDValue Ptr, MachinePointerInfo PtrInfo, EVT MemVT, MaybeAlign Alignment=MaybeAlign(), MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes())
Definition: SelectionDAG.cpp:8246
llvm::SelectionDAG::getSelect
SDValue getSelect(const SDLoc &DL, EVT VT, SDValue Cond, SDValue LHS, SDValue RHS)
Helper function to make it easier to build Select's if you just have operands and don't want to check...
Definition: SelectionDAG.h:1199
llvm::SelectionDAG::getVTList
SDVTList getVTList(EVT VT)
Return an SDVTList that represents the list of values specified.
Definition: SelectionDAG.cpp:9619
llvm::SelectionDAG::getSetCC
SDValue getSetCC(const SDLoc &DL, EVT VT, SDValue LHS, SDValue RHS, ISD::CondCode Cond, SDValue Chain=SDValue(), bool IsSignaling=false)
Helper function to make it easier to build SetCC's if you just have an ISD::CondCode instead of an SD...
Definition: SelectionDAG.h:1170
llvm::SelectionDAG::getConstantFP
SDValue getConstantFP(double Val, const SDLoc &DL, EVT VT, bool isTarget=false)
Create a ConstantFPSDNode wrapping a constant value.
Definition: SelectionDAG.cpp:1714
llvm::SelectionDAG::getLoad
SDValue getLoad(EVT VT, const SDLoc &dl, SDValue Chain, SDValue Ptr, MachinePointerInfo PtrInfo, MaybeAlign Alignment=MaybeAlign(), MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes(), const MDNode *Ranges=nullptr)
Loads are not normal binary operators: their result type is not determined by their operands,...
Definition: SelectionDAG.cpp:8229
llvm::SelectionDAG::getAtomic
SDValue getAtomic(unsigned Opcode, const SDLoc &dl, EVT MemVT, SDValue Chain, SDValue Ptr, SDValue Val, MachineMemOperand *MMO)
Gets a node for an atomic op, produces result (if relevant) and chain and takes 2 operands.
Definition: SelectionDAG.cpp:7950
llvm::SelectionDAG::getUNDEF
SDValue getUNDEF(EVT VT)
Return an UNDEF node. UNDEF does not have a useful SDLoc.
Definition: SelectionDAG.h:1061
llvm::SelectionDAG::getBitcast
SDValue getBitcast(EVT VT, SDValue V)
Return a bitcast using the SDLoc of the value operand, and casting to the provided type.
Definition: SelectionDAG.cpp:2280
llvm::SelectionDAG::getDataLayout
const DataLayout & getDataLayout() const
Definition: SelectionDAG.h:472
llvm::SelectionDAG::getConstant
SDValue getConstant(uint64_t Val, const SDLoc &DL, EVT VT, bool isTarget=false, bool isOpaque=false)
Create a ConstantSDNode wrapping a constant value.
Definition: SelectionDAG.cpp:1540
llvm::SelectionDAG::getVAArg
SDValue getVAArg(EVT VT, const SDLoc &dl, SDValue Chain, SDValue Ptr, SDValue SV, unsigned Align)
VAArg produces a result and token chain, and takes a pointer and a source value as input.
Definition: SelectionDAG.cpp:9307
llvm::SelectionDAG::getTruncStore
SDValue getTruncStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr, MachinePointerInfo PtrInfo, EVT SVT, Align Alignment, MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes())
Definition: SelectionDAG.cpp:8331
llvm::SelectionDAG::getStore
SDValue getStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr, MachinePointerInfo PtrInfo, Align Alignment, MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes())
Helper function to build ISD::STORE nodes.
Definition: SelectionDAG.cpp:8279
llvm::SelectionDAG::EVTToAPFloatSemantics
static const fltSemantics & EVTToAPFloatSemantics(EVT VT)
Returns an APFloat semantics tag appropriate for the given type.
Definition: SelectionDAG.h:1849
llvm::SelectionDAG::getSelectCC
SDValue getSelectCC(const SDLoc &DL, SDValue LHS, SDValue RHS, SDValue True, SDValue False, ISD::CondCode Cond)
Helper function to make it easier to build SelectCC's if you just have an ISD::CondCode instead of an...
Definition: SelectionDAG.h:1209
llvm::SelectionDAG::getIntPtrConstant
SDValue getIntPtrConstant(uint64_t Val, const SDLoc &DL, bool isTarget=false)
Definition: SelectionDAG.cpp:1660
llvm::SelectionDAG::getNode
SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT, ArrayRef< SDUse > Ops)
Gets or creates the specified node.
Definition: SelectionDAG.cpp:9313
llvm::SelectionDAG::getLibInfo
const TargetLibraryInfo & getLibInfo() const
Definition: SelectionDAG.h:479
llvm::SelectionDAG::getCondCode
SDValue getCondCode(ISD::CondCode Cond)
Definition: SelectionDAG.cpp:1925
llvm::SelectionDAG::getContext
LLVMContext * getContext() const
Definition: SelectionDAG.h:485
llvm::SelectionDAG::getShiftAmountConstant
SDValue getShiftAmountConstant(uint64_t Val, EVT VT, const SDLoc &DL, bool LegalTypes=true)
Definition: SelectionDAG.cpp:1665
llvm::SelectionDAG::UpdateNodeOperands
SDNode * UpdateNodeOperands(SDNode *N, SDValue Op)
Mutate the specified node in-place to have the specified operands.
Definition: SelectionDAG.cpp:9709
llvm::SelectionDAG::getEntryNode
SDValue getEntryNode() const
Return the token chain corresponding to the entry of the function.
Definition: SelectionDAG.h:554
llvm::SmallVector
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
Definition: SmallVector.h:1200
llvm::StoreSDNode
This class is used to represent ISD::STORE nodes.
Definition: SelectionDAGNodes.h:2380
llvm::TargetLibraryInfo::getIntSize
unsigned getIntSize() const
Get size of a C-level int or unsigned int, in bits.
Definition: TargetLibraryInfo.h:511
llvm::TargetLoweringBase::getShiftAmountTy
EVT getShiftAmountTy(EVT LHSTy, const DataLayout &DL, bool LegalTypes=true) const
Returns the type for the shift amount of a shift opcode.
Definition: TargetLoweringBase.cpp:919
llvm::TargetLoweringBase::getTypeToTransformTo
virtual EVT getTypeToTransformTo(LLVMContext &Context, EVT VT) const
For types supported by the target, this is an identity function.
Definition: TargetLowering.h:1030
llvm::TargetLoweringBase::getLibcallName
const char * getLibcallName(RTLIB::Libcall Call) const
Get the libcall routine name for the specified libcall.
Definition: TargetLowering.h:3219
llvm::TargetLowering::softenSetCCOperands
void softenSetCCOperands(SelectionDAG &DAG, EVT VT, SDValue &NewLHS, SDValue &NewRHS, ISD::CondCode &CCCode, const SDLoc &DL, const SDValue OldLHS, const SDValue OldRHS) const
Soften the operands of a comparison.
Definition: TargetLowering.cpp:289
llvm::TargetLowering::makeLibCall
std::pair< SDValue, SDValue > makeLibCall(SelectionDAG &DAG, RTLIB::Libcall LC, EVT RetVT, ArrayRef< SDValue > Ops, MakeLibCallOptions CallOptions, const SDLoc &dl, SDValue Chain=SDValue()) const
Returns a pair of (return value, chain).
Definition: TargetLowering.cpp:144
llvm::TargetLowering::expandVecReduceSeq
SDValue expandVecReduceSeq(SDNode *Node, SelectionDAG &DAG) const
Expand a VECREDUCE_SEQ_* into an explicit ordered calculation.
Definition: TargetLowering.cpp:10282
llvm::TargetLowering::expandFP_TO_INT_SAT
SDValue expandFP_TO_INT_SAT(SDNode *N, SelectionDAG &DAG) const
Expand FP_TO_[US]INT_SAT into FP_TO_[US]INT and selects or min/max.
Definition: TargetLowering.cpp:10333
llvm::TargetLowering::expandVecReduce
SDValue expandVecReduce(SDNode *Node, SelectionDAG &DAG) const
Expand a VECREDUCE_* into an explicit calculation.
Definition: TargetLowering.cpp:10242
llvm::TargetLowering::createSelectForFMINNUM_FMAXNUM
SDValue createSelectForFMINNUM_FMAXNUM(SDNode *Node, SelectionDAG &DAG) const
Try to convert the fminnum/fmaxnum to a compare/select sequence.
Definition: TargetLowering.cpp:7975
uint64_t
llvm_unreachable
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
Definition: ErrorHandling.h:143
llvm::ARM_MB::LD
@ LD
Definition: ARMBaseInfo.h:72
llvm::ARM_MB::ST
@ ST
Definition: ARMBaseInfo.h:73
llvm::BitmaskEnumDetail::Mask
constexpr std::underlying_type_t< E > Mask()
Get a bitmask with 1s in all places up to the high-order bit of E's largest value.
Definition: BitmaskEnum.h:119
llvm::CallingConv::C
@ C
The default llvm calling convention, compatible with C.
Definition: CallingConv.h:34
llvm::ISD::NodeType
NodeType
ISD::NodeType enum - This enum defines the target-independent operators for a SelectionDAG.
Definition: ISDOpcodes.h:40
llvm::ISD::SETCC
@ SETCC
SetCC operator - This evaluates to a true value iff the condition is true.
Definition: ISDOpcodes.h:749
llvm::ISD::MERGE_VALUES
@ MERGE_VALUES
MERGE_VALUES - This node takes multiple discrete operands and returns them all as its individual resu...
Definition: ISDOpcodes.h:236
llvm::ISD::STRICT_FSETCC
@ STRICT_FSETCC
STRICT_FSETCC/STRICT_FSETCCS - Constrained versions of SETCC, used for floating-point operands only.
Definition: ISDOpcodes.h:475
llvm::ISD::VECREDUCE_SEQ_FADD
@ VECREDUCE_SEQ_FADD
Generic reduction nodes.
Definition: ISDOpcodes.h:1276
llvm::ISD::ConstantFP
@ ConstantFP
Definition: ISDOpcodes.h:77
llvm::ISD::STRICT_FCEIL
@ STRICT_FCEIL
Definition: ISDOpcodes.h:425
llvm::ISD::FMAD
@ FMAD
FMAD - Perform a * b + c, while getting the same result as the separately rounded operations.
Definition: ISDOpcodes.h:486
llvm::ISD::LOAD
@ LOAD
LOAD and STORE have token chains as their first operand, then the same operands as an LLVM load/store...
Definition: ISDOpcodes.h:978
llvm::ISD::ANY_EXTEND
@ ANY_EXTEND
ANY_EXTEND - Used for integer types. The high bits are undefined.
Definition: ISDOpcodes.h:779
llvm::ISD::FMA
@ FMA
FMA - Perform a * b + c with no intermediate rounding step.
Definition: ISDOpcodes.h:482
llvm::ISD::SINT_TO_FP
@ SINT_TO_FP
[SU]INT_TO_FP - These operators convert integers (whose interpreted sign depends on the first letter)...
Definition: ISDOpcodes.h:786
llvm::ISD::VECREDUCE_FMAX
@ VECREDUCE_FMAX
FMIN/FMAX nodes can have flags, for NaN/NoNaN variants.
Definition: ISDOpcodes.h:1292
llvm::ISD::FADD
@ FADD
Simple binary floating point operators.
Definition: ISDOpcodes.h:390
llvm::ISD::STRICT_FSETCCS
@ STRICT_FSETCCS
Definition: ISDOpcodes.h:476
llvm::ISD::FP16_TO_FP
@ FP16_TO_FP
FP16_TO_FP, FP_TO_FP16 - These operators are used to perform promotions and truncation for half-preci...
Definition: ISDOpcodes.h:908
llvm::ISD::STRICT_FLOG2
@ STRICT_FLOG2
Definition: ISDOpcodes.h:420
llvm::ISD::BITCAST
@ BITCAST
BITCAST - This operator converts between integer, vector and FP values, as if the value was stored to...
Definition: ISDOpcodes.h:898
llvm::ISD::BUILD_PAIR
@ BUILD_PAIR
BUILD_PAIR - This is the opposite of EXTRACT_ELEMENT in some ways.
Definition: ISDOpcodes.h:229
llvm::ISD::STRICT_FSQRT
@ STRICT_FSQRT
Constrained versions of libm-equivalent floating point intrinsics.
Definition: ISDOpcodes.h:411
llvm::ISD::SIGN_EXTEND
@ SIGN_EXTEND
Conversion operators.
Definition: ISDOpcodes.h:773
llvm::ISD::STRICT_UINT_TO_FP
@ STRICT_UINT_TO_FP
Definition: ISDOpcodes.h:449
llvm::ISD::VECREDUCE_FADD
@ VECREDUCE_FADD
These reductions have relaxed evaluation order semantics, and have a single vector operand.
Definition: ISDOpcodes.h:1289
llvm::ISD::VECREDUCE_FMIN
@ VECREDUCE_FMIN
Definition: ISDOpcodes.h:1293
llvm::ISD::STRICT_LROUND
@ STRICT_LROUND
Definition: ISDOpcodes.h:430
llvm::ISD::FNEG
@ FNEG
Perform various unary floating-point operations inspired by libm.
Definition: ISDOpcodes.h:923
llvm::ISD::BR_CC
@ BR_CC
BR_CC - Conditional branch.
Definition: ISDOpcodes.h:1020
llvm::ISD::FCANONICALIZE
@ FCANONICALIZE
Returns platform specific canonical encoding of a floating point number.
Definition: ISDOpcodes.h:499
llvm::ISD::SELECT
@ SELECT
Select(COND, TRUEVAL, FALSEVAL).
Definition: ISDOpcodes.h:726
llvm::ISD::STRICT_FPOWI
@ STRICT_FPOWI
Definition: ISDOpcodes.h:413
llvm::ISD::UNDEF
@ UNDEF
UNDEF - An undefined node.
Definition: ISDOpcodes.h:211
llvm::ISD::EXTRACT_ELEMENT
@ EXTRACT_ELEMENT
EXTRACT_ELEMENT - This is used to get the lower or upper (determined by a Constant,...
Definition: ISDOpcodes.h:222
llvm::ISD::STRICT_FTRUNC
@ STRICT_FTRUNC
Definition: ISDOpcodes.h:429
llvm::ISD::ARITH_FENCE
@ ARITH_FENCE
ARITH_FENCE - This corresponds to a arithmetic fence intrinsic.
Definition: ISDOpcodes.h:1162
llvm::ISD::STRICT_FP_TO_FP16
@ STRICT_FP_TO_FP16
Definition: ISDOpcodes.h:911
llvm::ISD::STRICT_FP16_TO_FP
@ STRICT_FP16_TO_FP
Definition: ISDOpcodes.h:910
llvm::ISD::SHL
@ SHL
Shift and rotation operations.
Definition: ISDOpcodes.h:704
llvm::ISD::STRICT_FMAXNUM
@ STRICT_FMAXNUM
Definition: ISDOpcodes.h:423
llvm::ISD::EXTRACT_VECTOR_ELT
@ EXTRACT_VECTOR_ELT
EXTRACT_VECTOR_ELT(VECTOR, IDX) - Returns a single element from VECTOR identified by the (potentially...
Definition: ISDOpcodes.h:534
llvm::ISD::ZERO_EXTEND
@ ZERO_EXTEND
ZERO_EXTEND - Used for integer types, zeroing the new bits.
Definition: ISDOpcodes.h:776
llvm::ISD::FP_TO_UINT_SAT
@ FP_TO_UINT_SAT
Definition: ISDOpcodes.h:852
llvm::ISD::STRICT_FMINNUM
@ STRICT_FMINNUM
Definition: ISDOpcodes.h:424
llvm::ISD::SELECT_CC
@ SELECT_CC
Select with condition operator - This selects between a true value and a false value (ops #2 and #3) ...
Definition: ISDOpcodes.h:741
llvm::ISD::FMINNUM
@ FMINNUM
FMINNUM/FMAXNUM - Perform floating-point minimum or maximum on two values.
Definition: ISDOpcodes.h:955
llvm::ISD::STRICT_LRINT
@ STRICT_LRINT
Definition: ISDOpcodes.h:432
llvm::ISD::FP_EXTEND
@ FP_EXTEND
X = FP_EXTEND(Y) - Extend a smaller FP type into a larger FP type.
Definition: ISDOpcodes.h:883
llvm::ISD::STRICT_FROUND
@ STRICT_FROUND
Definition: ISDOpcodes.h:427
llvm::ISD::STRICT_SINT_TO_FP
@ STRICT_SINT_TO_FP
STRICT_[US]INT_TO_FP - Convert a signed or unsigned integer to a floating point value.
Definition: ISDOpcodes.h:448
llvm::ISD::STRICT_FFLOOR
@ STRICT_FFLOOR
Definition: ISDOpcodes.h:426
llvm::ISD::STRICT_FROUNDEVEN
@ STRICT_FROUNDEVEN
Definition: ISDOpcodes.h:428
llvm::ISD::BF16_TO_FP
@ BF16_TO_FP
BF16_TO_FP, FP_TO_BF16 - These operators are used to perform promotions and truncation for bfloat16.
Definition: ISDOpcodes.h:917
llvm::ISD::STRICT_FP_TO_UINT
@ STRICT_FP_TO_UINT
Definition: ISDOpcodes.h:442
llvm::ISD::STRICT_FP_ROUND
@ STRICT_FP_ROUND
X = STRICT_FP_ROUND(Y, TRUNC) - Rounding 'Y' from a larger floating point type down to the precision ...
Definition: ISDOpcodes.h:464
llvm::ISD::STRICT_FP_TO_SINT
@ STRICT_FP_TO_SINT
STRICT_FP_TO_[US]INT - Convert a floating point value to a signed or unsigned integer.
Definition: ISDOpcodes.h:441
llvm::ISD::FMINIMUM
@ FMINIMUM
FMINIMUM/FMAXIMUM - NaN-propagating minimum/maximum that also treat -0.0 as less than 0....
Definition: ISDOpcodes.h:968
llvm::ISD::FP_TO_SINT
@ FP_TO_SINT
FP_TO_[US]INT - Convert a floating point value to a signed or unsigned integer.
Definition: ISDOpcodes.h:832
llvm::ISD::STRICT_FP_EXTEND
@ STRICT_FP_EXTEND
X = STRICT_FP_EXTEND(Y) - Extend a smaller FP type into a larger FP type.
Definition: ISDOpcodes.h:469
llvm::ISD::AND
@ AND
Bitwise operators - logical and, logical or, logical xor.
Definition: ISDOpcodes.h:679
llvm::ISD::VECREDUCE_FMUL
@ VECREDUCE_FMUL
Definition: ISDOpcodes.h:1290
llvm::ISD::STRICT_FADD
@ STRICT_FADD
Constrained versions of the binary floating point operators.
Definition: ISDOpcodes.h:400
llvm::ISD::STRICT_FLOG10
@ STRICT_FLOG10
Definition: ISDOpcodes.h:419
llvm::ISD::STRICT_LLRINT
@ STRICT_LLRINT
Definition: ISDOpcodes.h:433
llvm::ISD::STRICT_FEXP2
@ STRICT_FEXP2
Definition: ISDOpcodes.h:417
llvm::ISD::ATOMIC_SWAP
@ ATOMIC_SWAP
Val, OUTCHAIN = ATOMIC_SWAP(INCHAIN, ptr, amt) Val, OUTCHAIN = ATOMIC_LOAD_[OpName](INCHAIN,...
Definition: ISDOpcodes.h:1199
llvm::ISD::FP_ROUND
@ FP_ROUND
X = FP_ROUND(Y, TRUNC) - Rounding 'Y' from a larger floating point type down to the precision of the ...
Definition: ISDOpcodes.h:865
llvm::ISD::STRICT_LLROUND
@ STRICT_LLROUND
Definition: ISDOpcodes.h:431
llvm::ISD::STRICT_FNEARBYINT
@ STRICT_FNEARBYINT
Definition: ISDOpcodes.h:422
llvm::ISD::FP_TO_SINT_SAT
@ FP_TO_SINT_SAT
FP_TO_[US]INT_SAT - Convert floating point value in operand 0 to a signed or unsigned scalar integer ...
Definition: ISDOpcodes.h:851
llvm::ISD::TRUNCATE
@ TRUNCATE
TRUNCATE - Completely drop the high bits.
Definition: ISDOpcodes.h:782
llvm::ISD::VAARG
@ VAARG
VAARG - VAARG has four operands: an input chain, a pointer, a SRCVALUE, and the alignment.
Definition: ISDOpcodes.h:1089
llvm::ISD::VECREDUCE_SEQ_FMUL
@ VECREDUCE_SEQ_FMUL
Definition: ISDOpcodes.h:1277
llvm::ISD::FCOPYSIGN
@ FCOPYSIGN
FCOPYSIGN(X, Y) - Return the value of X with the sign of Y.
Definition: ISDOpcodes.h:492
llvm::ISD::STRICT_FRINT
@ STRICT_FRINT
Definition: ISDOpcodes.h:421
llvm::ISD::BUILD_VECTOR
@ BUILD_VECTOR
BUILD_VECTOR(ELT0, ELT1, ELT2, ELT3,...) - Return a fixed-width vector with the specified,...
Definition: ISDOpcodes.h:514
llvm::ISD::isNormalStore
bool isNormalStore(const SDNode *N)
Returns true if the specified node is a non-truncating and unindexed store.
Definition: SelectionDAGNodes.h:3091
llvm::ISD::isUNINDEXEDLoad
bool isUNINDEXEDLoad(const SDNode *N)
Returns true if the specified node is an unindexed load.
Definition: SelectionDAGNodes.h:3084
llvm::ISD::isUNINDEXEDStore
bool isUNINDEXEDStore(const SDNode *N)
Returns true if the specified node is an unindexed store.
Definition: SelectionDAGNodes.h:3098
llvm::ISD::CondCode
CondCode
ISD::CondCode enum - These are ordered carefully to make the bitfields below work out,...
Definition: ISDOpcodes.h:1447
llvm::ISD::isNormalLoad
bool isNormalLoad(const SDNode *N)
Returns true if the specified node is a non-extending and unindexed load.
Definition: SelectionDAGNodes.h:3053
llvm::RTLIB::getPOWI
Libcall getPOWI(EVT RetVT)
getPOWI - Return the POWI_* value for the given types, or UNKNOWN_LIBCALL if there is none.
Definition: TargetLoweringBase.cpp:496
llvm::RTLIB::getSINTTOFP
Libcall getSINTTOFP(EVT OpVT, EVT RetVT)
getSINTTOFP - Return the SINTTOFP_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
Definition: TargetLoweringBase.cpp:406
llvm::RTLIB::getUINTTOFP
Libcall getUINTTOFP(EVT OpVT, EVT RetVT)
getUINTTOFP - Return the UINTTOFP_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
Definition: TargetLoweringBase.cpp:452
llvm::RTLIB::Libcall
Libcall
RTLIB::Libcall enum - This enum defines all of the runtime library calls the backend can emit.
Definition: RuntimeLibcalls.h:30
llvm::RTLIB::getFPTOUINT
Libcall getFPTOUINT(EVT OpVT, EVT RetVT)
getFPTOUINT - Return the FPTOUINT_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
Definition: TargetLoweringBase.cpp:357
llvm::RTLIB::getFPTOSINT
Libcall getFPTOSINT(EVT OpVT, EVT RetVT)
getFPTOSINT - Return the FPTOSINT_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
Definition: TargetLoweringBase.cpp:308
llvm::RTLIB::getFPEXT
Libcall getFPEXT(EVT OpVT, EVT RetVT)
getFPEXT - Return the FPEXT_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
Definition: TargetLoweringBase.cpp:233
llvm::RTLIB::getFPROUND
Libcall getFPROUND(EVT OpVT, EVT RetVT)
getFPROUND - Return the FPROUND_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
Definition: TargetLoweringBase.cpp:265
llvm::ms_demangle::QualifierMangleMode::Result
@ Result
llvm::ore::NV
DiagnosticInfoOptimizationBase::Argument NV
Definition: OptimizationRemarkEmitter.h:136
llvm::tgtok::FalseVal
@ FalseVal
Definition: TGLexer.h:62
llvm
This is an optimization pass for GlobalISel generic memory operations.
Definition: AddressRanges.h:18
llvm::Offset
@ Offset
Definition: DWP.cpp:406
llvm::dbgs
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
Definition: Debug.cpp:163
llvm::report_fatal_error
void report_fatal_error(Error Err, bool gen_crash_diag=true)
Report a serious error, calling any installed error handler.
Definition: Error.cpp:145
raw_ostream.h
N
#define N
llvm::APFloatBase::PPCDoubleDouble
static const fltSemantics & PPCDoubleDouble() LLVM_READNONE
Definition: APFloat.cpp:247
llvm::EVT
Extended Value Type.
Definition: ValueTypes.h:34
llvm::EVT::isSimple
bool isSimple() const
Test if the given EVT is simple (as opposed to being extended).
Definition: ValueTypes.h:129
llvm::EVT::getSizeInBits
TypeSize getSizeInBits() const
Return the size of the specified value type in bits.
Definition: ValueTypes.h:351
llvm::EVT::isByteSized
bool isByteSized() const
Return true if the bit size is a multiple of 8.
Definition: ValueTypes.h:226
llvm::EVT::getSimpleVT
MVT getSimpleVT() const
Return the SimpleValueType held in the specified simple EVT.
Definition: ValueTypes.h:299
llvm::EVT::getIntegerVT
static EVT getIntegerVT(LLVMContext &Context, unsigned BitWidth)
Returns the EVT that represents an integer with the given number of bits.
Definition: ValueTypes.h:64
llvm::EVT::bitsGE
bool bitsGE(EVT VT) const
Return true if this has no less bits than VT.
Definition: ValueTypes.h:275
llvm::EVT::getVectorElementType
EVT getVectorElementType() const
Given a vector type, return the type of each element.
Definition: ValueTypes.h:311
llvm::EVT::bitsLE
bool bitsLE(EVT VT) const
Return true if this has no more bits than VT.
Definition: ValueTypes.h:291
llvm::SDNodeFlags
These are IR-level optimization flags that may be propagated to SDNodes.
Definition: SelectionDAGNodes.h:379
llvm::TargetLowering::MakeLibCallOptions
This structure is used to pass arguments to makeLibCall function.
Definition: TargetLowering.h:4418
llvm::TargetLowering::MakeLibCallOptions::setSExt
MakeLibCallOptions & setSExt(bool Value=true)
Definition: TargetLowering.h:4433
llvm::TargetLowering::MakeLibCallOptions::setTypeListBeforeSoften
MakeLibCallOptions & setTypeListBeforeSoften(ArrayRef< EVT > OpsVT, EVT RetVT, bool Value=true)
Definition: TargetLowering.h:4453
Generated on Mon May 29 2023 12:59:04 for LLVM by doxygen 1.9.6