LLVM 18.0.0git
LegalizeDAG.cpp
Go to the documentation of this file.
1//===- LegalizeDAG.cpp - Implement SelectionDAG::Legalize -----------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements the SelectionDAG::Legalize method.
10//
11//===----------------------------------------------------------------------===//
12
13#include "llvm/ADT/APFloat.h"
14#include "llvm/ADT/APInt.h"
15#include "llvm/ADT/ArrayRef.h"
17#include "llvm/ADT/SetVector.h"
19#include "llvm/ADT/SmallSet.h"
35#include "llvm/IR/CallingConv.h"
36#include "llvm/IR/Constants.h"
37#include "llvm/IR/DataLayout.h"
39#include "llvm/IR/Function.h"
40#include "llvm/IR/Metadata.h"
41#include "llvm/IR/Type.h"
44#include "llvm/Support/Debug.h"
50#include <cassert>
51#include <cstdint>
52#include <tuple>
53#include <utility>
54
55using namespace llvm;
56
57#define DEBUG_TYPE "legalizedag"
58
59namespace {
60
61/// Keeps track of state when getting the sign of a floating-point value as an
62/// integer.
63struct FloatSignAsInt {
64 EVT FloatVT;
65 SDValue Chain;
66 SDValue FloatPtr;
67 SDValue IntPtr;
68 MachinePointerInfo IntPointerInfo;
69 MachinePointerInfo FloatPointerInfo;
70 SDValue IntValue;
71 APInt SignMask;
72 uint8_t SignBit;
73};
74
75//===----------------------------------------------------------------------===//
76/// This takes an arbitrary SelectionDAG as input and
77/// hacks on it until the target machine can handle it. This involves
78/// eliminating value sizes the machine cannot handle (promoting small sizes to
79/// large sizes or splitting up large values into small values) as well as
80/// eliminating operations the machine cannot handle.
81///
82/// This code also does a small amount of optimization and recognition of idioms
83/// as part of its processing. For example, if a target does not support a
84/// 'setcc' instruction efficiently, but does support 'brcc' instruction, this
85/// will attempt merge setcc and brc instructions into brcc's.
86class SelectionDAGLegalize {
87 const TargetMachine &TM;
88 const TargetLowering &TLI;
89 SelectionDAG &DAG;
90
91 /// The set of nodes which have already been legalized. We hold a
92 /// reference to it in order to update as necessary on node deletion.
93 SmallPtrSetImpl<SDNode *> &LegalizedNodes;
94
95 /// A set of all the nodes updated during legalization.
96 SmallSetVector<SDNode *, 16> *UpdatedNodes;
97
98 EVT getSetCCResultType(EVT VT) const {
99 return TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT);
100 }
101
102 // Libcall insertion helpers.
103
104public:
105 SelectionDAGLegalize(SelectionDAG &DAG,
106 SmallPtrSetImpl<SDNode *> &LegalizedNodes,
107 SmallSetVector<SDNode *, 16> *UpdatedNodes = nullptr)
108 : TM(DAG.getTarget()), TLI(DAG.getTargetLoweringInfo()), DAG(DAG),
109 LegalizedNodes(LegalizedNodes), UpdatedNodes(UpdatedNodes) {}
110
111 /// Legalizes the given operation.
112 void LegalizeOp(SDNode *Node);
113
114private:
115 SDValue OptimizeFloatStore(StoreSDNode *ST);
116
117 void LegalizeLoadOps(SDNode *Node);
118 void LegalizeStoreOps(SDNode *Node);
119
120 /// Some targets cannot handle a variable
121 /// insertion index for the INSERT_VECTOR_ELT instruction. In this case, it
122 /// is necessary to spill the vector being inserted into to memory, perform
123 /// the insert there, and then read the result back.
124 SDValue PerformInsertVectorEltInMemory(SDValue Vec, SDValue Val, SDValue Idx,
125 const SDLoc &dl);
126 SDValue ExpandINSERT_VECTOR_ELT(SDValue Vec, SDValue Val, SDValue Idx,
127 const SDLoc &dl);
128
129 /// Return a vector shuffle operation which
130 /// performs the same shuffe in terms of order or result bytes, but on a type
131 /// whose vector element type is narrower than the original shuffle type.
132 /// e.g. <v4i32> <0, 1, 0, 1> -> v8i16 <0, 1, 2, 3, 0, 1, 2, 3>
133 SDValue ShuffleWithNarrowerEltType(EVT NVT, EVT VT, const SDLoc &dl,
134 SDValue N1, SDValue N2,
135 ArrayRef<int> Mask) const;
136
137 std::pair<SDValue, SDValue> ExpandLibCall(RTLIB::Libcall LC, SDNode *Node,
139 std::pair<SDValue, SDValue> ExpandLibCall(RTLIB::Libcall LC, SDNode *Node, bool isSigned);
140
141 void ExpandFrexpLibCall(SDNode *Node, SmallVectorImpl<SDValue> &Results);
142 void ExpandFPLibCall(SDNode *Node, RTLIB::Libcall LC,
144 void ExpandFPLibCall(SDNode *Node, RTLIB::Libcall Call_F32,
145 RTLIB::Libcall Call_F64, RTLIB::Libcall Call_F80,
146 RTLIB::Libcall Call_F128,
147 RTLIB::Libcall Call_PPCF128,
149 SDValue ExpandIntLibCall(SDNode *Node, bool isSigned,
150 RTLIB::Libcall Call_I8,
151 RTLIB::Libcall Call_I16,
152 RTLIB::Libcall Call_I32,
153 RTLIB::Libcall Call_I64,
154 RTLIB::Libcall Call_I128);
155 void ExpandArgFPLibCall(SDNode *Node,
156 RTLIB::Libcall Call_F32, RTLIB::Libcall Call_F64,
157 RTLIB::Libcall Call_F80, RTLIB::Libcall Call_F128,
158 RTLIB::Libcall Call_PPCF128,
160 void ExpandDivRemLibCall(SDNode *Node, SmallVectorImpl<SDValue> &Results);
161 void ExpandSinCosLibCall(SDNode *Node, SmallVectorImpl<SDValue> &Results);
162
163 SDValue EmitStackConvert(SDValue SrcOp, EVT SlotVT, EVT DestVT,
164 const SDLoc &dl);
165 SDValue EmitStackConvert(SDValue SrcOp, EVT SlotVT, EVT DestVT,
166 const SDLoc &dl, SDValue ChainIn);
167 SDValue ExpandBUILD_VECTOR(SDNode *Node);
168 SDValue ExpandSPLAT_VECTOR(SDNode *Node);
169 SDValue ExpandSCALAR_TO_VECTOR(SDNode *Node);
170 void ExpandDYNAMIC_STACKALLOC(SDNode *Node,
172 void getSignAsIntValue(FloatSignAsInt &State, const SDLoc &DL,
173 SDValue Value) const;
174 SDValue modifySignAsInt(const FloatSignAsInt &State, const SDLoc &DL,
175 SDValue NewIntValue) const;
176 SDValue ExpandFCOPYSIGN(SDNode *Node) const;
177 SDValue ExpandFABS(SDNode *Node) const;
178 SDValue ExpandFNEG(SDNode *Node) const;
179 SDValue expandLdexp(SDNode *Node) const;
180 SDValue expandFrexp(SDNode *Node) const;
181
182 SDValue ExpandLegalINT_TO_FP(SDNode *Node, SDValue &Chain);
183 void PromoteLegalINT_TO_FP(SDNode *N, const SDLoc &dl,
185 void PromoteLegalFP_TO_INT(SDNode *N, const SDLoc &dl,
187 SDValue PromoteLegalFP_TO_INT_SAT(SDNode *Node, const SDLoc &dl);
188
189 SDValue ExpandPARITY(SDValue Op, const SDLoc &dl);
190
191 SDValue ExpandExtractFromVectorThroughStack(SDValue Op);
192 SDValue ExpandInsertToVectorThroughStack(SDValue Op);
193 SDValue ExpandVectorBuildThroughStack(SDNode* Node);
194
195 SDValue ExpandConstantFP(ConstantFPSDNode *CFP, bool UseCP);
196 SDValue ExpandConstant(ConstantSDNode *CP);
197
198 // if ExpandNode returns false, LegalizeOp falls back to ConvertNodeToLibcall
199 bool ExpandNode(SDNode *Node);
200 void ConvertNodeToLibcall(SDNode *Node);
201 void PromoteNode(SDNode *Node);
202
203public:
204 // Node replacement helpers
205
206 void ReplacedNode(SDNode *N) {
207 LegalizedNodes.erase(N);
208 if (UpdatedNodes)
209 UpdatedNodes->insert(N);
210 }
211
212 void ReplaceNode(SDNode *Old, SDNode *New) {
213 LLVM_DEBUG(dbgs() << " ... replacing: "; Old->dump(&DAG);
214 dbgs() << " with: "; New->dump(&DAG));
215
216 assert(Old->getNumValues() == New->getNumValues() &&
217 "Replacing one node with another that produces a different number "
218 "of values!");
219 DAG.ReplaceAllUsesWith(Old, New);
220 if (UpdatedNodes)
221 UpdatedNodes->insert(New);
222 ReplacedNode(Old);
223 }
224
225 void ReplaceNode(SDValue Old, SDValue New) {
226 LLVM_DEBUG(dbgs() << " ... replacing: "; Old->dump(&DAG);
227 dbgs() << " with: "; New->dump(&DAG));
228
229 DAG.ReplaceAllUsesWith(Old, New);
230 if (UpdatedNodes)
231 UpdatedNodes->insert(New.getNode());
232 ReplacedNode(Old.getNode());
233 }
234
235 void ReplaceNode(SDNode *Old, const SDValue *New) {
236 LLVM_DEBUG(dbgs() << " ... replacing: "; Old->dump(&DAG));
237
238 DAG.ReplaceAllUsesWith(Old, New);
239 for (unsigned i = 0, e = Old->getNumValues(); i != e; ++i) {
240 LLVM_DEBUG(dbgs() << (i == 0 ? " with: " : " and: ");
241 New[i]->dump(&DAG));
242 if (UpdatedNodes)
243 UpdatedNodes->insert(New[i].getNode());
244 }
245 ReplacedNode(Old);
246 }
247
248 void ReplaceNodeWithValue(SDValue Old, SDValue New) {
249 LLVM_DEBUG(dbgs() << " ... replacing: "; Old->dump(&DAG);
250 dbgs() << " with: "; New->dump(&DAG));
251
252 DAG.ReplaceAllUsesOfValueWith(Old, New);
253 if (UpdatedNodes)
254 UpdatedNodes->insert(New.getNode());
255 ReplacedNode(Old.getNode());
256 }
257};
258
259} // end anonymous namespace
260
261/// Return a vector shuffle operation which
262/// performs the same shuffle in terms of order or result bytes, but on a type
263/// whose vector element type is narrower than the original shuffle type.
264/// e.g. <v4i32> <0, 1, 0, 1> -> v8i16 <0, 1, 2, 3, 0, 1, 2, 3>
265SDValue SelectionDAGLegalize::ShuffleWithNarrowerEltType(
266 EVT NVT, EVT VT, const SDLoc &dl, SDValue N1, SDValue N2,
267 ArrayRef<int> Mask) const {
268 unsigned NumMaskElts = VT.getVectorNumElements();
269 unsigned NumDestElts = NVT.getVectorNumElements();
270 unsigned NumEltsGrowth = NumDestElts / NumMaskElts;
271
272 assert(NumEltsGrowth && "Cannot promote to vector type with fewer elts!");
273
274 if (NumEltsGrowth == 1)
275 return DAG.getVectorShuffle(NVT, dl, N1, N2, Mask);
276
277 SmallVector<int, 8> NewMask;
278 for (unsigned i = 0; i != NumMaskElts; ++i) {
279 int Idx = Mask[i];
280 for (unsigned j = 0; j != NumEltsGrowth; ++j) {
281 if (Idx < 0)
282 NewMask.push_back(-1);
283 else
284 NewMask.push_back(Idx * NumEltsGrowth + j);
285 }
286 }
287 assert(NewMask.size() == NumDestElts && "Non-integer NumEltsGrowth?");
288 assert(TLI.isShuffleMaskLegal(NewMask, NVT) && "Shuffle not legal?");
289 return DAG.getVectorShuffle(NVT, dl, N1, N2, NewMask);
290}
291
292/// Expands the ConstantFP node to an integer constant or
293/// a load from the constant pool.
295SelectionDAGLegalize::ExpandConstantFP(ConstantFPSDNode *CFP, bool UseCP) {
296 bool Extend = false;
297 SDLoc dl(CFP);
298
299 // If a FP immediate is precise when represented as a float and if the
300 // target can do an extending load from float to double, we put it into
301 // the constant pool as a float, even if it's is statically typed as a
302 // double. This shrinks FP constants and canonicalizes them for targets where
303 // an FP extending load is the same cost as a normal load (such as on the x87
304 // fp stack or PPC FP unit).
305 EVT VT = CFP->getValueType(0);
306 ConstantFP *LLVMC = const_cast<ConstantFP*>(CFP->getConstantFPValue());
307 if (!UseCP) {
308 assert((VT == MVT::f64 || VT == MVT::f32) && "Invalid type expansion");
309 return DAG.getConstant(LLVMC->getValueAPF().bitcastToAPInt(), dl,
310 (VT == MVT::f64) ? MVT::i64 : MVT::i32);
311 }
312
313 APFloat APF = CFP->getValueAPF();
314 EVT OrigVT = VT;
315 EVT SVT = VT;
316
317 // We don't want to shrink SNaNs. Converting the SNaN back to its real type
318 // can cause it to be changed into a QNaN on some platforms (e.g. on SystemZ).
319 if (!APF.isSignaling()) {
320 while (SVT != MVT::f32 && SVT != MVT::f16 && SVT != MVT::bf16) {
321 SVT = (MVT::SimpleValueType)(SVT.getSimpleVT().SimpleTy - 1);
323 // Only do this if the target has a native EXTLOAD instruction from
324 // smaller type.
325 TLI.isLoadExtLegal(ISD::EXTLOAD, OrigVT, SVT) &&
326 TLI.ShouldShrinkFPConstant(OrigVT)) {
327 Type *SType = SVT.getTypeForEVT(*DAG.getContext());
328 LLVMC = cast<ConstantFP>(ConstantFoldCastOperand(
329 Instruction::FPTrunc, LLVMC, SType, DAG.getDataLayout()));
330 VT = SVT;
331 Extend = true;
332 }
333 }
334 }
335
336 SDValue CPIdx =
337 DAG.getConstantPool(LLVMC, TLI.getPointerTy(DAG.getDataLayout()));
338 Align Alignment = cast<ConstantPoolSDNode>(CPIdx)->getAlign();
339 if (Extend) {
340 SDValue Result = DAG.getExtLoad(
341 ISD::EXTLOAD, dl, OrigVT, DAG.getEntryNode(), CPIdx,
342 MachinePointerInfo::getConstantPool(DAG.getMachineFunction()), VT,
343 Alignment);
344 return Result;
345 }
346 SDValue Result = DAG.getLoad(
347 OrigVT, dl, DAG.getEntryNode(), CPIdx,
348 MachinePointerInfo::getConstantPool(DAG.getMachineFunction()), Alignment);
349 return Result;
350}
351
352/// Expands the Constant node to a load from the constant pool.
353SDValue SelectionDAGLegalize::ExpandConstant(ConstantSDNode *CP) {
354 SDLoc dl(CP);
355 EVT VT = CP->getValueType(0);
356 SDValue CPIdx = DAG.getConstantPool(CP->getConstantIntValue(),
357 TLI.getPointerTy(DAG.getDataLayout()));
358 Align Alignment = cast<ConstantPoolSDNode>(CPIdx)->getAlign();
359 SDValue Result = DAG.getLoad(
360 VT, dl, DAG.getEntryNode(), CPIdx,
361 MachinePointerInfo::getConstantPool(DAG.getMachineFunction()), Alignment);
362 return Result;
363}
364
365/// Some target cannot handle a variable insertion index for the
366/// INSERT_VECTOR_ELT instruction. In this case, it
367/// is necessary to spill the vector being inserted into to memory, perform
368/// the insert there, and then read the result back.
369SDValue SelectionDAGLegalize::PerformInsertVectorEltInMemory(SDValue Vec,
370 SDValue Val,
371 SDValue Idx,
372 const SDLoc &dl) {
373 SDValue Tmp1 = Vec;
374 SDValue Tmp2 = Val;
375 SDValue Tmp3 = Idx;
376
377 // If the target doesn't support this, we have to spill the input vector
378 // to a temporary stack slot, update the element, then reload it. This is
379 // badness. We could also load the value into a vector register (either
380 // with a "move to register" or "extload into register" instruction, then
381 // permute it into place, if the idx is a constant and if the idx is
382 // supported by the target.
383 EVT VT = Tmp1.getValueType();
384 EVT EltVT = VT.getVectorElementType();
385 SDValue StackPtr = DAG.CreateStackTemporary(VT);
386
387 int SPFI = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex();
388
389 // Store the vector.
390 SDValue Ch = DAG.getStore(
391 DAG.getEntryNode(), dl, Tmp1, StackPtr,
392 MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), SPFI));
393
394 SDValue StackPtr2 = TLI.getVectorElementPointer(DAG, StackPtr, VT, Tmp3);
395
396 // Store the scalar value.
397 Ch = DAG.getTruncStore(
398 Ch, dl, Tmp2, StackPtr2,
399 MachinePointerInfo::getUnknownStack(DAG.getMachineFunction()), EltVT);
400 // Load the updated vector.
401 return DAG.getLoad(VT, dl, Ch, StackPtr, MachinePointerInfo::getFixedStack(
402 DAG.getMachineFunction(), SPFI));
403}
404
405SDValue SelectionDAGLegalize::ExpandINSERT_VECTOR_ELT(SDValue Vec, SDValue Val,
406 SDValue Idx,
407 const SDLoc &dl) {
408 if (ConstantSDNode *InsertPos = dyn_cast<ConstantSDNode>(Idx)) {
409 // SCALAR_TO_VECTOR requires that the type of the value being inserted
410 // match the element type of the vector being created, except for
411 // integers in which case the inserted value can be over width.
412 EVT EltVT = Vec.getValueType().getVectorElementType();
413 if (Val.getValueType() == EltVT ||
414 (EltVT.isInteger() && Val.getValueType().bitsGE(EltVT))) {
415 SDValue ScVec = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl,
416 Vec.getValueType(), Val);
417
418 unsigned NumElts = Vec.getValueType().getVectorNumElements();
419 // We generate a shuffle of InVec and ScVec, so the shuffle mask
420 // should be 0,1,2,3,4,5... with the appropriate element replaced with
421 // elt 0 of the RHS.
422 SmallVector<int, 8> ShufOps;
423 for (unsigned i = 0; i != NumElts; ++i)
424 ShufOps.push_back(i != InsertPos->getZExtValue() ? i : NumElts);
425
426 return DAG.getVectorShuffle(Vec.getValueType(), dl, Vec, ScVec, ShufOps);
427 }
428 }
429 return PerformInsertVectorEltInMemory(Vec, Val, Idx, dl);
430}
431
432SDValue SelectionDAGLegalize::OptimizeFloatStore(StoreSDNode* ST) {
433 if (!ISD::isNormalStore(ST))
434 return SDValue();
435
436 LLVM_DEBUG(dbgs() << "Optimizing float store operations\n");
437 // Turn 'store float 1.0, Ptr' -> 'store int 0x12345678, Ptr'
438 // FIXME: move this to the DAG Combiner! Note that we can't regress due
439 // to phase ordering between legalized code and the dag combiner. This
440 // probably means that we need to integrate dag combiner and legalizer
441 // together.
442 // We generally can't do this one for long doubles.
443 SDValue Chain = ST->getChain();
444 SDValue Ptr = ST->getBasePtr();
445 SDValue Value = ST->getValue();
446 MachineMemOperand::Flags MMOFlags = ST->getMemOperand()->getFlags();
447 AAMDNodes AAInfo = ST->getAAInfo();
448 SDLoc dl(ST);
449
450 // Don't optimise TargetConstantFP
451 if (Value.getOpcode() == ISD::TargetConstantFP)
452 return SDValue();
453
454 if (ConstantFPSDNode *CFP = dyn_cast<ConstantFPSDNode>(Value)) {
455 if (CFP->getValueType(0) == MVT::f32 &&
456 TLI.isTypeLegal(MVT::i32)) {
457 SDValue Con = DAG.getConstant(CFP->getValueAPF().
458 bitcastToAPInt().zextOrTrunc(32),
459 SDLoc(CFP), MVT::i32);
460 return DAG.getStore(Chain, dl, Con, Ptr, ST->getPointerInfo(),
461 ST->getOriginalAlign(), MMOFlags, AAInfo);
462 }
463
464 if (CFP->getValueType(0) == MVT::f64) {
465 // If this target supports 64-bit registers, do a single 64-bit store.
466 if (TLI.isTypeLegal(MVT::i64)) {
467 SDValue Con = DAG.getConstant(CFP->getValueAPF().bitcastToAPInt().
468 zextOrTrunc(64), SDLoc(CFP), MVT::i64);
469 return DAG.getStore(Chain, dl, Con, Ptr, ST->getPointerInfo(),
470 ST->getOriginalAlign(), MMOFlags, AAInfo);
471 }
472
473 if (TLI.isTypeLegal(MVT::i32) && !ST->isVolatile()) {
474 // Otherwise, if the target supports 32-bit registers, use 2 32-bit
475 // stores. If the target supports neither 32- nor 64-bits, this
476 // xform is certainly not worth it.
477 const APInt &IntVal = CFP->getValueAPF().bitcastToAPInt();
478 SDValue Lo = DAG.getConstant(IntVal.trunc(32), dl, MVT::i32);
479 SDValue Hi = DAG.getConstant(IntVal.lshr(32).trunc(32), dl, MVT::i32);
480 if (DAG.getDataLayout().isBigEndian())
481 std::swap(Lo, Hi);
482
483 Lo = DAG.getStore(Chain, dl, Lo, Ptr, ST->getPointerInfo(),
484 ST->getOriginalAlign(), MMOFlags, AAInfo);
485 Ptr = DAG.getMemBasePlusOffset(Ptr, TypeSize::getFixed(4), dl);
486 Hi = DAG.getStore(Chain, dl, Hi, Ptr,
487 ST->getPointerInfo().getWithOffset(4),
488 ST->getOriginalAlign(), MMOFlags, AAInfo);
489
490 return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo, Hi);
491 }
492 }
493 }
494 return SDValue();
495}
496
497void SelectionDAGLegalize::LegalizeStoreOps(SDNode *Node) {
498 StoreSDNode *ST = cast<StoreSDNode>(Node);
499 SDValue Chain = ST->getChain();
500 SDValue Ptr = ST->getBasePtr();
501 SDLoc dl(Node);
502
503 MachineMemOperand::Flags MMOFlags = ST->getMemOperand()->getFlags();
504 AAMDNodes AAInfo = ST->getAAInfo();
505
506 if (!ST->isTruncatingStore()) {
507 LLVM_DEBUG(dbgs() << "Legalizing store operation\n");
508 if (SDNode *OptStore = OptimizeFloatStore(ST).getNode()) {
509 ReplaceNode(ST, OptStore);
510 return;
511 }
512
513 SDValue Value = ST->getValue();
514 MVT VT = Value.getSimpleValueType();
515 switch (TLI.getOperationAction(ISD::STORE, VT)) {
516 default: llvm_unreachable("This action is not supported yet!");
517 case TargetLowering::Legal: {
518 // If this is an unaligned store and the target doesn't support it,
519 // expand it.
520 EVT MemVT = ST->getMemoryVT();
521 const DataLayout &DL = DAG.getDataLayout();
522 if (!TLI.allowsMemoryAccessForAlignment(*DAG.getContext(), DL, MemVT,
523 *ST->getMemOperand())) {
524 LLVM_DEBUG(dbgs() << "Expanding unsupported unaligned store\n");
525 SDValue Result = TLI.expandUnalignedStore(ST, DAG);
526 ReplaceNode(SDValue(ST, 0), Result);
527 } else
528 LLVM_DEBUG(dbgs() << "Legal store\n");
529 break;
530 }
531 case TargetLowering::Custom: {
532 LLVM_DEBUG(dbgs() << "Trying custom lowering\n");
533 SDValue Res = TLI.LowerOperation(SDValue(Node, 0), DAG);
534 if (Res && Res != SDValue(Node, 0))
535 ReplaceNode(SDValue(Node, 0), Res);
536 return;
537 }
538 case TargetLowering::Promote: {
539 MVT NVT = TLI.getTypeToPromoteTo(ISD::STORE, VT);
540 assert(NVT.getSizeInBits() == VT.getSizeInBits() &&
541 "Can only promote stores to same size type");
542 Value = DAG.getNode(ISD::BITCAST, dl, NVT, Value);
543 SDValue Result = DAG.getStore(Chain, dl, Value, Ptr, ST->getPointerInfo(),
544 ST->getOriginalAlign(), MMOFlags, AAInfo);
545 ReplaceNode(SDValue(Node, 0), Result);
546 break;
547 }
548 }
549 return;
550 }
551
552 LLVM_DEBUG(dbgs() << "Legalizing truncating store operations\n");
553 SDValue Value = ST->getValue();
554 EVT StVT = ST->getMemoryVT();
555 TypeSize StWidth = StVT.getSizeInBits();
556 TypeSize StSize = StVT.getStoreSizeInBits();
557 auto &DL = DAG.getDataLayout();
558
559 if (StWidth != StSize) {
560 // Promote to a byte-sized store with upper bits zero if not
561 // storing an integral number of bytes. For example, promote
562 // TRUNCSTORE:i1 X -> TRUNCSTORE:i8 (and X, 1)
563 EVT NVT = EVT::getIntegerVT(*DAG.getContext(), StSize.getFixedValue());
564 Value = DAG.getZeroExtendInReg(Value, dl, StVT);
566 DAG.getTruncStore(Chain, dl, Value, Ptr, ST->getPointerInfo(), NVT,
567 ST->getOriginalAlign(), MMOFlags, AAInfo);
568 ReplaceNode(SDValue(Node, 0), Result);
569 } else if (!StVT.isVector() && !isPowerOf2_64(StWidth.getFixedValue())) {
570 // If not storing a power-of-2 number of bits, expand as two stores.
571 assert(!StVT.isVector() && "Unsupported truncstore!");
572 unsigned StWidthBits = StWidth.getFixedValue();
573 unsigned LogStWidth = Log2_32(StWidthBits);
574 assert(LogStWidth < 32);
575 unsigned RoundWidth = 1 << LogStWidth;
576 assert(RoundWidth < StWidthBits);
577 unsigned ExtraWidth = StWidthBits - RoundWidth;
578 assert(ExtraWidth < RoundWidth);
579 assert(!(RoundWidth % 8) && !(ExtraWidth % 8) &&
580 "Store size not an integral number of bytes!");
581 EVT RoundVT = EVT::getIntegerVT(*DAG.getContext(), RoundWidth);
582 EVT ExtraVT = EVT::getIntegerVT(*DAG.getContext(), ExtraWidth);
583 SDValue Lo, Hi;
584 unsigned IncrementSize;
585
586 if (DL.isLittleEndian()) {
587 // TRUNCSTORE:i24 X -> TRUNCSTORE:i16 X, TRUNCSTORE@+2:i8 (srl X, 16)
588 // Store the bottom RoundWidth bits.
589 Lo = DAG.getTruncStore(Chain, dl, Value, Ptr, ST->getPointerInfo(),
590 RoundVT, ST->getOriginalAlign(), MMOFlags, AAInfo);
591
592 // Store the remaining ExtraWidth bits.
593 IncrementSize = RoundWidth / 8;
594 Ptr =
595 DAG.getMemBasePlusOffset(Ptr, TypeSize::getFixed(IncrementSize), dl);
596 Hi = DAG.getNode(
597 ISD::SRL, dl, Value.getValueType(), Value,
598 DAG.getConstant(RoundWidth, dl,
599 TLI.getShiftAmountTy(Value.getValueType(), DL)));
600 Hi = DAG.getTruncStore(Chain, dl, Hi, Ptr,
601 ST->getPointerInfo().getWithOffset(IncrementSize),
602 ExtraVT, ST->getOriginalAlign(), MMOFlags, AAInfo);
603 } else {
604 // Big endian - avoid unaligned stores.
605 // TRUNCSTORE:i24 X -> TRUNCSTORE:i16 (srl X, 8), TRUNCSTORE@+2:i8 X
606 // Store the top RoundWidth bits.
607 Hi = DAG.getNode(
608 ISD::SRL, dl, Value.getValueType(), Value,
609 DAG.getConstant(ExtraWidth, dl,
610 TLI.getShiftAmountTy(Value.getValueType(), DL)));
611 Hi = DAG.getTruncStore(Chain, dl, Hi, Ptr, ST->getPointerInfo(), RoundVT,
612 ST->getOriginalAlign(), MMOFlags, AAInfo);
613
614 // Store the remaining ExtraWidth bits.
615 IncrementSize = RoundWidth / 8;
616 Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr,
617 DAG.getConstant(IncrementSize, dl,
618 Ptr.getValueType()));
619 Lo = DAG.getTruncStore(Chain, dl, Value, Ptr,
620 ST->getPointerInfo().getWithOffset(IncrementSize),
621 ExtraVT, ST->getOriginalAlign(), MMOFlags, AAInfo);
622 }
623
624 // The order of the stores doesn't matter.
625 SDValue Result = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo, Hi);
626 ReplaceNode(SDValue(Node, 0), Result);
627 } else {
628 switch (TLI.getTruncStoreAction(ST->getValue().getValueType(), StVT)) {
629 default: llvm_unreachable("This action is not supported yet!");
630 case TargetLowering::Legal: {
631 EVT MemVT = ST->getMemoryVT();
632 // If this is an unaligned store and the target doesn't support it,
633 // expand it.
634 if (!TLI.allowsMemoryAccessForAlignment(*DAG.getContext(), DL, MemVT,
635 *ST->getMemOperand())) {
636 SDValue Result = TLI.expandUnalignedStore(ST, DAG);
637 ReplaceNode(SDValue(ST, 0), Result);
638 }
639 break;
640 }
641 case TargetLowering::Custom: {
642 SDValue Res = TLI.LowerOperation(SDValue(Node, 0), DAG);
643 if (Res && Res != SDValue(Node, 0))
644 ReplaceNode(SDValue(Node, 0), Res);
645 return;
646 }
647 case TargetLowering::Expand:
648 assert(!StVT.isVector() &&
649 "Vector Stores are handled in LegalizeVectorOps");
650
652
653 // TRUNCSTORE:i16 i32 -> STORE i16
654 if (TLI.isTypeLegal(StVT)) {
655 Value = DAG.getNode(ISD::TRUNCATE, dl, StVT, Value);
656 Result = DAG.getStore(Chain, dl, Value, Ptr, ST->getPointerInfo(),
657 ST->getOriginalAlign(), MMOFlags, AAInfo);
658 } else {
659 // The in-memory type isn't legal. Truncate to the type it would promote
660 // to, and then do a truncstore.
661 Value = DAG.getNode(ISD::TRUNCATE, dl,
662 TLI.getTypeToTransformTo(*DAG.getContext(), StVT),
663 Value);
664 Result =
665 DAG.getTruncStore(Chain, dl, Value, Ptr, ST->getPointerInfo(), StVT,
666 ST->getOriginalAlign(), MMOFlags, AAInfo);
667 }
668
669 ReplaceNode(SDValue(Node, 0), Result);
670 break;
671 }
672 }
673}
674
675void SelectionDAGLegalize::LegalizeLoadOps(SDNode *Node) {
676 LoadSDNode *LD = cast<LoadSDNode>(Node);
677 SDValue Chain = LD->getChain(); // The chain.
678 SDValue Ptr = LD->getBasePtr(); // The base pointer.
679 SDValue Value; // The value returned by the load op.
680 SDLoc dl(Node);
681
682 ISD::LoadExtType ExtType = LD->getExtensionType();
683 if (ExtType == ISD::NON_EXTLOAD) {
684 LLVM_DEBUG(dbgs() << "Legalizing non-extending load operation\n");
685 MVT VT = Node->getSimpleValueType(0);
686 SDValue RVal = SDValue(Node, 0);
687 SDValue RChain = SDValue(Node, 1);
688
689 switch (TLI.getOperationAction(Node->getOpcode(), VT)) {
690 default: llvm_unreachable("This action is not supported yet!");
691 case TargetLowering::Legal: {
692 EVT MemVT = LD->getMemoryVT();
693 const DataLayout &DL = DAG.getDataLayout();
694 // If this is an unaligned load and the target doesn't support it,
695 // expand it.
696 if (!TLI.allowsMemoryAccessForAlignment(*DAG.getContext(), DL, MemVT,
697 *LD->getMemOperand())) {
698 std::tie(RVal, RChain) = TLI.expandUnalignedLoad(LD, DAG);
699 }
700 break;
701 }
702 case TargetLowering::Custom:
703 if (SDValue Res = TLI.LowerOperation(RVal, DAG)) {
704 RVal = Res;
705 RChain = Res.getValue(1);
706 }
707 break;
708
709 case TargetLowering::Promote: {
710 MVT NVT = TLI.getTypeToPromoteTo(Node->getOpcode(), VT);
711 assert(NVT.getSizeInBits() == VT.getSizeInBits() &&
712 "Can only promote loads to same size type");
713
714 SDValue Res = DAG.getLoad(NVT, dl, Chain, Ptr, LD->getMemOperand());
715 RVal = DAG.getNode(ISD::BITCAST, dl, VT, Res);
716 RChain = Res.getValue(1);
717 break;
718 }
719 }
720 if (RChain.getNode() != Node) {
721 assert(RVal.getNode() != Node && "Load must be completely replaced");
722 DAG.ReplaceAllUsesOfValueWith(SDValue(Node, 0), RVal);
723 DAG.ReplaceAllUsesOfValueWith(SDValue(Node, 1), RChain);
724 if (UpdatedNodes) {
725 UpdatedNodes->insert(RVal.getNode());
726 UpdatedNodes->insert(RChain.getNode());
727 }
728 ReplacedNode(Node);
729 }
730 return;
731 }
732
733 LLVM_DEBUG(dbgs() << "Legalizing extending load operation\n");
734 EVT SrcVT = LD->getMemoryVT();
735 TypeSize SrcWidth = SrcVT.getSizeInBits();
736 MachineMemOperand::Flags MMOFlags = LD->getMemOperand()->getFlags();
737 AAMDNodes AAInfo = LD->getAAInfo();
738
739 if (SrcWidth != SrcVT.getStoreSizeInBits() &&
740 // Some targets pretend to have an i1 loading operation, and actually
741 // load an i8. This trick is correct for ZEXTLOAD because the top 7
742 // bits are guaranteed to be zero; it helps the optimizers understand
743 // that these bits are zero. It is also useful for EXTLOAD, since it
744 // tells the optimizers that those bits are undefined. It would be
745 // nice to have an effective generic way of getting these benefits...
746 // Until such a way is found, don't insist on promoting i1 here.
747 (SrcVT != MVT::i1 ||
748 TLI.getLoadExtAction(ExtType, Node->getValueType(0), MVT::i1) ==
749 TargetLowering::Promote)) {
750 // Promote to a byte-sized load if not loading an integral number of
751 // bytes. For example, promote EXTLOAD:i20 -> EXTLOAD:i24.
752 unsigned NewWidth = SrcVT.getStoreSizeInBits();
753 EVT NVT = EVT::getIntegerVT(*DAG.getContext(), NewWidth);
754 SDValue Ch;
755
756 // The extra bits are guaranteed to be zero, since we stored them that
757 // way. A zext load from NVT thus automatically gives zext from SrcVT.
758
759 ISD::LoadExtType NewExtType =
761
762 SDValue Result = DAG.getExtLoad(NewExtType, dl, Node->getValueType(0),
763 Chain, Ptr, LD->getPointerInfo(), NVT,
764 LD->getOriginalAlign(), MMOFlags, AAInfo);
765
766 Ch = Result.getValue(1); // The chain.
767
768 if (ExtType == ISD::SEXTLOAD)
769 // Having the top bits zero doesn't help when sign extending.
770 Result = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl,
771 Result.getValueType(),
772 Result, DAG.getValueType(SrcVT));
773 else if (ExtType == ISD::ZEXTLOAD || NVT == Result.getValueType())
774 // All the top bits are guaranteed to be zero - inform the optimizers.
775 Result = DAG.getNode(ISD::AssertZext, dl,
776 Result.getValueType(), Result,
777 DAG.getValueType(SrcVT));
778
779 Value = Result;
780 Chain = Ch;
781 } else if (!isPowerOf2_64(SrcWidth.getKnownMinValue())) {
782 // If not loading a power-of-2 number of bits, expand as two loads.
783 assert(!SrcVT.isVector() && "Unsupported extload!");
784 unsigned SrcWidthBits = SrcWidth.getFixedValue();
785 unsigned LogSrcWidth = Log2_32(SrcWidthBits);
786 assert(LogSrcWidth < 32);
787 unsigned RoundWidth = 1 << LogSrcWidth;
788 assert(RoundWidth < SrcWidthBits);
789 unsigned ExtraWidth = SrcWidthBits - RoundWidth;
790 assert(ExtraWidth < RoundWidth);
791 assert(!(RoundWidth % 8) && !(ExtraWidth % 8) &&
792 "Load size not an integral number of bytes!");
793 EVT RoundVT = EVT::getIntegerVT(*DAG.getContext(), RoundWidth);
794 EVT ExtraVT = EVT::getIntegerVT(*DAG.getContext(), ExtraWidth);
795 SDValue Lo, Hi, Ch;
796 unsigned IncrementSize;
797 auto &DL = DAG.getDataLayout();
798
799 if (DL.isLittleEndian()) {
800 // EXTLOAD:i24 -> ZEXTLOAD:i16 | (shl EXTLOAD@+2:i8, 16)
801 // Load the bottom RoundWidth bits.
802 Lo = DAG.getExtLoad(ISD::ZEXTLOAD, dl, Node->getValueType(0), Chain, Ptr,
803 LD->getPointerInfo(), RoundVT, LD->getOriginalAlign(),
804 MMOFlags, AAInfo);
805
806 // Load the remaining ExtraWidth bits.
807 IncrementSize = RoundWidth / 8;
808 Ptr =
809 DAG.getMemBasePlusOffset(Ptr, TypeSize::getFixed(IncrementSize), dl);
810 Hi = DAG.getExtLoad(ExtType, dl, Node->getValueType(0), Chain, Ptr,
811 LD->getPointerInfo().getWithOffset(IncrementSize),
812 ExtraVT, LD->getOriginalAlign(), MMOFlags, AAInfo);
813
814 // Build a factor node to remember that this load is independent of
815 // the other one.
816 Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1),
817 Hi.getValue(1));
818
819 // Move the top bits to the right place.
820 Hi = DAG.getNode(
821 ISD::SHL, dl, Hi.getValueType(), Hi,
822 DAG.getConstant(RoundWidth, dl,
823 TLI.getShiftAmountTy(Hi.getValueType(), DL)));
824
825 // Join the hi and lo parts.
826 Value = DAG.getNode(ISD::OR, dl, Node->getValueType(0), Lo, Hi);
827 } else {
828 // Big endian - avoid unaligned loads.
829 // EXTLOAD:i24 -> (shl EXTLOAD:i16, 8) | ZEXTLOAD@+2:i8
830 // Load the top RoundWidth bits.
831 Hi = DAG.getExtLoad(ExtType, dl, Node->getValueType(0), Chain, Ptr,
832 LD->getPointerInfo(), RoundVT, LD->getOriginalAlign(),
833 MMOFlags, AAInfo);
834
835 // Load the remaining ExtraWidth bits.
836 IncrementSize = RoundWidth / 8;
837 Ptr =
838 DAG.getMemBasePlusOffset(Ptr, TypeSize::getFixed(IncrementSize), dl);
839 Lo = DAG.getExtLoad(ISD::ZEXTLOAD, dl, Node->getValueType(0), Chain, Ptr,
840 LD->getPointerInfo().getWithOffset(IncrementSize),
841 ExtraVT, LD->getOriginalAlign(), MMOFlags, AAInfo);
842
843 // Build a factor node to remember that this load is independent of
844 // the other one.
845 Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1),
846 Hi.getValue(1));
847
848 // Move the top bits to the right place.
849 Hi = DAG.getNode(
850 ISD::SHL, dl, Hi.getValueType(), Hi,
851 DAG.getConstant(ExtraWidth, dl,
852 TLI.getShiftAmountTy(Hi.getValueType(), DL)));
853
854 // Join the hi and lo parts.
855 Value = DAG.getNode(ISD::OR, dl, Node->getValueType(0), Lo, Hi);
856 }
857
858 Chain = Ch;
859 } else {
860 bool isCustom = false;
861 switch (TLI.getLoadExtAction(ExtType, Node->getValueType(0),
862 SrcVT.getSimpleVT())) {
863 default: llvm_unreachable("This action is not supported yet!");
864 case TargetLowering::Custom:
865 isCustom = true;
866 [[fallthrough]];
867 case TargetLowering::Legal:
868 Value = SDValue(Node, 0);
869 Chain = SDValue(Node, 1);
870
871 if (isCustom) {
872 if (SDValue Res = TLI.LowerOperation(SDValue(Node, 0), DAG)) {
873 Value = Res;
874 Chain = Res.getValue(1);
875 }
876 } else {
877 // If this is an unaligned load and the target doesn't support it,
878 // expand it.
879 EVT MemVT = LD->getMemoryVT();
880 const DataLayout &DL = DAG.getDataLayout();
881 if (!TLI.allowsMemoryAccess(*DAG.getContext(), DL, MemVT,
882 *LD->getMemOperand())) {
883 std::tie(Value, Chain) = TLI.expandUnalignedLoad(LD, DAG);
884 }
885 }
886 break;
887
888 case TargetLowering::Expand: {
889 EVT DestVT = Node->getValueType(0);
890 if (!TLI.isLoadExtLegal(ISD::EXTLOAD, DestVT, SrcVT)) {
891 // If the source type is not legal, see if there is a legal extload to
892 // an intermediate type that we can then extend further.
893 EVT LoadVT = TLI.getRegisterType(SrcVT.getSimpleVT());
894 if ((LoadVT.isFloatingPoint() == SrcVT.isFloatingPoint()) &&
895 (TLI.isTypeLegal(SrcVT) || // Same as SrcVT == LoadVT?
896 TLI.isLoadExtLegal(ExtType, LoadVT, SrcVT))) {
897 // If we are loading a legal type, this is a non-extload followed by a
898 // full extend.
899 ISD::LoadExtType MidExtType =
900 (LoadVT == SrcVT) ? ISD::NON_EXTLOAD : ExtType;
901
902 SDValue Load = DAG.getExtLoad(MidExtType, dl, LoadVT, Chain, Ptr,
903 SrcVT, LD->getMemOperand());
904 unsigned ExtendOp =
906 Value = DAG.getNode(ExtendOp, dl, Node->getValueType(0), Load);
907 Chain = Load.getValue(1);
908 break;
909 }
910
911 // Handle the special case of fp16 extloads. EXTLOAD doesn't have the
912 // normal undefined upper bits behavior to allow using an in-reg extend
913 // with the illegal FP type, so load as an integer and do the
914 // from-integer conversion.
915 if (SrcVT.getScalarType() == MVT::f16) {
916 EVT ISrcVT = SrcVT.changeTypeToInteger();
917 EVT IDestVT = DestVT.changeTypeToInteger();
918 EVT ILoadVT = TLI.getRegisterType(IDestVT.getSimpleVT());
919
920 SDValue Result = DAG.getExtLoad(ISD::ZEXTLOAD, dl, ILoadVT, Chain,
921 Ptr, ISrcVT, LD->getMemOperand());
922 Value = DAG.getNode(ISD::FP16_TO_FP, dl, DestVT, Result);
923 Chain = Result.getValue(1);
924 break;
925 }
926 }
927
928 assert(!SrcVT.isVector() &&
929 "Vector Loads are handled in LegalizeVectorOps");
930
931 // FIXME: This does not work for vectors on most targets. Sign-
932 // and zero-extend operations are currently folded into extending
933 // loads, whether they are legal or not, and then we end up here
934 // without any support for legalizing them.
935 assert(ExtType != ISD::EXTLOAD &&
936 "EXTLOAD should always be supported!");
937 // Turn the unsupported load into an EXTLOAD followed by an
938 // explicit zero/sign extend inreg.
939 SDValue Result = DAG.getExtLoad(ISD::EXTLOAD, dl,
940 Node->getValueType(0),
941 Chain, Ptr, SrcVT,
942 LD->getMemOperand());
943 SDValue ValRes;
944 if (ExtType == ISD::SEXTLOAD)
945 ValRes = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl,
946 Result.getValueType(),
947 Result, DAG.getValueType(SrcVT));
948 else
949 ValRes = DAG.getZeroExtendInReg(Result, dl, SrcVT);
950 Value = ValRes;
951 Chain = Result.getValue(1);
952 break;
953 }
954 }
955 }
956
957 // Since loads produce two values, make sure to remember that we legalized
958 // both of them.
959 if (Chain.getNode() != Node) {
960 assert(Value.getNode() != Node && "Load must be completely replaced");
961 DAG.ReplaceAllUsesOfValueWith(SDValue(Node, 0), Value);
962 DAG.ReplaceAllUsesOfValueWith(SDValue(Node, 1), Chain);
963 if (UpdatedNodes) {
964 UpdatedNodes->insert(Value.getNode());
965 UpdatedNodes->insert(Chain.getNode());
966 }
967 ReplacedNode(Node);
968 }
969}
970
971/// Return a legal replacement for the given operation, with all legal operands.
972void SelectionDAGLegalize::LegalizeOp(SDNode *Node) {
973 LLVM_DEBUG(dbgs() << "\nLegalizing: "; Node->dump(&DAG));
974
975 // Allow illegal target nodes and illegal registers.
976 if (Node->getOpcode() == ISD::TargetConstant ||
977 Node->getOpcode() == ISD::Register)
978 return;
979
980#ifndef NDEBUG
981 for (unsigned i = 0, e = Node->getNumValues(); i != e; ++i)
982 assert(TLI.getTypeAction(*DAG.getContext(), Node->getValueType(i)) ==
983 TargetLowering::TypeLegal &&
984 "Unexpected illegal type!");
985
986 for (const SDValue &Op : Node->op_values())
987 assert((TLI.getTypeAction(*DAG.getContext(), Op.getValueType()) ==
988 TargetLowering::TypeLegal ||
989 Op.getOpcode() == ISD::TargetConstant ||
990 Op.getOpcode() == ISD::Register) &&
991 "Unexpected illegal type!");
992#endif
993
994 // Figure out the correct action; the way to query this varies by opcode
995 TargetLowering::LegalizeAction Action = TargetLowering::Legal;
996 bool SimpleFinishLegalizing = true;
997 switch (Node->getOpcode()) {
1001 case ISD::STACKSAVE:
1002 Action = TLI.getOperationAction(Node->getOpcode(), MVT::Other);
1003 break;
1005 Action = TLI.getOperationAction(Node->getOpcode(),
1006 Node->getValueType(0));
1007 break;
1008 case ISD::VAARG:
1009 Action = TLI.getOperationAction(Node->getOpcode(),
1010 Node->getValueType(0));
1011 if (Action != TargetLowering::Promote)
1012 Action = TLI.getOperationAction(Node->getOpcode(), MVT::Other);
1013 break;
1014 case ISD::SET_FPENV:
1015 case ISD::SET_FPMODE:
1016 Action = TLI.getOperationAction(Node->getOpcode(),
1017 Node->getOperand(1).getValueType());
1018 break;
1019 case ISD::FP_TO_FP16:
1020 case ISD::FP_TO_BF16:
1021 case ISD::SINT_TO_FP:
1022 case ISD::UINT_TO_FP:
1024 case ISD::LROUND:
1025 case ISD::LLROUND:
1026 case ISD::LRINT:
1027 case ISD::LLRINT:
1028 Action = TLI.getOperationAction(Node->getOpcode(),
1029 Node->getOperand(0).getValueType());
1030 break;
1034 case ISD::STRICT_LRINT:
1035 case ISD::STRICT_LLRINT:
1036 case ISD::STRICT_LROUND:
1038 // These pseudo-ops are the same as the other STRICT_ ops except
1039 // they are registered with setOperationAction() using the input type
1040 // instead of the output type.
1041 Action = TLI.getOperationAction(Node->getOpcode(),
1042 Node->getOperand(1).getValueType());
1043 break;
1045 EVT InnerType = cast<VTSDNode>(Node->getOperand(1))->getVT();
1046 Action = TLI.getOperationAction(Node->getOpcode(), InnerType);
1047 break;
1048 }
1049 case ISD::ATOMIC_STORE:
1050 Action = TLI.getOperationAction(Node->getOpcode(),
1051 Node->getOperand(1).getValueType());
1052 break;
1053 case ISD::SELECT_CC:
1054 case ISD::STRICT_FSETCC:
1056 case ISD::SETCC:
1057 case ISD::SETCCCARRY:
1058 case ISD::VP_SETCC:
1059 case ISD::BR_CC: {
1060 unsigned Opc = Node->getOpcode();
1061 unsigned CCOperand = Opc == ISD::SELECT_CC ? 4
1062 : Opc == ISD::STRICT_FSETCC ? 3
1063 : Opc == ISD::STRICT_FSETCCS ? 3
1064 : Opc == ISD::SETCCCARRY ? 3
1065 : (Opc == ISD::SETCC || Opc == ISD::VP_SETCC) ? 2
1066 : 1;
1067 unsigned CompareOperand = Opc == ISD::BR_CC ? 2
1068 : Opc == ISD::STRICT_FSETCC ? 1
1069 : Opc == ISD::STRICT_FSETCCS ? 1
1070 : 0;
1071 MVT OpVT = Node->getOperand(CompareOperand).getSimpleValueType();
1072 ISD::CondCode CCCode =
1073 cast<CondCodeSDNode>(Node->getOperand(CCOperand))->get();
1074 Action = TLI.getCondCodeAction(CCCode, OpVT);
1075 if (Action == TargetLowering::Legal) {
1076 if (Node->getOpcode() == ISD::SELECT_CC)
1077 Action = TLI.getOperationAction(Node->getOpcode(),
1078 Node->getValueType(0));
1079 else
1080 Action = TLI.getOperationAction(Node->getOpcode(), OpVT);
1081 }
1082 break;
1083 }
1084 case ISD::LOAD:
1085 case ISD::STORE:
1086 // FIXME: Model these properly. LOAD and STORE are complicated, and
1087 // STORE expects the unlegalized operand in some cases.
1088 SimpleFinishLegalizing = false;
1089 break;
1090 case ISD::CALLSEQ_START:
1091 case ISD::CALLSEQ_END:
1092 // FIXME: This shouldn't be necessary. These nodes have special properties
1093 // dealing with the recursive nature of legalization. Removing this
1094 // special case should be done as part of making LegalizeDAG non-recursive.
1095 SimpleFinishLegalizing = false;
1096 break;
1098 case ISD::GET_ROUNDING:
1099 case ISD::MERGE_VALUES:
1100 case ISD::EH_RETURN:
1102 case ISD::EH_DWARF_CFA:
1106 // These operations lie about being legal: when they claim to be legal,
1107 // they should actually be expanded.
1108 Action = TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0));
1109 if (Action == TargetLowering::Legal)
1110 Action = TargetLowering::Expand;
1111 break;
1114 case ISD::FRAMEADDR:
1115 case ISD::RETURNADDR:
1117 case ISD::SPONENTRY:
1118 // These operations lie about being legal: when they claim to be legal,
1119 // they should actually be custom-lowered.
1120 Action = TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0));
1121 if (Action == TargetLowering::Legal)
1122 Action = TargetLowering::Custom;
1123 break;
1125 // READCYCLECOUNTER returns an i64, even if type legalization might have
1126 // expanded that to several smaller types.
1127 Action = TLI.getOperationAction(Node->getOpcode(), MVT::i64);
1128 break;
1129 case ISD::READ_REGISTER:
1131 // Named register is legal in the DAG, but blocked by register name
1132 // selection if not implemented by target (to chose the correct register)
1133 // They'll be converted to Copy(To/From)Reg.
1134 Action = TargetLowering::Legal;
1135 break;
1136 case ISD::UBSANTRAP:
1137 Action = TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0));
1138 if (Action == TargetLowering::Expand) {
1139 // replace ISD::UBSANTRAP with ISD::TRAP
1140 SDValue NewVal;
1141 NewVal = DAG.getNode(ISD::TRAP, SDLoc(Node), Node->getVTList(),
1142 Node->getOperand(0));
1143 ReplaceNode(Node, NewVal.getNode());
1144 LegalizeOp(NewVal.getNode());
1145 return;
1146 }
1147 break;
1148 case ISD::DEBUGTRAP:
1149 Action = TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0));
1150 if (Action == TargetLowering::Expand) {
1151 // replace ISD::DEBUGTRAP with ISD::TRAP
1152 SDValue NewVal;
1153 NewVal = DAG.getNode(ISD::TRAP, SDLoc(Node), Node->getVTList(),
1154 Node->getOperand(0));
1155 ReplaceNode(Node, NewVal.getNode());
1156 LegalizeOp(NewVal.getNode());
1157 return;
1158 }
1159 break;
1160 case ISD::SADDSAT:
1161 case ISD::UADDSAT:
1162 case ISD::SSUBSAT:
1163 case ISD::USUBSAT:
1164 case ISD::SSHLSAT:
1165 case ISD::USHLSAT:
1168 Action = TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0));
1169 break;
1170 case ISD::SMULFIX:
1171 case ISD::SMULFIXSAT:
1172 case ISD::UMULFIX:
1173 case ISD::UMULFIXSAT:
1174 case ISD::SDIVFIX:
1175 case ISD::SDIVFIXSAT:
1176 case ISD::UDIVFIX:
1177 case ISD::UDIVFIXSAT: {
1178 unsigned Scale = Node->getConstantOperandVal(2);
1179 Action = TLI.getFixedPointOperationAction(Node->getOpcode(),
1180 Node->getValueType(0), Scale);
1181 break;
1182 }
1183 case ISD::MSCATTER:
1184 Action = TLI.getOperationAction(Node->getOpcode(),
1185 cast<MaskedScatterSDNode>(Node)->getValue().getValueType());
1186 break;
1187 case ISD::MSTORE:
1188 Action = TLI.getOperationAction(Node->getOpcode(),
1189 cast<MaskedStoreSDNode>(Node)->getValue().getValueType());
1190 break;
1191 case ISD::VP_SCATTER:
1192 Action = TLI.getOperationAction(
1193 Node->getOpcode(),
1194 cast<VPScatterSDNode>(Node)->getValue().getValueType());
1195 break;
1196 case ISD::VP_STORE:
1197 Action = TLI.getOperationAction(
1198 Node->getOpcode(),
1199 cast<VPStoreSDNode>(Node)->getValue().getValueType());
1200 break;
1201 case ISD::EXPERIMENTAL_VP_STRIDED_STORE:
1202 Action = TLI.getOperationAction(
1203 Node->getOpcode(),
1204 cast<VPStridedStoreSDNode>(Node)->getValue().getValueType());
1205 break;
1208 case ISD::VECREDUCE_ADD:
1209 case ISD::VECREDUCE_MUL:
1210 case ISD::VECREDUCE_AND:
1211 case ISD::VECREDUCE_OR:
1212 case ISD::VECREDUCE_XOR:
1221 case ISD::IS_FPCLASS:
1222 Action = TLI.getOperationAction(
1223 Node->getOpcode(), Node->getOperand(0).getValueType());
1224 break;
1227 case ISD::VP_REDUCE_FADD:
1228 case ISD::VP_REDUCE_FMUL:
1229 case ISD::VP_REDUCE_ADD:
1230 case ISD::VP_REDUCE_MUL:
1231 case ISD::VP_REDUCE_AND:
1232 case ISD::VP_REDUCE_OR:
1233 case ISD::VP_REDUCE_XOR:
1234 case ISD::VP_REDUCE_SMAX:
1235 case ISD::VP_REDUCE_SMIN:
1236 case ISD::VP_REDUCE_UMAX:
1237 case ISD::VP_REDUCE_UMIN:
1238 case ISD::VP_REDUCE_FMAX:
1239 case ISD::VP_REDUCE_FMIN:
1240 case ISD::VP_REDUCE_SEQ_FADD:
1241 case ISD::VP_REDUCE_SEQ_FMUL:
1242 Action = TLI.getOperationAction(
1243 Node->getOpcode(), Node->getOperand(1).getValueType());
1244 break;
1245 default:
1246 if (Node->getOpcode() >= ISD::BUILTIN_OP_END) {
1247 Action = TLI.getCustomOperationAction(*Node);
1248 } else {
1249 Action = TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0));
1250 }
1251 break;
1252 }
1253
1254 if (SimpleFinishLegalizing) {
1255 SDNode *NewNode = Node;
1256 switch (Node->getOpcode()) {
1257 default: break;
1258 case ISD::SHL:
1259 case ISD::SRL:
1260 case ISD::SRA:
1261 case ISD::ROTL:
1262 case ISD::ROTR: {
1263 // Legalizing shifts/rotates requires adjusting the shift amount
1264 // to the appropriate width.
1265 SDValue Op0 = Node->getOperand(0);
1266 SDValue Op1 = Node->getOperand(1);
1267 if (!Op1.getValueType().isVector()) {
1268 SDValue SAO = DAG.getShiftAmountOperand(Op0.getValueType(), Op1);
1269 // The getShiftAmountOperand() may create a new operand node or
1270 // return the existing one. If new operand is created we need
1271 // to update the parent node.
1272 // Do not try to legalize SAO here! It will be automatically legalized
1273 // in the next round.
1274 if (SAO != Op1)
1275 NewNode = DAG.UpdateNodeOperands(Node, Op0, SAO);
1276 }
1277 }
1278 break;
1279 case ISD::FSHL:
1280 case ISD::FSHR:
1281 case ISD::SRL_PARTS:
1282 case ISD::SRA_PARTS:
1283 case ISD::SHL_PARTS: {
1284 // Legalizing shifts/rotates requires adjusting the shift amount
1285 // to the appropriate width.
1286 SDValue Op0 = Node->getOperand(0);
1287 SDValue Op1 = Node->getOperand(1);
1288 SDValue Op2 = Node->getOperand(2);
1289 if (!Op2.getValueType().isVector()) {
1290 SDValue SAO = DAG.getShiftAmountOperand(Op0.getValueType(), Op2);
1291 // The getShiftAmountOperand() may create a new operand node or
1292 // return the existing one. If new operand is created we need
1293 // to update the parent node.
1294 if (SAO != Op2)
1295 NewNode = DAG.UpdateNodeOperands(Node, Op0, Op1, SAO);
1296 }
1297 break;
1298 }
1299 }
1300
1301 if (NewNode != Node) {
1302 ReplaceNode(Node, NewNode);
1303 Node = NewNode;
1304 }
1305 switch (Action) {
1306 case TargetLowering::Legal:
1307 LLVM_DEBUG(dbgs() << "Legal node: nothing to do\n");
1308 return;
1309 case TargetLowering::Custom:
1310 LLVM_DEBUG(dbgs() << "Trying custom legalization\n");
1311 // FIXME: The handling for custom lowering with multiple results is
1312 // a complete mess.
1313 if (SDValue Res = TLI.LowerOperation(SDValue(Node, 0), DAG)) {
1314 if (!(Res.getNode() != Node || Res.getResNo() != 0))
1315 return;
1316
1317 if (Node->getNumValues() == 1) {
1318 // Verify the new types match the original. Glue is waived because
1319 // ISD::ADDC can be legalized by replacing Glue with an integer type.
1320 assert((Res.getValueType() == Node->getValueType(0) ||
1321 Node->getValueType(0) == MVT::Glue) &&
1322 "Type mismatch for custom legalized operation");
1323 LLVM_DEBUG(dbgs() << "Successfully custom legalized node\n");
1324 // We can just directly replace this node with the lowered value.
1325 ReplaceNode(SDValue(Node, 0), Res);
1326 return;
1327 }
1328
1329 SmallVector<SDValue, 8> ResultVals;
1330 for (unsigned i = 0, e = Node->getNumValues(); i != e; ++i) {
1331 // Verify the new types match the original. Glue is waived because
1332 // ISD::ADDC can be legalized by replacing Glue with an integer type.
1333 assert((Res->getValueType(i) == Node->getValueType(i) ||
1334 Node->getValueType(i) == MVT::Glue) &&
1335 "Type mismatch for custom legalized operation");
1336 ResultVals.push_back(Res.getValue(i));
1337 }
1338 LLVM_DEBUG(dbgs() << "Successfully custom legalized node\n");
1339 ReplaceNode(Node, ResultVals.data());
1340 return;
1341 }
1342 LLVM_DEBUG(dbgs() << "Could not custom legalize node\n");
1343 [[fallthrough]];
1344 case TargetLowering::Expand:
1345 if (ExpandNode(Node))
1346 return;
1347 [[fallthrough]];
1348 case TargetLowering::LibCall:
1349 ConvertNodeToLibcall(Node);
1350 return;
1351 case TargetLowering::Promote:
1352 PromoteNode(Node);
1353 return;
1354 }
1355 }
1356
1357 switch (Node->getOpcode()) {
1358 default:
1359#ifndef NDEBUG
1360 dbgs() << "NODE: ";
1361 Node->dump( &DAG);
1362 dbgs() << "\n";
1363#endif
1364 llvm_unreachable("Do not know how to legalize this operator!");
1365
1366 case ISD::CALLSEQ_START:
1367 case ISD::CALLSEQ_END:
1368 break;
1369 case ISD::LOAD:
1370 return LegalizeLoadOps(Node);
1371 case ISD::STORE:
1372 return LegalizeStoreOps(Node);
1373 }
1374}
1375
1376SDValue SelectionDAGLegalize::ExpandExtractFromVectorThroughStack(SDValue Op) {
1377 SDValue Vec = Op.getOperand(0);
1378 SDValue Idx = Op.getOperand(1);
1379 SDLoc dl(Op);
1380
1381 // Before we generate a new store to a temporary stack slot, see if there is
1382 // already one that we can use. There often is because when we scalarize
1383 // vector operations (using SelectionDAG::UnrollVectorOp for example) a whole
1384 // series of EXTRACT_VECTOR_ELT nodes are generated, one for each element in
1385 // the vector. If all are expanded here, we don't want one store per vector
1386 // element.
1387
1388 // Caches for hasPredecessorHelper
1391 Visited.insert(Op.getNode());
1392 Worklist.push_back(Idx.getNode());
1393 SDValue StackPtr, Ch;
1394 for (SDNode *User : Vec.getNode()->uses()) {
1395 if (StoreSDNode *ST = dyn_cast<StoreSDNode>(User)) {
1396 if (ST->isIndexed() || ST->isTruncatingStore() ||
1397 ST->getValue() != Vec)
1398 continue;
1399
1400 // Make sure that nothing else could have stored into the destination of
1401 // this store.
1402 if (!ST->getChain().reachesChainWithoutSideEffects(DAG.getEntryNode()))
1403 continue;
1404
1405 // If the index is dependent on the store we will introduce a cycle when
1406 // creating the load (the load uses the index, and by replacing the chain
1407 // we will make the index dependent on the load). Also, the store might be
1408 // dependent on the extractelement and introduce a cycle when creating
1409 // the load.
1410 if (SDNode::hasPredecessorHelper(ST, Visited, Worklist) ||
1411 ST->hasPredecessor(Op.getNode()))
1412 continue;
1413
1414 StackPtr = ST->getBasePtr();
1415 Ch = SDValue(ST, 0);
1416 break;
1417 }
1418 }
1419
1420 EVT VecVT = Vec.getValueType();
1421
1422 if (!Ch.getNode()) {
1423 // Store the value to a temporary stack slot, then LOAD the returned part.
1424 StackPtr = DAG.CreateStackTemporary(VecVT);
1425 Ch = DAG.getStore(DAG.getEntryNode(), dl, Vec, StackPtr,
1427 }
1428
1429 SDValue NewLoad;
1430 Align ElementAlignment =
1431 std::min(cast<StoreSDNode>(Ch)->getAlign(),
1432 DAG.getDataLayout().getPrefTypeAlign(
1433 Op.getValueType().getTypeForEVT(*DAG.getContext())));
1434
1435 if (Op.getValueType().isVector()) {
1436 StackPtr = TLI.getVectorSubVecPointer(DAG, StackPtr, VecVT,
1437 Op.getValueType(), Idx);
1438 NewLoad = DAG.getLoad(Op.getValueType(), dl, Ch, StackPtr,
1439 MachinePointerInfo(), ElementAlignment);
1440 } else {
1441 StackPtr = TLI.getVectorElementPointer(DAG, StackPtr, VecVT, Idx);
1442 NewLoad = DAG.getExtLoad(ISD::EXTLOAD, dl, Op.getValueType(), Ch, StackPtr,
1444 ElementAlignment);
1445 }
1446
1447 // Replace the chain going out of the store, by the one out of the load.
1448 DAG.ReplaceAllUsesOfValueWith(Ch, SDValue(NewLoad.getNode(), 1));
1449
1450 // We introduced a cycle though, so update the loads operands, making sure
1451 // to use the original store's chain as an incoming chain.
1452 SmallVector<SDValue, 6> NewLoadOperands(NewLoad->op_begin(),
1453 NewLoad->op_end());
1454 NewLoadOperands[0] = Ch;
1455 NewLoad =
1456 SDValue(DAG.UpdateNodeOperands(NewLoad.getNode(), NewLoadOperands), 0);
1457 return NewLoad;
1458}
1459
1460SDValue SelectionDAGLegalize::ExpandInsertToVectorThroughStack(SDValue Op) {
1461 assert(Op.getValueType().isVector() && "Non-vector insert subvector!");
1462
1463 SDValue Vec = Op.getOperand(0);
1464 SDValue Part = Op.getOperand(1);
1465 SDValue Idx = Op.getOperand(2);
1466 SDLoc dl(Op);
1467
1468 // Store the value to a temporary stack slot, then LOAD the returned part.
1469 EVT VecVT = Vec.getValueType();
1470 EVT SubVecVT = Part.getValueType();
1471 SDValue StackPtr = DAG.CreateStackTemporary(VecVT);
1472 int FI = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex();
1473 MachinePointerInfo PtrInfo =
1474 MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), FI);
1475
1476 // First store the whole vector.
1477 SDValue Ch = DAG.getStore(DAG.getEntryNode(), dl, Vec, StackPtr, PtrInfo);
1478
1479 // Then store the inserted part.
1480 SDValue SubStackPtr =
1481 TLI.getVectorSubVecPointer(DAG, StackPtr, VecVT, SubVecVT, Idx);
1482
1483 // Store the subvector.
1484 Ch = DAG.getStore(
1485 Ch, dl, Part, SubStackPtr,
1486 MachinePointerInfo::getUnknownStack(DAG.getMachineFunction()));
1487
1488 // Finally, load the updated vector.
1489 return DAG.getLoad(Op.getValueType(), dl, Ch, StackPtr, PtrInfo);
1490}
1491
1492SDValue SelectionDAGLegalize::ExpandVectorBuildThroughStack(SDNode* Node) {
1493 assert((Node->getOpcode() == ISD::BUILD_VECTOR ||
1494 Node->getOpcode() == ISD::CONCAT_VECTORS) &&
1495 "Unexpected opcode!");
1496
1497 // We can't handle this case efficiently. Allocate a sufficiently
1498 // aligned object on the stack, store each operand into it, then load
1499 // the result as a vector.
1500 // Create the stack frame object.
1501 EVT VT = Node->getValueType(0);
1502 EVT MemVT = isa<BuildVectorSDNode>(Node) ? VT.getVectorElementType()
1503 : Node->getOperand(0).getValueType();
1504 SDLoc dl(Node);
1505 SDValue FIPtr = DAG.CreateStackTemporary(VT);
1506 int FI = cast<FrameIndexSDNode>(FIPtr.getNode())->getIndex();
1507 MachinePointerInfo PtrInfo =
1508 MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), FI);
1509
1510 // Emit a store of each element to the stack slot.
1512 unsigned TypeByteSize = MemVT.getSizeInBits() / 8;
1513 assert(TypeByteSize > 0 && "Vector element type too small for stack store!");
1514
1515 // If the destination vector element type of a BUILD_VECTOR is narrower than
1516 // the source element type, only store the bits necessary.
1517 bool Truncate = isa<BuildVectorSDNode>(Node) &&
1518 MemVT.bitsLT(Node->getOperand(0).getValueType());
1519
1520 // Store (in the right endianness) the elements to memory.
1521 for (unsigned i = 0, e = Node->getNumOperands(); i != e; ++i) {
1522 // Ignore undef elements.
1523 if (Node->getOperand(i).isUndef()) continue;
1524
1525 unsigned Offset = TypeByteSize*i;
1526
1527 SDValue Idx =
1528 DAG.getMemBasePlusOffset(FIPtr, TypeSize::getFixed(Offset), dl);
1529
1530 if (Truncate)
1531 Stores.push_back(DAG.getTruncStore(DAG.getEntryNode(), dl,
1532 Node->getOperand(i), Idx,
1533 PtrInfo.getWithOffset(Offset), MemVT));
1534 else
1535 Stores.push_back(DAG.getStore(DAG.getEntryNode(), dl, Node->getOperand(i),
1536 Idx, PtrInfo.getWithOffset(Offset)));
1537 }
1538
1539 SDValue StoreChain;
1540 if (!Stores.empty()) // Not all undef elements?
1541 StoreChain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Stores);
1542 else
1543 StoreChain = DAG.getEntryNode();
1544
1545 // Result is a load from the stack slot.
1546 return DAG.getLoad(VT, dl, StoreChain, FIPtr, PtrInfo);
1547}
1548
1549/// Bitcast a floating-point value to an integer value. Only bitcast the part
1550/// containing the sign bit if the target has no integer value capable of
1551/// holding all bits of the floating-point value.
1552void SelectionDAGLegalize::getSignAsIntValue(FloatSignAsInt &State,
1553 const SDLoc &DL,
1554 SDValue Value) const {
1555 EVT FloatVT = Value.getValueType();
1556 unsigned NumBits = FloatVT.getScalarSizeInBits();
1557 State.FloatVT = FloatVT;
1558 EVT IVT = EVT::getIntegerVT(*DAG.getContext(), NumBits);
1559 // Convert to an integer of the same size.
1560 if (TLI.isTypeLegal(IVT)) {
1561 State.IntValue = DAG.getNode(ISD::BITCAST, DL, IVT, Value);
1562 State.SignMask = APInt::getSignMask(NumBits);
1563 State.SignBit = NumBits - 1;
1564 return;
1565 }
1566
1567 auto &DataLayout = DAG.getDataLayout();
1568 // Store the float to memory, then load the sign part out as an integer.
1569 MVT LoadTy = TLI.getRegisterType(MVT::i8);
1570 // First create a temporary that is aligned for both the load and store.
1571 SDValue StackPtr = DAG.CreateStackTemporary(FloatVT, LoadTy);
1572 int FI = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex();
1573 // Then store the float to it.
1574 State.FloatPtr = StackPtr;
1575 MachineFunction &MF = DAG.getMachineFunction();
1576 State.FloatPointerInfo = MachinePointerInfo::getFixedStack(MF, FI);
1577 State.Chain = DAG.getStore(DAG.getEntryNode(), DL, Value, State.FloatPtr,
1578 State.FloatPointerInfo);
1579
1580 SDValue IntPtr;
1581 if (DataLayout.isBigEndian()) {
1582 assert(FloatVT.isByteSized() && "Unsupported floating point type!");
1583 // Load out a legal integer with the same sign bit as the float.
1584 IntPtr = StackPtr;
1585 State.IntPointerInfo = State.FloatPointerInfo;
1586 } else {
1587 // Advance the pointer so that the loaded byte will contain the sign bit.
1588 unsigned ByteOffset = (NumBits / 8) - 1;
1589 IntPtr =
1590 DAG.getMemBasePlusOffset(StackPtr, TypeSize::getFixed(ByteOffset), DL);
1591 State.IntPointerInfo = MachinePointerInfo::getFixedStack(MF, FI,
1592 ByteOffset);
1593 }
1594
1595 State.IntPtr = IntPtr;
1596 State.IntValue = DAG.getExtLoad(ISD::EXTLOAD, DL, LoadTy, State.Chain, IntPtr,
1597 State.IntPointerInfo, MVT::i8);
1598 State.SignMask = APInt::getOneBitSet(LoadTy.getScalarSizeInBits(), 7);
1599 State.SignBit = 7;
1600}
1601
1602/// Replace the integer value produced by getSignAsIntValue() with a new value
1603/// and cast the result back to a floating-point type.
1604SDValue SelectionDAGLegalize::modifySignAsInt(const FloatSignAsInt &State,
1605 const SDLoc &DL,
1606 SDValue NewIntValue) const {
1607 if (!State.Chain)
1608 return DAG.getNode(ISD::BITCAST, DL, State.FloatVT, NewIntValue);
1609
1610 // Override the part containing the sign bit in the value stored on the stack.
1611 SDValue Chain = DAG.getTruncStore(State.Chain, DL, NewIntValue, State.IntPtr,
1612 State.IntPointerInfo, MVT::i8);
1613 return DAG.getLoad(State.FloatVT, DL, Chain, State.FloatPtr,
1614 State.FloatPointerInfo);
1615}
1616
1617SDValue SelectionDAGLegalize::ExpandFCOPYSIGN(SDNode *Node) const {
1618 SDLoc DL(Node);
1619 SDValue Mag = Node->getOperand(0);
1620 SDValue Sign = Node->getOperand(1);
1621
1622 // Get sign bit into an integer value.
1623 FloatSignAsInt SignAsInt;
1624 getSignAsIntValue(SignAsInt, DL, Sign);
1625
1626 EVT IntVT = SignAsInt.IntValue.getValueType();
1627 SDValue SignMask = DAG.getConstant(SignAsInt.SignMask, DL, IntVT);
1628 SDValue SignBit = DAG.getNode(ISD::AND, DL, IntVT, SignAsInt.IntValue,
1629 SignMask);
1630
1631 // If FABS is legal transform FCOPYSIGN(x, y) => sign(x) ? -FABS(x) : FABS(X)
1632 EVT FloatVT = Mag.getValueType();
1633 if (TLI.isOperationLegalOrCustom(ISD::FABS, FloatVT) &&
1634 TLI.isOperationLegalOrCustom(ISD::FNEG, FloatVT)) {
1635 SDValue AbsValue = DAG.getNode(ISD::FABS, DL, FloatVT, Mag);
1636 SDValue NegValue = DAG.getNode(ISD::FNEG, DL, FloatVT, AbsValue);
1637 SDValue Cond = DAG.getSetCC(DL, getSetCCResultType(IntVT), SignBit,
1638 DAG.getConstant(0, DL, IntVT), ISD::SETNE);
1639 return DAG.getSelect(DL, FloatVT, Cond, NegValue, AbsValue);
1640 }
1641
1642 // Transform Mag value to integer, and clear the sign bit.
1643 FloatSignAsInt MagAsInt;
1644 getSignAsIntValue(MagAsInt, DL, Mag);
1645 EVT MagVT = MagAsInt.IntValue.getValueType();
1646 SDValue ClearSignMask = DAG.getConstant(~MagAsInt.SignMask, DL, MagVT);
1647 SDValue ClearedSign = DAG.getNode(ISD::AND, DL, MagVT, MagAsInt.IntValue,
1648 ClearSignMask);
1649
1650 // Get the signbit at the right position for MagAsInt.
1651 int ShiftAmount = SignAsInt.SignBit - MagAsInt.SignBit;
1652 EVT ShiftVT = IntVT;
1653 if (SignBit.getScalarValueSizeInBits() <
1654 ClearedSign.getScalarValueSizeInBits()) {
1655 SignBit = DAG.getNode(ISD::ZERO_EXTEND, DL, MagVT, SignBit);
1656 ShiftVT = MagVT;
1657 }
1658 if (ShiftAmount > 0) {
1659 SDValue ShiftCnst = DAG.getConstant(ShiftAmount, DL, ShiftVT);
1660 SignBit = DAG.getNode(ISD::SRL, DL, ShiftVT, SignBit, ShiftCnst);
1661 } else if (ShiftAmount < 0) {
1662 SDValue ShiftCnst = DAG.getConstant(-ShiftAmount, DL, ShiftVT);
1663 SignBit = DAG.getNode(ISD::SHL, DL, ShiftVT, SignBit, ShiftCnst);
1664 }
1665 if (SignBit.getScalarValueSizeInBits() >
1666 ClearedSign.getScalarValueSizeInBits()) {
1667 SignBit = DAG.getNode(ISD::TRUNCATE, DL, MagVT, SignBit);
1668 }
1669
1670 // Store the part with the modified sign and convert back to float.
1671 SDValue CopiedSign = DAG.getNode(ISD::OR, DL, MagVT, ClearedSign, SignBit);
1672 return modifySignAsInt(MagAsInt, DL, CopiedSign);
1673}
1674
1675SDValue SelectionDAGLegalize::ExpandFNEG(SDNode *Node) const {
1676 // Get the sign bit as an integer.
1677 SDLoc DL(Node);
1678 FloatSignAsInt SignAsInt;
1679 getSignAsIntValue(SignAsInt, DL, Node->getOperand(0));
1680 EVT IntVT = SignAsInt.IntValue.getValueType();
1681
1682 // Flip the sign.
1683 SDValue SignMask = DAG.getConstant(SignAsInt.SignMask, DL, IntVT);
1684 SDValue SignFlip =
1685 DAG.getNode(ISD::XOR, DL, IntVT, SignAsInt.IntValue, SignMask);
1686
1687 // Convert back to float.
1688 return modifySignAsInt(SignAsInt, DL, SignFlip);
1689}
1690
1691SDValue SelectionDAGLegalize::ExpandFABS(SDNode *Node) const {
1692 SDLoc DL(Node);
1693 SDValue Value = Node->getOperand(0);
1694
1695 // Transform FABS(x) => FCOPYSIGN(x, 0.0) if FCOPYSIGN is legal.
1696 EVT FloatVT = Value.getValueType();
1697 if (TLI.isOperationLegalOrCustom(ISD::FCOPYSIGN, FloatVT)) {
1698 SDValue Zero = DAG.getConstantFP(0.0, DL, FloatVT);
1699 return DAG.getNode(ISD::FCOPYSIGN, DL, FloatVT, Value, Zero);
1700 }
1701
1702 // Transform value to integer, clear the sign bit and transform back.
1703 FloatSignAsInt ValueAsInt;
1704 getSignAsIntValue(ValueAsInt, DL, Value);
1705 EVT IntVT = ValueAsInt.IntValue.getValueType();
1706 SDValue ClearSignMask = DAG.getConstant(~ValueAsInt.SignMask, DL, IntVT);
1707 SDValue ClearedSign = DAG.getNode(ISD::AND, DL, IntVT, ValueAsInt.IntValue,
1708 ClearSignMask);
1709 return modifySignAsInt(ValueAsInt, DL, ClearedSign);
1710}
1711
1712void SelectionDAGLegalize::ExpandDYNAMIC_STACKALLOC(SDNode* Node,
1714 Register SPReg = TLI.getStackPointerRegisterToSaveRestore();
1715 assert(SPReg && "Target cannot require DYNAMIC_STACKALLOC expansion and"
1716 " not tell us which reg is the stack pointer!");
1717 SDLoc dl(Node);
1718 EVT VT = Node->getValueType(0);
1719 SDValue Tmp1 = SDValue(Node, 0);
1720 SDValue Tmp2 = SDValue(Node, 1);
1721 SDValue Tmp3 = Node->getOperand(2);
1722 SDValue Chain = Tmp1.getOperand(0);
1723
1724 // Chain the dynamic stack allocation so that it doesn't modify the stack
1725 // pointer when other instructions are using the stack.
1726 Chain = DAG.getCALLSEQ_START(Chain, 0, 0, dl);
1727
1728 SDValue Size = Tmp2.getOperand(1);
1729 SDValue SP = DAG.getCopyFromReg(Chain, dl, SPReg, VT);
1730 Chain = SP.getValue(1);
1731 Align Alignment = cast<ConstantSDNode>(Tmp3)->getAlignValue();
1732 const TargetFrameLowering *TFL = DAG.getSubtarget().getFrameLowering();
1733 unsigned Opc =
1736
1738 Tmp1 = DAG.getNode(Opc, dl, VT, SP, Size); // Value
1739 if (Alignment > StackAlign)
1740 Tmp1 = DAG.getNode(ISD::AND, dl, VT, Tmp1,
1741 DAG.getConstant(-Alignment.value(), dl, VT));
1742 Chain = DAG.getCopyToReg(Chain, dl, SPReg, Tmp1); // Output chain
1743
1744 Tmp2 = DAG.getCALLSEQ_END(Chain, 0, 0, SDValue(), dl);
1745
1746 Results.push_back(Tmp1);
1747 Results.push_back(Tmp2);
1748}
1749
1750/// Emit a store/load combination to the stack. This stores
1751/// SrcOp to a stack slot of type SlotVT, truncating it if needed. It then does
1752/// a load from the stack slot to DestVT, extending it if needed.
1753/// The resultant code need not be legal.
1754SDValue SelectionDAGLegalize::EmitStackConvert(SDValue SrcOp, EVT SlotVT,
1755 EVT DestVT, const SDLoc &dl) {
1756 return EmitStackConvert(SrcOp, SlotVT, DestVT, dl, DAG.getEntryNode());
1757}
1758
1759SDValue SelectionDAGLegalize::EmitStackConvert(SDValue SrcOp, EVT SlotVT,
1760 EVT DestVT, const SDLoc &dl,
1761 SDValue Chain) {
1762 EVT SrcVT = SrcOp.getValueType();
1763 Type *DestType = DestVT.getTypeForEVT(*DAG.getContext());
1764 Align DestAlign = DAG.getDataLayout().getPrefTypeAlign(DestType);
1765
1766 // Don't convert with stack if the load/store is expensive.
1767 if ((SrcVT.bitsGT(SlotVT) &&
1768 !TLI.isTruncStoreLegalOrCustom(SrcOp.getValueType(), SlotVT)) ||
1769 (SlotVT.bitsLT(DestVT) &&
1770 !TLI.isLoadExtLegalOrCustom(ISD::EXTLOAD, DestVT, SlotVT)))
1771 return SDValue();
1772
1773 // Create the stack frame object.
1774 Align SrcAlign = DAG.getDataLayout().getPrefTypeAlign(
1775 SrcOp.getValueType().getTypeForEVT(*DAG.getContext()));
1776 SDValue FIPtr = DAG.CreateStackTemporary(SlotVT.getStoreSize(), SrcAlign);
1777
1778 FrameIndexSDNode *StackPtrFI = cast<FrameIndexSDNode>(FIPtr);
1779 int SPFI = StackPtrFI->getIndex();
1780 MachinePointerInfo PtrInfo =
1781 MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), SPFI);
1782
1783 // Emit a store to the stack slot. Use a truncstore if the input value is
1784 // later than DestVT.
1785 SDValue Store;
1786
1787 if (SrcVT.bitsGT(SlotVT))
1788 Store = DAG.getTruncStore(Chain, dl, SrcOp, FIPtr, PtrInfo,
1789 SlotVT, SrcAlign);
1790 else {
1791 assert(SrcVT.bitsEq(SlotVT) && "Invalid store");
1792 Store = DAG.getStore(Chain, dl, SrcOp, FIPtr, PtrInfo, SrcAlign);
1793 }
1794
1795 // Result is a load from the stack slot.
1796 if (SlotVT.bitsEq(DestVT))
1797 return DAG.getLoad(DestVT, dl, Store, FIPtr, PtrInfo, DestAlign);
1798
1799 assert(SlotVT.bitsLT(DestVT) && "Unknown extension!");
1800 return DAG.getExtLoad(ISD::EXTLOAD, dl, DestVT, Store, FIPtr, PtrInfo, SlotVT,
1801 DestAlign);
1802}
1803
1804SDValue SelectionDAGLegalize::ExpandSCALAR_TO_VECTOR(SDNode *Node) {
1805 SDLoc dl(Node);
1806 // Create a vector sized/aligned stack slot, store the value to element #0,
1807 // then load the whole vector back out.
1808 SDValue StackPtr = DAG.CreateStackTemporary(Node->getValueType(0));
1809
1810 FrameIndexSDNode *StackPtrFI = cast<FrameIndexSDNode>(StackPtr);
1811 int SPFI = StackPtrFI->getIndex();
1812
1813 SDValue Ch = DAG.getTruncStore(
1814 DAG.getEntryNode(), dl, Node->getOperand(0), StackPtr,
1815 MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), SPFI),
1816 Node->getValueType(0).getVectorElementType());
1817 return DAG.getLoad(
1818 Node->getValueType(0), dl, Ch, StackPtr,
1819 MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), SPFI));
1820}
1821
1822static bool
1824 const TargetLowering &TLI, SDValue &Res) {
1825 unsigned NumElems = Node->getNumOperands();
1826 SDLoc dl(Node);
1827 EVT VT = Node->getValueType(0);
1828
1829 // Try to group the scalars into pairs, shuffle the pairs together, then
1830 // shuffle the pairs of pairs together, etc. until the vector has
1831 // been built. This will work only if all of the necessary shuffle masks
1832 // are legal.
1833
1834 // We do this in two phases; first to check the legality of the shuffles,
1835 // and next, assuming that all shuffles are legal, to create the new nodes.
1836 for (int Phase = 0; Phase < 2; ++Phase) {
1838 NewIntermedVals;
1839 for (unsigned i = 0; i < NumElems; ++i) {
1840 SDValue V = Node->getOperand(i);
1841 if (V.isUndef())
1842 continue;
1843
1844 SDValue Vec;
1845 if (Phase)
1846 Vec = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, V);
1847 IntermedVals.push_back(std::make_pair(Vec, SmallVector<int, 16>(1, i)));
1848 }
1849
1850 while (IntermedVals.size() > 2) {
1851 NewIntermedVals.clear();
1852 for (unsigned i = 0, e = (IntermedVals.size() & ~1u); i < e; i += 2) {
1853 // This vector and the next vector are shuffled together (simply to
1854 // append the one to the other).
1855 SmallVector<int, 16> ShuffleVec(NumElems, -1);
1856
1857 SmallVector<int, 16> FinalIndices;
1858 FinalIndices.reserve(IntermedVals[i].second.size() +
1859 IntermedVals[i+1].second.size());
1860
1861 int k = 0;
1862 for (unsigned j = 0, f = IntermedVals[i].second.size(); j != f;
1863 ++j, ++k) {
1864 ShuffleVec[k] = j;
1865 FinalIndices.push_back(IntermedVals[i].second[j]);
1866 }
1867 for (unsigned j = 0, f = IntermedVals[i+1].second.size(); j != f;
1868 ++j, ++k) {
1869 ShuffleVec[k] = NumElems + j;
1870 FinalIndices.push_back(IntermedVals[i+1].second[j]);
1871 }
1872
1873 SDValue Shuffle;
1874 if (Phase)
1875 Shuffle = DAG.getVectorShuffle(VT, dl, IntermedVals[i].first,
1876 IntermedVals[i+1].first,
1877 ShuffleVec);
1878 else if (!TLI.isShuffleMaskLegal(ShuffleVec, VT))
1879 return false;
1880 NewIntermedVals.push_back(
1881 std::make_pair(Shuffle, std::move(FinalIndices)));
1882 }
1883
1884 // If we had an odd number of defined values, then append the last
1885 // element to the array of new vectors.
1886 if ((IntermedVals.size() & 1) != 0)
1887 NewIntermedVals.push_back(IntermedVals.back());
1888
1889 IntermedVals.swap(NewIntermedVals);
1890 }
1891
1892 assert(IntermedVals.size() <= 2 && IntermedVals.size() > 0 &&
1893 "Invalid number of intermediate vectors");
1894 SDValue Vec1 = IntermedVals[0].first;
1895 SDValue Vec2;
1896 if (IntermedVals.size() > 1)
1897 Vec2 = IntermedVals[1].first;
1898 else if (Phase)
1899 Vec2 = DAG.getUNDEF(VT);
1900
1901 SmallVector<int, 16> ShuffleVec(NumElems, -1);
1902 for (unsigned i = 0, e = IntermedVals[0].second.size(); i != e; ++i)
1903 ShuffleVec[IntermedVals[0].second[i]] = i;
1904 for (unsigned i = 0, e = IntermedVals[1].second.size(); i != e; ++i)
1905 ShuffleVec[IntermedVals[1].second[i]] = NumElems + i;
1906
1907 if (Phase)
1908 Res = DAG.getVectorShuffle(VT, dl, Vec1, Vec2, ShuffleVec);
1909 else if (!TLI.isShuffleMaskLegal(ShuffleVec, VT))
1910 return false;
1911 }
1912
1913 return true;
1914}
1915
1916/// Expand a BUILD_VECTOR node on targets that don't
1917/// support the operation, but do support the resultant vector type.
1918SDValue SelectionDAGLegalize::ExpandBUILD_VECTOR(SDNode *Node) {
1919 unsigned NumElems = Node->getNumOperands();
1920 SDValue Value1, Value2;
1921 SDLoc dl(Node);
1922 EVT VT = Node->getValueType(0);
1923 EVT OpVT = Node->getOperand(0).getValueType();
1924 EVT EltVT = VT.getVectorElementType();
1925
1926 // If the only non-undef value is the low element, turn this into a
1927 // SCALAR_TO_VECTOR node. If this is { X, X, X, X }, determine X.
1928 bool isOnlyLowElement = true;
1929 bool MoreThanTwoValues = false;
1930 bool isConstant = true;
1931 for (unsigned i = 0; i < NumElems; ++i) {
1932 SDValue V = Node->getOperand(i);
1933 if (V.isUndef())
1934 continue;
1935 if (i > 0)
1936 isOnlyLowElement = false;
1937 if (!isa<ConstantFPSDNode>(V) && !isa<ConstantSDNode>(V))
1938 isConstant = false;
1939
1940 if (!Value1.getNode()) {
1941 Value1 = V;
1942 } else if (!Value2.getNode()) {
1943 if (V != Value1)
1944 Value2 = V;
1945 } else if (V != Value1 && V != Value2) {
1946 MoreThanTwoValues = true;
1947 }
1948 }
1949
1950 if (!Value1.getNode())
1951 return DAG.getUNDEF(VT);
1952
1953 if (isOnlyLowElement)
1954 return DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, Node->getOperand(0));
1955
1956 // If all elements are constants, create a load from the constant pool.
1957 if (isConstant) {
1959 for (unsigned i = 0, e = NumElems; i != e; ++i) {
1960 if (ConstantFPSDNode *V =
1961 dyn_cast<ConstantFPSDNode>(Node->getOperand(i))) {
1962 CV.push_back(const_cast<ConstantFP *>(V->getConstantFPValue()));
1963 } else if (ConstantSDNode *V =
1964 dyn_cast<ConstantSDNode>(Node->getOperand(i))) {
1965 if (OpVT==EltVT)
1966 CV.push_back(const_cast<ConstantInt *>(V->getConstantIntValue()));
1967 else {
1968 // If OpVT and EltVT don't match, EltVT is not legal and the
1969 // element values have been promoted/truncated earlier. Undo this;
1970 // we don't want a v16i8 to become a v16i32 for example.
1971 const ConstantInt *CI = V->getConstantIntValue();
1972 CV.push_back(ConstantInt::get(EltVT.getTypeForEVT(*DAG.getContext()),
1973 CI->getZExtValue()));
1974 }
1975 } else {
1976 assert(Node->getOperand(i).isUndef());
1977 Type *OpNTy = EltVT.getTypeForEVT(*DAG.getContext());
1978 CV.push_back(UndefValue::get(OpNTy));
1979 }
1980 }
1982 SDValue CPIdx =
1983 DAG.getConstantPool(CP, TLI.getPointerTy(DAG.getDataLayout()));
1984 Align Alignment = cast<ConstantPoolSDNode>(CPIdx)->getAlign();
1985 return DAG.getLoad(
1986 VT, dl, DAG.getEntryNode(), CPIdx,
1987 MachinePointerInfo::getConstantPool(DAG.getMachineFunction()),
1988 Alignment);
1989 }
1990
1991 SmallSet<SDValue, 16> DefinedValues;
1992 for (unsigned i = 0; i < NumElems; ++i) {
1993 if (Node->getOperand(i).isUndef())
1994 continue;
1995 DefinedValues.insert(Node->getOperand(i));
1996 }
1997
1998 if (TLI.shouldExpandBuildVectorWithShuffles(VT, DefinedValues.size())) {
1999 if (!MoreThanTwoValues) {
2000 SmallVector<int, 8> ShuffleVec(NumElems, -1);
2001 for (unsigned i = 0; i < NumElems; ++i) {
2002 SDValue V = Node->getOperand(i);
2003 if (V.isUndef())
2004 continue;
2005 ShuffleVec[i] = V == Value1 ? 0 : NumElems;
2006 }
2007 if (TLI.isShuffleMaskLegal(ShuffleVec, Node->getValueType(0))) {
2008 // Get the splatted value into the low element of a vector register.
2009 SDValue Vec1 = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, Value1);
2010 SDValue Vec2;
2011 if (Value2.getNode())
2012 Vec2 = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, Value2);
2013 else
2014 Vec2 = DAG.getUNDEF(VT);
2015
2016 // Return shuffle(LowValVec, undef, <0,0,0,0>)
2017 return DAG.getVectorShuffle(VT, dl, Vec1, Vec2, ShuffleVec);
2018 }
2019 } else {
2020 SDValue Res;
2021 if (ExpandBVWithShuffles(Node, DAG, TLI, Res))
2022 return Res;
2023 }
2024 }
2025
2026 // Otherwise, we can't handle this case efficiently.
2027 return ExpandVectorBuildThroughStack(Node);
2028}
2029
2030SDValue SelectionDAGLegalize::ExpandSPLAT_VECTOR(SDNode *Node) {
2031 SDLoc DL(Node);
2032 EVT VT = Node->getValueType(0);
2033 SDValue SplatVal = Node->getOperand(0);
2034
2035 return DAG.getSplatBuildVector(VT, DL, SplatVal);
2036}
2037
2038// Expand a node into a call to a libcall, returning the value as the first
2039// result and the chain as the second. If the result value does not fit into a
2040// register, return the lo part and set the hi part to the by-reg argument in
2041// the first. If it does fit into a single register, return the result and
2042// leave the Hi part unset.
2043std::pair<SDValue, SDValue> SelectionDAGLegalize::ExpandLibCall(RTLIB::Libcall LC, SDNode *Node,
2045 bool isSigned) {
2046 SDValue Callee = DAG.getExternalSymbol(TLI.getLibcallName(LC),
2047 TLI.getPointerTy(DAG.getDataLayout()));
2048
2049 EVT RetVT = Node->getValueType(0);
2050 Type *RetTy = RetVT.getTypeForEVT(*DAG.getContext());
2051
2052 // By default, the input chain to this libcall is the entry node of the
2053 // function. If the libcall is going to be emitted as a tail call then
2054 // TLI.isUsedByReturnOnly will change it to the right chain if the return
2055 // node which is being folded has a non-entry input chain.
2056 SDValue InChain = DAG.getEntryNode();
2057
2058 // isTailCall may be true since the callee does not reference caller stack
2059 // frame. Check if it's in the right position and that the return types match.
2060 SDValue TCChain = InChain;
2061 const Function &F = DAG.getMachineFunction().getFunction();
2062 bool isTailCall =
2063 TLI.isInTailCallPosition(DAG, Node, TCChain) &&
2064 (RetTy == F.getReturnType() || F.getReturnType()->isVoidTy());
2065 if (isTailCall)
2066 InChain = TCChain;
2067
2069 bool signExtend = TLI.shouldSignExtendTypeInLibCall(RetVT, isSigned);
2070 CLI.setDebugLoc(SDLoc(Node))
2071 .setChain(InChain)
2072 .setLibCallee(TLI.getLibcallCallingConv(LC), RetTy, Callee,
2073 std::move(Args))
2074 .setTailCall(isTailCall)
2075 .setSExtResult(signExtend)
2076 .setZExtResult(!signExtend)
2077 .setIsPostTypeLegalization(true);
2078
2079 std::pair<SDValue, SDValue> CallInfo = TLI.LowerCallTo(CLI);
2080
2081 if (!CallInfo.second.getNode()) {
2082 LLVM_DEBUG(dbgs() << "Created tailcall: "; DAG.getRoot().dump(&DAG));
2083 // It's a tailcall, return the chain (which is the DAG root).
2084 return {DAG.getRoot(), DAG.getRoot()};
2085 }
2086
2087 LLVM_DEBUG(dbgs() << "Created libcall: "; CallInfo.first.dump(&DAG));
2088 return CallInfo;
2089}
2090
2091std::pair<SDValue, SDValue> SelectionDAGLegalize::ExpandLibCall(RTLIB::Libcall LC, SDNode *Node,
2092 bool isSigned) {
2095 for (const SDValue &Op : Node->op_values()) {
2096 EVT ArgVT = Op.getValueType();
2097 Type *ArgTy = ArgVT.getTypeForEVT(*DAG.getContext());
2098 Entry.Node = Op;
2099 Entry.Ty = ArgTy;
2100 Entry.IsSExt = TLI.shouldSignExtendTypeInLibCall(ArgVT, isSigned);
2101 Entry.IsZExt = !Entry.IsSExt;
2102 Args.push_back(Entry);
2103 }
2104
2105 return ExpandLibCall(LC, Node, std::move(Args), isSigned);
2106}
2107
2108void SelectionDAGLegalize::ExpandFrexpLibCall(
2110 SDLoc dl(Node);
2111 EVT VT = Node->getValueType(0);
2112 EVT ExpVT = Node->getValueType(1);
2113
2114 SDValue FPOp = Node->getOperand(0);
2115
2116 EVT ArgVT = FPOp.getValueType();
2117 Type *ArgTy = ArgVT.getTypeForEVT(*DAG.getContext());
2118
2120 FPArgEntry.Node = FPOp;
2121 FPArgEntry.Ty = ArgTy;
2122
2123 SDValue StackSlot = DAG.CreateStackTemporary(ExpVT);
2124 TargetLowering::ArgListEntry PtrArgEntry;
2125 PtrArgEntry.Node = StackSlot;
2126 PtrArgEntry.Ty = PointerType::get(*DAG.getContext(),
2127 DAG.getDataLayout().getAllocaAddrSpace());
2128
2129 TargetLowering::ArgListTy Args = {FPArgEntry, PtrArgEntry};
2130
2132 auto [Call, Chain] = ExpandLibCall(LC, Node, std::move(Args), false);
2133
2134 // FIXME: Get type of int for libcall declaration and cast
2135
2136 int FrameIdx = cast<FrameIndexSDNode>(StackSlot)->getIndex();
2137 auto PtrInfo =
2138 MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), FrameIdx);
2139
2140 SDValue LoadExp = DAG.getLoad(ExpVT, dl, Chain, StackSlot, PtrInfo);
2141 SDValue OutputChain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other,
2142 LoadExp.getValue(1), DAG.getRoot());
2143 DAG.setRoot(OutputChain);
2144
2145 Results.push_back(Call);
2146 Results.push_back(LoadExp);
2147}
2148
2149void SelectionDAGLegalize::ExpandFPLibCall(SDNode* Node,
2150 RTLIB::Libcall LC,
2152 if (LC == RTLIB::UNKNOWN_LIBCALL)
2153 llvm_unreachable("Can't create an unknown libcall!");
2154
2155 if (Node->isStrictFPOpcode()) {
2156 EVT RetVT = Node->getValueType(0);
2159 // FIXME: This doesn't support tail calls.
2160 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, RetVT,
2161 Ops, CallOptions,
2162 SDLoc(Node),
2163 Node->getOperand(0));
2164 Results.push_back(Tmp.first);
2165 Results.push_back(Tmp.second);
2166 } else {
2167 SDValue Tmp = ExpandLibCall(LC, Node, false).first;
2168 Results.push_back(Tmp);
2169 }
2170}
2171
2172/// Expand the node to a libcall based on the result type.
2173void SelectionDAGLegalize::ExpandFPLibCall(SDNode* Node,
2174 RTLIB::Libcall Call_F32,
2175 RTLIB::Libcall Call_F64,
2176 RTLIB::Libcall Call_F80,
2177 RTLIB::Libcall Call_F128,
2178 RTLIB::Libcall Call_PPCF128,
2180 RTLIB::Libcall LC = RTLIB::getFPLibCall(Node->getSimpleValueType(0),
2181 Call_F32, Call_F64, Call_F80,
2182 Call_F128, Call_PPCF128);
2183 ExpandFPLibCall(Node, LC, Results);
2184}
2185
2186SDValue SelectionDAGLegalize::ExpandIntLibCall(SDNode* Node, bool isSigned,
2187 RTLIB::Libcall Call_I8,
2188 RTLIB::Libcall Call_I16,
2189 RTLIB::Libcall Call_I32,
2190 RTLIB::Libcall Call_I64,
2191 RTLIB::Libcall Call_I128) {
2192 RTLIB::Libcall LC;
2193 switch (Node->getSimpleValueType(0).SimpleTy) {
2194 default: llvm_unreachable("Unexpected request for libcall!");
2195 case MVT::i8: LC = Call_I8; break;
2196 case MVT::i16: LC = Call_I16; break;
2197 case MVT::i32: LC = Call_I32; break;
2198 case MVT::i64: LC = Call_I64; break;
2199 case MVT::i128: LC = Call_I128; break;
2200 }
2201 return ExpandLibCall(LC, Node, isSigned).first;
2202}
2203
2204/// Expand the node to a libcall based on first argument type (for instance
2205/// lround and its variant).
2206void SelectionDAGLegalize::ExpandArgFPLibCall(SDNode* Node,
2207 RTLIB::Libcall Call_F32,
2208 RTLIB::Libcall Call_F64,
2209 RTLIB::Libcall Call_F80,
2210 RTLIB::Libcall Call_F128,
2211 RTLIB::Libcall Call_PPCF128,
2213 EVT InVT = Node->getOperand(Node->isStrictFPOpcode() ? 1 : 0).getValueType();
2215 Call_F32, Call_F64, Call_F80,
2216 Call_F128, Call_PPCF128);
2217 ExpandFPLibCall(Node, LC, Results);
2218}
2219
2220/// Issue libcalls to __{u}divmod to compute div / rem pairs.
2221void
2222SelectionDAGLegalize::ExpandDivRemLibCall(SDNode *Node,
2224 unsigned Opcode = Node->getOpcode();
2225 bool isSigned = Opcode == ISD::SDIVREM;
2226
2227 RTLIB::Libcall LC;
2228 switch (Node->getSimpleValueType(0).SimpleTy) {
2229 default: llvm_unreachable("Unexpected request for libcall!");
2230 case MVT::i8: LC= isSigned ? RTLIB::SDIVREM_I8 : RTLIB::UDIVREM_I8; break;
2231 case MVT::i16: LC= isSigned ? RTLIB::SDIVREM_I16 : RTLIB::UDIVREM_I16; break;
2232 case MVT::i32: LC= isSigned ? RTLIB::SDIVREM_I32 : RTLIB::UDIVREM_I32; break;
2233 case MVT::i64: LC= isSigned ? RTLIB::SDIVREM_I64 : RTLIB::UDIVREM_I64; break;
2234 case MVT::i128: LC= isSigned ? RTLIB::SDIVREM_I128:RTLIB::UDIVREM_I128; break;
2235 }
2236
2237 // The input chain to this libcall is the entry node of the function.
2238 // Legalizing the call will automatically add the previous call to the
2239 // dependence.
2240 SDValue InChain = DAG.getEntryNode();
2241
2242 EVT RetVT = Node->getValueType(0);
2243 Type *RetTy = RetVT.getTypeForEVT(*DAG.getContext());
2244
2247 for (const SDValue &Op : Node->op_values()) {
2248 EVT ArgVT = Op.getValueType();
2249 Type *ArgTy = ArgVT.getTypeForEVT(*DAG.getContext());
2250 Entry.Node = Op;
2251 Entry.Ty = ArgTy;
2252 Entry.IsSExt = isSigned;
2253 Entry.IsZExt = !isSigned;
2254 Args.push_back(Entry);
2255 }
2256
2257 // Also pass the return address of the remainder.
2258 SDValue FIPtr = DAG.CreateStackTemporary(RetVT);
2259 Entry.Node = FIPtr;
2260 Entry.Ty = PointerType::getUnqual(RetTy->getContext());
2261 Entry.IsSExt = isSigned;
2262 Entry.IsZExt = !isSigned;
2263 Args.push_back(Entry);
2264
2265 SDValue Callee = DAG.getExternalSymbol(TLI.getLibcallName(LC),
2266 TLI.getPointerTy(DAG.getDataLayout()));
2267
2268 SDLoc dl(Node);
2270 CLI.setDebugLoc(dl)
2271 .setChain(InChain)
2272 .setLibCallee(TLI.getLibcallCallingConv(LC), RetTy, Callee,
2273 std::move(Args))
2274 .setSExtResult(isSigned)
2275 .setZExtResult(!isSigned);
2276
2277 std::pair<SDValue, SDValue> CallInfo = TLI.LowerCallTo(CLI);
2278
2279 // Remainder is loaded back from the stack frame.
2280 SDValue Rem =
2281 DAG.getLoad(RetVT, dl, CallInfo.second, FIPtr, MachinePointerInfo());
2282 Results.push_back(CallInfo.first);
2283 Results.push_back(Rem);
2284}
2285
2286/// Return true if sincos libcall is available.
2288 RTLIB::Libcall LC;
2289 switch (Node->getSimpleValueType(0).SimpleTy) {
2290 default: llvm_unreachable("Unexpected request for libcall!");
2291 case MVT::f32: LC = RTLIB::SINCOS_F32; break;
2292 case MVT::f64: LC = RTLIB::SINCOS_F64; break;
2293 case MVT::f80: LC = RTLIB::SINCOS_F80; break;
2294 case MVT::f128: LC = RTLIB::SINCOS_F128; break;
2295 case MVT::ppcf128: LC = RTLIB::SINCOS_PPCF128; break;
2296 }
2297 return TLI.getLibcallName(LC) != nullptr;
2298}
2299
2300/// Only issue sincos libcall if both sin and cos are needed.
2301static bool useSinCos(SDNode *Node) {
2302 unsigned OtherOpcode = Node->getOpcode() == ISD::FSIN
2303 ? ISD::FCOS : ISD::FSIN;
2304
2305 SDValue Op0 = Node->getOperand(0);
2306 for (const SDNode *User : Op0.getNode()->uses()) {
2307 if (User == Node)
2308 continue;
2309 // The other user might have been turned into sincos already.
2310 if (User->getOpcode() == OtherOpcode || User->getOpcode() == ISD::FSINCOS)
2311 return true;
2312 }
2313 return false;
2314}
2315
2316/// Issue libcalls to sincos to compute sin / cos pairs.
2317void
2318SelectionDAGLegalize::ExpandSinCosLibCall(SDNode *Node,
2320 RTLIB::Libcall LC;
2321 switch (Node->getSimpleValueType(0).SimpleTy) {
2322 default: llvm_unreachable("Unexpected request for libcall!");
2323 case MVT::f32: LC = RTLIB::SINCOS_F32; break;
2324 case MVT::f64: LC = RTLIB::SINCOS_F64; break;
2325 case MVT::f80: LC = RTLIB::SINCOS_F80; break;
2326 case MVT::f128: LC = RTLIB::SINCOS_F128; break;
2327 case MVT::ppcf128: LC = RTLIB::SINCOS_PPCF128; break;
2328 }
2329
2330 // The input chain to this libcall is the entry node of the function.
2331 // Legalizing the call will automatically add the previous call to the
2332 // dependence.
2333 SDValue InChain = DAG.getEntryNode();
2334
2335 EVT RetVT = Node->getValueType(0);
2336 Type *RetTy = RetVT.getTypeForEVT(*DAG.getContext());
2337
2340
2341 // Pass the argument.
2342 Entry.Node = Node->getOperand(0);
2343 Entry.Ty = RetTy;
2344 Entry.IsSExt = false;
2345 Entry.IsZExt = false;
2346 Args.push_back(Entry);
2347
2348 // Pass the return address of sin.
2349 SDValue SinPtr = DAG.CreateStackTemporary(RetVT);
2350 Entry.Node = SinPtr;
2351 Entry.Ty = PointerType::getUnqual(RetTy->getContext());
2352 Entry.IsSExt = false;
2353 Entry.IsZExt = false;
2354 Args.push_back(Entry);
2355
2356 // Also pass the return address of the cos.
2357 SDValue CosPtr = DAG.CreateStackTemporary(RetVT);
2358 Entry.Node = CosPtr;
2359 Entry.Ty = PointerType::getUnqual(RetTy->getContext());
2360 Entry.IsSExt = false;
2361 Entry.IsZExt = false;
2362 Args.push_back(Entry);
2363
2364 SDValue Callee = DAG.getExternalSymbol(TLI.getLibcallName(LC),
2365 TLI.getPointerTy(DAG.getDataLayout()));
2366
2367 SDLoc dl(Node);
2369 CLI.setDebugLoc(dl).setChain(InChain).setLibCallee(
2370 TLI.getLibcallCallingConv(LC), Type::getVoidTy(*DAG.getContext()), Callee,
2371 std::move(Args));
2372
2373 std::pair<SDValue, SDValue> CallInfo = TLI.LowerCallTo(CLI);
2374
2375 Results.push_back(
2376 DAG.getLoad(RetVT, dl, CallInfo.second, SinPtr, MachinePointerInfo()));
2377 Results.push_back(
2378 DAG.getLoad(RetVT, dl, CallInfo.second, CosPtr, MachinePointerInfo()));
2379}
2380
2381SDValue SelectionDAGLegalize::expandLdexp(SDNode *Node) const {
2382 SDLoc dl(Node);
2383 EVT VT = Node->getValueType(0);
2384 SDValue X = Node->getOperand(0);
2385 SDValue N = Node->getOperand(1);
2386 EVT ExpVT = N.getValueType();
2387 EVT AsIntVT = VT.changeTypeToInteger();
2388 if (AsIntVT == EVT()) // TODO: How to handle f80?
2389 return SDValue();
2390
2391 if (Node->getOpcode() == ISD::STRICT_FLDEXP) // TODO
2392 return SDValue();
2393
2394 SDNodeFlags NSW;
2395 NSW.setNoSignedWrap(true);
2396 SDNodeFlags NUW_NSW;
2397 NUW_NSW.setNoUnsignedWrap(true);
2398 NUW_NSW.setNoSignedWrap(true);
2399
2400 EVT SetCCVT =
2401 TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), ExpVT);
2403
2404 const APFloat::ExponentType MaxExpVal = APFloat::semanticsMaxExponent(FltSem);
2405 const APFloat::ExponentType MinExpVal = APFloat::semanticsMinExponent(FltSem);
2406 const int Precision = APFloat::semanticsPrecision(FltSem);
2407
2408 const SDValue MaxExp = DAG.getConstant(MaxExpVal, dl, ExpVT);
2409 const SDValue MinExp = DAG.getConstant(MinExpVal, dl, ExpVT);
2410
2411 const SDValue DoubleMaxExp = DAG.getConstant(2 * MaxExpVal, dl, ExpVT);
2412
2413 const APFloat One(FltSem, "1.0");
2414 APFloat ScaleUpK = scalbn(One, MaxExpVal, APFloat::rmNearestTiesToEven);
2415
2416 // Offset by precision to avoid denormal range.
2417 APFloat ScaleDownK =
2418 scalbn(One, MinExpVal + Precision, APFloat::rmNearestTiesToEven);
2419
2420 // TODO: Should really introduce control flow and use a block for the >
2421 // MaxExp, < MinExp cases
2422
2423 // First, handle exponents Exp > MaxExp and scale down.
2424 SDValue NGtMaxExp = DAG.getSetCC(dl, SetCCVT, N, MaxExp, ISD::SETGT);
2425
2426 SDValue DecN0 = DAG.getNode(ISD::SUB, dl, ExpVT, N, MaxExp, NSW);
2427 SDValue ClampMaxVal = DAG.getConstant(3 * MaxExpVal, dl, ExpVT);
2428 SDValue ClampN_Big = DAG.getNode(ISD::SMIN, dl, ExpVT, N, ClampMaxVal);
2429 SDValue DecN1 =
2430 DAG.getNode(ISD::SUB, dl, ExpVT, ClampN_Big, DoubleMaxExp, NSW);
2431
2432 SDValue ScaleUpTwice =
2433 DAG.getSetCC(dl, SetCCVT, N, DoubleMaxExp, ISD::SETUGT);
2434
2435 const SDValue ScaleUpVal = DAG.getConstantFP(ScaleUpK, dl, VT);
2436 SDValue ScaleUp0 = DAG.getNode(ISD::FMUL, dl, VT, X, ScaleUpVal);
2437 SDValue ScaleUp1 = DAG.getNode(ISD::FMUL, dl, VT, ScaleUp0, ScaleUpVal);
2438
2439 SDValue SelectN_Big =
2440 DAG.getNode(ISD::SELECT, dl, ExpVT, ScaleUpTwice, DecN1, DecN0);
2441 SDValue SelectX_Big =
2442 DAG.getNode(ISD::SELECT, dl, VT, ScaleUpTwice, ScaleUp1, ScaleUp0);
2443
2444 // Now handle exponents Exp < MinExp
2445 SDValue NLtMinExp = DAG.getSetCC(dl, SetCCVT, N, MinExp, ISD::SETLT);
2446
2447 SDValue Increment0 = DAG.getConstant(-(MinExpVal + Precision), dl, ExpVT);
2448 SDValue Increment1 = DAG.getConstant(-2 * (MinExpVal + Precision), dl, ExpVT);
2449
2450 SDValue IncN0 = DAG.getNode(ISD::ADD, dl, ExpVT, N, Increment0, NUW_NSW);
2451
2452 SDValue ClampMinVal =
2453 DAG.getConstant(3 * MinExpVal + 2 * Precision, dl, ExpVT);
2454 SDValue ClampN_Small = DAG.getNode(ISD::SMAX, dl, ExpVT, N, ClampMinVal);
2455 SDValue IncN1 =
2456 DAG.getNode(ISD::ADD, dl, ExpVT, ClampN_Small, Increment1, NSW);
2457
2458 const SDValue ScaleDownVal = DAG.getConstantFP(ScaleDownK, dl, VT);
2459 SDValue ScaleDown0 = DAG.getNode(ISD::FMUL, dl, VT, X, ScaleDownVal);
2460 SDValue ScaleDown1 = DAG.getNode(ISD::FMUL, dl, VT, ScaleDown0, ScaleDownVal);
2461
2462 SDValue ScaleDownTwice = DAG.getSetCC(
2463 dl, SetCCVT, N, DAG.getConstant(2 * MinExpVal + Precision, dl, ExpVT),
2464 ISD::SETULT);
2465
2466 SDValue SelectN_Small =
2467 DAG.getNode(ISD::SELECT, dl, ExpVT, ScaleDownTwice, IncN1, IncN0);
2468 SDValue SelectX_Small =
2469 DAG.getNode(ISD::SELECT, dl, VT, ScaleDownTwice, ScaleDown1, ScaleDown0);
2470
2471 // Now combine the two out of range exponent handling cases with the base
2472 // case.
2473 SDValue NewX = DAG.getNode(
2474 ISD::SELECT, dl, VT, NGtMaxExp, SelectX_Big,
2475 DAG.getNode(ISD::SELECT, dl, VT, NLtMinExp, SelectX_Small, X));
2476
2477 SDValue NewN = DAG.getNode(
2478 ISD::SELECT, dl, ExpVT, NGtMaxExp, SelectN_Big,
2479 DAG.getNode(ISD::SELECT, dl, ExpVT, NLtMinExp, SelectN_Small, N));
2480
2481 SDValue BiasedN = DAG.getNode(ISD::ADD, dl, ExpVT, NewN, MaxExp, NSW);
2482
2483 SDValue ExponentShiftAmt =
2484 DAG.getShiftAmountConstant(Precision - 1, ExpVT, dl);
2485 SDValue CastExpToValTy = DAG.getZExtOrTrunc(BiasedN, dl, AsIntVT);
2486
2487 SDValue AsInt = DAG.getNode(ISD::SHL, dl, AsIntVT, CastExpToValTy,
2488 ExponentShiftAmt, NUW_NSW);
2489 SDValue AsFP = DAG.getNode(ISD::BITCAST, dl, VT, AsInt);
2490 return DAG.getNode(ISD::FMUL, dl, VT, NewX, AsFP);
2491}
2492
2493SDValue SelectionDAGLegalize::expandFrexp(SDNode *Node) const {
2494 SDLoc dl(Node);
2495 SDValue Val = Node->getOperand(0);
2496 EVT VT = Val.getValueType();
2497 EVT ExpVT = Node->getValueType(1);
2498 EVT AsIntVT = VT.changeTypeToInteger();
2499 if (AsIntVT == EVT()) // TODO: How to handle f80?
2500 return SDValue();
2501
2503 const APFloat::ExponentType MinExpVal = APFloat::semanticsMinExponent(FltSem);
2504 const unsigned Precision = APFloat::semanticsPrecision(FltSem);
2505 const unsigned BitSize = VT.getScalarSizeInBits();
2506
2507 // TODO: Could introduce control flow and skip over the denormal handling.
2508
2509 // scale_up = fmul value, scalbn(1.0, precision + 1)
2510 // extracted_exp = (bitcast value to uint) >> precision - 1
2511 // biased_exp = extracted_exp + min_exp
2512 // extracted_fract = (bitcast value to uint) & (fract_mask | sign_mask)
2513 //
2514 // is_denormal = val < smallest_normalized
2515 // computed_fract = is_denormal ? scale_up : extracted_fract
2516 // computed_exp = is_denormal ? biased_exp + (-precision - 1) : biased_exp
2517 //
2518 // result_0 = (!isfinite(val) || iszero(val)) ? val : computed_fract
2519 // result_1 = (!isfinite(val) || iszero(val)) ? 0 : computed_exp
2520
2521 SDValue NegSmallestNormalizedInt = DAG.getConstant(
2522 APFloat::getSmallestNormalized(FltSem, true).bitcastToAPInt(), dl,
2523 AsIntVT);
2524
2525 SDValue SmallestNormalizedInt = DAG.getConstant(
2526 APFloat::getSmallestNormalized(FltSem, false).bitcastToAPInt(), dl,
2527 AsIntVT);
2528
2529 // Masks out the exponent bits.
2530 SDValue ExpMask =
2531 DAG.getConstant(APFloat::getInf(FltSem).bitcastToAPInt(), dl, AsIntVT);
2532
2533 // Mask out the exponent part of the value.
2534 //
2535 // e.g, for f32 FractSignMaskVal = 0x807fffff
2536 APInt FractSignMaskVal = APInt::getBitsSet(BitSize, 0, Precision - 1);
2537 FractSignMaskVal.setBit(BitSize - 1); // Set the sign bit
2538
2539 APInt SignMaskVal = APInt::getSignedMaxValue(BitSize);
2540 SDValue SignMask = DAG.getConstant(SignMaskVal, dl, AsIntVT);
2541
2542 SDValue FractSignMask = DAG.getConstant(FractSignMaskVal, dl, AsIntVT);
2543
2544 const APFloat One(FltSem, "1.0");
2545 // Scale a possible denormal input.
2546 // e.g., for f64, 0x1p+54
2547 APFloat ScaleUpKVal =
2548 scalbn(One, Precision + 1, APFloat::rmNearestTiesToEven);
2549
2550 SDValue ScaleUpK = DAG.getConstantFP(ScaleUpKVal, dl, VT);
2551 SDValue ScaleUp = DAG.getNode(ISD::FMUL, dl, VT, Val, ScaleUpK);
2552
2553 EVT SetCCVT =
2554 TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT);
2555
2556 SDValue AsInt = DAG.getNode(ISD::BITCAST, dl, AsIntVT, Val);
2557
2558 SDValue Abs = DAG.getNode(ISD::AND, dl, AsIntVT, AsInt, SignMask);
2559
2560 SDValue AddNegSmallestNormal =
2561 DAG.getNode(ISD::ADD, dl, AsIntVT, Abs, NegSmallestNormalizedInt);
2562 SDValue DenormOrZero = DAG.getSetCC(dl, SetCCVT, AddNegSmallestNormal,
2563 NegSmallestNormalizedInt, ISD::SETULE);
2564
2565 SDValue IsDenormal =
2566 DAG.getSetCC(dl, SetCCVT, Abs, SmallestNormalizedInt, ISD::SETULT);
2567
2568 SDValue MinExp = DAG.getConstant(MinExpVal, dl, ExpVT);
2569 SDValue Zero = DAG.getConstant(0, dl, ExpVT);
2570
2571 SDValue ScaledAsInt = DAG.getNode(ISD::BITCAST, dl, AsIntVT, ScaleUp);
2572 SDValue ScaledSelect =
2573 DAG.getNode(ISD::SELECT, dl, AsIntVT, IsDenormal, ScaledAsInt, AsInt);
2574
2575 SDValue ExpMaskScaled =
2576 DAG.getNode(ISD::AND, dl, AsIntVT, ScaledAsInt, ExpMask);
2577
2578 SDValue ScaledValue =
2579 DAG.getNode(ISD::SELECT, dl, AsIntVT, IsDenormal, ExpMaskScaled, Abs);
2580
2581 // Extract the exponent bits.
2582 SDValue ExponentShiftAmt =
2583 DAG.getShiftAmountConstant(Precision - 1, AsIntVT, dl);
2584 SDValue ShiftedExp =
2585 DAG.getNode(ISD::SRL, dl, AsIntVT, ScaledValue, ExponentShiftAmt);
2586 SDValue Exp = DAG.getSExtOrTrunc(ShiftedExp, dl, ExpVT);
2587
2588 SDValue NormalBiasedExp = DAG.getNode(ISD::ADD, dl, ExpVT, Exp, MinExp);
2589 SDValue DenormalOffset = DAG.getConstant(-Precision - 1, dl, ExpVT);
2590 SDValue DenormalExpBias =
2591 DAG.getNode(ISD::SELECT, dl, ExpVT, IsDenormal, DenormalOffset, Zero);
2592
2593 SDValue MaskedFractAsInt =
2594 DAG.getNode(ISD::AND, dl, AsIntVT, ScaledSelect, FractSignMask);
2595 const APFloat Half(FltSem, "0.5");
2596 SDValue FPHalf = DAG.getConstant(Half.bitcastToAPInt(), dl, AsIntVT);
2597 SDValue Or = DAG.getNode(ISD::OR, dl, AsIntVT, MaskedFractAsInt, FPHalf);
2598 SDValue MaskedFract = DAG.getNode(ISD::BITCAST, dl, VT, Or);
2599
2600 SDValue ComputedExp =
2601 DAG.getNode(ISD::ADD, dl, ExpVT, NormalBiasedExp, DenormalExpBias);
2602
2603 SDValue Result0 =
2604 DAG.getNode(ISD::SELECT, dl, VT, DenormOrZero, Val, MaskedFract);
2605
2606 SDValue Result1 =
2607 DAG.getNode(ISD::SELECT, dl, ExpVT, DenormOrZero, Zero, ComputedExp);
2608
2609 return DAG.getMergeValues({Result0, Result1}, dl);
2610}
2611
2612/// This function is responsible for legalizing a
2613/// INT_TO_FP operation of the specified operand when the target requests that
2614/// we expand it. At this point, we know that the result and operand types are
2615/// legal for the target.
2616SDValue SelectionDAGLegalize::ExpandLegalINT_TO_FP(SDNode *Node,
2617 SDValue &Chain) {
2618 bool isSigned = (Node->getOpcode() == ISD::STRICT_SINT_TO_FP ||
2619 Node->getOpcode() == ISD::SINT_TO_FP);
2620 EVT DestVT = Node->getValueType(0);
2621 SDLoc dl(Node);
2622 unsigned OpNo = Node->isStrictFPOpcode() ? 1 : 0;
2623 SDValue Op0 = Node->getOperand(OpNo);
2624 EVT SrcVT = Op0.getValueType();
2625
2626 // TODO: Should any fast-math-flags be set for the created nodes?
2627 LLVM_DEBUG(dbgs() << "Legalizing INT_TO_FP\n");
2628 if (SrcVT == MVT::i32 && TLI.isTypeLegal(MVT::f64) &&
2629 (DestVT.bitsLE(MVT::f64) ||
2630 TLI.isOperationLegal(Node->isStrictFPOpcode() ? ISD::STRICT_FP_EXTEND
2632 DestVT))) {
2633 LLVM_DEBUG(dbgs() << "32-bit [signed|unsigned] integer to float/double "
2634 "expansion\n");
2635
2636 // Get the stack frame index of a 8 byte buffer.
2637 SDValue StackSlot = DAG.CreateStackTemporary(MVT::f64);
2638
2639 SDValue Lo = Op0;
2640 // if signed map to unsigned space
2641 if (isSigned) {
2642 // Invert sign bit (signed to unsigned mapping).
2643 Lo = DAG.getNode(ISD::XOR, dl, MVT::i32, Lo,
2644 DAG.getConstant(0x80000000u, dl, MVT::i32));
2645 }
2646 // Initial hi portion of constructed double.
2647 SDValue Hi = DAG.getConstant(0x43300000u, dl, MVT::i32);
2648
2649 // If this a big endian target, swap the lo and high data.
2650 if (DAG.getDataLayout().isBigEndian())
2651 std::swap(Lo, Hi);
2652
2653 SDValue MemChain = DAG.getEntryNode();
2654
2655 // Store the lo of the constructed double.
2656 SDValue Store1 = DAG.getStore(MemChain, dl, Lo, StackSlot,
2658 // Store the hi of the constructed double.
2659 SDValue HiPtr =
2660 DAG.getMemBasePlusOffset(StackSlot, TypeSize::getFixed(4), dl);
2661 SDValue Store2 =
2662 DAG.getStore(MemChain, dl, Hi, HiPtr, MachinePointerInfo());
2663 MemChain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Store1, Store2);
2664
2665 // load the constructed double
2666 SDValue Load =
2667 DAG.getLoad(MVT::f64, dl, MemChain, StackSlot, MachinePointerInfo());
2668 // FP constant to bias correct the final result
2669 SDValue Bias = DAG.getConstantFP(
2670 isSigned ? llvm::bit_cast<double>(0x4330000080000000ULL)
2671 : llvm::bit_cast<double>(0x4330000000000000ULL),
2672 dl, MVT::f64);
2673 // Subtract the bias and get the final result.
2674 SDValue Sub;
2676 if (Node->isStrictFPOpcode()) {
2677 Sub = DAG.getNode(ISD::STRICT_FSUB, dl, {MVT::f64, MVT::Other},
2678 {Node->getOperand(0), Load, Bias});
2679 Chain = Sub.getValue(1);
2680 if (DestVT != Sub.getValueType()) {
2681 std::pair<SDValue, SDValue> ResultPair;
2682 ResultPair =
2683 DAG.getStrictFPExtendOrRound(Sub, Chain, dl, DestVT);
2684 Result = ResultPair.first;
2685 Chain = ResultPair.second;
2686 }
2687 else
2688 Result = Sub;
2689 } else {
2690 Sub = DAG.getNode(ISD::FSUB, dl, MVT::f64, Load, Bias);
2691 Result = DAG.getFPExtendOrRound(Sub, dl, DestVT);
2692 }
2693 return Result;
2694 }
2695
2696 if (isSigned)
2697 return SDValue();
2698
2699 // TODO: Generalize this for use with other types.
2700 if (((SrcVT == MVT::i32 || SrcVT == MVT::i64) && DestVT == MVT::f32) ||
2701 (SrcVT == MVT::i64 && DestVT == MVT::f64)) {
2702 LLVM_DEBUG(dbgs() << "Converting unsigned i32/i64 to f32/f64\n");
2703 // For unsigned conversions, convert them to signed conversions using the
2704 // algorithm from the x86_64 __floatundisf in compiler_rt. That method
2705 // should be valid for i32->f32 as well.
2706
2707 // More generally this transform should be valid if there are 3 more bits
2708 // in the integer type than the significand. Rounding uses the first bit
2709 // after the width of the significand and the OR of all bits after that. So
2710 // we need to be able to OR the shifted out bit into one of the bits that
2711 // participate in the OR.
2712
2713 // TODO: This really should be implemented using a branch rather than a
2714 // select. We happen to get lucky and machinesink does the right
2715 // thing most of the time. This would be a good candidate for a
2716 // pseudo-op, or, even better, for whole-function isel.
2717 EVT SetCCVT = getSetCCResultType(SrcVT);
2718
2719 SDValue SignBitTest = DAG.getSetCC(
2720 dl, SetCCVT, Op0, DAG.getConstant(0, dl, SrcVT), ISD::SETLT);
2721
2722 EVT ShiftVT = TLI.getShiftAmountTy(SrcVT, DAG.getDataLayout());
2723 SDValue ShiftConst = DAG.getConstant(1, dl, ShiftVT);
2724 SDValue Shr = DAG.getNode(ISD::SRL, dl, SrcVT, Op0, ShiftConst);
2725 SDValue AndConst = DAG.getConstant(1, dl, SrcVT);
2726 SDValue And = DAG.getNode(ISD::AND, dl, SrcVT, Op0, AndConst);
2727 SDValue Or = DAG.getNode(ISD::OR, dl, SrcVT, And, Shr);
2728
2729 SDValue Slow, Fast;
2730 if (Node->isStrictFPOpcode()) {
2731 // In strict mode, we must avoid spurious exceptions, and therefore
2732 // must make sure to only emit a single STRICT_SINT_TO_FP.
2733 SDValue InCvt = DAG.getSelect(dl, SrcVT, SignBitTest, Or, Op0);
2734 Fast = DAG.getNode(ISD::STRICT_SINT_TO_FP, dl, { DestVT, MVT::Other },
2735 { Node->getOperand(0), InCvt });
2736 Slow = DAG.getNode(ISD::STRICT_FADD, dl, { DestVT, MVT::Other },
2737 { Fast.getValue(1), Fast, Fast });
2738 Chain = Slow.getValue(1);
2739 // The STRICT_SINT_TO_FP inherits the exception mode from the
2740 // incoming STRICT_UINT_TO_FP node; the STRICT_FADD node can
2741 // never raise any exception.
2743 Flags.setNoFPExcept(Node->getFlags().hasNoFPExcept());
2744 Fast->setFlags(Flags);
2745 Flags.setNoFPExcept(true);
2746 Slow->setFlags(Flags);
2747 } else {
2748 SDValue SignCvt = DAG.getNode(ISD::SINT_TO_FP, dl, DestVT, Or);
2749 Slow = DAG.getNode(ISD::FADD, dl, DestVT, SignCvt, SignCvt);
2750 Fast = DAG.getNode(ISD::SINT_TO_FP, dl, DestVT, Op0);
2751 }
2752
2753 return DAG.getSelect(dl, DestVT, SignBitTest, Slow, Fast);
2754 }
2755
2756 // Don't expand it if there isn't cheap fadd.
2757 if (!TLI.isOperationLegalOrCustom(
2758 Node->isStrictFPOpcode() ? ISD::STRICT_FADD : ISD::FADD, DestVT))
2759 return SDValue();
2760
2761 // The following optimization is valid only if every value in SrcVT (when
2762 // treated as signed) is representable in DestVT. Check that the mantissa
2763 // size of DestVT is >= than the number of bits in SrcVT -1.
2764 assert(APFloat::semanticsPrecision(DAG.EVTToAPFloatSemantics(DestVT)) >=
2765 SrcVT.getSizeInBits() - 1 &&
2766 "Cannot perform lossless SINT_TO_FP!");
2767
2768 SDValue Tmp1;
2769 if (Node->isStrictFPOpcode()) {
2770 Tmp1 = DAG.getNode(ISD::STRICT_SINT_TO_FP, dl, { DestVT, MVT::Other },
2771 { Node->getOperand(0), Op0 });
2772 } else
2773 Tmp1 = DAG.getNode(ISD::SINT_TO_FP, dl, DestVT, Op0);
2774
2775 SDValue SignSet = DAG.getSetCC(dl, getSetCCResultType(SrcVT), Op0,
2776 DAG.getConstant(0, dl, SrcVT), ISD::SETLT);
2777 SDValue Zero = DAG.getIntPtrConstant(0, dl),
2778 Four = DAG.getIntPtrConstant(4, dl);
2779 SDValue CstOffset = DAG.getSelect(dl, Zero.getValueType(),
2780 SignSet, Four, Zero);
2781
2782 // If the sign bit of the integer is set, the large number will be treated
2783 // as a negative number. To counteract this, the dynamic code adds an
2784 // offset depending on the data type.
2785 uint64_t FF;
2786 switch (SrcVT.getSimpleVT().SimpleTy) {
2787 default:
2788 return SDValue();
2789 case MVT::i8 : FF = 0x43800000ULL; break; // 2^8 (as a float)
2790 case MVT::i16: FF = 0x47800000ULL; break; // 2^16 (as a float)
2791 case MVT::i32: FF = 0x4F800000ULL; break; // 2^32 (as a float)
2792 case MVT::i64: FF = 0x5F800000ULL; break; // 2^64 (as a float)
2793 }
2794 if (DAG.getDataLayout().isLittleEndian())
2795 FF <<= 32;
2796 Constant *FudgeFactor = ConstantInt::get(
2797 Type::getInt64Ty(*DAG.getContext()), FF);
2798
2799 SDValue CPIdx =
2800 DAG.getConstantPool(FudgeFactor, TLI.getPointerTy(DAG.getDataLayout()));
2801 Align Alignment = cast<ConstantPoolSDNode>(CPIdx)->getAlign();
2802 CPIdx = DAG.getNode(ISD::ADD, dl, CPIdx.getValueType(), CPIdx, CstOffset);
2803 Alignment = commonAlignment(Alignment, 4);
2804 SDValue FudgeInReg;
2805 if (DestVT == MVT::f32)
2806 FudgeInReg = DAG.getLoad(
2807 MVT::f32, dl, DAG.getEntryNode(), CPIdx,
2808 MachinePointerInfo::getConstantPool(DAG.getMachineFunction()),
2809 Alignment);
2810 else {
2811 SDValue Load = DAG.getExtLoad(
2812 ISD::EXTLOAD, dl, DestVT, DAG.getEntryNode(), CPIdx,
2813 MachinePointerInfo::getConstantPool(DAG.getMachineFunction()), MVT::f32,
2814 Alignment);
2815 HandleSDNode Handle(Load);
2816 LegalizeOp(Load.getNode());
2817 FudgeInReg = Handle.getValue();
2818 }
2819
2820 if (Node->isStrictFPOpcode()) {
2821 SDValue Result = DAG.getNode(ISD::STRICT_FADD, dl, { DestVT, MVT::Other },
2822 { Tmp1.getValue(1), Tmp1, FudgeInReg });
2823 Chain = Result.getValue(1);
2824 return Result;
2825 }
2826
2827 return DAG.getNode(ISD::FADD, dl, DestVT, Tmp1, FudgeInReg);
2828}
2829
2830/// This function is responsible for legalizing a
2831/// *INT_TO_FP operation of the specified operand when the target requests that
2832/// we promote it. At this point, we know that the result and operand types are
2833/// legal for the target, and that there is a legal UINT_TO_FP or SINT_TO_FP
2834/// operation that takes a larger input.
2835void SelectionDAGLegalize::PromoteLegalINT_TO_FP(
2837 bool IsStrict = N->isStrictFPOpcode();
2838 bool IsSigned = N->getOpcode() == ISD::SINT_TO_FP ||
2839 N->getOpcode() == ISD::STRICT_SINT_TO_FP;
2840 EVT DestVT = N->getValueType(0);
2841 SDValue LegalOp = N->getOperand(IsStrict ? 1 : 0);
2842 unsigned UIntOp = IsStrict ? ISD::STRICT_UINT_TO_FP : ISD::UINT_TO_FP;
2843 unsigned SIntOp = IsStrict ? ISD::STRICT_SINT_TO_FP : ISD::SINT_TO_FP;
2844
2845 // First step, figure out the appropriate *INT_TO_FP operation to use.
2846 EVT NewInTy = LegalOp.getValueType();
2847
2848 unsigned OpToUse = 0;
2849
2850 // Scan for the appropriate larger type to use.
2851 while (true) {
2852 NewInTy = (MVT::SimpleValueType)(NewInTy.getSimpleVT().SimpleTy+1);
2853 assert(NewInTy.isInteger() && "Ran out of possibilities!");
2854
2855 // If the target supports SINT_TO_FP of this type, use it.
2856 if (TLI.isOperationLegalOrCustom(SIntOp, NewInTy)) {
2857 OpToUse = SIntOp;
2858 break;
2859 }
2860 if (IsSigned)
2861 continue;
2862
2863 // If the target supports UINT_TO_FP of this type, use it.
2864 if (TLI.isOperationLegalOrCustom(UIntOp, NewInTy)) {
2865 OpToUse = UIntOp;
2866 break;
2867 }
2868
2869 // Otherwise, try a larger type.
2870 }
2871
2872 // Okay, we found the operation and type to use. Zero extend our input to the
2873 // desired type then run the operation on it.
2874 if (IsStrict) {
2875 SDValue Res =
2876 DAG.getNode(OpToUse, dl, {DestVT, MVT::Other},
2877 {N->getOperand(0),
2878 DAG.getNode(IsSigned ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND,
2879 dl, NewInTy, LegalOp)});
2880 Results.push_back(Res);
2881 Results.push_back(Res.getValue(1));
2882 return;
2883 }
2884
2885 Results.push_back(
2886 DAG.getNode(OpToUse, dl, DestVT,
2887 DAG.getNode(IsSigned ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND,
2888 dl, NewInTy, LegalOp)));
2889}
2890
2891/// This function is responsible for legalizing a
2892/// FP_TO_*INT operation of the specified operand when the target requests that
2893/// we promote it. At this point, we know that the result and operand types are
2894/// legal for the target, and that there is a legal FP_TO_UINT or FP_TO_SINT
2895/// operation that returns a larger result.
2896void SelectionDAGLegalize::PromoteLegalFP_TO_INT(SDNode *N, const SDLoc &dl,
2898 bool IsStrict = N->isStrictFPOpcode();
2899 bool IsSigned = N->getOpcode() == ISD::FP_TO_SINT ||
2900 N->getOpcode() == ISD::STRICT_FP_TO_SINT;
2901 EVT DestVT = N->getValueType(0);
2902 SDValue LegalOp = N->getOperand(IsStrict ? 1 : 0);
2903 // First step, figure out the appropriate FP_TO*INT operation to use.
2904 EVT NewOutTy = DestVT;
2905
2906 unsigned OpToUse = 0;
2907
2908 // Scan for the appropriate larger type to use.
2909 while (true) {
2910 NewOutTy = (MVT::SimpleValueType)(NewOutTy.getSimpleVT().SimpleTy+1);
2911 assert(NewOutTy.isInteger() && "Ran out of possibilities!");
2912
2913 // A larger signed type can hold all unsigned values of the requested type,
2914 // so using FP_TO_SINT is valid
2915 OpToUse = IsStrict ? ISD::STRICT_FP_TO_SINT : ISD::FP_TO_SINT;
2916 if (TLI.isOperationLegalOrCustom(OpToUse, NewOutTy))
2917 break;
2918
2919 // However, if the value may be < 0.0, we *must* use some FP_TO_SINT.
2920 OpToUse = IsStrict ? ISD::STRICT_FP_TO_UINT : ISD::FP_TO_UINT;
2921 if (!IsSigned && TLI.isOperationLegalOrCustom(OpToUse, NewOutTy))
2922 break;
2923
2924 // Otherwise, try a larger type.
2925 }
2926
2927 // Okay, we found the operation and type to use.
2929 if (IsStrict) {
2930 SDVTList VTs = DAG.getVTList(NewOutTy, MVT::Other);
2931 Operation = DAG.getNode(OpToUse, dl, VTs, N->getOperand(0), LegalOp);
2932 } else
2933 Operation = DAG.getNode(OpToUse, dl, NewOutTy, LegalOp);
2934
2935 // Truncate the result of the extended FP_TO_*INT operation to the desired
2936 // size.
2937 SDValue Trunc = DAG.getNode(ISD::TRUNCATE, dl, DestVT, Operation);
2938 Results.push_back(Trunc);
2939 if (IsStrict)
2940 Results.push_back(Operation.getValue(1));
2941}
2942
2943/// Promote FP_TO_*INT_SAT operation to a larger result type. At this point
2944/// the result and operand types are legal and there must be a legal
2945/// FP_TO_*INT_SAT operation for a larger result type.
2946SDValue SelectionDAGLegalize::PromoteLegalFP_TO_INT_SAT(SDNode *Node,
2947 const SDLoc &dl) {
2948 unsigned Opcode = Node->getOpcode();
2949
2950 // Scan for the appropriate larger type to use.
2951 EVT NewOutTy = Node->getValueType(0);
2952 while (true) {
2953 NewOutTy = (MVT::SimpleValueType)(NewOutTy.getSimpleVT().SimpleTy + 1);
2954 assert(NewOutTy.isInteger() && "Ran out of possibilities!");
2955
2956 if (TLI.isOperationLegalOrCustom(Opcode, NewOutTy))
2957 break;
2958 }
2959
2960 // Saturation width is determined by second operand, so we don't have to
2961 // perform any fixup and can directly truncate the result.
2962 SDValue Result = DAG.getNode(Opcode, dl, NewOutTy, Node->getOperand(0),
2963 Node->getOperand(1));
2964 return DAG.getNode(ISD::TRUNCATE, dl, Node->getValueType(0), Result);
2965}
2966
2967/// Open code the operations for PARITY of the specified operation.
2968SDValue SelectionDAGLegalize::ExpandPARITY(SDValue Op, const SDLoc &dl) {
2969 EVT VT = Op.getValueType();
2970 EVT ShVT = TLI.getShiftAmountTy(VT, DAG.getDataLayout());
2971 unsigned Sz = VT.getScalarSizeInBits();
2972
2973 // If CTPOP is legal, use it. Otherwise use shifts and xor.
2975 if (TLI.isOperationLegalOrPromote(ISD::CTPOP, VT)) {
2976 Result = DAG.getNode(ISD::CTPOP, dl, VT, Op);
2977 } else {
2978 Result = Op;
2979 for (unsigned i = Log2_32_Ceil(Sz); i != 0;) {
2980 SDValue Shift = DAG.getNode(ISD::SRL, dl, VT, Result,
2981 DAG.getConstant(1ULL << (--i), dl, ShVT));
2982 Result = DAG.getNode(ISD::XOR, dl, VT, Result, Shift);
2983 }
2984 }
2985
2986 return DAG.getNode(ISD::AND, dl, VT, Result, DAG.getConstant(1, dl, VT));
2987}
2988
2989bool SelectionDAGLegalize::ExpandNode(SDNode *Node) {
2990 LLVM_DEBUG(dbgs() << "Trying to expand node\n");
2992 SDLoc dl(Node);
2993 SDValue Tmp1, Tmp2, Tmp3, Tmp4;
2994 bool NeedInvert;
2995 switch (Node->getOpcode()) {
2996 case ISD::ABS:
2997 if ((Tmp1 = TLI.expandABS(Node, DAG)))
2998 Results.push_back(Tmp1);
2999 break;
3000 case ISD::ABDS:
3001 case ISD::ABDU:
3002 if ((Tmp1 = TLI.expandABD(Node, DAG)))
3003 Results.push_back(Tmp1);
3004 break;
3005 case ISD::CTPOP:
3006 if ((Tmp1 = TLI.expandCTPOP(Node, DAG)))
3007 Results.push_back(Tmp1);
3008 break;
3009 case ISD::CTLZ:
3011 if ((Tmp1 = TLI.expandCTLZ(Node, DAG)))
3012 Results.push_back(Tmp1);
3013 break;
3014 case ISD::CTTZ:
3016 if ((Tmp1 = TLI.expandCTTZ(Node, DAG)))
3017 Results.push_back(Tmp1);
3018 break;
3019 case ISD::BITREVERSE:
3020 if ((Tmp1 = TLI.expandBITREVERSE(Node, DAG)))
3021 Results.push_back(Tmp1);
3022 break;
3023 case ISD::BSWAP:
3024 if ((Tmp1 = TLI.expandBSWAP(Node, DAG)))
3025 Results.push_back(Tmp1);
3026 break;
3027 case ISD::PARITY:
3028 Results.push_back(ExpandPARITY(Node->getOperand(0), dl));
3029 break;
3030 case ISD::FRAMEADDR:
3031 case ISD::RETURNADDR:
3033 Results.push_back(DAG.getConstant(0, dl, Node->getValueType(0)));
3034 break;
3035 case ISD::EH_DWARF_CFA: {
3036 SDValue CfaArg = DAG.getSExtOrTrunc(Node->getOperand(0), dl,
3037 TLI.getPointerTy(DAG.getDataLayout()));
3038 SDValue Offset = DAG.getNode(ISD::ADD, dl,
3039 CfaArg.getValueType(),
3040 DAG.getNode(ISD::FRAME_TO_ARGS_OFFSET, dl,
3041 CfaArg.getValueType()),
3042 CfaArg);
3043 SDValue FA = DAG.getNode(
3044 ISD::FRAMEADDR, dl, TLI.getPointerTy(DAG.getDataLayout()),
3045 DAG.getConstant(0, dl, TLI.getPointerTy(DAG.getDataLayout())));
3046 Results.push_back(DAG.getNode(ISD::ADD, dl, FA.getValueType(),
3047 FA, Offset));
3048 break;
3049 }
3050 case ISD::GET_ROUNDING:
3051 Results.push_back(DAG.getConstant(1, dl, Node->getValueType(0)));
3052 Results.push_back(Node->getOperand(0));
3053 break;
3054 case ISD::EH_RETURN:
3055 case ISD::EH_LABEL:
3056 case ISD::PREFETCH:
3057 case ISD::VAEND:
3059 // If the target didn't expand these, there's nothing to do, so just
3060 // preserve the chain and be done.
3061 Results.push_back(Node->getOperand(0));
3062 break;
3064 // If the target didn't expand this, just return 'zero' and preserve the
3065 // chain.
3066 Results.append(Node->getNumValues() - 1,
3067 DAG.getConstant(0, dl, Node->getValueType(0)));
3068 Results.push_back(Node->getOperand(0));
3069 break;
3071 // If the target didn't expand this, just return 'zero' and preserve the
3072 // chain.
3073 Results.push_back(DAG.getConstant(0, dl, MVT::i32));
3074 Results.push_back(Node->getOperand(0));
3075 break;
3076 case ISD::ATOMIC_LOAD: {
3077 // There is no libcall for atomic load; fake it with ATOMIC_CMP_SWAP.
3078 SDValue Zero = DAG.getConstant(0, dl, Node->getValueType(0));
3079 SDVTList VTs = DAG.getVTList(Node->getValueType(0), MVT::Other);
3080 SDValue Swap = DAG.getAtomicCmpSwap(
3081 ISD::ATOMIC_CMP_SWAP, dl, cast<AtomicSDNode>(Node)->getMemoryVT(), VTs,
3082 Node->getOperand(0), Node->getOperand(1), Zero, Zero,
3083 cast<AtomicSDNode>(Node)->getMemOperand());
3084 Results.push_back(Swap.getValue(0));
3085 Results.push_back(Swap.getValue(1));
3086 break;
3087 }
3088 case ISD::ATOMIC_STORE: {
3089 // There is no libcall for atomic store; fake it with ATOMIC_SWAP.
3090 SDValue Swap = DAG.getAtomic(
3091 ISD::ATOMIC_SWAP, dl, cast<AtomicSDNode>(Node)->getMemoryVT(),
3092 Node->getOperand(0), Node->getOperand(2), Node->getOperand(1),
3093 cast<AtomicSDNode>(Node)->getMemOperand());
3094 Results.push_back(Swap.getValue(1));
3095 break;
3096 }
3098 // Expanding an ATOMIC_CMP_SWAP_WITH_SUCCESS produces an ATOMIC_CMP_SWAP and
3099 // splits out the success value as a comparison. Expanding the resulting
3100 // ATOMIC_CMP_SWAP will produce a libcall.
3101 SDVTList VTs = DAG.getVTList(Node->getValueType(0), MVT::Other);
3102 SDValue Res = DAG.getAtomicCmpSwap(
3103 ISD::ATOMIC_CMP_SWAP, dl, cast<AtomicSDNode>(Node)->getMemoryVT(), VTs,
3104 Node->getOperand(0), Node->getOperand(1), Node->getOperand(2),
3105 Node->getOperand(3), cast<MemSDNode>(Node)->getMemOperand());
3106
3107 SDValue ExtRes = Res;
3108 SDValue LHS = Res;
3109 SDValue RHS = Node->getOperand(1);
3110
3111 EVT AtomicType = cast<AtomicSDNode>(Node)->getMemoryVT();
3112 EVT OuterType = Node->getValueType(0);
3113 switch (TLI.getExtendForAtomicOps()) {
3114 case ISD::SIGN_EXTEND:
3115 LHS = DAG.getNode(ISD::AssertSext, dl, OuterType, Res,
3116 DAG.getValueType(AtomicType));
3117 RHS = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, OuterType,
3118 Node->getOperand(2), DAG.getValueType(AtomicType));
3119 ExtRes = LHS;
3120 break;
3121 case ISD::ZERO_EXTEND:
3122 LHS = DAG.getNode(ISD::AssertZext, dl, OuterType, Res,
3123 DAG.getValueType(AtomicType));
3124 RHS = DAG.getZeroExtendInReg(Node->getOperand(2), dl, AtomicType);
3125 ExtRes = LHS;
3126 break;
3127 case ISD::ANY_EXTEND:
3128 LHS = DAG.getZeroExtendInReg(Res, dl, AtomicType);
3129 RHS = DAG.getZeroExtendInReg(Node->getOperand(2), dl, AtomicType);
3130 break;
3131 default:
3132 llvm_unreachable("Invalid atomic op extension");
3133 }
3134
3136 DAG.getSetCC(dl, Node->getValueType(1), LHS, RHS, ISD::SETEQ);
3137
3138 Results.push_back(ExtRes.getValue(0));
3139 Results.push_back(Success);
3140 Results.push_back(Res.getValue(1));
3141 break;
3142 }
3143 case ISD::ATOMIC_LOAD_SUB: {
3144 SDLoc DL(Node);
3145 EVT VT = Node->getValueType(0);
3146 SDValue RHS = Node->getOperand(2);
3147 AtomicSDNode *AN = cast<AtomicSDNode>(Node);
3148 if (RHS->getOpcode() == ISD::SIGN_EXTEND_INREG &&
3149 cast<VTSDNode>(RHS->getOperand(1))->getVT() == AN->getMemoryVT())
3150 RHS = RHS->getOperand(0);
3151 SDValue NewRHS =
3152 DAG.getNode(ISD::SUB, DL, VT, DAG.getConstant(0, DL, VT), RHS);
3153 SDValue Res = DAG.getAtomic(ISD::ATOMIC_LOAD_ADD, DL, AN->getMemoryVT(),
3154 Node->getOperand(0), Node->getOperand(1),
3155 NewRHS, AN->getMemOperand());
3156 Results.push_back(Res);
3157 Results.push_back(Res.getValue(1));
3158 break;
3159 }
3161 ExpandDYNAMIC_STACKALLOC(Node, Results);
3162 break;
3163 case ISD::MERGE_VALUES:
3164 for (unsigned i = 0; i < Node->getNumValues(); i++)
3165 Results.push_back(Node->getOperand(i));
3166 break;
3167 case ISD::UNDEF: {
3168 EVT VT = Node->getValueType(0);
3169 if (VT.isInteger())
3170 Results.push_back(DAG.getConstant(0, dl, VT));
3171 else {
3172 assert(VT.isFloatingPoint() && "Unknown value type!");
3173 Results.push_back(DAG.getConstantFP(0, dl, VT));
3174 }
3175 break;
3176 }
3178 // When strict mode is enforced we can't do expansion because it
3179 // does not honor the "strict" properties. Only libcall is allowed.
3180 if (TLI.isStrictFPEnabled())
3181 break;
3182 // We might as well mutate to FP_ROUND when FP_ROUND operation is legal
3183 // since this operation is more efficient than stack operation.
3184 if (TLI.getStrictFPOperationAction(Node->getOpcode(),
3185 Node->getValueType(0))
3186 == TargetLowering::Legal)
3187 break;
3188 // We fall back to use stack operation when the FP_ROUND operation
3189 // isn't available.
3190 if ((Tmp1 = EmitStackConvert(Node->getOperand(1), Node->getValueType(0),
3191 Node->getValueType(0), dl,
3192 Node->getOperand(0)))) {
3193 ReplaceNode(Node, Tmp1.getNode());
3194 LLVM_DEBUG(dbgs() << "Successfully expanded STRICT_FP_ROUND node\n");
3195 return true;
3196 }
3197 break;
3198 case ISD::FP_ROUND:
3199 case ISD::BITCAST:
3200 if ((Tmp1 = EmitStackConvert(Node->getOperand(0), Node->getValueType(0),
3201 Node->getValueType(0), dl)))
3202 Results.push_back(Tmp1);
3203 break;
3205 // When strict mode is enforced we can't do expansion because it
3206 // does not honor the "strict" properties. Only libcall is allowed.
3207 if (TLI.isStrictFPEnabled())
3208 break;
3209 // We might as well mutate to FP_EXTEND when FP_EXTEND operation is legal
3210 // since this operation is more efficient than stack operation.
3211 if (TLI.getStrictFPOperationAction(Node->getOpcode(),
3212 Node->getValueType(0))
3213 == TargetLowering::Legal)
3214 break;
3215 // We fall back to use stack operation when the FP_EXTEND operation
3216 // isn't available.
3217 if ((Tmp1 = EmitStackConvert(
3218 Node->getOperand(1), Node->getOperand(1).getValueType(),
3219 Node->getValueType(0), dl, Node->getOperand(0)))) {
3220 ReplaceNode(Node, Tmp1.getNode());
3221 LLVM_DEBUG(dbgs() << "Successfully expanded STRICT_FP_EXTEND node\n");
3222 return true;
3223 }
3224 break;
3225 case ISD::FP_EXTEND:
3226 if ((Tmp1 = EmitStackConvert(Node->getOperand(0),
3227 Node->getOperand(0).getValueType(),
3228 Node->getValueType(0), dl)))
3229 Results.push_back(Tmp1);
3230 break;
3231 case ISD::BF16_TO_FP: {
3232 // Always expand bf16 to f32 casts, they lower to ext + shift.
3233 //
3234 // Note that the operand of this code can be bf16 or an integer type in case
3235 // bf16 is not supported on the target and was softened.
3236 SDValue Op = Node->getOperand(0);
3237 if (Op.getValueType() == MVT::bf16) {
3238 Op = DAG.getNode(ISD::ANY_EXTEND, dl, MVT::i32,
3239 DAG.getNode(ISD::BITCAST, dl, MVT::i16, Op));
3240 } else {
3241 Op = DAG.getAnyExtOrTrunc(Op, dl, MVT::i32);
3242 }
3243 Op = DAG.getNode(
3244 ISD::SHL, dl, MVT::i32, Op,
3245 DAG.getConstant(16, dl,
3246 TLI.getShiftAmountTy(MVT::i32, DAG.getDataLayout())));
3247 Op = DAG.getNode(ISD::BITCAST, dl, MVT::f32, Op);
3248 // Add fp_extend in case the output is bigger than f32.
3249 if (Node->getValueType(0) != MVT::f32)
3250 Op = DAG.getNode(ISD::FP_EXTEND, dl, Node->getValueType(0), Op);
3251 Results.push_back(Op);
3252 break;
3253 }
3254 case ISD::FP_TO_BF16: {
3255 SDValue Op = Node->getOperand(0);
3256 if (Op.getValueType() != MVT::f32)
3257 Op = DAG.getNode(ISD::FP_ROUND, dl, MVT::f32, Op,
3258 DAG.getIntPtrConstant(0, dl, /*isTarget=*/true));
3259 Op = DAG.getNode(
3260 ISD::SRL, dl, MVT::i32, DAG.getNode(ISD::BITCAST, dl, MVT::i32, Op),
3261 DAG.getConstant(16, dl,
3262 TLI.getShiftAmountTy(MVT::i32, DAG.getDataLayout())));
3263 // The result of this node can be bf16 or an integer type in case bf16 is
3264 // not supported on the target and was softened to i16 for storage.
3265 if (Node->getValueType(0) == MVT::bf16) {
3266 Op = DAG.getNode(ISD::BITCAST, dl, MVT::bf16,
3267 DAG.getNode(ISD::TRUNCATE, dl, MVT::i16, Op));
3268 } else {
3269 Op = DAG.getAnyExtOrTrunc(Op, dl, Node->getValueType(0));
3270 }
3271 Results.push_back(Op);
3272 break;
3273 }
3275 EVT ExtraVT = cast<VTSDNode>(Node->getOperand(1))->getVT();
3276 EVT VT = Node->getValueType(0);
3277
3278 // An in-register sign-extend of a boolean is a negation:
3279 // 'true' (1) sign-extended is -1.
3280 // 'false' (0) sign-extended is 0.
3281 // However, we must mask the high bits of the source operand because the
3282 // SIGN_EXTEND_INREG does not guarantee that the high bits are already zero.
3283
3284 // TODO: Do this for vectors too?
3285 if (ExtraVT.isScalarInteger() && ExtraVT.getSizeInBits() == 1) {
3286 SDValue One = DAG.getConstant(1, dl, VT);
3287 SDValue And = DAG.getNode(ISD::AND, dl, VT, Node->getOperand(0), One);
3288 SDValue Zero = DAG.getConstant(0, dl, VT);
3289 SDValue Neg = DAG.getNode(ISD::SUB, dl, VT, Zero, And);
3290 Results.push_back(Neg);
3291 break;
3292 }
3293
3294 // NOTE: we could fall back on load/store here too for targets without
3295 // SRA. However, it is doubtful that any exist.
3296 EVT ShiftAmountTy = TLI.getShiftAmountTy(VT, DAG.getDataLayout());
3297 unsigned BitsDiff = VT.getScalarSizeInBits() -
3298 ExtraVT.getScalarSizeInBits();
3299 SDValue ShiftCst = DAG.getConstant(BitsDiff, dl, ShiftAmountTy);
3300 Tmp1 = DAG.getNode(ISD::SHL, dl, Node->getValueType(0),
3301 Node->getOperand(0), ShiftCst);
3302 Tmp1 = DAG.getNode(ISD::SRA, dl, Node->getValueType(0), Tmp1, ShiftCst);
3303 Results.push_back(Tmp1);
3304 break;
3305 }
3306 case ISD::UINT_TO_FP:
3308 if (TLI.expandUINT_TO_FP(Node, Tmp1, Tmp2, DAG)) {
3309 Results.push_back(Tmp1);
3310 if (Node->isStrictFPOpcode())
3311 Results.push_back(Tmp2);
3312 break;
3313 }
3314 [[fallthrough]];
3315 case ISD::SINT_TO_FP:
3317 if ((Tmp1 = ExpandLegalINT_TO_FP(Node, Tmp2))) {
3318 Results.push_back(Tmp1);
3319 if (Node->isStrictFPOpcode())
3320 Results.push_back(Tmp2);
3321 }
3322 break;
3323 case ISD::FP_TO_SINT:
3324 if (TLI.expandFP_TO_SINT(Node, Tmp1, DAG))
3325 Results.push_back(Tmp1);
3326 break;
3328 if (TLI.expandFP_TO_SINT(Node, Tmp1, DAG)) {
3329 ReplaceNode(Node, Tmp1.getNode());
3330 LLVM_DEBUG(dbgs() << "Successfully expanded STRICT_FP_TO_SINT node\n");
3331 return true;
3332 }
3333 break;
3334 case ISD::FP_TO_UINT:
3335 if (TLI.expandFP_TO_UINT(Node, Tmp1, Tmp2, DAG))
3336 Results.push_back(Tmp1);
3337 break;
3339 if (TLI.expandFP_TO_UINT(Node, Tmp1, Tmp2, DAG)) {
3340 // Relink the chain.
3341 DAG.ReplaceAllUsesOfValueWith(SDValue(Node,1), Tmp2);
3342 // Replace the new UINT result.
3343 ReplaceNodeWithValue(SDValue(Node, 0), Tmp1);
3344 LLVM_DEBUG(dbgs() << "Successfully expanded STRICT_FP_TO_UINT node\n");
3345 return true;
3346 }
3347 break;
3350 Results.push_back(TLI.expandFP_TO_INT_SAT(Node, DAG));
3351 break;
3352 case ISD::VAARG:
3353 Results.push_back(DAG.expandVAArg(Node));
3354 Results.push_back(Results[0].getValue(1));
3355 break;
3356 case ISD::VACOPY:
3357 Results.push_back(DAG.expandVACopy(Node));
3358 break;
3360 if (Node->getOperand(0).getValueType().getVectorElementCount().isScalar())
3361 // This must be an access of the only element. Return it.
3362 Tmp1 = DAG.getNode(ISD::BITCAST, dl, Node->getValueType(0),
3363 Node->getOperand(0));
3364 else
3365 Tmp1 = ExpandExtractFromVectorThroughStack(SDValue(Node, 0));
3366 Results.push_back(Tmp1);
3367 break;
3369 Results.push_back(ExpandExtractFromVectorThroughStack(SDValue(Node, 0)));
3370 break;
3372 Results.push_back(ExpandInsertToVectorThroughStack(SDValue(Node, 0)));
3373 break;
3375 Results.push_back(ExpandVectorBuildThroughStack(Node));
3376 break;
3378 Results.push_back(ExpandSCALAR_TO_VECTOR(Node));
3379 break;
3381 Results.push_back(ExpandINSERT_VECTOR_ELT(Node->getOperand(0),
3382 Node->getOperand(1),
3383 Node->getOperand(2), dl));
3384 break;
3385 case ISD::VECTOR_SHUFFLE: {
3386 SmallVector<int, 32> NewMask;
3387 ArrayRef<int> Mask = cast<ShuffleVectorSDNode>(Node)->getMask();
3388
3389 EVT VT = Node->getValueType(0);
3390 EVT EltVT = VT.getVectorElementType();
3391 SDValue Op0 = Node->getOperand(0);
3392 SDValue Op1 = Node->getOperand(1);
3393 if (!TLI.isTypeLegal(EltVT)) {
3394 EVT NewEltVT = TLI.getTypeToTransformTo(*DAG.getContext(), EltVT);
3395
3396 // BUILD_VECTOR operands are allowed to be wider than the element type.
3397 // But if NewEltVT is smaller that EltVT the BUILD_VECTOR does not accept
3398 // it.
3399 if (NewEltVT.bitsLT(EltVT)) {
3400 // Convert shuffle node.
3401 // If original node was v4i64 and the new EltVT is i32,
3402 // cast operands to v8i32 and re-build the mask.
3403
3404 // Calculate new VT, the size of the new VT should be equal to original.
3405 EVT NewVT =
3406 EVT::getVectorVT(*DAG.getContext(), NewEltVT,
3407 VT.getSizeInBits() / NewEltVT.getSizeInBits());
3408 assert(NewVT.bitsEq(VT));
3409
3410 // cast operands to new VT
3411 Op0 = DAG.getNode(ISD::BITCAST, dl, NewVT, Op0);
3412 Op1 = DAG.getNode(ISD::BITCAST, dl, NewVT, Op1);
3413
3414 // Convert the shuffle mask
3415 unsigned int factor =
3417
3418 // EltVT gets smaller
3419 assert(factor > 0);
3420
3421 for (unsigned i = 0; i < VT.getVectorNumElements(); ++i) {
3422 if (Mask[i] < 0) {
3423 for (unsigned fi = 0; fi < factor; ++fi)
3424 NewMask.push_back(Mask[i]);
3425 }
3426 else {
3427 for (unsigned fi = 0; fi < factor; ++fi)
3428 NewMask.push_back(Mask[i]*factor+fi);
3429 }
3430 }
3431 Mask = NewMask;
3432 VT = NewVT;
3433 }
3434 EltVT = NewEltVT;
3435 }
3436 unsigned NumElems = VT.getVectorNumElements();
3438 for (unsigned i = 0; i != NumElems; ++i) {
3439 if (Mask[i] < 0) {
3440 Ops.push_back(DAG.getUNDEF(EltVT));
3441 continue;
3442 }
3443 unsigned Idx = Mask[i];
3444 if (Idx < NumElems)
3445 Ops.push_back(DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT, Op0,
3446 DAG.getVectorIdxConstant(Idx, dl)));
3447 else
3448 Ops.push_back(
3449 DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT, Op1,
3450 DAG.getVectorIdxConstant(Idx - NumElems, dl)));
3451 }
3452
3453 Tmp1 = DAG.getBuildVector(VT, dl, Ops);
3454 // We may have changed the BUILD_VECTOR type. Cast it back to the Node type.
3455 Tmp1 = DAG.getNode(ISD::BITCAST, dl, Node->getValueType(0), Tmp1);
3456 Results.push_back(Tmp1);
3457 break;
3458 }
3459 case ISD::VECTOR_SPLICE: {
3460 Results.push_back(TLI.expandVectorSplice(Node, DAG));
3461 break;
3462 }
3463 case ISD::EXTRACT_ELEMENT: {
3464 EVT OpTy = Node->getOperand(0).getValueType();
3465 if (Node->getConstantOperandVal(1)) {
3466 // 1 -> Hi
3467 Tmp1 = DAG.getNode(ISD::SRL, dl, OpTy, Node->getOperand(0),
3468 DAG.getConstant(OpTy.getSizeInBits() / 2, dl,
3469 TLI.getShiftAmountTy(
3470 Node->getOperand(0).getValueType(),
3471 DAG.getDataLayout())));
3472 Tmp1 = DAG.getNode(ISD::TRUNCATE, dl, Node->getValueType(0), Tmp1);
3473 } else {
3474 // 0 -> Lo
3475 Tmp1 = DAG.getNode(ISD::TRUNCATE, dl, Node->getValueType(0),
3476 Node->getOperand(0));
3477 }
3478 Results.push_back(Tmp1);
3479 break;
3480 }
3481 case ISD::STACKSAVE:
3482 // Expand to CopyFromReg if the target set
3483 // StackPointerRegisterToSaveRestore.
3484 if (Register SP = TLI.getStackPointerRegisterToSaveRestore()) {
3485 Results.push_back(DAG.getCopyFromReg(Node->getOperand(0), dl, SP,
3486 Node->getValueType(0)));
3487 Results.push_back(Results[0].getValue(1));
3488 } else {
3489 Results.push_back(DAG.getUNDEF(Node->getValueType(0)));
3490 Results.push_back(Node->getOperand(0));
3491 }
3492 break;
3493 case ISD::STACKRESTORE:
3494 // Expand to CopyToReg if the target set
3495 // StackPointerRegisterToSaveRestore.
3496 if (Register SP = TLI.getStackPointerRegisterToSaveRestore()) {
3497 Results.push_back(DAG.getCopyToReg(Node->getOperand(0), dl, SP,
3498 Node->getOperand(1)));
3499 } else {
3500 Results.push_back(Node->getOperand(0));
3501 }
3502 break;
3504 Results.push_back(DAG.getConstant(0, dl, Node->getValueType(0)));
3505 Results.push_back(Results[0].getValue(0));
3506 break;
3507 case ISD::FCOPYSIGN:
3508 Results.push_back(ExpandFCOPYSIGN(Node));
3509 break;
3510 case ISD::FNEG:
3511 Results.push_back(ExpandFNEG(Node));
3512 break;
3513 case ISD::FABS:
3514 Results.push_back(ExpandFABS(Node));
3515 break;
3516 case ISD::IS_FPCLASS: {
3517 auto CNode = cast<ConstantSDNode>(Node->getOperand(1));
3518 auto Test = static_cast<FPClassTest>(CNode->getZExtValue());
3519 if (SDValue Expanded =
3520 TLI.expandIS_FPCLASS(Node->getValueType(0), Node->getOperand(0),
3521 Test, Node->getFlags(), SDLoc(Node), DAG))
3522 Results.push_back(Expanded);
3523 break;
3524 }
3525 case ISD::SMIN:
3526 case ISD::SMAX:
3527 case ISD::UMIN:
3528 case ISD::UMAX: {
3529 // Expand Y = MAX(A, B) -> Y = (A > B) ? A : B
3530 ISD::CondCode Pred;
3531 switch (Node->getOpcode()) {
3532 default: llvm_unreachable("How did we get here?");
3533 case ISD::SMAX: Pred = ISD::SETGT; break;
3534 case ISD::SMIN: Pred = ISD::SETLT; break;
3535 case ISD::UMAX: Pred = ISD::SETUGT; break;
3536 case ISD::UMIN: Pred = ISD::SETULT; break;
3537 }
3538 Tmp1 = Node->getOperand(0);
3539 Tmp2 = Node->getOperand(1);
3540 Tmp1 = DAG.getSelectCC(dl, Tmp1, Tmp2, Tmp1, Tmp2, Pred);
3541 Results.push_back(Tmp1);
3542 break;
3543 }
3544 case ISD::FMINNUM:
3545 case ISD::FMAXNUM: {
3546 if (SDValue Expanded = TLI.expandFMINNUM_FMAXNUM(Node, DAG))
3547 Results.push_back(Expanded);
3548 break;
3549 }
3550 case ISD::FSIN:
3551 case ISD::FCOS: {
3552 EVT VT = Node->getValueType(0);
3553 // Turn fsin / fcos into ISD::FSINCOS node if there are a pair of fsin /
3554 // fcos which share the same operand and both are used.
3555 if ((TLI.isOperationLegalOrCustom(ISD::FSINCOS, VT) ||
3557 && useSinCos(Node)) {
3558 SDVTList VTs = DAG.getVTList(VT, VT);
3559 Tmp1 = DAG.getNode(ISD::FSINCOS, dl, VTs, Node->getOperand(0));
3560 if (Node->getOpcode() == ISD::FCOS)
3561 Tmp1 = Tmp1.getValue(1);
3562 Results.push_back(Tmp1);
3563 }
3564 break;
3565 }
3566 case ISD::FLDEXP:
3567 case ISD::STRICT_FLDEXP: {
3568 EVT VT = Node->getValueType(0);
3570 // Use the LibCall instead, it is very likely faster
3571 // FIXME: Use separate LibCall action.
3572 if (TLI.getLibcallName(LC))
3573 break;
3574
3575 if (SDValue Expanded = expandLdexp(Node)) {
3576 Results.push_back(Expanded);
3577 if (Node->getOpcode() == ISD::STRICT_FLDEXP)
3578 Results.push_back(Expanded.getValue(1));
3579 }
3580
3581 break;
3582 }
3583 case ISD::FFREXP: {
3584 RTLIB::Libcall LC = RTLIB::getFREXP(Node->getValueType(0));
3585 // Use the LibCall instead, it is very likely faster
3586 // FIXME: Use separate LibCall action.
3587 if (TLI.getLibcallName(LC))
3588 break;
3589
3590 if (SDValue Expanded = expandFrexp(Node)) {
3591 Results.push_back(Expanded);
3592 Results.push_back(Expanded.getValue(1));
3593 }
3594 break;
3595 }
3596 case ISD::FMAD:
3597 llvm_unreachable("Illegal fmad should never be formed");
3598
3599 case ISD::FP16_TO_FP:
3600 if (Node->getValueType(0) != MVT::f32) {
3601 // We can extend to types bigger than f32 in two steps without changing
3602 // the result. Since "f16 -> f32" is much more commonly available, give
3603 // CodeGen the option of emitting that before resorting to a libcall.
3604 SDValue Res =
3605 DAG.getNode(ISD::FP16_TO_FP, dl, MVT::f32, Node->getOperand(0));
3606 Results.push_back(
3607 DAG.getNode(ISD::FP_EXTEND, dl, Node->getValueType(0), Res));
3608 }
3609 break;
3611 if (Node->getValueType(0) != MVT::f32) {
3612 // We can extend to types bigger than f32 in two steps without changing
3613 // the result. Since "f16 -> f32" is much more commonly available, give
3614 // CodeGen the option of emitting that before resorting to a libcall.
3615 SDValue Res =
3616 DAG.getNode(ISD::STRICT_FP16_TO_FP, dl, {MVT::f32, MVT::Other},
3617 {Node->getOperand(0), Node->getOperand(1)});
3618 Res = DAG.getNode(ISD::STRICT_FP_EXTEND, dl,
3619 {Node->getValueType(0), MVT::Other},
3620 {Res.getValue(1), Res});
3621 Results.push_back(Res);
3622 Results.push_back(Res.getValue(1));
3623 }
3624 break;
3625 case ISD::FP_TO_FP16:
3626 LLVM_DEBUG(dbgs() << "Legalizing FP_TO_FP16\n");
3627 if (!TLI.useSoftFloat() && TM.Options.UnsafeFPMath) {
3628 SDValue Op = Node->getOperand(0);
3629 MVT SVT = Op.getSimpleValueType();
3630 if ((SVT == MVT::f64 || SVT == MVT::f80) &&
3631 TLI.isOperationLegalOrCustom(ISD::FP_TO_FP16, MVT::f32)) {
3632 // Under fastmath, we can expand this node into a fround followed by
3633 // a float-half conversion.
3634 SDValue FloatVal =
3635 DAG.getNode(ISD::FP_ROUND, dl, MVT::f32, Op,
3636 DAG.getIntPtrConstant(0, dl, /*isTarget=*/true));
3637 Results.push_back(
3638 DAG.getNode(ISD::FP_TO_FP16, dl, Node->getValueType(0), FloatVal));
3639 }
3640 }
3641 break;
3642 case ISD::ConstantFP: {
3643 ConstantFPSDNode *CFP = cast<ConstantFPSDNode>(Node);
3644 // Check to see if this FP immediate is already legal.
3645 // If this is a legal constant, turn it into a TargetConstantFP node.
3646 if (!TLI.isFPImmLegal(CFP->getValueAPF(), Node->getValueType(0),
3647 DAG.shouldOptForSize()))
3648 Results.push_back(ExpandConstantFP(CFP, true));
3649 break;
3650 }
3651 case ISD::Constant: {
3652 ConstantSDNode *CP = cast<ConstantSDNode>(Node);
3653 Results.push_back(ExpandConstant(CP));
3654 break;
3655 }
3656 case ISD::FSUB: {
3657 EVT VT = Node->getValueType(0);
3658 if (TLI.isOperationLegalOrCustom(ISD::FADD, VT) &&
3659 TLI.isOperationLegalOrCustom(ISD::FNEG, VT)) {
3660 const SDNodeFlags Flags = Node->getFlags();
3661 Tmp1 = DAG.getNode(ISD::FNEG, dl, VT, Node->getOperand(1));
3662 Tmp1 = DAG.getNode(ISD::FADD, dl, VT, Node->getOperand(0), Tmp1, Flags);
3663 Results.push_back(Tmp1);
3664 }
3665 break;
3666 }
3667 case ISD::SUB: {
3668 EVT VT = Node->getValueType(0);
3669 assert(TLI.isOperationLegalOrCustom(ISD::ADD, VT) &&
3670 TLI.isOperationLegalOrCustom(ISD::XOR, VT) &&
3671 "Don't know how to expand this subtraction!");
3672 Tmp1 = DAG.getNOT(dl, Node->getOperand(1), VT);
3673 Tmp1 = DAG.getNode(ISD::ADD, dl, VT, Tmp1, DAG.getConstant(1, dl, VT));
3674 Results.push_back(DAG.getNode(ISD::ADD, dl, VT, Node->getOperand(0), Tmp1));
3675 break;
3676 }
3677 case ISD::UREM:
3678 case ISD::SREM:
3679 if (TLI.expandREM(Node, Tmp1, DAG))
3680 Results.push_back(Tmp1);
3681 break;
3682 case ISD::UDIV:
3683 case ISD::SDIV: {
3684 bool isSigned = Node->getOpcode() == ISD::SDIV;
3685 unsigned DivRemOpc = isSigned ? ISD::SDIVREM : ISD::UDIVREM;
3686 EVT VT = Node->getValueType(0);
3687 if (TLI.isOperationLegalOrCustom(DivRemOpc, VT)) {
3688 SDVTList VTs = DAG.getVTList(VT, VT);
3689 Tmp1 = DAG.getNode(DivRemOpc, dl, VTs, Node->getOperand(0),
3690 Node->getOperand(1));
3691 Results.push_back(Tmp1);
3692 }
3693 break;
3694 }
3695 case ISD::MULHU:
3696 case ISD::MULHS: {
3697 unsigned ExpandOpcode =
3698 Node->getOpcode() == ISD::MULHU ? ISD::UMUL_LOHI : ISD::SMUL_LOHI;
3699 EVT VT = Node->getValueType(0);
3700 SDVTList VTs = DAG.getVTList(VT, VT);
3701
3702 Tmp1 = DAG.getNode(ExpandOpcode, dl, VTs, Node->getOperand(0),
3703 Node->getOperand(1));
3704 Results.push_back(Tmp1.getValue(1));
3705 break;
3706 }
3707 case ISD::UMUL_LOHI:
3708 case ISD::SMUL_LOHI: {
3709 SDValue LHS = Node->getOperand(0);
3710 SDValue RHS = Node->getOperand(1);
3711 MVT VT = LHS.getSimpleValueType();
3712 unsigned MULHOpcode =
3713 Node->getOpcode() == ISD::UMUL_LOHI ? ISD::MULHU : ISD::MULHS;
3714
3715 if (TLI.isOperationLegalOrCustom(MULHOpcode, VT)) {
3716 Results.push_back(DAG.getNode(ISD::MUL, dl, VT, LHS, RHS));
3717 Results.push_back(DAG.getNode(MULHOpcode, dl, VT, LHS, RHS));
3718 break;
3719 }
3720
3722 EVT HalfType = EVT(VT).getHalfSizedIntegerVT(*DAG.getContext());
3723 assert(TLI.isTypeLegal(HalfType));
3724 if (TLI.expandMUL_LOHI(Node->getOpcode(), VT, dl, LHS, RHS, Halves,
3725 HalfType, DAG,
3726 TargetLowering::MulExpansionKind::Always)) {
3727 for (unsigned i = 0; i < 2; ++i) {
3728 SDValue Lo = DAG.getNode(ISD::ZERO_EXTEND, dl, VT, Halves[2 * i]);
3729 SDValue Hi = DAG.getNode(ISD::ANY_EXTEND, dl, VT, Halves[2 * i + 1]);
3730 SDValue Shift = DAG.getConstant(
3731 HalfType.getScalarSizeInBits(), dl,
3732 TLI.getShiftAmountTy(HalfType, DAG.getDataLayout()));
3733 Hi = DAG.getNode(ISD::SHL, dl, VT, Hi, Shift);
3734 Results.push_back(DAG.getNode(ISD::OR, dl, VT, Lo, Hi));
3735 }
3736 break;
3737 }
3738 break;
3739 }
3740 case ISD::MUL: {
3741 EVT VT = Node->getValueType(0);
3742 SDVTList VTs = DAG.getVTList(VT, VT);
3743 // See if multiply or divide can be lowered using two-result operations.
3744 // We just need the low half of the multiply; try both the signed
3745 // and unsigned forms. If the target supports both SMUL_LOHI and
3746 // UMUL_LOHI, form a preference by checking which forms of plain
3747 // MULH it supports.
3748 bool HasSMUL_LOHI = TLI.isOperationLegalOrCustom(ISD::SMUL_LOHI, VT);
3749 bool HasUMUL_LOHI = TLI.isOperationLegalOrCustom(ISD::UMUL_LOHI, VT);
3750 bool HasMULHS = TLI.isOperationLegalOrCustom(ISD::MULHS, VT);
3751 bool HasMULHU = TLI.isOperationLegalOrCustom(ISD::MULHU, VT);
3752 unsigned OpToUse = 0;
3753 if (HasSMUL_LOHI && !HasMULHS) {
3754 OpToUse = ISD::SMUL_LOHI;
3755 } else if (HasUMUL_LOHI && !HasMULHU) {
3756 OpToUse = ISD::UMUL_LOHI;
3757 } else if (HasSMUL_LOHI) {
3758 OpToUse = ISD::SMUL_LOHI;
3759 } else if (HasUMUL_LOHI) {
3760 OpToUse = ISD::UMUL_LOHI;
3761 }
3762 if (OpToUse) {
3763 Results.push_back(DAG.getNode(OpToUse, dl, VTs, Node->getOperand(0),
3764 Node->getOperand(1)));
3765 break;
3766 }
3767
3768 SDValue Lo, Hi;
3769 EVT HalfType = VT.getHalfSizedIntegerVT(*DAG.getContext());
3770 if (TLI.isOperationLegalOrCustom(ISD::ZERO_EXTEND, VT) &&
3771 TLI.isOperationLegalOrCustom(ISD::ANY_EXTEND, VT) &&
3772 TLI.isOperationLegalOrCustom(ISD::SHL, VT) &&
3773 TLI.isOperationLegalOrCustom(ISD::OR, VT) &&
3774 TLI.expandMUL(Node, Lo, Hi, HalfType, DAG,
3775 TargetLowering::MulExpansionKind::OnlyLegalOrCustom)) {
3776 Lo = DAG.getNode(ISD::ZERO_EXTEND, dl, VT, Lo);
3777 Hi = DAG.getNode(ISD::ANY_EXTEND, dl, VT, Hi);
3778 SDValue Shift =
3779 DAG.getConstant(HalfType.getSizeInBits(), dl,
3780 TLI.getShiftAmountTy(HalfType, DAG.getDataLayout()));
3781 Hi = DAG.getNode(ISD::SHL, dl, VT, Hi, Shift);
3782 Results.push_back(DAG.getNode(ISD::OR, dl, VT, Lo, Hi));
3783 }
3784 break;
3785 }
3786 case ISD::FSHL:
3787 case ISD::FSHR:
3788 if (SDValue Expanded = TLI.expandFunnelShift(Node, DAG))
3789 Results.push_back(Expanded);
3790 break;
3791 case ISD::ROTL:
3792 case ISD::ROTR:
3793 if (SDValue Expanded = TLI.expandROT(Node, true /*AllowVectorOps*/, DAG))
3794 Results.push_back(Expanded);
3795 break;
3796 case ISD::SADDSAT:
3797 case ISD::UADDSAT:
3798 case ISD::SSUBSAT:
3799 case ISD::USUBSAT:
3800 Results.push_back(TLI.expandAddSubSat(Node, DAG));
3801 break;
3802 case ISD::SSHLSAT:
3803 case ISD::USHLSAT:
3804 Results.push_back(TLI.expandShlSat(Node, DAG));
3805 break;
3806 case ISD::SMULFIX:
3807 case ISD::SMULFIXSAT:
3808 case ISD::UMULFIX:
3809 case ISD::UMULFIXSAT:
3810 Results.push_back(TLI.expandFixedPointMul(Node, DAG));
3811 break;
3812 case ISD::SDIVFIX:
3813 case ISD::SDIVFIXSAT:
3814 case ISD::UDIVFIX:
3815 case ISD::UDIVFIXSAT:
3816 if (SDValue V = TLI.expandFixedPointDiv(Node->getOpcode(), SDLoc(Node),
3817 Node->getOperand(0),
3818 Node->getOperand(1),
3819 Node->getConstantOperandVal(2),
3820 DAG)) {
3821 Results.push_back(V);
3822 break;
3823 }
3824 // FIXME: We might want to retry here with a wider type if we fail, if that
3825 // type is legal.
3826 // FIXME: Technically, so long as we only have sdivfixes where BW+Scale is
3827 // <= 128 (which is the case for all of the default Embedded-C types),
3828 // we will only get here with types and scales that we could always expand
3829 // if we were allowed to generate libcalls to division functions of illegal
3830 // type. But we cannot do that.
3831 llvm_unreachable("Cannot expand DIVFIX!");
3832 case ISD::UADDO_CARRY:
3833 case ISD::USUBO_CARRY: {
3834 SDValue LHS = Node->getOperand(0);
3835 SDValue RHS = Node->getOperand(1);
3836 SDValue Carry = Node->getOperand(2);
3837
3838 bool IsAdd = Node->getOpcode() == ISD::UADDO_CARRY;
3839
3840 // Initial add of the 2 operands.
3841 unsigned Op = IsAdd ? ISD::ADD : ISD::SUB;
3842 EVT VT = LHS.getValueType();
3843 SDValue Sum = DAG.getNode(Op, dl, VT, LHS, RHS);
3844
3845 // Initial check for overflow.
3846 EVT CarryType = Node->getValueType(1);
3847 EVT SetCCType = getSetCCResultType(Node->getValueType(0));
3849 SDValue Overflow = DAG.getSetCC(dl, SetCCType, Sum, LHS, CC);
3850
3851 // Add of the sum and the carry.
3852 SDValue One = DAG.getConstant(1, dl, VT);
3853 SDValue CarryExt =
3854 DAG.getNode(ISD::AND, dl, VT, DAG.getZExtOrTrunc(Carry, dl, VT), One);
3855 SDValue Sum2 = DAG.getNode(Op, dl, VT, Sum, CarryExt);
3856
3857 // Second check for overflow. If we are adding, we can only overflow if the
3858 // initial sum is all 1s ang the carry is set, resulting in a new sum of 0.
3859 // If we are subtracting, we can only overflow if the initial sum is 0 and
3860 // the carry is set, resulting in a new sum of all 1s.
3861 SDValue Zero = DAG.getConstant(0, dl, VT);
3862 SDValue Overflow2 =
3863 IsAdd ? DAG.getSetCC(dl, SetCCType, Sum2, Zero, ISD::SETEQ)
3864 : DAG.getSetCC(dl, SetCCType, Sum, Zero, ISD::SETEQ);
3865 Overflow2 = DAG.getNode(ISD::AND, dl, SetCCType, Overflow2,
3866 DAG.getZExtOrTrunc(Carry, dl, SetCCType));
3867
3868 SDValue ResultCarry =
3869 DAG.getNode(ISD::OR, dl, SetCCType, Overflow, Overflow2);
3870
3871 Results.push_back(Sum2);
3872 Results.push_back(DAG.getBoolExtOrTrunc(ResultCarry, dl, CarryType, VT));
3873 break;
3874 }
3875 case ISD::SADDO:
3876 case ISD::SSUBO: {
3877 SDValue Result, Overflow;
3878 TLI.expandSADDSUBO(Node, Result, Overflow, DAG);
3879 Results.push_back(Result);
3880 Results.push_back(Overflow);
3881 break;
3882 }
3883 case ISD::UADDO:
3884 case ISD::USUBO: {
3885 SDValue Result, Overflow;
3886 TLI.expandUADDSUBO(Node, Result, Overflow, DAG);
3887 Results.push_back(Result);
3888 Results.push_back(Overflow);
3889 break;
3890 }
3891 case ISD::UMULO:
3892 case ISD::SMULO: {
3893 SDValue Result, Overflow;
3894 if (TLI.expandMULO(Node, Result, Overflow, DAG)) {
3895 Results.push_back(Result);
3896 Results.push_back(Overflow);
3897 }
3898 break;
3899 }
3900 case ISD::BUILD_PAIR: {
3901 EVT PairTy = Node->getValueType(0);
3902 Tmp1 = DAG.getNode(ISD::ZERO_EXTEND, dl, PairTy, Node->getOperand(0));
3903 Tmp2 = DAG.getNode(ISD::ANY_EXTEND, dl, PairTy, Node->getOperand(1));
3904 Tmp2 = DAG.getNode(
3905 ISD::SHL, dl, PairTy, Tmp2,
3906 DAG.getConstant(PairTy.getSizeInBits() / 2, dl,
3907 TLI.getShiftAmountTy(PairTy, DAG.getDataLayout())));
3908 Results.push_back(DAG.getNode(ISD::OR, dl, PairTy, Tmp1, Tmp2));
3909 break;
3910 }
3911 case ISD::SELECT:
3912 Tmp1 = Node->getOperand(0);
3913 Tmp2 = Node->getOperand(1);
3914 Tmp3 = Node->getOperand(2);
3915 if (Tmp1.getOpcode() == ISD::SETCC) {
3916 Tmp1 = DAG.getSelectCC(dl, Tmp1.getOperand(0), Tmp1.getOperand(1),
3917 Tmp2, Tmp3,
3918 cast<CondCodeSDNode>(Tmp1.getOperand(2))->get());
3919 } else {
3920 Tmp1 = DAG.getSelectCC(dl, Tmp1,
3921 DAG.getConstant(0, dl, Tmp1.getValueType()),
3922 Tmp2, Tmp3, ISD::SETNE);
3923 }
3924 Tmp1->setFlags(Node->getFlags());
3925 Results.push_back(Tmp1);
3926 break;
3927 case ISD::BR_JT: {
3928 SDValue Chain = Node->getOperand(0);
3929 SDValue Table = Node->getOperand(1);
3930 SDValue Index = Node->getOperand(2);
3931 int JTI = cast<JumpTableSDNode>(Table.getNode())->getIndex();
3932
3933 const DataLayout &TD = DAG.getDataLayout();
3934 EVT PTy = TLI.getPointerTy(TD);
3935
3936 unsigned EntrySize =
3937 DAG.getMachineFunction().getJumpTableInfo()->getEntrySize(TD);
3938
3939 // For power-of-two jumptable entry sizes convert multiplication to a shift.
3940 // This transformation needs to be done here since otherwise the MIPS
3941 // backend will end up emitting a three instruction multiply sequence
3942 // instead of a single shift and MSP430 will call a runtime function.
3943 if (llvm::isPowerOf2_32(EntrySize))
3944 Index = DAG.getNode(
3945 ISD::SHL, dl, Index.getValueType(), Index,
3946 DAG.getConstant(llvm::Log2_32(EntrySize), dl, Index.getValueType()));
3947 else
3948 Index = DAG.getNode(ISD::MUL, dl, Index.getValueType(), Index,
3949 DAG.getConstant(EntrySize, dl, Index.getValueType()));
3950 SDValue Addr = DAG.getNode(ISD::ADD, dl, Index.getValueType(),
3951 Index, Table);
3952
3953 EVT MemVT = EVT::getIntegerVT(*DAG.getContext(), EntrySize * 8);
3954 SDValue LD = DAG.getExtLoad(
3955 ISD::SEXTLOAD, dl, PTy, Chain, Addr,
3956 MachinePointerInfo::getJumpTable(DAG.getMachineFunction()), MemVT);
3957 Addr = LD;
3958 if (TLI.isJumpTableRelative()) {
3959 // For PIC, the sequence is:
3960 // BRIND(load(Jumptable + index) + RelocBase)
3961 // RelocBase can be JumpTable, GOT or some sort of global base.
3962 Addr = DAG.getNode(ISD::ADD, dl, PTy, Addr,
3963 TLI.getPICJumpTableRelocBase(Table, DAG));
3964 }
3965
3966 Tmp1 = TLI.expandIndirectJTBranch(dl, LD.getValue(1), Addr, JTI, DAG);
3967 Results.push_back(Tmp1);
3968 break;
3969 }
3970 case ISD::BRCOND:
3971 // Expand brcond's setcc into its constituent parts and create a BR_CC
3972 // Node.
3973 Tmp1 = Node->getOperand(0);
3974 Tmp2 = Node->getOperand(1);
3975 if (Tmp2.getOpcode() == ISD::SETCC &&
3976 TLI.isOperationLegalOrCustom(ISD::BR_CC,
3977 Tmp2.getOperand(0).getValueType())) {
3978 Tmp1 = DAG.getNode(ISD::BR_CC, dl, MVT::Other, Tmp1, Tmp2.getOperand(2),
3979 Tmp2.getOperand(0), Tmp2.getOperand(1),
3980 Node->getOperand(2));
3981 } else {
3982 // We test only the i1 bit. Skip the AND if UNDEF or another AND.
3983 if (Tmp2.isUndef() ||
3984 (Tmp2.getOpcode() == ISD::AND && isOneConstant(Tmp2.getOperand(1))))
3985 Tmp3 = Tmp2;
3986 else
3987 Tmp3 = DAG.getNode(ISD::AND, dl, Tmp2.getValueType(), Tmp2,
3988 DAG.getConstant(1, dl, Tmp2.getValueType()));
3989 Tmp1 = DAG.getNode(ISD::BR_CC, dl, MVT::Other, Tmp1,
3990 DAG.getCondCode(ISD::SETNE), Tmp3,
3991 DAG.getConstant(0, dl, Tmp3.getValueType()),
3992 Node->getOperand(2));
3993 }
3994 Results.push_back(Tmp1);
3995 break;
3996 case ISD::SETCC:
3997 case ISD::VP_SETCC:
3998 case ISD::STRICT_FSETCC:
3999 case ISD::STRICT_FSETCCS: {
4000 bool IsVP = Node->getOpcode() == ISD::VP_SETCC;
4001 bool IsStrict = Node->getOpcode() == ISD::STRICT_FSETCC ||
4002 Node->getOpcode() == ISD::STRICT_FSETCCS;
4003 bool IsSignaling = Node->getOpcode() == ISD::STRICT_FSETCCS;
4004 SDValue Chain = IsStrict ? Node->getOperand(0) : SDValue();
4005 unsigned Offset = IsStrict ? 1 : 0;
4006 Tmp1 = Node->getOperand(0 + Offset);
4007 Tmp2 = Node->getOperand(1 + Offset);
4008 Tmp3 = Node->getOperand(2 + Offset);
4009 SDValue Mask, EVL;
4010 if (IsVP) {
4011 Mask = Node->getOperand(3 + Offset);
4012 EVL = Node->getOperand(4 + Offset);
4013 }
4014 bool Legalized = TLI.LegalizeSetCCCondCode(
4015 DAG, Node->getValueType(0), Tmp1, Tmp2, Tmp3, Mask, EVL, NeedInvert, dl,
4016 Chain, IsSignaling);
4017
4018 if (Legalized) {
4019 // If we expanded the SETCC by swapping LHS and RHS, or by inverting the
4020 // condition code, create a new SETCC node.
4021 if (Tmp3.getNode()) {
4022 if (IsStrict) {
4023 Tmp1 = DAG.getNode(Node->getOpcode(), dl, Node->getVTList(),
4024 {Chain, Tmp1, Tmp2, Tmp3}, Node->getFlags());
4025 Chain = Tmp1.getValue(1);
4026 } else if (IsVP) {
4027 Tmp1 = DAG.getNode(Node->getOpcode(), dl, Node->getValueType(0),
4028 {Tmp1, Tmp2, Tmp3, Mask, EVL}, Node->getFlags());
4029 } else {
4030 Tmp1 = DAG.getNode(Node->getOpcode(), dl, Node->getValueType(0), Tmp1,
4031 Tmp2, Tmp3, Node->getFlags());
4032 }
4033 }
4034
4035 // If we expanded the SETCC by inverting the condition code, then wrap
4036 // the existing SETCC in a NOT to restore the intended condition.
4037 if (NeedInvert) {
4038 if (!IsVP)
4039 Tmp1 = DAG.getLogicalNOT(dl, Tmp1, Tmp1->getValueType(0));
4040 else
4041 Tmp1 =
4042 DAG.getVPLogicalNOT(dl, Tmp1, Mask, EVL, Tmp1->getValueType(0));
4043 }
4044
4045 Results.push_back(Tmp1);
4046 if (IsStrict)
4047 Results.push_back(Chain);
4048
4049 break;
4050 }
4051
4052 // FIXME: It seems Legalized is false iff CCCode is Legal. I don't
4053 // understand if this code is useful for strict nodes.
4054 assert(!IsStrict && "Don't know how to expand for strict nodes.");
4055
4056 // Otherwise, SETCC for the given comparison type must be completely
4057 // illegal; expand it into a SELECT_CC.
4058 // FIXME: This drops the mask/evl for VP_SETCC.
4059 EVT VT = Node->getValueType(0);
4060 EVT Tmp1VT = Tmp1.getValueType();
4061 Tmp1 = DAG.getNode(ISD::SELECT_CC, dl, VT, Tmp1, Tmp2,
4062 DAG.getBoolConstant(true, dl, VT, Tmp1VT),
4063 DAG.getBoolConstant(false, dl, VT, Tmp1VT), Tmp3);
4064 Tmp1->setFlags(Node->getFlags());
4065 Results.push_back(Tmp1);
4066 break;
4067 }
4068 case ISD::SELECT_CC: {
4069 // TODO: need to add STRICT_SELECT_CC and STRICT_SELECT_CCS
4070 Tmp1 = Node->getOperand(0); // LHS
4071 Tmp2 = Node->getOperand(1); // RHS
4072 Tmp3 = Node->getOperand(2); // True
4073 Tmp4 = Node->getOperand(3); // False
4074 EVT VT = Node->getValueType(0);
4075 SDValue Chain;
4076 SDValue CC = Node->getOperand(4);
4077 ISD::CondCode CCOp = cast<CondCodeSDNode>(CC)->get();
4078
4079 if (TLI.isCondCodeLegalOrCustom(CCOp, Tmp1.getSimpleValueType())) {
4080 // If the condition code is legal, then we need to expand this
4081 // node using SETCC and SELECT.
4082 EVT CmpVT = Tmp1.getValueType();
4083 assert(!TLI.isOperationExpand(ISD::SELECT, VT) &&
4084 "Cannot expand ISD::SELECT_CC when ISD::SELECT also needs to be "
4085 "expanded.");
4086 EVT CCVT = getSetCCResultType(CmpVT);
4087 SDValue Cond = DAG.getNode(ISD::SETCC, dl, CCVT, Tmp1, Tmp2, CC, Node->getFlags());
4088 Results.push_back(DAG.getSelect(dl, VT, Cond, Tmp3, Tmp4));
4089 break;
4090 }
4091
4092 // SELECT_CC is legal, so the condition code must not be.
4093 bool Legalized = false;
4094 // Try to legalize by inverting the condition. This is for targets that
4095 // might support an ordered version of a condition, but not the unordered
4096 // version (or vice versa).
4097 ISD::CondCode InvCC = ISD::getSetCCInverse(CCOp, Tmp1.getValueType());
4098 if (TLI.isCondCodeLegalOrCustom(InvCC, Tmp1.getSimpleValueType())) {
4099 // Use the new condition code and swap true and false
4100 Legalized = true;
4101 Tmp1 = DAG.getSelectCC(dl, Tmp1, Tmp2, Tmp4, Tmp3, InvCC);
4102 Tmp1->setFlags(Node->getFlags());
4103 } else {
4104 // If The inverse is not legal, then try to swap the arguments using
4105 // the inverse condition code.
4107 if (TLI.isCondCodeLegalOrCustom(SwapInvCC, Tmp1.getSimpleValueType())) {
4108 // The swapped inverse condition is legal, so swap true and false,
4109 // lhs and rhs.
4110 Legalized = true;
4111 Tmp1 = DAG.getSelectCC(dl, Tmp2, Tmp1, Tmp4, Tmp3, SwapInvCC);
4112 Tmp1->setFlags(Node->getFlags());
4113 }
4114 }
4115
4116 if (!Legalized) {
4117 Legalized = TLI.LegalizeSetCCCondCode(
4118 DAG, getSetCCResultType(Tmp1.getValueType()), Tmp1, Tmp2, CC,
4119 /*Mask*/ SDValue(), /*EVL*/ SDValue(), NeedInvert, dl, Chain);
4120
4121 assert(Legalized && "Can't legalize SELECT_CC with legal condition!");
4122
4123 // If we expanded the SETCC by inverting the condition code, then swap
4124 // the True/False operands to match.
4125 if (NeedInvert)
4126 std::swap(Tmp3, Tmp4);
4127
4128 // If we expanded the SETCC by swapping LHS and RHS, or by inverting the
4129 // condition code, create a new SELECT_CC node.
4130 if (CC.getNode()) {
4131 Tmp1 = DAG.getNode(ISD::SELECT_CC, dl, Node->getValueType(0),
4132 Tmp1, Tmp2, Tmp3, Tmp4, CC);
4133 } else {
4134 Tmp2 = DAG.getConstant(0, dl, Tmp1.getValueType());
4135 CC = DAG.getCondCode(ISD::SETNE);
4136 Tmp1 = DAG.getNode(ISD::SELECT_CC, dl, Node->getValueType(0), Tmp1,
4137 Tmp2, Tmp3, Tmp4, CC);
4138 }
4139 Tmp1->setFlags(Node->getFlags());
4140 }
4141 Results.push_back(Tmp1);
4142 break;
4143 }
4144 case ISD::BR_CC: {
4145 // TODO: need to add STRICT_BR_CC and STRICT_BR_CCS
4146 SDValue Chain;
4147 Tmp1 = Node->getOperand(0); // Chain
4148 Tmp2 = Node->getOperand(2); // LHS
4149 Tmp3 = Node->getOperand(3); // RHS
4150 Tmp4 = Node->getOperand(1); // CC
4151
4152 bool Legalized = TLI.LegalizeSetCCCondCode(
4153 DAG, getSetCCResultType(Tmp2.getValueType()), Tmp2, Tmp3, Tmp4,
4154 /*Mask*/ SDValue(), /*EVL*/ SDValue(), NeedInvert, dl, Chain);
4155 (void)Legalized;
4156 assert(Legalized && "Can't legalize BR_CC with legal condition!");
4157
4158 // If we expanded the SETCC by swapping LHS and RHS, create a new BR_CC
4159 // node.
4160 if (Tmp4.getNode()) {
4161 assert(!NeedInvert && "Don't know how to invert BR_CC!");
4162
4163 Tmp1 = DAG.getNode(ISD::BR_CC, dl, Node->getValueType(0), Tmp1,
4164 Tmp4, Tmp2, Tmp3, Node->getOperand(4));
4165 } else {
4166 Tmp3 = DAG.getConstant(0, dl, Tmp2.getValueType());
4167 Tmp4 = DAG.getCondCode(NeedInvert ? ISD::SETEQ : ISD::SETNE);
4168 Tmp1 = DAG.getNode(ISD::BR_CC, dl, Node->getValueType(0), Tmp1, Tmp4,
4169 Tmp2, Tmp3, Node->getOperand(4));
4170 }
4171 Results.push_back(Tmp1);
4172 break;
4173 }
4174 case ISD::BUILD_VECTOR:
4175 Results.push_back(ExpandBUILD_VECTOR(Node));
4176 break;
4177 case ISD::SPLAT_VECTOR:
4178 Results.push_back(ExpandSPLAT_VECTOR(Node));
4179 break;
4180 case ISD::SRA:
4181 case ISD::SRL:
4182 case ISD::SHL: {
4183 // Scalarize vector SRA/SRL/SHL.
4184 EVT VT = Node->getValueType(0);
4185 assert(VT.isVector() && "Unable to legalize non-vector shift");
4186 assert(TLI.isTypeLegal(VT.getScalarType())&& "Element type must be legal");
4187 unsigned NumElem = VT.getVectorNumElements();
4188
4190 for (unsigned Idx = 0; Idx < NumElem; Idx++) {
4191 SDValue Ex =
4193 Node->getOperand(0), DAG.getVectorIdxConstant(Idx, dl));
4194 SDValue Sh =
4196 Node->getOperand(1), DAG.getVectorIdxConstant(Idx, dl));
4197 Scalars.push_back(DAG.getNode(Node->getOpcode(), dl,
4198 VT.getScalarType(), Ex, Sh));
4199 }
4200
4201 SDValue Result = DAG.getBuildVector(Node->getValueType(0), dl, Scalars);
4202 Results.push_back(Result);
4203 break;
4204 }
4207 case ISD::VECREDUCE_ADD:
4208 case ISD::VECREDUCE_MUL:
4209 case ISD::VECREDUCE_AND:
4210 case ISD::VECREDUCE_OR:
4211 case ISD::VECREDUCE_XOR:
4220 Results.push_back(TLI.expandVecReduce(Node, DAG));
4221 break;
4223 case ISD::GlobalAddress:
4226 case ISD::ConstantPool:
4227 case ISD::JumpTable:
4231 // FIXME: Custom lowering for these operations shouldn't return null!
4232 // Return true so that we don't call ConvertNodeToLibcall which also won't
4233 // do anything.
4234 return true;
4235 }
4236
4237 if (!TLI.isStrictFPEnabled() && Results.empty() && Node->isStrictFPOpcode()) {
4238 // FIXME: We were asked to expand a strict floating-point operation,
4239 // but there is currently no expansion implemented that would preserve
4240 // the "strict" properties. For now, we just fall back to the non-strict
4241 // version if that is legal on the target. The actual mutation of the
4242 // operation will happen in SelectionDAGISel::DoInstructionSelection.
4243 switch (Node->getOpcode()) {
4244 default:
4245 if (TLI.getStrictFPOperationAction(Node->getOpcode(),
4246 Node->getValueType(0))
4247 == TargetLowering::Legal)
4248 return true;
4249 break;
4250 case ISD::STRICT_FSUB: {
4251 if (TLI.getStrictFPOperationAction(
4252 ISD::STRICT_FSUB, Node->getValueType(0)) == TargetLowering::Legal)
4253 return true;
4254 if (TLI.getStrictFPOperationAction(
4255 ISD::STRICT_FADD, Node->getValueType(0)) != TargetLowering::Legal)
4256 break;
4257
4258 EVT VT = Node->getValueType(0);
4259 const SDNodeFlags Flags = Node->getFlags();
4260 SDValue Neg = DAG.getNode(ISD::FNEG, dl, VT, Node->getOperand(2), Flags);
4261 SDValue Fadd = DAG.getNode(ISD::STRICT_FADD, dl, Node->getVTList(),
4262 {Node->getOperand(0), Node->getOperand(1), Neg},
4263 Flags);
4264
4265 Results.push_back(Fadd);
4266 Results.push_back(Fadd.getValue(1));
4267 break;
4268 }
4271 case ISD::STRICT_LRINT:
4272 case ISD::STRICT_LLRINT:
4273 case ISD::STRICT_LROUND:
4275 // These are registered by the operand type instead of the value
4276 // type. Reflect that here.
4277 if (TLI.getStrictFPOperationAction(Node->getOpcode(),
4278 Node->getOperand(1).getValueType())
4279 == TargetLowering::Legal)
4280 return true;
4281 break;
4282 }
4283 }
4284
4285 // Replace the original node with the legalized result.
4286 if (Results.empty()) {
4287 LLVM_DEBUG(dbgs() << "Cannot expand node\n");
4288 return false;
4289 }
4290
4291 LLVM_DEBUG(dbgs() << "Successfully expanded node\n");
4292 ReplaceNode(Node, Results.data());
4293 return true;
4294}
4295
4296void SelectionDAGLegalize::ConvertNodeToLibcall(SDNode *Node) {
4297 LLVM_DEBUG(dbgs() << "Trying to convert node to libcall\n");
4299 SDLoc dl(Node);
4300 // FIXME: Check flags on the node to see if we can use a finite call.
4301 unsigned Opc = Node->getOpcode();
4302 switch (Opc) {
4303 case ISD::ATOMIC_FENCE: {
4304 // If the target didn't lower this, lower it to '__sync_synchronize()' call
4305 // FIXME: handle "fence singlethread" more efficiently.
4307
4309 CLI.setDebugLoc(dl)
4310 .setChain(Node->getOperand(0))
4311 .setLibCallee(
4312 CallingConv::C, Type::getVoidTy(*DAG.getContext()),
4313 DAG.getExternalSymbol("__sync_synchronize",
4314 TLI.getPointerTy(DAG.getDataLayout())),
4315 std::move(Args));
4316
4317 std::pair<SDValue, SDValue> CallResult = TLI.LowerCallTo(CLI);
4318
4319 Results.push_back(CallResult.second);
4320 break;
4321 }
4322 // By default, atomic intrinsics are marked Legal and lowered. Targets
4323 // which don't support them directly, however, may want libcalls, in which
4324 // case they mark them Expand, and we get here.
4325 case ISD::ATOMIC_SWAP:
4337 case ISD::ATOMIC_CMP_SWAP: {
4338 MVT VT = cast<AtomicSDNode>(Node)->getMemoryVT().getSimpleVT();
4339 AtomicOrdering Order = cast<AtomicSDNode>(Node)->getMergedOrdering();
4340 RTLIB::Libcall LC = RTLIB::getOUTLINE_ATOMIC(Opc, Order, VT);
4341 EVT RetVT = Node->getValueType(0);
4344 if (TLI.getLibcallName(LC)) {
4345 // If outline atomic available, prepare its arguments and expand.
4346 Ops.append(Node->op_begin() + 2, Node->op_end());
4347 Ops.push_back(Node->getOperand(1));
4348
4349 } else {
4350 LC = RTLIB::getSYNC(Opc, VT);
4351 assert(LC != RTLIB::UNKNOWN_LIBCALL &&
4352 "Unexpected atomic op or value type!");
4353 // Arguments for expansion to sync libcall
4354 Ops.append(Node->op_begin() + 1, Node->op_end());
4355 }
4356 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, RetVT,
4357 Ops, CallOptions,
4358 SDLoc(Node),
4359 Node->getOperand(0));
4360 Results.push_back(Tmp.first);
4361 Results.push_back(Tmp.second);
4362 break;
4363 }
4364 case ISD::TRAP: {
4365 // If this operation is not supported, lower it to 'abort()' call
4368 CLI.setDebugLoc(dl)
4369 .setChain(Node->getOperand(0))
4370 .setLibCallee(CallingConv::C, Type::getVoidTy(*DAG.getContext()),
4371 DAG.getExternalSymbol(
4372 "abort", TLI.getPointerTy(DAG.getDataLayout())),
4373 std::move(Args));
4374 std::pair<SDValue, SDValue> CallResult = TLI.LowerCallTo(CLI);
4375
4376 Results.push_back(CallResult.second);
4377 break;
4378 }
4379 case ISD::FMINNUM:
4381 ExpandFPLibCall(Node, RTLIB::FMIN_F32, RTLIB::FMIN_F64,
4382 RTLIB::FMIN_F80, RTLIB::FMIN_F128,
4383 RTLIB::FMIN_PPCF128, Results);
4384 break;
4385 // FIXME: We do not have libcalls for FMAXIMUM and FMINIMUM. So, we cannot use
4386 // libcall legalization for these nodes, but there is no default expasion for
4387 // these nodes either (see PR63267 for example).
4388 case ISD::FMAXNUM:
4390 ExpandFPLibCall(Node, RTLIB::FMAX_F32, RTLIB::FMAX_F64,
4391 RTLIB::FMAX_F80, RTLIB::FMAX_F128,
4392 RTLIB::FMAX_PPCF128, Results);
4393 break;
4394 case ISD::FSQRT:
4395 case ISD::STRICT_FSQRT:
4396 ExpandFPLibCall(Node, RTLIB::SQRT_F32, RTLIB::SQRT_F64,
4397 RTLIB::SQRT_F80, RTLIB::SQRT_F128,
4398 RTLIB::SQRT_PPCF128, Results);
4399 break;
4400 case ISD::FCBRT:
4401 ExpandFPLibCall(Node, RTLIB::CBRT_F32, RTLIB::CBRT_F64,
4402 RTLIB::CBRT_F80, RTLIB::CBRT_F128,
4403 RTLIB::CBRT_PPCF128, Results);
4404 break;
4405 case ISD::FSIN:
4406 case ISD::STRICT_FSIN:
4407 ExpandFPLibCall(Node, RTLIB::SIN_F32, RTLIB::SIN_F64,
4408 RTLIB::SIN_F80, RTLIB::SIN_F128,
4409 RTLIB::SIN_PPCF128, Results);
4410 break;
4411 case ISD::FCOS:
4412 case ISD::STRICT_FCOS:
4413 ExpandFPLibCall(Node, RTLIB::COS_F32, RTLIB::COS_F64,
4414 RTLIB::COS_F80, RTLIB::COS_F128,
4415 RTLIB::COS_PPCF128, Results);
4416 break;
4417 case ISD::FSINCOS:
4418 // Expand into sincos libcall.
4419 ExpandSinCosLibCall(Node, Results);
4420 break;
4421 case ISD::FLOG:
4422 case ISD::STRICT_FLOG:
4423 ExpandFPLibCall(Node, RTLIB::LOG_F32, RTLIB::LOG_F64, RTLIB::LOG_F80,
4424 RTLIB::LOG_F128, RTLIB::LOG_PPCF128, Results);
4425 break;
4426 case ISD::FLOG2:
4427 case ISD::STRICT_FLOG2:
4428 ExpandFPLibCall(Node, RTLIB::LOG2_F32, RTLIB::LOG2_F64, RTLIB::LOG2_F80,
4429 RTLIB::LOG2_F128, RTLIB::LOG2_PPCF128, Results);
4430 break;
4431 case ISD::FLOG10:
4432 case ISD::STRICT_FLOG10:
4433 ExpandFPLibCall(Node, RTLIB::LOG10_F32, RTLIB::LOG10_F64, RTLIB::LOG10_F80,
4434 RTLIB::LOG10_F128, RTLIB::LOG10_PPCF128, Results);
4435 break;
4436 case ISD::FEXP:
4437 case ISD::STRICT_FEXP:
4438 ExpandFPLibCall(Node, RTLIB::EXP_F32, RTLIB::EXP_F64, RTLIB::EXP_F80,
4439 RTLIB::EXP_F128, RTLIB::EXP_PPCF128, Results);
4440 break;
4441 case ISD::FEXP2:
4442 case ISD::STRICT_FEXP2:
4443 ExpandFPLibCall(Node, RTLIB::EXP2_F32, RTLIB::EXP2_F64, RTLIB::EXP2_F80,
4444 RTLIB::EXP2_F128, RTLIB::EXP2_PPCF128, Results);
4445 break;
4446 case ISD::FEXP10:
4447 ExpandFPLibCall(Node, RTLIB::EXP10_F32, RTLIB::EXP10_F64, RTLIB::EXP10_F80,
4448 RTLIB::EXP10_F128, RTLIB::EXP10_PPCF128, Results);
4449 break;
4450 case ISD::FTRUNC:
4451 case ISD::STRICT_FTRUNC:
4452 ExpandFPLibCall(Node, RTLIB::TRUNC_F32, RTLIB::TRUNC_F64,
4453 RTLIB::TRUNC_F80, RTLIB::TRUNC_F128,
4454 RTLIB::TRUNC_PPCF128, Results);
4455 break;
4456 case ISD::FFLOOR:
4457 case ISD::STRICT_FFLOOR:
4458 ExpandFPLibCall(Node, RTLIB::FLOOR_F32, RTLIB::FLOOR_F64,
4459 RTLIB::FLOOR_F80, RTLIB::FLOOR_F128,
4460 RTLIB::FLOOR_PPCF128, Results);
4461 break;
4462 case ISD::FCEIL:
4463 case ISD::STRICT_FCEIL:
4464 ExpandFPLibCall(Node, RTLIB::CEIL_F32, RTLIB::CEIL_F64,
4465 RTLIB::CEIL_F80, RTLIB::CEIL_F128,
4466 RTLIB::CEIL_PPCF128, Results);
4467 break;
4468 case ISD::FRINT:
4469 case ISD::STRICT_FRINT:
4470 ExpandFPLibCall(Node, RTLIB::RINT_F32, RTLIB::RINT_F64,
4471 RTLIB::RINT_F80, RTLIB::RINT_F128,
4472 RTLIB::RINT_PPCF128, Results);
4473 break;
4474 case ISD::FNEARBYINT:
4476 ExpandFPLibCall(Node, RTLIB::NEARBYINT_F32,
4477 RTLIB::NEARBYINT_F64,
4478 RTLIB::NEARBYINT_F80,
4479 RTLIB::NEARBYINT_F128,
4480 RTLIB::NEARBYINT_PPCF128, Results);
4481 break;
4482 case ISD::FROUND:
4483 case ISD::STRICT_FROUND:
4484 ExpandFPLibCall(Node, RTLIB::ROUND_F32,
4485 RTLIB::ROUND_F64,
4486 RTLIB::ROUND_F80,
4487 RTLIB::ROUND_F128,
4488 RTLIB::ROUND_PPCF128, Results);
4489 break;
4490 case ISD::FROUNDEVEN:
4492 ExpandFPLibCall(Node, RTLIB::ROUNDEVEN_F32,
4493 RTLIB::ROUNDEVEN_F64,
4494 RTLIB::ROUNDEVEN_F80,
4495 RTLIB::ROUNDEVEN_F128,
4496 RTLIB::ROUNDEVEN_PPCF128, Results);
4497 break;
4498 case ISD::FLDEXP:
4499 case ISD::STRICT_FLDEXP:
4500 ExpandFPLibCall(Node, RTLIB::LDEXP_F32, RTLIB::LDEXP_F64, RTLIB::LDEXP_F80,
4501 RTLIB::LDEXP_F128, RTLIB::LDEXP_PPCF128, Results);
4502 break;
4503 case ISD::FFREXP: {
4504 ExpandFrexpLibCall(Node, Results);
4505 break;
4506 }
4507 case ISD::FPOWI:
4508 case ISD::STRICT_FPOWI: {
4509 RTLIB::Libcall LC = RTLIB::getPOWI(Node->getSimpleValueType(0));
4510 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected fpowi.");
4511 if (!TLI.getLibcallName(LC)) {
4512 // Some targets don't have a powi libcall; use pow instead.
4513 if (Node->isStrictFPOpcode()) {
4515 DAG.getNode(ISD::STRICT_SINT_TO_FP, SDLoc(Node),
4516 {Node->getValueType(0), Node->getValueType(1)},
4517 {Node->getOperand(0), Node->getOperand(2)});
4518 SDValue FPOW =
4519 DAG.getNode(ISD::STRICT_FPOW, SDLoc(Node),
4520 {Node->getValueType(0), Node->getValueType(1)},
4521 {Exponent.getValue(1), Node->getOperand(1), Exponent});
4522 Results.push_back(FPOW);
4523 Results.push_back(FPOW.getValue(1));
4524 } else {
4526 DAG.getNode(ISD::SINT_TO_FP, SDLoc(Node), Node->getValueType(0),
4527 Node->getOperand(1));
4528 Results.push_back(DAG.getNode(ISD::FPOW, SDLoc(Node),
4529 Node->getValueType(0),
4530 Node->getOperand(0), Exponent));
4531 }
4532 break;
4533 }
4534 unsigned Offset = Node->isStrictFPOpcode() ? 1 : 0;
4535 bool ExponentHasSizeOfInt =
4536 DAG.getLibInfo().getIntSize() ==
4537 Node->getOperand(1 + Offset).getValueType().getSizeInBits();
4538 if (!ExponentHasSizeOfInt) {
4539 // If the exponent does not match with sizeof(int) a libcall to
4540 // RTLIB::POWI would use the wrong type for the argument.
4541 DAG.getContext()->emitError("POWI exponent does not match sizeof(int)");
4542 Results.push_back(DAG.getUNDEF(Node->getValueType(0)));
4543 break;
4544 }
4545 ExpandFPLibCall(Node, LC, Results);
4546 break;
4547 }
4548 case ISD::FPOW:
4549 case ISD::STRICT_FPOW:
4550 ExpandFPLibCall(Node, RTLIB::POW_F32, RTLIB::POW_F64, RTLIB::POW_F80,
4551 RTLIB::POW_F128, RTLIB::POW_PPCF128, Results);
4552 break;
4553 case ISD::LROUND:
4554 case ISD::STRICT_LROUND:
4555 ExpandArgFPLibCall(Node, RTLIB::LROUND_F32,
4556 RTLIB::LROUND_F64, RTLIB::LROUND_F80,
4557 RTLIB::LROUND_F128,
4558 RTLIB::LROUND_PPCF128, Results);
4559 break;
4560 case ISD::LLROUND:
4562 ExpandArgFPLibCall(Node, RTLIB::LLROUND_F32,
4563 RTLIB::LLROUND_F64, RTLIB::LLROUND_F80,
4564 RTLIB::LLROUND_F128,
4565 RTLIB::LLROUND_PPCF128, Results);
4566 break;
4567 case ISD::LRINT:
4568 case ISD::STRICT_LRINT:
4569 ExpandArgFPLibCall(Node, RTLIB::LRINT_F32,
4570 RTLIB::LRINT_F64, RTLIB::LRINT_F80,
4571 RTLIB::LRINT_F128,
4572 RTLIB::LRINT_PPCF128, Results);
4573 break;
4574 case ISD::LLRINT:
4575 case ISD::STRICT_LLRINT:
4576 ExpandArgFPLibCall(Node, RTLIB::LLRINT_F32,
4577 RTLIB::LLRINT_F64, RTLIB::LLRINT_F80,
4578 RTLIB::LLRINT_F128,
4579 RTLIB::LLRINT_PPCF128, Results);
4580 break;
4581 case ISD::FDIV:
4582 case ISD::STRICT_FDIV:
4583 ExpandFPLibCall(Node, RTLIB::DIV_F32, RTLIB::DIV_F64,
4584 RTLIB::DIV_F80, RTLIB::DIV_F128,
4585 RTLIB::DIV_PPCF128, Results);
4586 break;
4587 case ISD::FREM:
4588 case ISD::STRICT_FREM:
4589 ExpandFPLibCall(Node, RTLIB::REM_F32, RTLIB::REM_F64,
4590 RTLIB::REM_F80, RTLIB::REM_F128,
4591 RTLIB::REM_PPCF128, Results);
4592 break;
4593 case ISD::FMA:
4594 case ISD::STRICT_FMA:
4595 ExpandFPLibCall(Node, RTLIB::FMA_F32, RTLIB::FMA_F64,
4596 RTLIB::FMA_F80, RTLIB::FMA_F128,
4597 RTLIB::FMA_PPCF128, Results);
4598 break;
4599 case ISD::FADD:
4600 case ISD::STRICT_FADD:
4601 ExpandFPLibCall(Node, RTLIB::ADD_F32, RTLIB::ADD_F64,
4602 RTLIB::ADD_F80, RTLIB::ADD_F128,
4603 RTLIB::ADD_PPCF128, Results);
4604 break;
4605 case ISD::FMUL:
4606 case ISD::STRICT_FMUL:
4607 ExpandFPLibCall(Node, RTLIB::MUL_F32, RTLIB::MUL_F64,
4608 RTLIB::MUL_F80, RTLIB::MUL_F128,
4609 RTLIB::MUL_PPCF128, Results);
4610 break;
4611 case ISD::FP16_TO_FP:
4612 if (Node->getValueType(0) == MVT::f32) {
4613 Results.push_back(ExpandLibCall(RTLIB::FPEXT_F16_F32, Node, false).first);
4614 }
4615 break;
4617 if (Node->getValueType(0) == MVT::f32) {
4619 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(
4620 DAG, RTLIB::FPEXT_F16_F32, MVT::f32, Node->getOperand(1), CallOptions,
4621 SDLoc(Node), Node->getOperand(0));
4622 Results.push_back(Tmp.first);
4623 Results.push_back(Tmp.second);
4624 }
4625 break;
4626 }
4627 case ISD::FP_TO_FP16: {
4628 RTLIB::Libcall LC =
4629 RTLIB::getFPROUND(Node->getOperand(0).getValueType(), MVT::f16);
4630 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unable to expand fp_to_fp16");
4631 Results.push_back(ExpandLibCall(LC, Node, false).first);
4632 break;
4633 }
4634 case ISD::FP_TO_BF16: {
4635 RTLIB::Libcall LC =
4636 RTLIB::getFPROUND(Node->getOperand(0).getValueType(), MVT::bf16);
4637 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unable to expand fp_to_bf16");
4638 Results.push_back(ExpandLibCall(LC, Node, false).first);
4639 break;
4640 }
4643 case ISD::SINT_TO_FP:
4644