LLVM 19.0.0git
LegalizeDAG.cpp
Go to the documentation of this file.
1//===- LegalizeDAG.cpp - Implement SelectionDAG::Legalize -----------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements the SelectionDAG::Legalize method.
10//
11//===----------------------------------------------------------------------===//
12
13#include "llvm/ADT/APFloat.h"
14#include "llvm/ADT/APInt.h"
15#include "llvm/ADT/ArrayRef.h"
17#include "llvm/ADT/SetVector.h"
19#include "llvm/ADT/SmallSet.h"
36#include "llvm/IR/CallingConv.h"
37#include "llvm/IR/Constants.h"
38#include "llvm/IR/DataLayout.h"
40#include "llvm/IR/Function.h"
41#include "llvm/IR/Metadata.h"
42#include "llvm/IR/Type.h"
45#include "llvm/Support/Debug.h"
51#include <cassert>
52#include <cstdint>
53#include <tuple>
54#include <utility>
55
56using namespace llvm;
57
58#define DEBUG_TYPE "legalizedag"
59
60namespace {
61
62/// Keeps track of state when getting the sign of a floating-point value as an
63/// integer.
64struct FloatSignAsInt {
65 EVT FloatVT;
66 SDValue Chain;
67 SDValue FloatPtr;
68 SDValue IntPtr;
69 MachinePointerInfo IntPointerInfo;
70 MachinePointerInfo FloatPointerInfo;
71 SDValue IntValue;
72 APInt SignMask;
73 uint8_t SignBit;
74};
75
76//===----------------------------------------------------------------------===//
77/// This takes an arbitrary SelectionDAG as input and
78/// hacks on it until the target machine can handle it. This involves
79/// eliminating value sizes the machine cannot handle (promoting small sizes to
80/// large sizes or splitting up large values into small values) as well as
81/// eliminating operations the machine cannot handle.
82///
83/// This code also does a small amount of optimization and recognition of idioms
84/// as part of its processing. For example, if a target does not support a
85/// 'setcc' instruction efficiently, but does support 'brcc' instruction, this
86/// will attempt merge setcc and brc instructions into brcc's.
87class SelectionDAGLegalize {
88 const TargetMachine &TM;
89 const TargetLowering &TLI;
90 SelectionDAG &DAG;
91
92 /// The set of nodes which have already been legalized. We hold a
93 /// reference to it in order to update as necessary on node deletion.
94 SmallPtrSetImpl<SDNode *> &LegalizedNodes;
95
96 /// A set of all the nodes updated during legalization.
97 SmallSetVector<SDNode *, 16> *UpdatedNodes;
98
99 EVT getSetCCResultType(EVT VT) const {
100 return TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT);
101 }
102
103 // Libcall insertion helpers.
104
105public:
106 SelectionDAGLegalize(SelectionDAG &DAG,
107 SmallPtrSetImpl<SDNode *> &LegalizedNodes,
108 SmallSetVector<SDNode *, 16> *UpdatedNodes = nullptr)
109 : TM(DAG.getTarget()), TLI(DAG.getTargetLoweringInfo()), DAG(DAG),
110 LegalizedNodes(LegalizedNodes), UpdatedNodes(UpdatedNodes) {}
111
112 /// Legalizes the given operation.
113 void LegalizeOp(SDNode *Node);
114
115private:
116 SDValue OptimizeFloatStore(StoreSDNode *ST);
117
118 void LegalizeLoadOps(SDNode *Node);
119 void LegalizeStoreOps(SDNode *Node);
120
121 SDValue ExpandINSERT_VECTOR_ELT(SDValue Op);
122
123 /// Return a vector shuffle operation which
124 /// performs the same shuffe in terms of order or result bytes, but on a type
125 /// whose vector element type is narrower than the original shuffle type.
126 /// e.g. <v4i32> <0, 1, 0, 1> -> v8i16 <0, 1, 2, 3, 0, 1, 2, 3>
127 SDValue ShuffleWithNarrowerEltType(EVT NVT, EVT VT, const SDLoc &dl,
128 SDValue N1, SDValue N2,
129 ArrayRef<int> Mask) const;
130
131 std::pair<SDValue, SDValue> ExpandLibCall(RTLIB::Libcall LC, SDNode *Node,
133 std::pair<SDValue, SDValue> ExpandLibCall(RTLIB::Libcall LC, SDNode *Node, bool isSigned);
134
135 void ExpandFrexpLibCall(SDNode *Node, SmallVectorImpl<SDValue> &Results);
136 void ExpandFPLibCall(SDNode *Node, RTLIB::Libcall LC,
138 void ExpandFPLibCall(SDNode *Node, RTLIB::Libcall Call_F32,
139 RTLIB::Libcall Call_F64, RTLIB::Libcall Call_F80,
140 RTLIB::Libcall Call_F128,
141 RTLIB::Libcall Call_PPCF128,
143 SDValue ExpandIntLibCall(SDNode *Node, bool isSigned,
144 RTLIB::Libcall Call_I8,
145 RTLIB::Libcall Call_I16,
146 RTLIB::Libcall Call_I32,
147 RTLIB::Libcall Call_I64,
148 RTLIB::Libcall Call_I128);
149 void ExpandArgFPLibCall(SDNode *Node,
150 RTLIB::Libcall Call_F32, RTLIB::Libcall Call_F64,
151 RTLIB::Libcall Call_F80, RTLIB::Libcall Call_F128,
152 RTLIB::Libcall Call_PPCF128,
154 void ExpandDivRemLibCall(SDNode *Node, SmallVectorImpl<SDValue> &Results);
155 void ExpandSinCosLibCall(SDNode *Node, SmallVectorImpl<SDValue> &Results);
156
157 SDValue EmitStackConvert(SDValue SrcOp, EVT SlotVT, EVT DestVT,
158 const SDLoc &dl);
159 SDValue EmitStackConvert(SDValue SrcOp, EVT SlotVT, EVT DestVT,
160 const SDLoc &dl, SDValue ChainIn);
161 SDValue ExpandBUILD_VECTOR(SDNode *Node);
162 SDValue ExpandSPLAT_VECTOR(SDNode *Node);
163 SDValue ExpandSCALAR_TO_VECTOR(SDNode *Node);
164 void ExpandDYNAMIC_STACKALLOC(SDNode *Node,
166 void getSignAsIntValue(FloatSignAsInt &State, const SDLoc &DL,
167 SDValue Value) const;
168 SDValue modifySignAsInt(const FloatSignAsInt &State, const SDLoc &DL,
169 SDValue NewIntValue) const;
170 SDValue ExpandFCOPYSIGN(SDNode *Node) const;
171 SDValue ExpandFABS(SDNode *Node) const;
172 SDValue ExpandFNEG(SDNode *Node) const;
173 SDValue expandLdexp(SDNode *Node) const;
174 SDValue expandFrexp(SDNode *Node) const;
175
176 SDValue ExpandLegalINT_TO_FP(SDNode *Node, SDValue &Chain);
177 void PromoteLegalINT_TO_FP(SDNode *N, const SDLoc &dl,
179 void PromoteLegalFP_TO_INT(SDNode *N, const SDLoc &dl,
181 SDValue PromoteLegalFP_TO_INT_SAT(SDNode *Node, const SDLoc &dl);
182
183 SDValue ExpandPARITY(SDValue Op, const SDLoc &dl);
184
185 SDValue ExpandExtractFromVectorThroughStack(SDValue Op);
186 SDValue ExpandInsertToVectorThroughStack(SDValue Op);
187 SDValue ExpandVectorBuildThroughStack(SDNode* Node);
188
189 SDValue ExpandConstantFP(ConstantFPSDNode *CFP, bool UseCP);
190 SDValue ExpandConstant(ConstantSDNode *CP);
191
192 // if ExpandNode returns false, LegalizeOp falls back to ConvertNodeToLibcall
193 bool ExpandNode(SDNode *Node);
194 void ConvertNodeToLibcall(SDNode *Node);
195 void PromoteNode(SDNode *Node);
196
197public:
198 // Node replacement helpers
199
200 void ReplacedNode(SDNode *N) {
201 LegalizedNodes.erase(N);
202 if (UpdatedNodes)
203 UpdatedNodes->insert(N);
204 }
205
206 void ReplaceNode(SDNode *Old, SDNode *New) {
207 LLVM_DEBUG(dbgs() << " ... replacing: "; Old->dump(&DAG);
208 dbgs() << " with: "; New->dump(&DAG));
209
210 assert(Old->getNumValues() == New->getNumValues() &&
211 "Replacing one node with another that produces a different number "
212 "of values!");
213 DAG.ReplaceAllUsesWith(Old, New);
214 if (UpdatedNodes)
215 UpdatedNodes->insert(New);
216 ReplacedNode(Old);
217 }
218
219 void ReplaceNode(SDValue Old, SDValue New) {
220 LLVM_DEBUG(dbgs() << " ... replacing: "; Old->dump(&DAG);
221 dbgs() << " with: "; New->dump(&DAG));
222
223 DAG.ReplaceAllUsesWith(Old, New);
224 if (UpdatedNodes)
225 UpdatedNodes->insert(New.getNode());
226 ReplacedNode(Old.getNode());
227 }
228
229 void ReplaceNode(SDNode *Old, const SDValue *New) {
230 LLVM_DEBUG(dbgs() << " ... replacing: "; Old->dump(&DAG));
231
232 DAG.ReplaceAllUsesWith(Old, New);
233 for (unsigned i = 0, e = Old->getNumValues(); i != e; ++i) {
234 LLVM_DEBUG(dbgs() << (i == 0 ? " with: " : " and: ");
235 New[i]->dump(&DAG));
236 if (UpdatedNodes)
237 UpdatedNodes->insert(New[i].getNode());
238 }
239 ReplacedNode(Old);
240 }
241
242 void ReplaceNodeWithValue(SDValue Old, SDValue New) {
243 LLVM_DEBUG(dbgs() << " ... replacing: "; Old->dump(&DAG);
244 dbgs() << " with: "; New->dump(&DAG));
245
246 DAG.ReplaceAllUsesOfValueWith(Old, New);
247 if (UpdatedNodes)
248 UpdatedNodes->insert(New.getNode());
249 ReplacedNode(Old.getNode());
250 }
251};
252
253} // end anonymous namespace
254
255// Helper function that generates an MMO that considers the alignment of the
256// stack, and the size of the stack object
258 MachineFunction &MF,
259 bool isObjectScalable) {
260 auto &MFI = MF.getFrameInfo();
261 int FI = cast<FrameIndexSDNode>(StackPtr)->getIndex();
263 LocationSize ObjectSize = isObjectScalable
265 : LocationSize::precise(MFI.getObjectSize(FI));
267 ObjectSize, MFI.getObjectAlign(FI));
268}
269
270/// Return a vector shuffle operation which
271/// performs the same shuffle in terms of order or result bytes, but on a type
272/// whose vector element type is narrower than the original shuffle type.
273/// e.g. <v4i32> <0, 1, 0, 1> -> v8i16 <0, 1, 2, 3, 0, 1, 2, 3>
274SDValue SelectionDAGLegalize::ShuffleWithNarrowerEltType(
275 EVT NVT, EVT VT, const SDLoc &dl, SDValue N1, SDValue N2,
276 ArrayRef<int> Mask) const {
277 unsigned NumMaskElts = VT.getVectorNumElements();
278 unsigned NumDestElts = NVT.getVectorNumElements();
279 unsigned NumEltsGrowth = NumDestElts / NumMaskElts;
280
281 assert(NumEltsGrowth && "Cannot promote to vector type with fewer elts!");
282
283 if (NumEltsGrowth == 1)
284 return DAG.getVectorShuffle(NVT, dl, N1, N2, Mask);
285
286 SmallVector<int, 8> NewMask;
287 for (unsigned i = 0; i != NumMaskElts; ++i) {
288 int Idx = Mask[i];
289 for (unsigned j = 0; j != NumEltsGrowth; ++j) {
290 if (Idx < 0)
291 NewMask.push_back(-1);
292 else
293 NewMask.push_back(Idx * NumEltsGrowth + j);
294 }
295 }
296 assert(NewMask.size() == NumDestElts && "Non-integer NumEltsGrowth?");
297 assert(TLI.isShuffleMaskLegal(NewMask, NVT) && "Shuffle not legal?");
298 return DAG.getVectorShuffle(NVT, dl, N1, N2, NewMask);
299}
300
301/// Expands the ConstantFP node to an integer constant or
302/// a load from the constant pool.
304SelectionDAGLegalize::ExpandConstantFP(ConstantFPSDNode *CFP, bool UseCP) {
305 bool Extend = false;
306 SDLoc dl(CFP);
307
308 // If a FP immediate is precise when represented as a float and if the
309 // target can do an extending load from float to double, we put it into
310 // the constant pool as a float, even if it's is statically typed as a
311 // double. This shrinks FP constants and canonicalizes them for targets where
312 // an FP extending load is the same cost as a normal load (such as on the x87
313 // fp stack or PPC FP unit).
314 EVT VT = CFP->getValueType(0);
315 ConstantFP *LLVMC = const_cast<ConstantFP*>(CFP->getConstantFPValue());
316 if (!UseCP) {
317 assert((VT == MVT::f64 || VT == MVT::f32) && "Invalid type expansion");
318 return DAG.getConstant(LLVMC->getValueAPF().bitcastToAPInt(), dl,
319 (VT == MVT::f64) ? MVT::i64 : MVT::i32);
320 }
321
322 APFloat APF = CFP->getValueAPF();
323 EVT OrigVT = VT;
324 EVT SVT = VT;
325
326 // We don't want to shrink SNaNs. Converting the SNaN back to its real type
327 // can cause it to be changed into a QNaN on some platforms (e.g. on SystemZ).
328 if (!APF.isSignaling()) {
329 while (SVT != MVT::f32 && SVT != MVT::f16 && SVT != MVT::bf16) {
330 SVT = (MVT::SimpleValueType)(SVT.getSimpleVT().SimpleTy - 1);
332 // Only do this if the target has a native EXTLOAD instruction from
333 // smaller type.
334 TLI.isLoadExtLegal(ISD::EXTLOAD, OrigVT, SVT) &&
335 TLI.ShouldShrinkFPConstant(OrigVT)) {
336 Type *SType = SVT.getTypeForEVT(*DAG.getContext());
337 LLVMC = cast<ConstantFP>(ConstantFoldCastOperand(
338 Instruction::FPTrunc, LLVMC, SType, DAG.getDataLayout()));
339 VT = SVT;
340 Extend = true;
341 }
342 }
343 }
344
345 SDValue CPIdx =
346 DAG.getConstantPool(LLVMC, TLI.getPointerTy(DAG.getDataLayout()));
347 Align Alignment = cast<ConstantPoolSDNode>(CPIdx)->getAlign();
348 if (Extend) {
349 SDValue Result = DAG.getExtLoad(
350 ISD::EXTLOAD, dl, OrigVT, DAG.getEntryNode(), CPIdx,
351 MachinePointerInfo::getConstantPool(DAG.getMachineFunction()), VT,
352 Alignment);
353 return Result;
354 }
355 SDValue Result = DAG.getLoad(
356 OrigVT, dl, DAG.getEntryNode(), CPIdx,
357 MachinePointerInfo::getConstantPool(DAG.getMachineFunction()), Alignment);
358 return Result;
359}
360
361/// Expands the Constant node to a load from the constant pool.
362SDValue SelectionDAGLegalize::ExpandConstant(ConstantSDNode *CP) {
363 SDLoc dl(CP);
364 EVT VT = CP->getValueType(0);
365 SDValue CPIdx = DAG.getConstantPool(CP->getConstantIntValue(),
366 TLI.getPointerTy(DAG.getDataLayout()));
367 Align Alignment = cast<ConstantPoolSDNode>(CPIdx)->getAlign();
368 SDValue Result = DAG.getLoad(
369 VT, dl, DAG.getEntryNode(), CPIdx,
370 MachinePointerInfo::getConstantPool(DAG.getMachineFunction()), Alignment);
371 return Result;
372}
373
374SDValue SelectionDAGLegalize::ExpandINSERT_VECTOR_ELT(SDValue Op) {
375 SDValue Vec = Op.getOperand(0);
376 SDValue Val = Op.getOperand(1);
377 SDValue Idx = Op.getOperand(2);
378 SDLoc dl(Op);
379
380 if (ConstantSDNode *InsertPos = dyn_cast<ConstantSDNode>(Idx)) {
381 // SCALAR_TO_VECTOR requires that the type of the value being inserted
382 // match the element type of the vector being created, except for
383 // integers in which case the inserted value can be over width.
384 EVT EltVT = Vec.getValueType().getVectorElementType();
385 if (Val.getValueType() == EltVT ||
386 (EltVT.isInteger() && Val.getValueType().bitsGE(EltVT))) {
387 SDValue ScVec = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl,
388 Vec.getValueType(), Val);
389
390 unsigned NumElts = Vec.getValueType().getVectorNumElements();
391 // We generate a shuffle of InVec and ScVec, so the shuffle mask
392 // should be 0,1,2,3,4,5... with the appropriate element replaced with
393 // elt 0 of the RHS.
394 SmallVector<int, 8> ShufOps;
395 for (unsigned i = 0; i != NumElts; ++i)
396 ShufOps.push_back(i != InsertPos->getZExtValue() ? i : NumElts);
397
398 return DAG.getVectorShuffle(Vec.getValueType(), dl, Vec, ScVec, ShufOps);
399 }
400 }
401 return ExpandInsertToVectorThroughStack(Op);
402}
403
404SDValue SelectionDAGLegalize::OptimizeFloatStore(StoreSDNode* ST) {
405 if (!ISD::isNormalStore(ST))
406 return SDValue();
407
408 LLVM_DEBUG(dbgs() << "Optimizing float store operations\n");
409 // Turn 'store float 1.0, Ptr' -> 'store int 0x12345678, Ptr'
410 // FIXME: move this to the DAG Combiner! Note that we can't regress due
411 // to phase ordering between legalized code and the dag combiner. This
412 // probably means that we need to integrate dag combiner and legalizer
413 // together.
414 // We generally can't do this one for long doubles.
415 SDValue Chain = ST->getChain();
416 SDValue Ptr = ST->getBasePtr();
417 SDValue Value = ST->getValue();
418 MachineMemOperand::Flags MMOFlags = ST->getMemOperand()->getFlags();
419 AAMDNodes AAInfo = ST->getAAInfo();
420 SDLoc dl(ST);
421
422 // Don't optimise TargetConstantFP
423 if (Value.getOpcode() == ISD::TargetConstantFP)
424 return SDValue();
425
426 if (ConstantFPSDNode *CFP = dyn_cast<ConstantFPSDNode>(Value)) {
427 if (CFP->getValueType(0) == MVT::f32 &&
428 TLI.isTypeLegal(MVT::i32)) {
429 SDValue Con = DAG.getConstant(CFP->getValueAPF().
430 bitcastToAPInt().zextOrTrunc(32),
431 SDLoc(CFP), MVT::i32);
432 return DAG.getStore(Chain, dl, Con, Ptr, ST->getPointerInfo(),
433 ST->getOriginalAlign(), MMOFlags, AAInfo);
434 }
435
436 if (CFP->getValueType(0) == MVT::f64 &&
437 !TLI.isFPImmLegal(CFP->getValueAPF(), MVT::f64)) {
438 // If this target supports 64-bit registers, do a single 64-bit store.
439 if (TLI.isTypeLegal(MVT::i64)) {
440 SDValue Con = DAG.getConstant(CFP->getValueAPF().bitcastToAPInt().
441 zextOrTrunc(64), SDLoc(CFP), MVT::i64);
442 return DAG.getStore(Chain, dl, Con, Ptr, ST->getPointerInfo(),
443 ST->getOriginalAlign(), MMOFlags, AAInfo);
444 }
445
446 if (TLI.isTypeLegal(MVT::i32) && !ST->isVolatile()) {
447 // Otherwise, if the target supports 32-bit registers, use 2 32-bit
448 // stores. If the target supports neither 32- nor 64-bits, this
449 // xform is certainly not worth it.
450 const APInt &IntVal = CFP->getValueAPF().bitcastToAPInt();
451 SDValue Lo = DAG.getConstant(IntVal.trunc(32), dl, MVT::i32);
452 SDValue Hi = DAG.getConstant(IntVal.lshr(32).trunc(32), dl, MVT::i32);
453 if (DAG.getDataLayout().isBigEndian())
454 std::swap(Lo, Hi);
455
456 Lo = DAG.getStore(Chain, dl, Lo, Ptr, ST->getPointerInfo(),
457 ST->getOriginalAlign(), MMOFlags, AAInfo);
458 Ptr = DAG.getMemBasePlusOffset(Ptr, TypeSize::getFixed(4), dl);
459 Hi = DAG.getStore(Chain, dl, Hi, Ptr,
460 ST->getPointerInfo().getWithOffset(4),
461 ST->getOriginalAlign(), MMOFlags, AAInfo);
462
463 return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo, Hi);
464 }
465 }
466 }
467 return SDValue();
468}
469
470void SelectionDAGLegalize::LegalizeStoreOps(SDNode *Node) {
471 StoreSDNode *ST = cast<StoreSDNode>(Node);
472 SDValue Chain = ST->getChain();
473 SDValue Ptr = ST->getBasePtr();
474 SDLoc dl(Node);
475
476 MachineMemOperand::Flags MMOFlags = ST->getMemOperand()->getFlags();
477 AAMDNodes AAInfo = ST->getAAInfo();
478
479 if (!ST->isTruncatingStore()) {
480 LLVM_DEBUG(dbgs() << "Legalizing store operation\n");
481 if (SDNode *OptStore = OptimizeFloatStore(ST).getNode()) {
482 ReplaceNode(ST, OptStore);
483 return;
484 }
485
486 SDValue Value = ST->getValue();
487 MVT VT = Value.getSimpleValueType();
488 switch (TLI.getOperationAction(ISD::STORE, VT)) {
489 default: llvm_unreachable("This action is not supported yet!");
490 case TargetLowering::Legal: {
491 // If this is an unaligned store and the target doesn't support it,
492 // expand it.
493 EVT MemVT = ST->getMemoryVT();
494 const DataLayout &DL = DAG.getDataLayout();
495 if (!TLI.allowsMemoryAccessForAlignment(*DAG.getContext(), DL, MemVT,
496 *ST->getMemOperand())) {
497 LLVM_DEBUG(dbgs() << "Expanding unsupported unaligned store\n");
498 SDValue Result = TLI.expandUnalignedStore(ST, DAG);
499 ReplaceNode(SDValue(ST, 0), Result);
500 } else
501 LLVM_DEBUG(dbgs() << "Legal store\n");
502 break;
503 }
504 case TargetLowering::Custom: {
505 LLVM_DEBUG(dbgs() << "Trying custom lowering\n");
506 SDValue Res = TLI.LowerOperation(SDValue(Node, 0), DAG);
507 if (Res && Res != SDValue(Node, 0))
508 ReplaceNode(SDValue(Node, 0), Res);
509 return;
510 }
511 case TargetLowering::Promote: {
512 MVT NVT = TLI.getTypeToPromoteTo(ISD::STORE, VT);
513 assert(NVT.getSizeInBits() == VT.getSizeInBits() &&
514 "Can only promote stores to same size type");
515 Value = DAG.getNode(ISD::BITCAST, dl, NVT, Value);
516 SDValue Result = DAG.getStore(Chain, dl, Value, Ptr, ST->getPointerInfo(),
517 ST->getOriginalAlign(), MMOFlags, AAInfo);
518 ReplaceNode(SDValue(Node, 0), Result);
519 break;
520 }
521 }
522 return;
523 }
524
525 LLVM_DEBUG(dbgs() << "Legalizing truncating store operations\n");
526 SDValue Value = ST->getValue();
527 EVT StVT = ST->getMemoryVT();
528 TypeSize StWidth = StVT.getSizeInBits();
529 TypeSize StSize = StVT.getStoreSizeInBits();
530 auto &DL = DAG.getDataLayout();
531
532 if (StWidth != StSize) {
533 // Promote to a byte-sized store with upper bits zero if not
534 // storing an integral number of bytes. For example, promote
535 // TRUNCSTORE:i1 X -> TRUNCSTORE:i8 (and X, 1)
536 EVT NVT = EVT::getIntegerVT(*DAG.getContext(), StSize.getFixedValue());
537 Value = DAG.getZeroExtendInReg(Value, dl, StVT);
539 DAG.getTruncStore(Chain, dl, Value, Ptr, ST->getPointerInfo(), NVT,
540 ST->getOriginalAlign(), MMOFlags, AAInfo);
541 ReplaceNode(SDValue(Node, 0), Result);
542 } else if (!StVT.isVector() && !isPowerOf2_64(StWidth.getFixedValue())) {
543 // If not storing a power-of-2 number of bits, expand as two stores.
544 assert(!StVT.isVector() && "Unsupported truncstore!");
545 unsigned StWidthBits = StWidth.getFixedValue();
546 unsigned LogStWidth = Log2_32(StWidthBits);
547 assert(LogStWidth < 32);
548 unsigned RoundWidth = 1 << LogStWidth;
549 assert(RoundWidth < StWidthBits);
550 unsigned ExtraWidth = StWidthBits - RoundWidth;
551 assert(ExtraWidth < RoundWidth);
552 assert(!(RoundWidth % 8) && !(ExtraWidth % 8) &&
553 "Store size not an integral number of bytes!");
554 EVT RoundVT = EVT::getIntegerVT(*DAG.getContext(), RoundWidth);
555 EVT ExtraVT = EVT::getIntegerVT(*DAG.getContext(), ExtraWidth);
556 SDValue Lo, Hi;
557 unsigned IncrementSize;
558
559 if (DL.isLittleEndian()) {
560 // TRUNCSTORE:i24 X -> TRUNCSTORE:i16 X, TRUNCSTORE@+2:i8 (srl X, 16)
561 // Store the bottom RoundWidth bits.
562 Lo = DAG.getTruncStore(Chain, dl, Value, Ptr, ST->getPointerInfo(),
563 RoundVT, ST->getOriginalAlign(), MMOFlags, AAInfo);
564
565 // Store the remaining ExtraWidth bits.
566 IncrementSize = RoundWidth / 8;
567 Ptr =
568 DAG.getMemBasePlusOffset(Ptr, TypeSize::getFixed(IncrementSize), dl);
569 Hi = DAG.getNode(
570 ISD::SRL, dl, Value.getValueType(), Value,
571 DAG.getConstant(RoundWidth, dl,
572 TLI.getShiftAmountTy(Value.getValueType(), DL)));
573 Hi = DAG.getTruncStore(Chain, dl, Hi, Ptr,
574 ST->getPointerInfo().getWithOffset(IncrementSize),
575 ExtraVT, ST->getOriginalAlign(), MMOFlags, AAInfo);
576 } else {
577 // Big endian - avoid unaligned stores.
578 // TRUNCSTORE:i24 X -> TRUNCSTORE:i16 (srl X, 8), TRUNCSTORE@+2:i8 X
579 // Store the top RoundWidth bits.
580 Hi = DAG.getNode(
581 ISD::SRL, dl, Value.getValueType(), Value,
582 DAG.getConstant(ExtraWidth, dl,
583 TLI.getShiftAmountTy(Value.getValueType(), DL)));
584 Hi = DAG.getTruncStore(Chain, dl, Hi, Ptr, ST->getPointerInfo(), RoundVT,
585 ST->getOriginalAlign(), MMOFlags, AAInfo);
586
587 // Store the remaining ExtraWidth bits.
588 IncrementSize = RoundWidth / 8;
589 Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr,
590 DAG.getConstant(IncrementSize, dl,
591 Ptr.getValueType()));
592 Lo = DAG.getTruncStore(Chain, dl, Value, Ptr,
593 ST->getPointerInfo().getWithOffset(IncrementSize),
594 ExtraVT, ST->getOriginalAlign(), MMOFlags, AAInfo);
595 }
596
597 // The order of the stores doesn't matter.
598 SDValue Result = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo, Hi);
599 ReplaceNode(SDValue(Node, 0), Result);
600 } else {
601 switch (TLI.getTruncStoreAction(ST->getValue().getValueType(), StVT)) {
602 default: llvm_unreachable("This action is not supported yet!");
603 case TargetLowering::Legal: {
604 EVT MemVT = ST->getMemoryVT();
605 // If this is an unaligned store and the target doesn't support it,
606 // expand it.
607 if (!TLI.allowsMemoryAccessForAlignment(*DAG.getContext(), DL, MemVT,
608 *ST->getMemOperand())) {
609 SDValue Result = TLI.expandUnalignedStore(ST, DAG);
610 ReplaceNode(SDValue(ST, 0), Result);
611 }
612 break;
613 }
614 case TargetLowering::Custom: {
615 SDValue Res = TLI.LowerOperation(SDValue(Node, 0), DAG);
616 if (Res && Res != SDValue(Node, 0))
617 ReplaceNode(SDValue(Node, 0), Res);
618 return;
619 }
620 case TargetLowering::Expand:
621 assert(!StVT.isVector() &&
622 "Vector Stores are handled in LegalizeVectorOps");
623
625
626 // TRUNCSTORE:i16 i32 -> STORE i16
627 if (TLI.isTypeLegal(StVT)) {
628 Value = DAG.getNode(ISD::TRUNCATE, dl, StVT, Value);
629 Result = DAG.getStore(Chain, dl, Value, Ptr, ST->getPointerInfo(),
630 ST->getOriginalAlign(), MMOFlags, AAInfo);
631 } else {
632 // The in-memory type isn't legal. Truncate to the type it would promote
633 // to, and then do a truncstore.
634 Value = DAG.getNode(ISD::TRUNCATE, dl,
635 TLI.getTypeToTransformTo(*DAG.getContext(), StVT),
636 Value);
637 Result =
638 DAG.getTruncStore(Chain, dl, Value, Ptr, ST->getPointerInfo(), StVT,
639 ST->getOriginalAlign(), MMOFlags, AAInfo);
640 }
641
642 ReplaceNode(SDValue(Node, 0), Result);
643 break;
644 }
645 }
646}
647
648void SelectionDAGLegalize::LegalizeLoadOps(SDNode *Node) {
649 LoadSDNode *LD = cast<LoadSDNode>(Node);
650 SDValue Chain = LD->getChain(); // The chain.
651 SDValue Ptr = LD->getBasePtr(); // The base pointer.
652 SDValue Value; // The value returned by the load op.
653 SDLoc dl(Node);
654
655 ISD::LoadExtType ExtType = LD->getExtensionType();
656 if (ExtType == ISD::NON_EXTLOAD) {
657 LLVM_DEBUG(dbgs() << "Legalizing non-extending load operation\n");
658 MVT VT = Node->getSimpleValueType(0);
659 SDValue RVal = SDValue(Node, 0);
660 SDValue RChain = SDValue(Node, 1);
661
662 switch (TLI.getOperationAction(Node->getOpcode(), VT)) {
663 default: llvm_unreachable("This action is not supported yet!");
664 case TargetLowering::Legal: {
665 EVT MemVT = LD->getMemoryVT();
666 const DataLayout &DL = DAG.getDataLayout();
667 // If this is an unaligned load and the target doesn't support it,
668 // expand it.
669 if (!TLI.allowsMemoryAccessForAlignment(*DAG.getContext(), DL, MemVT,
670 *LD->getMemOperand())) {
671 std::tie(RVal, RChain) = TLI.expandUnalignedLoad(LD, DAG);
672 }
673 break;
674 }
675 case TargetLowering::Custom:
676 if (SDValue Res = TLI.LowerOperation(RVal, DAG)) {
677 RVal = Res;
678 RChain = Res.getValue(1);
679 }
680 break;
681
682 case TargetLowering::Promote: {
683 MVT NVT = TLI.getTypeToPromoteTo(Node->getOpcode(), VT);
684 assert(NVT.getSizeInBits() == VT.getSizeInBits() &&
685 "Can only promote loads to same size type");
686
687 SDValue Res = DAG.getLoad(NVT, dl, Chain, Ptr, LD->getMemOperand());
688 RVal = DAG.getNode(ISD::BITCAST, dl, VT, Res);
689 RChain = Res.getValue(1);
690 break;
691 }
692 }
693 if (RChain.getNode() != Node) {
694 assert(RVal.getNode() != Node && "Load must be completely replaced");
695 DAG.ReplaceAllUsesOfValueWith(SDValue(Node, 0), RVal);
696 DAG.ReplaceAllUsesOfValueWith(SDValue(Node, 1), RChain);
697 if (UpdatedNodes) {
698 UpdatedNodes->insert(RVal.getNode());
699 UpdatedNodes->insert(RChain.getNode());
700 }
701 ReplacedNode(Node);
702 }
703 return;
704 }
705
706 LLVM_DEBUG(dbgs() << "Legalizing extending load operation\n");
707 EVT SrcVT = LD->getMemoryVT();
708 TypeSize SrcWidth = SrcVT.getSizeInBits();
709 MachineMemOperand::Flags MMOFlags = LD->getMemOperand()->getFlags();
710 AAMDNodes AAInfo = LD->getAAInfo();
711
712 if (SrcWidth != SrcVT.getStoreSizeInBits() &&
713 // Some targets pretend to have an i1 loading operation, and actually
714 // load an i8. This trick is correct for ZEXTLOAD because the top 7
715 // bits are guaranteed to be zero; it helps the optimizers understand
716 // that these bits are zero. It is also useful for EXTLOAD, since it
717 // tells the optimizers that those bits are undefined. It would be
718 // nice to have an effective generic way of getting these benefits...
719 // Until such a way is found, don't insist on promoting i1 here.
720 (SrcVT != MVT::i1 ||
721 TLI.getLoadExtAction(ExtType, Node->getValueType(0), MVT::i1) ==
722 TargetLowering::Promote)) {
723 // Promote to a byte-sized load if not loading an integral number of
724 // bytes. For example, promote EXTLOAD:i20 -> EXTLOAD:i24.
725 unsigned NewWidth = SrcVT.getStoreSizeInBits();
726 EVT NVT = EVT::getIntegerVT(*DAG.getContext(), NewWidth);
727 SDValue Ch;
728
729 // The extra bits are guaranteed to be zero, since we stored them that
730 // way. A zext load from NVT thus automatically gives zext from SrcVT.
731
732 ISD::LoadExtType NewExtType =
734
735 SDValue Result = DAG.getExtLoad(NewExtType, dl, Node->getValueType(0),
736 Chain, Ptr, LD->getPointerInfo(), NVT,
737 LD->getOriginalAlign(), MMOFlags, AAInfo);
738
739 Ch = Result.getValue(1); // The chain.
740
741 if (ExtType == ISD::SEXTLOAD)
742 // Having the top bits zero doesn't help when sign extending.
743 Result = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl,
744 Result.getValueType(),
745 Result, DAG.getValueType(SrcVT));
746 else if (ExtType == ISD::ZEXTLOAD || NVT == Result.getValueType())
747 // All the top bits are guaranteed to be zero - inform the optimizers.
748 Result = DAG.getNode(ISD::AssertZext, dl,
749 Result.getValueType(), Result,
750 DAG.getValueType(SrcVT));
751
752 Value = Result;
753 Chain = Ch;
754 } else if (!isPowerOf2_64(SrcWidth.getKnownMinValue())) {
755 // If not loading a power-of-2 number of bits, expand as two loads.
756 assert(!SrcVT.isVector() && "Unsupported extload!");
757 unsigned SrcWidthBits = SrcWidth.getFixedValue();
758 unsigned LogSrcWidth = Log2_32(SrcWidthBits);
759 assert(LogSrcWidth < 32);
760 unsigned RoundWidth = 1 << LogSrcWidth;
761 assert(RoundWidth < SrcWidthBits);
762 unsigned ExtraWidth = SrcWidthBits - RoundWidth;
763 assert(ExtraWidth < RoundWidth);
764 assert(!(RoundWidth % 8) && !(ExtraWidth % 8) &&
765 "Load size not an integral number of bytes!");
766 EVT RoundVT = EVT::getIntegerVT(*DAG.getContext(), RoundWidth);
767 EVT ExtraVT = EVT::getIntegerVT(*DAG.getContext(), ExtraWidth);
768 SDValue Lo, Hi, Ch;
769 unsigned IncrementSize;
770 auto &DL = DAG.getDataLayout();
771
772 if (DL.isLittleEndian()) {
773 // EXTLOAD:i24 -> ZEXTLOAD:i16 | (shl EXTLOAD@+2:i8, 16)
774 // Load the bottom RoundWidth bits.
775 Lo = DAG.getExtLoad(ISD::ZEXTLOAD, dl, Node->getValueType(0), Chain, Ptr,
776 LD->getPointerInfo(), RoundVT, LD->getOriginalAlign(),
777 MMOFlags, AAInfo);
778
779 // Load the remaining ExtraWidth bits.
780 IncrementSize = RoundWidth / 8;
781 Ptr =
782 DAG.getMemBasePlusOffset(Ptr, TypeSize::getFixed(IncrementSize), dl);
783 Hi = DAG.getExtLoad(ExtType, dl, Node->getValueType(0), Chain, Ptr,
784 LD->getPointerInfo().getWithOffset(IncrementSize),
785 ExtraVT, LD->getOriginalAlign(), MMOFlags, AAInfo);
786
787 // Build a factor node to remember that this load is independent of
788 // the other one.
789 Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1),
790 Hi.getValue(1));
791
792 // Move the top bits to the right place.
793 Hi = DAG.getNode(
794 ISD::SHL, dl, Hi.getValueType(), Hi,
795 DAG.getConstant(RoundWidth, dl,
796 TLI.getShiftAmountTy(Hi.getValueType(), DL)));
797
798 // Join the hi and lo parts.
799 Value = DAG.getNode(ISD::OR, dl, Node->getValueType(0), Lo, Hi);
800 } else {
801 // Big endian - avoid unaligned loads.
802 // EXTLOAD:i24 -> (shl EXTLOAD:i16, 8) | ZEXTLOAD@+2:i8
803 // Load the top RoundWidth bits.
804 Hi = DAG.getExtLoad(ExtType, dl, Node->getValueType(0), Chain, Ptr,
805 LD->getPointerInfo(), RoundVT, LD->getOriginalAlign(),
806 MMOFlags, AAInfo);
807
808 // Load the remaining ExtraWidth bits.
809 IncrementSize = RoundWidth / 8;
810 Ptr =
811 DAG.getMemBasePlusOffset(Ptr, TypeSize::getFixed(IncrementSize), dl);
812 Lo = DAG.getExtLoad(ISD::ZEXTLOAD, dl, Node->getValueType(0), Chain, Ptr,
813 LD->getPointerInfo().getWithOffset(IncrementSize),
814 ExtraVT, LD->getOriginalAlign(), MMOFlags, AAInfo);
815
816 // Build a factor node to remember that this load is independent of
817 // the other one.
818 Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1),
819 Hi.getValue(1));
820
821 // Move the top bits to the right place.
822 Hi = DAG.getNode(
823 ISD::SHL, dl, Hi.getValueType(), Hi,
824 DAG.getConstant(ExtraWidth, dl,
825 TLI.getShiftAmountTy(Hi.getValueType(), DL)));
826
827 // Join the hi and lo parts.
828 Value = DAG.getNode(ISD::OR, dl, Node->getValueType(0), Lo, Hi);
829 }
830
831 Chain = Ch;
832 } else {
833 bool isCustom = false;
834 switch (TLI.getLoadExtAction(ExtType, Node->getValueType(0),
835 SrcVT.getSimpleVT())) {
836 default: llvm_unreachable("This action is not supported yet!");
837 case TargetLowering::Custom:
838 isCustom = true;
839 [[fallthrough]];
840 case TargetLowering::Legal:
841 Value = SDValue(Node, 0);
842 Chain = SDValue(Node, 1);
843
844 if (isCustom) {
845 if (SDValue Res = TLI.LowerOperation(SDValue(Node, 0), DAG)) {
846 Value = Res;
847 Chain = Res.getValue(1);
848 }
849 } else {
850 // If this is an unaligned load and the target doesn't support it,
851 // expand it.
852 EVT MemVT = LD->getMemoryVT();
853 const DataLayout &DL = DAG.getDataLayout();
854 if (!TLI.allowsMemoryAccess(*DAG.getContext(), DL, MemVT,
855 *LD->getMemOperand())) {
856 std::tie(Value, Chain) = TLI.expandUnalignedLoad(LD, DAG);
857 }
858 }
859 break;
860
861 case TargetLowering::Expand: {
862 EVT DestVT = Node->getValueType(0);
863 if (!TLI.isLoadExtLegal(ISD::EXTLOAD, DestVT, SrcVT)) {
864 // If the source type is not legal, see if there is a legal extload to
865 // an intermediate type that we can then extend further.
866 EVT LoadVT = TLI.getRegisterType(SrcVT.getSimpleVT());
867 if ((LoadVT.isFloatingPoint() == SrcVT.isFloatingPoint()) &&
868 (TLI.isTypeLegal(SrcVT) || // Same as SrcVT == LoadVT?
869 TLI.isLoadExtLegal(ExtType, LoadVT, SrcVT))) {
870 // If we are loading a legal type, this is a non-extload followed by a
871 // full extend.
872 ISD::LoadExtType MidExtType =
873 (LoadVT == SrcVT) ? ISD::NON_EXTLOAD : ExtType;
874
875 SDValue Load = DAG.getExtLoad(MidExtType, dl, LoadVT, Chain, Ptr,
876 SrcVT, LD->getMemOperand());
877 unsigned ExtendOp =
879 Value = DAG.getNode(ExtendOp, dl, Node->getValueType(0), Load);
880 Chain = Load.getValue(1);
881 break;
882 }
883
884 // Handle the special case of fp16 extloads. EXTLOAD doesn't have the
885 // normal undefined upper bits behavior to allow using an in-reg extend
886 // with the illegal FP type, so load as an integer and do the
887 // from-integer conversion.
888 EVT SVT = SrcVT.getScalarType();
889 if (SVT == MVT::f16 || SVT == MVT::bf16) {
890 EVT ISrcVT = SrcVT.changeTypeToInteger();
891 EVT IDestVT = DestVT.changeTypeToInteger();
892 EVT ILoadVT = TLI.getRegisterType(IDestVT.getSimpleVT());
893
894 SDValue Result = DAG.getExtLoad(ISD::ZEXTLOAD, dl, ILoadVT, Chain,
895 Ptr, ISrcVT, LD->getMemOperand());
896 Value =
897 DAG.getNode(SVT == MVT::f16 ? ISD::FP16_TO_FP : ISD::BF16_TO_FP,
898 dl, DestVT, Result);
899 Chain = Result.getValue(1);
900 break;
901 }
902 }
903
904 assert(!SrcVT.isVector() &&
905 "Vector Loads are handled in LegalizeVectorOps");
906
907 // FIXME: This does not work for vectors on most targets. Sign-
908 // and zero-extend operations are currently folded into extending
909 // loads, whether they are legal or not, and then we end up here
910 // without any support for legalizing them.
911 assert(ExtType != ISD::EXTLOAD &&
912 "EXTLOAD should always be supported!");
913 // Turn the unsupported load into an EXTLOAD followed by an
914 // explicit zero/sign extend inreg.
915 SDValue Result = DAG.getExtLoad(ISD::EXTLOAD, dl,
916 Node->getValueType(0),
917 Chain, Ptr, SrcVT,
918 LD->getMemOperand());
919 SDValue ValRes;
920 if (ExtType == ISD::SEXTLOAD)
921 ValRes = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl,
922 Result.getValueType(),
923 Result, DAG.getValueType(SrcVT));
924 else
925 ValRes = DAG.getZeroExtendInReg(Result, dl, SrcVT);
926 Value = ValRes;
927 Chain = Result.getValue(1);
928 break;
929 }
930 }
931 }
932
933 // Since loads produce two values, make sure to remember that we legalized
934 // both of them.
935 if (Chain.getNode() != Node) {
936 assert(Value.getNode() != Node && "Load must be completely replaced");
937 DAG.ReplaceAllUsesOfValueWith(SDValue(Node, 0), Value);
938 DAG.ReplaceAllUsesOfValueWith(SDValue(Node, 1), Chain);
939 if (UpdatedNodes) {
940 UpdatedNodes->insert(Value.getNode());
941 UpdatedNodes->insert(Chain.getNode());
942 }
943 ReplacedNode(Node);
944 }
945}
946
947/// Return a legal replacement for the given operation, with all legal operands.
948void SelectionDAGLegalize::LegalizeOp(SDNode *Node) {
949 LLVM_DEBUG(dbgs() << "\nLegalizing: "; Node->dump(&DAG));
950
951 // Allow illegal target nodes and illegal registers.
952 if (Node->getOpcode() == ISD::TargetConstant ||
953 Node->getOpcode() == ISD::Register)
954 return;
955
956#ifndef NDEBUG
957 for (unsigned i = 0, e = Node->getNumValues(); i != e; ++i)
958 assert(TLI.getTypeAction(*DAG.getContext(), Node->getValueType(i)) ==
959 TargetLowering::TypeLegal &&
960 "Unexpected illegal type!");
961
962 for (const SDValue &Op : Node->op_values())
963 assert((TLI.getTypeAction(*DAG.getContext(), Op.getValueType()) ==
964 TargetLowering::TypeLegal ||
965 Op.getOpcode() == ISD::TargetConstant ||
966 Op.getOpcode() == ISD::Register) &&
967 "Unexpected illegal type!");
968#endif
969
970 // Figure out the correct action; the way to query this varies by opcode
971 TargetLowering::LegalizeAction Action = TargetLowering::Legal;
972 bool SimpleFinishLegalizing = true;
973 switch (Node->getOpcode()) {
977 case ISD::STACKSAVE:
978 Action = TLI.getOperationAction(Node->getOpcode(), MVT::Other);
979 break;
981 Action = TLI.getOperationAction(Node->getOpcode(),
982 Node->getValueType(0));
983 break;
984 case ISD::VAARG:
985 Action = TLI.getOperationAction(Node->getOpcode(),
986 Node->getValueType(0));
987 if (Action != TargetLowering::Promote)
988 Action = TLI.getOperationAction(Node->getOpcode(), MVT::Other);
989 break;
990 case ISD::SET_FPENV:
991 case ISD::SET_FPMODE:
992 Action = TLI.getOperationAction(Node->getOpcode(),
993 Node->getOperand(1).getValueType());
994 break;
995 case ISD::FP_TO_FP16:
996 case ISD::FP_TO_BF16:
997 case ISD::SINT_TO_FP:
998 case ISD::UINT_TO_FP:
1000 case ISD::LROUND:
1001 case ISD::LLROUND:
1002 case ISD::LRINT:
1003 case ISD::LLRINT:
1004 Action = TLI.getOperationAction(Node->getOpcode(),
1005 Node->getOperand(0).getValueType());
1006 break;
1011 case ISD::STRICT_LRINT:
1012 case ISD::STRICT_LLRINT:
1013 case ISD::STRICT_LROUND:
1015 // These pseudo-ops are the same as the other STRICT_ ops except
1016 // they are registered with setOperationAction() using the input type
1017 // instead of the output type.
1018 Action = TLI.getOperationAction(Node->getOpcode(),
1019 Node->getOperand(1).getValueType());
1020 break;
1022 EVT InnerType = cast<VTSDNode>(Node->getOperand(1))->getVT();
1023 Action = TLI.getOperationAction(Node->getOpcode(), InnerType);
1024 break;
1025 }
1026 case ISD::ATOMIC_STORE:
1027 Action = TLI.getOperationAction(Node->getOpcode(),
1028 Node->getOperand(1).getValueType());
1029 break;
1030 case ISD::SELECT_CC:
1031 case ISD::STRICT_FSETCC:
1033 case ISD::SETCC:
1034 case ISD::SETCCCARRY:
1035 case ISD::VP_SETCC:
1036 case ISD::BR_CC: {
1037 unsigned Opc = Node->getOpcode();
1038 unsigned CCOperand = Opc == ISD::SELECT_CC ? 4
1039 : Opc == ISD::STRICT_FSETCC ? 3
1040 : Opc == ISD::STRICT_FSETCCS ? 3
1041 : Opc == ISD::SETCCCARRY ? 3
1042 : (Opc == ISD::SETCC || Opc == ISD::VP_SETCC) ? 2
1043 : 1;
1044 unsigned CompareOperand = Opc == ISD::BR_CC ? 2
1045 : Opc == ISD::STRICT_FSETCC ? 1
1046 : Opc == ISD::STRICT_FSETCCS ? 1
1047 : 0;
1048 MVT OpVT = Node->getOperand(CompareOperand).getSimpleValueType();
1049 ISD::CondCode CCCode =
1050 cast<CondCodeSDNode>(Node->getOperand(CCOperand))->get();
1051 Action = TLI.getCondCodeAction(CCCode, OpVT);
1052 if (Action == TargetLowering::Legal) {
1053 if (Node->getOpcode() == ISD::SELECT_CC)
1054 Action = TLI.getOperationAction(Node->getOpcode(),
1055 Node->getValueType(0));
1056 else
1057 Action = TLI.getOperationAction(Node->getOpcode(), OpVT);
1058 }
1059 break;
1060 }
1061 case ISD::LOAD:
1062 case ISD::STORE:
1063 // FIXME: Model these properly. LOAD and STORE are complicated, and
1064 // STORE expects the unlegalized operand in some cases.
1065 SimpleFinishLegalizing = false;
1066 break;
1067 case ISD::CALLSEQ_START:
1068 case ISD::CALLSEQ_END:
1069 // FIXME: This shouldn't be necessary. These nodes have special properties
1070 // dealing with the recursive nature of legalization. Removing this
1071 // special case should be done as part of making LegalizeDAG non-recursive.
1072 SimpleFinishLegalizing = false;
1073 break;
1075 case ISD::GET_ROUNDING:
1076 case ISD::MERGE_VALUES:
1077 case ISD::EH_RETURN:
1079 case ISD::EH_DWARF_CFA:
1083 // These operations lie about being legal: when they claim to be legal,
1084 // they should actually be expanded.
1085 Action = TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0));
1086 if (Action == TargetLowering::Legal)
1087 Action = TargetLowering::Expand;
1088 break;
1091 case ISD::FRAMEADDR:
1092 case ISD::RETURNADDR:
1094 case ISD::SPONENTRY:
1095 // These operations lie about being legal: when they claim to be legal,
1096 // they should actually be custom-lowered.
1097 Action = TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0));
1098 if (Action == TargetLowering::Legal)
1099 Action = TargetLowering::Custom;
1100 break;
1103 // READCYCLECOUNTER and READSTEADYCOUNTER return a i64, even if type
1104 // legalization might have expanded that to several smaller types.
1105 Action = TLI.getOperationAction(Node->getOpcode(), MVT::i64);
1106 break;
1107 case ISD::READ_REGISTER:
1109 // Named register is legal in the DAG, but blocked by register name
1110 // selection if not implemented by target (to chose the correct register)
1111 // They'll be converted to Copy(To/From)Reg.
1112 Action = TargetLowering::Legal;
1113 break;
1114 case ISD::UBSANTRAP:
1115 Action = TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0));
1116 if (Action == TargetLowering::Expand) {
1117 // replace ISD::UBSANTRAP with ISD::TRAP
1118 SDValue NewVal;
1119 NewVal = DAG.getNode(ISD::TRAP, SDLoc(Node), Node->getVTList(),
1120 Node->getOperand(0));
1121 ReplaceNode(Node, NewVal.getNode());
1122 LegalizeOp(NewVal.getNode());
1123 return;
1124 }
1125 break;
1126 case ISD::DEBUGTRAP:
1127 Action = TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0));
1128 if (Action == TargetLowering::Expand) {
1129 // replace ISD::DEBUGTRAP with ISD::TRAP
1130 SDValue NewVal;
1131 NewVal = DAG.getNode(ISD::TRAP, SDLoc(Node), Node->getVTList(),
1132 Node->getOperand(0));
1133 ReplaceNode(Node, NewVal.getNode());
1134 LegalizeOp(NewVal.getNode());
1135 return;
1136 }
1137 break;
1138 case ISD::SADDSAT:
1139 case ISD::UADDSAT:
1140 case ISD::SSUBSAT:
1141 case ISD::USUBSAT:
1142 case ISD::SSHLSAT:
1143 case ISD::USHLSAT:
1146 Action = TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0));
1147 break;
1148 case ISD::SMULFIX:
1149 case ISD::SMULFIXSAT:
1150 case ISD::UMULFIX:
1151 case ISD::UMULFIXSAT:
1152 case ISD::SDIVFIX:
1153 case ISD::SDIVFIXSAT:
1154 case ISD::UDIVFIX:
1155 case ISD::UDIVFIXSAT: {
1156 unsigned Scale = Node->getConstantOperandVal(2);
1157 Action = TLI.getFixedPointOperationAction(Node->getOpcode(),
1158 Node->getValueType(0), Scale);
1159 break;
1160 }
1161 case ISD::MSCATTER:
1162 Action = TLI.getOperationAction(Node->getOpcode(),
1163 cast<MaskedScatterSDNode>(Node)->getValue().getValueType());
1164 break;
1165 case ISD::MSTORE:
1166 Action = TLI.getOperationAction(Node->getOpcode(),
1167 cast<MaskedStoreSDNode>(Node)->getValue().getValueType());
1168 break;
1169 case ISD::VP_SCATTER:
1170 Action = TLI.getOperationAction(
1171 Node->getOpcode(),
1172 cast<VPScatterSDNode>(Node)->getValue().getValueType());
1173 break;
1174 case ISD::VP_STORE:
1175 Action = TLI.getOperationAction(
1176 Node->getOpcode(),
1177 cast<VPStoreSDNode>(Node)->getValue().getValueType());
1178 break;
1179 case ISD::EXPERIMENTAL_VP_STRIDED_STORE:
1180 Action = TLI.getOperationAction(
1181 Node->getOpcode(),
1182 cast<VPStridedStoreSDNode>(Node)->getValue().getValueType());
1183 break;
1186 case ISD::VECREDUCE_ADD:
1187 case ISD::VECREDUCE_MUL:
1188 case ISD::VECREDUCE_AND:
1189 case ISD::VECREDUCE_OR:
1190 case ISD::VECREDUCE_XOR:
1199 case ISD::IS_FPCLASS:
1200 Action = TLI.getOperationAction(
1201 Node->getOpcode(), Node->getOperand(0).getValueType());
1202 break;
1205 case ISD::VP_REDUCE_FADD:
1206 case ISD::VP_REDUCE_FMUL:
1207 case ISD::VP_REDUCE_ADD:
1208 case ISD::VP_REDUCE_MUL:
1209 case ISD::VP_REDUCE_AND:
1210 case ISD::VP_REDUCE_OR:
1211 case ISD::VP_REDUCE_XOR:
1212 case ISD::VP_REDUCE_SMAX:
1213 case ISD::VP_REDUCE_SMIN:
1214 case ISD::VP_REDUCE_UMAX:
1215 case ISD::VP_REDUCE_UMIN:
1216 case ISD::VP_REDUCE_FMAX:
1217 case ISD::VP_REDUCE_FMIN:
1218 case ISD::VP_REDUCE_SEQ_FADD:
1219 case ISD::VP_REDUCE_SEQ_FMUL:
1220 Action = TLI.getOperationAction(
1221 Node->getOpcode(), Node->getOperand(1).getValueType());
1222 break;
1223 default:
1224 if (Node->getOpcode() >= ISD::BUILTIN_OP_END) {
1225 Action = TLI.getCustomOperationAction(*Node);
1226 } else {
1227 Action = TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0));
1228 }
1229 break;
1230 }
1231
1232 if (SimpleFinishLegalizing) {
1233 SDNode *NewNode = Node;
1234 switch (Node->getOpcode()) {
1235 default: break;
1236 case ISD::SHL:
1237 case ISD::SRL:
1238 case ISD::SRA:
1239 case ISD::ROTL:
1240 case ISD::ROTR: {
1241 // Legalizing shifts/rotates requires adjusting the shift amount
1242 // to the appropriate width.
1243 SDValue Op0 = Node->getOperand(0);
1244 SDValue Op1 = Node->getOperand(1);
1245 if (!Op1.getValueType().isVector()) {
1246 SDValue SAO = DAG.getShiftAmountOperand(Op0.getValueType(), Op1);
1247 // The getShiftAmountOperand() may create a new operand node or
1248 // return the existing one. If new operand is created we need
1249 // to update the parent node.
1250 // Do not try to legalize SAO here! It will be automatically legalized
1251 // in the next round.
1252 if (SAO != Op1)
1253 NewNode = DAG.UpdateNodeOperands(Node, Op0, SAO);
1254 }
1255 }
1256 break;
1257 case ISD::FSHL:
1258 case ISD::FSHR:
1259 case ISD::SRL_PARTS:
1260 case ISD::SRA_PARTS:
1261 case ISD::SHL_PARTS: {
1262 // Legalizing shifts/rotates requires adjusting the shift amount
1263 // to the appropriate width.
1264 SDValue Op0 = Node->getOperand(0);
1265 SDValue Op1 = Node->getOperand(1);
1266 SDValue Op2 = Node->getOperand(2);
1267 if (!Op2.getValueType().isVector()) {
1268 SDValue SAO = DAG.getShiftAmountOperand(Op0.getValueType(), Op2);
1269 // The getShiftAmountOperand() may create a new operand node or
1270 // return the existing one. If new operand is created we need
1271 // to update the parent node.
1272 if (SAO != Op2)
1273 NewNode = DAG.UpdateNodeOperands(Node, Op0, Op1, SAO);
1274 }
1275 break;
1276 }
1277 }
1278
1279 if (NewNode != Node) {
1280 ReplaceNode(Node, NewNode);
1281 Node = NewNode;
1282 }
1283 switch (Action) {
1284 case TargetLowering::Legal:
1285 LLVM_DEBUG(dbgs() << "Legal node: nothing to do\n");
1286 return;
1287 case TargetLowering::Custom:
1288 LLVM_DEBUG(dbgs() << "Trying custom legalization\n");
1289 // FIXME: The handling for custom lowering with multiple results is
1290 // a complete mess.
1291 if (SDValue Res = TLI.LowerOperation(SDValue(Node, 0), DAG)) {
1292 if (!(Res.getNode() != Node || Res.getResNo() != 0))
1293 return;
1294
1295 if (Node->getNumValues() == 1) {
1296 // Verify the new types match the original. Glue is waived because
1297 // ISD::ADDC can be legalized by replacing Glue with an integer type.
1298 assert((Res.getValueType() == Node->getValueType(0) ||
1299 Node->getValueType(0) == MVT::Glue) &&
1300 "Type mismatch for custom legalized operation");
1301 LLVM_DEBUG(dbgs() << "Successfully custom legalized node\n");
1302 // We can just directly replace this node with the lowered value.
1303 ReplaceNode(SDValue(Node, 0), Res);
1304 return;
1305 }
1306
1307 SmallVector<SDValue, 8> ResultVals;
1308 for (unsigned i = 0, e = Node->getNumValues(); i != e; ++i) {
1309 // Verify the new types match the original. Glue is waived because
1310 // ISD::ADDC can be legalized by replacing Glue with an integer type.
1311 assert((Res->getValueType(i) == Node->getValueType(i) ||
1312 Node->getValueType(i) == MVT::Glue) &&
1313 "Type mismatch for custom legalized operation");
1314 ResultVals.push_back(Res.getValue(i));
1315 }
1316 LLVM_DEBUG(dbgs() << "Successfully custom legalized node\n");
1317 ReplaceNode(Node, ResultVals.data());
1318 return;
1319 }
1320 LLVM_DEBUG(dbgs() << "Could not custom legalize node\n");
1321 [[fallthrough]];
1322 case TargetLowering::Expand:
1323 if (ExpandNode(Node))
1324 return;
1325 [[fallthrough]];
1326 case TargetLowering::LibCall:
1327 ConvertNodeToLibcall(Node);
1328 return;
1329 case TargetLowering::Promote:
1330 PromoteNode(Node);
1331 return;
1332 }
1333 }
1334
1335 switch (Node->getOpcode()) {
1336 default:
1337#ifndef NDEBUG
1338 dbgs() << "NODE: ";
1339 Node->dump( &DAG);
1340 dbgs() << "\n";
1341#endif
1342 llvm_unreachable("Do not know how to legalize this operator!");
1343
1344 case ISD::CALLSEQ_START:
1345 case ISD::CALLSEQ_END:
1346 break;
1347 case ISD::LOAD:
1348 return LegalizeLoadOps(Node);
1349 case ISD::STORE:
1350 return LegalizeStoreOps(Node);
1351 }
1352}
1353
1354SDValue SelectionDAGLegalize::ExpandExtractFromVectorThroughStack(SDValue Op) {
1355 SDValue Vec = Op.getOperand(0);
1356 SDValue Idx = Op.getOperand(1);
1357 SDLoc dl(Op);
1358
1359 // Before we generate a new store to a temporary stack slot, see if there is
1360 // already one that we can use. There often is because when we scalarize
1361 // vector operations (using SelectionDAG::UnrollVectorOp for example) a whole
1362 // series of EXTRACT_VECTOR_ELT nodes are generated, one for each element in
1363 // the vector. If all are expanded here, we don't want one store per vector
1364 // element.
1365
1366 // Caches for hasPredecessorHelper
1369 Visited.insert(Op.getNode());
1370 Worklist.push_back(Idx.getNode());
1371 SDValue StackPtr, Ch;
1372 for (SDNode *User : Vec.getNode()->uses()) {
1373 if (StoreSDNode *ST = dyn_cast<StoreSDNode>(User)) {
1374 if (ST->isIndexed() || ST->isTruncatingStore() ||
1375 ST->getValue() != Vec)
1376 continue;
1377
1378 // Make sure that nothing else could have stored into the destination of
1379 // this store.
1380 if (!ST->getChain().reachesChainWithoutSideEffects(DAG.getEntryNode()))
1381 continue;
1382
1383 // If the index is dependent on the store we will introduce a cycle when
1384 // creating the load (the load uses the index, and by replacing the chain
1385 // we will make the index dependent on the load). Also, the store might be
1386 // dependent on the extractelement and introduce a cycle when creating
1387 // the load.
1388 if (SDNode::hasPredecessorHelper(ST, Visited, Worklist) ||
1389 ST->hasPredecessor(Op.getNode()))
1390 continue;
1391
1392 StackPtr = ST->getBasePtr();
1393 Ch = SDValue(ST, 0);
1394 break;
1395 }
1396 }
1397
1398 EVT VecVT = Vec.getValueType();
1399
1400 if (!Ch.getNode()) {
1401 // Store the value to a temporary stack slot, then LOAD the returned part.
1402 StackPtr = DAG.CreateStackTemporary(VecVT);
1404 StackPtr, DAG.getMachineFunction(), VecVT.isScalableVector());
1405 Ch = DAG.getStore(DAG.getEntryNode(), dl, Vec, StackPtr, StoreMMO);
1406 }
1407
1408 SDValue NewLoad;
1409 Align ElementAlignment =
1410 std::min(cast<StoreSDNode>(Ch)->getAlign(),
1411 DAG.getDataLayout().getPrefTypeAlign(
1412 Op.getValueType().getTypeForEVT(*DAG.getContext())));
1413
1414 if (Op.getValueType().isVector()) {
1415 StackPtr = TLI.getVectorSubVecPointer(DAG, StackPtr, VecVT,
1416 Op.getValueType(), Idx);
1417 NewLoad = DAG.getLoad(Op.getValueType(), dl, Ch, StackPtr,
1418 MachinePointerInfo(), ElementAlignment);
1419 } else {
1420 StackPtr = TLI.getVectorElementPointer(DAG, StackPtr, VecVT, Idx);
1421 NewLoad = DAG.getExtLoad(ISD::EXTLOAD, dl, Op.getValueType(), Ch, StackPtr,
1423 ElementAlignment);
1424 }
1425
1426 // Replace the chain going out of the store, by the one out of the load.
1427 DAG.ReplaceAllUsesOfValueWith(Ch, SDValue(NewLoad.getNode(), 1));
1428
1429 // We introduced a cycle though, so update the loads operands, making sure
1430 // to use the original store's chain as an incoming chain.
1431 SmallVector<SDValue, 6> NewLoadOperands(NewLoad->op_begin(),
1432 NewLoad->op_end());
1433 NewLoadOperands[0] = Ch;
1434 NewLoad =
1435 SDValue(DAG.UpdateNodeOperands(NewLoad.getNode(), NewLoadOperands), 0);
1436 return NewLoad;
1437}
1438
1439SDValue SelectionDAGLegalize::ExpandInsertToVectorThroughStack(SDValue Op) {
1440 assert(Op.getValueType().isVector() && "Non-vector insert subvector!");
1441
1442 SDValue Vec = Op.getOperand(0);
1443 SDValue Part = Op.getOperand(1);
1444 SDValue Idx = Op.getOperand(2);
1445 SDLoc dl(Op);
1446
1447 // Store the value to a temporary stack slot, then LOAD the returned part.
1448 EVT VecVT = Vec.getValueType();
1449 EVT PartVT = Part.getValueType();
1450 SDValue StackPtr = DAG.CreateStackTemporary(VecVT);
1451 int FI = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex();
1452 MachinePointerInfo PtrInfo =
1453 MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), FI);
1454
1455 // First store the whole vector.
1456 SDValue Ch = DAG.getStore(DAG.getEntryNode(), dl, Vec, StackPtr, PtrInfo);
1457
1458 // Freeze the index so we don't poison the clamping code we're about to emit.
1459 Idx = DAG.getFreeze(Idx);
1460
1461 // Then store the inserted part.
1462 if (PartVT.isVector()) {
1463 SDValue SubStackPtr =
1464 TLI.getVectorSubVecPointer(DAG, StackPtr, VecVT, PartVT, Idx);
1465
1466 // Store the subvector.
1467 Ch = DAG.getStore(
1468 Ch, dl, Part, SubStackPtr,
1469 MachinePointerInfo::getUnknownStack(DAG.getMachineFunction()));
1470 } else {
1471 SDValue SubStackPtr =
1472 TLI.getVectorElementPointer(DAG, StackPtr, VecVT, Idx);
1473
1474 // Store the scalar value.
1475 Ch = DAG.getTruncStore(
1476 Ch, dl, Part, SubStackPtr,
1477 MachinePointerInfo::getUnknownStack(DAG.getMachineFunction()),
1478 VecVT.getVectorElementType());
1479 }
1480
1481 // Finally, load the updated vector.
1482 return DAG.getLoad(Op.getValueType(), dl, Ch, StackPtr, PtrInfo);
1483}
1484
1485SDValue SelectionDAGLegalize::ExpandVectorBuildThroughStack(SDNode* Node) {
1486 assert((Node->getOpcode() == ISD::BUILD_VECTOR ||
1487 Node->getOpcode() == ISD::CONCAT_VECTORS) &&
1488 "Unexpected opcode!");
1489
1490 // We can't handle this case efficiently. Allocate a sufficiently
1491 // aligned object on the stack, store each operand into it, then load
1492 // the result as a vector.
1493 // Create the stack frame object.
1494 EVT VT = Node->getValueType(0);
1495 EVT MemVT = isa<BuildVectorSDNode>(Node) ? VT.getVectorElementType()
1496 : Node->getOperand(0).getValueType();
1497 SDLoc dl(Node);
1498 SDValue FIPtr = DAG.CreateStackTemporary(VT);
1499 int FI = cast<FrameIndexSDNode>(FIPtr.getNode())->getIndex();
1500 MachinePointerInfo PtrInfo =
1501 MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), FI);
1502
1503 // Emit a store of each element to the stack slot.
1505 unsigned TypeByteSize = MemVT.getSizeInBits() / 8;
1506 assert(TypeByteSize > 0 && "Vector element type too small for stack store!");
1507
1508 // If the destination vector element type of a BUILD_VECTOR is narrower than
1509 // the source element type, only store the bits necessary.
1510 bool Truncate = isa<BuildVectorSDNode>(Node) &&
1511 MemVT.bitsLT(Node->getOperand(0).getValueType());
1512
1513 // Store (in the right endianness) the elements to memory.
1514 for (unsigned i = 0, e = Node->getNumOperands(); i != e; ++i) {
1515 // Ignore undef elements.
1516 if (Node->getOperand(i).isUndef()) continue;
1517
1518 unsigned Offset = TypeByteSize*i;
1519
1520 SDValue Idx =
1521 DAG.getMemBasePlusOffset(FIPtr, TypeSize::getFixed(Offset), dl);
1522
1523 if (Truncate)
1524 Stores.push_back(DAG.getTruncStore(DAG.getEntryNode(), dl,
1525 Node->getOperand(i), Idx,
1526 PtrInfo.getWithOffset(Offset), MemVT));
1527 else
1528 Stores.push_back(DAG.getStore(DAG.getEntryNode(), dl, Node->getOperand(i),
1529 Idx, PtrInfo.getWithOffset(Offset)));
1530 }
1531
1532 SDValue StoreChain;
1533 if (!Stores.empty()) // Not all undef elements?
1534 StoreChain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Stores);
1535 else
1536 StoreChain = DAG.getEntryNode();
1537
1538 // Result is a load from the stack slot.
1539 return DAG.getLoad(VT, dl, StoreChain, FIPtr, PtrInfo);
1540}
1541
1542/// Bitcast a floating-point value to an integer value. Only bitcast the part
1543/// containing the sign bit if the target has no integer value capable of
1544/// holding all bits of the floating-point value.
1545void SelectionDAGLegalize::getSignAsIntValue(FloatSignAsInt &State,
1546 const SDLoc &DL,
1547 SDValue Value) const {
1548 EVT FloatVT = Value.getValueType();
1549 unsigned NumBits = FloatVT.getScalarSizeInBits();
1550 State.FloatVT = FloatVT;
1551 EVT IVT = EVT::getIntegerVT(*DAG.getContext(), NumBits);
1552 // Convert to an integer of the same size.
1553 if (TLI.isTypeLegal(IVT)) {
1554 State.IntValue = DAG.getNode(ISD::BITCAST, DL, IVT, Value);
1555 State.SignMask = APInt::getSignMask(NumBits);
1556 State.SignBit = NumBits - 1;
1557 return;
1558 }
1559
1560 auto &DataLayout = DAG.getDataLayout();
1561 // Store the float to memory, then load the sign part out as an integer.
1562 MVT LoadTy = TLI.getRegisterType(MVT::i8);
1563 // First create a temporary that is aligned for both the load and store.
1564 SDValue StackPtr = DAG.CreateStackTemporary(FloatVT, LoadTy);
1565 int FI = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex();
1566 // Then store the float to it.
1567 State.FloatPtr = StackPtr;
1568 MachineFunction &MF = DAG.getMachineFunction();
1569 State.FloatPointerInfo = MachinePointerInfo::getFixedStack(MF, FI);
1570 State.Chain = DAG.getStore(DAG.getEntryNode(), DL, Value, State.FloatPtr,
1571 State.FloatPointerInfo);
1572
1573 SDValue IntPtr;
1574 if (DataLayout.isBigEndian()) {
1575 assert(FloatVT.isByteSized() && "Unsupported floating point type!");
1576 // Load out a legal integer with the same sign bit as the float.
1577 IntPtr = StackPtr;
1578 State.IntPointerInfo = State.FloatPointerInfo;
1579 } else {
1580 // Advance the pointer so that the loaded byte will contain the sign bit.
1581 unsigned ByteOffset = (NumBits / 8) - 1;
1582 IntPtr =
1583 DAG.getMemBasePlusOffset(StackPtr, TypeSize::getFixed(ByteOffset), DL);
1584 State.IntPointerInfo = MachinePointerInfo::getFixedStack(MF, FI,
1585 ByteOffset);
1586 }
1587
1588 State.IntPtr = IntPtr;
1589 State.IntValue = DAG.getExtLoad(ISD::EXTLOAD, DL, LoadTy, State.Chain, IntPtr,
1590 State.IntPointerInfo, MVT::i8);
1591 State.SignMask = APInt::getOneBitSet(LoadTy.getScalarSizeInBits(), 7);
1592 State.SignBit = 7;
1593}
1594
1595/// Replace the integer value produced by getSignAsIntValue() with a new value
1596/// and cast the result back to a floating-point type.
1597SDValue SelectionDAGLegalize::modifySignAsInt(const FloatSignAsInt &State,
1598 const SDLoc &DL,
1599 SDValue NewIntValue) const {
1600 if (!State.Chain)
1601 return DAG.getNode(ISD::BITCAST, DL, State.FloatVT, NewIntValue);
1602
1603 // Override the part containing the sign bit in the value stored on the stack.
1604 SDValue Chain = DAG.getTruncStore(State.Chain, DL, NewIntValue, State.IntPtr,
1605 State.IntPointerInfo, MVT::i8);
1606 return DAG.getLoad(State.FloatVT, DL, Chain, State.FloatPtr,
1607 State.FloatPointerInfo);
1608}
1609
1610SDValue SelectionDAGLegalize::ExpandFCOPYSIGN(SDNode *Node) const {
1611 SDLoc DL(Node);
1612 SDValue Mag = Node->getOperand(0);
1613 SDValue Sign = Node->getOperand(1);
1614
1615 // Get sign bit into an integer value.
1616 FloatSignAsInt SignAsInt;
1617 getSignAsIntValue(SignAsInt, DL, Sign);
1618
1619 EVT IntVT = SignAsInt.IntValue.getValueType();
1620 SDValue SignMask = DAG.getConstant(SignAsInt.SignMask, DL, IntVT);
1621 SDValue SignBit = DAG.getNode(ISD::AND, DL, IntVT, SignAsInt.IntValue,
1622 SignMask);
1623
1624 // If FABS is legal transform FCOPYSIGN(x, y) => sign(x) ? -FABS(x) : FABS(X)
1625 EVT FloatVT = Mag.getValueType();
1626 if (TLI.isOperationLegalOrCustom(ISD::FABS, FloatVT) &&
1627 TLI.isOperationLegalOrCustom(ISD::FNEG, FloatVT)) {
1628 SDValue AbsValue = DAG.getNode(ISD::FABS, DL, FloatVT, Mag);
1629 SDValue NegValue = DAG.getNode(ISD::FNEG, DL, FloatVT, AbsValue);
1630 SDValue Cond = DAG.getSetCC(DL, getSetCCResultType(IntVT), SignBit,
1631 DAG.getConstant(0, DL, IntVT), ISD::SETNE);
1632 return DAG.getSelect(DL, FloatVT, Cond, NegValue, AbsValue);
1633 }
1634
1635 // Transform Mag value to integer, and clear the sign bit.
1636 FloatSignAsInt MagAsInt;
1637 getSignAsIntValue(MagAsInt, DL, Mag);
1638 EVT MagVT = MagAsInt.IntValue.getValueType();
1639 SDValue ClearSignMask = DAG.getConstant(~MagAsInt.SignMask, DL, MagVT);
1640 SDValue ClearedSign = DAG.getNode(ISD::AND, DL, MagVT, MagAsInt.IntValue,
1641 ClearSignMask);
1642
1643 // Get the signbit at the right position for MagAsInt.
1644 int ShiftAmount = SignAsInt.SignBit - MagAsInt.SignBit;
1645 EVT ShiftVT = IntVT;
1646 if (SignBit.getScalarValueSizeInBits() <
1647 ClearedSign.getScalarValueSizeInBits()) {
1648 SignBit = DAG.getNode(ISD::ZERO_EXTEND, DL, MagVT, SignBit);
1649 ShiftVT = MagVT;
1650 }
1651 if (ShiftAmount > 0) {
1652 SDValue ShiftCnst = DAG.getConstant(ShiftAmount, DL, ShiftVT);
1653 SignBit = DAG.getNode(ISD::SRL, DL, ShiftVT, SignBit, ShiftCnst);
1654 } else if (ShiftAmount < 0) {
1655 SDValue ShiftCnst = DAG.getConstant(-ShiftAmount, DL, ShiftVT);
1656 SignBit = DAG.getNode(ISD::SHL, DL, ShiftVT, SignBit, ShiftCnst);
1657 }
1658 if (SignBit.getScalarValueSizeInBits() >
1659 ClearedSign.getScalarValueSizeInBits()) {
1660 SignBit = DAG.getNode(ISD::TRUNCATE, DL, MagVT, SignBit);
1661 }
1662
1663 // Store the part with the modified sign and convert back to float.
1664 SDValue CopiedSign = DAG.getNode(ISD::OR, DL, MagVT, ClearedSign, SignBit);
1665 return modifySignAsInt(MagAsInt, DL, CopiedSign);
1666}
1667
1668SDValue SelectionDAGLegalize::ExpandFNEG(SDNode *Node) const {
1669 // Get the sign bit as an integer.
1670 SDLoc DL(Node);
1671 FloatSignAsInt SignAsInt;
1672 getSignAsIntValue(SignAsInt, DL, Node->getOperand(0));
1673 EVT IntVT = SignAsInt.IntValue.getValueType();
1674
1675 // Flip the sign.
1676 SDValue SignMask = DAG.getConstant(SignAsInt.SignMask, DL, IntVT);
1677 SDValue SignFlip =
1678 DAG.getNode(ISD::XOR, DL, IntVT, SignAsInt.IntValue, SignMask);
1679
1680 // Convert back to float.
1681 return modifySignAsInt(SignAsInt, DL, SignFlip);
1682}
1683
1684SDValue SelectionDAGLegalize::ExpandFABS(SDNode *Node) const {
1685 SDLoc DL(Node);
1686 SDValue Value = Node->getOperand(0);
1687
1688 // Transform FABS(x) => FCOPYSIGN(x, 0.0) if FCOPYSIGN is legal.
1689 EVT FloatVT = Value.getValueType();
1690 if (TLI.isOperationLegalOrCustom(ISD::FCOPYSIGN, FloatVT)) {
1691 SDValue Zero = DAG.getConstantFP(0.0, DL, FloatVT);
1692 return DAG.getNode(ISD::FCOPYSIGN, DL, FloatVT, Value, Zero);
1693 }
1694
1695 // Transform value to integer, clear the sign bit and transform back.
1696 FloatSignAsInt ValueAsInt;
1697 getSignAsIntValue(ValueAsInt, DL, Value);
1698 EVT IntVT = ValueAsInt.IntValue.getValueType();
1699 SDValue ClearSignMask = DAG.getConstant(~ValueAsInt.SignMask, DL, IntVT);
1700 SDValue ClearedSign = DAG.getNode(ISD::AND, DL, IntVT, ValueAsInt.IntValue,
1701 ClearSignMask);
1702 return modifySignAsInt(ValueAsInt, DL, ClearedSign);
1703}
1704
1705void SelectionDAGLegalize::ExpandDYNAMIC_STACKALLOC(SDNode* Node,
1707 Register SPReg = TLI.getStackPointerRegisterToSaveRestore();
1708 assert(SPReg && "Target cannot require DYNAMIC_STACKALLOC expansion and"
1709 " not tell us which reg is the stack pointer!");
1710 SDLoc dl(Node);
1711 EVT VT = Node->getValueType(0);
1712 SDValue Tmp1 = SDValue(Node, 0);
1713 SDValue Tmp2 = SDValue(Node, 1);
1714 SDValue Tmp3 = Node->getOperand(2);
1715 SDValue Chain = Tmp1.getOperand(0);
1716
1717 // Chain the dynamic stack allocation so that it doesn't modify the stack
1718 // pointer when other instructions are using the stack.
1719 Chain = DAG.getCALLSEQ_START(Chain, 0, 0, dl);
1720
1721 SDValue Size = Tmp2.getOperand(1);
1722 SDValue SP = DAG.getCopyFromReg(Chain, dl, SPReg, VT);
1723 Chain = SP.getValue(1);
1724 Align Alignment = cast<ConstantSDNode>(Tmp3)->getAlignValue();
1725 const TargetFrameLowering *TFL = DAG.getSubtarget().getFrameLowering();
1726 unsigned Opc =
1729
1730 Align StackAlign = TFL->getStackAlign();
1731 Tmp1 = DAG.getNode(Opc, dl, VT, SP, Size); // Value
1732 if (Alignment > StackAlign)
1733 Tmp1 = DAG.getNode(ISD::AND, dl, VT, Tmp1,
1734 DAG.getConstant(-Alignment.value(), dl, VT));
1735 Chain = DAG.getCopyToReg(Chain, dl, SPReg, Tmp1); // Output chain
1736
1737 Tmp2 = DAG.getCALLSEQ_END(Chain, 0, 0, SDValue(), dl);
1738
1739 Results.push_back(Tmp1);
1740 Results.push_back(Tmp2);
1741}
1742
1743/// Emit a store/load combination to the stack. This stores
1744/// SrcOp to a stack slot of type SlotVT, truncating it if needed. It then does
1745/// a load from the stack slot to DestVT, extending it if needed.
1746/// The resultant code need not be legal.
1747SDValue SelectionDAGLegalize::EmitStackConvert(SDValue SrcOp, EVT SlotVT,
1748 EVT DestVT, const SDLoc &dl) {
1749 return EmitStackConvert(SrcOp, SlotVT, DestVT, dl, DAG.getEntryNode());
1750}
1751
1752SDValue SelectionDAGLegalize::EmitStackConvert(SDValue SrcOp, EVT SlotVT,
1753 EVT DestVT, const SDLoc &dl,
1754 SDValue Chain) {
1755 EVT SrcVT = SrcOp.getValueType();
1756 Type *DestType = DestVT.getTypeForEVT(*DAG.getContext());
1757 Align DestAlign = DAG.getDataLayout().getPrefTypeAlign(DestType);
1758
1759 // Don't convert with stack if the load/store is expensive.
1760 if ((SrcVT.bitsGT(SlotVT) &&
1761 !TLI.isTruncStoreLegalOrCustom(SrcOp.getValueType(), SlotVT)) ||
1762 (SlotVT.bitsLT(DestVT) &&
1763 !TLI.isLoadExtLegalOrCustom(ISD::EXTLOAD, DestVT, SlotVT)))
1764 return SDValue();
1765
1766 // Create the stack frame object.
1767 Align SrcAlign = DAG.getDataLayout().getPrefTypeAlign(
1768 SrcOp.getValueType().getTypeForEVT(*DAG.getContext()));
1769 SDValue FIPtr = DAG.CreateStackTemporary(SlotVT.getStoreSize(), SrcAlign);
1770
1771 FrameIndexSDNode *StackPtrFI = cast<FrameIndexSDNode>(FIPtr);
1772 int SPFI = StackPtrFI->getIndex();
1773 MachinePointerInfo PtrInfo =
1774 MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), SPFI);
1775
1776 // Emit a store to the stack slot. Use a truncstore if the input value is
1777 // later than DestVT.
1778 SDValue Store;
1779
1780 if (SrcVT.bitsGT(SlotVT))
1781 Store = DAG.getTruncStore(Chain, dl, SrcOp, FIPtr, PtrInfo,
1782 SlotVT, SrcAlign);
1783 else {
1784 assert(SrcVT.bitsEq(SlotVT) && "Invalid store");
1785 Store = DAG.getStore(Chain, dl, SrcOp, FIPtr, PtrInfo, SrcAlign);
1786 }
1787
1788 // Result is a load from the stack slot.
1789 if (SlotVT.bitsEq(DestVT))
1790 return DAG.getLoad(DestVT, dl, Store, FIPtr, PtrInfo, DestAlign);
1791
1792 assert(SlotVT.bitsLT(DestVT) && "Unknown extension!");
1793 return DAG.getExtLoad(ISD::EXTLOAD, dl, DestVT, Store, FIPtr, PtrInfo, SlotVT,
1794 DestAlign);
1795}
1796
1797SDValue SelectionDAGLegalize::ExpandSCALAR_TO_VECTOR(SDNode *Node) {
1798 SDLoc dl(Node);
1799 // Create a vector sized/aligned stack slot, store the value to element #0,
1800 // then load the whole vector back out.
1801 SDValue StackPtr = DAG.CreateStackTemporary(Node->getValueType(0));
1802
1803 FrameIndexSDNode *StackPtrFI = cast<FrameIndexSDNode>(StackPtr);
1804 int SPFI = StackPtrFI->getIndex();
1805
1806 SDValue Ch = DAG.getTruncStore(
1807 DAG.getEntryNode(), dl, Node->getOperand(0), StackPtr,
1808 MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), SPFI),
1809 Node->getValueType(0).getVectorElementType());
1810 return DAG.getLoad(
1811 Node->getValueType(0), dl, Ch, StackPtr,
1812 MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), SPFI));
1813}
1814
1815static bool
1817 const TargetLowering &TLI, SDValue &Res) {
1818 unsigned NumElems = Node->getNumOperands();
1819 SDLoc dl(Node);
1820 EVT VT = Node->getValueType(0);
1821
1822 // Try to group the scalars into pairs, shuffle the pairs together, then
1823 // shuffle the pairs of pairs together, etc. until the vector has
1824 // been built. This will work only if all of the necessary shuffle masks
1825 // are legal.
1826
1827 // We do this in two phases; first to check the legality of the shuffles,
1828 // and next, assuming that all shuffles are legal, to create the new nodes.
1829 for (int Phase = 0; Phase < 2; ++Phase) {
1831 NewIntermedVals;
1832 for (unsigned i = 0; i < NumElems; ++i) {
1833 SDValue V = Node->getOperand(i);
1834 if (V.isUndef())
1835 continue;
1836
1837 SDValue Vec;
1838 if (Phase)
1839 Vec = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, V);
1840 IntermedVals.push_back(std::make_pair(Vec, SmallVector<int, 16>(1, i)));
1841 }
1842
1843 while (IntermedVals.size() > 2) {
1844 NewIntermedVals.clear();
1845 for (unsigned i = 0, e = (IntermedVals.size() & ~1u); i < e; i += 2) {
1846 // This vector and the next vector are shuffled together (simply to
1847 // append the one to the other).
1848 SmallVector<int, 16> ShuffleVec(NumElems, -1);
1849
1850 SmallVector<int, 16> FinalIndices;
1851 FinalIndices.reserve(IntermedVals[i].second.size() +
1852 IntermedVals[i+1].second.size());
1853
1854 int k = 0;
1855 for (unsigned j = 0, f = IntermedVals[i].second.size(); j != f;
1856 ++j, ++k) {
1857 ShuffleVec[k] = j;
1858 FinalIndices.push_back(IntermedVals[i].second[j]);
1859 }
1860 for (unsigned j = 0, f = IntermedVals[i+1].second.size(); j != f;
1861 ++j, ++k) {
1862 ShuffleVec[k] = NumElems + j;
1863 FinalIndices.push_back(IntermedVals[i+1].second[j]);
1864 }
1865
1866 SDValue Shuffle;
1867 if (Phase)
1868 Shuffle = DAG.getVectorShuffle(VT, dl, IntermedVals[i].first,
1869 IntermedVals[i+1].first,
1870 ShuffleVec);
1871 else if (!TLI.isShuffleMaskLegal(ShuffleVec, VT))
1872 return false;
1873 NewIntermedVals.push_back(
1874 std::make_pair(Shuffle, std::move(FinalIndices)));
1875 }
1876
1877 // If we had an odd number of defined values, then append the last
1878 // element to the array of new vectors.
1879 if ((IntermedVals.size() & 1) != 0)
1880 NewIntermedVals.push_back(IntermedVals.back());
1881
1882 IntermedVals.swap(NewIntermedVals);
1883 }
1884
1885 assert(IntermedVals.size() <= 2 && IntermedVals.size() > 0 &&
1886 "Invalid number of intermediate vectors");
1887 SDValue Vec1 = IntermedVals[0].first;
1888 SDValue Vec2;
1889 if (IntermedVals.size() > 1)
1890 Vec2 = IntermedVals[1].first;
1891 else if (Phase)
1892 Vec2 = DAG.getUNDEF(VT);
1893
1894 SmallVector<int, 16> ShuffleVec(NumElems, -1);
1895 for (unsigned i = 0, e = IntermedVals[0].second.size(); i != e; ++i)
1896 ShuffleVec[IntermedVals[0].second[i]] = i;
1897 for (unsigned i = 0, e = IntermedVals[1].second.size(); i != e; ++i)
1898 ShuffleVec[IntermedVals[1].second[i]] = NumElems + i;
1899
1900 if (Phase)
1901 Res = DAG.getVectorShuffle(VT, dl, Vec1, Vec2, ShuffleVec);
1902 else if (!TLI.isShuffleMaskLegal(ShuffleVec, VT))
1903 return false;
1904 }
1905
1906 return true;
1907}
1908
1909/// Expand a BUILD_VECTOR node on targets that don't
1910/// support the operation, but do support the resultant vector type.
1911SDValue SelectionDAGLegalize::ExpandBUILD_VECTOR(SDNode *Node) {
1912 unsigned NumElems = Node->getNumOperands();
1913 SDValue Value1, Value2;
1914 SDLoc dl(Node);
1915 EVT VT = Node->getValueType(0);
1916 EVT OpVT = Node->getOperand(0).getValueType();
1917 EVT EltVT = VT.getVectorElementType();
1918
1919 // If the only non-undef value is the low element, turn this into a
1920 // SCALAR_TO_VECTOR node. If this is { X, X, X, X }, determine X.
1921 bool isOnlyLowElement = true;
1922 bool MoreThanTwoValues = false;
1923 bool isConstant = true;
1924 for (unsigned i = 0; i < NumElems; ++i) {
1925 SDValue V = Node->getOperand(i);
1926 if (V.isUndef())
1927 continue;
1928 if (i > 0)
1929 isOnlyLowElement = false;
1930 if (!isa<ConstantFPSDNode>(V) && !isa<ConstantSDNode>(V))
1931 isConstant = false;
1932
1933 if (!Value1.getNode()) {
1934 Value1 = V;
1935 } else if (!Value2.getNode()) {
1936 if (V != Value1)
1937 Value2 = V;
1938 } else if (V != Value1 && V != Value2) {
1939 MoreThanTwoValues = true;
1940 }
1941 }
1942
1943 if (!Value1.getNode())
1944 return DAG.getUNDEF(VT);
1945
1946 if (isOnlyLowElement)
1947 return DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, Node->getOperand(0));
1948
1949 // If all elements are constants, create a load from the constant pool.
1950 if (isConstant) {
1952 for (unsigned i = 0, e = NumElems; i != e; ++i) {
1953 if (ConstantFPSDNode *V =
1954 dyn_cast<ConstantFPSDNode>(Node->getOperand(i))) {
1955 CV.push_back(const_cast<ConstantFP *>(V->getConstantFPValue()));
1956 } else if (ConstantSDNode *V =
1957 dyn_cast<ConstantSDNode>(Node->getOperand(i))) {
1958 if (OpVT==EltVT)
1959 CV.push_back(const_cast<ConstantInt *>(V->getConstantIntValue()));
1960 else {
1961 // If OpVT and EltVT don't match, EltVT is not legal and the
1962 // element values have been promoted/truncated earlier. Undo this;
1963 // we don't want a v16i8 to become a v16i32 for example.
1964 const ConstantInt *CI = V->getConstantIntValue();
1965 CV.push_back(ConstantInt::get(EltVT.getTypeForEVT(*DAG.getContext()),
1966 CI->getZExtValue()));
1967 }
1968 } else {
1969 assert(Node->getOperand(i).isUndef());
1970 Type *OpNTy = EltVT.getTypeForEVT(*DAG.getContext());
1971 CV.push_back(UndefValue::get(OpNTy));
1972 }
1973 }
1975 SDValue CPIdx =
1976 DAG.getConstantPool(CP, TLI.getPointerTy(DAG.getDataLayout()));
1977 Align Alignment = cast<ConstantPoolSDNode>(CPIdx)->getAlign();
1978 return DAG.getLoad(
1979 VT, dl, DAG.getEntryNode(), CPIdx,
1980 MachinePointerInfo::getConstantPool(DAG.getMachineFunction()),
1981 Alignment);
1982 }
1983
1984 SmallSet<SDValue, 16> DefinedValues;
1985 for (unsigned i = 0; i < NumElems; ++i) {
1986 if (Node->getOperand(i).isUndef())
1987 continue;
1988 DefinedValues.insert(Node->getOperand(i));
1989 }
1990
1991 if (TLI.shouldExpandBuildVectorWithShuffles(VT, DefinedValues.size())) {
1992 if (!MoreThanTwoValues) {
1993 SmallVector<int, 8> ShuffleVec(NumElems, -1);
1994 for (unsigned i = 0; i < NumElems; ++i) {
1995 SDValue V = Node->getOperand(i);
1996 if (V.isUndef())
1997 continue;
1998 ShuffleVec[i] = V == Value1 ? 0 : NumElems;
1999 }
2000 if (TLI.isShuffleMaskLegal(ShuffleVec, Node->getValueType(0))) {
2001 // Get the splatted value into the low element of a vector register.
2002 SDValue Vec1 = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, Value1);
2003 SDValue Vec2;
2004 if (Value2.getNode())
2005 Vec2 = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, Value2);
2006 else
2007 Vec2 = DAG.getUNDEF(VT);
2008
2009 // Return shuffle(LowValVec, undef, <0,0,0,0>)
2010 return DAG.getVectorShuffle(VT, dl, Vec1, Vec2, ShuffleVec);
2011 }
2012 } else {
2013 SDValue Res;
2014 if (ExpandBVWithShuffles(Node, DAG, TLI, Res))
2015 return Res;
2016 }
2017 }
2018
2019 // Otherwise, we can't handle this case efficiently.
2020 return ExpandVectorBuildThroughStack(Node);
2021}
2022
2023SDValue SelectionDAGLegalize::ExpandSPLAT_VECTOR(SDNode *Node) {
2024 SDLoc DL(Node);
2025 EVT VT = Node->getValueType(0);
2026 SDValue SplatVal = Node->getOperand(0);
2027
2028 return DAG.getSplatBuildVector(VT, DL, SplatVal);
2029}
2030
2031// Expand a node into a call to a libcall, returning the value as the first
2032// result and the chain as the second. If the result value does not fit into a
2033// register, return the lo part and set the hi part to the by-reg argument in
2034// the first. If it does fit into a single register, return the result and
2035// leave the Hi part unset.
2036std::pair<SDValue, SDValue> SelectionDAGLegalize::ExpandLibCall(RTLIB::Libcall LC, SDNode *Node,
2038 bool isSigned) {
2039 SDValue Callee = DAG.getExternalSymbol(TLI.getLibcallName(LC),
2040 TLI.getPointerTy(DAG.getDataLayout()));
2041
2042 EVT RetVT = Node->getValueType(0);
2043 Type *RetTy = RetVT.getTypeForEVT(*DAG.getContext());
2044
2045 // By default, the input chain to this libcall is the entry node of the
2046 // function. If the libcall is going to be emitted as a tail call then
2047 // TLI.isUsedByReturnOnly will change it to the right chain if the return
2048 // node which is being folded has a non-entry input chain.
2049 SDValue InChain = DAG.getEntryNode();
2050
2051 // isTailCall may be true since the callee does not reference caller stack
2052 // frame. Check if it's in the right position and that the return types match.
2053 SDValue TCChain = InChain;
2054 const Function &F = DAG.getMachineFunction().getFunction();
2055 bool isTailCall =
2056 TLI.isInTailCallPosition(DAG, Node, TCChain) &&
2057 (RetTy == F.getReturnType() || F.getReturnType()->isVoidTy());
2058 if (isTailCall)
2059 InChain = TCChain;
2060
2062 bool signExtend = TLI.shouldSignExtendTypeInLibCall(RetVT, isSigned);
2063 CLI.setDebugLoc(SDLoc(Node))
2064 .setChain(InChain)
2065 .setLibCallee(TLI.getLibcallCallingConv(LC), RetTy, Callee,
2066 std::move(Args))
2067 .setTailCall(isTailCall)
2068 .setSExtResult(signExtend)
2069 .setZExtResult(!signExtend)
2070 .setIsPostTypeLegalization(true);
2071
2072 std::pair<SDValue, SDValue> CallInfo = TLI.LowerCallTo(CLI);
2073
2074 if (!CallInfo.second.getNode()) {
2075 LLVM_DEBUG(dbgs() << "Created tailcall: "; DAG.getRoot().dump(&DAG));
2076 // It's a tailcall, return the chain (which is the DAG root).
2077 return {DAG.getRoot(), DAG.getRoot()};
2078 }
2079
2080 LLVM_DEBUG(dbgs() << "Created libcall: "; CallInfo.first.dump(&DAG));
2081 return CallInfo;
2082}
2083
2084std::pair<SDValue, SDValue> SelectionDAGLegalize::ExpandLibCall(RTLIB::Libcall LC, SDNode *Node,
2085 bool isSigned) {
2088 for (const SDValue &Op : Node->op_values()) {
2089 EVT ArgVT = Op.getValueType();
2090 Type *ArgTy = ArgVT.getTypeForEVT(*DAG.getContext());
2091 Entry.Node = Op;
2092 Entry.Ty = ArgTy;
2093 Entry.IsSExt = TLI.shouldSignExtendTypeInLibCall(ArgVT, isSigned);
2094 Entry.IsZExt = !Entry.IsSExt;
2095 Args.push_back(Entry);
2096 }
2097
2098 return ExpandLibCall(LC, Node, std::move(Args), isSigned);
2099}
2100
2101void SelectionDAGLegalize::ExpandFrexpLibCall(
2103 SDLoc dl(Node);
2104 EVT VT = Node->getValueType(0);
2105 EVT ExpVT = Node->getValueType(1);
2106
2107 SDValue FPOp = Node->getOperand(0);
2108
2109 EVT ArgVT = FPOp.getValueType();
2110 Type *ArgTy = ArgVT.getTypeForEVT(*DAG.getContext());
2111
2113 FPArgEntry.Node = FPOp;
2114 FPArgEntry.Ty = ArgTy;
2115
2116 SDValue StackSlot = DAG.CreateStackTemporary(ExpVT);
2117 TargetLowering::ArgListEntry PtrArgEntry;
2118 PtrArgEntry.Node = StackSlot;
2119 PtrArgEntry.Ty = PointerType::get(*DAG.getContext(),
2120 DAG.getDataLayout().getAllocaAddrSpace());
2121
2122 TargetLowering::ArgListTy Args = {FPArgEntry, PtrArgEntry};
2123
2125 auto [Call, Chain] = ExpandLibCall(LC, Node, std::move(Args), false);
2126
2127 // FIXME: Get type of int for libcall declaration and cast
2128
2129 int FrameIdx = cast<FrameIndexSDNode>(StackSlot)->getIndex();
2130 auto PtrInfo =
2131 MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), FrameIdx);
2132
2133 SDValue LoadExp = DAG.getLoad(ExpVT, dl, Chain, StackSlot, PtrInfo);
2134 SDValue OutputChain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other,
2135 LoadExp.getValue(1), DAG.getRoot());
2136 DAG.setRoot(OutputChain);
2137
2138 Results.push_back(Call);
2139 Results.push_back(LoadExp);
2140}
2141
2142void SelectionDAGLegalize::ExpandFPLibCall(SDNode* Node,
2143 RTLIB::Libcall LC,
2145 if (LC == RTLIB::UNKNOWN_LIBCALL)
2146 llvm_unreachable("Can't create an unknown libcall!");
2147
2148 if (Node->isStrictFPOpcode()) {
2149 EVT RetVT = Node->getValueType(0);
2152 // FIXME: This doesn't support tail calls.
2153 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, RetVT,
2154 Ops, CallOptions,
2155 SDLoc(Node),
2156 Node->getOperand(0));
2157 Results.push_back(Tmp.first);
2158 Results.push_back(Tmp.second);
2159 } else {
2160 SDValue Tmp = ExpandLibCall(LC, Node, false).first;
2161 Results.push_back(Tmp);
2162 }
2163}
2164
2165/// Expand the node to a libcall based on the result type.
2166void SelectionDAGLegalize::ExpandFPLibCall(SDNode* Node,
2167 RTLIB::Libcall Call_F32,
2168 RTLIB::Libcall Call_F64,
2169 RTLIB::Libcall Call_F80,
2170 RTLIB::Libcall Call_F128,
2171 RTLIB::Libcall Call_PPCF128,
2173 RTLIB::Libcall LC = RTLIB::getFPLibCall(Node->getSimpleValueType(0),
2174 Call_F32, Call_F64, Call_F80,
2175 Call_F128, Call_PPCF128);
2176 ExpandFPLibCall(Node, LC, Results);
2177}
2178
2179SDValue SelectionDAGLegalize::ExpandIntLibCall(SDNode* Node, bool isSigned,
2180 RTLIB::Libcall Call_I8,
2181 RTLIB::Libcall Call_I16,
2182 RTLIB::Libcall Call_I32,
2183 RTLIB::Libcall Call_I64,
2184 RTLIB::Libcall Call_I128) {
2185 RTLIB::Libcall LC;
2186 switch (Node->getSimpleValueType(0).SimpleTy) {
2187 default: llvm_unreachable("Unexpected request for libcall!");
2188 case MVT::i8: LC = Call_I8; break;
2189 case MVT::i16: LC = Call_I16; break;
2190 case MVT::i32: LC = Call_I32; break;
2191 case MVT::i64: LC = Call_I64; break;
2192 case MVT::i128: LC = Call_I128; break;
2193 }
2194 return ExpandLibCall(LC, Node, isSigned).first;
2195}
2196
2197/// Expand the node to a libcall based on first argument type (for instance
2198/// lround and its variant).
2199void SelectionDAGLegalize::ExpandArgFPLibCall(SDNode* Node,
2200 RTLIB::Libcall Call_F32,
2201 RTLIB::Libcall Call_F64,
2202 RTLIB::Libcall Call_F80,
2203 RTLIB::Libcall Call_F128,
2204 RTLIB::Libcall Call_PPCF128,
2206 EVT InVT = Node->getOperand(Node->isStrictFPOpcode() ? 1 : 0).getValueType();
2208 Call_F32, Call_F64, Call_F80,
2209 Call_F128, Call_PPCF128);
2210 ExpandFPLibCall(Node, LC, Results);
2211}
2212
2213/// Issue libcalls to __{u}divmod to compute div / rem pairs.
2214void
2215SelectionDAGLegalize::ExpandDivRemLibCall(SDNode *Node,
2217 unsigned Opcode = Node->getOpcode();
2218 bool isSigned = Opcode == ISD::SDIVREM;
2219
2220 RTLIB::Libcall LC;
2221 switch (Node->getSimpleValueType(0).SimpleTy) {
2222 default: llvm_unreachable("Unexpected request for libcall!");
2223 case MVT::i8: LC= isSigned ? RTLIB::SDIVREM_I8 : RTLIB::UDIVREM_I8; break;
2224 case MVT::i16: LC= isSigned ? RTLIB::SDIVREM_I16 : RTLIB::UDIVREM_I16; break;
2225 case MVT::i32: LC= isSigned ? RTLIB::SDIVREM_I32 : RTLIB::UDIVREM_I32; break;
2226 case MVT::i64: LC= isSigned ? RTLIB::SDIVREM_I64 : RTLIB::UDIVREM_I64; break;
2227 case MVT::i128: LC= isSigned ? RTLIB::SDIVREM_I128:RTLIB::UDIVREM_I128; break;
2228 }
2229
2230 // The input chain to this libcall is the entry node of the function.
2231 // Legalizing the call will automatically add the previous call to the
2232 // dependence.
2233 SDValue InChain = DAG.getEntryNode();
2234
2235 EVT RetVT = Node->getValueType(0);
2236 Type *RetTy = RetVT.getTypeForEVT(*DAG.getContext());
2237
2240 for (const SDValue &Op : Node->op_values()) {
2241 EVT ArgVT = Op.getValueType();
2242 Type *ArgTy = ArgVT.getTypeForEVT(*DAG.getContext());
2243 Entry.Node = Op;
2244 Entry.Ty = ArgTy;
2245 Entry.IsSExt = isSigned;
2246 Entry.IsZExt = !isSigned;
2247 Args.push_back(Entry);
2248 }
2249
2250 // Also pass the return address of the remainder.
2251 SDValue FIPtr = DAG.CreateStackTemporary(RetVT);
2252 Entry.Node = FIPtr;
2253 Entry.Ty = PointerType::getUnqual(RetTy->getContext());
2254 Entry.IsSExt = isSigned;
2255 Entry.IsZExt = !isSigned;
2256 Args.push_back(Entry);
2257
2258 SDValue Callee = DAG.getExternalSymbol(TLI.getLibcallName(LC),
2259 TLI.getPointerTy(DAG.getDataLayout()));
2260
2261 SDLoc dl(Node);
2263 CLI.setDebugLoc(dl)
2264 .setChain(InChain)
2265 .setLibCallee(TLI.getLibcallCallingConv(LC), RetTy, Callee,
2266 std::move(Args))
2267 .setSExtResult(isSigned)
2268 .setZExtResult(!isSigned);
2269
2270 std::pair<SDValue, SDValue> CallInfo = TLI.LowerCallTo(CLI);
2271
2272 // Remainder is loaded back from the stack frame.
2273 SDValue Rem =
2274 DAG.getLoad(RetVT, dl, CallInfo.second, FIPtr, MachinePointerInfo());
2275 Results.push_back(CallInfo.first);
2276 Results.push_back(Rem);
2277}
2278
2279/// Return true if sincos libcall is available.
2281 RTLIB::Libcall LC;
2282 switch (Node->getSimpleValueType(0).SimpleTy) {
2283 default: llvm_unreachable("Unexpected request for libcall!");
2284 case MVT::f32: LC = RTLIB::SINCOS_F32; break;
2285 case MVT::f64: LC = RTLIB::SINCOS_F64; break;
2286 case MVT::f80: LC = RTLIB::SINCOS_F80; break;
2287 case MVT::f128: LC = RTLIB::SINCOS_F128; break;
2288 case MVT::ppcf128: LC = RTLIB::SINCOS_PPCF128; break;
2289 }
2290 return TLI.getLibcallName(LC) != nullptr;
2291}
2292
2293/// Only issue sincos libcall if both sin and cos are needed.
2294static bool useSinCos(SDNode *Node) {
2295 unsigned OtherOpcode = Node->getOpcode() == ISD::FSIN
2296 ? ISD::FCOS : ISD::FSIN;
2297
2298 SDValue Op0 = Node->getOperand(0);
2299 for (const SDNode *User : Op0.getNode()->uses()) {
2300 if (User == Node)
2301 continue;
2302 // The other user might have been turned into sincos already.
2303 if (User->getOpcode() == OtherOpcode || User->getOpcode() == ISD::FSINCOS)
2304 return true;
2305 }
2306 return false;
2307}
2308
2309/// Issue libcalls to sincos to compute sin / cos pairs.
2310void
2311SelectionDAGLegalize::ExpandSinCosLibCall(SDNode *Node,
2313 RTLIB::Libcall LC;
2314 switch (Node->getSimpleValueType(0).SimpleTy) {
2315 default: llvm_unreachable("Unexpected request for libcall!");
2316 case MVT::f32: LC = RTLIB::SINCOS_F32; break;
2317 case MVT::f64: LC = RTLIB::SINCOS_F64; break;
2318 case MVT::f80: LC = RTLIB::SINCOS_F80; break;
2319 case MVT::f128: LC = RTLIB::SINCOS_F128; break;
2320 case MVT::ppcf128: LC = RTLIB::SINCOS_PPCF128; break;
2321 }
2322
2323 // The input chain to this libcall is the entry node of the function.
2324 // Legalizing the call will automatically add the previous call to the
2325 // dependence.
2326 SDValue InChain = DAG.getEntryNode();
2327
2328 EVT RetVT = Node->getValueType(0);
2329 Type *RetTy = RetVT.getTypeForEVT(*DAG.getContext());
2330
2333
2334 // Pass the argument.
2335 Entry.Node = Node->getOperand(0);
2336 Entry.Ty = RetTy;
2337 Entry.IsSExt = false;
2338 Entry.IsZExt = false;
2339 Args.push_back(Entry);
2340
2341 // Pass the return address of sin.
2342 SDValue SinPtr = DAG.CreateStackTemporary(RetVT);
2343 Entry.Node = SinPtr;
2344 Entry.Ty = PointerType::getUnqual(RetTy->getContext());
2345 Entry.IsSExt = false;
2346 Entry.IsZExt = false;
2347 Args.push_back(Entry);
2348
2349 // Also pass the return address of the cos.
2350 SDValue CosPtr = DAG.CreateStackTemporary(RetVT);
2351 Entry.Node = CosPtr;
2352 Entry.Ty = PointerType::getUnqual(RetTy->getContext());
2353 Entry.IsSExt = false;
2354 Entry.IsZExt = false;
2355 Args.push_back(Entry);
2356
2357 SDValue Callee = DAG.getExternalSymbol(TLI.getLibcallName(LC),
2358 TLI.getPointerTy(DAG.getDataLayout()));
2359
2360 SDLoc dl(Node);
2362 CLI.setDebugLoc(dl).setChain(InChain).setLibCallee(
2363 TLI.getLibcallCallingConv(LC), Type::getVoidTy(*DAG.getContext()), Callee,
2364 std::move(Args));
2365
2366 std::pair<SDValue, SDValue> CallInfo = TLI.LowerCallTo(CLI);
2367
2368 Results.push_back(
2369 DAG.getLoad(RetVT, dl, CallInfo.second, SinPtr, MachinePointerInfo()));
2370 Results.push_back(
2371 DAG.getLoad(RetVT, dl, CallInfo.second, CosPtr, MachinePointerInfo()));
2372}
2373
2374SDValue SelectionDAGLegalize::expandLdexp(SDNode *Node) const {
2375 SDLoc dl(Node);
2376 EVT VT = Node->getValueType(0);
2377 SDValue X = Node->getOperand(0);
2378 SDValue N = Node->getOperand(1);
2379 EVT ExpVT = N.getValueType();
2380 EVT AsIntVT = VT.changeTypeToInteger();
2381 if (AsIntVT == EVT()) // TODO: How to handle f80?
2382 return SDValue();
2383
2384 if (Node->getOpcode() == ISD::STRICT_FLDEXP) // TODO
2385 return SDValue();
2386
2387 SDNodeFlags NSW;
2388 NSW.setNoSignedWrap(true);
2389 SDNodeFlags NUW_NSW;
2390 NUW_NSW.setNoUnsignedWrap(true);
2391 NUW_NSW.setNoSignedWrap(true);
2392
2393 EVT SetCCVT =
2394 TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), ExpVT);
2396
2397 const APFloat::ExponentType MaxExpVal = APFloat::semanticsMaxExponent(FltSem);
2398 const APFloat::ExponentType MinExpVal = APFloat::semanticsMinExponent(FltSem);
2399 const int Precision = APFloat::semanticsPrecision(FltSem);
2400
2401 const SDValue MaxExp = DAG.getConstant(MaxExpVal, dl, ExpVT);
2402 const SDValue MinExp = DAG.getConstant(MinExpVal, dl, ExpVT);
2403
2404 const SDValue DoubleMaxExp = DAG.getConstant(2 * MaxExpVal, dl, ExpVT);
2405
2406 const APFloat One(FltSem, "1.0");
2407 APFloat ScaleUpK = scalbn(One, MaxExpVal, APFloat::rmNearestTiesToEven);
2408
2409 // Offset by precision to avoid denormal range.
2410 APFloat ScaleDownK =
2411 scalbn(One, MinExpVal + Precision, APFloat::rmNearestTiesToEven);
2412
2413 // TODO: Should really introduce control flow and use a block for the >
2414 // MaxExp, < MinExp cases
2415
2416 // First, handle exponents Exp > MaxExp and scale down.
2417 SDValue NGtMaxExp = DAG.getSetCC(dl, SetCCVT, N, MaxExp, ISD::SETGT);
2418
2419 SDValue DecN0 = DAG.getNode(ISD::SUB, dl, ExpVT, N, MaxExp, NSW);
2420 SDValue ClampMaxVal = DAG.getConstant(3 * MaxExpVal, dl, ExpVT);
2421 SDValue ClampN_Big = DAG.getNode(ISD::SMIN, dl, ExpVT, N, ClampMaxVal);
2422 SDValue DecN1 =
2423 DAG.getNode(ISD::SUB, dl, ExpVT, ClampN_Big, DoubleMaxExp, NSW);
2424
2425 SDValue ScaleUpTwice =
2426 DAG.getSetCC(dl, SetCCVT, N, DoubleMaxExp, ISD::SETUGT);
2427
2428 const SDValue ScaleUpVal = DAG.getConstantFP(ScaleUpK, dl, VT);
2429 SDValue ScaleUp0 = DAG.getNode(ISD::FMUL, dl, VT, X, ScaleUpVal);
2430 SDValue ScaleUp1 = DAG.getNode(ISD::FMUL, dl, VT, ScaleUp0, ScaleUpVal);
2431
2432 SDValue SelectN_Big =
2433 DAG.getNode(ISD::SELECT, dl, ExpVT, ScaleUpTwice, DecN1, DecN0);
2434 SDValue SelectX_Big =
2435 DAG.getNode(ISD::SELECT, dl, VT, ScaleUpTwice, ScaleUp1, ScaleUp0);
2436
2437 // Now handle exponents Exp < MinExp
2438 SDValue NLtMinExp = DAG.getSetCC(dl, SetCCVT, N, MinExp, ISD::SETLT);
2439
2440 SDValue Increment0 = DAG.getConstant(-(MinExpVal + Precision), dl, ExpVT);
2441 SDValue Increment1 = DAG.getConstant(-2 * (MinExpVal + Precision), dl, ExpVT);
2442
2443 SDValue IncN0 = DAG.getNode(ISD::ADD, dl, ExpVT, N, Increment0, NUW_NSW);
2444
2445 SDValue ClampMinVal =
2446 DAG.getConstant(3 * MinExpVal + 2 * Precision, dl, ExpVT);
2447 SDValue ClampN_Small = DAG.getNode(ISD::SMAX, dl, ExpVT, N, ClampMinVal);
2448 SDValue IncN1 =
2449 DAG.getNode(ISD::ADD, dl, ExpVT, ClampN_Small, Increment1, NSW);
2450
2451 const SDValue ScaleDownVal = DAG.getConstantFP(ScaleDownK, dl, VT);
2452 SDValue ScaleDown0 = DAG.getNode(ISD::FMUL, dl, VT, X, ScaleDownVal);
2453 SDValue ScaleDown1 = DAG.getNode(ISD::FMUL, dl, VT, ScaleDown0, ScaleDownVal);
2454
2455 SDValue ScaleDownTwice = DAG.getSetCC(
2456 dl, SetCCVT, N, DAG.getConstant(2 * MinExpVal + Precision, dl, ExpVT),
2457 ISD::SETULT);
2458
2459 SDValue SelectN_Small =
2460 DAG.getNode(ISD::SELECT, dl, ExpVT, ScaleDownTwice, IncN1, IncN0);
2461 SDValue SelectX_Small =
2462 DAG.getNode(ISD::SELECT, dl, VT, ScaleDownTwice, ScaleDown1, ScaleDown0);
2463
2464 // Now combine the two out of range exponent handling cases with the base
2465 // case.
2466 SDValue NewX = DAG.getNode(
2467 ISD::SELECT, dl, VT, NGtMaxExp, SelectX_Big,
2468 DAG.getNode(ISD::SELECT, dl, VT, NLtMinExp, SelectX_Small, X));
2469
2470 SDValue NewN = DAG.getNode(
2471 ISD::SELECT, dl, ExpVT, NGtMaxExp, SelectN_Big,
2472 DAG.getNode(ISD::SELECT, dl, ExpVT, NLtMinExp, SelectN_Small, N));
2473
2474 SDValue BiasedN = DAG.getNode(ISD::ADD, dl, ExpVT, NewN, MaxExp, NSW);
2475
2476 SDValue ExponentShiftAmt =
2477 DAG.getShiftAmountConstant(Precision - 1, ExpVT, dl);
2478 SDValue CastExpToValTy = DAG.getZExtOrTrunc(BiasedN, dl, AsIntVT);
2479
2480 SDValue AsInt = DAG.getNode(ISD::SHL, dl, AsIntVT, CastExpToValTy,
2481 ExponentShiftAmt, NUW_NSW);
2482 SDValue AsFP = DAG.getNode(ISD::BITCAST, dl, VT, AsInt);
2483 return DAG.getNode(ISD::FMUL, dl, VT, NewX, AsFP);
2484}
2485
2486SDValue SelectionDAGLegalize::expandFrexp(SDNode *Node) const {
2487 SDLoc dl(Node);
2488 SDValue Val = Node->getOperand(0);
2489 EVT VT = Val.getValueType();
2490 EVT ExpVT = Node->getValueType(1);
2491 EVT AsIntVT = VT.changeTypeToInteger();
2492 if (AsIntVT == EVT()) // TODO: How to handle f80?
2493 return SDValue();
2494
2496 const APFloat::ExponentType MinExpVal = APFloat::semanticsMinExponent(FltSem);
2497 const unsigned Precision = APFloat::semanticsPrecision(FltSem);
2498 const unsigned BitSize = VT.getScalarSizeInBits();
2499
2500 // TODO: Could introduce control flow and skip over the denormal handling.
2501
2502 // scale_up = fmul value, scalbn(1.0, precision + 1)
2503 // extracted_exp = (bitcast value to uint) >> precision - 1
2504 // biased_exp = extracted_exp + min_exp
2505 // extracted_fract = (bitcast value to uint) & (fract_mask | sign_mask)
2506 //
2507 // is_denormal = val < smallest_normalized
2508 // computed_fract = is_denormal ? scale_up : extracted_fract
2509 // computed_exp = is_denormal ? biased_exp + (-precision - 1) : biased_exp
2510 //
2511 // result_0 = (!isfinite(val) || iszero(val)) ? val : computed_fract
2512 // result_1 = (!isfinite(val) || iszero(val)) ? 0 : computed_exp
2513
2514 SDValue NegSmallestNormalizedInt = DAG.getConstant(
2515 APFloat::getSmallestNormalized(FltSem, true).bitcastToAPInt(), dl,
2516 AsIntVT);
2517
2518 SDValue SmallestNormalizedInt = DAG.getConstant(
2519 APFloat::getSmallestNormalized(FltSem, false).bitcastToAPInt(), dl,
2520 AsIntVT);
2521
2522 // Masks out the exponent bits.
2523 SDValue ExpMask =
2524 DAG.getConstant(APFloat::getInf(FltSem).bitcastToAPInt(), dl, AsIntVT);
2525
2526 // Mask out the exponent part of the value.
2527 //
2528 // e.g, for f32 FractSignMaskVal = 0x807fffff
2529 APInt FractSignMaskVal = APInt::getBitsSet(BitSize, 0, Precision - 1);
2530 FractSignMaskVal.setBit(BitSize - 1); // Set the sign bit
2531
2532 APInt SignMaskVal = APInt::getSignedMaxValue(BitSize);
2533 SDValue SignMask = DAG.getConstant(SignMaskVal, dl, AsIntVT);
2534
2535 SDValue FractSignMask = DAG.getConstant(FractSignMaskVal, dl, AsIntVT);
2536
2537 const APFloat One(FltSem, "1.0");
2538 // Scale a possible denormal input.
2539 // e.g., for f64, 0x1p+54
2540 APFloat ScaleUpKVal =
2541 scalbn(One, Precision + 1, APFloat::rmNearestTiesToEven);
2542
2543 SDValue ScaleUpK = DAG.getConstantFP(ScaleUpKVal, dl, VT);
2544 SDValue ScaleUp = DAG.getNode(ISD::FMUL, dl, VT, Val, ScaleUpK);
2545
2546 EVT SetCCVT =
2547 TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT);
2548
2549 SDValue AsInt = DAG.getNode(ISD::BITCAST, dl, AsIntVT, Val);
2550
2551 SDValue Abs = DAG.getNode(ISD::AND, dl, AsIntVT, AsInt, SignMask);
2552
2553 SDValue AddNegSmallestNormal =
2554 DAG.getNode(ISD::ADD, dl, AsIntVT, Abs, NegSmallestNormalizedInt);
2555 SDValue DenormOrZero = DAG.getSetCC(dl, SetCCVT, AddNegSmallestNormal,
2556 NegSmallestNormalizedInt, ISD::SETULE);
2557
2558 SDValue IsDenormal =
2559 DAG.getSetCC(dl, SetCCVT, Abs, SmallestNormalizedInt, ISD::SETULT);
2560
2561 SDValue MinExp = DAG.getConstant(MinExpVal, dl, ExpVT);
2562 SDValue Zero = DAG.getConstant(0, dl, ExpVT);
2563
2564 SDValue ScaledAsInt = DAG.getNode(ISD::BITCAST, dl, AsIntVT, ScaleUp);
2565 SDValue ScaledSelect =
2566 DAG.getNode(ISD::SELECT, dl, AsIntVT, IsDenormal, ScaledAsInt, AsInt);
2567
2568 SDValue ExpMaskScaled =
2569 DAG.getNode(ISD::AND, dl, AsIntVT, ScaledAsInt, ExpMask);
2570
2571 SDValue ScaledValue =
2572 DAG.getNode(ISD::SELECT, dl, AsIntVT, IsDenormal, ExpMaskScaled, Abs);
2573
2574 // Extract the exponent bits.
2575 SDValue ExponentShiftAmt =
2576 DAG.getShiftAmountConstant(Precision - 1, AsIntVT, dl);
2577 SDValue ShiftedExp =
2578 DAG.getNode(ISD::SRL, dl, AsIntVT, ScaledValue, ExponentShiftAmt);
2579 SDValue Exp = DAG.getSExtOrTrunc(ShiftedExp, dl, ExpVT);
2580
2581 SDValue NormalBiasedExp = DAG.getNode(ISD::ADD, dl, ExpVT, Exp, MinExp);
2582 SDValue DenormalOffset = DAG.getConstant(-Precision - 1, dl, ExpVT);
2583 SDValue DenormalExpBias =
2584 DAG.getNode(ISD::SELECT, dl, ExpVT, IsDenormal, DenormalOffset, Zero);
2585
2586 SDValue MaskedFractAsInt =
2587 DAG.getNode(ISD::AND, dl, AsIntVT, ScaledSelect, FractSignMask);
2588 const APFloat Half(FltSem, "0.5");
2589 SDValue FPHalf = DAG.getConstant(Half.bitcastToAPInt(), dl, AsIntVT);
2590 SDValue Or = DAG.getNode(ISD::OR, dl, AsIntVT, MaskedFractAsInt, FPHalf);
2591 SDValue MaskedFract = DAG.getNode(ISD::BITCAST, dl, VT, Or);
2592
2593 SDValue ComputedExp =
2594 DAG.getNode(ISD::ADD, dl, ExpVT, NormalBiasedExp, DenormalExpBias);
2595
2596 SDValue Result0 =
2597 DAG.getNode(ISD::SELECT, dl, VT, DenormOrZero, Val, MaskedFract);
2598
2599 SDValue Result1 =
2600 DAG.getNode(ISD::SELECT, dl, ExpVT, DenormOrZero, Zero, ComputedExp);
2601
2602 return DAG.getMergeValues({Result0, Result1}, dl);
2603}
2604
2605/// This function is responsible for legalizing a
2606/// INT_TO_FP operation of the specified operand when the target requests that
2607/// we expand it. At this point, we know that the result and operand types are
2608/// legal for the target.
2609SDValue SelectionDAGLegalize::ExpandLegalINT_TO_FP(SDNode *Node,
2610 SDValue &Chain) {
2611 bool isSigned = (Node->getOpcode() == ISD::STRICT_SINT_TO_FP ||
2612 Node->getOpcode() == ISD::SINT_TO_FP);
2613 EVT DestVT = Node->getValueType(0);
2614 SDLoc dl(Node);
2615 unsigned OpNo = Node->isStrictFPOpcode() ? 1 : 0;
2616 SDValue Op0 = Node->getOperand(OpNo);
2617 EVT SrcVT = Op0.getValueType();
2618
2619 // TODO: Should any fast-math-flags be set for the created nodes?
2620 LLVM_DEBUG(dbgs() << "Legalizing INT_TO_FP\n");
2621 if (SrcVT == MVT::i32 && TLI.isTypeLegal(MVT::f64) &&
2622 (DestVT.bitsLE(MVT::f64) ||
2623 TLI.isOperationLegal(Node->isStrictFPOpcode() ? ISD::STRICT_FP_EXTEND
2625 DestVT))) {
2626 LLVM_DEBUG(dbgs() << "32-bit [signed|unsigned] integer to float/double "
2627 "expansion\n");
2628
2629 // Get the stack frame index of a 8 byte buffer.
2630 SDValue StackSlot = DAG.CreateStackTemporary(MVT::f64);
2631
2632 SDValue Lo = Op0;
2633 // if signed map to unsigned space
2634 if (isSigned) {
2635 // Invert sign bit (signed to unsigned mapping).
2636 Lo = DAG.getNode(ISD::XOR, dl, MVT::i32, Lo,
2637 DAG.getConstant(0x80000000u, dl, MVT::i32));
2638 }
2639 // Initial hi portion of constructed double.
2640 SDValue Hi = DAG.getConstant(0x43300000u, dl, MVT::i32);
2641
2642 // If this a big endian target, swap the lo and high data.
2643 if (DAG.getDataLayout().isBigEndian())
2644 std::swap(Lo, Hi);
2645
2646 SDValue MemChain = DAG.getEntryNode();
2647
2648 // Store the lo of the constructed double.
2649 SDValue Store1 = DAG.getStore(MemChain, dl, Lo, StackSlot,
2651 // Store the hi of the constructed double.
2652 SDValue HiPtr =
2653 DAG.getMemBasePlusOffset(StackSlot, TypeSize::getFixed(4), dl);
2654 SDValue Store2 =
2655 DAG.getStore(MemChain, dl, Hi, HiPtr, MachinePointerInfo());
2656 MemChain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Store1, Store2);
2657
2658 // load the constructed double
2659 SDValue Load =
2660 DAG.getLoad(MVT::f64, dl, MemChain, StackSlot, MachinePointerInfo());
2661 // FP constant to bias correct the final result
2662 SDValue Bias = DAG.getConstantFP(
2663 isSigned ? llvm::bit_cast<double>(0x4330000080000000ULL)
2664 : llvm::bit_cast<double>(0x4330000000000000ULL),
2665 dl, MVT::f64);
2666 // Subtract the bias and get the final result.
2667 SDValue Sub;
2669 if (Node->isStrictFPOpcode()) {
2670 Sub = DAG.getNode(ISD::STRICT_FSUB, dl, {MVT::f64, MVT::Other},
2671 {Node->getOperand(0), Load, Bias});
2672 Chain = Sub.getValue(1);
2673 if (DestVT != Sub.getValueType()) {
2674 std::pair<SDValue, SDValue> ResultPair;
2675 ResultPair =
2676 DAG.getStrictFPExtendOrRound(Sub, Chain, dl, DestVT);
2677 Result = ResultPair.first;
2678 Chain = ResultPair.second;
2679 }
2680 else
2681 Result = Sub;
2682 } else {
2683 Sub = DAG.getNode(ISD::FSUB, dl, MVT::f64, Load, Bias);
2684 Result = DAG.getFPExtendOrRound(Sub, dl, DestVT);
2685 }
2686 return Result;
2687 }
2688
2689 if (isSigned)
2690 return SDValue();
2691
2692 // TODO: Generalize this for use with other types.
2693 if (((SrcVT == MVT::i32 || SrcVT == MVT::i64) && DestVT == MVT::f32) ||
2694 (SrcVT == MVT::i64 && DestVT == MVT::f64)) {
2695 LLVM_DEBUG(dbgs() << "Converting unsigned i32/i64 to f32/f64\n");
2696 // For unsigned conversions, convert them to signed conversions using the
2697 // algorithm from the x86_64 __floatundisf in compiler_rt. That method
2698 // should be valid for i32->f32 as well.
2699
2700 // More generally this transform should be valid if there are 3 more bits
2701 // in the integer type than the significand. Rounding uses the first bit
2702 // after the width of the significand and the OR of all bits after that. So
2703 // we need to be able to OR the shifted out bit into one of the bits that
2704 // participate in the OR.
2705
2706 // TODO: This really should be implemented using a branch rather than a
2707 // select. We happen to get lucky and machinesink does the right
2708 // thing most of the time. This would be a good candidate for a
2709 // pseudo-op, or, even better, for whole-function isel.
2710 EVT SetCCVT = getSetCCResultType(SrcVT);
2711
2712 SDValue SignBitTest = DAG.getSetCC(
2713 dl, SetCCVT, Op0, DAG.getConstant(0, dl, SrcVT), ISD::SETLT);
2714
2715 EVT ShiftVT = TLI.getShiftAmountTy(SrcVT, DAG.getDataLayout());
2716 SDValue ShiftConst = DAG.getConstant(1, dl, ShiftVT);
2717 SDValue Shr = DAG.getNode(ISD::SRL, dl, SrcVT, Op0, ShiftConst);
2718 SDValue AndConst = DAG.getConstant(1, dl, SrcVT);
2719 SDValue And = DAG.getNode(ISD::AND, dl, SrcVT, Op0, AndConst);
2720 SDValue Or = DAG.getNode(ISD::OR, dl, SrcVT, And, Shr);
2721
2722 SDValue Slow, Fast;
2723 if (Node->isStrictFPOpcode()) {
2724 // In strict mode, we must avoid spurious exceptions, and therefore
2725 // must make sure to only emit a single STRICT_SINT_TO_FP.
2726 SDValue InCvt = DAG.getSelect(dl, SrcVT, SignBitTest, Or, Op0);
2727 Fast = DAG.getNode(ISD::STRICT_SINT_TO_FP, dl, { DestVT, MVT::Other },
2728 { Node->getOperand(0), InCvt });
2729 Slow = DAG.getNode(ISD::STRICT_FADD, dl, { DestVT, MVT::Other },
2730 { Fast.getValue(1), Fast, Fast });
2731 Chain = Slow.getValue(1);
2732 // The STRICT_SINT_TO_FP inherits the exception mode from the
2733 // incoming STRICT_UINT_TO_FP node; the STRICT_FADD node can
2734 // never raise any exception.
2736 Flags.setNoFPExcept(Node->getFlags().hasNoFPExcept());
2737 Fast->setFlags(Flags);
2738 Flags.setNoFPExcept(true);
2739 Slow->setFlags(Flags);
2740 } else {
2741 SDValue SignCvt = DAG.getNode(ISD::SINT_TO_FP, dl, DestVT, Or);
2742 Slow = DAG.getNode(ISD::FADD, dl, DestVT, SignCvt, SignCvt);
2743 Fast = DAG.getNode(ISD::SINT_TO_FP, dl, DestVT, Op0);
2744 }
2745
2746 return DAG.getSelect(dl, DestVT, SignBitTest, Slow, Fast);
2747 }
2748
2749 // Don't expand it if there isn't cheap fadd.
2750 if (!TLI.isOperationLegalOrCustom(
2751 Node->isStrictFPOpcode() ? ISD::STRICT_FADD : ISD::FADD, DestVT))
2752 return SDValue();
2753
2754 // The following optimization is valid only if every value in SrcVT (when
2755 // treated as signed) is representable in DestVT. Check that the mantissa
2756 // size of DestVT is >= than the number of bits in SrcVT -1.
2757 assert(APFloat::semanticsPrecision(DAG.EVTToAPFloatSemantics(DestVT)) >=
2758 SrcVT.getSizeInBits() - 1 &&
2759 "Cannot perform lossless SINT_TO_FP!");
2760
2761 SDValue Tmp1;
2762 if (Node->isStrictFPOpcode()) {
2763 Tmp1 = DAG.getNode(ISD::STRICT_SINT_TO_FP, dl, { DestVT, MVT::Other },
2764 { Node->getOperand(0), Op0 });
2765 } else
2766 Tmp1 = DAG.getNode(ISD::SINT_TO_FP, dl, DestVT, Op0);
2767
2768 SDValue SignSet = DAG.getSetCC(dl, getSetCCResultType(SrcVT), Op0,
2769 DAG.getConstant(0, dl, SrcVT), ISD::SETLT);
2770 SDValue Zero = DAG.getIntPtrConstant(0, dl),
2771 Four = DAG.getIntPtrConstant(4, dl);
2772 SDValue CstOffset = DAG.getSelect(dl, Zero.getValueType(),
2773 SignSet, Four, Zero);
2774
2775 // If the sign bit of the integer is set, the large number will be treated
2776 // as a negative number. To counteract this, the dynamic code adds an
2777 // offset depending on the data type.
2778 uint64_t FF;
2779 switch (SrcVT.getSimpleVT().SimpleTy) {
2780 default:
2781 return SDValue();
2782 case MVT::i8 : FF = 0x43800000ULL; break; // 2^8 (as a float)
2783 case MVT::i16: FF = 0x47800000ULL; break; // 2^16 (as a float)
2784 case MVT::i32: FF = 0x4F800000ULL; break; // 2^32 (as a float)
2785 case MVT::i64: FF = 0x5F800000ULL; break; // 2^64 (as a float)
2786 }
2787 if (DAG.getDataLayout().isLittleEndian())
2788 FF <<= 32;
2789 Constant *FudgeFactor = ConstantInt::get(
2790 Type::getInt64Ty(*DAG.getContext()), FF);
2791
2792 SDValue CPIdx =
2793 DAG.getConstantPool(FudgeFactor, TLI.getPointerTy(DAG.getDataLayout()));
2794 Align Alignment = cast<ConstantPoolSDNode>(CPIdx)->getAlign();
2795 CPIdx = DAG.getNode(ISD::ADD, dl, CPIdx.getValueType(), CPIdx, CstOffset);
2796 Alignment = commonAlignment(Alignment, 4);
2797 SDValue FudgeInReg;
2798 if (DestVT == MVT::f32)
2799 FudgeInReg = DAG.getLoad(
2800 MVT::f32, dl, DAG.getEntryNode(), CPIdx,
2801 MachinePointerInfo::getConstantPool(DAG.getMachineFunction()),
2802 Alignment);
2803 else {
2804 SDValue Load = DAG.getExtLoad(
2805 ISD::EXTLOAD, dl, DestVT, DAG.getEntryNode(), CPIdx,
2806 MachinePointerInfo::getConstantPool(DAG.getMachineFunction()), MVT::f32,
2807 Alignment);
2808 HandleSDNode Handle(Load);
2809 LegalizeOp(Load.getNode());
2810 FudgeInReg = Handle.getValue();
2811 }
2812
2813 if (Node->isStrictFPOpcode()) {
2814 SDValue Result = DAG.getNode(ISD::STRICT_FADD, dl, { DestVT, MVT::Other },
2815 { Tmp1.getValue(1), Tmp1, FudgeInReg });
2816 Chain = Result.getValue(1);
2817 return Result;
2818 }
2819
2820 return DAG.getNode(ISD::FADD, dl, DestVT, Tmp1, FudgeInReg);
2821}
2822
2823/// This function is responsible for legalizing a
2824/// *INT_TO_FP operation of the specified operand when the target requests that
2825/// we promote it. At this point, we know that the result and operand types are
2826/// legal for the target, and that there is a legal UINT_TO_FP or SINT_TO_FP
2827/// operation that takes a larger input.
2828void SelectionDAGLegalize::PromoteLegalINT_TO_FP(
2830 bool IsStrict = N->isStrictFPOpcode();
2831 bool IsSigned = N->getOpcode() == ISD::SINT_TO_FP ||
2832 N->getOpcode() == ISD::STRICT_SINT_TO_FP;
2833 EVT DestVT = N->getValueType(0);
2834 SDValue LegalOp = N->getOperand(IsStrict ? 1 : 0);
2835 unsigned UIntOp = IsStrict ? ISD::STRICT_UINT_TO_FP : ISD::UINT_TO_FP;
2836 unsigned SIntOp = IsStrict ? ISD::STRICT_SINT_TO_FP : ISD::SINT_TO_FP;
2837
2838 // First step, figure out the appropriate *INT_TO_FP operation to use.
2839 EVT NewInTy = LegalOp.getValueType();
2840
2841 unsigned OpToUse = 0;
2842
2843 // Scan for the appropriate larger type to use.
2844 while (true) {
2845 NewInTy = (MVT::SimpleValueType)(NewInTy.getSimpleVT().SimpleTy+1);
2846 assert(NewInTy.isInteger() && "Ran out of possibilities!");
2847
2848 // If the target supports SINT_TO_FP of this type, use it.
2849 if (TLI.isOperationLegalOrCustom(SIntOp, NewInTy)) {
2850 OpToUse = SIntOp;
2851 break;
2852 }
2853 if (IsSigned)
2854 continue;
2855
2856 // If the target supports UINT_TO_FP of this type, use it.
2857 if (TLI.isOperationLegalOrCustom(UIntOp, NewInTy)) {
2858 OpToUse = UIntOp;
2859 break;
2860 }
2861
2862 // Otherwise, try a larger type.
2863 }
2864
2865 // Okay, we found the operation and type to use. Zero extend our input to the
2866 // desired type then run the operation on it.
2867 if (IsStrict) {
2868 SDValue Res =
2869 DAG.getNode(OpToUse, dl, {DestVT, MVT::Other},
2870 {N->getOperand(0),
2871 DAG.getNode(IsSigned ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND,
2872 dl, NewInTy, LegalOp)});
2873 Results.push_back(Res);
2874 Results.push_back(Res.getValue(1));
2875 return;
2876 }
2877
2878 Results.push_back(
2879 DAG.getNode(OpToUse, dl, DestVT,
2880 DAG.getNode(IsSigned ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND,
2881 dl, NewInTy, LegalOp)));
2882}
2883
2884/// This function is responsible for legalizing a
2885/// FP_TO_*INT operation of the specified operand when the target requests that
2886/// we promote it. At this point, we know that the result and operand types are
2887/// legal for the target, and that there is a legal FP_TO_UINT or FP_TO_SINT
2888/// operation that returns a larger result.
2889void SelectionDAGLegalize::PromoteLegalFP_TO_INT(SDNode *N, const SDLoc &dl,
2891 bool IsStrict = N->isStrictFPOpcode();
2892 bool IsSigned = N->getOpcode() == ISD::FP_TO_SINT ||
2893 N->getOpcode() == ISD::STRICT_FP_TO_SINT;
2894 EVT DestVT = N->getValueType(0);
2895 SDValue LegalOp = N->getOperand(IsStrict ? 1 : 0);
2896 // First step, figure out the appropriate FP_TO*INT operation to use.
2897 EVT NewOutTy = DestVT;
2898
2899 unsigned OpToUse = 0;
2900
2901 // Scan for the appropriate larger type to use.
2902 while (true) {
2903 NewOutTy = (MVT::SimpleValueType)(NewOutTy.getSimpleVT().SimpleTy+1);
2904 assert(NewOutTy.isInteger() && "Ran out of possibilities!");
2905
2906 // A larger signed type can hold all unsigned values of the requested type,
2907 // so using FP_TO_SINT is valid
2908 OpToUse = IsStrict ? ISD::STRICT_FP_TO_SINT : ISD::FP_TO_SINT;
2909 if (TLI.isOperationLegalOrCustom(OpToUse, NewOutTy))
2910 break;
2911
2912 // However, if the value may be < 0.0, we *must* use some FP_TO_SINT.
2913 OpToUse = IsStrict ? ISD::STRICT_FP_TO_UINT : ISD::FP_TO_UINT;
2914 if (!IsSigned && TLI.isOperationLegalOrCustom(OpToUse, NewOutTy))
2915 break;
2916
2917 // Otherwise, try a larger type.
2918 }
2919
2920 // Okay, we found the operation and type to use.
2922 if (IsStrict) {
2923 SDVTList VTs = DAG.getVTList(NewOutTy, MVT::Other);
2924 Operation = DAG.getNode(OpToUse, dl, VTs, N->getOperand(0), LegalOp);
2925 } else
2926 Operation = DAG.getNode(OpToUse, dl, NewOutTy, LegalOp);
2927
2928 // Truncate the result of the extended FP_TO_*INT operation to the desired
2929 // size.
2930 SDValue Trunc = DAG.getNode(ISD::TRUNCATE, dl, DestVT, Operation);
2931 Results.push_back(Trunc);
2932 if (IsStrict)
2933 Results.push_back(Operation.getValue(1));
2934}
2935
2936/// Promote FP_TO_*INT_SAT operation to a larger result type. At this point
2937/// the result and operand types are legal and there must be a legal
2938/// FP_TO_*INT_SAT operation for a larger result type.
2939SDValue SelectionDAGLegalize::PromoteLegalFP_TO_INT_SAT(SDNode *Node,
2940 const SDLoc &dl) {
2941 unsigned Opcode = Node->getOpcode();
2942
2943 // Scan for the appropriate larger type to use.
2944 EVT NewOutTy = Node->getValueType(0);
2945 while (true) {
2946 NewOutTy = (MVT::SimpleValueType)(NewOutTy.getSimpleVT().SimpleTy + 1);
2947 assert(NewOutTy.isInteger() && "Ran out of possibilities!");
2948
2949 if (TLI.isOperationLegalOrCustom(Opcode, NewOutTy))
2950 break;
2951 }
2952
2953 // Saturation width is determined by second operand, so we don't have to
2954 // perform any fixup and can directly truncate the result.
2955 SDValue Result = DAG.getNode(Opcode, dl, NewOutTy, Node->getOperand(0),
2956 Node->getOperand(1));
2957 return DAG.getNode(ISD::TRUNCATE, dl, Node->getValueType(0), Result);
2958}
2959
2960/// Open code the operations for PARITY of the specified operation.
2961SDValue SelectionDAGLegalize::ExpandPARITY(SDValue Op, const SDLoc &dl) {
2962 EVT VT = Op.getValueType();
2963 EVT ShVT = TLI.getShiftAmountTy(VT, DAG.getDataLayout());
2964 unsigned Sz = VT.getScalarSizeInBits();
2965
2966 // If CTPOP is legal, use it. Otherwise use shifts and xor.
2968 if (TLI.isOperationLegalOrPromote(ISD::CTPOP, VT)) {
2969 Result = DAG.getNode(ISD::CTPOP, dl, VT, Op);
2970 } else {
2971 Result = Op;
2972 for (unsigned i = Log2_32_Ceil(Sz); i != 0;) {
2973 SDValue Shift = DAG.getNode(ISD::SRL, dl, VT, Result,
2974 DAG.getConstant(1ULL << (--i), dl, ShVT));
2975 Result = DAG.getNode(ISD::XOR, dl, VT, Result, Shift);
2976 }
2977 }
2978
2979 return DAG.getNode(ISD::AND, dl, VT, Result, DAG.getConstant(1, dl, VT));
2980}
2981
2982bool SelectionDAGLegalize::ExpandNode(SDNode *Node) {
2983 LLVM_DEBUG(dbgs() << "Trying to expand node\n");
2985 SDLoc dl(Node);
2986 SDValue Tmp1, Tmp2, Tmp3, Tmp4;
2987 bool NeedInvert;
2988 switch (Node->getOpcode()) {
2989 case ISD::ABS:
2990 if ((Tmp1 = TLI.expandABS(Node, DAG)))
2991 Results.push_back(Tmp1);
2992 break;
2993 case ISD::ABDS:
2994 case ISD::ABDU:
2995 if ((Tmp1 = TLI.expandABD(Node, DAG)))
2996 Results.push_back(Tmp1);
2997 break;
2998 case ISD::CTPOP:
2999 if ((Tmp1 = TLI.expandCTPOP(Node, DAG)))
3000 Results.push_back(Tmp1);
3001 break;
3002 case ISD::CTLZ:
3004 if ((Tmp1 = TLI.expandCTLZ(Node, DAG)))
3005 Results.push_back(Tmp1);
3006 break;
3007 case ISD::CTTZ:
3009 if ((Tmp1 = TLI.expandCTTZ(Node, DAG)))
3010 Results.push_back(Tmp1);
3011 break;
3012 case ISD::BITREVERSE:
3013 if ((Tmp1 = TLI.expandBITREVERSE(Node, DAG)))
3014 Results.push_back(Tmp1);
3015 break;
3016 case ISD::BSWAP:
3017 if ((Tmp1 = TLI.expandBSWAP(Node, DAG)))
3018 Results.push_back(Tmp1);
3019 break;
3020 case ISD::PARITY:
3021 Results.push_back(ExpandPARITY(Node->getOperand(0), dl));
3022 break;
3023 case ISD::FRAMEADDR:
3024 case ISD::RETURNADDR:
3026 Results.push_back(DAG.getConstant(0, dl, Node->getValueType(0)));
3027 break;
3028 case ISD::EH_DWARF_CFA: {
3029 SDValue CfaArg = DAG.getSExtOrTrunc(Node->getOperand(0), dl,
3030 TLI.getPointerTy(DAG.getDataLayout()));
3031 SDValue Offset = DAG.getNode(ISD::ADD, dl,
3032 CfaArg.getValueType(),
3033 DAG.getNode(ISD::FRAME_TO_ARGS_OFFSET, dl,
3034 CfaArg.getValueType()),
3035 CfaArg);
3036 SDValue FA = DAG.getNode(
3037 ISD::FRAMEADDR, dl, TLI.getPointerTy(DAG.getDataLayout()),
3038 DAG.getConstant(0, dl, TLI.getPointerTy(DAG.getDataLayout())));
3039 Results.push_back(DAG.getNode(ISD::ADD, dl, FA.getValueType(),
3040 FA, Offset));
3041 break;
3042 }
3043 case ISD::GET_ROUNDING:
3044 Results.push_back(DAG.getConstant(1, dl, Node->getValueType(0)));
3045 Results.push_back(Node->getOperand(0));
3046 break;
3047 case ISD::EH_RETURN:
3048 case ISD::EH_LABEL:
3049 case ISD::PREFETCH:
3050 case ISD::VAEND:
3052 // If the target didn't expand these, there's nothing to do, so just
3053 // preserve the chain and be done.
3054 Results.push_back(Node->getOperand(0));
3055 break;
3058 // If the target didn't expand this, just return 'zero' and preserve the
3059 // chain.
3060 Results.append(Node->getNumValues() - 1,
3061 DAG.getConstant(0, dl, Node->getValueType(0)));
3062 Results.push_back(Node->getOperand(0));
3063 break;
3065 // If the target didn't expand this, just return 'zero' and preserve the
3066 // chain.
3067 Results.push_back(DAG.getConstant(0, dl, MVT::i32));
3068 Results.push_back(Node->getOperand(0));
3069 break;
3070 case ISD::ATOMIC_LOAD: {
3071 // There is no libcall for atomic load; fake it with ATOMIC_CMP_SWAP.
3072 SDValue Zero = DAG.getConstant(0, dl, Node->getValueType(0));
3073 SDVTList VTs = DAG.getVTList(Node->getValueType(0), MVT::Other);
3074 SDValue Swap = DAG.getAtomicCmpSwap(
3075 ISD::ATOMIC_CMP_SWAP, dl, cast<AtomicSDNode>(Node)->getMemoryVT(), VTs,
3076 Node->getOperand(0), Node->getOperand(1), Zero, Zero,
3077 cast<AtomicSDNode>(Node)->getMemOperand());
3078 Results.push_back(Swap.getValue(0));
3079 Results.push_back(Swap.getValue(1));
3080 break;
3081 }
3082 case ISD::ATOMIC_STORE: {
3083 // There is no libcall for atomic store; fake it with ATOMIC_SWAP.
3084 SDValue Swap = DAG.getAtomic(
3085 ISD::ATOMIC_SWAP, dl, cast<AtomicSDNode>(Node)->getMemoryVT(),
3086 Node->getOperand(0), Node->getOperand(2), Node->getOperand(1),
3087 cast<AtomicSDNode>(Node)->getMemOperand());
3088 Results.push_back(Swap.getValue(1));
3089 break;
3090 }
3092 // Expanding an ATOMIC_CMP_SWAP_WITH_SUCCESS produces an ATOMIC_CMP_SWAP and
3093 // splits out the success value as a comparison. Expanding the resulting
3094 // ATOMIC_CMP_SWAP will produce a libcall.
3095 SDVTList VTs = DAG.getVTList(Node->getValueType(0), MVT::Other);
3096 SDValue Res = DAG.getAtomicCmpSwap(
3097 ISD::ATOMIC_CMP_SWAP, dl, cast<AtomicSDNode>(Node)->getMemoryVT(), VTs,
3098 Node->getOperand(0), Node->getOperand(1), Node->getOperand(2),
3099 Node->getOperand(3), cast<MemSDNode>(Node)->getMemOperand());
3100
3101 SDValue ExtRes = Res;
3102 SDValue LHS = Res;
3103 SDValue RHS = Node->getOperand(1);
3104
3105 EVT AtomicType = cast<AtomicSDNode>(Node)->getMemoryVT();
3106 EVT OuterType = Node->getValueType(0);
3107 switch (TLI.getExtendForAtomicOps()) {
3108 case ISD::SIGN_EXTEND:
3109 LHS = DAG.getNode(ISD::AssertSext, dl, OuterType, Res,
3110 DAG.getValueType(AtomicType));
3111 RHS = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, OuterType,
3112 Node->getOperand(2), DAG.getValueType(AtomicType));
3113 ExtRes = LHS;
3114 break;
3115 case ISD::ZERO_EXTEND:
3116 LHS = DAG.getNode(ISD::AssertZext, dl, OuterType, Res,
3117 DAG.getValueType(AtomicType));
3118 RHS = DAG.getZeroExtendInReg(Node->getOperand(2), dl, AtomicType);
3119 ExtRes = LHS;
3120 break;
3121 case ISD::ANY_EXTEND:
3122 LHS = DAG.getZeroExtendInReg(Res, dl, AtomicType);
3123 RHS = DAG.getZeroExtendInReg(Node->getOperand(2), dl, AtomicType);
3124 break;
3125 default:
3126 llvm_unreachable("Invalid atomic op extension");
3127 }
3128
3130 DAG.getSetCC(dl, Node->getValueType(1), LHS, RHS, ISD::SETEQ);
3131
3132 Results.push_back(ExtRes.getValue(0));
3133 Results.push_back(Success);
3134 Results.push_back(Res.getValue(1));
3135 break;
3136 }
3137 case ISD::ATOMIC_LOAD_SUB: {
3138 SDLoc DL(Node);
3139 EVT VT = Node->getValueType(0);
3140 SDValue RHS = Node->getOperand(2);
3141 AtomicSDNode *AN = cast<AtomicSDNode>(Node);
3142 if (RHS->getOpcode() == ISD::SIGN_EXTEND_INREG &&
3143 cast<VTSDNode>(RHS->getOperand(1))->getVT() == AN->getMemoryVT())
3144 RHS = RHS->getOperand(0);
3145 SDValue NewRHS =
3146 DAG.getNode(ISD::SUB, DL, VT, DAG.getConstant(0, DL, VT), RHS);
3147 SDValue Res = DAG.getAtomic(ISD::ATOMIC_LOAD_ADD, DL, AN->getMemoryVT(),
3148 Node->getOperand(0), Node->getOperand(1),
3149 NewRHS, AN->getMemOperand());
3150 Results.push_back(Res);
3151 Results.push_back(Res.getValue(1));
3152 break;
3153 }
3155 ExpandDYNAMIC_STACKALLOC(Node, Results);
3156 break;
3157 case ISD::MERGE_VALUES:
3158 for (unsigned i = 0; i < Node->getNumValues(); i++)
3159 Results.push_back(Node->getOperand(i));
3160 break;
3161 case ISD::UNDEF: {
3162 EVT VT = Node->getValueType(0);
3163 if (VT.isInteger())
3164 Results.push_back(DAG.getConstant(0, dl, VT));
3165 else {
3166 assert(VT.isFloatingPoint() && "Unknown value type!");
3167 Results.push_back(DAG.getConstantFP(0, dl, VT));
3168 }
3169 break;
3170 }
3172 // When strict mode is enforced we can't do expansion because it
3173 // does not honor the "strict" properties. Only libcall is allowed.
3174 if (TLI.isStrictFPEnabled())
3175 break;
3176 // We might as well mutate to FP_ROUND when FP_ROUND operation is legal
3177 // since this operation is more efficient than stack operation.
3178 if (TLI.getStrictFPOperationAction(Node->getOpcode(),
3179 Node->getValueType(0))
3180 == TargetLowering::Legal)
3181 break;
3182 // We fall back to use stack operation when the FP_ROUND operation
3183 // isn't available.
3184 if ((Tmp1 = EmitStackConvert(Node->getOperand(1), Node->getValueType(0),
3185 Node->getValueType(0), dl,
3186 Node->getOperand(0)))) {
3187 ReplaceNode(Node, Tmp1.getNode());
3188 LLVM_DEBUG(dbgs() << "Successfully expanded STRICT_FP_ROUND node\n");
3189 return true;
3190 }
3191 break;
3192 case ISD::FP_ROUND: {
3193 if ((Tmp1 = TLI.expandFP_ROUND(Node, DAG))) {
3194 Results.push_back(Tmp1);
3195 break;
3196 }
3197
3199 }
3200 case ISD::BITCAST:
3201 if ((Tmp1 = EmitStackConvert(Node->getOperand(0), Node->getValueType(0),
3202 Node->getValueType(0), dl)))
3203 Results.push_back(Tmp1);
3204 break;
3206 // When strict mode is enforced we can't do expansion because it
3207 // does not honor the "strict" properties. Only libcall is allowed.
3208 if (TLI.isStrictFPEnabled())
3209 break;
3210 // We might as well mutate to FP_EXTEND when FP_EXTEND operation is legal
3211 // since this operation is more efficient than stack operation.
3212 if (TLI.getStrictFPOperationAction(Node->getOpcode(),
3213 Node->getValueType(0))
3214 == TargetLowering::Legal)
3215 break;
3216 // We fall back to use stack operation when the FP_EXTEND operation
3217 // isn't available.
3218 if ((Tmp1 = EmitStackConvert(
3219 Node->getOperand(1), Node->getOperand(1).getValueType(),
3220 Node->getValueType(0), dl, Node->getOperand(0)))) {
3221 ReplaceNode(Node, Tmp1.getNode());
3222 LLVM_DEBUG(dbgs() << "Successfully expanded STRICT_FP_EXTEND node\n");
3223 return true;
3224 }
3225 break;
3226 case ISD::FP_EXTEND: {
3227 SDValue Op = Node->getOperand(0);
3228 EVT SrcVT = Op.getValueType();
3229 EVT DstVT = Node->getValueType(0);
3230 if (SrcVT.getScalarType() == MVT::bf16) {
3231 Results.push_back(DAG.getNode(ISD::BF16_TO_FP, SDLoc(Node), DstVT, Op));
3232 break;
3233 }
3234
3235 if ((Tmp1 = EmitStackConvert(Op, SrcVT, DstVT, dl)))
3236 Results.push_back(Tmp1);
3237 break;
3238 }
3239 case ISD::BF16_TO_FP: {
3240 // Always expand bf16 to f32 casts, they lower to ext + shift.
3241 //
3242 // Note that the operand of this code can be bf16 or an integer type in case
3243 // bf16 is not supported on the target and was softened.
3244 SDValue Op = Node->getOperand(0);
3245 if (Op.getValueType() == MVT::bf16) {
3246 Op = DAG.getNode(ISD::ANY_EXTEND, dl, MVT::i32,
3247 DAG.getNode(ISD::BITCAST, dl, MVT::i16, Op));
3248 } else {
3249 Op = DAG.getAnyExtOrTrunc(Op, dl, MVT::i32);
3250 }
3251 Op = DAG.getNode(
3252 ISD::SHL, dl, MVT::i32, Op,
3253 DAG.getConstant(16, dl,
3254 TLI.getShiftAmountTy(MVT::i32, DAG.getDataLayout())));
3255 Op = DAG.getNode(ISD::BITCAST, dl, MVT::f32, Op);
3256 // Add fp_extend in case the output is bigger than f32.
3257 if (Node->getValueType(0) != MVT::f32)
3258 Op = DAG.getNode(ISD::FP_EXTEND, dl, Node->getValueType(0), Op);
3259 Results.push_back(Op);
3260 break;
3261 }
3262 case ISD::FP_TO_BF16: {
3263 SDValue Op = Node->getOperand(0);
3264 if (Op.getValueType() != MVT::f32)
3265 Op = DAG.getNode(ISD::FP_ROUND, dl, MVT::f32, Op,
3266 DAG.getIntPtrConstant(0, dl, /*isTarget=*/true));
3267 // Certain SNaNs will turn into infinities if we do a simple shift right.
3268 if (!DAG.isKnownNeverSNaN(Op)) {
3269 Op = DAG.getNode(ISD::FCANONICALIZE, dl, MVT::f32, Op, Node->getFlags());
3270 }
3271 Op = DAG.getNode(
3272 ISD::SRL, dl, MVT::i32, DAG.getNode(ISD::BITCAST, dl, MVT::i32, Op),
3273 DAG.getConstant(16, dl,
3274 TLI.getShiftAmountTy(MVT::i32, DAG.getDataLayout())));
3275 // The result of this node can be bf16 or an integer type in case bf16 is
3276 // not supported on the target and was softened to i16 for storage.
3277 if (Node->getValueType(0) == MVT::bf16) {
3278 Op = DAG.getNode(ISD::BITCAST, dl, MVT::bf16,
3279 DAG.getNode(ISD::TRUNCATE, dl, MVT::i16, Op));
3280 } else {
3281 Op = DAG.getAnyExtOrTrunc(Op, dl, Node->getValueType(0));
3282 }
3283 Results.push_back(Op);
3284 break;
3285 }
3287 EVT ExtraVT = cast<VTSDNode>(Node->getOperand(1))->getVT();
3288 EVT VT = Node->getValueType(0);
3289
3290 // An in-register sign-extend of a boolean is a negation:
3291 // 'true' (1) sign-extended is -1.
3292 // 'false' (0) sign-extended is 0.
3293 // However, we must mask the high bits of the source operand because the
3294 // SIGN_EXTEND_INREG does not guarantee that the high bits are already zero.
3295
3296 // TODO: Do this for vectors too?
3297 if (ExtraVT.isScalarInteger() && ExtraVT.getSizeInBits() == 1) {
3298 SDValue One = DAG.getConstant(1, dl, VT);
3299 SDValue And = DAG.getNode(ISD::AND, dl, VT, Node->getOperand(0), One);
3300 SDValue Zero = DAG.getConstant(0, dl, VT);
3301 SDValue Neg = DAG.getNode(ISD::SUB, dl, VT, Zero, And);
3302 Results.push_back(Neg);
3303 break;
3304 }
3305
3306 // NOTE: we could fall back on load/store here too for targets without
3307 // SRA. However, it is doubtful that any exist.
3308 EVT ShiftAmountTy = TLI.getShiftAmountTy(VT, DAG.getDataLayout());
3309 unsigned BitsDiff = VT.getScalarSizeInBits() -
3310 ExtraVT.getScalarSizeInBits();
3311 SDValue ShiftCst = DAG.getConstant(BitsDiff, dl, ShiftAmountTy);
3312 Tmp1 = DAG.getNode(ISD::SHL, dl, Node->getValueType(0),
3313 Node->getOperand(0), ShiftCst);
3314 Tmp1 = DAG.getNode(ISD::SRA, dl, Node->getValueType(0), Tmp1, ShiftCst);
3315 Results.push_back(Tmp1);
3316 break;
3317 }
3318 case ISD::UINT_TO_FP:
3320 if (TLI.expandUINT_TO_FP(Node, Tmp1, Tmp2, DAG)) {
3321 Results.push_back(Tmp1);
3322 if (Node->isStrictFPOpcode())
3323 Results.push_back(Tmp2);
3324 break;
3325 }
3326 [[fallthrough]];
3327 case ISD::SINT_TO_FP:
3329 if ((Tmp1 = ExpandLegalINT_TO_FP(Node, Tmp2))) {
3330 Results.push_back(Tmp1);
3331 if (Node->isStrictFPOpcode())
3332 Results.push_back(Tmp2);
3333 }
3334 break;
3335 case ISD::FP_TO_SINT:
3336 if (TLI.expandFP_TO_SINT(Node, Tmp1, DAG))
3337 Results.push_back(Tmp1);
3338 break;
3340 if (TLI.expandFP_TO_SINT(Node, Tmp1, DAG)) {
3341 ReplaceNode(Node, Tmp1.getNode());
3342 LLVM_DEBUG(dbgs() << "Successfully expanded STRICT_FP_TO_SINT node\n");
3343 return true;
3344 }
3345 break;
3346 case ISD::FP_TO_UINT:
3347 if (TLI.expandFP_TO_UINT(Node, Tmp1, Tmp2, DAG))
3348 Results.push_back(Tmp1);
3349 break;
3351 if (TLI.expandFP_TO_UINT(Node, Tmp1, Tmp2, DAG)) {
3352 // Relink the chain.
3353 DAG.ReplaceAllUsesOfValueWith(SDValue(Node,1), Tmp2);
3354 // Replace the new UINT result.
3355 ReplaceNodeWithValue(SDValue(Node, 0), Tmp1);
3356 LLVM_DEBUG(dbgs() << "Successfully expanded STRICT_FP_TO_UINT node\n");
3357 return true;
3358 }
3359 break;
3362 Results.push_back(TLI.expandFP_TO_INT_SAT(Node, DAG));
3363 break;
3364 case ISD::VAARG:
3365 Results.push_back(DAG.expandVAArg(Node));
3366 Results.push_back(Results[0].getValue(1));
3367 break;
3368 case ISD::VACOPY:
3369 Results.push_back(DAG.expandVACopy(Node));
3370 break;
3372 if (Node->getOperand(0).getValueType().getVectorElementCount().isScalar())
3373 // This must be an access of the only element. Return it.
3374 Tmp1 = DAG.getNode(ISD::BITCAST, dl, Node->getValueType(0),
3375 Node->getOperand(0));
3376 else
3377 Tmp1 = ExpandExtractFromVectorThroughStack(SDValue(Node, 0));
3378 Results.push_back(Tmp1);
3379 break;
3381 Results.push_back(ExpandExtractFromVectorThroughStack(SDValue(Node, 0)));
3382 break;
3384 Results.push_back(ExpandInsertToVectorThroughStack(SDValue(Node, 0)));
3385 break;
3387 Results.push_back(ExpandVectorBuildThroughStack(Node));
3388 break;
3390 Results.push_back(ExpandSCALAR_TO_VECTOR(Node));
3391 break;
3393 Results.push_back(ExpandINSERT_VECTOR_ELT(SDValue(Node, 0)));
3394 break;
3395 case ISD::VECTOR_SHUFFLE: {
3396 SmallVector<int, 32> NewMask;
3397 ArrayRef<int> Mask = cast<ShuffleVectorSDNode>(Node)->getMask();
3398
3399 EVT VT = Node->getValueType(0);
3400 EVT EltVT = VT.getVectorElementType();
3401 SDValue Op0 = Node->getOperand(0);
3402 SDValue Op1 = Node->getOperand(1);
3403 if (!TLI.isTypeLegal(EltVT)) {
3404 EVT NewEltVT = TLI.getTypeToTransformTo(*DAG.getContext(), EltVT);
3405
3406 // BUILD_VECTOR operands are allowed to be wider than the element type.
3407 // But if NewEltVT is smaller that EltVT the BUILD_VECTOR does not accept
3408 // it.
3409 if (NewEltVT.bitsLT(EltVT)) {
3410 // Convert shuffle node.
3411 // If original node was v4i64 and the new EltVT is i32,
3412 // cast operands to v8i32 and re-build the mask.
3413
3414 // Calculate new VT, the size of the new VT should be equal to original.
3415 EVT NewVT =
3416 EVT::getVectorVT(*DAG.getContext(), NewEltVT,
3417 VT.getSizeInBits() / NewEltVT.getSizeInBits());
3418 assert(NewVT.bitsEq(VT));
3419
3420 // cast operands to new VT
3421 Op0 = DAG.getNode(ISD::BITCAST, dl, NewVT, Op0);
3422 Op1 = DAG.getNode(ISD::BITCAST, dl, NewVT, Op1);
3423
3424 // Convert the shuffle mask
3425 unsigned int factor =
3427
3428 // EltVT gets smaller
3429 assert(factor > 0);
3430
3431 for (unsigned i = 0; i < VT.getVectorNumElements(); ++i) {
3432 if (Mask[i] < 0) {
3433 for (unsigned fi = 0; fi < factor; ++fi)
3434 NewMask.push_back(Mask[i]);
3435 }
3436 else {
3437 for (unsigned fi = 0; fi < factor; ++fi)
3438 NewMask.push_back(Mask[i]*factor+fi);
3439 }
3440 }
3441 Mask = NewMask;
3442 VT = NewVT;
3443 }
3444 EltVT = NewEltVT;
3445 }
3446 unsigned NumElems = VT.getVectorNumElements();
3448 for (unsigned i = 0; i != NumElems; ++i) {
3449 if (Mask[i] < 0) {
3450 Ops.push_back(DAG.getUNDEF(EltVT));
3451 continue;
3452 }
3453 unsigned Idx = Mask[i];
3454 if (Idx < NumElems)
3455 Ops.push_back(DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT, Op0,
3456 DAG.getVectorIdxConstant(Idx, dl)));
3457 else
3458 Ops.push_back(
3459 DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT, Op1,
3460 DAG.getVectorIdxConstant(Idx - NumElems, dl)));
3461 }
3462
3463 Tmp1 = DAG.getBuildVector(VT, dl, Ops);
3464 // We may have changed the BUILD_VECTOR type. Cast it back to the Node type.
3465 Tmp1 = DAG.getNode(ISD::BITCAST, dl, Node->getValueType(0), Tmp1);
3466 Results.push_back(Tmp1);
3467 break;
3468 }
3469 case ISD::VECTOR_SPLICE: {
3470 Results.push_back(TLI.expandVectorSplice(Node, DAG));
3471 break;
3472 }
3473 case ISD::EXTRACT_ELEMENT: {
3474 EVT OpTy = Node->getOperand(0).getValueType();
3475 if (Node->getConstantOperandVal(1)) {
3476 // 1 -> Hi
3477 Tmp1 = DAG.getNode(ISD::SRL, dl, OpTy, Node->getOperand(0),
3478 DAG.getConstant(OpTy.getSizeInBits() / 2, dl,
3479 TLI.getShiftAmountTy(
3480 Node->getOperand(0).getValueType(),
3481 DAG.getDataLayout())));
3482 Tmp1 = DAG.getNode(ISD::TRUNCATE, dl, Node->getValueType(0), Tmp1);
3483 } else {
3484 // 0 -> Lo
3485 Tmp1 = DAG.getNode(ISD::TRUNCATE, dl, Node->getValueType(0),
3486 Node->getOperand(0));
3487 }
3488 Results.push_back(Tmp1);
3489 break;
3490 }
3491 case ISD::STACKSAVE:
3492 // Expand to CopyFromReg if the target set
3493 // StackPointerRegisterToSaveRestore.
3494 if (Register SP = TLI.getStackPointerRegisterToSaveRestore()) {
3495 Results.push_back(DAG.getCopyFromReg(Node->getOperand(0), dl, SP,
3496 Node->getValueType(0)));
3497 Results.push_back(Results[0].getValue(1));
3498 } else {
3499 Results.push_back(DAG.getUNDEF(Node->getValueType(0)));
3500 Results.push_back(Node->getOperand(0));
3501 }
3502 break;
3503 case ISD::STACKRESTORE:
3504 // Expand to CopyToReg if the target set
3505 // StackPointerRegisterToSaveRestore.
3506 if (Register SP = TLI.getStackPointerRegisterToSaveRestore()) {
3507 Results.push_back(DAG.getCopyToReg(Node->getOperand(0), dl, SP,
3508 Node->getOperand(1)));
3509 } else {
3510 Results.push_back(Node->getOperand(0));
3511 }
3512 break;
3514 Results.push_back(DAG.getConstant(0, dl, Node->getValueType(0)));
3515 Results.push_back(Results[0].getValue(0));
3516 break;
3517 case ISD::FCOPYSIGN:
3518 Results.push_back(ExpandFCOPYSIGN(Node));
3519 break;
3520 case ISD::FNEG:
3521 Results.push_back(ExpandFNEG(Node));
3522 break;
3523 case ISD::FABS:
3524 Results.push_back(ExpandFABS(Node));
3525 break;
3526 case ISD::IS_FPCLASS: {
3527 auto Test = static_cast<FPClassTest>(Node->getConstantOperandVal(1));
3528 if (SDValue Expanded =
3529 TLI.expandIS_FPCLASS(Node->getValueType(0), Node->getOperand(0),
3530 Test, Node->getFlags(), SDLoc(Node), DAG))
3531 Results.push_back(Expanded);
3532 break;
3533 }
3534 case ISD::SMIN:
3535 case ISD::SMAX:
3536 case ISD::UMIN:
3537 case ISD::UMAX: {
3538 // Expand Y = MAX(A, B) -> Y = (A > B) ? A : B
3539 ISD::CondCode Pred;
3540 switch (Node->getOpcode()) {
3541 default: llvm_unreachable("How did we get here?");
3542 case ISD::SMAX: Pred = ISD::SETGT; break;
3543 case ISD::SMIN: Pred = ISD::SETLT; break;
3544 case ISD::UMAX: Pred = ISD::SETUGT; break;
3545 case ISD::UMIN: Pred = ISD::SETULT; break;
3546 }
3547 Tmp1 = Node->getOperand(0);
3548 Tmp2 = Node->getOperand(1);
3549 Tmp1 = DAG.getSelectCC(dl, Tmp1, Tmp2, Tmp1, Tmp2, Pred);
3550 Results.push_back(Tmp1);
3551 break;
3552 }
3553 case ISD::FMINNUM:
3554 case ISD::FMAXNUM: {
3555 if (SDValue Expanded = TLI.expandFMINNUM_FMAXNUM(Node, DAG))
3556 Results.push_back(Expanded);
3557 break;
3558 }
3559 case ISD::FSIN:
3560 case ISD::FCOS: {
3561 EVT VT = Node->getValueType(0);
3562 // Turn fsin / fcos into ISD::FSINCOS node if there are a pair of fsin /
3563 // fcos which share the same operand and both are used.
3564 if ((TLI.isOperationLegalOrCustom(ISD::FSINCOS, VT) ||
3566 && useSinCos(Node)) {
3567 SDVTList VTs = DAG.getVTList(VT, VT);
3568 Tmp1 = DAG.getNode(ISD::FSINCOS, dl, VTs, Node->getOperand(0));
3569 if (Node->getOpcode() == ISD::FCOS)
3570 Tmp1 = Tmp1.getValue(1);
3571 Results.push_back(Tmp1);
3572 }
3573 break;
3574 }
3575 case ISD::FLDEXP:
3576 case ISD::STRICT_FLDEXP: {
3577 EVT VT = Node->getValueType(0);
3579 // Use the LibCall instead, it is very likely faster
3580 // FIXME: Use separate LibCall action.
3581 if (TLI.getLibcallName(LC))
3582 break;
3583
3584 if (SDValue Expanded = expandLdexp(Node)) {
3585 Results.push_back(Expanded);
3586 if (Node->getOpcode() == ISD::STRICT_FLDEXP)
3587 Results.push_back(Expanded.getValue(1));
3588 }
3589
3590 break;
3591 }
3592 case ISD::FFREXP: {
3593 RTLIB::Libcall LC = RTLIB::getFREXP(Node->getValueType(0));
3594 // Use the LibCall instead, it is very likely faster
3595 // FIXME: Use separate LibCall action.
3596 if (TLI.getLibcallName(LC))
3597 break;
3598
3599 if (SDValue Expanded = expandFrexp(Node)) {
3600 Results.push_back(Expanded);
3601 Results.push_back(Expanded.getValue(1));
3602 }
3603 break;
3604 }
3605 case ISD::FMAD:
3606 llvm_unreachable("Illegal fmad should never be formed");
3607
3608 case ISD::FP16_TO_FP:
3609 if (Node->getValueType(0) != MVT::f32) {
3610 // We can extend to types bigger than f32 in two steps without changing
3611 // the result. Since "f16 -> f32" is much more commonly available, give
3612 // CodeGen the option of emitting that before resorting to a libcall.
3613 SDValue Res =
3614 DAG.getNode(ISD::FP16_TO_FP, dl, MVT::f32, Node->getOperand(0));
3615 Results.push_back(
3616 DAG.getNode(ISD::FP_EXTEND, dl, Node->getValueType(0), Res));
3617 }
3618 break;
3621 if (Node->getValueType(0) != MVT::f32) {
3622 // We can extend to types bigger than f32 in two steps without changing
3623 // the result. Since "f16 -> f32" is much more commonly available, give
3624 // CodeGen the option of emitting that before resorting to a libcall.
3625 SDValue Res = DAG.getNode(Node->getOpcode(), dl, {MVT::f32, MVT::Other},
3626 {Node->getOperand(0), Node->getOperand(1)});
3627 Res = DAG.getNode(ISD::STRICT_FP_EXTEND, dl,
3628 {Node->getValueType(0), MVT::Other},
3629 {Res.getValue(1), Res});
3630 Results.push_back(Res);
3631 Results.push_back(Res.getValue(1));
3632 }
3633 break;
3634 case ISD::FP_TO_FP16:
3635 LLVM_DEBUG(dbgs() << "Legalizing FP_TO_FP16\n");
3636 if (!TLI.useSoftFloat() && TM.Options.UnsafeFPMath) {
3637 SDValue Op = Node->getOperand(0);
3638 MVT SVT = Op.getSimpleValueType();
3639 if ((SVT == MVT::f64 || SVT == MVT::f80) &&
3640 TLI.isOperationLegalOrCustom(ISD::FP_TO_FP16, MVT::f32)) {
3641 // Under fastmath, we can expand this node into a fround followed by
3642 // a float-half conversion.
3643 SDValue FloatVal =
3644 DAG.getNode(ISD::FP_ROUND, dl, MVT::f32, Op,
3645 DAG.getIntPtrConstant(0, dl, /*isTarget=*/true));
3646 Results.push_back(
3647 DAG.getNode(ISD::FP_TO_FP16, dl, Node->getValueType(0), FloatVal));
3648 }
3649 }
3650 break;
3651 case ISD::ConstantFP: {
3652 ConstantFPSDNode *CFP = cast<ConstantFPSDNode>(Node);
3653 // Check to see if this FP immediate is already legal.
3654 // If this is a legal constant, turn it into a TargetConstantFP node.
3655 if (!TLI.isFPImmLegal(CFP->getValueAPF(), Node->getValueType(0),
3656 DAG.shouldOptForSize()))
3657 Results.push_back(ExpandConstantFP(CFP, true));
3658 break;
3659 }
3660 case ISD::Constant: {
3661 ConstantSDNode *CP = cast<ConstantSDNode>(Node);
3662 Results.push_back(ExpandConstant(CP));
3663 break;
3664 }
3665 case ISD::FSUB: {
3666 EVT VT = Node->getValueType(0);
3667 if (TLI.isOperationLegalOrCustom(ISD::FADD, VT) &&
3668 TLI.isOperationLegalOrCustom(ISD::FNEG, VT)) {
3669 const SDNodeFlags Flags = Node->getFlags();
3670 Tmp1 = DAG.getNode(ISD::FNEG, dl, VT, Node->getOperand(1));
3671 Tmp1 = DAG.getNode(ISD::FADD, dl, VT, Node->getOperand(0), Tmp1, Flags);
3672 Results.push_back(Tmp1);
3673 }
3674 break;
3675 }
3676 case ISD::SUB: {
3677 EVT VT = Node->getValueType(0);
3678 assert(TLI.isOperationLegalOrCustom(ISD::ADD, VT) &&
3679 TLI.isOperationLegalOrCustom(ISD::XOR, VT) &&
3680 "Don't know how to expand this subtraction!");
3681 Tmp1 = DAG.getNOT(dl, Node->getOperand(1), VT);
3682 Tmp1 = DAG.getNode(ISD::ADD, dl, VT, Tmp1, DAG.getConstant(1, dl, VT));
3683 Results.push_back(DAG.getNode(ISD::ADD, dl, VT, Node->getOperand(0), Tmp1));
3684 break;
3685 }
3686 case ISD::UREM:
3687 case ISD::SREM:
3688 if (TLI.expandREM(Node, Tmp1, DAG))
3689 Results.push_back(Tmp1);
3690 break;
3691 case ISD::UDIV:
3692 case ISD::SDIV: {
3693 bool isSigned = Node->getOpcode() == ISD::SDIV;
3694 unsigned DivRemOpc = isSigned ? ISD::SDIVREM : ISD::UDIVREM;
3695 EVT VT = Node->getValueType(0);
3696 if (TLI.isOperationLegalOrCustom(DivRemOpc, VT)) {
3697 SDVTList VTs = DAG.getVTList(VT, VT);
3698 Tmp1 = DAG.getNode(DivRemOpc, dl, VTs, Node->getOperand(0),
3699 Node->getOperand(1));
3700 Results.push_back(Tmp1);
3701 }
3702 break;
3703 }
3704 case ISD::MULHU:
3705 case ISD::MULHS: {
3706 unsigned ExpandOpcode =
3707 Node->getOpcode() == ISD::MULHU ? ISD::UMUL_LOHI : ISD::SMUL_LOHI;
3708 EVT VT = Node->getValueType(0);
3709 SDVTList VTs = DAG.getVTList(VT, VT);
3710
3711 Tmp1 = DAG.getNode(ExpandOpcode, dl, VTs, Node->getOperand(0),
3712 Node->getOperand(1));
3713 Results.push_back(Tmp1.getValue(1));
3714 break;
3715 }
3716 case ISD::UMUL_LOHI:
3717 case ISD::SMUL_LOHI: {
3718 SDValue LHS = Node->getOperand(0);
3719 SDValue RHS = Node->getOperand(1);
3720 MVT VT = LHS.getSimpleValueType();
3721 unsigned MULHOpcode =
3722 Node->getOpcode() == ISD::UMUL_LOHI ? ISD::MULHU : ISD::MULHS;
3723
3724 if (TLI.isOperationLegalOrCustom(MULHOpcode, VT)) {
3725 Results.push_back(DAG.getNode(ISD::MUL, dl, VT, LHS, RHS));
3726 Results.push_back(DAG.getNode(MULHOpcode, dl, VT, LHS, RHS));
3727 break;
3728 }
3729
3731 EVT HalfType = EVT(VT).getHalfSizedIntegerVT(*DAG.getContext());
3732 assert(TLI.isTypeLegal(HalfType));
3733 if (TLI.expandMUL_LOHI(Node->getOpcode(), VT, dl, LHS, RHS, Halves,
3734 HalfType, DAG,
3735 TargetLowering::MulExpansionKind::Always)) {
3736 for (unsigned i = 0; i < 2; ++i) {
3737 SDValue Lo = DAG.getNode(ISD::ZERO_EXTEND, dl, VT, Halves[2 * i]);
3738 SDValue Hi = DAG.getNode(ISD::ANY_EXTEND, dl, VT, Halves[2 * i + 1]);
3739 SDValue Shift = DAG.getConstant(
3740 HalfType.getScalarSizeInBits(), dl,
3741 TLI.getShiftAmountTy(HalfType, DAG.getDataLayout()));
3742 Hi = DAG.getNode(ISD::SHL, dl, VT, Hi, Shift);
3743 Results.push_back(DAG.getNode(ISD::OR, dl, VT, Lo, Hi));
3744 }
3745 break;
3746 }
3747 break;
3748 }
3749 case ISD::MUL: {
3750 EVT VT = Node->getValueType(0);
3751 SDVTList VTs = DAG.getVTList(VT, VT);
3752 // See if multiply or divide can be lowered using two-result operations.
3753 // We just need the low half of the multiply; try both the signed
3754 // and unsigned forms. If the target supports both SMUL_LOHI and
3755 // UMUL_LOHI, form a preference by checking which forms of plain
3756 // MULH it supports.
3757 bool HasSMUL_LOHI = TLI.isOperationLegalOrCustom(ISD::SMUL_LOHI, VT);
3758 bool HasUMUL_LOHI = TLI.isOperationLegalOrCustom(ISD::UMUL_LOHI, VT);
3759 bool HasMULHS = TLI.isOperationLegalOrCustom(ISD::MULHS, VT);
3760 bool HasMULHU = TLI.isOperationLegalOrCustom(ISD::MULHU, VT);
3761 unsigned OpToUse = 0;
3762 if (HasSMUL_LOHI && !HasMULHS) {
3763 OpToUse = ISD::SMUL_LOHI;
3764 } else if (HasUMUL_LOHI && !HasMULHU) {
3765 OpToUse = ISD::UMUL_LOHI;
3766 } else if (HasSMUL_LOHI) {
3767 OpToUse = ISD::SMUL_LOHI;
3768 } else if (HasUMUL_LOHI) {
3769 OpToUse = ISD::UMUL_LOHI;
3770 }
3771 if (OpToUse) {
3772 Results.push_back(DAG.getNode(OpToUse, dl, VTs, Node->getOperand(0),
3773 Node->getOperand(1)));
3774 break;
3775 }
3776
3777 SDValue Lo, Hi;
3778 EVT HalfType = VT.getHalfSizedIntegerVT(*DAG.getContext());
3779 if (TLI.isOperationLegalOrCustom(ISD::ZERO_EXTEND, VT) &&
3780 TLI.isOperationLegalOrCustom(ISD::ANY_EXTEND, VT) &&
3781 TLI.isOperationLegalOrCustom(ISD::SHL, VT) &&
3782 TLI.isOperationLegalOrCustom(ISD::OR, VT) &&
3783 TLI.expandMUL(Node, Lo, Hi, HalfType, DAG,
3784 TargetLowering::MulExpansionKind::OnlyLegalOrCustom)) {
3785 Lo = DAG.getNode(ISD::ZERO_EXTEND, dl, VT, Lo);
3786 Hi = DAG.getNode(ISD::ANY_EXTEND, dl, VT, Hi);
3787 SDValue Shift =
3788 DAG.getConstant(HalfType.getSizeInBits(), dl,
3789 TLI.getShiftAmountTy(HalfType, DAG.getDataLayout()));
3790 Hi = DAG.getNode(ISD::SHL, dl, VT, Hi, Shift);
3791 Results.push_back(DAG.getNode(ISD::OR, dl, VT, Lo, Hi));
3792 }
3793 break;
3794 }
3795 case ISD::FSHL:
3796 case ISD::FSHR:
3797 if (SDValue Expanded = TLI.expandFunnelShift(Node, DAG))
3798 Results.push_back(Expanded);
3799 break;
3800 case ISD::ROTL:
3801 case ISD::ROTR:
3802 if (SDValue Expanded = TLI.expandROT(Node, true /*AllowVectorOps*/, DAG))
3803 Results.push_back(Expanded);
3804 break;
3805 case ISD::SADDSAT:
3806 case ISD::UADDSAT:
3807 case ISD::SSUBSAT:
3808 case ISD::USUBSAT:
3809 Results.push_back(TLI.expandAddSubSat(Node, DAG));
3810 break;
3811 case ISD::SSHLSAT:
3812 case ISD::USHLSAT:
3813 Results.push_back(TLI.expandShlSat(Node, DAG));
3814 break;
3815 case ISD::SMULFIX:
3816 case ISD::SMULFIXSAT:
3817 case ISD::UMULFIX:
3818 case ISD::UMULFIXSAT:
3819 Results.push_back(TLI.expandFixedPointMul(Node, DAG));
3820 break;
3821 case ISD::SDIVFIX:
3822 case ISD::SDIVFIXSAT:
3823 case ISD::UDIVFIX:
3824 case ISD::UDIVFIXSAT:
3825 if (SDValue V = TLI.expandFixedPointDiv(Node->getOpcode(), SDLoc(Node),
3826 Node->getOperand(0),
3827 Node->getOperand(1),
3828 Node->getConstantOperandVal(2),
3829 DAG)) {
3830 Results.push_back(V);
3831 break;
3832 }
3833 // FIXME: We might want to retry here with a wider type if we fail, if that
3834 // type is legal.
3835 // FIXME: Technically, so long as we only have sdivfixes where BW+Scale is
3836 // <= 128 (which is the case for all of the default Embedded-C types),
3837 // we will only get here with types and scales that we could always expand
3838 // if we were allowed to generate libcalls to division functions of illegal
3839 // type. But we cannot do that.
3840 llvm_unreachable("Cannot expand DIVFIX!");
3841 case ISD::UADDO_CARRY:
3842 case ISD::USUBO_CARRY: {
3843 SDValue LHS = Node->getOperand(0);
3844 SDValue RHS = Node->getOperand(1);
3845 SDValue Carry = Node->getOperand(2);
3846
3847 bool IsAdd = Node->getOpcode() == ISD::UADDO_CARRY;
3848
3849 // Initial add of the 2 operands.
3850 unsigned Op = IsAdd ? ISD::ADD : ISD::SUB;
3851 EVT VT = LHS.getValueType();
3852 SDValue Sum = DAG.getNode(Op, dl, VT, LHS, RHS);
3853
3854 // Initial check for overflow.
3855 EVT CarryType = Node->getValueType(1);
3856 EVT SetCCType = getSetCCResultType(Node->getValueType(0));
3858 SDValue Overflow = DAG.getSetCC(dl, SetCCType, Sum, LHS, CC);
3859
3860 // Add of the sum and the carry.
3861 SDValue One = DAG.getConstant(1, dl, VT);
3862 SDValue CarryExt =
3863 DAG.getNode(ISD::AND, dl, VT, DAG.getZExtOrTrunc(Carry, dl, VT), One);
3864 SDValue Sum2 = DAG.getNode(Op, dl, VT, Sum, CarryExt);
3865
3866 // Second check for overflow. If we are adding, we can only overflow if the
3867 // initial sum is all 1s ang the carry is set, resulting in a new sum of 0.
3868 // If we are subtracting, we can only overflow if the initial sum is 0 and
3869 // the carry is set, resulting in a new sum of all 1s.
3870 SDValue Zero = DAG.getConstant(0, dl, VT);
3871 SDValue Overflow2 =
3872 IsAdd ? DAG.getSetCC(dl, SetCCType, Sum2, Zero, ISD::SETEQ)
3873 : DAG.getSetCC(dl, SetCCType, Sum, Zero, ISD::SETEQ);
3874 Overflow2 = DAG.getNode(ISD::AND, dl, SetCCType, Overflow2,
3875 DAG.getZExtOrTrunc(Carry, dl, SetCCType));
3876
3877 SDValue ResultCarry =
3878 DAG.getNode(ISD::OR, dl, SetCCType, Overflow, Overflow2);
3879
3880 Results.push_back(Sum2);
3881 Results.push_back(DAG.getBoolExtOrTrunc(ResultCarry, dl, CarryType, VT));
3882 break;
3883 }
3884 case ISD::SADDO:
3885 case ISD::SSUBO: {
3886 SDValue Result, Overflow;
3887 TLI.expandSADDSUBO(Node, Result, Overflow, DAG);
3888 Results.push_back(Result);
3889 Results.push_back(Overflow);
3890 break;
3891 }
3892 case ISD::UADDO:
3893 case ISD::USUBO: {
3894 SDValue Result, Overflow;
3895 TLI.expandUADDSUBO(Node, Result, Overflow, DAG);
3896 Results.push_back(Result);
3897 Results.push_back(Overflow);
3898 break;
3899 }
3900 case ISD::UMULO:
3901 case ISD::SMULO: {
3902 SDValue Result, Overflow;
3903 if (TLI.expandMULO(Node, Result, Overflow, DAG)) {
3904 Results.push_back(Result);
3905 Results.push_back(Overflow);
3906 }
3907 break;
3908 }
3909 case ISD::BUILD_PAIR: {
3910 EVT PairTy = Node->getValueType(0);
3911 Tmp1 = DAG.getNode(ISD::ZERO_EXTEND, dl, PairTy, Node->getOperand(0));
3912 Tmp2 = DAG.getNode(ISD::ANY_EXTEND, dl, PairTy, Node->getOperand(1));
3913 Tmp2 = DAG.getNode(
3914 ISD::SHL, dl, PairTy, Tmp2,
3915 DAG.getConstant(PairTy.getSizeInBits() / 2, dl,
3916 TLI.getShiftAmountTy(PairTy, DAG.getDataLayout())));
3917 Results.push_back(DAG.getNode(ISD::OR, dl, PairTy, Tmp1, Tmp2));
3918 break;
3919 }
3920 case ISD::SELECT:
3921 Tmp1 = Node->getOperand(0);
3922 Tmp2 = Node->getOperand(1);
3923 Tmp3 = Node->getOperand(2);
3924 if (Tmp1.getOpcode() == ISD::SETCC) {
3925 Tmp1 = DAG.getSelectCC(dl, Tmp1.getOperand(0), Tmp1.getOperand(1),
3926 Tmp2, Tmp3,
3927 cast<CondCodeSDNode>(Tmp1.getOperand(2))->get());
3928 } else {
3929 Tmp1 = DAG.getSelectCC(dl, Tmp1,
3930 DAG.getConstant(0, dl, Tmp1.getValueType()),
3931 Tmp2, Tmp3, ISD::SETNE);
3932 }
3933 Tmp1->setFlags(Node->getFlags());
3934 Results.push_back(Tmp1);
3935 break;
3936 case ISD::BR_JT: {
3937 SDValue Chain = Node->getOperand(0);
3938 SDValue Table = Node->getOperand(1);
3939 SDValue Index = Node->getOperand(2);
3940 int JTI = cast<JumpTableSDNode>(Table.getNode())->getIndex();
3941
3942 const DataLayout &TD = DAG.getDataLayout();
3943 EVT PTy = TLI.getPointerTy(TD);
3944
3945 unsigned EntrySize =
3946 DAG.getMachineFunction().getJumpTableInfo()->getEntrySize(TD);
3947
3948 // For power-of-two jumptable entry sizes convert multiplication to a shift.
3949 // This transformation needs to be done here since otherwise the MIPS
3950 // backend will end up emitting a three instruction multiply sequence
3951 // instead of a single shift and MSP430 will call a runtime function.
3952 if (llvm::isPowerOf2_32(EntrySize))
3953 Index = DAG.getNode(
3954 ISD::SHL, dl, Index.getValueType(), Index,
3955 DAG.getConstant(llvm::Log2_32(EntrySize), dl, Index.getValueType()));
3956 else
3957 Index = DAG.getNode(ISD::MUL, dl, Index.getValueType(), Index,
3958 DAG.getConstant(EntrySize, dl, Index.getValueType()));
3959 SDValue Addr = DAG.getNode(ISD::ADD, dl, Index.getValueType(),
3960 Index, Table);
3961
3962 EVT MemVT = EVT::getIntegerVT(*DAG.getContext(), EntrySize * 8);
3963 SDValue LD = DAG.getExtLoad(
3964 ISD::SEXTLOAD, dl, PTy, Chain, Addr,
3965 MachinePointerInfo::getJumpTable(DAG.getMachineFunction()), MemVT);
3966 Addr = LD;
3967 if (TLI.isJumpTableRelative()) {
3968 // For PIC, the sequence is:
3969 // BRIND(load(Jumptable + index) + RelocBase)
3970 // RelocBase can be JumpTable, GOT or some sort of global base.
3971 Addr = DAG.getNode(ISD::ADD, dl, PTy, Addr,
3972 TLI.getPICJumpTableRelocBase(Table, DAG));
3973 }
3974
3975 Tmp1 = TLI.expandIndirectJTBranch(dl, LD.getValue(1), Addr, JTI, DAG);
3976 Results.push_back(Tmp1);
3977 break;
3978 }
3979 case ISD::BRCOND:
3980 // Expand brcond's setcc into its constituent parts and create a BR_CC
3981 // Node.
3982 Tmp1 = Node->getOperand(0);
3983 Tmp2 = Node->getOperand(1);
3984 if (Tmp2.getOpcode() == ISD::SETCC &&
3985 TLI.isOperationLegalOrCustom(ISD::BR_CC,
3986 Tmp2.getOperand(0).getValueType())) {
3987 Tmp1 = DAG.getNode(ISD::BR_CC, dl, MVT::Other, Tmp1, Tmp2.getOperand(2),
3988 Tmp2.getOperand(0), Tmp2.getOperand(1),
3989 Node->getOperand(2));
3990 } else {
3991 // We test only the i1 bit. Skip the AND if UNDEF or another AND.
3992 if (Tmp2.isUndef() ||
3993 (Tmp2.getOpcode() == ISD::AND && isOneConstant(Tmp2.getOperand(1))))
3994 Tmp3 = Tmp2;
3995 else
3996 Tmp3 = DAG.getNode(ISD::AND, dl, Tmp2.getValueType(), Tmp2,
3997 DAG.getConstant(1, dl, Tmp2.getValueType()));
3998 Tmp1 = DAG.getNode(ISD::BR_CC, dl, MVT::Other, Tmp1,
3999 DAG.getCondCode(ISD::SETNE), Tmp3,
4000 DAG.getConstant(0, dl, Tmp3.getValueType()),
4001 Node->getOperand(2));
4002 }
4003 Results.push_back(Tmp1);
4004 break;
4005 case ISD::SETCC:
4006 case ISD::VP_SETCC:
4007 case ISD::STRICT_FSETCC:
4008 case ISD::STRICT_FSETCCS: {
4009 bool IsVP = Node->getOpcode() == ISD::VP_SETCC;
4010 bool IsStrict = Node->getOpcode() == ISD::STRICT_FSETCC ||
4011 Node->getOpcode() == ISD::STRICT_FSETCCS;
4012 bool IsSignaling = Node->getOpcode() == ISD::STRICT_FSETCCS;
4013 SDValue Chain = IsStrict ? Node->getOperand(0) : SDValue();
4014 unsigned Offset = IsStrict ? 1 : 0;
4015 Tmp1 = Node->getOperand(0 + Offset);
4016 Tmp2 = Node->getOperand(1 + Offset);
4017 Tmp3 = Node->getOperand(2 + Offset);
4018 SDValue Mask, EVL;
4019 if (IsVP) {
4020 Mask = Node->getOperand(3 + Offset);
4021 EVL = Node->getOperand(4 + Offset);
4022 }
4023 bool Legalized = TLI.LegalizeSetCCCondCode(
4024 DAG, Node->getValueType(0), Tmp1, Tmp2, Tmp3, Mask, EVL, NeedInvert, dl,
4025 Chain, IsSignaling);
4026
4027 if (Legalized) {
4028 // If we expanded the SETCC by swapping LHS and RHS, or by inverting the
4029 // condition code, create a new SETCC node.
4030 if (Tmp3.getNode()) {
4031 if (IsStrict) {
4032 Tmp1 = DAG.getNode(Node->getOpcode(), dl, Node->getVTList(),
4033 {Chain, Tmp1, Tmp2, Tmp3}, Node->getFlags());
4034 Chain = Tmp1.getValue(1);
4035 } else if (IsVP) {
4036 Tmp1 = DAG.getNode(Node->getOpcode(), dl, Node->getValueType(0),
4037 {Tmp1, Tmp2, Tmp3, Mask, EVL}, Node->getFlags());
4038 } else {
4039 Tmp1 = DAG.getNode(Node->getOpcode(), dl, Node->getValueType(0), Tmp1,
4040 Tmp2, Tmp3, Node->getFlags());
4041 }
4042 }
4043
4044 // If we expanded the SETCC by inverting the condition code, then wrap
4045 // the existing SETCC in a NOT to restore the intended condition.
4046 if (NeedInvert) {
4047 if (!IsVP)
4048 Tmp1 = DAG.getLogicalNOT(dl, Tmp1, Tmp1->getValueType(0));
4049 else
4050 Tmp1 =
4051 DAG.getVPLogicalNOT(dl, Tmp1, Mask, EVL, Tmp1->getValueType(0));
4052 }
4053
4054 Results.push_back(Tmp1);
4055 if (IsStrict)
4056 Results.push_back(Chain);
4057
4058 break;
4059 }
4060
4061 // FIXME: It seems Legalized is false iff CCCode is Legal. I don't
4062 // understand if this code is useful for strict nodes.
4063 assert(!IsStrict && "Don't know how to expand for strict nodes.");
4064
4065 // Otherwise, SETCC for the given comparison type must be completely
4066 // illegal; expand it into a SELECT_CC.
4067 // FIXME: This drops the mask/evl for VP_SETCC.
4068 EVT VT = Node->getValueType(0);
4069 EVT Tmp1VT = Tmp1.getValueType();
4070 Tmp1 = DAG.getNode(ISD::SELECT_CC, dl, VT, Tmp1, Tmp2,
4071 DAG.getBoolConstant(true, dl, VT, Tmp1VT),
4072 DAG.getBoolConstant(false, dl, VT, Tmp1VT), Tmp3);
4073 Tmp1->setFlags(Node->getFlags());
4074 Results.push_back(Tmp1);
4075 break;
4076 }
4077 case ISD::SELECT_CC: {
4078 // TODO: need to add STRICT_SELECT_CC and STRICT_SELECT_CCS
4079 Tmp1 = Node->getOperand(0); // LHS
4080 Tmp2 = Node->getOperand(1); // RHS
4081 Tmp3 = Node->getOperand(2); // True
4082 Tmp4 = Node->getOperand(3); // False
4083 EVT VT = Node->getValueType(0);
4084 SDValue Chain;
4085 SDValue CC = Node->getOperand(4);
4086 ISD::CondCode CCOp = cast<CondCodeSDNode>(CC)->get();
4087
4088 if (TLI.isCondCodeLegalOrCustom(CCOp, Tmp1.getSimpleValueType())) {
4089 // If the condition code is legal, then we need to expand this
4090 // node using SETCC and SELECT.
4091 EVT CmpVT = Tmp1.getValueType();
4092 assert(!TLI.isOperationExpand(ISD::SELECT, VT) &&
4093 "Cannot expand ISD::SELECT_CC when ISD::SELECT also needs to be "
4094 "expanded.");
4095 EVT CCVT = getSetCCResultType(CmpVT);
4096 SDValue Cond = DAG.getNode(ISD::SETCC, dl, CCVT, Tmp1, Tmp2, CC, Node->getFlags());
4097 Results.push_back(DAG.getSelect(dl, VT, Cond, Tmp3, Tmp4));
4098 break;
4099 }
4100
4101 // SELECT_CC is legal, so the condition code must not be.
4102 bool Legalized = false;
4103 // Try to legalize by inverting the condition. This is for targets that
4104 // might support an ordered version of a condition, but not the unordered
4105 // version (or vice versa).
4106 ISD::CondCode InvCC = ISD::getSetCCInverse(CCOp, Tmp1.getValueType());
4107 if (TLI.isCondCodeLegalOrCustom(InvCC, Tmp1.getSimpleValueType())) {
4108 // Use the new condition code and swap true and false
4109 Legalized = true;
4110 Tmp1 = DAG.getSelectCC(dl, Tmp1, Tmp2, Tmp4, Tmp3, InvCC);
4111 Tmp1->setFlags(Node->getFlags());
4112 } else {
4113 // If The inverse is not legal, then try to swap the arguments using
4114 // the inverse condition code.
4116 if (TLI.isCondCodeLegalOrCustom(SwapInvCC, Tmp1.getSimpleValueType())) {
4117 // The swapped inverse condition is legal, so swap true and false,
4118 // lhs and rhs.
4119 Legalized = true;
4120 Tmp1 = DAG.getSelectCC(dl, Tmp2, Tmp1, Tmp4, Tmp3, SwapInvCC);
4121 Tmp1->setFlags(Node->getFlags());
4122 }
4123 }
4124
4125 if (!Legalized) {
4126 Legalized = TLI.LegalizeSetCCCondCode(
4127 DAG, getSetCCResultType(Tmp1.getValueType()), Tmp1, Tmp2, CC,
4128 /*Mask*/ SDValue(), /*EVL*/ SDValue(), NeedInvert, dl, Chain);
4129
4130 assert(Legalized && "Can't legalize SELECT_CC with legal condition!");
4131
4132 // If we expanded the SETCC by inverting the condition code, then swap
4133 // the True/False operands to match.
4134 if (NeedInvert)
4135 std::swap(Tmp3, Tmp4);
4136
4137 // If we expanded the SETCC by swapping LHS and RHS, or by inverting the
4138 // condition code, create a new SELECT_CC node.
4139 if (CC.getNode()) {
4140 Tmp1 = DAG.getNode(ISD::SELECT_CC, dl, Node->getValueType(0),
4141 Tmp1, Tmp2, Tmp3, Tmp4, CC);
4142 } else {
4143 Tmp2 = DAG.getConstant(0, dl, Tmp1.getValueType());
4144 CC = DAG.getCondCode(ISD::SETNE);
4145 Tmp1 = DAG.getNode(ISD::SELECT_CC, dl, Node->getValueType(0), Tmp1,
4146 Tmp2, Tmp3, Tmp4, CC);
4147 }
4148 Tmp1->setFlags(Node->getFlags());
4149 }
4150 Results.push_back(Tmp1);
4151 break;
4152 }
4153 case ISD::BR_CC: {
4154 // TODO: need to add STRICT_BR_CC and STRICT_BR_CCS
4155 SDValue Chain;
4156 Tmp1 = Node->getOperand(0); // Chain
4157 Tmp2 = Node->getOperand(2); // LHS
4158 Tmp3 = Node->getOperand(3); // RHS
4159 Tmp4 = Node->getOperand(1); // CC
4160
4161 bool Legalized = TLI.LegalizeSetCCCondCode(
4162 DAG, getSetCCResultType(Tmp2.getValueType()), Tmp2, Tmp3, Tmp4,
4163 /*Mask*/ SDValue(), /*EVL*/ SDValue(), NeedInvert, dl, Chain);
4164 (void)Legalized;
4165 assert(Legalized && "Can't legalize BR_CC with legal condition!");
4166
4167 // If we expanded the SETCC by swapping LHS and RHS, create a new BR_CC
4168 // node.
4169 if (Tmp4.getNode()) {
4170 assert(!NeedInvert && "Don't know how to invert BR_CC!");
4171
4172 Tmp1 = DAG.getNode(ISD::BR_CC, dl, Node->getValueType(0), Tmp1,
4173 Tmp4, Tmp2, Tmp3, Node->getOperand(4));
4174 } else {
4175 Tmp3 = DAG.getConstant(0, dl, Tmp2.getValueType());
4176 Tmp4 = DAG.getCondCode(NeedInvert ? ISD::SETEQ : ISD::SETNE);
4177 Tmp1 = DAG.getNode(ISD::BR_CC, dl, Node->getValueType(0), Tmp1, Tmp4,
4178 Tmp2, Tmp3, Node->getOperand(4));
4179 }
4180 Results.push_back(Tmp1);
4181 break;
4182 }
4183 case ISD::BUILD_VECTOR:
4184 Results.push_back(ExpandBUILD_VECTOR(Node));
4185 break;
4186 case ISD::SPLAT_VECTOR:
4187 Results.push_back(ExpandSPLAT_VECTOR(Node));
4188 break;
4189 case ISD::SRA:
4190 case ISD::SRL:
4191 case ISD::SHL: {
4192 // Scalarize vector SRA/SRL/SHL.
4193 EVT VT = Node->getValueType(0);
4194 assert(VT.isVector() && "Unable to legalize non-vector shift");
4195 assert(TLI.isTypeLegal(VT.getScalarType())&& "Element type must be legal");
4196 unsigned NumElem = VT.getVectorNumElements();
4197
4199 for (unsigned Idx = 0; Idx < NumElem; Idx++) {
4200 SDValue Ex =
4202 Node->getOperand(0), DAG.getVectorIdxConstant(Idx, dl));
4203 SDValue Sh =
4205 Node->getOperand(1), DAG.getVectorIdxConstant(Idx, dl));
4206 Scalars.push_back(DAG.getNode(Node->getOpcode(), dl,
4207 VT.getScalarType(), Ex, Sh));
4208 }
4209
4210 SDValue Result = DAG.getBuildVector(Node->getValueType(0), dl, Scalars);
4211 Results.push_back(Result);
4212 break;
4213 }
4216 case ISD::VECREDUCE_ADD:
4217 case ISD::VECREDUCE_MUL:
4218 case ISD::VECREDUCE_AND:
4219 case ISD::VECREDUCE_OR:
4220 case ISD::VECREDUCE_XOR:
4229 Results.push_back(TLI.expandVecReduce(Node, DAG));
4230 break;
4232 case ISD::GlobalAddress:
4235 case ISD::ConstantPool:
4236 case ISD::JumpTable:
4240 // FIXME: Custom lowering for these operations shouldn't return null!
4241 // Return true so that we don't call ConvertNodeToLibcall which also won't
4242 // do anything.
4243 return true;
4244 }
4245
4246 if (!TLI.isStrictFPEnabled() && Results.empty() && Node->isStrictFPOpcode()) {
4247 // FIXME: We were asked to expand a strict floating-point operation,
4248 // but there is currently no expansion implemented that would preserve
4249 // the "strict" properties. For now, we just fall back to the non-strict
4250 // version if that is legal on the target. The actual mutation of the
4251 // operation will happen in SelectionDAGISel::DoInstructionSelection.
4252 switch (Node->getOpcode()) {
4253 default:
4254 if (TLI.getStrictFPOperationAction(Node->getOpcode(),
4255 Node->getValueType(0))
4256 == TargetLowering::Legal)
4257 return true;
4258 break;
4259 case ISD::STRICT_FSUB: {
4260 if (TLI.getStrictFPOperationAction(
4261 ISD::STRICT_FSUB, Node->getValueType(0)) == TargetLowering::Legal)
4262 return true;
4263 if (TLI.getStrictFPOperationAction(
4264 ISD::STRICT_FADD, Node->getValueType(0)) != TargetLowering::Legal)
4265 break;
4266
4267 EVT VT = Node->getValueType(0);
4268 const SDNodeFlags Flags = Node->getFlags();
4269 SDValue Neg = DAG.getNode(ISD::FNEG, dl, VT, Node->getOperand(2), Flags);
4270 SDValue Fadd = DAG.getNode(ISD::STRICT_FADD, dl, Node->getVTList(),
4271 {Node->getOperand(0), Node->getOperand(1), Neg},
4272 Flags);
4273
4274 Results.push_back(Fadd);
4275 Results.push_back(Fadd.getValue(1));
4276 break;
4277 }
4280 case ISD::STRICT_LRINT:
4281 case ISD::STRICT_LLRINT:
4282 case ISD::STRICT_LROUND:
4284 // These are registered by the operand type instead of the value
4285 // type. Reflect that here.
4286 if (TLI.getStrictFPOperationAction(Node->getOpcode(),
4287 Node->getOperand(1).getValueType())
4288 == TargetLowering::Legal)
4289 return true;
4290 break;
4291 }
4292 }
4293
4294 // Replace the original node with the legalized result.
4295 if (Results.empty()) {
4296 LLVM_DEBUG(dbgs() << "Cannot expand node\n");
4297 return false;
4298 }
4299
4300 LLVM_DEBUG(dbgs() << "Successfully expanded node\n");
4301 ReplaceNode(Node, Results.data());
4302 return true;
4303}
4304
4305void SelectionDAGLegalize::ConvertNodeToLibcall(SDNode *Node) {
4306 LLVM_DEBUG(dbgs() << "Trying to convert node to libcall\n");
4308 SDLoc dl(Node);
4309 // FIXME: Check flags on the node to see if we can use a finite call.
4310 unsigned Opc = Node->getOpcode();
4311 switch (Opc) {
4312 case ISD::ATOMIC_FENCE: {
4313 // If the target didn't lower this, lower it to '__sync_synchronize()' call
4314 // FIXME: handle "fence singlethread" more efficiently.
4316
4318 CLI.setDebugLoc(dl)
4319 .setChain(Node->getOperand(0))
4320 .setLibCallee(
4321 CallingConv::C, Type::getVoidTy(*DAG.getContext()),
4322 DAG.getExternalSymbol("__sync_synchronize",
4323 TLI.getPointerTy(DAG.getDataLayout())),
4324 std::move(Args));
4325
4326 std::pair<SDValue, SDValue> CallResult = TLI.LowerCallTo(CLI);
4327
4328 Results.push_back(CallResult.second);
4329 break;
4330 }
4331 // By default, atomic intrinsics are marked Legal and lowered. Targets
4332 // which don't support them directly, however, may want libcalls, in which
4333 // case they mark them Expand, and we get here.
4334 case ISD::ATOMIC_SWAP:
4346 case ISD::ATOMIC_CMP_SWAP: {
4347 MVT VT = cast<AtomicSDNode>(Node)->getMemoryVT().getSimpleVT();
4348 AtomicOrdering Order = cast<AtomicSDNode>(Node)->getMergedOrdering();
4349 RTLIB::Libcall LC = RTLIB::getOUTLINE_ATOMIC(Opc, Order, VT);
4350 EVT RetVT = Node->getValueType(0);
4353 if (TLI.getLibcallName(LC)) {
4354 // If outline atomic available, prepare its arguments and expand.
4355 Ops.append(Node->op_begin() + 2, Node->op_end());
4356 Ops.push_back(Node->getOperand(1));
4357
4358 } else {
4359 LC = RTLIB::getSYNC(Opc, VT);
4360 assert(LC != RTLIB::UNKNOWN_LIBCALL &&
4361 "Unexpected atomic op or value type!");
4362 // Arguments for expansion to sync libcall
4363 Ops.append(Node->op_begin() + 1, Node->op_end());
4364 }
4365 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, RetVT,
4366 Ops, CallOptions,
4367 SDLoc(Node),
4368 Node->getOperand(0));
4369 Results.push_back(Tmp.first);
4370 Results.push_back(Tmp.second);
4371 break;
4372 }
4373 case ISD::TRAP: {
4374 // If this operation is not supported, lower it to 'abort()' call
4377 CLI.setDebugLoc(dl)
4378 .setChain(Node->getOperand(0))
4379 .setLibCallee(CallingConv::C, Type::getVoidTy(*DAG.getContext()),
4380 DAG.getExternalSymbol(
4381 "abort", TLI.getPointerTy(DAG.getDataLayout())),
4382 std::move(Args));
4383 std::pair<SDValue, SDValue> CallResult = TLI.LowerCallTo(CLI);
4384
4385 Results.push_back(CallResult.second);
4386 break;
4387 }
4388 case ISD::FMINNUM:
4390 ExpandFPLibCall(Node, RTLIB::FMIN_F32, RTLIB::FMIN_F64,
4391 RTLIB::FMIN_F80, RTLIB::FMIN_F128,
4392 RTLIB::FMIN_PPCF128, Results);
4393 break;
4394 // FIXME: We do not have libcalls for FMAXIMUM and FMINIMUM. So, we cannot use
4395 // libcall legalization for these nodes, but there is no default expasion for
4396 // these nodes either (see PR63267 for example).
4397 case ISD::FMAXNUM:
4399 ExpandFPLibCall(Node, RTLIB::FMAX_F32, RTLIB::FMAX_F64,
4400 RTLIB::FMAX_F80, RTLIB::FMAX_F128,
4401 RTLIB::FMAX_PPCF128, Results);
4402 break;
4403 case ISD::FSQRT:
4404 case ISD::STRICT_FSQRT:
4405 ExpandFPLibCall(Node, RTLIB::SQRT_F32, RTLIB::SQRT_F64,
4406 RTLIB::SQRT_F80, RTLIB::SQRT_F128,
4407 RTLIB::SQRT_PPCF128, Results);
4408 break;
4409 case ISD::FCBRT:
4410 ExpandFPLibCall(Node, RTLIB::CBRT_F32, RTLIB::CBRT_F64,
4411 RTLIB::CBRT_F80, RTLIB::CBRT_F128,
4412 RTLIB::CBRT_PPCF128, Results);
4413 break;
4414 case ISD::FSIN:
4415 case ISD::STRICT_FSIN:
4416 ExpandFPLibCall(Node, RTLIB::SIN_F32, RTLIB::SIN_F64,
4417 RTLIB::SIN_F80, RTLIB::SIN_F128,
4418 RTLIB::SIN_PPCF128, Results);
4419 break;
4420 case ISD::FCOS:
4421 case ISD::STRICT_FCOS:
4422 ExpandFPLibCall(Node, RTLIB::COS_F32, RTLIB::COS_F64,
4423 RTLIB::COS_F80, RTLIB::COS_F128,
4424 RTLIB::COS_PPCF128, Results);
4425 break;
4426 case ISD::FSINCOS:
4427 // Expand into sincos libcall.
4428 ExpandSinCosLibCall(Node, Results);
4429 break;
4430 case ISD::FLOG:
4431 case ISD::STRICT_FLOG:
4432 ExpandFPLibCall(Node, RTLIB::LOG_F32, RTLIB::LOG_F64, RTLIB::LOG_F80,
4433 RTLIB::LOG_F128, RTLIB::LOG_PPCF128, Results);
4434 break;
4435 case ISD::FLOG2:
4436 case ISD::STRICT_FLOG2:
4437 ExpandFPLibCall(Node, RTLIB::LOG2_F32, RTLIB::LOG2_F64, RTLIB::LOG2_F80,
4438 RTLIB::LOG2_F128, RTLIB::LOG2_PPCF128, Results);
4439 break;
4440 case ISD::FLOG10:
4441 case ISD::STRICT_FLOG10:
4442 ExpandFPLibCall(Node, RTLIB::LOG10_F32, RTLIB::LOG10_F64, RTLIB::LOG10_F80,
4443 RTLIB::LOG10_F128, RTLIB::LOG10_PPCF128, Results);
4444 break;
4445 case ISD::FEXP:
4446 case ISD::STRICT_FEXP:
4447 ExpandFPLibCall(Node, RTLIB::EXP_F32, RTLIB::EXP_F64, RTLIB::EXP_F80,
4448 RTLIB::EXP_F128, RTLIB::EXP_PPCF128, Results);
4449 break;
4450 case ISD::FEXP2:
4451 case ISD::STRICT_FEXP2:
4452 ExpandFPLibCall(Node, RTLIB::EXP2_F32, RTLIB::EXP2_F64, RTLIB::EXP2_F80,
4453 RTLIB::EXP2_F128, RTLIB::EXP2_PPCF128, Results);
4454 break;
4455 case ISD::FEXP10:
4456 ExpandFPLibCall(Node, RTLIB::EXP10_F32, RTLIB::EXP10_F64, RTLIB::EXP10_F80,
4457 RTLIB::EXP10_F128, RTLIB::EXP10_PPCF128, Results);
4458 break;
4459 case ISD::FTRUNC:
4460 case ISD::STRICT_FTRUNC:
4461 ExpandFPLibCall(Node, RTLIB::TRUNC_F32, RTLIB::TRUNC_F64,
4462 RTLIB::TRUNC_F80, RTLIB::TRUNC_F128,
4463 RTLIB::TRUNC_PPCF128, Results);
4464 break;
4465 case ISD::FFLOOR:
4466 case ISD::STRICT_FFLOOR:
4467 ExpandFPLibCall(Node, RTLIB::FLOOR_F32, RTLIB::FLOOR_F64,
4468 RTLIB::FLOOR_F80, RTLIB::FLOOR_F128,
4469 RTLIB::FLOOR_PPCF128, Results);
4470 break;
4471 case ISD::FCEIL:
4472 case ISD::STRICT_FCEIL:
4473 ExpandFPLibCall(Node, RTLIB::CEIL_F32, RTLIB::CEIL_F64,
4474 RTLIB::CEIL_F80, RTLIB::CEIL_F128,
4475 RTLIB::CEIL_PPCF128, Results);
4476 break;
4477 case ISD::FRINT:
4478 case ISD::STRICT_FRINT:
4479 ExpandFPLibCall(Node, RTLIB::RINT_F32, RTLIB::RINT_F64,
4480 RTLIB::RINT_F80, RTLIB::RINT_F128,
4481 RTLIB::RINT_PPCF128, Results);
4482 break;
4483 case ISD::FNEARBYINT:
4485 ExpandFPLibCall(Node, RTLIB::NEARBYINT_F32,
4486 RTLIB::NEARBYINT_F64,
4487 RTLIB::NEARBYINT_F80,
4488 RTLIB::NEARBYINT_F128,
4489 RTLIB::NEARBYINT_PPCF128, Results);
4490 break;
4491 case ISD::FROUND:
4492 case ISD::STRICT_FROUND:
4493 ExpandFPLibCall(Node, RTLIB::ROUND_F32,
4494 RTLIB::ROUND_F64,
4495 RTLIB::ROUND_F80,
4496 RTLIB::ROUND_F128,
4497 RTLIB::ROUND_PPCF128, Results);
4498 break;
4499 case ISD::FROUNDEVEN:
4501 ExpandFPLibCall(Node, RTLIB::ROUNDEVEN_F32,
4502 RTLIB::ROUNDEVEN_F64,
4503 RTLIB::ROUNDEVEN_F80,
4504 RTLIB::ROUNDEVEN_F128,
4505 RTLIB::ROUNDEVEN_PPCF128, Results);
4506 break;
4507 case ISD::FLDEXP:
4508 case ISD::STRICT_FLDEXP:
4509 ExpandFPLibCall(Node, RTLIB::LDEXP_F32, RTLIB::LDEXP_F64, RTLIB::LDEXP_F80,
4510 RTLIB::LDEXP_F128, RTLIB::LDEXP_PPCF128, Results);
4511 break;
4512 case ISD::FFREXP: {
4513 ExpandFrexpLibCall(Node, Results);
4514 break;
4515 }
4516 case ISD::FPOWI:
4517 case ISD::STRICT_FPOWI: {
4518 RTLIB::Libcall LC = RTLIB::getPOWI(Node->getSimpleValueType(0));
4519 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected fpowi.");
4520 if (!TLI.getLibcallName(LC)) {
4521 // Some targets don't have a powi libcall; use pow instead.
4522 if (Node->isStrictFPOpcode()) {
4524 DAG.getNode(ISD::STRICT_SINT_TO_FP, SDLoc(Node),
4525 {Node->getValueType(0), Node->getValueType(1)},
4526 {Node->getOperand(0), Node->getOperand(2)});
4527 SDValue FPOW =
4528 DAG.getNode(ISD::STRICT_FPOW, SDLoc(Node),
4529 {Node->getValueType(0), Node->getValueType(1)},
4530 {Exponent.getValue(1), Node->getOperand(1), Exponent});
4531 Results.push_back(FPOW);
4532 Results.push_back(FPOW.getValue(1));
4533 } else {
4535 DAG.getNode(ISD::SINT_TO_FP, SDLoc(Node), Node->getValueType(0),
4536 Node->getOperand(1));
4537 Results.push_back(DAG.getNode(ISD::FPOW, SDLoc(Node),
4538 Node->getValueType(0),
4539 Node->getOperand(0), Exponent));
4540 }
4541 break;
4542 }
4543 unsigned Offset = Node->isStrictFPOpcode() ? 1 : 0;
4544 bool ExponentHasSizeOfInt =
4545 DAG.getLibInfo().getIntSize() ==
4546 Node->getOperand(1 + Offset).getValueType().getSizeInBits();
4547 if (!ExponentHasSizeOfInt) {
4548 // If the exponent does not match with sizeof(int) a libcall to
4549 // RTLIB::POWI would use the wrong type for the argument.
4550 DAG.getContext()->emitError("POWI exponent does not match sizeof(int)");
4551 Results.push_back(DAG.getUNDEF(Node->getValueType(0)));
4552 break;
4553 }
4554 ExpandFPLibCall(Node, LC, Results);
4555 break;
4556 }
4557 case ISD::FPOW:
4558 case ISD::STRICT_FPOW:
4559 ExpandFPLibCall(Node, RTLIB::POW_F32, RTLIB::POW_F64, RTLIB::POW_F80,
4560 RTLIB::POW_F128, RTLIB::POW_PPCF128, Results);
4561 break;
4562 case ISD::LROUND:
4563 case ISD::STRICT_LROUND:
4564 ExpandArgFPLibCall(Node, RTLIB::LROUND_F32,
4565 RTLIB::LROUND_F64, RTLIB::LROUND_F80,
4566 RTLIB::LROUND_F128,
4567 RTLIB::LROUND_PPCF128, Results);
4568 break;
4569 case ISD::LLROUND:
4571 ExpandArgFPLibCall(Node, RTLIB::LLROUND_F32,
4572 RTLIB::LLROUND_F64, RTLIB::LLROUND_F80,
4573 RTLIB::LLROUND_F128,
4574 RTLIB::LLROUND_PPCF128, Results);
4575 break;
4576 case ISD::LRINT:
4577 case ISD::STRICT_LRINT:
4578 ExpandArgFPLibCall(Node, RTLIB::LRINT_F32,
4579 RTLIB::LRINT_F64, RTLIB::LRINT_F80,
4580 RTLIB::LRINT_F128,
4581 RTLIB::LRINT_PPCF128, Results);
4582 break;
4583 case ISD::LLRINT:
4584 case ISD::STRICT_LLRINT:
4585 ExpandArgFPLibCall(Node, RTLIB::LLRINT_F32,
4586 RTLIB::LLRINT_F64, RTLIB::LLRINT_F80,
4587 RTLIB::LLRINT_F128,
4588 RTLIB::LLRINT_PPCF128, Results);
4589 break;
4590 case ISD::FDIV:
4591 case ISD::STRICT_FDIV:
4592 ExpandFPLibCall(Node, RTLIB::DIV_F32, RTLIB::DIV_F64,
4593 RTLIB::DIV_F80, RTLIB::DIV_F128,
4594 RTLIB::DIV_PPCF128, Results);
4595 break;
4596 case ISD::FREM:
4597 case ISD::STRICT_FREM:
4598 ExpandFPLibCall(Node, RTLIB::REM_F32, RTLIB::REM_F64,
4599 RTLIB::REM_F80, RTLIB::REM_F128,
4600 RTLIB::REM_PPCF128, Results);
4601 break;
4602 case ISD::FMA:
4603 case ISD::STRICT_FMA:
4604 ExpandFPLibCall(Node, RTLIB::FMA_F32, RTLIB::FMA_F64,
4605 RTLIB::FMA_F80, RTLIB::FMA_F128,
4606 RTLIB::FMA_PPCF128, Results);
4607 break;
4608 case ISD::FADD:
4609 case ISD::STRICT_FADD:
4610 ExpandFPLibCall(Node, RTLIB::ADD_F32, RTLIB::ADD_F64,
4611 RTLIB::ADD_F80, RTLIB::ADD_F128,
4612 RTLIB::ADD_PPCF128, Results);
4613 break;
4614 case ISD::FMUL:
4615 case ISD::STRICT_FMUL:
4616 ExpandFPLibCall(Node, RTLIB::MUL_F32, RTLIB::MUL_F64,
4617 RTLIB::MUL_F80, RTLIB::MUL_F128,
4618 RTLIB::MUL_PPCF128, Results);
4619 break;
4620 case ISD::FP16_TO_FP:
4621 if (Node->getValueType(0) == MVT::f32) {
4622 Results.push_back(ExpandLibCall(RTLIB::FPEXT_F16_F32, Node, false).first);
4623 }
4624 break;
4626 if (Node->getValueType(0) == MVT::f32) {
4628 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(
4629 DAG, RTLIB::FPEXT_BF16_F32, MVT::f32, Node->getOperand(1),
4630 CallOptions, SDLoc(Node), Node->getOperand(0));
4631 Results.push_back(Tmp.first);
4632 Results.push_back(Tmp.second);
4633 }
4634 break;
4636 if (Node->getValueType(0) == MVT::f32) {
4638 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(
4639 DAG, RTLIB::FPEXT_F16_F32, MVT::f32, Node->getOperand(1), CallOptions,
4640 SDLoc(Node), Node->getOperand(0));
4641 Results.push_back(Tmp.first);
4642 Results.push_back(Tmp.second);
4643 }
4644 break;
4645 }