LLVM 19.0.0git
LegalizeDAG.cpp
Go to the documentation of this file.
1//===- LegalizeDAG.cpp - Implement SelectionDAG::Legalize -----------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements the SelectionDAG::Legalize method.
10//
11//===----------------------------------------------------------------------===//
12
13#include "llvm/ADT/APFloat.h"
14#include "llvm/ADT/APInt.h"
15#include "llvm/ADT/ArrayRef.h"
17#include "llvm/ADT/SetVector.h"
19#include "llvm/ADT/SmallSet.h"
36#include "llvm/IR/CallingConv.h"
37#include "llvm/IR/Constants.h"
38#include "llvm/IR/DataLayout.h"
40#include "llvm/IR/Function.h"
41#include "llvm/IR/Metadata.h"
42#include "llvm/IR/Type.h"
45#include "llvm/Support/Debug.h"
51#include <cassert>
52#include <cstdint>
53#include <tuple>
54#include <utility>
55
56using namespace llvm;
57
58#define DEBUG_TYPE "legalizedag"
59
60namespace {
61
62/// Keeps track of state when getting the sign of a floating-point value as an
63/// integer.
64struct FloatSignAsInt {
65 EVT FloatVT;
66 SDValue Chain;
67 SDValue FloatPtr;
68 SDValue IntPtr;
69 MachinePointerInfo IntPointerInfo;
70 MachinePointerInfo FloatPointerInfo;
71 SDValue IntValue;
72 APInt SignMask;
73 uint8_t SignBit;
74};
75
76//===----------------------------------------------------------------------===//
77/// This takes an arbitrary SelectionDAG as input and
78/// hacks on it until the target machine can handle it. This involves
79/// eliminating value sizes the machine cannot handle (promoting small sizes to
80/// large sizes or splitting up large values into small values) as well as
81/// eliminating operations the machine cannot handle.
82///
83/// This code also does a small amount of optimization and recognition of idioms
84/// as part of its processing. For example, if a target does not support a
85/// 'setcc' instruction efficiently, but does support 'brcc' instruction, this
86/// will attempt merge setcc and brc instructions into brcc's.
87class SelectionDAGLegalize {
88 const TargetMachine &TM;
89 const TargetLowering &TLI;
90 SelectionDAG &DAG;
91
92 /// The set of nodes which have already been legalized. We hold a
93 /// reference to it in order to update as necessary on node deletion.
94 SmallPtrSetImpl<SDNode *> &LegalizedNodes;
95
96 /// A set of all the nodes updated during legalization.
97 SmallSetVector<SDNode *, 16> *UpdatedNodes;
98
99 EVT getSetCCResultType(EVT VT) const {
100 return TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT);
101 }
102
103 // Libcall insertion helpers.
104
105public:
106 SelectionDAGLegalize(SelectionDAG &DAG,
107 SmallPtrSetImpl<SDNode *> &LegalizedNodes,
108 SmallSetVector<SDNode *, 16> *UpdatedNodes = nullptr)
109 : TM(DAG.getTarget()), TLI(DAG.getTargetLoweringInfo()), DAG(DAG),
110 LegalizedNodes(LegalizedNodes), UpdatedNodes(UpdatedNodes) {}
111
112 /// Legalizes the given operation.
113 void LegalizeOp(SDNode *Node);
114
115private:
116 SDValue OptimizeFloatStore(StoreSDNode *ST);
117
118 void LegalizeLoadOps(SDNode *Node);
119 void LegalizeStoreOps(SDNode *Node);
120
121 /// Some targets cannot handle a variable
122 /// insertion index for the INSERT_VECTOR_ELT instruction. In this case, it
123 /// is necessary to spill the vector being inserted into to memory, perform
124 /// the insert there, and then read the result back.
125 SDValue PerformInsertVectorEltInMemory(SDValue Vec, SDValue Val, SDValue Idx,
126 const SDLoc &dl);
127 SDValue ExpandINSERT_VECTOR_ELT(SDValue Vec, SDValue Val, SDValue Idx,
128 const SDLoc &dl);
129
130 /// Return a vector shuffle operation which
131 /// performs the same shuffe in terms of order or result bytes, but on a type
132 /// whose vector element type is narrower than the original shuffle type.
133 /// e.g. <v4i32> <0, 1, 0, 1> -> v8i16 <0, 1, 2, 3, 0, 1, 2, 3>
134 SDValue ShuffleWithNarrowerEltType(EVT NVT, EVT VT, const SDLoc &dl,
135 SDValue N1, SDValue N2,
136 ArrayRef<int> Mask) const;
137
138 std::pair<SDValue, SDValue> ExpandLibCall(RTLIB::Libcall LC, SDNode *Node,
140 std::pair<SDValue, SDValue> ExpandLibCall(RTLIB::Libcall LC, SDNode *Node, bool isSigned);
141
142 void ExpandFrexpLibCall(SDNode *Node, SmallVectorImpl<SDValue> &Results);
143 void ExpandFPLibCall(SDNode *Node, RTLIB::Libcall LC,
145 void ExpandFPLibCall(SDNode *Node, RTLIB::Libcall Call_F32,
146 RTLIB::Libcall Call_F64, RTLIB::Libcall Call_F80,
147 RTLIB::Libcall Call_F128,
148 RTLIB::Libcall Call_PPCF128,
150 SDValue ExpandIntLibCall(SDNode *Node, bool isSigned,
151 RTLIB::Libcall Call_I8,
152 RTLIB::Libcall Call_I16,
153 RTLIB::Libcall Call_I32,
154 RTLIB::Libcall Call_I64,
155 RTLIB::Libcall Call_I128);
156 void ExpandArgFPLibCall(SDNode *Node,
157 RTLIB::Libcall Call_F32, RTLIB::Libcall Call_F64,
158 RTLIB::Libcall Call_F80, RTLIB::Libcall Call_F128,
159 RTLIB::Libcall Call_PPCF128,
161 void ExpandDivRemLibCall(SDNode *Node, SmallVectorImpl<SDValue> &Results);
162 void ExpandSinCosLibCall(SDNode *Node, SmallVectorImpl<SDValue> &Results);
163
164 SDValue EmitStackConvert(SDValue SrcOp, EVT SlotVT, EVT DestVT,
165 const SDLoc &dl);
166 SDValue EmitStackConvert(SDValue SrcOp, EVT SlotVT, EVT DestVT,
167 const SDLoc &dl, SDValue ChainIn);
168 SDValue ExpandBUILD_VECTOR(SDNode *Node);
169 SDValue ExpandSPLAT_VECTOR(SDNode *Node);
170 SDValue ExpandSCALAR_TO_VECTOR(SDNode *Node);
171 void ExpandDYNAMIC_STACKALLOC(SDNode *Node,
173 void getSignAsIntValue(FloatSignAsInt &State, const SDLoc &DL,
174 SDValue Value) const;
175 SDValue modifySignAsInt(const FloatSignAsInt &State, const SDLoc &DL,
176 SDValue NewIntValue) const;
177 SDValue ExpandFCOPYSIGN(SDNode *Node) const;
178 SDValue ExpandFABS(SDNode *Node) const;
179 SDValue ExpandFNEG(SDNode *Node) const;
180 SDValue expandLdexp(SDNode *Node) const;
181 SDValue expandFrexp(SDNode *Node) const;
182
183 SDValue ExpandLegalINT_TO_FP(SDNode *Node, SDValue &Chain);
184 void PromoteLegalINT_TO_FP(SDNode *N, const SDLoc &dl,
186 void PromoteLegalFP_TO_INT(SDNode *N, const SDLoc &dl,
188 SDValue PromoteLegalFP_TO_INT_SAT(SDNode *Node, const SDLoc &dl);
189
190 SDValue ExpandPARITY(SDValue Op, const SDLoc &dl);
191
192 SDValue ExpandExtractFromVectorThroughStack(SDValue Op);
193 SDValue ExpandInsertToVectorThroughStack(SDValue Op);
194 SDValue ExpandVectorBuildThroughStack(SDNode* Node);
195
196 SDValue ExpandConstantFP(ConstantFPSDNode *CFP, bool UseCP);
197 SDValue ExpandConstant(ConstantSDNode *CP);
198
199 // if ExpandNode returns false, LegalizeOp falls back to ConvertNodeToLibcall
200 bool ExpandNode(SDNode *Node);
201 void ConvertNodeToLibcall(SDNode *Node);
202 void PromoteNode(SDNode *Node);
203
204public:
205 // Node replacement helpers
206
207 void ReplacedNode(SDNode *N) {
208 LegalizedNodes.erase(N);
209 if (UpdatedNodes)
210 UpdatedNodes->insert(N);
211 }
212
213 void ReplaceNode(SDNode *Old, SDNode *New) {
214 LLVM_DEBUG(dbgs() << " ... replacing: "; Old->dump(&DAG);
215 dbgs() << " with: "; New->dump(&DAG));
216
217 assert(Old->getNumValues() == New->getNumValues() &&
218 "Replacing one node with another that produces a different number "
219 "of values!");
220 DAG.ReplaceAllUsesWith(Old, New);
221 if (UpdatedNodes)
222 UpdatedNodes->insert(New);
223 ReplacedNode(Old);
224 }
225
226 void ReplaceNode(SDValue Old, SDValue New) {
227 LLVM_DEBUG(dbgs() << " ... replacing: "; Old->dump(&DAG);
228 dbgs() << " with: "; New->dump(&DAG));
229
230 DAG.ReplaceAllUsesWith(Old, New);
231 if (UpdatedNodes)
232 UpdatedNodes->insert(New.getNode());
233 ReplacedNode(Old.getNode());
234 }
235
236 void ReplaceNode(SDNode *Old, const SDValue *New) {
237 LLVM_DEBUG(dbgs() << " ... replacing: "; Old->dump(&DAG));
238
239 DAG.ReplaceAllUsesWith(Old, New);
240 for (unsigned i = 0, e = Old->getNumValues(); i != e; ++i) {
241 LLVM_DEBUG(dbgs() << (i == 0 ? " with: " : " and: ");
242 New[i]->dump(&DAG));
243 if (UpdatedNodes)
244 UpdatedNodes->insert(New[i].getNode());
245 }
246 ReplacedNode(Old);
247 }
248
249 void ReplaceNodeWithValue(SDValue Old, SDValue New) {
250 LLVM_DEBUG(dbgs() << " ... replacing: "; Old->dump(&DAG);
251 dbgs() << " with: "; New->dump(&DAG));
252
253 DAG.ReplaceAllUsesOfValueWith(Old, New);
254 if (UpdatedNodes)
255 UpdatedNodes->insert(New.getNode());
256 ReplacedNode(Old.getNode());
257 }
258};
259
260} // end anonymous namespace
261
262// Helper function that generates an MMO that considers the alignment of the
263// stack, and the size of the stack object
265 MachineFunction &MF,
266 bool isObjectScalable) {
267 auto &MFI = MF.getFrameInfo();
268 int FI = cast<FrameIndexSDNode>(StackPtr)->getIndex();
270 LocationSize ObjectSize = isObjectScalable
272 : LocationSize::precise(MFI.getObjectSize(FI));
274 ObjectSize, MFI.getObjectAlign(FI));
275}
276
277/// Return a vector shuffle operation which
278/// performs the same shuffle in terms of order or result bytes, but on a type
279/// whose vector element type is narrower than the original shuffle type.
280/// e.g. <v4i32> <0, 1, 0, 1> -> v8i16 <0, 1, 2, 3, 0, 1, 2, 3>
281SDValue SelectionDAGLegalize::ShuffleWithNarrowerEltType(
282 EVT NVT, EVT VT, const SDLoc &dl, SDValue N1, SDValue N2,
283 ArrayRef<int> Mask) const {
284 unsigned NumMaskElts = VT.getVectorNumElements();
285 unsigned NumDestElts = NVT.getVectorNumElements();
286 unsigned NumEltsGrowth = NumDestElts / NumMaskElts;
287
288 assert(NumEltsGrowth && "Cannot promote to vector type with fewer elts!");
289
290 if (NumEltsGrowth == 1)
291 return DAG.getVectorShuffle(NVT, dl, N1, N2, Mask);
292
293 SmallVector<int, 8> NewMask;
294 for (unsigned i = 0; i != NumMaskElts; ++i) {
295 int Idx = Mask[i];
296 for (unsigned j = 0; j != NumEltsGrowth; ++j) {
297 if (Idx < 0)
298 NewMask.push_back(-1);
299 else
300 NewMask.push_back(Idx * NumEltsGrowth + j);
301 }
302 }
303 assert(NewMask.size() == NumDestElts && "Non-integer NumEltsGrowth?");
304 assert(TLI.isShuffleMaskLegal(NewMask, NVT) && "Shuffle not legal?");
305 return DAG.getVectorShuffle(NVT, dl, N1, N2, NewMask);
306}
307
308/// Expands the ConstantFP node to an integer constant or
309/// a load from the constant pool.
311SelectionDAGLegalize::ExpandConstantFP(ConstantFPSDNode *CFP, bool UseCP) {
312 bool Extend = false;
313 SDLoc dl(CFP);
314
315 // If a FP immediate is precise when represented as a float and if the
316 // target can do an extending load from float to double, we put it into
317 // the constant pool as a float, even if it's is statically typed as a
318 // double. This shrinks FP constants and canonicalizes them for targets where
319 // an FP extending load is the same cost as a normal load (such as on the x87
320 // fp stack or PPC FP unit).
321 EVT VT = CFP->getValueType(0);
322 ConstantFP *LLVMC = const_cast<ConstantFP*>(CFP->getConstantFPValue());
323 if (!UseCP) {
324 assert((VT == MVT::f64 || VT == MVT::f32) && "Invalid type expansion");
325 return DAG.getConstant(LLVMC->getValueAPF().bitcastToAPInt(), dl,
326 (VT == MVT::f64) ? MVT::i64 : MVT::i32);
327 }
328
329 APFloat APF = CFP->getValueAPF();
330 EVT OrigVT = VT;
331 EVT SVT = VT;
332
333 // We don't want to shrink SNaNs. Converting the SNaN back to its real type
334 // can cause it to be changed into a QNaN on some platforms (e.g. on SystemZ).
335 if (!APF.isSignaling()) {
336 while (SVT != MVT::f32 && SVT != MVT::f16 && SVT != MVT::bf16) {
337 SVT = (MVT::SimpleValueType)(SVT.getSimpleVT().SimpleTy - 1);
339 // Only do this if the target has a native EXTLOAD instruction from
340 // smaller type.
341 TLI.isLoadExtLegal(ISD::EXTLOAD, OrigVT, SVT) &&
342 TLI.ShouldShrinkFPConstant(OrigVT)) {
343 Type *SType = SVT.getTypeForEVT(*DAG.getContext());
344 LLVMC = cast<ConstantFP>(ConstantFoldCastOperand(
345 Instruction::FPTrunc, LLVMC, SType, DAG.getDataLayout()));
346 VT = SVT;
347 Extend = true;
348 }
349 }
350 }
351
352 SDValue CPIdx =
353 DAG.getConstantPool(LLVMC, TLI.getPointerTy(DAG.getDataLayout()));
354 Align Alignment = cast<ConstantPoolSDNode>(CPIdx)->getAlign();
355 if (Extend) {
356 SDValue Result = DAG.getExtLoad(
357 ISD::EXTLOAD, dl, OrigVT, DAG.getEntryNode(), CPIdx,
358 MachinePointerInfo::getConstantPool(DAG.getMachineFunction()), VT,
359 Alignment);
360 return Result;
361 }
362 SDValue Result = DAG.getLoad(
363 OrigVT, dl, DAG.getEntryNode(), CPIdx,
364 MachinePointerInfo::getConstantPool(DAG.getMachineFunction()), Alignment);
365 return Result;
366}
367
368/// Expands the Constant node to a load from the constant pool.
369SDValue SelectionDAGLegalize::ExpandConstant(ConstantSDNode *CP) {
370 SDLoc dl(CP);
371 EVT VT = CP->getValueType(0);
372 SDValue CPIdx = DAG.getConstantPool(CP->getConstantIntValue(),
373 TLI.getPointerTy(DAG.getDataLayout()));
374 Align Alignment = cast<ConstantPoolSDNode>(CPIdx)->getAlign();
375 SDValue Result = DAG.getLoad(
376 VT, dl, DAG.getEntryNode(), CPIdx,
377 MachinePointerInfo::getConstantPool(DAG.getMachineFunction()), Alignment);
378 return Result;
379}
380
381/// Some target cannot handle a variable insertion index for the
382/// INSERT_VECTOR_ELT instruction. In this case, it
383/// is necessary to spill the vector being inserted into to memory, perform
384/// the insert there, and then read the result back.
385SDValue SelectionDAGLegalize::PerformInsertVectorEltInMemory(SDValue Vec,
386 SDValue Val,
387 SDValue Idx,
388 const SDLoc &dl) {
389 // If the target doesn't support this, we have to spill the input vector
390 // to a temporary stack slot, update the element, then reload it. This is
391 // badness. We could also load the value into a vector register (either
392 // with a "move to register" or "extload into register" instruction, then
393 // permute it into place, if the idx is a constant and if the idx is
394 // supported by the target.
395 EVT VT = Vec.getValueType();
396 EVT EltVT = VT.getVectorElementType();
397 SDValue StackPtr = DAG.CreateStackTemporary(VT);
398
399 int SPFI = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex();
400
401 // Store the vector.
402 SDValue Ch = DAG.getStore(
403 DAG.getEntryNode(), dl, Vec, StackPtr,
404 MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), SPFI));
405
406 SDValue StackPtr2 = TLI.getVectorElementPointer(DAG, StackPtr, VT, Idx);
407
408 // Store the scalar value.
409 Ch = DAG.getTruncStore(
410 Ch, dl, Val, StackPtr2,
411 MachinePointerInfo::getUnknownStack(DAG.getMachineFunction()), EltVT);
412 // Load the updated vector.
413 return DAG.getLoad(VT, dl, Ch, StackPtr, MachinePointerInfo::getFixedStack(
414 DAG.getMachineFunction(), SPFI));
415}
416
417SDValue SelectionDAGLegalize::ExpandINSERT_VECTOR_ELT(SDValue Vec, SDValue Val,
418 SDValue Idx,
419 const SDLoc &dl) {
420 if (ConstantSDNode *InsertPos = dyn_cast<ConstantSDNode>(Idx)) {
421 // SCALAR_TO_VECTOR requires that the type of the value being inserted
422 // match the element type of the vector being created, except for
423 // integers in which case the inserted value can be over width.
424 EVT EltVT = Vec.getValueType().getVectorElementType();
425 if (Val.getValueType() == EltVT ||
426 (EltVT.isInteger() && Val.getValueType().bitsGE(EltVT))) {
427 SDValue ScVec = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl,
428 Vec.getValueType(), Val);
429
430 unsigned NumElts = Vec.getValueType().getVectorNumElements();
431 // We generate a shuffle of InVec and ScVec, so the shuffle mask
432 // should be 0,1,2,3,4,5... with the appropriate element replaced with
433 // elt 0 of the RHS.
434 SmallVector<int, 8> ShufOps;
435 for (unsigned i = 0; i != NumElts; ++i)
436 ShufOps.push_back(i != InsertPos->getZExtValue() ? i : NumElts);
437
438 return DAG.getVectorShuffle(Vec.getValueType(), dl, Vec, ScVec, ShufOps);
439 }
440 }
441 return PerformInsertVectorEltInMemory(Vec, Val, Idx, dl);
442}
443
444SDValue SelectionDAGLegalize::OptimizeFloatStore(StoreSDNode* ST) {
445 if (!ISD::isNormalStore(ST))
446 return SDValue();
447
448 LLVM_DEBUG(dbgs() << "Optimizing float store operations\n");
449 // Turn 'store float 1.0, Ptr' -> 'store int 0x12345678, Ptr'
450 // FIXME: move this to the DAG Combiner! Note that we can't regress due
451 // to phase ordering between legalized code and the dag combiner. This
452 // probably means that we need to integrate dag combiner and legalizer
453 // together.
454 // We generally can't do this one for long doubles.
455 SDValue Chain = ST->getChain();
456 SDValue Ptr = ST->getBasePtr();
457 SDValue Value = ST->getValue();
458 MachineMemOperand::Flags MMOFlags = ST->getMemOperand()->getFlags();
459 AAMDNodes AAInfo = ST->getAAInfo();
460 SDLoc dl(ST);
461
462 // Don't optimise TargetConstantFP
463 if (Value.getOpcode() == ISD::TargetConstantFP)
464 return SDValue();
465
466 if (ConstantFPSDNode *CFP = dyn_cast<ConstantFPSDNode>(Value)) {
467 if (CFP->getValueType(0) == MVT::f32 &&
468 TLI.isTypeLegal(MVT::i32)) {
469 SDValue Con = DAG.getConstant(CFP->getValueAPF().
470 bitcastToAPInt().zextOrTrunc(32),
471 SDLoc(CFP), MVT::i32);
472 return DAG.getStore(Chain, dl, Con, Ptr, ST->getPointerInfo(),
473 ST->getOriginalAlign(), MMOFlags, AAInfo);
474 }
475
476 if (CFP->getValueType(0) == MVT::f64 &&
477 !TLI.isFPImmLegal(CFP->getValueAPF(), MVT::f64)) {
478 // If this target supports 64-bit registers, do a single 64-bit store.
479 if (TLI.isTypeLegal(MVT::i64)) {
480 SDValue Con = DAG.getConstant(CFP->getValueAPF().bitcastToAPInt().
481 zextOrTrunc(64), SDLoc(CFP), MVT::i64);
482 return DAG.getStore(Chain, dl, Con, Ptr, ST->getPointerInfo(),
483 ST->getOriginalAlign(), MMOFlags, AAInfo);
484 }
485
486 if (TLI.isTypeLegal(MVT::i32) && !ST->isVolatile()) {
487 // Otherwise, if the target supports 32-bit registers, use 2 32-bit
488 // stores. If the target supports neither 32- nor 64-bits, this
489 // xform is certainly not worth it.
490 const APInt &IntVal = CFP->getValueAPF().bitcastToAPInt();
491 SDValue Lo = DAG.getConstant(IntVal.trunc(32), dl, MVT::i32);
492 SDValue Hi = DAG.getConstant(IntVal.lshr(32).trunc(32), dl, MVT::i32);
493 if (DAG.getDataLayout().isBigEndian())
494 std::swap(Lo, Hi);
495
496 Lo = DAG.getStore(Chain, dl, Lo, Ptr, ST->getPointerInfo(),
497 ST->getOriginalAlign(), MMOFlags, AAInfo);
498 Ptr = DAG.getMemBasePlusOffset(Ptr, TypeSize::getFixed(4), dl);
499 Hi = DAG.getStore(Chain, dl, Hi, Ptr,
500 ST->getPointerInfo().getWithOffset(4),
501 ST->getOriginalAlign(), MMOFlags, AAInfo);
502
503 return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo, Hi);
504 }
505 }
506 }
507 return SDValue();
508}
509
510void SelectionDAGLegalize::LegalizeStoreOps(SDNode *Node) {
511 StoreSDNode *ST = cast<StoreSDNode>(Node);
512 SDValue Chain = ST->getChain();
513 SDValue Ptr = ST->getBasePtr();
514 SDLoc dl(Node);
515
516 MachineMemOperand::Flags MMOFlags = ST->getMemOperand()->getFlags();
517 AAMDNodes AAInfo = ST->getAAInfo();
518
519 if (!ST->isTruncatingStore()) {
520 LLVM_DEBUG(dbgs() << "Legalizing store operation\n");
521 if (SDNode *OptStore = OptimizeFloatStore(ST).getNode()) {
522 ReplaceNode(ST, OptStore);
523 return;
524 }
525
526 SDValue Value = ST->getValue();
527 MVT VT = Value.getSimpleValueType();
528 switch (TLI.getOperationAction(ISD::STORE, VT)) {
529 default: llvm_unreachable("This action is not supported yet!");
530 case TargetLowering::Legal: {
531 // If this is an unaligned store and the target doesn't support it,
532 // expand it.
533 EVT MemVT = ST->getMemoryVT();
534 const DataLayout &DL = DAG.getDataLayout();
535 if (!TLI.allowsMemoryAccessForAlignment(*DAG.getContext(), DL, MemVT,
536 *ST->getMemOperand())) {
537 LLVM_DEBUG(dbgs() << "Expanding unsupported unaligned store\n");
538 SDValue Result = TLI.expandUnalignedStore(ST, DAG);
539 ReplaceNode(SDValue(ST, 0), Result);
540 } else
541 LLVM_DEBUG(dbgs() << "Legal store\n");
542 break;
543 }
544 case TargetLowering::Custom: {
545 LLVM_DEBUG(dbgs() << "Trying custom lowering\n");
546 SDValue Res = TLI.LowerOperation(SDValue(Node, 0), DAG);
547 if (Res && Res != SDValue(Node, 0))
548 ReplaceNode(SDValue(Node, 0), Res);
549 return;
550 }
551 case TargetLowering::Promote: {
552 MVT NVT = TLI.getTypeToPromoteTo(ISD::STORE, VT);
553 assert(NVT.getSizeInBits() == VT.getSizeInBits() &&
554 "Can only promote stores to same size type");
555 Value = DAG.getNode(ISD::BITCAST, dl, NVT, Value);
556 SDValue Result = DAG.getStore(Chain, dl, Value, Ptr, ST->getPointerInfo(),
557 ST->getOriginalAlign(), MMOFlags, AAInfo);
558 ReplaceNode(SDValue(Node, 0), Result);
559 break;
560 }
561 }
562 return;
563 }
564
565 LLVM_DEBUG(dbgs() << "Legalizing truncating store operations\n");
566 SDValue Value = ST->getValue();
567 EVT StVT = ST->getMemoryVT();
568 TypeSize StWidth = StVT.getSizeInBits();
569 TypeSize StSize = StVT.getStoreSizeInBits();
570 auto &DL = DAG.getDataLayout();
571
572 if (StWidth != StSize) {
573 // Promote to a byte-sized store with upper bits zero if not
574 // storing an integral number of bytes. For example, promote
575 // TRUNCSTORE:i1 X -> TRUNCSTORE:i8 (and X, 1)
576 EVT NVT = EVT::getIntegerVT(*DAG.getContext(), StSize.getFixedValue());
577 Value = DAG.getZeroExtendInReg(Value, dl, StVT);
579 DAG.getTruncStore(Chain, dl, Value, Ptr, ST->getPointerInfo(), NVT,
580 ST->getOriginalAlign(), MMOFlags, AAInfo);
581 ReplaceNode(SDValue(Node, 0), Result);
582 } else if (!StVT.isVector() && !isPowerOf2_64(StWidth.getFixedValue())) {
583 // If not storing a power-of-2 number of bits, expand as two stores.
584 assert(!StVT.isVector() && "Unsupported truncstore!");
585 unsigned StWidthBits = StWidth.getFixedValue();
586 unsigned LogStWidth = Log2_32(StWidthBits);
587 assert(LogStWidth < 32);
588 unsigned RoundWidth = 1 << LogStWidth;
589 assert(RoundWidth < StWidthBits);
590 unsigned ExtraWidth = StWidthBits - RoundWidth;
591 assert(ExtraWidth < RoundWidth);
592 assert(!(RoundWidth % 8) && !(ExtraWidth % 8) &&
593 "Store size not an integral number of bytes!");
594 EVT RoundVT = EVT::getIntegerVT(*DAG.getContext(), RoundWidth);
595 EVT ExtraVT = EVT::getIntegerVT(*DAG.getContext(), ExtraWidth);
596 SDValue Lo, Hi;
597 unsigned IncrementSize;
598
599 if (DL.isLittleEndian()) {
600 // TRUNCSTORE:i24 X -> TRUNCSTORE:i16 X, TRUNCSTORE@+2:i8 (srl X, 16)
601 // Store the bottom RoundWidth bits.
602 Lo = DAG.getTruncStore(Chain, dl, Value, Ptr, ST->getPointerInfo(),
603 RoundVT, ST->getOriginalAlign(), MMOFlags, AAInfo);
604
605 // Store the remaining ExtraWidth bits.
606 IncrementSize = RoundWidth / 8;
607 Ptr =
608 DAG.getMemBasePlusOffset(Ptr, TypeSize::getFixed(IncrementSize), dl);
609 Hi = DAG.getNode(
610 ISD::SRL, dl, Value.getValueType(), Value,
611 DAG.getConstant(RoundWidth, dl,
612 TLI.getShiftAmountTy(Value.getValueType(), DL)));
613 Hi = DAG.getTruncStore(Chain, dl, Hi, Ptr,
614 ST->getPointerInfo().getWithOffset(IncrementSize),
615 ExtraVT, ST->getOriginalAlign(), MMOFlags, AAInfo);
616 } else {
617 // Big endian - avoid unaligned stores.
618 // TRUNCSTORE:i24 X -> TRUNCSTORE:i16 (srl X, 8), TRUNCSTORE@+2:i8 X
619 // Store the top RoundWidth bits.
620 Hi = DAG.getNode(
621 ISD::SRL, dl, Value.getValueType(), Value,
622 DAG.getConstant(ExtraWidth, dl,
623 TLI.getShiftAmountTy(Value.getValueType(), DL)));
624 Hi = DAG.getTruncStore(Chain, dl, Hi, Ptr, ST->getPointerInfo(), RoundVT,
625 ST->getOriginalAlign(), MMOFlags, AAInfo);
626
627 // Store the remaining ExtraWidth bits.
628 IncrementSize = RoundWidth / 8;
629 Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr,
630 DAG.getConstant(IncrementSize, dl,
631 Ptr.getValueType()));
632 Lo = DAG.getTruncStore(Chain, dl, Value, Ptr,
633 ST->getPointerInfo().getWithOffset(IncrementSize),
634 ExtraVT, ST->getOriginalAlign(), MMOFlags, AAInfo);
635 }
636
637 // The order of the stores doesn't matter.
638 SDValue Result = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo, Hi);
639 ReplaceNode(SDValue(Node, 0), Result);
640 } else {
641 switch (TLI.getTruncStoreAction(ST->getValue().getValueType(), StVT)) {
642 default: llvm_unreachable("This action is not supported yet!");
643 case TargetLowering::Legal: {
644 EVT MemVT = ST->getMemoryVT();
645 // If this is an unaligned store and the target doesn't support it,
646 // expand it.
647 if (!TLI.allowsMemoryAccessForAlignment(*DAG.getContext(), DL, MemVT,
648 *ST->getMemOperand())) {
649 SDValue Result = TLI.expandUnalignedStore(ST, DAG);
650 ReplaceNode(SDValue(ST, 0), Result);
651 }
652 break;
653 }
654 case TargetLowering::Custom: {
655 SDValue Res = TLI.LowerOperation(SDValue(Node, 0), DAG);
656 if (Res && Res != SDValue(Node, 0))
657 ReplaceNode(SDValue(Node, 0), Res);
658 return;
659 }
660 case TargetLowering::Expand:
661 assert(!StVT.isVector() &&
662 "Vector Stores are handled in LegalizeVectorOps");
663
665
666 // TRUNCSTORE:i16 i32 -> STORE i16
667 if (TLI.isTypeLegal(StVT)) {
668 Value = DAG.getNode(ISD::TRUNCATE, dl, StVT, Value);
669 Result = DAG.getStore(Chain, dl, Value, Ptr, ST->getPointerInfo(),
670 ST->getOriginalAlign(), MMOFlags, AAInfo);
671 } else {
672 // The in-memory type isn't legal. Truncate to the type it would promote
673 // to, and then do a truncstore.
674 Value = DAG.getNode(ISD::TRUNCATE, dl,
675 TLI.getTypeToTransformTo(*DAG.getContext(), StVT),
676 Value);
677 Result =
678 DAG.getTruncStore(Chain, dl, Value, Ptr, ST->getPointerInfo(), StVT,
679 ST->getOriginalAlign(), MMOFlags, AAInfo);
680 }
681
682 ReplaceNode(SDValue(Node, 0), Result);
683 break;
684 }
685 }
686}
687
688void SelectionDAGLegalize::LegalizeLoadOps(SDNode *Node) {
689 LoadSDNode *LD = cast<LoadSDNode>(Node);
690 SDValue Chain = LD->getChain(); // The chain.
691 SDValue Ptr = LD->getBasePtr(); // The base pointer.
692 SDValue Value; // The value returned by the load op.
693 SDLoc dl(Node);
694
695 ISD::LoadExtType ExtType = LD->getExtensionType();
696 if (ExtType == ISD::NON_EXTLOAD) {
697 LLVM_DEBUG(dbgs() << "Legalizing non-extending load operation\n");
698 MVT VT = Node->getSimpleValueType(0);
699 SDValue RVal = SDValue(Node, 0);
700 SDValue RChain = SDValue(Node, 1);
701
702 switch (TLI.getOperationAction(Node->getOpcode(), VT)) {
703 default: llvm_unreachable("This action is not supported yet!");
704 case TargetLowering::Legal: {
705 EVT MemVT = LD->getMemoryVT();
706 const DataLayout &DL = DAG.getDataLayout();
707 // If this is an unaligned load and the target doesn't support it,
708 // expand it.
709 if (!TLI.allowsMemoryAccessForAlignment(*DAG.getContext(), DL, MemVT,
710 *LD->getMemOperand())) {
711 std::tie(RVal, RChain) = TLI.expandUnalignedLoad(LD, DAG);
712 }
713 break;
714 }
715 case TargetLowering::Custom:
716 if (SDValue Res = TLI.LowerOperation(RVal, DAG)) {
717 RVal = Res;
718 RChain = Res.getValue(1);
719 }
720 break;
721
722 case TargetLowering::Promote: {
723 MVT NVT = TLI.getTypeToPromoteTo(Node->getOpcode(), VT);
724 assert(NVT.getSizeInBits() == VT.getSizeInBits() &&
725 "Can only promote loads to same size type");
726
727 SDValue Res = DAG.getLoad(NVT, dl, Chain, Ptr, LD->getMemOperand());
728 RVal = DAG.getNode(ISD::BITCAST, dl, VT, Res);
729 RChain = Res.getValue(1);
730 break;
731 }
732 }
733 if (RChain.getNode() != Node) {
734 assert(RVal.getNode() != Node && "Load must be completely replaced");
735 DAG.ReplaceAllUsesOfValueWith(SDValue(Node, 0), RVal);
736 DAG.ReplaceAllUsesOfValueWith(SDValue(Node, 1), RChain);
737 if (UpdatedNodes) {
738 UpdatedNodes->insert(RVal.getNode());
739 UpdatedNodes->insert(RChain.getNode());
740 }
741 ReplacedNode(Node);
742 }
743 return;
744 }
745
746 LLVM_DEBUG(dbgs() << "Legalizing extending load operation\n");
747 EVT SrcVT = LD->getMemoryVT();
748 TypeSize SrcWidth = SrcVT.getSizeInBits();
749 MachineMemOperand::Flags MMOFlags = LD->getMemOperand()->getFlags();
750 AAMDNodes AAInfo = LD->getAAInfo();
751
752 if (SrcWidth != SrcVT.getStoreSizeInBits() &&
753 // Some targets pretend to have an i1 loading operation, and actually
754 // load an i8. This trick is correct for ZEXTLOAD because the top 7
755 // bits are guaranteed to be zero; it helps the optimizers understand
756 // that these bits are zero. It is also useful for EXTLOAD, since it
757 // tells the optimizers that those bits are undefined. It would be
758 // nice to have an effective generic way of getting these benefits...
759 // Until such a way is found, don't insist on promoting i1 here.
760 (SrcVT != MVT::i1 ||
761 TLI.getLoadExtAction(ExtType, Node->getValueType(0), MVT::i1) ==
762 TargetLowering::Promote)) {
763 // Promote to a byte-sized load if not loading an integral number of
764 // bytes. For example, promote EXTLOAD:i20 -> EXTLOAD:i24.
765 unsigned NewWidth = SrcVT.getStoreSizeInBits();
766 EVT NVT = EVT::getIntegerVT(*DAG.getContext(), NewWidth);
767 SDValue Ch;
768
769 // The extra bits are guaranteed to be zero, since we stored them that
770 // way. A zext load from NVT thus automatically gives zext from SrcVT.
771
772 ISD::LoadExtType NewExtType =
774
775 SDValue Result = DAG.getExtLoad(NewExtType, dl, Node->getValueType(0),
776 Chain, Ptr, LD->getPointerInfo(), NVT,
777 LD->getOriginalAlign(), MMOFlags, AAInfo);
778
779 Ch = Result.getValue(1); // The chain.
780
781 if (ExtType == ISD::SEXTLOAD)
782 // Having the top bits zero doesn't help when sign extending.
783 Result = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl,
784 Result.getValueType(),
785 Result, DAG.getValueType(SrcVT));
786 else if (ExtType == ISD::ZEXTLOAD || NVT == Result.getValueType())
787 // All the top bits are guaranteed to be zero - inform the optimizers.
788 Result = DAG.getNode(ISD::AssertZext, dl,
789 Result.getValueType(), Result,
790 DAG.getValueType(SrcVT));
791
792 Value = Result;
793 Chain = Ch;
794 } else if (!isPowerOf2_64(SrcWidth.getKnownMinValue())) {
795 // If not loading a power-of-2 number of bits, expand as two loads.
796 assert(!SrcVT.isVector() && "Unsupported extload!");
797 unsigned SrcWidthBits = SrcWidth.getFixedValue();
798 unsigned LogSrcWidth = Log2_32(SrcWidthBits);
799 assert(LogSrcWidth < 32);
800 unsigned RoundWidth = 1 << LogSrcWidth;
801 assert(RoundWidth < SrcWidthBits);
802 unsigned ExtraWidth = SrcWidthBits - RoundWidth;
803 assert(ExtraWidth < RoundWidth);
804 assert(!(RoundWidth % 8) && !(ExtraWidth % 8) &&
805 "Load size not an integral number of bytes!");
806 EVT RoundVT = EVT::getIntegerVT(*DAG.getContext(), RoundWidth);
807 EVT ExtraVT = EVT::getIntegerVT(*DAG.getContext(), ExtraWidth);
808 SDValue Lo, Hi, Ch;
809 unsigned IncrementSize;
810 auto &DL = DAG.getDataLayout();
811
812 if (DL.isLittleEndian()) {
813 // EXTLOAD:i24 -> ZEXTLOAD:i16 | (shl EXTLOAD@+2:i8, 16)
814 // Load the bottom RoundWidth bits.
815 Lo = DAG.getExtLoad(ISD::ZEXTLOAD, dl, Node->getValueType(0), Chain, Ptr,
816 LD->getPointerInfo(), RoundVT, LD->getOriginalAlign(),
817 MMOFlags, AAInfo);
818
819 // Load the remaining ExtraWidth bits.
820 IncrementSize = RoundWidth / 8;
821 Ptr =
822 DAG.getMemBasePlusOffset(Ptr, TypeSize::getFixed(IncrementSize), dl);
823 Hi = DAG.getExtLoad(ExtType, dl, Node->getValueType(0), Chain, Ptr,
824 LD->getPointerInfo().getWithOffset(IncrementSize),
825 ExtraVT, LD->getOriginalAlign(), MMOFlags, AAInfo);
826
827 // Build a factor node to remember that this load is independent of
828 // the other one.
829 Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1),
830 Hi.getValue(1));
831
832 // Move the top bits to the right place.
833 Hi = DAG.getNode(
834 ISD::SHL, dl, Hi.getValueType(), Hi,
835 DAG.getConstant(RoundWidth, dl,
836 TLI.getShiftAmountTy(Hi.getValueType(), DL)));
837
838 // Join the hi and lo parts.
839 Value = DAG.getNode(ISD::OR, dl, Node->getValueType(0), Lo, Hi);
840 } else {
841 // Big endian - avoid unaligned loads.
842 // EXTLOAD:i24 -> (shl EXTLOAD:i16, 8) | ZEXTLOAD@+2:i8
843 // Load the top RoundWidth bits.
844 Hi = DAG.getExtLoad(ExtType, dl, Node->getValueType(0), Chain, Ptr,
845 LD->getPointerInfo(), RoundVT, LD->getOriginalAlign(),
846 MMOFlags, AAInfo);
847
848 // Load the remaining ExtraWidth bits.
849 IncrementSize = RoundWidth / 8;
850 Ptr =
851 DAG.getMemBasePlusOffset(Ptr, TypeSize::getFixed(IncrementSize), dl);
852 Lo = DAG.getExtLoad(ISD::ZEXTLOAD, dl, Node->getValueType(0), Chain, Ptr,
853 LD->getPointerInfo().getWithOffset(IncrementSize),
854 ExtraVT, LD->getOriginalAlign(), MMOFlags, AAInfo);
855
856 // Build a factor node to remember that this load is independent of
857 // the other one.
858 Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1),
859 Hi.getValue(1));
860
861 // Move the top bits to the right place.
862 Hi = DAG.getNode(
863 ISD::SHL, dl, Hi.getValueType(), Hi,
864 DAG.getConstant(ExtraWidth, dl,
865 TLI.getShiftAmountTy(Hi.getValueType(), DL)));
866
867 // Join the hi and lo parts.
868 Value = DAG.getNode(ISD::OR, dl, Node->getValueType(0), Lo, Hi);
869 }
870
871 Chain = Ch;
872 } else {
873 bool isCustom = false;
874 switch (TLI.getLoadExtAction(ExtType, Node->getValueType(0),
875 SrcVT.getSimpleVT())) {
876 default: llvm_unreachable("This action is not supported yet!");
877 case TargetLowering::Custom:
878 isCustom = true;
879 [[fallthrough]];
880 case TargetLowering::Legal:
881 Value = SDValue(Node, 0);
882 Chain = SDValue(Node, 1);
883
884 if (isCustom) {
885 if (SDValue Res = TLI.LowerOperation(SDValue(Node, 0), DAG)) {
886 Value = Res;
887 Chain = Res.getValue(1);
888 }
889 } else {
890 // If this is an unaligned load and the target doesn't support it,
891 // expand it.
892 EVT MemVT = LD->getMemoryVT();
893 const DataLayout &DL = DAG.getDataLayout();
894 if (!TLI.allowsMemoryAccess(*DAG.getContext(), DL, MemVT,
895 *LD->getMemOperand())) {
896 std::tie(Value, Chain) = TLI.expandUnalignedLoad(LD, DAG);
897 }
898 }
899 break;
900
901 case TargetLowering::Expand: {
902 EVT DestVT = Node->getValueType(0);
903 if (!TLI.isLoadExtLegal(ISD::EXTLOAD, DestVT, SrcVT)) {
904 // If the source type is not legal, see if there is a legal extload to
905 // an intermediate type that we can then extend further.
906 EVT LoadVT = TLI.getRegisterType(SrcVT.getSimpleVT());
907 if ((LoadVT.isFloatingPoint() == SrcVT.isFloatingPoint()) &&
908 (TLI.isTypeLegal(SrcVT) || // Same as SrcVT == LoadVT?
909 TLI.isLoadExtLegal(ExtType, LoadVT, SrcVT))) {
910 // If we are loading a legal type, this is a non-extload followed by a
911 // full extend.
912 ISD::LoadExtType MidExtType =
913 (LoadVT == SrcVT) ? ISD::NON_EXTLOAD : ExtType;
914
915 SDValue Load = DAG.getExtLoad(MidExtType, dl, LoadVT, Chain, Ptr,
916 SrcVT, LD->getMemOperand());
917 unsigned ExtendOp =
919 Value = DAG.getNode(ExtendOp, dl, Node->getValueType(0), Load);
920 Chain = Load.getValue(1);
921 break;
922 }
923
924 // Handle the special case of fp16 extloads. EXTLOAD doesn't have the
925 // normal undefined upper bits behavior to allow using an in-reg extend
926 // with the illegal FP type, so load as an integer and do the
927 // from-integer conversion.
928 EVT SVT = SrcVT.getScalarType();
929 if (SVT == MVT::f16 || SVT == MVT::bf16) {
930 EVT ISrcVT = SrcVT.changeTypeToInteger();
931 EVT IDestVT = DestVT.changeTypeToInteger();
932 EVT ILoadVT = TLI.getRegisterType(IDestVT.getSimpleVT());
933
934 SDValue Result = DAG.getExtLoad(ISD::ZEXTLOAD, dl, ILoadVT, Chain,
935 Ptr, ISrcVT, LD->getMemOperand());
936 Value =
937 DAG.getNode(SVT == MVT::f16 ? ISD::FP16_TO_FP : ISD::BF16_TO_FP,
938 dl, DestVT, Result);
939 Chain = Result.getValue(1);
940 break;
941 }
942 }
943
944 assert(!SrcVT.isVector() &&
945 "Vector Loads are handled in LegalizeVectorOps");
946
947 // FIXME: This does not work for vectors on most targets. Sign-
948 // and zero-extend operations are currently folded into extending
949 // loads, whether they are legal or not, and then we end up here
950 // without any support for legalizing them.
951 assert(ExtType != ISD::EXTLOAD &&
952 "EXTLOAD should always be supported!");
953 // Turn the unsupported load into an EXTLOAD followed by an
954 // explicit zero/sign extend inreg.
955 SDValue Result = DAG.getExtLoad(ISD::EXTLOAD, dl,
956 Node->getValueType(0),
957 Chain, Ptr, SrcVT,
958 LD->getMemOperand());
959 SDValue ValRes;
960 if (ExtType == ISD::SEXTLOAD)
961 ValRes = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl,
962 Result.getValueType(),
963 Result, DAG.getValueType(SrcVT));
964 else
965 ValRes = DAG.getZeroExtendInReg(Result, dl, SrcVT);
966 Value = ValRes;
967 Chain = Result.getValue(1);
968 break;
969 }
970 }
971 }
972
973 // Since loads produce two values, make sure to remember that we legalized
974 // both of them.
975 if (Chain.getNode() != Node) {
976 assert(Value.getNode() != Node && "Load must be completely replaced");
977 DAG.ReplaceAllUsesOfValueWith(SDValue(Node, 0), Value);
978 DAG.ReplaceAllUsesOfValueWith(SDValue(Node, 1), Chain);
979 if (UpdatedNodes) {
980 UpdatedNodes->insert(Value.getNode());
981 UpdatedNodes->insert(Chain.getNode());
982 }
983 ReplacedNode(Node);
984 }
985}
986
987/// Return a legal replacement for the given operation, with all legal operands.
988void SelectionDAGLegalize::LegalizeOp(SDNode *Node) {
989 LLVM_DEBUG(dbgs() << "\nLegalizing: "; Node->dump(&DAG));
990
991 // Allow illegal target nodes and illegal registers.
992 if (Node->getOpcode() == ISD::TargetConstant ||
993 Node->getOpcode() == ISD::Register)
994 return;
995
996#ifndef NDEBUG
997 for (unsigned i = 0, e = Node->getNumValues(); i != e; ++i)
998 assert(TLI.getTypeAction(*DAG.getContext(), Node->getValueType(i)) ==
999 TargetLowering::TypeLegal &&
1000 "Unexpected illegal type!");
1001
1002 for (const SDValue &Op : Node->op_values())
1003 assert((TLI.getTypeAction(*DAG.getContext(), Op.getValueType()) ==
1004 TargetLowering::TypeLegal ||
1005 Op.getOpcode() == ISD::TargetConstant ||
1006 Op.getOpcode() == ISD::Register) &&
1007 "Unexpected illegal type!");
1008#endif
1009
1010 // Figure out the correct action; the way to query this varies by opcode
1011 TargetLowering::LegalizeAction Action = TargetLowering::Legal;
1012 bool SimpleFinishLegalizing = true;
1013 switch (Node->getOpcode()) {
1017 case ISD::STACKSAVE:
1018 Action = TLI.getOperationAction(Node->getOpcode(), MVT::Other);
1019 break;
1021 Action = TLI.getOperationAction(Node->getOpcode(),
1022 Node->getValueType(0));
1023 break;
1024 case ISD::VAARG:
1025 Action = TLI.getOperationAction(Node->getOpcode(),
1026 Node->getValueType(0));
1027 if (Action != TargetLowering::Promote)
1028 Action = TLI.getOperationAction(Node->getOpcode(), MVT::Other);
1029 break;
1030 case ISD::SET_FPENV:
1031 case ISD::SET_FPMODE:
1032 Action = TLI.getOperationAction(Node->getOpcode(),
1033 Node->getOperand(1).getValueType());
1034 break;
1035 case ISD::FP_TO_FP16:
1036 case ISD::FP_TO_BF16:
1037 case ISD::SINT_TO_FP:
1038 case ISD::UINT_TO_FP:
1040 case ISD::LROUND:
1041 case ISD::LLROUND:
1042 case ISD::LRINT:
1043 case ISD::LLRINT:
1044 Action = TLI.getOperationAction(Node->getOpcode(),
1045 Node->getOperand(0).getValueType());
1046 break;
1051 case ISD::STRICT_LRINT:
1052 case ISD::STRICT_LLRINT:
1053 case ISD::STRICT_LROUND:
1055 // These pseudo-ops are the same as the other STRICT_ ops except
1056 // they are registered with setOperationAction() using the input type
1057 // instead of the output type.
1058 Action = TLI.getOperationAction(Node->getOpcode(),
1059 Node->getOperand(1).getValueType());
1060 break;
1062 EVT InnerType = cast<VTSDNode>(Node->getOperand(1))->getVT();
1063 Action = TLI.getOperationAction(Node->getOpcode(), InnerType);
1064 break;
1065 }
1066 case ISD::ATOMIC_STORE:
1067 Action = TLI.getOperationAction(Node->getOpcode(),
1068 Node->getOperand(1).getValueType());
1069 break;
1070 case ISD::SELECT_CC:
1071 case ISD::STRICT_FSETCC:
1073 case ISD::SETCC:
1074 case ISD::SETCCCARRY:
1075 case ISD::VP_SETCC:
1076 case ISD::BR_CC: {
1077 unsigned Opc = Node->getOpcode();
1078 unsigned CCOperand = Opc == ISD::SELECT_CC ? 4
1079 : Opc == ISD::STRICT_FSETCC ? 3
1080 : Opc == ISD::STRICT_FSETCCS ? 3
1081 : Opc == ISD::SETCCCARRY ? 3
1082 : (Opc == ISD::SETCC || Opc == ISD::VP_SETCC) ? 2
1083 : 1;
1084 unsigned CompareOperand = Opc == ISD::BR_CC ? 2
1085 : Opc == ISD::STRICT_FSETCC ? 1
1086 : Opc == ISD::STRICT_FSETCCS ? 1
1087 : 0;
1088 MVT OpVT = Node->getOperand(CompareOperand).getSimpleValueType();
1089 ISD::CondCode CCCode =
1090 cast<CondCodeSDNode>(Node->getOperand(CCOperand))->get();
1091 Action = TLI.getCondCodeAction(CCCode, OpVT);
1092 if (Action == TargetLowering::Legal) {
1093 if (Node->getOpcode() == ISD::SELECT_CC)
1094 Action = TLI.getOperationAction(Node->getOpcode(),
1095 Node->getValueType(0));
1096 else
1097 Action = TLI.getOperationAction(Node->getOpcode(), OpVT);
1098 }
1099 break;
1100 }
1101 case ISD::LOAD:
1102 case ISD::STORE:
1103 // FIXME: Model these properly. LOAD and STORE are complicated, and
1104 // STORE expects the unlegalized operand in some cases.
1105 SimpleFinishLegalizing = false;
1106 break;
1107 case ISD::CALLSEQ_START:
1108 case ISD::CALLSEQ_END:
1109 // FIXME: This shouldn't be necessary. These nodes have special properties
1110 // dealing with the recursive nature of legalization. Removing this
1111 // special case should be done as part of making LegalizeDAG non-recursive.
1112 SimpleFinishLegalizing = false;
1113 break;
1115 case ISD::GET_ROUNDING:
1116 case ISD::MERGE_VALUES:
1117 case ISD::EH_RETURN:
1119 case ISD::EH_DWARF_CFA:
1123 // These operations lie about being legal: when they claim to be legal,
1124 // they should actually be expanded.
1125 Action = TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0));
1126 if (Action == TargetLowering::Legal)
1127 Action = TargetLowering::Expand;
1128 break;
1131 case ISD::FRAMEADDR:
1132 case ISD::RETURNADDR:
1134 case ISD::SPONENTRY:
1135 // These operations lie about being legal: when they claim to be legal,
1136 // they should actually be custom-lowered.
1137 Action = TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0));
1138 if (Action == TargetLowering::Legal)
1139 Action = TargetLowering::Custom;
1140 break;
1143 // READCYCLECOUNTER and READSTEADYCOUNTER return a i64, even if type
1144 // legalization might have expanded that to several smaller types.
1145 Action = TLI.getOperationAction(Node->getOpcode(), MVT::i64);
1146 break;
1147 case ISD::READ_REGISTER:
1149 // Named register is legal in the DAG, but blocked by register name
1150 // selection if not implemented by target (to chose the correct register)
1151 // They'll be converted to Copy(To/From)Reg.
1152 Action = TargetLowering::Legal;
1153 break;
1154 case ISD::UBSANTRAP:
1155 Action = TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0));
1156 if (Action == TargetLowering::Expand) {
1157 // replace ISD::UBSANTRAP with ISD::TRAP
1158 SDValue NewVal;
1159 NewVal = DAG.getNode(ISD::TRAP, SDLoc(Node), Node->getVTList(),
1160 Node->getOperand(0));
1161 ReplaceNode(Node, NewVal.getNode());
1162 LegalizeOp(NewVal.getNode());
1163 return;
1164 }
1165 break;
1166 case ISD::DEBUGTRAP:
1167 Action = TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0));
1168 if (Action == TargetLowering::Expand) {
1169 // replace ISD::DEBUGTRAP with ISD::TRAP
1170 SDValue NewVal;
1171 NewVal = DAG.getNode(ISD::TRAP, SDLoc(Node), Node->getVTList(),
1172 Node->getOperand(0));
1173 ReplaceNode(Node, NewVal.getNode());
1174 LegalizeOp(NewVal.getNode());
1175 return;
1176 }
1177 break;
1178 case ISD::SADDSAT:
1179 case ISD::UADDSAT:
1180 case ISD::SSUBSAT:
1181 case ISD::USUBSAT:
1182 case ISD::SSHLSAT:
1183 case ISD::USHLSAT:
1186 Action = TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0));
1187 break;
1188 case ISD::SMULFIX:
1189 case ISD::SMULFIXSAT:
1190 case ISD::UMULFIX:
1191 case ISD::UMULFIXSAT:
1192 case ISD::SDIVFIX:
1193 case ISD::SDIVFIXSAT:
1194 case ISD::UDIVFIX:
1195 case ISD::UDIVFIXSAT: {
1196 unsigned Scale = Node->getConstantOperandVal(2);
1197 Action = TLI.getFixedPointOperationAction(Node->getOpcode(),
1198 Node->getValueType(0), Scale);
1199 break;
1200 }
1201 case ISD::MSCATTER:
1202 Action = TLI.getOperationAction(Node->getOpcode(),
1203 cast<MaskedScatterSDNode>(Node)->getValue().getValueType());
1204 break;
1205 case ISD::MSTORE:
1206 Action = TLI.getOperationAction(Node->getOpcode(),
1207 cast<MaskedStoreSDNode>(Node)->getValue().getValueType());
1208 break;
1209 case ISD::VP_SCATTER:
1210 Action = TLI.getOperationAction(
1211 Node->getOpcode(),
1212 cast<VPScatterSDNode>(Node)->getValue().getValueType());
1213 break;
1214 case ISD::VP_STORE:
1215 Action = TLI.getOperationAction(
1216 Node->getOpcode(),
1217 cast<VPStoreSDNode>(Node)->getValue().getValueType());
1218 break;
1219 case ISD::EXPERIMENTAL_VP_STRIDED_STORE:
1220 Action = TLI.getOperationAction(
1221 Node->getOpcode(),
1222 cast<VPStridedStoreSDNode>(Node)->getValue().getValueType());
1223 break;
1226 case ISD::VECREDUCE_ADD:
1227 case ISD::VECREDUCE_MUL:
1228 case ISD::VECREDUCE_AND:
1229 case ISD::VECREDUCE_OR:
1230 case ISD::VECREDUCE_XOR:
1239 case ISD::IS_FPCLASS:
1240 Action = TLI.getOperationAction(
1241 Node->getOpcode(), Node->getOperand(0).getValueType());
1242 break;
1245 case ISD::VP_REDUCE_FADD:
1246 case ISD::VP_REDUCE_FMUL:
1247 case ISD::VP_REDUCE_ADD:
1248 case ISD::VP_REDUCE_MUL:
1249 case ISD::VP_REDUCE_AND:
1250 case ISD::VP_REDUCE_OR:
1251 case ISD::VP_REDUCE_XOR:
1252 case ISD::VP_REDUCE_SMAX:
1253 case ISD::VP_REDUCE_SMIN:
1254 case ISD::VP_REDUCE_UMAX:
1255 case ISD::VP_REDUCE_UMIN:
1256 case ISD::VP_REDUCE_FMAX:
1257 case ISD::VP_REDUCE_FMIN:
1258 case ISD::VP_REDUCE_SEQ_FADD:
1259 case ISD::VP_REDUCE_SEQ_FMUL:
1260 Action = TLI.getOperationAction(
1261 Node->getOpcode(), Node->getOperand(1).getValueType());
1262 break;
1263 default:
1264 if (Node->getOpcode() >= ISD::BUILTIN_OP_END) {
1265 Action = TLI.getCustomOperationAction(*Node);
1266 } else {
1267 Action = TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0));
1268 }
1269 break;
1270 }
1271
1272 if (SimpleFinishLegalizing) {
1273 SDNode *NewNode = Node;
1274 switch (Node->getOpcode()) {
1275 default: break;
1276 case ISD::SHL:
1277 case ISD::SRL:
1278 case ISD::SRA:
1279 case ISD::ROTL:
1280 case ISD::ROTR: {
1281 // Legalizing shifts/rotates requires adjusting the shift amount
1282 // to the appropriate width.
1283 SDValue Op0 = Node->getOperand(0);
1284 SDValue Op1 = Node->getOperand(1);
1285 if (!Op1.getValueType().isVector()) {
1286 SDValue SAO = DAG.getShiftAmountOperand(Op0.getValueType(), Op1);
1287 // The getShiftAmountOperand() may create a new operand node or
1288 // return the existing one. If new operand is created we need
1289 // to update the parent node.
1290 // Do not try to legalize SAO here! It will be automatically legalized
1291 // in the next round.
1292 if (SAO != Op1)
1293 NewNode = DAG.UpdateNodeOperands(Node, Op0, SAO);
1294 }
1295 }
1296 break;
1297 case ISD::FSHL:
1298 case ISD::FSHR:
1299 case ISD::SRL_PARTS:
1300 case ISD::SRA_PARTS:
1301 case ISD::SHL_PARTS: {
1302 // Legalizing shifts/rotates requires adjusting the shift amount
1303 // to the appropriate width.
1304 SDValue Op0 = Node->getOperand(0);
1305 SDValue Op1 = Node->getOperand(1);
1306 SDValue Op2 = Node->getOperand(2);
1307 if (!Op2.getValueType().isVector()) {
1308 SDValue SAO = DAG.getShiftAmountOperand(Op0.getValueType(), Op2);
1309 // The getShiftAmountOperand() may create a new operand node or
1310 // return the existing one. If new operand is created we need
1311 // to update the parent node.
1312 if (SAO != Op2)
1313 NewNode = DAG.UpdateNodeOperands(Node, Op0, Op1, SAO);
1314 }
1315 break;
1316 }
1317 }
1318
1319 if (NewNode != Node) {
1320 ReplaceNode(Node, NewNode);
1321 Node = NewNode;
1322 }
1323 switch (Action) {
1324 case TargetLowering::Legal:
1325 LLVM_DEBUG(dbgs() << "Legal node: nothing to do\n");
1326 return;
1327 case TargetLowering::Custom:
1328 LLVM_DEBUG(dbgs() << "Trying custom legalization\n");
1329 // FIXME: The handling for custom lowering with multiple results is
1330 // a complete mess.
1331 if (SDValue Res = TLI.LowerOperation(SDValue(Node, 0), DAG)) {
1332 if (!(Res.getNode() != Node || Res.getResNo() != 0))
1333 return;
1334
1335 if (Node->getNumValues() == 1) {
1336 // Verify the new types match the original. Glue is waived because
1337 // ISD::ADDC can be legalized by replacing Glue with an integer type.
1338 assert((Res.getValueType() == Node->getValueType(0) ||
1339 Node->getValueType(0) == MVT::Glue) &&
1340 "Type mismatch for custom legalized operation");
1341 LLVM_DEBUG(dbgs() << "Successfully custom legalized node\n");
1342 // We can just directly replace this node with the lowered value.
1343 ReplaceNode(SDValue(Node, 0), Res);
1344 return;
1345 }
1346
1347 SmallVector<SDValue, 8> ResultVals;
1348 for (unsigned i = 0, e = Node->getNumValues(); i != e; ++i) {
1349 // Verify the new types match the original. Glue is waived because
1350 // ISD::ADDC can be legalized by replacing Glue with an integer type.
1351 assert((Res->getValueType(i) == Node->getValueType(i) ||
1352 Node->getValueType(i) == MVT::Glue) &&
1353 "Type mismatch for custom legalized operation");
1354 ResultVals.push_back(Res.getValue(i));
1355 }
1356 LLVM_DEBUG(dbgs() << "Successfully custom legalized node\n");
1357 ReplaceNode(Node, ResultVals.data());
1358 return;
1359 }
1360 LLVM_DEBUG(dbgs() << "Could not custom legalize node\n");
1361 [[fallthrough]];
1362 case TargetLowering::Expand:
1363 if (ExpandNode(Node))
1364 return;
1365 [[fallthrough]];
1366 case TargetLowering::LibCall:
1367 ConvertNodeToLibcall(Node);
1368 return;
1369 case TargetLowering::Promote:
1370 PromoteNode(Node);
1371 return;
1372 }
1373 }
1374
1375 switch (Node->getOpcode()) {
1376 default:
1377#ifndef NDEBUG
1378 dbgs() << "NODE: ";
1379 Node->dump( &DAG);
1380 dbgs() << "\n";
1381#endif
1382 llvm_unreachable("Do not know how to legalize this operator!");
1383
1384 case ISD::CALLSEQ_START:
1385 case ISD::CALLSEQ_END:
1386 break;
1387 case ISD::LOAD:
1388 return LegalizeLoadOps(Node);
1389 case ISD::STORE:
1390 return LegalizeStoreOps(Node);
1391 }
1392}
1393
1394SDValue SelectionDAGLegalize::ExpandExtractFromVectorThroughStack(SDValue Op) {
1395 SDValue Vec = Op.getOperand(0);
1396 SDValue Idx = Op.getOperand(1);
1397 SDLoc dl(Op);
1398
1399 // Before we generate a new store to a temporary stack slot, see if there is
1400 // already one that we can use. There often is because when we scalarize
1401 // vector operations (using SelectionDAG::UnrollVectorOp for example) a whole
1402 // series of EXTRACT_VECTOR_ELT nodes are generated, one for each element in
1403 // the vector. If all are expanded here, we don't want one store per vector
1404 // element.
1405
1406 // Caches for hasPredecessorHelper
1409 Visited.insert(Op.getNode());
1410 Worklist.push_back(Idx.getNode());
1411 SDValue StackPtr, Ch;
1412 for (SDNode *User : Vec.getNode()->uses()) {
1413 if (StoreSDNode *ST = dyn_cast<StoreSDNode>(User)) {
1414 if (ST->isIndexed() || ST->isTruncatingStore() ||
1415 ST->getValue() != Vec)
1416 continue;
1417
1418 // Make sure that nothing else could have stored into the destination of
1419 // this store.
1420 if (!ST->getChain().reachesChainWithoutSideEffects(DAG.getEntryNode()))
1421 continue;
1422
1423 // If the index is dependent on the store we will introduce a cycle when
1424 // creating the load (the load uses the index, and by replacing the chain
1425 // we will make the index dependent on the load). Also, the store might be
1426 // dependent on the extractelement and introduce a cycle when creating
1427 // the load.
1428 if (SDNode::hasPredecessorHelper(ST, Visited, Worklist) ||
1429 ST->hasPredecessor(Op.getNode()))
1430 continue;
1431
1432 StackPtr = ST->getBasePtr();
1433 Ch = SDValue(ST, 0);
1434 break;
1435 }
1436 }
1437
1438 EVT VecVT = Vec.getValueType();
1439
1440 if (!Ch.getNode()) {
1441 // Store the value to a temporary stack slot, then LOAD the returned part.
1442 StackPtr = DAG.CreateStackTemporary(VecVT);
1444 StackPtr, DAG.getMachineFunction(), VecVT.isScalableVector());
1445 Ch = DAG.getStore(DAG.getEntryNode(), dl, Vec, StackPtr, StoreMMO);
1446 }
1447
1448 SDValue NewLoad;
1449 Align ElementAlignment =
1450 std::min(cast<StoreSDNode>(Ch)->getAlign(),
1451 DAG.getDataLayout().getPrefTypeAlign(
1452 Op.getValueType().getTypeForEVT(*DAG.getContext())));
1453
1454 if (Op.getValueType().isVector()) {
1455 StackPtr = TLI.getVectorSubVecPointer(DAG, StackPtr, VecVT,
1456 Op.getValueType(), Idx);
1457 NewLoad = DAG.getLoad(Op.getValueType(), dl, Ch, StackPtr,
1458 MachinePointerInfo(), ElementAlignment);
1459 } else {
1460 StackPtr = TLI.getVectorElementPointer(DAG, StackPtr, VecVT, Idx);
1461 NewLoad = DAG.getExtLoad(ISD::EXTLOAD, dl, Op.getValueType(), Ch, StackPtr,
1463 ElementAlignment);
1464 }
1465
1466 // Replace the chain going out of the store, by the one out of the load.
1467 DAG.ReplaceAllUsesOfValueWith(Ch, SDValue(NewLoad.getNode(), 1));
1468
1469 // We introduced a cycle though, so update the loads operands, making sure
1470 // to use the original store's chain as an incoming chain.
1471 SmallVector<SDValue, 6> NewLoadOperands(NewLoad->op_begin(),
1472 NewLoad->op_end());
1473 NewLoadOperands[0] = Ch;
1474 NewLoad =
1475 SDValue(DAG.UpdateNodeOperands(NewLoad.getNode(), NewLoadOperands), 0);
1476 return NewLoad;
1477}
1478
1479SDValue SelectionDAGLegalize::ExpandInsertToVectorThroughStack(SDValue Op) {
1480 assert(Op.getValueType().isVector() && "Non-vector insert subvector!");
1481
1482 SDValue Vec = Op.getOperand(0);
1483 SDValue Part = Op.getOperand(1);
1484 SDValue Idx = Op.getOperand(2);
1485 SDLoc dl(Op);
1486
1487 // Store the value to a temporary stack slot, then LOAD the returned part.
1488 EVT VecVT = Vec.getValueType();
1489 EVT SubVecVT = Part.getValueType();
1490 SDValue StackPtr = DAG.CreateStackTemporary(VecVT);
1491 int FI = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex();
1492 MachinePointerInfo PtrInfo =
1493 MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), FI);
1494
1495 // First store the whole vector.
1496 SDValue Ch = DAG.getStore(DAG.getEntryNode(), dl, Vec, StackPtr, PtrInfo);
1497
1498 // Then store the inserted part.
1499 SDValue SubStackPtr =
1500 TLI.getVectorSubVecPointer(DAG, StackPtr, VecVT, SubVecVT, Idx);
1501
1502 // Store the subvector.
1503 Ch = DAG.getStore(
1504 Ch, dl, Part, SubStackPtr,
1505 MachinePointerInfo::getUnknownStack(DAG.getMachineFunction()));
1506
1507 // Finally, load the updated vector.
1508 return DAG.getLoad(Op.getValueType(), dl, Ch, StackPtr, PtrInfo);
1509}
1510
1511SDValue SelectionDAGLegalize::ExpandVectorBuildThroughStack(SDNode* Node) {
1512 assert((Node->getOpcode() == ISD::BUILD_VECTOR ||
1513 Node->getOpcode() == ISD::CONCAT_VECTORS) &&
1514 "Unexpected opcode!");
1515
1516 // We can't handle this case efficiently. Allocate a sufficiently
1517 // aligned object on the stack, store each operand into it, then load
1518 // the result as a vector.
1519 // Create the stack frame object.
1520 EVT VT = Node->getValueType(0);
1521 EVT MemVT = isa<BuildVectorSDNode>(Node) ? VT.getVectorElementType()
1522 : Node->getOperand(0).getValueType();
1523 SDLoc dl(Node);
1524 SDValue FIPtr = DAG.CreateStackTemporary(VT);
1525 int FI = cast<FrameIndexSDNode>(FIPtr.getNode())->getIndex();
1526 MachinePointerInfo PtrInfo =
1527 MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), FI);
1528
1529 // Emit a store of each element to the stack slot.
1531 unsigned TypeByteSize = MemVT.getSizeInBits() / 8;
1532 assert(TypeByteSize > 0 && "Vector element type too small for stack store!");
1533
1534 // If the destination vector element type of a BUILD_VECTOR is narrower than
1535 // the source element type, only store the bits necessary.
1536 bool Truncate = isa<BuildVectorSDNode>(Node) &&
1537 MemVT.bitsLT(Node->getOperand(0).getValueType());
1538
1539 // Store (in the right endianness) the elements to memory.
1540 for (unsigned i = 0, e = Node->getNumOperands(); i != e; ++i) {
1541 // Ignore undef elements.
1542 if (Node->getOperand(i).isUndef()) continue;
1543
1544 unsigned Offset = TypeByteSize*i;
1545
1546 SDValue Idx =
1547 DAG.getMemBasePlusOffset(FIPtr, TypeSize::getFixed(Offset), dl);
1548
1549 if (Truncate)
1550 Stores.push_back(DAG.getTruncStore(DAG.getEntryNode(), dl,
1551 Node->getOperand(i), Idx,
1552 PtrInfo.getWithOffset(Offset), MemVT));
1553 else
1554 Stores.push_back(DAG.getStore(DAG.getEntryNode(), dl, Node->getOperand(i),
1555 Idx, PtrInfo.getWithOffset(Offset)));
1556 }
1557
1558 SDValue StoreChain;
1559 if (!Stores.empty()) // Not all undef elements?
1560 StoreChain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Stores);
1561 else
1562 StoreChain = DAG.getEntryNode();
1563
1564 // Result is a load from the stack slot.
1565 return DAG.getLoad(VT, dl, StoreChain, FIPtr, PtrInfo);
1566}
1567
1568/// Bitcast a floating-point value to an integer value. Only bitcast the part
1569/// containing the sign bit if the target has no integer value capable of
1570/// holding all bits of the floating-point value.
1571void SelectionDAGLegalize::getSignAsIntValue(FloatSignAsInt &State,
1572 const SDLoc &DL,
1573 SDValue Value) const {
1574 EVT FloatVT = Value.getValueType();
1575 unsigned NumBits = FloatVT.getScalarSizeInBits();
1576 State.FloatVT = FloatVT;
1577 EVT IVT = EVT::getIntegerVT(*DAG.getContext(), NumBits);
1578 // Convert to an integer of the same size.
1579 if (TLI.isTypeLegal(IVT)) {
1580 State.IntValue = DAG.getNode(ISD::BITCAST, DL, IVT, Value);
1581 State.SignMask = APInt::getSignMask(NumBits);
1582 State.SignBit = NumBits - 1;
1583 return;
1584 }
1585
1586 auto &DataLayout = DAG.getDataLayout();
1587 // Store the float to memory, then load the sign part out as an integer.
1588 MVT LoadTy = TLI.getRegisterType(MVT::i8);
1589 // First create a temporary that is aligned for both the load and store.
1590 SDValue StackPtr = DAG.CreateStackTemporary(FloatVT, LoadTy);
1591 int FI = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex();
1592 // Then store the float to it.
1593 State.FloatPtr = StackPtr;
1594 MachineFunction &MF = DAG.getMachineFunction();
1595 State.FloatPointerInfo = MachinePointerInfo::getFixedStack(MF, FI);
1596 State.Chain = DAG.getStore(DAG.getEntryNode(), DL, Value, State.FloatPtr,
1597 State.FloatPointerInfo);
1598
1599 SDValue IntPtr;
1600 if (DataLayout.isBigEndian()) {
1601 assert(FloatVT.isByteSized() && "Unsupported floating point type!");
1602 // Load out a legal integer with the same sign bit as the float.
1603 IntPtr = StackPtr;
1604 State.IntPointerInfo = State.FloatPointerInfo;
1605 } else {
1606 // Advance the pointer so that the loaded byte will contain the sign bit.
1607 unsigned ByteOffset = (NumBits / 8) - 1;
1608 IntPtr =
1609 DAG.getMemBasePlusOffset(StackPtr, TypeSize::getFixed(ByteOffset), DL);
1610 State.IntPointerInfo = MachinePointerInfo::getFixedStack(MF, FI,
1611 ByteOffset);
1612 }
1613
1614 State.IntPtr = IntPtr;
1615 State.IntValue = DAG.getExtLoad(ISD::EXTLOAD, DL, LoadTy, State.Chain, IntPtr,
1616 State.IntPointerInfo, MVT::i8);
1617 State.SignMask = APInt::getOneBitSet(LoadTy.getScalarSizeInBits(), 7);
1618 State.SignBit = 7;
1619}
1620
1621/// Replace the integer value produced by getSignAsIntValue() with a new value
1622/// and cast the result back to a floating-point type.
1623SDValue SelectionDAGLegalize::modifySignAsInt(const FloatSignAsInt &State,
1624 const SDLoc &DL,
1625 SDValue NewIntValue) const {
1626 if (!State.Chain)
1627 return DAG.getNode(ISD::BITCAST, DL, State.FloatVT, NewIntValue);
1628
1629 // Override the part containing the sign bit in the value stored on the stack.
1630 SDValue Chain = DAG.getTruncStore(State.Chain, DL, NewIntValue, State.IntPtr,
1631 State.IntPointerInfo, MVT::i8);
1632 return DAG.getLoad(State.FloatVT, DL, Chain, State.FloatPtr,
1633 State.FloatPointerInfo);
1634}
1635
1636SDValue SelectionDAGLegalize::ExpandFCOPYSIGN(SDNode *Node) const {
1637 SDLoc DL(Node);
1638 SDValue Mag = Node->getOperand(0);
1639 SDValue Sign = Node->getOperand(1);
1640
1641 // Get sign bit into an integer value.
1642 FloatSignAsInt SignAsInt;
1643 getSignAsIntValue(SignAsInt, DL, Sign);
1644
1645 EVT IntVT = SignAsInt.IntValue.getValueType();
1646 SDValue SignMask = DAG.getConstant(SignAsInt.SignMask, DL, IntVT);
1647 SDValue SignBit = DAG.getNode(ISD::AND, DL, IntVT, SignAsInt.IntValue,
1648 SignMask);
1649
1650 // If FABS is legal transform FCOPYSIGN(x, y) => sign(x) ? -FABS(x) : FABS(X)
1651 EVT FloatVT = Mag.getValueType();
1652 if (TLI.isOperationLegalOrCustom(ISD::FABS, FloatVT) &&
1653 TLI.isOperationLegalOrCustom(ISD::FNEG, FloatVT)) {
1654 SDValue AbsValue = DAG.getNode(ISD::FABS, DL, FloatVT, Mag);
1655 SDValue NegValue = DAG.getNode(ISD::FNEG, DL, FloatVT, AbsValue);
1656 SDValue Cond = DAG.getSetCC(DL, getSetCCResultType(IntVT), SignBit,
1657 DAG.getConstant(0, DL, IntVT), ISD::SETNE);
1658 return DAG.getSelect(DL, FloatVT, Cond, NegValue, AbsValue);
1659 }
1660
1661 // Transform Mag value to integer, and clear the sign bit.
1662 FloatSignAsInt MagAsInt;
1663 getSignAsIntValue(MagAsInt, DL, Mag);
1664 EVT MagVT = MagAsInt.IntValue.getValueType();
1665 SDValue ClearSignMask = DAG.getConstant(~MagAsInt.SignMask, DL, MagVT);
1666 SDValue ClearedSign = DAG.getNode(ISD::AND, DL, MagVT, MagAsInt.IntValue,
1667 ClearSignMask);
1668
1669 // Get the signbit at the right position for MagAsInt.
1670 int ShiftAmount = SignAsInt.SignBit - MagAsInt.SignBit;
1671 EVT ShiftVT = IntVT;
1672 if (SignBit.getScalarValueSizeInBits() <
1673 ClearedSign.getScalarValueSizeInBits()) {
1674 SignBit = DAG.getNode(ISD::ZERO_EXTEND, DL, MagVT, SignBit);
1675 ShiftVT = MagVT;
1676 }
1677 if (ShiftAmount > 0) {
1678 SDValue ShiftCnst = DAG.getConstant(ShiftAmount, DL, ShiftVT);
1679 SignBit = DAG.getNode(ISD::SRL, DL, ShiftVT, SignBit, ShiftCnst);
1680 } else if (ShiftAmount < 0) {
1681 SDValue ShiftCnst = DAG.getConstant(-ShiftAmount, DL, ShiftVT);
1682 SignBit = DAG.getNode(ISD::SHL, DL, ShiftVT, SignBit, ShiftCnst);
1683 }
1684 if (SignBit.getScalarValueSizeInBits() >
1685 ClearedSign.getScalarValueSizeInBits()) {
1686 SignBit = DAG.getNode(ISD::TRUNCATE, DL, MagVT, SignBit);
1687 }
1688
1689 // Store the part with the modified sign and convert back to float.
1690 SDValue CopiedSign = DAG.getNode(ISD::OR, DL, MagVT, ClearedSign, SignBit);
1691 return modifySignAsInt(MagAsInt, DL, CopiedSign);
1692}
1693
1694SDValue SelectionDAGLegalize::ExpandFNEG(SDNode *Node) const {
1695 // Get the sign bit as an integer.
1696 SDLoc DL(Node);
1697 FloatSignAsInt SignAsInt;
1698 getSignAsIntValue(SignAsInt, DL, Node->getOperand(0));
1699 EVT IntVT = SignAsInt.IntValue.getValueType();
1700
1701 // Flip the sign.
1702 SDValue SignMask = DAG.getConstant(SignAsInt.SignMask, DL, IntVT);
1703 SDValue SignFlip =
1704 DAG.getNode(ISD::XOR, DL, IntVT, SignAsInt.IntValue, SignMask);
1705
1706 // Convert back to float.
1707 return modifySignAsInt(SignAsInt, DL, SignFlip);
1708}
1709
1710SDValue SelectionDAGLegalize::ExpandFABS(SDNode *Node) const {
1711 SDLoc DL(Node);
1712 SDValue Value = Node->getOperand(0);
1713
1714 // Transform FABS(x) => FCOPYSIGN(x, 0.0) if FCOPYSIGN is legal.
1715 EVT FloatVT = Value.getValueType();
1716 if (TLI.isOperationLegalOrCustom(ISD::FCOPYSIGN, FloatVT)) {
1717 SDValue Zero = DAG.getConstantFP(0.0, DL, FloatVT);
1718 return DAG.getNode(ISD::FCOPYSIGN, DL, FloatVT, Value, Zero);
1719 }
1720
1721 // Transform value to integer, clear the sign bit and transform back.
1722 FloatSignAsInt ValueAsInt;
1723 getSignAsIntValue(ValueAsInt, DL, Value);
1724 EVT IntVT = ValueAsInt.IntValue.getValueType();
1725 SDValue ClearSignMask = DAG.getConstant(~ValueAsInt.SignMask, DL, IntVT);
1726 SDValue ClearedSign = DAG.getNode(ISD::AND, DL, IntVT, ValueAsInt.IntValue,
1727 ClearSignMask);
1728 return modifySignAsInt(ValueAsInt, DL, ClearedSign);
1729}
1730
1731void SelectionDAGLegalize::ExpandDYNAMIC_STACKALLOC(SDNode* Node,
1733 Register SPReg = TLI.getStackPointerRegisterToSaveRestore();
1734 assert(SPReg && "Target cannot require DYNAMIC_STACKALLOC expansion and"
1735 " not tell us which reg is the stack pointer!");
1736 SDLoc dl(Node);
1737 EVT VT = Node->getValueType(0);
1738 SDValue Tmp1 = SDValue(Node, 0);
1739 SDValue Tmp2 = SDValue(Node, 1);
1740 SDValue Tmp3 = Node->getOperand(2);
1741 SDValue Chain = Tmp1.getOperand(0);
1742
1743 // Chain the dynamic stack allocation so that it doesn't modify the stack
1744 // pointer when other instructions are using the stack.
1745 Chain = DAG.getCALLSEQ_START(Chain, 0, 0, dl);
1746
1747 SDValue Size = Tmp2.getOperand(1);
1748 SDValue SP = DAG.getCopyFromReg(Chain, dl, SPReg, VT);
1749 Chain = SP.getValue(1);
1750 Align Alignment = cast<ConstantSDNode>(Tmp3)->getAlignValue();
1751 const TargetFrameLowering *TFL = DAG.getSubtarget().getFrameLowering();
1752 unsigned Opc =
1755
1756 Align StackAlign = TFL->getStackAlign();
1757 Tmp1 = DAG.getNode(Opc, dl, VT, SP, Size); // Value
1758 if (Alignment > StackAlign)
1759 Tmp1 = DAG.getNode(ISD::AND, dl, VT, Tmp1,
1760 DAG.getConstant(-Alignment.value(), dl, VT));
1761 Chain = DAG.getCopyToReg(Chain, dl, SPReg, Tmp1); // Output chain
1762
1763 Tmp2 = DAG.getCALLSEQ_END(Chain, 0, 0, SDValue(), dl);
1764
1765 Results.push_back(Tmp1);
1766 Results.push_back(Tmp2);
1767}
1768
1769/// Emit a store/load combination to the stack. This stores
1770/// SrcOp to a stack slot of type SlotVT, truncating it if needed. It then does
1771/// a load from the stack slot to DestVT, extending it if needed.
1772/// The resultant code need not be legal.
1773SDValue SelectionDAGLegalize::EmitStackConvert(SDValue SrcOp, EVT SlotVT,
1774 EVT DestVT, const SDLoc &dl) {
1775 return EmitStackConvert(SrcOp, SlotVT, DestVT, dl, DAG.getEntryNode());
1776}
1777
1778SDValue SelectionDAGLegalize::EmitStackConvert(SDValue SrcOp, EVT SlotVT,
1779 EVT DestVT, const SDLoc &dl,
1780 SDValue Chain) {
1781 EVT SrcVT = SrcOp.getValueType();
1782 Type *DestType = DestVT.getTypeForEVT(*DAG.getContext());
1783 Align DestAlign = DAG.getDataLayout().getPrefTypeAlign(DestType);
1784
1785 // Don't convert with stack if the load/store is expensive.
1786 if ((SrcVT.bitsGT(SlotVT) &&
1787 !TLI.isTruncStoreLegalOrCustom(SrcOp.getValueType(), SlotVT)) ||
1788 (SlotVT.bitsLT(DestVT) &&
1789 !TLI.isLoadExtLegalOrCustom(ISD::EXTLOAD, DestVT, SlotVT)))
1790 return SDValue();
1791
1792 // Create the stack frame object.
1793 Align SrcAlign = DAG.getDataLayout().getPrefTypeAlign(
1794 SrcOp.getValueType().getTypeForEVT(*DAG.getContext()));
1795 SDValue FIPtr = DAG.CreateStackTemporary(SlotVT.getStoreSize(), SrcAlign);
1796
1797 FrameIndexSDNode *StackPtrFI = cast<FrameIndexSDNode>(FIPtr);
1798 int SPFI = StackPtrFI->getIndex();
1799 MachinePointerInfo PtrInfo =
1800 MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), SPFI);
1801
1802 // Emit a store to the stack slot. Use a truncstore if the input value is
1803 // later than DestVT.
1804 SDValue Store;
1805
1806 if (SrcVT.bitsGT(SlotVT))
1807 Store = DAG.getTruncStore(Chain, dl, SrcOp, FIPtr, PtrInfo,
1808 SlotVT, SrcAlign);
1809 else {
1810 assert(SrcVT.bitsEq(SlotVT) && "Invalid store");
1811 Store = DAG.getStore(Chain, dl, SrcOp, FIPtr, PtrInfo, SrcAlign);
1812 }
1813
1814 // Result is a load from the stack slot.
1815 if (SlotVT.bitsEq(DestVT))
1816 return DAG.getLoad(DestVT, dl, Store, FIPtr, PtrInfo, DestAlign);
1817
1818 assert(SlotVT.bitsLT(DestVT) && "Unknown extension!");
1819 return DAG.getExtLoad(ISD::EXTLOAD, dl, DestVT, Store, FIPtr, PtrInfo, SlotVT,
1820 DestAlign);
1821}
1822
1823SDValue SelectionDAGLegalize::ExpandSCALAR_TO_VECTOR(SDNode *Node) {
1824 SDLoc dl(Node);
1825 // Create a vector sized/aligned stack slot, store the value to element #0,
1826 // then load the whole vector back out.
1827 SDValue StackPtr = DAG.CreateStackTemporary(Node->getValueType(0));
1828
1829 FrameIndexSDNode *StackPtrFI = cast<FrameIndexSDNode>(StackPtr);
1830 int SPFI = StackPtrFI->getIndex();
1831
1832 SDValue Ch = DAG.getTruncStore(
1833 DAG.getEntryNode(), dl, Node->getOperand(0), StackPtr,
1834 MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), SPFI),
1835 Node->getValueType(0).getVectorElementType());
1836 return DAG.getLoad(
1837 Node->getValueType(0), dl, Ch, StackPtr,
1838 MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), SPFI));
1839}
1840
1841static bool
1843 const TargetLowering &TLI, SDValue &Res) {
1844 unsigned NumElems = Node->getNumOperands();
1845 SDLoc dl(Node);
1846 EVT VT = Node->getValueType(0);
1847
1848 // Try to group the scalars into pairs, shuffle the pairs together, then
1849 // shuffle the pairs of pairs together, etc. until the vector has
1850 // been built. This will work only if all of the necessary shuffle masks
1851 // are legal.
1852
1853 // We do this in two phases; first to check the legality of the shuffles,
1854 // and next, assuming that all shuffles are legal, to create the new nodes.
1855 for (int Phase = 0; Phase < 2; ++Phase) {
1857 NewIntermedVals;
1858 for (unsigned i = 0; i < NumElems; ++i) {
1859 SDValue V = Node->getOperand(i);
1860 if (V.isUndef())
1861 continue;
1862
1863 SDValue Vec;
1864 if (Phase)
1865 Vec = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, V);
1866 IntermedVals.push_back(std::make_pair(Vec, SmallVector<int, 16>(1, i)));
1867 }
1868
1869 while (IntermedVals.size() > 2) {
1870 NewIntermedVals.clear();
1871 for (unsigned i = 0, e = (IntermedVals.size() & ~1u); i < e; i += 2) {
1872 // This vector and the next vector are shuffled together (simply to
1873 // append the one to the other).
1874 SmallVector<int, 16> ShuffleVec(NumElems, -1);
1875
1876 SmallVector<int, 16> FinalIndices;
1877 FinalIndices.reserve(IntermedVals[i].second.size() +
1878 IntermedVals[i+1].second.size());
1879
1880 int k = 0;
1881 for (unsigned j = 0, f = IntermedVals[i].second.size(); j != f;
1882 ++j, ++k) {
1883 ShuffleVec[k] = j;
1884 FinalIndices.push_back(IntermedVals[i].second[j]);
1885 }
1886 for (unsigned j = 0, f = IntermedVals[i+1].second.size(); j != f;
1887 ++j, ++k) {
1888 ShuffleVec[k] = NumElems + j;
1889 FinalIndices.push_back(IntermedVals[i+1].second[j]);
1890 }
1891
1892 SDValue Shuffle;
1893 if (Phase)
1894 Shuffle = DAG.getVectorShuffle(VT, dl, IntermedVals[i].first,
1895 IntermedVals[i+1].first,
1896 ShuffleVec);
1897 else if (!TLI.isShuffleMaskLegal(ShuffleVec, VT))
1898 return false;
1899 NewIntermedVals.push_back(
1900 std::make_pair(Shuffle, std::move(FinalIndices)));
1901 }
1902
1903 // If we had an odd number of defined values, then append the last
1904 // element to the array of new vectors.
1905 if ((IntermedVals.size() & 1) != 0)
1906 NewIntermedVals.push_back(IntermedVals.back());
1907
1908 IntermedVals.swap(NewIntermedVals);
1909 }
1910
1911 assert(IntermedVals.size() <= 2 && IntermedVals.size() > 0 &&
1912 "Invalid number of intermediate vectors");
1913 SDValue Vec1 = IntermedVals[0].first;
1914 SDValue Vec2;
1915 if (IntermedVals.size() > 1)
1916 Vec2 = IntermedVals[1].first;
1917 else if (Phase)
1918 Vec2 = DAG.getUNDEF(VT);
1919
1920 SmallVector<int, 16> ShuffleVec(NumElems, -1);
1921 for (unsigned i = 0, e = IntermedVals[0].second.size(); i != e; ++i)
1922 ShuffleVec[IntermedVals[0].second[i]] = i;
1923 for (unsigned i = 0, e = IntermedVals[1].second.size(); i != e; ++i)
1924 ShuffleVec[IntermedVals[1].second[i]] = NumElems + i;
1925
1926 if (Phase)
1927 Res = DAG.getVectorShuffle(VT, dl, Vec1, Vec2, ShuffleVec);
1928 else if (!TLI.isShuffleMaskLegal(ShuffleVec, VT))
1929 return false;
1930 }
1931
1932 return true;
1933}
1934
1935/// Expand a BUILD_VECTOR node on targets that don't
1936/// support the operation, but do support the resultant vector type.
1937SDValue SelectionDAGLegalize::ExpandBUILD_VECTOR(SDNode *Node) {
1938 unsigned NumElems = Node->getNumOperands();
1939 SDValue Value1, Value2;
1940 SDLoc dl(Node);
1941 EVT VT = Node->getValueType(0);
1942 EVT OpVT = Node->getOperand(0).getValueType();
1943 EVT EltVT = VT.getVectorElementType();
1944
1945 // If the only non-undef value is the low element, turn this into a
1946 // SCALAR_TO_VECTOR node. If this is { X, X, X, X }, determine X.
1947 bool isOnlyLowElement = true;
1948 bool MoreThanTwoValues = false;
1949 bool isConstant = true;
1950 for (unsigned i = 0; i < NumElems; ++i) {
1951 SDValue V = Node->getOperand(i);
1952 if (V.isUndef())
1953 continue;
1954 if (i > 0)
1955 isOnlyLowElement = false;
1956 if (!isa<ConstantFPSDNode>(V) && !isa<ConstantSDNode>(V))
1957 isConstant = false;
1958
1959 if (!Value1.getNode()) {
1960 Value1 = V;
1961 } else if (!Value2.getNode()) {
1962 if (V != Value1)
1963 Value2 = V;
1964 } else if (V != Value1 && V != Value2) {
1965 MoreThanTwoValues = true;
1966 }
1967 }
1968
1969 if (!Value1.getNode())
1970 return DAG.getUNDEF(VT);
1971
1972 if (isOnlyLowElement)
1973 return DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, Node->getOperand(0));
1974
1975 // If all elements are constants, create a load from the constant pool.
1976 if (isConstant) {
1978 for (unsigned i = 0, e = NumElems; i != e; ++i) {
1979 if (ConstantFPSDNode *V =
1980 dyn_cast<ConstantFPSDNode>(Node->getOperand(i))) {
1981 CV.push_back(const_cast<ConstantFP *>(V->getConstantFPValue()));
1982 } else if (ConstantSDNode *V =
1983 dyn_cast<ConstantSDNode>(Node->getOperand(i))) {
1984 if (OpVT==EltVT)
1985 CV.push_back(const_cast<ConstantInt *>(V->getConstantIntValue()));
1986 else {
1987 // If OpVT and EltVT don't match, EltVT is not legal and the
1988 // element values have been promoted/truncated earlier. Undo this;
1989 // we don't want a v16i8 to become a v16i32 for example.
1990 const ConstantInt *CI = V->getConstantIntValue();
1991 CV.push_back(ConstantInt::get(EltVT.getTypeForEVT(*DAG.getContext()),
1992 CI->getZExtValue()));
1993 }
1994 } else {
1995 assert(Node->getOperand(i).isUndef());
1996 Type *OpNTy = EltVT.getTypeForEVT(*DAG.getContext());
1997 CV.push_back(UndefValue::get(OpNTy));
1998 }
1999 }
2001 SDValue CPIdx =
2002 DAG.getConstantPool(CP, TLI.getPointerTy(DAG.getDataLayout()));
2003 Align Alignment = cast<ConstantPoolSDNode>(CPIdx)->getAlign();
2004 return DAG.getLoad(
2005 VT, dl, DAG.getEntryNode(), CPIdx,
2006 MachinePointerInfo::getConstantPool(DAG.getMachineFunction()),
2007 Alignment);
2008 }
2009
2010 SmallSet<SDValue, 16> DefinedValues;
2011 for (unsigned i = 0; i < NumElems; ++i) {
2012 if (Node->getOperand(i).isUndef())
2013 continue;
2014 DefinedValues.insert(Node->getOperand(i));
2015 }
2016
2017 if (TLI.shouldExpandBuildVectorWithShuffles(VT, DefinedValues.size())) {
2018 if (!MoreThanTwoValues) {
2019 SmallVector<int, 8> ShuffleVec(NumElems, -1);
2020 for (unsigned i = 0; i < NumElems; ++i) {
2021 SDValue V = Node->getOperand(i);
2022 if (V.isUndef())
2023 continue;
2024 ShuffleVec[i] = V == Value1 ? 0 : NumElems;
2025 }
2026 if (TLI.isShuffleMaskLegal(ShuffleVec, Node->getValueType(0))) {
2027 // Get the splatted value into the low element of a vector register.
2028 SDValue Vec1 = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, Value1);
2029 SDValue Vec2;
2030 if (Value2.getNode())
2031 Vec2 = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, Value2);
2032 else
2033 Vec2 = DAG.getUNDEF(VT);
2034
2035 // Return shuffle(LowValVec, undef, <0,0,0,0>)
2036 return DAG.getVectorShuffle(VT, dl, Vec1, Vec2, ShuffleVec);
2037 }
2038 } else {
2039 SDValue Res;
2040 if (ExpandBVWithShuffles(Node, DAG, TLI, Res))
2041 return Res;
2042 }
2043 }
2044
2045 // Otherwise, we can't handle this case efficiently.
2046 return ExpandVectorBuildThroughStack(Node);
2047}
2048
2049SDValue SelectionDAGLegalize::ExpandSPLAT_VECTOR(SDNode *Node) {
2050 SDLoc DL(Node);
2051 EVT VT = Node->getValueType(0);
2052 SDValue SplatVal = Node->getOperand(0);
2053
2054 return DAG.getSplatBuildVector(VT, DL, SplatVal);
2055}
2056
2057// Expand a node into a call to a libcall, returning the value as the first
2058// result and the chain as the second. If the result value does not fit into a
2059// register, return the lo part and set the hi part to the by-reg argument in
2060// the first. If it does fit into a single register, return the result and
2061// leave the Hi part unset.
2062std::pair<SDValue, SDValue> SelectionDAGLegalize::ExpandLibCall(RTLIB::Libcall LC, SDNode *Node,
2064 bool isSigned) {
2065 SDValue Callee = DAG.getExternalSymbol(TLI.getLibcallName(LC),
2066 TLI.getPointerTy(DAG.getDataLayout()));
2067
2068 EVT RetVT = Node->getValueType(0);
2069 Type *RetTy = RetVT.getTypeForEVT(*DAG.getContext());
2070
2071 // By default, the input chain to this libcall is the entry node of the
2072 // function. If the libcall is going to be emitted as a tail call then
2073 // TLI.isUsedByReturnOnly will change it to the right chain if the return
2074 // node which is being folded has a non-entry input chain.
2075 SDValue InChain = DAG.getEntryNode();
2076
2077 // isTailCall may be true since the callee does not reference caller stack
2078 // frame. Check if it's in the right position and that the return types match.
2079 SDValue TCChain = InChain;
2080 const Function &F = DAG.getMachineFunction().getFunction();
2081 bool isTailCall =
2082 TLI.isInTailCallPosition(DAG, Node, TCChain) &&
2083 (RetTy == F.getReturnType() || F.getReturnType()->isVoidTy());
2084 if (isTailCall)
2085 InChain = TCChain;
2086
2088 bool signExtend = TLI.shouldSignExtendTypeInLibCall(RetVT, isSigned);
2089 CLI.setDebugLoc(SDLoc(Node))
2090 .setChain(InChain)
2091 .setLibCallee(TLI.getLibcallCallingConv(LC), RetTy, Callee,
2092 std::move(Args))
2093 .setTailCall(isTailCall)
2094 .setSExtResult(signExtend)
2095 .setZExtResult(!signExtend)
2096 .setIsPostTypeLegalization(true);
2097
2098 std::pair<SDValue, SDValue> CallInfo = TLI.LowerCallTo(CLI);
2099
2100 if (!CallInfo.second.getNode()) {
2101 LLVM_DEBUG(dbgs() << "Created tailcall: "; DAG.getRoot().dump(&DAG));
2102 // It's a tailcall, return the chain (which is the DAG root).
2103 return {DAG.getRoot(), DAG.getRoot()};
2104 }
2105
2106 LLVM_DEBUG(dbgs() << "Created libcall: "; CallInfo.first.dump(&DAG));
2107 return CallInfo;
2108}
2109
2110std::pair<SDValue, SDValue> SelectionDAGLegalize::ExpandLibCall(RTLIB::Libcall LC, SDNode *Node,
2111 bool isSigned) {
2114 for (const SDValue &Op : Node->op_values()) {
2115 EVT ArgVT = Op.getValueType();
2116 Type *ArgTy = ArgVT.getTypeForEVT(*DAG.getContext());
2117 Entry.Node = Op;
2118 Entry.Ty = ArgTy;
2119 Entry.IsSExt = TLI.shouldSignExtendTypeInLibCall(ArgVT, isSigned);
2120 Entry.IsZExt = !Entry.IsSExt;
2121 Args.push_back(Entry);
2122 }
2123
2124 return ExpandLibCall(LC, Node, std::move(Args), isSigned);
2125}
2126
2127void SelectionDAGLegalize::ExpandFrexpLibCall(
2129 SDLoc dl(Node);
2130 EVT VT = Node->getValueType(0);
2131 EVT ExpVT = Node->getValueType(1);
2132
2133 SDValue FPOp = Node->getOperand(0);
2134
2135 EVT ArgVT = FPOp.getValueType();
2136 Type *ArgTy = ArgVT.getTypeForEVT(*DAG.getContext());
2137
2139 FPArgEntry.Node = FPOp;
2140 FPArgEntry.Ty = ArgTy;
2141
2142 SDValue StackSlot = DAG.CreateStackTemporary(ExpVT);
2143 TargetLowering::ArgListEntry PtrArgEntry;
2144 PtrArgEntry.Node = StackSlot;
2145 PtrArgEntry.Ty = PointerType::get(*DAG.getContext(),
2146 DAG.getDataLayout().getAllocaAddrSpace());
2147
2148 TargetLowering::ArgListTy Args = {FPArgEntry, PtrArgEntry};
2149
2151 auto [Call, Chain] = ExpandLibCall(LC, Node, std::move(Args), false);
2152
2153 // FIXME: Get type of int for libcall declaration and cast
2154
2155 int FrameIdx = cast<FrameIndexSDNode>(StackSlot)->getIndex();
2156 auto PtrInfo =
2157 MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), FrameIdx);
2158
2159 SDValue LoadExp = DAG.getLoad(ExpVT, dl, Chain, StackSlot, PtrInfo);
2160 SDValue OutputChain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other,
2161 LoadExp.getValue(1), DAG.getRoot());
2162 DAG.setRoot(OutputChain);
2163
2164 Results.push_back(Call);
2165 Results.push_back(LoadExp);
2166}
2167
2168void SelectionDAGLegalize::ExpandFPLibCall(SDNode* Node,
2169 RTLIB::Libcall LC,
2171 if (LC == RTLIB::UNKNOWN_LIBCALL)
2172 llvm_unreachable("Can't create an unknown libcall!");
2173
2174 if (Node->isStrictFPOpcode()) {
2175 EVT RetVT = Node->getValueType(0);
2178 // FIXME: This doesn't support tail calls.
2179 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, RetVT,
2180 Ops, CallOptions,
2181 SDLoc(Node),
2182 Node->getOperand(0));
2183 Results.push_back(Tmp.first);
2184 Results.push_back(Tmp.second);
2185 } else {
2186 SDValue Tmp = ExpandLibCall(LC, Node, false).first;
2187 Results.push_back(Tmp);
2188 }
2189}
2190
2191/// Expand the node to a libcall based on the result type.
2192void SelectionDAGLegalize::ExpandFPLibCall(SDNode* Node,
2193 RTLIB::Libcall Call_F32,
2194 RTLIB::Libcall Call_F64,
2195 RTLIB::Libcall Call_F80,
2196 RTLIB::Libcall Call_F128,
2197 RTLIB::Libcall Call_PPCF128,
2199 RTLIB::Libcall LC = RTLIB::getFPLibCall(Node->getSimpleValueType(0),
2200 Call_F32, Call_F64, Call_F80,
2201 Call_F128, Call_PPCF128);
2202 ExpandFPLibCall(Node, LC, Results);
2203}
2204
2205SDValue SelectionDAGLegalize::ExpandIntLibCall(SDNode* Node, bool isSigned,
2206 RTLIB::Libcall Call_I8,
2207 RTLIB::Libcall Call_I16,
2208 RTLIB::Libcall Call_I32,
2209 RTLIB::Libcall Call_I64,
2210 RTLIB::Libcall Call_I128) {
2211 RTLIB::Libcall LC;
2212 switch (Node->getSimpleValueType(0).SimpleTy) {
2213 default: llvm_unreachable("Unexpected request for libcall!");
2214 case MVT::i8: LC = Call_I8; break;
2215 case MVT::i16: LC = Call_I16; break;
2216 case MVT::i32: LC = Call_I32; break;
2217 case MVT::i64: LC = Call_I64; break;
2218 case MVT::i128: LC = Call_I128; break;
2219 }
2220 return ExpandLibCall(LC, Node, isSigned).first;
2221}
2222
2223/// Expand the node to a libcall based on first argument type (for instance
2224/// lround and its variant).
2225void SelectionDAGLegalize::ExpandArgFPLibCall(SDNode* Node,
2226 RTLIB::Libcall Call_F32,
2227 RTLIB::Libcall Call_F64,
2228 RTLIB::Libcall Call_F80,
2229 RTLIB::Libcall Call_F128,
2230 RTLIB::Libcall Call_PPCF128,
2232 EVT InVT = Node->getOperand(Node->isStrictFPOpcode() ? 1 : 0).getValueType();
2234 Call_F32, Call_F64, Call_F80,
2235 Call_F128, Call_PPCF128);
2236 ExpandFPLibCall(Node, LC, Results);
2237}
2238
2239/// Issue libcalls to __{u}divmod to compute div / rem pairs.
2240void
2241SelectionDAGLegalize::ExpandDivRemLibCall(SDNode *Node,
2243 unsigned Opcode = Node->getOpcode();
2244 bool isSigned = Opcode == ISD::SDIVREM;
2245
2246 RTLIB::Libcall LC;
2247 switch (Node->getSimpleValueType(0).SimpleTy) {
2248 default: llvm_unreachable("Unexpected request for libcall!");
2249 case MVT::i8: LC= isSigned ? RTLIB::SDIVREM_I8 : RTLIB::UDIVREM_I8; break;
2250 case MVT::i16: LC= isSigned ? RTLIB::SDIVREM_I16 : RTLIB::UDIVREM_I16; break;
2251 case MVT::i32: LC= isSigned ? RTLIB::SDIVREM_I32 : RTLIB::UDIVREM_I32; break;
2252 case MVT::i64: LC= isSigned ? RTLIB::SDIVREM_I64 : RTLIB::UDIVREM_I64; break;
2253 case MVT::i128: LC= isSigned ? RTLIB::SDIVREM_I128:RTLIB::UDIVREM_I128; break;
2254 }
2255
2256 // The input chain to this libcall is the entry node of the function.
2257 // Legalizing the call will automatically add the previous call to the
2258 // dependence.
2259 SDValue InChain = DAG.getEntryNode();
2260
2261 EVT RetVT = Node->getValueType(0);
2262 Type *RetTy = RetVT.getTypeForEVT(*DAG.getContext());
2263
2266 for (const SDValue &Op : Node->op_values()) {
2267 EVT ArgVT = Op.getValueType();
2268 Type *ArgTy = ArgVT.getTypeForEVT(*DAG.getContext());
2269 Entry.Node = Op;
2270 Entry.Ty = ArgTy;
2271 Entry.IsSExt = isSigned;
2272 Entry.IsZExt = !isSigned;
2273 Args.push_back(Entry);
2274 }
2275
2276 // Also pass the return address of the remainder.
2277 SDValue FIPtr = DAG.CreateStackTemporary(RetVT);
2278 Entry.Node = FIPtr;
2279 Entry.Ty = PointerType::getUnqual(RetTy->getContext());
2280 Entry.IsSExt = isSigned;
2281 Entry.IsZExt = !isSigned;
2282 Args.push_back(Entry);
2283
2284 SDValue Callee = DAG.getExternalSymbol(TLI.getLibcallName(LC),
2285 TLI.getPointerTy(DAG.getDataLayout()));
2286
2287 SDLoc dl(Node);
2289 CLI.setDebugLoc(dl)
2290 .setChain(InChain)
2291 .setLibCallee(TLI.getLibcallCallingConv(LC), RetTy, Callee,
2292 std::move(Args))
2293 .setSExtResult(isSigned)
2294 .setZExtResult(!isSigned);
2295
2296 std::pair<SDValue, SDValue> CallInfo = TLI.LowerCallTo(CLI);
2297
2298 // Remainder is loaded back from the stack frame.
2299 SDValue Rem =
2300 DAG.getLoad(RetVT, dl, CallInfo.second, FIPtr, MachinePointerInfo());
2301 Results.push_back(CallInfo.first);
2302 Results.push_back(Rem);
2303}
2304
2305/// Return true if sincos libcall is available.
2307 RTLIB::Libcall LC;
2308 switch (Node->getSimpleValueType(0).SimpleTy) {
2309 default: llvm_unreachable("Unexpected request for libcall!");
2310 case MVT::f32: LC = RTLIB::SINCOS_F32; break;
2311 case MVT::f64: LC = RTLIB::SINCOS_F64; break;
2312 case MVT::f80: LC = RTLIB::SINCOS_F80; break;
2313 case MVT::f128: LC = RTLIB::SINCOS_F128; break;
2314 case MVT::ppcf128: LC = RTLIB::SINCOS_PPCF128; break;
2315 }
2316 return TLI.getLibcallName(LC) != nullptr;
2317}
2318
2319/// Only issue sincos libcall if both sin and cos are needed.
2320static bool useSinCos(SDNode *Node) {
2321 unsigned OtherOpcode = Node->getOpcode() == ISD::FSIN
2322 ? ISD::FCOS : ISD::FSIN;
2323
2324 SDValue Op0 = Node->getOperand(0);
2325 for (const SDNode *User : Op0.getNode()->uses()) {
2326 if (User == Node)
2327 continue;
2328 // The other user might have been turned into sincos already.
2329 if (User->getOpcode() == OtherOpcode || User->getOpcode() == ISD::FSINCOS)
2330 return true;
2331 }
2332 return false;
2333}
2334
2335/// Issue libcalls to sincos to compute sin / cos pairs.
2336void
2337SelectionDAGLegalize::ExpandSinCosLibCall(SDNode *Node,
2339 RTLIB::Libcall LC;
2340 switch (Node->getSimpleValueType(0).SimpleTy) {
2341 default: llvm_unreachable("Unexpected request for libcall!");
2342 case MVT::f32: LC = RTLIB::SINCOS_F32; break;
2343 case MVT::f64: LC = RTLIB::SINCOS_F64; break;
2344 case MVT::f80: LC = RTLIB::SINCOS_F80; break;
2345 case MVT::f128: LC = RTLIB::SINCOS_F128; break;
2346 case MVT::ppcf128: LC = RTLIB::SINCOS_PPCF128; break;
2347 }
2348
2349 // The input chain to this libcall is the entry node of the function.
2350 // Legalizing the call will automatically add the previous call to the
2351 // dependence.
2352 SDValue InChain = DAG.getEntryNode();
2353
2354 EVT RetVT = Node->getValueType(0);
2355 Type *RetTy = RetVT.getTypeForEVT(*DAG.getContext());
2356
2359
2360 // Pass the argument.
2361 Entry.Node = Node->getOperand(0);
2362 Entry.Ty = RetTy;
2363 Entry.IsSExt = false;
2364 Entry.IsZExt = false;
2365 Args.push_back(Entry);
2366
2367 // Pass the return address of sin.
2368 SDValue SinPtr = DAG.CreateStackTemporary(RetVT);
2369 Entry.Node = SinPtr;
2370 Entry.Ty = PointerType::getUnqual(RetTy->getContext());
2371 Entry.IsSExt = false;
2372 Entry.IsZExt = false;
2373 Args.push_back(Entry);
2374
2375 // Also pass the return address of the cos.
2376 SDValue CosPtr = DAG.CreateStackTemporary(RetVT);
2377 Entry.Node = CosPtr;
2378 Entry.Ty = PointerType::getUnqual(RetTy->getContext());
2379 Entry.IsSExt = false;
2380 Entry.IsZExt = false;
2381 Args.push_back(Entry);
2382
2383 SDValue Callee = DAG.getExternalSymbol(TLI.getLibcallName(LC),
2384 TLI.getPointerTy(DAG.getDataLayout()));
2385
2386 SDLoc dl(Node);
2388 CLI.setDebugLoc(dl).setChain(InChain).setLibCallee(
2389 TLI.getLibcallCallingConv(LC), Type::getVoidTy(*DAG.getContext()), Callee,
2390 std::move(Args));
2391
2392 std::pair<SDValue, SDValue> CallInfo = TLI.LowerCallTo(CLI);
2393
2394 Results.push_back(
2395 DAG.getLoad(RetVT, dl, CallInfo.second, SinPtr, MachinePointerInfo()));
2396 Results.push_back(
2397 DAG.getLoad(RetVT, dl, CallInfo.second, CosPtr, MachinePointerInfo()));
2398}
2399
2400SDValue SelectionDAGLegalize::expandLdexp(SDNode *Node) const {
2401 SDLoc dl(Node);
2402 EVT VT = Node->getValueType(0);
2403 SDValue X = Node->getOperand(0);
2404 SDValue N = Node->getOperand(1);
2405 EVT ExpVT = N.getValueType();
2406 EVT AsIntVT = VT.changeTypeToInteger();
2407 if (AsIntVT == EVT()) // TODO: How to handle f80?
2408 return SDValue();
2409
2410 if (Node->getOpcode() == ISD::STRICT_FLDEXP) // TODO
2411 return SDValue();
2412
2413 SDNodeFlags NSW;
2414 NSW.setNoSignedWrap(true);
2415 SDNodeFlags NUW_NSW;
2416 NUW_NSW.setNoUnsignedWrap(true);
2417 NUW_NSW.setNoSignedWrap(true);
2418
2419 EVT SetCCVT =
2420 TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), ExpVT);
2422
2423 const APFloat::ExponentType MaxExpVal = APFloat::semanticsMaxExponent(FltSem);
2424 const APFloat::ExponentType MinExpVal = APFloat::semanticsMinExponent(FltSem);
2425 const int Precision = APFloat::semanticsPrecision(FltSem);
2426
2427 const SDValue MaxExp = DAG.getConstant(MaxExpVal, dl, ExpVT);
2428 const SDValue MinExp = DAG.getConstant(MinExpVal, dl, ExpVT);
2429
2430 const SDValue DoubleMaxExp = DAG.getConstant(2 * MaxExpVal, dl, ExpVT);
2431
2432 const APFloat One(FltSem, "1.0");
2433 APFloat ScaleUpK = scalbn(One, MaxExpVal, APFloat::rmNearestTiesToEven);
2434
2435 // Offset by precision to avoid denormal range.
2436 APFloat ScaleDownK =
2437 scalbn(One, MinExpVal + Precision, APFloat::rmNearestTiesToEven);
2438
2439 // TODO: Should really introduce control flow and use a block for the >
2440 // MaxExp, < MinExp cases
2441
2442 // First, handle exponents Exp > MaxExp and scale down.
2443 SDValue NGtMaxExp = DAG.getSetCC(dl, SetCCVT, N, MaxExp, ISD::SETGT);
2444
2445 SDValue DecN0 = DAG.getNode(ISD::SUB, dl, ExpVT, N, MaxExp, NSW);
2446 SDValue ClampMaxVal = DAG.getConstant(3 * MaxExpVal, dl, ExpVT);
2447 SDValue ClampN_Big = DAG.getNode(ISD::SMIN, dl, ExpVT, N, ClampMaxVal);
2448 SDValue DecN1 =
2449 DAG.getNode(ISD::SUB, dl, ExpVT, ClampN_Big, DoubleMaxExp, NSW);
2450
2451 SDValue ScaleUpTwice =
2452 DAG.getSetCC(dl, SetCCVT, N, DoubleMaxExp, ISD::SETUGT);
2453
2454 const SDValue ScaleUpVal = DAG.getConstantFP(ScaleUpK, dl, VT);
2455 SDValue ScaleUp0 = DAG.getNode(ISD::FMUL, dl, VT, X, ScaleUpVal);
2456 SDValue ScaleUp1 = DAG.getNode(ISD::FMUL, dl, VT, ScaleUp0, ScaleUpVal);
2457
2458 SDValue SelectN_Big =
2459 DAG.getNode(ISD::SELECT, dl, ExpVT, ScaleUpTwice, DecN1, DecN0);
2460 SDValue SelectX_Big =
2461 DAG.getNode(ISD::SELECT, dl, VT, ScaleUpTwice, ScaleUp1, ScaleUp0);
2462
2463 // Now handle exponents Exp < MinExp
2464 SDValue NLtMinExp = DAG.getSetCC(dl, SetCCVT, N, MinExp, ISD::SETLT);
2465
2466 SDValue Increment0 = DAG.getConstant(-(MinExpVal + Precision), dl, ExpVT);
2467 SDValue Increment1 = DAG.getConstant(-2 * (MinExpVal + Precision), dl, ExpVT);
2468
2469 SDValue IncN0 = DAG.getNode(ISD::ADD, dl, ExpVT, N, Increment0, NUW_NSW);
2470
2471 SDValue ClampMinVal =
2472 DAG.getConstant(3 * MinExpVal + 2 * Precision, dl, ExpVT);
2473 SDValue ClampN_Small = DAG.getNode(ISD::SMAX, dl, ExpVT, N, ClampMinVal);
2474 SDValue IncN1 =
2475 DAG.getNode(ISD::ADD, dl, ExpVT, ClampN_Small, Increment1, NSW);
2476
2477 const SDValue ScaleDownVal = DAG.getConstantFP(ScaleDownK, dl, VT);
2478 SDValue ScaleDown0 = DAG.getNode(ISD::FMUL, dl, VT, X, ScaleDownVal);
2479 SDValue ScaleDown1 = DAG.getNode(ISD::FMUL, dl, VT, ScaleDown0, ScaleDownVal);
2480
2481 SDValue ScaleDownTwice = DAG.getSetCC(
2482 dl, SetCCVT, N, DAG.getConstant(2 * MinExpVal + Precision, dl, ExpVT),
2483 ISD::SETULT);
2484
2485 SDValue SelectN_Small =
2486 DAG.getNode(ISD::SELECT, dl, ExpVT, ScaleDownTwice, IncN1, IncN0);
2487 SDValue SelectX_Small =
2488 DAG.getNode(ISD::SELECT, dl, VT, ScaleDownTwice, ScaleDown1, ScaleDown0);
2489
2490 // Now combine the two out of range exponent handling cases with the base
2491 // case.
2492 SDValue NewX = DAG.getNode(
2493 ISD::SELECT, dl, VT, NGtMaxExp, SelectX_Big,
2494 DAG.getNode(ISD::SELECT, dl, VT, NLtMinExp, SelectX_Small, X));
2495
2496 SDValue NewN = DAG.getNode(
2497 ISD::SELECT, dl, ExpVT, NGtMaxExp, SelectN_Big,
2498 DAG.getNode(ISD::SELECT, dl, ExpVT, NLtMinExp, SelectN_Small, N));
2499
2500 SDValue BiasedN = DAG.getNode(ISD::ADD, dl, ExpVT, NewN, MaxExp, NSW);
2501
2502 SDValue ExponentShiftAmt =
2503 DAG.getShiftAmountConstant(Precision - 1, ExpVT, dl);
2504 SDValue CastExpToValTy = DAG.getZExtOrTrunc(BiasedN, dl, AsIntVT);
2505
2506 SDValue AsInt = DAG.getNode(ISD::SHL, dl, AsIntVT, CastExpToValTy,
2507 ExponentShiftAmt, NUW_NSW);
2508 SDValue AsFP = DAG.getNode(ISD::BITCAST, dl, VT, AsInt);
2509 return DAG.getNode(ISD::FMUL, dl, VT, NewX, AsFP);
2510}
2511
2512SDValue SelectionDAGLegalize::expandFrexp(SDNode *Node) const {
2513 SDLoc dl(Node);
2514 SDValue Val = Node->getOperand(0);
2515 EVT VT = Val.getValueType();
2516 EVT ExpVT = Node->getValueType(1);
2517 EVT AsIntVT = VT.changeTypeToInteger();
2518 if (AsIntVT == EVT()) // TODO: How to handle f80?
2519 return SDValue();
2520
2522 const APFloat::ExponentType MinExpVal = APFloat::semanticsMinExponent(FltSem);
2523 const unsigned Precision = APFloat::semanticsPrecision(FltSem);
2524 const unsigned BitSize = VT.getScalarSizeInBits();
2525
2526 // TODO: Could introduce control flow and skip over the denormal handling.
2527
2528 // scale_up = fmul value, scalbn(1.0, precision + 1)
2529 // extracted_exp = (bitcast value to uint) >> precision - 1
2530 // biased_exp = extracted_exp + min_exp
2531 // extracted_fract = (bitcast value to uint) & (fract_mask | sign_mask)
2532 //
2533 // is_denormal = val < smallest_normalized
2534 // computed_fract = is_denormal ? scale_up : extracted_fract
2535 // computed_exp = is_denormal ? biased_exp + (-precision - 1) : biased_exp
2536 //
2537 // result_0 = (!isfinite(val) || iszero(val)) ? val : computed_fract
2538 // result_1 = (!isfinite(val) || iszero(val)) ? 0 : computed_exp
2539
2540 SDValue NegSmallestNormalizedInt = DAG.getConstant(
2541 APFloat::getSmallestNormalized(FltSem, true).bitcastToAPInt(), dl,
2542 AsIntVT);
2543
2544 SDValue SmallestNormalizedInt = DAG.getConstant(
2545 APFloat::getSmallestNormalized(FltSem, false).bitcastToAPInt(), dl,
2546 AsIntVT);
2547
2548 // Masks out the exponent bits.
2549 SDValue ExpMask =
2550 DAG.getConstant(APFloat::getInf(FltSem).bitcastToAPInt(), dl, AsIntVT);
2551
2552 // Mask out the exponent part of the value.
2553 //
2554 // e.g, for f32 FractSignMaskVal = 0x807fffff
2555 APInt FractSignMaskVal = APInt::getBitsSet(BitSize, 0, Precision - 1);
2556 FractSignMaskVal.setBit(BitSize - 1); // Set the sign bit
2557
2558 APInt SignMaskVal = APInt::getSignedMaxValue(BitSize);
2559 SDValue SignMask = DAG.getConstant(SignMaskVal, dl, AsIntVT);
2560
2561 SDValue FractSignMask = DAG.getConstant(FractSignMaskVal, dl, AsIntVT);
2562
2563 const APFloat One(FltSem, "1.0");
2564 // Scale a possible denormal input.
2565 // e.g., for f64, 0x1p+54
2566 APFloat ScaleUpKVal =
2567 scalbn(One, Precision + 1, APFloat::rmNearestTiesToEven);
2568
2569 SDValue ScaleUpK = DAG.getConstantFP(ScaleUpKVal, dl, VT);
2570 SDValue ScaleUp = DAG.getNode(ISD::FMUL, dl, VT, Val, ScaleUpK);
2571
2572 EVT SetCCVT =
2573 TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT);
2574
2575 SDValue AsInt = DAG.getNode(ISD::BITCAST, dl, AsIntVT, Val);
2576
2577 SDValue Abs = DAG.getNode(ISD::AND, dl, AsIntVT, AsInt, SignMask);
2578
2579 SDValue AddNegSmallestNormal =
2580 DAG.getNode(ISD::ADD, dl, AsIntVT, Abs, NegSmallestNormalizedInt);
2581 SDValue DenormOrZero = DAG.getSetCC(dl, SetCCVT, AddNegSmallestNormal,
2582 NegSmallestNormalizedInt, ISD::SETULE);
2583
2584 SDValue IsDenormal =
2585 DAG.getSetCC(dl, SetCCVT, Abs, SmallestNormalizedInt, ISD::SETULT);
2586
2587 SDValue MinExp = DAG.getConstant(MinExpVal, dl, ExpVT);
2588 SDValue Zero = DAG.getConstant(0, dl, ExpVT);
2589
2590 SDValue ScaledAsInt = DAG.getNode(ISD::BITCAST, dl, AsIntVT, ScaleUp);
2591 SDValue ScaledSelect =
2592 DAG.getNode(ISD::SELECT, dl, AsIntVT, IsDenormal, ScaledAsInt, AsInt);
2593
2594 SDValue ExpMaskScaled =
2595 DAG.getNode(ISD::AND, dl, AsIntVT, ScaledAsInt, ExpMask);
2596
2597 SDValue ScaledValue =
2598 DAG.getNode(ISD::SELECT, dl, AsIntVT, IsDenormal, ExpMaskScaled, Abs);
2599
2600 // Extract the exponent bits.
2601 SDValue ExponentShiftAmt =
2602 DAG.getShiftAmountConstant(Precision - 1, AsIntVT, dl);
2603 SDValue ShiftedExp =
2604 DAG.getNode(ISD::SRL, dl, AsIntVT, ScaledValue, ExponentShiftAmt);
2605 SDValue Exp = DAG.getSExtOrTrunc(ShiftedExp, dl, ExpVT);
2606
2607 SDValue NormalBiasedExp = DAG.getNode(ISD::ADD, dl, ExpVT, Exp, MinExp);
2608 SDValue DenormalOffset = DAG.getConstant(-Precision - 1, dl, ExpVT);
2609 SDValue DenormalExpBias =
2610 DAG.getNode(ISD::SELECT, dl, ExpVT, IsDenormal, DenormalOffset, Zero);
2611
2612 SDValue MaskedFractAsInt =
2613 DAG.getNode(ISD::AND, dl, AsIntVT, ScaledSelect, FractSignMask);
2614 const APFloat Half(FltSem, "0.5");
2615 SDValue FPHalf = DAG.getConstant(Half.bitcastToAPInt(), dl, AsIntVT);
2616 SDValue Or = DAG.getNode(ISD::OR, dl, AsIntVT, MaskedFractAsInt, FPHalf);
2617 SDValue MaskedFract = DAG.getNode(ISD::BITCAST, dl, VT, Or);
2618
2619 SDValue ComputedExp =
2620 DAG.getNode(ISD::ADD, dl, ExpVT, NormalBiasedExp, DenormalExpBias);
2621
2622 SDValue Result0 =
2623 DAG.getNode(ISD::SELECT, dl, VT, DenormOrZero, Val, MaskedFract);
2624
2625 SDValue Result1 =
2626 DAG.getNode(ISD::SELECT, dl, ExpVT, DenormOrZero, Zero, ComputedExp);
2627
2628 return DAG.getMergeValues({Result0, Result1}, dl);
2629}
2630
2631/// This function is responsible for legalizing a
2632/// INT_TO_FP operation of the specified operand when the target requests that
2633/// we expand it. At this point, we know that the result and operand types are
2634/// legal for the target.
2635SDValue SelectionDAGLegalize::ExpandLegalINT_TO_FP(SDNode *Node,
2636 SDValue &Chain) {
2637 bool isSigned = (Node->getOpcode() == ISD::STRICT_SINT_TO_FP ||
2638 Node->getOpcode() == ISD::SINT_TO_FP);
2639 EVT DestVT = Node->getValueType(0);
2640 SDLoc dl(Node);
2641 unsigned OpNo = Node->isStrictFPOpcode() ? 1 : 0;
2642 SDValue Op0 = Node->getOperand(OpNo);
2643 EVT SrcVT = Op0.getValueType();
2644
2645 // TODO: Should any fast-math-flags be set for the created nodes?
2646 LLVM_DEBUG(dbgs() << "Legalizing INT_TO_FP\n");
2647 if (SrcVT == MVT::i32 && TLI.isTypeLegal(MVT::f64) &&
2648 (DestVT.bitsLE(MVT::f64) ||
2649 TLI.isOperationLegal(Node->isStrictFPOpcode() ? ISD::STRICT_FP_EXTEND
2651 DestVT))) {
2652 LLVM_DEBUG(dbgs() << "32-bit [signed|unsigned] integer to float/double "
2653 "expansion\n");
2654
2655 // Get the stack frame index of a 8 byte buffer.
2656 SDValue StackSlot = DAG.CreateStackTemporary(MVT::f64);
2657
2658 SDValue Lo = Op0;
2659 // if signed map to unsigned space
2660 if (isSigned) {
2661 // Invert sign bit (signed to unsigned mapping).
2662 Lo = DAG.getNode(ISD::XOR, dl, MVT::i32, Lo,
2663 DAG.getConstant(0x80000000u, dl, MVT::i32));
2664 }
2665 // Initial hi portion of constructed double.
2666 SDValue Hi = DAG.getConstant(0x43300000u, dl, MVT::i32);
2667
2668 // If this a big endian target, swap the lo and high data.
2669 if (DAG.getDataLayout().isBigEndian())
2670 std::swap(Lo, Hi);
2671
2672 SDValue MemChain = DAG.getEntryNode();
2673
2674 // Store the lo of the constructed double.
2675 SDValue Store1 = DAG.getStore(MemChain, dl, Lo, StackSlot,
2677 // Store the hi of the constructed double.
2678 SDValue HiPtr =
2679 DAG.getMemBasePlusOffset(StackSlot, TypeSize::getFixed(4), dl);
2680 SDValue Store2 =
2681 DAG.getStore(MemChain, dl, Hi, HiPtr, MachinePointerInfo());
2682 MemChain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Store1, Store2);
2683
2684 // load the constructed double
2685 SDValue Load =
2686 DAG.getLoad(MVT::f64, dl, MemChain, StackSlot, MachinePointerInfo());
2687 // FP constant to bias correct the final result
2688 SDValue Bias = DAG.getConstantFP(
2689 isSigned ? llvm::bit_cast<double>(0x4330000080000000ULL)
2690 : llvm::bit_cast<double>(0x4330000000000000ULL),
2691 dl, MVT::f64);
2692 // Subtract the bias and get the final result.
2693 SDValue Sub;
2695 if (Node->isStrictFPOpcode()) {
2696 Sub = DAG.getNode(ISD::STRICT_FSUB, dl, {MVT::f64, MVT::Other},
2697 {Node->getOperand(0), Load, Bias});
2698 Chain = Sub.getValue(1);
2699 if (DestVT != Sub.getValueType()) {
2700 std::pair<SDValue, SDValue> ResultPair;
2701 ResultPair =
2702 DAG.getStrictFPExtendOrRound(Sub, Chain, dl, DestVT);
2703 Result = ResultPair.first;
2704 Chain = ResultPair.second;
2705 }
2706 else
2707 Result = Sub;
2708 } else {
2709 Sub = DAG.getNode(ISD::FSUB, dl, MVT::f64, Load, Bias);
2710 Result = DAG.getFPExtendOrRound(Sub, dl, DestVT);
2711 }
2712 return Result;
2713 }
2714
2715 if (isSigned)
2716 return SDValue();
2717
2718 // TODO: Generalize this for use with other types.
2719 if (((SrcVT == MVT::i32 || SrcVT == MVT::i64) && DestVT == MVT::f32) ||
2720 (SrcVT == MVT::i64 && DestVT == MVT::f64)) {
2721 LLVM_DEBUG(dbgs() << "Converting unsigned i32/i64 to f32/f64\n");
2722 // For unsigned conversions, convert them to signed conversions using the
2723 // algorithm from the x86_64 __floatundisf in compiler_rt. That method
2724 // should be valid for i32->f32 as well.
2725
2726 // More generally this transform should be valid if there are 3 more bits
2727 // in the integer type than the significand. Rounding uses the first bit
2728 // after the width of the significand and the OR of all bits after that. So
2729 // we need to be able to OR the shifted out bit into one of the bits that
2730 // participate in the OR.
2731
2732 // TODO: This really should be implemented using a branch rather than a
2733 // select. We happen to get lucky and machinesink does the right
2734 // thing most of the time. This would be a good candidate for a
2735 // pseudo-op, or, even better, for whole-function isel.
2736 EVT SetCCVT = getSetCCResultType(SrcVT);
2737
2738 SDValue SignBitTest = DAG.getSetCC(
2739 dl, SetCCVT, Op0, DAG.getConstant(0, dl, SrcVT), ISD::SETLT);
2740
2741 EVT ShiftVT = TLI.getShiftAmountTy(SrcVT, DAG.getDataLayout());
2742 SDValue ShiftConst = DAG.getConstant(1, dl, ShiftVT);
2743 SDValue Shr = DAG.getNode(ISD::SRL, dl, SrcVT, Op0, ShiftConst);
2744 SDValue AndConst = DAG.getConstant(1, dl, SrcVT);
2745 SDValue And = DAG.getNode(ISD::AND, dl, SrcVT, Op0, AndConst);
2746 SDValue Or = DAG.getNode(ISD::OR, dl, SrcVT, And, Shr);
2747
2748 SDValue Slow, Fast;
2749 if (Node->isStrictFPOpcode()) {
2750 // In strict mode, we must avoid spurious exceptions, and therefore
2751 // must make sure to only emit a single STRICT_SINT_TO_FP.
2752 SDValue InCvt = DAG.getSelect(dl, SrcVT, SignBitTest, Or, Op0);
2753 Fast = DAG.getNode(ISD::STRICT_SINT_TO_FP, dl, { DestVT, MVT::Other },
2754 { Node->getOperand(0), InCvt });
2755 Slow = DAG.getNode(ISD::STRICT_FADD, dl, { DestVT, MVT::Other },
2756 { Fast.getValue(1), Fast, Fast });
2757 Chain = Slow.getValue(1);
2758 // The STRICT_SINT_TO_FP inherits the exception mode from the
2759 // incoming STRICT_UINT_TO_FP node; the STRICT_FADD node can
2760 // never raise any exception.
2762 Flags.setNoFPExcept(Node->getFlags().hasNoFPExcept());
2763 Fast->setFlags(Flags);
2764 Flags.setNoFPExcept(true);
2765 Slow->setFlags(Flags);
2766 } else {
2767 SDValue SignCvt = DAG.getNode(ISD::SINT_TO_FP, dl, DestVT, Or);
2768 Slow = DAG.getNode(ISD::FADD, dl, DestVT, SignCvt, SignCvt);
2769 Fast = DAG.getNode(ISD::SINT_TO_FP, dl, DestVT, Op0);
2770 }
2771
2772 return DAG.getSelect(dl, DestVT, SignBitTest, Slow, Fast);
2773 }
2774
2775 // Don't expand it if there isn't cheap fadd.
2776 if (!TLI.isOperationLegalOrCustom(
2777 Node->isStrictFPOpcode() ? ISD::STRICT_FADD : ISD::FADD, DestVT))
2778 return SDValue();
2779
2780 // The following optimization is valid only if every value in SrcVT (when
2781 // treated as signed) is representable in DestVT. Check that the mantissa
2782 // size of DestVT is >= than the number of bits in SrcVT -1.
2783 assert(APFloat::semanticsPrecision(DAG.EVTToAPFloatSemantics(DestVT)) >=
2784 SrcVT.getSizeInBits() - 1 &&
2785 "Cannot perform lossless SINT_TO_FP!");
2786
2787 SDValue Tmp1;
2788 if (Node->isStrictFPOpcode()) {
2789 Tmp1 = DAG.getNode(ISD::STRICT_SINT_TO_FP, dl, { DestVT, MVT::Other },
2790 { Node->getOperand(0), Op0 });
2791 } else
2792 Tmp1 = DAG.getNode(ISD::SINT_TO_FP, dl, DestVT, Op0);
2793
2794 SDValue SignSet = DAG.getSetCC(dl, getSetCCResultType(SrcVT), Op0,
2795 DAG.getConstant(0, dl, SrcVT), ISD::SETLT);
2796 SDValue Zero = DAG.getIntPtrConstant(0, dl),
2797 Four = DAG.getIntPtrConstant(4, dl);
2798 SDValue CstOffset = DAG.getSelect(dl, Zero.getValueType(),
2799 SignSet, Four, Zero);
2800
2801 // If the sign bit of the integer is set, the large number will be treated
2802 // as a negative number. To counteract this, the dynamic code adds an
2803 // offset depending on the data type.
2804 uint64_t FF;
2805 switch (SrcVT.getSimpleVT().SimpleTy) {
2806 default:
2807 return SDValue();
2808 case MVT::i8 : FF = 0x43800000ULL; break; // 2^8 (as a float)
2809 case MVT::i16: FF = 0x47800000ULL; break; // 2^16 (as a float)
2810 case MVT::i32: FF = 0x4F800000ULL; break; // 2^32 (as a float)
2811 case MVT::i64: FF = 0x5F800000ULL; break; // 2^64 (as a float)
2812 }
2813 if (DAG.getDataLayout().isLittleEndian())
2814 FF <<= 32;
2815 Constant *FudgeFactor = ConstantInt::get(
2816 Type::getInt64Ty(*DAG.getContext()), FF);
2817
2818 SDValue CPIdx =
2819 DAG.getConstantPool(FudgeFactor, TLI.getPointerTy(DAG.getDataLayout()));
2820 Align Alignment = cast<ConstantPoolSDNode>(CPIdx)->getAlign();
2821 CPIdx = DAG.getNode(ISD::ADD, dl, CPIdx.getValueType(), CPIdx, CstOffset);
2822 Alignment = commonAlignment(Alignment, 4);
2823 SDValue FudgeInReg;
2824 if (DestVT == MVT::f32)
2825 FudgeInReg = DAG.getLoad(
2826 MVT::f32, dl, DAG.getEntryNode(), CPIdx,
2827 MachinePointerInfo::getConstantPool(DAG.getMachineFunction()),
2828 Alignment);
2829 else {
2830 SDValue Load = DAG.getExtLoad(
2831 ISD::EXTLOAD, dl, DestVT, DAG.getEntryNode(), CPIdx,
2832 MachinePointerInfo::getConstantPool(DAG.getMachineFunction()), MVT::f32,
2833 Alignment);
2834 HandleSDNode Handle(Load);
2835 LegalizeOp(Load.getNode());
2836 FudgeInReg = Handle.getValue();
2837 }
2838
2839 if (Node->isStrictFPOpcode()) {
2840 SDValue Result = DAG.getNode(ISD::STRICT_FADD, dl, { DestVT, MVT::Other },
2841 { Tmp1.getValue(1), Tmp1, FudgeInReg });
2842 Chain = Result.getValue(1);
2843 return Result;
2844 }
2845
2846 return DAG.getNode(ISD::FADD, dl, DestVT, Tmp1, FudgeInReg);
2847}
2848
2849/// This function is responsible for legalizing a
2850/// *INT_TO_FP operation of the specified operand when the target requests that
2851/// we promote it. At this point, we know that the result and operand types are
2852/// legal for the target, and that there is a legal UINT_TO_FP or SINT_TO_FP
2853/// operation that takes a larger input.
2854void SelectionDAGLegalize::PromoteLegalINT_TO_FP(
2856 bool IsStrict = N->isStrictFPOpcode();
2857 bool IsSigned = N->getOpcode() == ISD::SINT_TO_FP ||
2858 N->getOpcode() == ISD::STRICT_SINT_TO_FP;
2859 EVT DestVT = N->getValueType(0);
2860 SDValue LegalOp = N->getOperand(IsStrict ? 1 : 0);
2861 unsigned UIntOp = IsStrict ? ISD::STRICT_UINT_TO_FP : ISD::UINT_TO_FP;
2862 unsigned SIntOp = IsStrict ? ISD::STRICT_SINT_TO_FP : ISD::SINT_TO_FP;
2863
2864 // First step, figure out the appropriate *INT_TO_FP operation to use.
2865 EVT NewInTy = LegalOp.getValueType();
2866
2867 unsigned OpToUse = 0;
2868
2869 // Scan for the appropriate larger type to use.
2870 while (true) {
2871 NewInTy = (MVT::SimpleValueType)(NewInTy.getSimpleVT().SimpleTy+1);
2872 assert(NewInTy.isInteger() && "Ran out of possibilities!");
2873
2874 // If the target supports SINT_TO_FP of this type, use it.
2875 if (TLI.isOperationLegalOrCustom(SIntOp, NewInTy)) {
2876 OpToUse = SIntOp;
2877 break;
2878 }
2879 if (IsSigned)
2880 continue;
2881
2882 // If the target supports UINT_TO_FP of this type, use it.
2883 if (TLI.isOperationLegalOrCustom(UIntOp, NewInTy)) {
2884 OpToUse = UIntOp;
2885 break;
2886 }
2887
2888 // Otherwise, try a larger type.
2889 }
2890
2891 // Okay, we found the operation and type to use. Zero extend our input to the
2892 // desired type then run the operation on it.
2893 if (IsStrict) {
2894 SDValue Res =
2895 DAG.getNode(OpToUse, dl, {DestVT, MVT::Other},
2896 {N->getOperand(0),
2897 DAG.getNode(IsSigned ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND,
2898 dl, NewInTy, LegalOp)});
2899 Results.push_back(Res);
2900 Results.push_back(Res.getValue(1));
2901 return;
2902 }
2903
2904 Results.push_back(
2905 DAG.getNode(OpToUse, dl, DestVT,
2906 DAG.getNode(IsSigned ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND,
2907 dl, NewInTy, LegalOp)));
2908}
2909
2910/// This function is responsible for legalizing a
2911/// FP_TO_*INT operation of the specified operand when the target requests that
2912/// we promote it. At this point, we know that the result and operand types are
2913/// legal for the target, and that there is a legal FP_TO_UINT or FP_TO_SINT
2914/// operation that returns a larger result.
2915void SelectionDAGLegalize::PromoteLegalFP_TO_INT(SDNode *N, const SDLoc &dl,
2917 bool IsStrict = N->isStrictFPOpcode();
2918 bool IsSigned = N->getOpcode() == ISD::FP_TO_SINT ||
2919 N->getOpcode() == ISD::STRICT_FP_TO_SINT;
2920 EVT DestVT = N->getValueType(0);
2921 SDValue LegalOp = N->getOperand(IsStrict ? 1 : 0);
2922 // First step, figure out the appropriate FP_TO*INT operation to use.
2923 EVT NewOutTy = DestVT;
2924
2925 unsigned OpToUse = 0;
2926
2927 // Scan for the appropriate larger type to use.
2928 while (true) {
2929 NewOutTy = (MVT::SimpleValueType)(NewOutTy.getSimpleVT().SimpleTy+1);
2930 assert(NewOutTy.isInteger() && "Ran out of possibilities!");
2931
2932 // A larger signed type can hold all unsigned values of the requested type,
2933 // so using FP_TO_SINT is valid
2934 OpToUse = IsStrict ? ISD::STRICT_FP_TO_SINT : ISD::FP_TO_SINT;
2935 if (TLI.isOperationLegalOrCustom(OpToUse, NewOutTy))
2936 break;
2937
2938 // However, if the value may be < 0.0, we *must* use some FP_TO_SINT.
2939 OpToUse = IsStrict ? ISD::STRICT_FP_TO_UINT : ISD::FP_TO_UINT;
2940 if (!IsSigned && TLI.isOperationLegalOrCustom(OpToUse, NewOutTy))
2941 break;
2942
2943 // Otherwise, try a larger type.
2944 }
2945
2946 // Okay, we found the operation and type to use.
2948 if (IsStrict) {
2949 SDVTList VTs = DAG.getVTList(NewOutTy, MVT::Other);
2950 Operation = DAG.getNode(OpToUse, dl, VTs, N->getOperand(0), LegalOp);
2951 } else
2952 Operation = DAG.getNode(OpToUse, dl, NewOutTy, LegalOp);
2953
2954 // Truncate the result of the extended FP_TO_*INT operation to the desired
2955 // size.
2956 SDValue Trunc = DAG.getNode(ISD::TRUNCATE, dl, DestVT, Operation);
2957 Results.push_back(Trunc);
2958 if (IsStrict)
2959 Results.push_back(Operation.getValue(1));
2960}
2961
2962/// Promote FP_TO_*INT_SAT operation to a larger result type. At this point
2963/// the result and operand types are legal and there must be a legal
2964/// FP_TO_*INT_SAT operation for a larger result type.
2965SDValue SelectionDAGLegalize::PromoteLegalFP_TO_INT_SAT(SDNode *Node,
2966 const SDLoc &dl) {
2967 unsigned Opcode = Node->getOpcode();
2968
2969 // Scan for the appropriate larger type to use.
2970 EVT NewOutTy = Node->getValueType(0);
2971 while (true) {
2972 NewOutTy = (MVT::SimpleValueType)(NewOutTy.getSimpleVT().SimpleTy + 1);
2973 assert(NewOutTy.isInteger() && "Ran out of possibilities!");
2974
2975 if (TLI.isOperationLegalOrCustom(Opcode, NewOutTy))
2976 break;
2977 }
2978
2979 // Saturation width is determined by second operand, so we don't have to
2980 // perform any fixup and can directly truncate the result.
2981 SDValue Result = DAG.getNode(Opcode, dl, NewOutTy, Node->getOperand(0),
2982 Node->getOperand(1));
2983 return DAG.getNode(ISD::TRUNCATE, dl, Node->getValueType(0), Result);
2984}
2985
2986/// Open code the operations for PARITY of the specified operation.
2987SDValue SelectionDAGLegalize::ExpandPARITY(SDValue Op, const SDLoc &dl) {
2988 EVT VT = Op.getValueType();
2989 EVT ShVT = TLI.getShiftAmountTy(VT, DAG.getDataLayout());
2990 unsigned Sz = VT.getScalarSizeInBits();
2991
2992 // If CTPOP is legal, use it. Otherwise use shifts and xor.
2994 if (TLI.isOperationLegalOrPromote(ISD::CTPOP, VT)) {
2995 Result = DAG.getNode(ISD::CTPOP, dl, VT, Op);
2996 } else {
2997 Result = Op;
2998 for (unsigned i = Log2_32_Ceil(Sz); i != 0;) {
2999 SDValue Shift = DAG.getNode(ISD::SRL, dl, VT, Result,
3000 DAG.getConstant(1ULL << (--i), dl, ShVT));
3001 Result = DAG.getNode(ISD::XOR, dl, VT, Result, Shift);
3002 }
3003 }
3004
3005 return DAG.getNode(ISD::AND, dl, VT, Result, DAG.getConstant(1, dl, VT));
3006}
3007
3008bool SelectionDAGLegalize::ExpandNode(SDNode *Node) {
3009 LLVM_DEBUG(dbgs() << "Trying to expand node\n");
3011 SDLoc dl(Node);
3012 SDValue Tmp1, Tmp2, Tmp3, Tmp4;
3013 bool NeedInvert;
3014 switch (Node->getOpcode()) {
3015 case ISD::ABS:
3016 if ((Tmp1 = TLI.expandABS(Node, DAG)))
3017 Results.push_back(Tmp1);
3018 break;
3019 case ISD::ABDS:
3020 case ISD::ABDU:
3021 if ((Tmp1 = TLI.expandABD(Node, DAG)))
3022 Results.push_back(Tmp1);
3023 break;
3024 case ISD::CTPOP:
3025 if ((Tmp1 = TLI.expandCTPOP(Node, DAG)))
3026 Results.push_back(Tmp1);
3027 break;
3028 case ISD::CTLZ:
3030 if ((Tmp1 = TLI.expandCTLZ(Node, DAG)))
3031 Results.push_back(Tmp1);
3032 break;
3033 case ISD::CTTZ:
3035 if ((Tmp1 = TLI.expandCTTZ(Node, DAG)))
3036 Results.push_back(Tmp1);
3037 break;
3038 case ISD::BITREVERSE:
3039 if ((Tmp1 = TLI.expandBITREVERSE(Node, DAG)))
3040 Results.push_back(Tmp1);
3041 break;
3042 case ISD::BSWAP:
3043 if ((Tmp1 = TLI.expandBSWAP(Node, DAG)))
3044 Results.push_back(Tmp1);
3045 break;
3046 case ISD::PARITY:
3047 Results.push_back(ExpandPARITY(Node->getOperand(0), dl));
3048 break;
3049 case ISD::FRAMEADDR:
3050 case ISD::RETURNADDR:
3052 Results.push_back(DAG.getConstant(0, dl, Node->getValueType(0)));
3053 break;
3054 case ISD::EH_DWARF_CFA: {
3055 SDValue CfaArg = DAG.getSExtOrTrunc(Node->getOperand(0), dl,
3056 TLI.getPointerTy(DAG.getDataLayout()));
3057 SDValue Offset = DAG.getNode(ISD::ADD, dl,
3058 CfaArg.getValueType(),
3059 DAG.getNode(ISD::FRAME_TO_ARGS_OFFSET, dl,
3060 CfaArg.getValueType()),
3061 CfaArg);
3062 SDValue FA = DAG.getNode(
3063 ISD::FRAMEADDR, dl, TLI.getPointerTy(DAG.getDataLayout()),
3064 DAG.getConstant(0, dl, TLI.getPointerTy(DAG.getDataLayout())));
3065 Results.push_back(DAG.getNode(ISD::ADD, dl, FA.getValueType(),
3066 FA, Offset));
3067 break;
3068 }
3069 case ISD::GET_ROUNDING:
3070 Results.push_back(DAG.getConstant(1, dl, Node->getValueType(0)));
3071 Results.push_back(Node->getOperand(0));
3072 break;
3073 case ISD::EH_RETURN:
3074 case ISD::EH_LABEL:
3075 case ISD::PREFETCH:
3076 case ISD::VAEND:
3078 // If the target didn't expand these, there's nothing to do, so just
3079 // preserve the chain and be done.
3080 Results.push_back(Node->getOperand(0));
3081 break;
3084 // If the target didn't expand this, just return 'zero' and preserve the
3085 // chain.
3086 Results.append(Node->getNumValues() - 1,
3087 DAG.getConstant(0, dl, Node->getValueType(0)));
3088 Results.push_back(Node->getOperand(0));
3089 break;
3091 // If the target didn't expand this, just return 'zero' and preserve the
3092 // chain.
3093 Results.push_back(DAG.getConstant(0, dl, MVT::i32));
3094 Results.push_back(Node->getOperand(0));
3095 break;
3096 case ISD::ATOMIC_LOAD: {
3097 // There is no libcall for atomic load; fake it with ATOMIC_CMP_SWAP.
3098 SDValue Zero = DAG.getConstant(0, dl, Node->getValueType(0));
3099 SDVTList VTs = DAG.getVTList(Node->getValueType(0), MVT::Other);
3100 SDValue Swap = DAG.getAtomicCmpSwap(
3101 ISD::ATOMIC_CMP_SWAP, dl, cast<AtomicSDNode>(Node)->getMemoryVT(), VTs,
3102 Node->getOperand(0), Node->getOperand(1), Zero, Zero,
3103 cast<AtomicSDNode>(Node)->getMemOperand());
3104 Results.push_back(Swap.getValue(0));
3105 Results.push_back(Swap.getValue(1));
3106 break;
3107 }
3108 case ISD::ATOMIC_STORE: {
3109 // There is no libcall for atomic store; fake it with ATOMIC_SWAP.
3110 SDValue Swap = DAG.getAtomic(
3111 ISD::ATOMIC_SWAP, dl, cast<AtomicSDNode>(Node)->getMemoryVT(),
3112 Node->getOperand(0), Node->getOperand(2), Node->getOperand(1),
3113 cast<AtomicSDNode>(Node)->getMemOperand());
3114 Results.push_back(Swap.getValue(1));
3115 break;
3116 }
3118 // Expanding an ATOMIC_CMP_SWAP_WITH_SUCCESS produces an ATOMIC_CMP_SWAP and
3119 // splits out the success value as a comparison. Expanding the resulting
3120 // ATOMIC_CMP_SWAP will produce a libcall.
3121 SDVTList VTs = DAG.getVTList(Node->getValueType(0), MVT::Other);
3122 SDValue Res = DAG.getAtomicCmpSwap(
3123 ISD::ATOMIC_CMP_SWAP, dl, cast<AtomicSDNode>(Node)->getMemoryVT(), VTs,
3124 Node->getOperand(0), Node->getOperand(1), Node->getOperand(2),
3125 Node->getOperand(3), cast<MemSDNode>(Node)->getMemOperand());
3126
3127 SDValue ExtRes = Res;
3128 SDValue LHS = Res;
3129 SDValue RHS = Node->getOperand(1);
3130
3131 EVT AtomicType = cast<AtomicSDNode>(Node)->getMemoryVT();
3132 EVT OuterType = Node->getValueType(0);
3133 switch (TLI.getExtendForAtomicOps()) {
3134 case ISD::SIGN_EXTEND:
3135 LHS = DAG.getNode(ISD::AssertSext, dl, OuterType, Res,
3136 DAG.getValueType(AtomicType));
3137 RHS = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, OuterType,
3138 Node->getOperand(2), DAG.getValueType(AtomicType));
3139 ExtRes = LHS;
3140 break;
3141 case ISD::ZERO_EXTEND:
3142 LHS = DAG.getNode(ISD::AssertZext, dl, OuterType, Res,
3143 DAG.getValueType(AtomicType));
3144 RHS = DAG.getZeroExtendInReg(Node->getOperand(2), dl, AtomicType);
3145 ExtRes = LHS;
3146 break;
3147 case ISD::ANY_EXTEND:
3148 LHS = DAG.getZeroExtendInReg(Res, dl, AtomicType);
3149 RHS = DAG.getZeroExtendInReg(Node->getOperand(2), dl, AtomicType);
3150 break;
3151 default:
3152 llvm_unreachable("Invalid atomic op extension");
3153 }
3154
3156 DAG.getSetCC(dl, Node->getValueType(1), LHS, RHS, ISD::SETEQ);
3157
3158 Results.push_back(ExtRes.getValue(0));
3159 Results.push_back(Success);
3160 Results.push_back(Res.getValue(1));
3161 break;
3162 }
3163 case ISD::ATOMIC_LOAD_SUB: {
3164 SDLoc DL(Node);
3165 EVT VT = Node->getValueType(0);
3166 SDValue RHS = Node->getOperand(2);
3167 AtomicSDNode *AN = cast<AtomicSDNode>(Node);
3168 if (RHS->getOpcode() == ISD::SIGN_EXTEND_INREG &&
3169 cast<VTSDNode>(RHS->getOperand(1))->getVT() == AN->getMemoryVT())
3170 RHS = RHS->getOperand(0);
3171 SDValue NewRHS =
3172 DAG.getNode(ISD::SUB, DL, VT, DAG.getConstant(0, DL, VT), RHS);
3173 SDValue Res = DAG.getAtomic(ISD::ATOMIC_LOAD_ADD, DL, AN->getMemoryVT(),
3174 Node->getOperand(0), Node->getOperand(1),
3175 NewRHS, AN->getMemOperand());
3176 Results.push_back(Res);
3177 Results.push_back(Res.getValue(1));
3178 break;
3179 }
3181 ExpandDYNAMIC_STACKALLOC(Node, Results);
3182 break;
3183 case ISD::MERGE_VALUES:
3184 for (unsigned i = 0; i < Node->getNumValues(); i++)
3185 Results.push_back(Node->getOperand(i));
3186 break;
3187 case ISD::UNDEF: {
3188 EVT VT = Node->getValueType(0);
3189 if (VT.isInteger())
3190 Results.push_back(DAG.getConstant(0, dl, VT));
3191 else {
3192 assert(VT.isFloatingPoint() && "Unknown value type!");
3193 Results.push_back(DAG.getConstantFP(0, dl, VT));
3194 }
3195 break;
3196 }
3198 // When strict mode is enforced we can't do expansion because it
3199 // does not honor the "strict" properties. Only libcall is allowed.
3200 if (TLI.isStrictFPEnabled())
3201 break;
3202 // We might as well mutate to FP_ROUND when FP_ROUND operation is legal
3203 // since this operation is more efficient than stack operation.
3204 if (TLI.getStrictFPOperationAction(Node->getOpcode(),
3205 Node->getValueType(0))
3206 == TargetLowering::Legal)
3207 break;
3208 // We fall back to use stack operation when the FP_ROUND operation
3209 // isn't available.
3210 if ((Tmp1 = EmitStackConvert(Node->getOperand(1), Node->getValueType(0),
3211 Node->getValueType(0), dl,
3212 Node->getOperand(0)))) {
3213 ReplaceNode(Node, Tmp1.getNode());
3214 LLVM_DEBUG(dbgs() << "Successfully expanded STRICT_FP_ROUND node\n");
3215 return true;
3216 }
3217 break;
3218 case ISD::FP_ROUND: {
3219 if ((Tmp1 = TLI.expandFP_ROUND(Node, DAG))) {
3220 Results.push_back(Tmp1);
3221 break;
3222 }
3223
3225 }
3226 case ISD::BITCAST:
3227 if ((Tmp1 = EmitStackConvert(Node->getOperand(0), Node->getValueType(0),
3228 Node->getValueType(0), dl)))
3229 Results.push_back(Tmp1);
3230 break;
3232 // When strict mode is enforced we can't do expansion because it
3233 // does not honor the "strict" properties. Only libcall is allowed.
3234 if (TLI.isStrictFPEnabled())
3235 break;
3236 // We might as well mutate to FP_EXTEND when FP_EXTEND operation is legal
3237 // since this operation is more efficient than stack operation.
3238 if (TLI.getStrictFPOperationAction(Node->getOpcode(),
3239 Node->getValueType(0))
3240 == TargetLowering::Legal)
3241 break;
3242 // We fall back to use stack operation when the FP_EXTEND operation
3243 // isn't available.
3244 if ((Tmp1 = EmitStackConvert(
3245 Node->getOperand(1), Node->getOperand(1).getValueType(),
3246 Node->getValueType(0), dl, Node->getOperand(0)))) {
3247 ReplaceNode(Node, Tmp1.getNode());
3248 LLVM_DEBUG(dbgs() << "Successfully expanded STRICT_FP_EXTEND node\n");
3249 return true;
3250 }
3251 break;
3252 case ISD::FP_EXTEND: {
3253 SDValue Op = Node->getOperand(0);
3254 EVT SrcVT = Op.getValueType();
3255 EVT DstVT = Node->getValueType(0);
3256 if (SrcVT.getScalarType() == MVT::bf16) {
3257 Results.push_back(DAG.getNode(ISD::BF16_TO_FP, SDLoc(Node), DstVT, Op));
3258 break;
3259 }
3260
3261 if ((Tmp1 = EmitStackConvert(Op, SrcVT, DstVT, dl)))
3262 Results.push_back(Tmp1);
3263 break;
3264 }
3265 case ISD::BF16_TO_FP: {
3266 // Always expand bf16 to f32 casts, they lower to ext + shift.
3267 //
3268 // Note that the operand of this code can be bf16 or an integer type in case
3269 // bf16 is not supported on the target and was softened.
3270 SDValue Op = Node->getOperand(0);
3271 if (Op.getValueType() == MVT::bf16) {
3272 Op = DAG.getNode(ISD::ANY_EXTEND, dl, MVT::i32,
3273 DAG.getNode(ISD::BITCAST, dl, MVT::i16, Op));
3274 } else {
3275 Op = DAG.getAnyExtOrTrunc(Op, dl, MVT::i32);
3276 }
3277 Op = DAG.getNode(
3278 ISD::SHL, dl, MVT::i32, Op,
3279 DAG.getConstant(16, dl,
3280 TLI.getShiftAmountTy(MVT::i32, DAG.getDataLayout())));
3281 Op = DAG.getNode(ISD::BITCAST, dl, MVT::f32, Op);
3282 // Add fp_extend in case the output is bigger than f32.
3283 if (Node->getValueType(0) != MVT::f32)
3284 Op = DAG.getNode(ISD::FP_EXTEND, dl, Node->getValueType(0), Op);
3285 Results.push_back(Op);
3286 break;
3287 }
3288 case ISD::FP_TO_BF16: {
3289 SDValue Op = Node->getOperand(0);
3290 if (Op.getValueType() != MVT::f32)
3291 Op = DAG.getNode(ISD::FP_ROUND, dl, MVT::f32, Op,
3292 DAG.getIntPtrConstant(0, dl, /*isTarget=*/true));
3293 // Certain SNaNs will turn into infinities if we do a simple shift right.
3294 if (!DAG.isKnownNeverSNaN(Op)) {
3295 Op = DAG.getNode(ISD::FCANONICALIZE, dl, MVT::f32, Op, Node->getFlags());
3296 }
3297 Op = DAG.getNode(
3298 ISD::SRL, dl, MVT::i32, DAG.getNode(ISD::BITCAST, dl, MVT::i32, Op),
3299 DAG.getConstant(16, dl,
3300 TLI.getShiftAmountTy(MVT::i32, DAG.getDataLayout())));
3301 // The result of this node can be bf16 or an integer type in case bf16 is
3302 // not supported on the target and was softened to i16 for storage.
3303 if (Node->getValueType(0) == MVT::bf16) {
3304 Op = DAG.getNode(ISD::BITCAST, dl, MVT::bf16,
3305 DAG.getNode(ISD::TRUNCATE, dl, MVT::i16, Op));
3306 } else {
3307 Op = DAG.getAnyExtOrTrunc(Op, dl, Node->getValueType(0));
3308 }
3309 Results.push_back(Op);
3310 break;
3311 }
3313 EVT ExtraVT = cast<VTSDNode>(Node->getOperand(1))->getVT();
3314 EVT VT = Node->getValueType(0);
3315
3316 // An in-register sign-extend of a boolean is a negation:
3317 // 'true' (1) sign-extended is -1.
3318 // 'false' (0) sign-extended is 0.
3319 // However, we must mask the high bits of the source operand because the
3320 // SIGN_EXTEND_INREG does not guarantee that the high bits are already zero.
3321
3322 // TODO: Do this for vectors too?
3323 if (ExtraVT.isScalarInteger() && ExtraVT.getSizeInBits() == 1) {
3324 SDValue One = DAG.getConstant(1, dl, VT);
3325 SDValue And = DAG.getNode(ISD::AND, dl, VT, Node->getOperand(0), One);
3326 SDValue Zero = DAG.getConstant(0, dl, VT);
3327 SDValue Neg = DAG.getNode(ISD::SUB, dl, VT, Zero, And);
3328 Results.push_back(Neg);
3329 break;
3330 }
3331
3332 // NOTE: we could fall back on load/store here too for targets without
3333 // SRA. However, it is doubtful that any exist.
3334 EVT ShiftAmountTy = TLI.getShiftAmountTy(VT, DAG.getDataLayout());
3335 unsigned BitsDiff = VT.getScalarSizeInBits() -
3336 ExtraVT.getScalarSizeInBits();
3337 SDValue ShiftCst = DAG.getConstant(BitsDiff, dl, ShiftAmountTy);
3338 Tmp1 = DAG.getNode(ISD::SHL, dl, Node->getValueType(0),
3339 Node->getOperand(0), ShiftCst);
3340 Tmp1 = DAG.getNode(ISD::SRA, dl, Node->getValueType(0), Tmp1, ShiftCst);
3341 Results.push_back(Tmp1);
3342 break;
3343 }
3344 case ISD::UINT_TO_FP:
3346 if (TLI.expandUINT_TO_FP(Node, Tmp1, Tmp2, DAG)) {
3347 Results.push_back(Tmp1);
3348 if (Node->isStrictFPOpcode())
3349 Results.push_back(Tmp2);
3350 break;
3351 }
3352 [[fallthrough]];
3353 case ISD::SINT_TO_FP:
3355 if ((Tmp1 = ExpandLegalINT_TO_FP(Node, Tmp2))) {
3356 Results.push_back(Tmp1);
3357 if (Node->isStrictFPOpcode())
3358 Results.push_back(Tmp2);
3359 }
3360 break;
3361 case ISD::FP_TO_SINT:
3362 if (TLI.expandFP_TO_SINT(Node, Tmp1, DAG))
3363 Results.push_back(Tmp1);
3364 break;
3366 if (TLI.expandFP_TO_SINT(Node, Tmp1, DAG)) {
3367 ReplaceNode(Node, Tmp1.getNode());
3368 LLVM_DEBUG(dbgs() << "Successfully expanded STRICT_FP_TO_SINT node\n");
3369 return true;
3370 }
3371 break;
3372 case ISD::FP_TO_UINT:
3373 if (TLI.expandFP_TO_UINT(Node, Tmp1, Tmp2, DAG))
3374 Results.push_back(Tmp1);
3375 break;
3377 if (TLI.expandFP_TO_UINT(Node, Tmp1, Tmp2, DAG)) {
3378 // Relink the chain.
3379 DAG.ReplaceAllUsesOfValueWith(SDValue(Node,1), Tmp2);
3380 // Replace the new UINT result.
3381 ReplaceNodeWithValue(SDValue(Node, 0), Tmp1);
3382 LLVM_DEBUG(dbgs() << "Successfully expanded STRICT_FP_TO_UINT node\n");
3383 return true;
3384 }
3385 break;
3388 Results.push_back(TLI.expandFP_TO_INT_SAT(Node, DAG));
3389 break;
3390 case ISD::VAARG:
3391 Results.push_back(DAG.expandVAArg(Node));
3392 Results.push_back(Results[0].getValue(1));
3393 break;
3394 case ISD::VACOPY:
3395 Results.push_back(DAG.expandVACopy(Node));
3396 break;
3398 if (Node->getOperand(0).getValueType().getVectorElementCount().isScalar())
3399 // This must be an access of the only element. Return it.
3400 Tmp1 = DAG.getNode(ISD::BITCAST, dl, Node->getValueType(0),
3401 Node->getOperand(0));
3402 else
3403 Tmp1 = ExpandExtractFromVectorThroughStack(SDValue(Node, 0));
3404 Results.push_back(Tmp1);
3405 break;
3407 Results.push_back(ExpandExtractFromVectorThroughStack(SDValue(Node, 0)));
3408 break;
3410 Results.push_back(ExpandInsertToVectorThroughStack(SDValue(Node, 0)));
3411 break;
3413 Results.push_back(ExpandVectorBuildThroughStack(Node));
3414 break;
3416 Results.push_back(ExpandSCALAR_TO_VECTOR(Node));
3417 break;
3419 Results.push_back(ExpandINSERT_VECTOR_ELT(Node->getOperand(0),
3420 Node->getOperand(1),
3421 Node->getOperand(2), dl));
3422 break;
3423 case ISD::VECTOR_SHUFFLE: {
3424 SmallVector<int, 32> NewMask;
3425 ArrayRef<int> Mask = cast<ShuffleVectorSDNode>(Node)->getMask();
3426
3427 EVT VT = Node->getValueType(0);
3428 EVT EltVT = VT.getVectorElementType();
3429 SDValue Op0 = Node->getOperand(0);
3430 SDValue Op1 = Node->getOperand(1);
3431 if (!TLI.isTypeLegal(EltVT)) {
3432 EVT NewEltVT = TLI.getTypeToTransformTo(*DAG.getContext(), EltVT);
3433
3434 // BUILD_VECTOR operands are allowed to be wider than the element type.
3435 // But if NewEltVT is smaller that EltVT the BUILD_VECTOR does not accept
3436 // it.
3437 if (NewEltVT.bitsLT(EltVT)) {
3438 // Convert shuffle node.
3439 // If original node was v4i64 and the new EltVT is i32,
3440 // cast operands to v8i32 and re-build the mask.
3441
3442 // Calculate new VT, the size of the new VT should be equal to original.
3443 EVT NewVT =
3444 EVT::getVectorVT(*DAG.getContext(), NewEltVT,
3445 VT.getSizeInBits() / NewEltVT.getSizeInBits());
3446 assert(NewVT.bitsEq(VT));
3447
3448 // cast operands to new VT
3449 Op0 = DAG.getNode(ISD::BITCAST, dl, NewVT, Op0);
3450 Op1 = DAG.getNode(ISD::BITCAST, dl, NewVT, Op1);
3451
3452 // Convert the shuffle mask
3453 unsigned int factor =
3455
3456 // EltVT gets smaller
3457 assert(factor > 0);
3458
3459 for (unsigned i = 0; i < VT.getVectorNumElements(); ++i) {
3460 if (Mask[i] < 0) {
3461 for (unsigned fi = 0; fi < factor; ++fi)
3462 NewMask.push_back(Mask[i]);
3463 }
3464 else {
3465 for (unsigned fi = 0; fi < factor; ++fi)
3466 NewMask.push_back(Mask[i]*factor+fi);
3467 }
3468 }
3469 Mask = NewMask;
3470 VT = NewVT;
3471 }
3472 EltVT = NewEltVT;
3473 }
3474 unsigned NumElems = VT.getVectorNumElements();
3476 for (unsigned i = 0; i != NumElems; ++i) {
3477 if (Mask[i] < 0) {
3478 Ops.push_back(DAG.getUNDEF(EltVT));
3479 continue;
3480 }
3481 unsigned Idx = Mask[i];
3482 if (Idx < NumElems)
3483 Ops.push_back(DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT, Op0,
3484 DAG.getVectorIdxConstant(Idx, dl)));
3485 else
3486 Ops.push_back(
3487 DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT, Op1,
3488 DAG.getVectorIdxConstant(Idx - NumElems, dl)));
3489 }
3490
3491 Tmp1 = DAG.getBuildVector(VT, dl, Ops);
3492 // We may have changed the BUILD_VECTOR type. Cast it back to the Node type.
3493 Tmp1 = DAG.getNode(ISD::BITCAST, dl, Node->getValueType(0), Tmp1);
3494 Results.push_back(Tmp1);
3495 break;
3496 }
3497 case ISD::VECTOR_SPLICE: {
3498 Results.push_back(TLI.expandVectorSplice(Node, DAG));
3499 break;
3500 }
3501 case ISD::EXTRACT_ELEMENT: {
3502 EVT OpTy = Node->getOperand(0).getValueType();
3503 if (Node->getConstantOperandVal(1)) {
3504 // 1 -> Hi
3505 Tmp1 = DAG.getNode(ISD::SRL, dl, OpTy, Node->getOperand(0),
3506 DAG.getConstant(OpTy.getSizeInBits() / 2, dl,
3507 TLI.getShiftAmountTy(
3508 Node->getOperand(0).getValueType(),
3509 DAG.getDataLayout())));
3510 Tmp1 = DAG.getNode(ISD::TRUNCATE, dl, Node->getValueType(0), Tmp1);
3511 } else {
3512 // 0 -> Lo
3513 Tmp1 = DAG.getNode(ISD::TRUNCATE, dl, Node->getValueType(0),
3514 Node->getOperand(0));
3515 }
3516 Results.push_back(Tmp1);
3517 break;
3518 }
3519 case ISD::STACKSAVE:
3520 // Expand to CopyFromReg if the target set
3521 // StackPointerRegisterToSaveRestore.
3522 if (Register SP = TLI.getStackPointerRegisterToSaveRestore()) {
3523 Results.push_back(DAG.getCopyFromReg(Node->getOperand(0), dl, SP,
3524 Node->getValueType(0)));
3525 Results.push_back(Results[0].getValue(1));
3526 } else {
3527 Results.push_back(DAG.getUNDEF(Node->getValueType(0)));
3528 Results.push_back(Node->getOperand(0));
3529 }
3530 break;
3531 case ISD::STACKRESTORE:
3532 // Expand to CopyToReg if the target set
3533 // StackPointerRegisterToSaveRestore.
3534 if (Register SP = TLI.getStackPointerRegisterToSaveRestore()) {
3535 Results.push_back(DAG.getCopyToReg(Node->getOperand(0), dl, SP,
3536 Node->getOperand(1)));
3537 } else {
3538 Results.push_back(Node->getOperand(0));
3539 }
3540 break;
3542 Results.push_back(DAG.getConstant(0, dl, Node->getValueType(0)));
3543 Results.push_back(Results[0].getValue(0));
3544 break;
3545 case ISD::FCOPYSIGN:
3546 Results.push_back(ExpandFCOPYSIGN(Node));
3547 break;
3548 case ISD::FNEG:
3549 Results.push_back(ExpandFNEG(Node));
3550 break;
3551 case ISD::FABS:
3552 Results.push_back(ExpandFABS(Node));
3553 break;
3554 case ISD::IS_FPCLASS: {
3555 auto Test = static_cast<FPClassTest>(Node->getConstantOperandVal(1));
3556 if (SDValue Expanded =
3557 TLI.expandIS_FPCLASS(Node->getValueType(0), Node->getOperand(0),
3558 Test, Node->getFlags(), SDLoc(Node), DAG))
3559 Results.push_back(Expanded);
3560 break;
3561 }
3562 case ISD::SMIN:
3563 case ISD::SMAX:
3564 case ISD::UMIN:
3565 case ISD::UMAX: {
3566 // Expand Y = MAX(A, B) -> Y = (A > B) ? A : B
3567 ISD::CondCode Pred;
3568 switch (Node->getOpcode()) {
3569 default: llvm_unreachable("How did we get here?");
3570 case ISD::SMAX: Pred = ISD::SETGT; break;
3571 case ISD::SMIN: Pred = ISD::SETLT; break;
3572 case ISD::UMAX: Pred = ISD::SETUGT; break;
3573 case ISD::UMIN: Pred = ISD::SETULT; break;
3574 }
3575 Tmp1 = Node->getOperand(0);
3576 Tmp2 = Node->getOperand(1);
3577 Tmp1 = DAG.getSelectCC(dl, Tmp1, Tmp2, Tmp1, Tmp2, Pred);
3578 Results.push_back(Tmp1);
3579 break;
3580 }
3581 case ISD::FMINNUM:
3582 case ISD::FMAXNUM: {
3583 if (SDValue Expanded = TLI.expandFMINNUM_FMAXNUM(Node, DAG))
3584 Results.push_back(Expanded);
3585 break;
3586 }
3587 case ISD::FSIN:
3588 case ISD::FCOS: {
3589 EVT VT = Node->getValueType(0);
3590 // Turn fsin / fcos into ISD::FSINCOS node if there are a pair of fsin /
3591 // fcos which share the same operand and both are used.
3592 if ((TLI.isOperationLegalOrCustom(ISD::FSINCOS, VT) ||
3594 && useSinCos(Node)) {
3595 SDVTList VTs = DAG.getVTList(VT, VT);
3596 Tmp1 = DAG.getNode(ISD::FSINCOS, dl, VTs, Node->getOperand(0));
3597 if (Node->getOpcode() == ISD::FCOS)
3598 Tmp1 = Tmp1.getValue(1);
3599 Results.push_back(Tmp1);
3600 }
3601 break;
3602 }
3603 case ISD::FLDEXP:
3604 case ISD::STRICT_FLDEXP: {
3605 EVT VT = Node->getValueType(0);
3607 // Use the LibCall instead, it is very likely faster
3608 // FIXME: Use separate LibCall action.
3609 if (TLI.getLibcallName(LC))
3610 break;
3611
3612 if (SDValue Expanded = expandLdexp(Node)) {
3613 Results.push_back(Expanded);
3614 if (Node->getOpcode() == ISD::STRICT_FLDEXP)
3615 Results.push_back(Expanded.getValue(1));
3616 }
3617
3618 break;
3619 }
3620 case ISD::FFREXP: {
3621 RTLIB::Libcall LC = RTLIB::getFREXP(Node->getValueType(0));
3622 // Use the LibCall instead, it is very likely faster
3623 // FIXME: Use separate LibCall action.
3624 if (TLI.getLibcallName(LC))
3625 break;
3626
3627 if (SDValue Expanded = expandFrexp(Node)) {
3628 Results.push_back(Expanded);
3629 Results.push_back(Expanded.getValue(1));
3630 }
3631 break;
3632 }
3633 case ISD::FMAD:
3634 llvm_unreachable("Illegal fmad should never be formed");
3635
3636 case ISD::FP16_TO_FP:
3637 if (Node->getValueType(0) != MVT::f32) {
3638 // We can extend to types bigger than f32 in two steps without changing
3639 // the result. Since "f16 -> f32" is much more commonly available, give
3640 // CodeGen the option of emitting that before resorting to a libcall.
3641 SDValue Res =
3642 DAG.getNode(ISD::FP16_TO_FP, dl, MVT::f32, Node->getOperand(0));
3643 Results.push_back(
3644 DAG.getNode(ISD::FP_EXTEND, dl, Node->getValueType(0), Res));
3645 }
3646 break;
3649 if (Node->getValueType(0) != MVT::f32) {
3650 // We can extend to types bigger than f32 in two steps without changing
3651 // the result. Since "f16 -> f32" is much more commonly available, give
3652 // CodeGen the option of emitting that before resorting to a libcall.
3653 SDValue Res = DAG.getNode(Node->getOpcode(), dl, {MVT::f32, MVT::Other},
3654 {Node->getOperand(0), Node->getOperand(1)});
3655 Res = DAG.getNode(ISD::STRICT_FP_EXTEND, dl,
3656 {Node->getValueType(0), MVT::Other},
3657 {Res.getValue(1), Res});
3658 Results.push_back(Res);
3659 Results.push_back(Res.getValue(1));
3660 }
3661 break;
3662 case ISD::FP_TO_FP16:
3663 LLVM_DEBUG(dbgs() << "Legalizing FP_TO_FP16\n");
3664 if (!TLI.useSoftFloat() && TM.Options.UnsafeFPMath) {
3665 SDValue Op = Node->getOperand(0);
3666 MVT SVT = Op.getSimpleValueType();
3667 if ((SVT == MVT::f64 || SVT == MVT::f80) &&
3668 TLI.isOperationLegalOrCustom(ISD::FP_TO_FP16, MVT::f32)) {
3669 // Under fastmath, we can expand this node into a fround followed by
3670 // a float-half conversion.
3671 SDValue FloatVal =
3672 DAG.getNode(ISD::FP_ROUND, dl, MVT::f32, Op,
3673 DAG.getIntPtrConstant(0, dl, /*isTarget=*/true));
3674 Results.push_back(
3675 DAG.getNode(ISD::FP_TO_FP16, dl, Node->getValueType(0), FloatVal));
3676 }
3677 }
3678 break;
3679 case ISD::ConstantFP: {
3680 ConstantFPSDNode *CFP = cast<ConstantFPSDNode>(Node);
3681 // Check to see if this FP immediate is already legal.
3682 // If this is a legal constant, turn it into a TargetConstantFP node.
3683 if (!TLI.isFPImmLegal(CFP->getValueAPF(), Node->getValueType(0),
3684 DAG.shouldOptForSize()))
3685 Results.push_back(ExpandConstantFP(CFP, true));
3686 break;
3687 }
3688 case ISD::Constant: {
3689 ConstantSDNode *CP = cast<ConstantSDNode>(Node);
3690 Results.push_back(ExpandConstant(CP));
3691 break;
3692 }
3693 case ISD::FSUB: {
3694 EVT VT = Node->getValueType(0);
3695 if (TLI.isOperationLegalOrCustom(ISD::FADD, VT) &&
3696 TLI.isOperationLegalOrCustom(ISD::FNEG, VT)) {
3697 const SDNodeFlags Flags = Node->getFlags();
3698 Tmp1 = DAG.getNode(ISD::FNEG, dl, VT, Node->getOperand(1));
3699 Tmp1 = DAG.getNode(ISD::FADD, dl, VT, Node->getOperand(0), Tmp1, Flags);
3700 Results.push_back(Tmp1);
3701 }
3702 break;
3703 }
3704 case ISD::SUB: {
3705 EVT VT = Node->getValueType(0);
3706 assert(TLI.isOperationLegalOrCustom(ISD::ADD, VT) &&
3707 TLI.isOperationLegalOrCustom(ISD::XOR, VT) &&
3708 "Don't know how to expand this subtraction!");
3709 Tmp1 = DAG.getNOT(dl, Node->getOperand(1), VT);
3710 Tmp1 = DAG.getNode(ISD::ADD, dl, VT, Tmp1, DAG.getConstant(1, dl, VT));
3711 Results.push_back(DAG.getNode(ISD::ADD, dl, VT, Node->getOperand(0), Tmp1));
3712 break;
3713 }
3714 case ISD::UREM:
3715 case ISD::SREM:
3716 if (TLI.expandREM(Node, Tmp1, DAG))
3717 Results.push_back(Tmp1);
3718 break;
3719 case ISD::UDIV:
3720 case ISD::SDIV: {
3721 bool isSigned = Node->getOpcode() == ISD::SDIV;
3722 unsigned DivRemOpc = isSigned ? ISD::SDIVREM : ISD::UDIVREM;
3723 EVT VT = Node->getValueType(0);
3724 if (TLI.isOperationLegalOrCustom(DivRemOpc, VT)) {
3725 SDVTList VTs = DAG.getVTList(VT, VT);
3726 Tmp1 = DAG.getNode(DivRemOpc, dl, VTs, Node->getOperand(0),
3727 Node->getOperand(1));
3728 Results.push_back(Tmp1);
3729 }
3730 break;
3731 }
3732 case ISD::MULHU:
3733 case ISD::MULHS: {
3734 unsigned ExpandOpcode =
3735 Node->getOpcode() == ISD::MULHU ? ISD::UMUL_LOHI : ISD::SMUL_LOHI;
3736 EVT VT = Node->getValueType(0);
3737 SDVTList VTs = DAG.getVTList(VT, VT);
3738
3739 Tmp1 = DAG.getNode(ExpandOpcode, dl, VTs, Node->getOperand(0),
3740 Node->getOperand(1));
3741 Results.push_back(Tmp1.getValue(1));
3742 break;
3743 }
3744 case ISD::UMUL_LOHI:
3745 case ISD::SMUL_LOHI: {
3746 SDValue LHS = Node->getOperand(0);
3747 SDValue RHS = Node->getOperand(1);
3748 MVT VT = LHS.getSimpleValueType();
3749 unsigned MULHOpcode =
3750 Node->getOpcode() == ISD::UMUL_LOHI ? ISD::MULHU : ISD::MULHS;
3751
3752 if (TLI.isOperationLegalOrCustom(MULHOpcode, VT)) {
3753 Results.push_back(DAG.getNode(ISD::MUL, dl, VT, LHS, RHS));
3754 Results.push_back(DAG.getNode(MULHOpcode, dl, VT, LHS, RHS));
3755 break;
3756 }
3757
3759 EVT HalfType = EVT(VT).getHalfSizedIntegerVT(*DAG.getContext());
3760 assert(TLI.isTypeLegal(HalfType));
3761 if (TLI.expandMUL_LOHI(Node->getOpcode(), VT, dl, LHS, RHS, Halves,
3762 HalfType, DAG,
3763 TargetLowering::MulExpansionKind::Always)) {
3764 for (unsigned i = 0; i < 2; ++i) {
3765 SDValue Lo = DAG.getNode(ISD::ZERO_EXTEND, dl, VT, Halves[2 * i]);
3766 SDValue Hi = DAG.getNode(ISD::ANY_EXTEND, dl, VT, Halves[2 * i + 1]);
3767 SDValue Shift = DAG.getConstant(
3768 HalfType.getScalarSizeInBits(), dl,
3769 TLI.getShiftAmountTy(HalfType, DAG.getDataLayout()));
3770 Hi = DAG.getNode(ISD::SHL, dl, VT, Hi, Shift);
3771 Results.push_back(DAG.getNode(ISD::OR, dl, VT, Lo, Hi));
3772 }
3773 break;
3774 }
3775 break;
3776 }
3777 case ISD::MUL: {
3778 EVT VT = Node->getValueType(0);
3779 SDVTList VTs = DAG.getVTList(VT, VT);
3780 // See if multiply or divide can be lowered using two-result operations.
3781 // We just need the low half of the multiply; try both the signed
3782 // and unsigned forms. If the target supports both SMUL_LOHI and
3783 // UMUL_LOHI, form a preference by checking which forms of plain
3784 // MULH it supports.
3785 bool HasSMUL_LOHI = TLI.isOperationLegalOrCustom(ISD::SMUL_LOHI, VT);
3786 bool HasUMUL_LOHI = TLI.isOperationLegalOrCustom(ISD::UMUL_LOHI, VT);
3787 bool HasMULHS = TLI.isOperationLegalOrCustom(ISD::MULHS, VT);
3788 bool HasMULHU = TLI.isOperationLegalOrCustom(ISD::MULHU, VT);
3789 unsigned OpToUse = 0;
3790 if (HasSMUL_LOHI && !HasMULHS) {
3791 OpToUse = ISD::SMUL_LOHI;
3792 } else if (HasUMUL_LOHI && !HasMULHU) {
3793 OpToUse = ISD::UMUL_LOHI;
3794 } else if (HasSMUL_LOHI) {
3795 OpToUse = ISD::SMUL_LOHI;
3796 } else if (HasUMUL_LOHI) {
3797 OpToUse = ISD::UMUL_LOHI;
3798 }
3799 if (OpToUse) {
3800 Results.push_back(DAG.getNode(OpToUse, dl, VTs, Node->getOperand(0),
3801 Node->getOperand(1)));
3802 break;
3803 }
3804
3805 SDValue Lo, Hi;
3806 EVT HalfType = VT.getHalfSizedIntegerVT(*DAG.getContext());
3807 if (TLI.isOperationLegalOrCustom(ISD::ZERO_EXTEND, VT) &&
3808 TLI.isOperationLegalOrCustom(ISD::ANY_EXTEND, VT) &&
3809 TLI.isOperationLegalOrCustom(ISD::SHL, VT) &&
3810 TLI.isOperationLegalOrCustom(ISD::OR, VT) &&
3811 TLI.expandMUL(Node, Lo, Hi, HalfType, DAG,
3812 TargetLowering::MulExpansionKind::OnlyLegalOrCustom)) {
3813 Lo = DAG.getNode(ISD::ZERO_EXTEND, dl, VT, Lo);
3814 Hi = DAG.getNode(ISD::ANY_EXTEND, dl, VT, Hi);
3815 SDValue Shift =
3816 DAG.getConstant(HalfType.getSizeInBits(), dl,
3817 TLI.getShiftAmountTy(HalfType, DAG.getDataLayout()));
3818 Hi = DAG.getNode(ISD::SHL, dl, VT, Hi, Shift);
3819 Results.push_back(DAG.getNode(ISD::OR, dl, VT, Lo, Hi));
3820 }
3821 break;
3822 }
3823 case ISD::FSHL:
3824 case ISD::FSHR:
3825 if (SDValue Expanded = TLI.expandFunnelShift(Node, DAG))
3826 Results.push_back(Expanded);
3827 break;
3828 case ISD::ROTL:
3829 case ISD::ROTR:
3830 if (SDValue Expanded = TLI.expandROT(Node, true /*AllowVectorOps*/, DAG))
3831 Results.push_back(Expanded);
3832 break;
3833 case ISD::SADDSAT:
3834 case ISD::UADDSAT:
3835 case ISD::SSUBSAT:
3836 case ISD::USUBSAT:
3837 Results.push_back(TLI.expandAddSubSat(Node, DAG));
3838 break;
3839 case ISD::SSHLSAT:
3840 case ISD::USHLSAT:
3841 Results.push_back(TLI.expandShlSat(Node, DAG));
3842 break;
3843 case ISD::SMULFIX:
3844 case ISD::SMULFIXSAT:
3845 case ISD::UMULFIX:
3846 case ISD::UMULFIXSAT:
3847 Results.push_back(TLI.expandFixedPointMul(Node, DAG));
3848 break;
3849 case ISD::SDIVFIX:
3850 case ISD::SDIVFIXSAT:
3851 case ISD::UDIVFIX:
3852 case ISD::UDIVFIXSAT:
3853 if (SDValue V = TLI.expandFixedPointDiv(Node->getOpcode(), SDLoc(Node),
3854 Node->getOperand(0),
3855 Node->getOperand(1),
3856 Node->getConstantOperandVal(2),
3857 DAG)) {
3858 Results.push_back(V);
3859 break;
3860 }
3861 // FIXME: We might want to retry here with a wider type if we fail, if that
3862 // type is legal.
3863 // FIXME: Technically, so long as we only have sdivfixes where BW+Scale is
3864 // <= 128 (which is the case for all of the default Embedded-C types),
3865 // we will only get here with types and scales that we could always expand
3866 // if we were allowed to generate libcalls to division functions of illegal
3867 // type. But we cannot do that.
3868 llvm_unreachable("Cannot expand DIVFIX!");
3869 case ISD::UADDO_CARRY:
3870 case ISD::USUBO_CARRY: {
3871 SDValue LHS = Node->getOperand(0);
3872 SDValue RHS = Node->getOperand(1);
3873 SDValue Carry = Node->getOperand(2);
3874
3875 bool IsAdd = Node->getOpcode() == ISD::UADDO_CARRY;
3876
3877 // Initial add of the 2 operands.
3878 unsigned Op = IsAdd ? ISD::ADD : ISD::SUB;
3879 EVT VT = LHS.getValueType();
3880 SDValue Sum = DAG.getNode(Op, dl, VT, LHS, RHS);
3881
3882 // Initial check for overflow.
3883 EVT CarryType = Node->getValueType(1);
3884 EVT SetCCType = getSetCCResultType(Node->getValueType(0));
3886 SDValue Overflow = DAG.getSetCC(dl, SetCCType, Sum, LHS, CC);
3887
3888 // Add of the sum and the carry.
3889 SDValue One = DAG.getConstant(1, dl, VT);
3890 SDValue CarryExt =
3891 DAG.getNode(ISD::AND, dl, VT, DAG.getZExtOrTrunc(Carry, dl, VT), One);
3892 SDValue Sum2 = DAG.getNode(Op, dl, VT, Sum, CarryExt);
3893
3894 // Second check for overflow. If we are adding, we can only overflow if the
3895 // initial sum is all 1s ang the carry is set, resulting in a new sum of 0.
3896 // If we are subtracting, we can only overflow if the initial sum is 0 and
3897 // the carry is set, resulting in a new sum of all 1s.
3898 SDValue Zero = DAG.getConstant(0, dl, VT);
3899 SDValue Overflow2 =
3900 IsAdd ? DAG.getSetCC(dl, SetCCType, Sum2, Zero, ISD::SETEQ)
3901 : DAG.getSetCC(dl, SetCCType, Sum, Zero, ISD::SETEQ);
3902 Overflow2 = DAG.getNode(ISD::AND, dl, SetCCType, Overflow2,
3903 DAG.getZExtOrTrunc(Carry, dl, SetCCType));
3904
3905 SDValue ResultCarry =
3906 DAG.getNode(ISD::OR, dl, SetCCType, Overflow, Overflow2);
3907
3908 Results.push_back(Sum2);
3909 Results.push_back(DAG.getBoolExtOrTrunc(ResultCarry, dl, CarryType, VT));
3910 break;
3911 }
3912 case ISD::SADDO:
3913 case ISD::SSUBO: {
3914 SDValue Result, Overflow;
3915 TLI.expandSADDSUBO(Node, Result, Overflow, DAG);
3916 Results.push_back(Result);
3917 Results.push_back(Overflow);
3918 break;
3919 }
3920 case ISD::UADDO:
3921 case ISD::USUBO: {
3922 SDValue Result, Overflow;
3923 TLI.expandUADDSUBO(Node, Result, Overflow, DAG);
3924 Results.push_back(Result);
3925 Results.push_back(Overflow);
3926 break;
3927 }
3928 case ISD::UMULO:
3929 case ISD::SMULO: {
3930 SDValue Result, Overflow;
3931 if (TLI.expandMULO(Node, Result, Overflow, DAG)) {
3932 Results.push_back(Result);
3933 Results.push_back(Overflow);
3934 }
3935 break;
3936 }
3937 case ISD::BUILD_PAIR: {
3938 EVT PairTy = Node->getValueType(0);
3939 Tmp1 = DAG.getNode(ISD::ZERO_EXTEND, dl, PairTy, Node->getOperand(0));
3940 Tmp2 = DAG.getNode(ISD::ANY_EXTEND, dl, PairTy, Node->getOperand(1));
3941 Tmp2 = DAG.getNode(
3942 ISD::SHL, dl, PairTy, Tmp2,
3943 DAG.getConstant(PairTy.getSizeInBits() / 2, dl,
3944 TLI.getShiftAmountTy(PairTy, DAG.getDataLayout())));
3945 Results.push_back(DAG.getNode(ISD::OR, dl, PairTy, Tmp1, Tmp2));
3946 break;
3947 }
3948 case ISD::SELECT:
3949 Tmp1 = Node->getOperand(0);
3950 Tmp2 = Node->getOperand(1);
3951 Tmp3 = Node->getOperand(2);
3952 if (Tmp1.getOpcode() == ISD::SETCC) {
3953 Tmp1 = DAG.getSelectCC(dl, Tmp1.getOperand(0), Tmp1.getOperand(1),
3954 Tmp2, Tmp3,
3955 cast<CondCodeSDNode>(Tmp1.getOperand(2))->get());
3956 } else {
3957 Tmp1 = DAG.getSelectCC(dl, Tmp1,
3958 DAG.getConstant(0, dl, Tmp1.getValueType()),
3959 Tmp2, Tmp3, ISD::SETNE);
3960 }
3961 Tmp1->setFlags(Node->getFlags());
3962 Results.push_back(Tmp1);
3963 break;
3964 case ISD::BR_JT: {
3965 SDValue Chain = Node->getOperand(0);
3966 SDValue Table = Node->getOperand(1);
3967 SDValue Index = Node->getOperand(2);
3968 int JTI = cast<JumpTableSDNode>(Table.getNode())->getIndex();
3969
3970 const DataLayout &TD = DAG.getDataLayout();
3971 EVT PTy = TLI.getPointerTy(TD);
3972
3973 unsigned EntrySize =
3974 DAG.getMachineFunction().getJumpTableInfo()->getEntrySize(TD);
3975
3976 // For power-of-two jumptable entry sizes convert multiplication to a shift.
3977 // This transformation needs to be done here since otherwise the MIPS
3978 // backend will end up emitting a three instruction multiply sequence
3979 // instead of a single shift and MSP430 will call a runtime function.
3980 if (llvm::isPowerOf2_32(EntrySize))
3981 Index = DAG.getNode(
3982 ISD::SHL, dl, Index.getValueType(), Index,
3983 DAG.getConstant(llvm::Log2_32(EntrySize), dl, Index.getValueType()));
3984 else
3985 Index = DAG.getNode(ISD::MUL, dl, Index.getValueType(), Index,
3986 DAG.getConstant(EntrySize, dl, Index.getValueType()));
3987 SDValue Addr = DAG.getNode(ISD::ADD, dl, Index.getValueType(),
3988 Index, Table);
3989
3990 EVT MemVT = EVT::getIntegerVT(*DAG.getContext(), EntrySize * 8);
3991 SDValue LD = DAG.getExtLoad(
3992 ISD::SEXTLOAD, dl, PTy, Chain, Addr,
3993 MachinePointerInfo::getJumpTable(DAG.getMachineFunction()), MemVT);
3994 Addr = LD;
3995 if (TLI.isJumpTableRelative()) {
3996 // For PIC, the sequence is:
3997 // BRIND(load(Jumptable + index) + RelocBase)
3998 // RelocBase can be JumpTable, GOT or some sort of global base.
3999 Addr = DAG.getNode(ISD::ADD, dl, PTy, Addr,
4000 TLI.getPICJumpTableRelocBase(Table, DAG));
4001 }
4002
4003 Tmp1 = TLI.expandIndirectJTBranch(dl, LD.getValue(1), Addr, JTI, DAG);
4004 Results.push_back(Tmp1);
4005 break;
4006 }
4007 case ISD::BRCOND:
4008 // Expand brcond's setcc into its constituent parts and create a BR_CC
4009 // Node.
4010 Tmp1 = Node->getOperand(0);
4011 Tmp2 = Node->getOperand(1);
4012 if (Tmp2.getOpcode() == ISD::SETCC &&
4013 TLI.isOperationLegalOrCustom(ISD::BR_CC,
4014 Tmp2.getOperand(0).getValueType())) {
4015 Tmp1 = DAG.getNode(ISD::BR_CC, dl, MVT::Other, Tmp1, Tmp2.getOperand(2),
4016 Tmp2.getOperand(0), Tmp2.getOperand(1),
4017 Node->getOperand(2));
4018 } else {
4019 // We test only the i1 bit. Skip the AND if UNDEF or another AND.
4020 if (Tmp2.isUndef() ||
4021 (Tmp2.getOpcode() == ISD::AND && isOneConstant(Tmp2.getOperand(1))))
4022 Tmp3 = Tmp2;
4023 else
4024 Tmp3 = DAG.getNode(ISD::AND, dl, Tmp2.getValueType(), Tmp2,
4025 DAG.getConstant(1, dl, Tmp2.getValueType()));
4026 Tmp1 = DAG.getNode(ISD::BR_CC, dl, MVT::Other, Tmp1,
4027 DAG.getCondCode(ISD::SETNE), Tmp3,
4028 DAG.getConstant(0, dl, Tmp3.getValueType()),
4029 Node->getOperand(2));
4030 }
4031 Results.push_back(Tmp1);
4032 break;
4033 case ISD::SETCC:
4034 case ISD::VP_SETCC:
4035 case ISD::STRICT_FSETCC:
4036 case ISD::STRICT_FSETCCS: {
4037 bool IsVP = Node->getOpcode() == ISD::VP_SETCC;
4038 bool IsStrict = Node->getOpcode() == ISD::STRICT_FSETCC ||
4039 Node->getOpcode() == ISD::STRICT_FSETCCS;
4040 bool IsSignaling = Node->getOpcode() == ISD::STRICT_FSETCCS;
4041 SDValue Chain = IsStrict ? Node->getOperand(0) : SDValue();
4042 unsigned Offset = IsStrict ? 1 : 0;
4043 Tmp1 = Node->getOperand(0 + Offset);
4044 Tmp2 = Node->getOperand(1 + Offset);
4045 Tmp3 = Node->getOperand(2 + Offset);
4046 SDValue Mask, EVL;
4047 if (IsVP) {
4048 Mask = Node->getOperand(3 + Offset);
4049 EVL = Node->getOperand(4 + Offset);
4050 }
4051 bool Legalized = TLI.LegalizeSetCCCondCode(
4052 DAG, Node->getValueType(0), Tmp1, Tmp2, Tmp3, Mask, EVL, NeedInvert, dl,
4053 Chain, IsSignaling);
4054
4055 if (Legalized) {
4056 // If we expanded the SETCC by swapping LHS and RHS, or by inverting the
4057 // condition code, create a new SETCC node.
4058 if (Tmp3.getNode()) {
4059 if (IsStrict) {
4060 Tmp1 = DAG.getNode(Node->getOpcode(), dl, Node->getVTList(),
4061 {Chain, Tmp1, Tmp2, Tmp3}, Node->getFlags());
4062 Chain = Tmp1.getValue(1);
4063 } else if (IsVP) {
4064 Tmp1 = DAG.getNode(Node->getOpcode(), dl, Node->getValueType(0),
4065 {Tmp1, Tmp2, Tmp3, Mask, EVL}, Node->getFlags());
4066 } else {
4067 Tmp1 = DAG.getNode(Node->getOpcode(), dl, Node->getValueType(0), Tmp1,
4068 Tmp2, Tmp3, Node->getFlags());
4069 }
4070 }
4071
4072 // If we expanded the SETCC by inverting the condition code, then wrap
4073 // the existing SETCC in a NOT to restore the intended condition.
4074 if (NeedInvert) {
4075 if (!IsVP)
4076 Tmp1 = DAG.getLogicalNOT(dl, Tmp1, Tmp1->getValueType(0));
4077 else
4078 Tmp1 =
4079 DAG.getVPLogicalNOT(dl, Tmp1, Mask, EVL, Tmp1->getValueType(0));
4080 }
4081
4082 Results.push_back(Tmp1);
4083 if (IsStrict)
4084 Results.push_back(Chain);
4085
4086 break;
4087 }
4088
4089 // FIXME: It seems Legalized is false iff CCCode is Legal. I don't
4090 // understand if this code is useful for strict nodes.
4091 assert(!IsStrict && "Don't know how to expand for strict nodes.");
4092
4093 // Otherwise, SETCC for the given comparison type must be completely
4094 // illegal; expand it into a SELECT_CC.
4095 // FIXME: This drops the mask/evl for VP_SETCC.
4096 EVT VT = Node->getValueType(0);
4097 EVT Tmp1VT = Tmp1.getValueType();
4098 Tmp1 = DAG.getNode(ISD::SELECT_CC, dl, VT, Tmp1, Tmp2,
4099 DAG.getBoolConstant(true, dl, VT, Tmp1VT),
4100 DAG.getBoolConstant(false, dl, VT, Tmp1VT), Tmp3);
4101 Tmp1->setFlags(Node->getFlags());
4102 Results.push_back(Tmp1);
4103 break;
4104 }
4105 case ISD::SELECT_CC: {
4106 // TODO: need to add STRICT_SELECT_CC and STRICT_SELECT_CCS
4107 Tmp1 = Node->getOperand(0); // LHS
4108 Tmp2 = Node->getOperand(1); // RHS
4109 Tmp3 = Node->getOperand(2); // True
4110 Tmp4 = Node->getOperand(3); // False
4111 EVT VT = Node->getValueType(0);
4112 SDValue Chain;
4113 SDValue CC = Node->getOperand(4);
4114 ISD::CondCode CCOp = cast<CondCodeSDNode>(CC)->get();
4115
4116 if (TLI.isCondCodeLegalOrCustom(CCOp, Tmp1.getSimpleValueType())) {
4117 // If the condition code is legal, then we need to expand this
4118 // node using SETCC and SELECT.
4119 EVT CmpVT = Tmp1.getValueType();
4120 assert(!TLI.isOperationExpand(ISD::SELECT, VT) &&
4121 "Cannot expand ISD::SELECT_CC when ISD::SELECT also needs to be "
4122 "expanded.");
4123 EVT CCVT = getSetCCResultType(CmpVT);
4124 SDValue Cond = DAG.getNode(ISD::SETCC, dl, CCVT, Tmp1, Tmp2, CC, Node->getFlags());
4125 Results.push_back(DAG.getSelect(dl, VT, Cond, Tmp3, Tmp4));
4126 break;
4127 }
4128
4129 // SELECT_CC is legal, so the condition code must not be.
4130 bool Legalized = false;
4131 // Try to legalize by inverting the condition. This is for targets that
4132 // might support an ordered version of a condition, but not the unordered
4133 // version (or vice versa).
4134 ISD::CondCode InvCC = ISD::getSetCCInverse(CCOp, Tmp1.getValueType());
4135 if (TLI.isCondCodeLegalOrCustom(InvCC, Tmp1.getSimpleValueType())) {
4136 // Use the new condition code and swap true and false
4137 Legalized = true;
4138 Tmp1 = DAG.getSelectCC(dl, Tmp1, Tmp2, Tmp4, Tmp3, InvCC);
4139 Tmp1->setFlags(Node->getFlags());
4140 } else {
4141 // If The inverse is not legal, then try to swap the arguments using
4142 // the inverse condition code.
4144 if (TLI.isCondCodeLegalOrCustom(SwapInvCC, Tmp1.getSimpleValueType())) {
4145 // The swapped inverse condition is legal, so swap true and false,
4146 // lhs and rhs.
4147 Legalized = true;
4148 Tmp1 = DAG.getSelectCC(dl, Tmp2, Tmp1, Tmp4, Tmp3, SwapInvCC);
4149 Tmp1->setFlags(Node->getFlags());
4150 }
4151 }
4152
4153 if (!Legalized) {
4154 Legalized = TLI.LegalizeSetCCCondCode(
4155 DAG, getSetCCResultType(Tmp1.getValueType()), Tmp1, Tmp2, CC,
4156 /*Mask*/ SDValue(), /*EVL*/ SDValue(), NeedInvert, dl, Chain);
4157
4158 assert(Legalized && "Can't legalize SELECT_CC with legal condition!");
4159
4160 // If we expanded the SETCC by inverting the condition code, then swap
4161 // the True/False operands to match.
4162 if (NeedInvert)
4163 std::swap(Tmp3, Tmp4);
4164
4165 // If we expanded the SETCC by swapping LHS and RHS, or by inverting the
4166 // condition code, create a new SELECT_CC node.
4167 if (CC.getNode()) {
4168 Tmp1 = DAG.getNode(ISD::SELECT_CC, dl, Node->getValueType(0),
4169 Tmp1, Tmp2, Tmp3, Tmp4, CC);
4170 } else {
4171 Tmp2 = DAG.getConstant(0, dl, Tmp1.getValueType());
4172 CC = DAG.getCondCode(ISD::SETNE);
4173 Tmp1 = DAG.getNode(ISD::SELECT_CC, dl, Node->getValueType(0), Tmp1,
4174 Tmp2, Tmp3, Tmp4, CC);
4175 }
4176 Tmp1->setFlags(Node->getFlags());
4177 }
4178 Results.push_back(Tmp1);
4179 break;
4180 }
4181 case ISD::BR_CC: {
4182 // TODO: need to add STRICT_BR_CC and STRICT_BR_CCS
4183 SDValue Chain;
4184 Tmp1 = Node->getOperand(0); // Chain
4185 Tmp2 = Node->getOperand(2); // LHS
4186 Tmp3 = Node->getOperand(3); // RHS
4187 Tmp4 = Node->getOperand(1); // CC
4188
4189 bool Legalized = TLI.LegalizeSetCCCondCode(
4190 DAG, getSetCCResultType(Tmp2.getValueType()), Tmp2, Tmp3, Tmp4,
4191 /*Mask*/ SDValue(), /*EVL*/ SDValue(), NeedInvert, dl, Chain);
4192 (void)Legalized;
4193 assert(Legalized && "Can't legalize BR_CC with legal condition!");
4194
4195 // If we expanded the SETCC by swapping LHS and RHS, create a new BR_CC
4196 // node.
4197 if (Tmp4.getNode()) {
4198 assert(!NeedInvert && "Don't know how to invert BR_CC!");
4199
4200 Tmp1 = DAG.getNode(ISD::BR_CC, dl, Node->getValueType(0), Tmp1,
4201 Tmp4, Tmp2, Tmp3, Node->getOperand(4));
4202 } else {
4203 Tmp3 = DAG.getConstant(0, dl, Tmp2.getValueType());
4204 Tmp4 = DAG.getCondCode(NeedInvert ? ISD::SETEQ : ISD::SETNE);
4205 Tmp1 = DAG.getNode(ISD::BR_CC, dl, Node->getValueType(0), Tmp1, Tmp4,
4206 Tmp2, Tmp3, Node->getOperand(4));
4207 }
4208 Results.push_back(Tmp1);
4209 break;
4210 }
4211 case ISD::BUILD_VECTOR:
4212 Results.push_back(ExpandBUILD_VECTOR(Node));
4213 break;
4214 case ISD::SPLAT_VECTOR:
4215 Results.push_back(ExpandSPLAT_VECTOR(Node));
4216 break;
4217 case ISD::SRA:
4218 case ISD::SRL:
4219 case ISD::SHL: {
4220 // Scalarize vector SRA/SRL/SHL.
4221 EVT VT = Node->getValueType(0);
4222 assert(VT.isVector() && "Unable to legalize non-vector shift");
4223 assert(TLI.isTypeLegal(VT.getScalarType())&& "Element type must be legal");
4224 unsigned NumElem = VT.getVectorNumElements();
4225
4227 for (unsigned Idx = 0; Idx < NumElem; Idx++) {
4228 SDValue Ex =
4230 Node->getOperand(0), DAG.getVectorIdxConstant(Idx, dl));
4231 SDValue Sh =
4233 Node->getOperand(1), DAG.getVectorIdxConstant(Idx, dl));
4234 Scalars.push_back(DAG.getNode(Node->getOpcode(), dl,
4235 VT.getScalarType(), Ex, Sh));
4236 }
4237
4238 SDValue Result = DAG.getBuildVector(Node->getValueType(0), dl, Scalars);
4239 Results.push_back(Result);
4240 break;
4241 }
4244 case ISD::VECREDUCE_ADD:
4245 case ISD::VECREDUCE_MUL:
4246 case ISD::VECREDUCE_AND:
4247 case ISD::VECREDUCE_OR:
4248 case ISD::VECREDUCE_XOR:
4257 Results.push_back(TLI.expandVecReduce(Node, DAG));
4258 break;
4260 case ISD::GlobalAddress:
4263 case ISD::ConstantPool:
4264 case ISD::JumpTable:
4268 // FIXME: Custom lowering for these operations shouldn't return null!
4269 // Return true so that we don't call ConvertNodeToLibcall which also won't
4270 // do anything.
4271 return true;
4272 }
4273
4274 if (!TLI.isStrictFPEnabled() && Results.empty() && Node->isStrictFPOpcode()) {
4275 // FIXME: We were asked to expand a strict floating-point operation,
4276 // but there is currently no expansion implemented that would preserve
4277 // the "strict" properties. For now, we just fall back to the non-strict
4278 // version if that is legal on the target. The actual mutation of the
4279 // operation will happen in SelectionDAGISel::DoInstructionSelection.
4280 switch (Node->getOpcode()) {
4281 default:
4282 if (TLI.getStrictFPOperationAction(Node->getOpcode(),
4283 Node->getValueType(0))
4284 == TargetLowering::Legal)
4285 return true;
4286 break;
4287 case ISD::STRICT_FSUB: {
4288 if (TLI.getStrictFPOperationAction(
4289 ISD::STRICT_FSUB, Node->getValueType(0)) == TargetLowering::Legal)
4290 return true;
4291 if (TLI.getStrictFPOperationAction(
4292 ISD::STRICT_FADD, Node->getValueType(0)) != TargetLowering::Legal)
4293 break;
4294
4295 EVT VT = Node->getValueType(0);
4296 const SDNodeFlags Flags = Node->getFlags();
4297 SDValue Neg = DAG.getNode(ISD::FNEG, dl, VT, Node->getOperand(2), Flags);
4298 SDValue Fadd = DAG.getNode(ISD::STRICT_FADD, dl, Node->getVTList(),
4299 {Node->getOperand(0), Node->getOperand(1), Neg},
4300 Flags);
4301
4302 Results.push_back(Fadd);
4303 Results.push_back(Fadd.getValue(1));
4304 break;
4305 }
4308 case ISD::STRICT_LRINT:
4309 case ISD::STRICT_LLRINT:
4310 case ISD::STRICT_LROUND:
4312 // These are registered by the operand type instead of the value
4313 // type. Reflect that here.
4314 if (TLI.getStrictFPOperationAction(Node->getOpcode(),
4315 Node->getOperand(1).getValueType())
4316 == TargetLowering::Legal)
4317 return true;
4318 break;
4319 }
4320 }
4321
4322 // Replace the original node with the legalized result.
4323 if (Results.empty()) {
4324 LLVM_DEBUG(dbgs() << "Cannot expand node\n");
4325 return false;
4326 }
4327
4328 LLVM_DEBUG(dbgs() << "Successfully expanded node\n");
4329 ReplaceNode(Node, Results.data());
4330 return true;
4331}
4332
4333void SelectionDAGLegalize::ConvertNodeToLibcall(SDNode *Node) {
4334 LLVM_DEBUG(dbgs() << "Trying to convert node to libcall\n");
4336 SDLoc dl(Node);
4337 // FIXME: Check flags on the node to see if we can use a finite call.
4338 unsigned Opc = Node->getOpcode();
4339 switch (Opc) {
4340 case ISD::ATOMIC_FENCE: {
4341 // If the target didn't lower this, lower it to '__sync_synchronize()' call
4342 // FIXME: handle "fence singlethread" more efficiently.
4344
4346 CLI.setDebugLoc(dl)
4347 .setChain(Node->getOperand(0))
4348 .setLibCallee(
4349 CallingConv::C, Type::getVoidTy(*DAG.getContext()),
4350 DAG.getExternalSymbol("__sync_synchronize",
4351 TLI.getPointerTy(DAG.getDataLayout())),
4352 std::move(Args));
4353
4354 std::pair<SDValue, SDValue> CallResult = TLI.LowerCallTo(CLI);
4355
4356 Results.push_back(CallResult.second);
4357 break;
4358 }
4359 // By default, atomic intrinsics are marked Legal and lowered. Targets
4360 // which don't support them directly, however, may want libcalls, in which
4361 // case they mark them Expand, and we get here.
4362 case ISD::ATOMIC_SWAP:
4374 case ISD::ATOMIC_CMP_SWAP: {
4375 MVT VT = cast<AtomicSDNode>(Node)->getMemoryVT().getSimpleVT();
4376 AtomicOrdering Order = cast<AtomicSDNode>(Node)->getMergedOrdering();
4377 RTLIB::Libcall LC = RTLIB::getOUTLINE_ATOMIC(Opc, Order, VT);
4378 EVT RetVT = Node->getValueType(0);
4381 if (TLI.getLibcallName(LC)) {
4382 // If outline atomic available, prepare its arguments and expand.
4383 Ops.append(Node->op_begin() + 2, Node->op_end());
4384 Ops.push_back(Node->getOperand(1));
4385
4386 } else {
4387 LC = RTLIB::getSYNC(Opc, VT);
4388 assert(LC != RTLIB::UNKNOWN_LIBCALL &&
4389 "Unexpected atomic op or value type!");
4390 // Arguments for expansion to sync libcall
4391 Ops.append(Node->op_begin() + 1, Node->op_end());
4392 }
4393 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, RetVT,
4394 Ops, CallOptions,
4395 SDLoc(Node),
4396 Node->getOperand(0));
4397 Results.push_back(Tmp.first);
4398 Results.push_back(Tmp.second);
4399 break;
4400 }
4401 case ISD::TRAP: {
4402 // If this operation is not supported, lower it to 'abort()' call
4405 CLI.setDebugLoc(dl)
4406 .setChain(Node->getOperand(0))
4407 .setLibCallee(CallingConv::C, Type::getVoidTy(*DAG.getContext()),
4408 DAG.getExternalSymbol(
4409 "abort", TLI.getPointerTy(DAG.getDataLayout())),
4410 std::move(Args));
4411 std::pair<SDValue, SDValue> CallResult = TLI.LowerCallTo(CLI);
4412
4413 Results.push_back(CallResult.second);
4414 break;
4415 }
4416 case ISD::FMINNUM:
4418 ExpandFPLibCall(Node, RTLIB::FMIN_F32, RTLIB::FMIN_F64,
4419 RTLIB::FMIN_F80, RTLIB::FMIN_F128,
4420 RTLIB::FMIN_PPCF128, Results);
4421 break;
4422 // FIXME: We do not have libcalls for FMAXIMUM and FMINIMUM. So, we cannot use
4423 // libcall legalization for these nodes, but there is no default expasion for
4424 // these nodes either (see PR63267 for example).
4425 case ISD::FMAXNUM:
4427 ExpandFPLibCall(Node, RTLIB::FMAX_F32, RTLIB::FMAX_F64,
4428 RTLIB::FMAX_F80, RTLIB::FMAX_F128,
4429 RTLIB::FMAX_PPCF128, Results);
4430 break;
4431 case ISD::FSQRT:
4432 case ISD::STRICT_FSQRT:
4433 ExpandFPLibCall(Node, RTLIB::SQRT_F32, RTLIB::SQRT_F64,
4434 RTLIB::SQRT_F80, RTLIB::SQRT_F128,
4435 RTLIB::SQRT_PPCF128, Results);
4436 break;
4437 case ISD::FCBRT:
4438 ExpandFPLibCall(Node, RTLIB::CBRT_F32, RTLIB::CBRT_F64,
4439 RTLIB::CBRT_F80, RTLIB::CBRT_F128,
4440 RTLIB::CBRT_PPCF128, Results);
4441 break;
4442 case ISD::FSIN:
4443 case ISD::STRICT_FSIN:
4444 ExpandFPLibCall(Node, RTLIB::SIN_F32, RTLIB::SIN_F64,
4445 RTLIB::SIN_F80, RTLIB::SIN_F128,
4446 RTLIB::SIN_PPCF128, Results);
4447 break;
4448 case ISD::FCOS:
4449 case ISD::STRICT_FCOS:
4450 ExpandFPLibCall(Node, RTLIB::COS_F32, RTLIB::COS_F64,
4451 RTLIB::COS_F80, RTLIB::COS_F128,
4452 RTLIB::COS_PPCF128, Results);
4453 break;
4454 case ISD::FSINCOS:
4455 // Expand into sincos libcall.
4456 ExpandSinCosLibCall(Node, Results);
4457 break;
4458 case ISD::FLOG:
4459 case ISD::STRICT_FLOG:
4460 ExpandFPLibCall(Node, RTLIB::LOG_F32, RTLIB::LOG_F64, RTLIB::LOG_F80,
4461 RTLIB::LOG_F128, RTLIB::LOG_PPCF128, Results);
4462 break;
4463 case ISD::FLOG2:
4464 case ISD::STRICT_FLOG2:
4465 ExpandFPLibCall(Node, RTLIB::LOG2_F32, RTLIB::LOG2_F64, RTLIB::LOG2_F80,
4466 RTLIB::LOG2_F128, RTLIB::LOG2_PPCF128, Results);
4467 break;
4468 case ISD::FLOG10:
4469 case ISD::STRICT_FLOG10:
4470 ExpandFPLibCall(Node, RTLIB::LOG10_F32, RTLIB::LOG10_F64, RTLIB::LOG10_F80,
4471 RTLIB::LOG10_F128, RTLIB::LOG10_PPCF128, Results);
4472 break;
4473 case ISD::FEXP:
4474 case ISD::STRICT_FEXP:
4475 ExpandFPLibCall(Node, RTLIB::EXP_F32, RTLIB::EXP_F64, RTLIB::EXP_F80,
4476 RTLIB::EXP_F128, RTLIB::EXP_PPCF128, Results);
4477 break;
4478 case ISD::FEXP2:
4479 case ISD::STRICT_FEXP2:
4480 ExpandFPLibCall(Node, RTLIB::EXP2_F32, RTLIB::EXP2_F64, RTLIB::EXP2_F80,
4481 RTLIB::EXP2_F128, RTLIB::EXP2_PPCF128, Results);
4482 break;
4483 case ISD::FEXP10:
4484 ExpandFPLibCall(Node, RTLIB::EXP10_F32, RTLIB::EXP10_F64, RTLIB::EXP10_F80,
4485 RTLIB::EXP10_F128, RTLIB::EXP10_PPCF128, Results);
4486 break;
4487 case ISD::FTRUNC:
4488 case ISD::STRICT_FTRUNC:
4489 ExpandFPLibCall(Node, RTLIB::TRUNC_F32, RTLIB::TRUNC_F64,
4490 RTLIB::TRUNC_F80, RTLIB::TRUNC_F128,
4491 RTLIB::TRUNC_PPCF128, Results);
4492 break;
4493 case ISD::FFLOOR:
4494 case ISD::STRICT_FFLOOR:
4495 ExpandFPLibCall(Node, RTLIB::FLOOR_F32, RTLIB::FLOOR_F64,
4496 RTLIB::FLOOR_F80, RTLIB::FLOOR_F128,
4497 RTLIB::FLOOR_PPCF128, Results);
4498 break;
4499 case ISD::FCEIL:
4500 case ISD::STRICT_FCEIL:
4501 ExpandFPLibCall(Node, RTLIB::CEIL_F32, RTLIB::CEIL_F64,
4502 RTLIB::CEIL_F80, RTLIB::CEIL_F128,
4503 RTLIB::CEIL_PPCF128, Results);
4504 break;
4505 case ISD::FRINT:
4506 case ISD::STRICT_FRINT:
4507 ExpandFPLibCall(Node, RTLIB::RINT_F32, RTLIB::RINT_F64,
4508 RTLIB::RINT_F80, RTLIB::RINT_F128,
4509 RTLIB::RINT_PPCF128, Results);
4510 break;
4511 case ISD::FNEARBYINT:
4513 ExpandFPLibCall(Node, RTLIB::NEARBYINT_F32,
4514 RTLIB::NEARBYINT_F64,
4515 RTLIB::NEARBYINT_F80,
4516 RTLIB::NEARBYINT_F128,
4517 RTLIB::NEARBYINT_PPCF128, Results);
4518 break;
4519 case ISD::FROUND:
4520 case ISD::STRICT_FROUND:
4521 ExpandFPLibCall(Node, RTLIB::ROUND_F32,
4522 RTLIB::ROUND_F64,
4523 RTLIB::ROUND_F80,
4524 RTLIB::ROUND_F128,
4525 RTLIB::ROUND_PPCF128, Results);
4526 break;
4527 case ISD::FROUNDEVEN:
4529 ExpandFPLibCall(Node, RTLIB::ROUNDEVEN_F32,
4530 RTLIB::ROUNDEVEN_F64,
4531 RTLIB::ROUNDEVEN_F80,
4532 RTLIB::ROUNDEVEN_F128,
4533 RTLIB::ROUNDEVEN_PPCF128, Results);
4534 break;
4535 case ISD::FLDEXP:
4536 case ISD::STRICT_FLDEXP:
4537 ExpandFPLibCall(Node, RTLIB::LDEXP_F32, RTLIB::LDEXP_F64, RTLIB::LDEXP_F80,
4538 RTLIB::LDEXP_F128, RTLIB::LDEXP_PPCF128, Results);
4539 break;
4540 case ISD::FFREXP: {
4541 ExpandFrexpLibCall(Node, Results);
4542 break;
4543 }
4544 case ISD::FPOWI:
4545 case ISD::STRICT_FPOWI: {
4546 RTLIB::Libcall LC = RTLIB::getPOWI(Node->getSimpleValueType(0));
4547 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected fpowi.");
4548 if (!TLI.getLibcallName(LC)) {
4549 // Some targets don't have a powi libcall; use pow instead.
4550 if (Node->isStrictFPOpcode()) {
4552 DAG.getNode(ISD::STRICT_SINT_TO_FP, SDLoc(Node),
4553 {Node->getValueType(0), Node->getValueType(1)},
4554 {Node->getOperand(0), Node->getOperand(2)});
4555 SDValue FPOW =
4556 DAG.getNode(ISD::STRICT_FPOW, SDLoc(Node),
4557 {Node->getValueType(0), Node->getValueType(1)},
4558 {Exponent.getValue(1), Node->getOperand(1), Exponent});
4559 Results.push_back(FPOW);
4560 Results.push_back(FPOW.getValue(1));
4561 } else {
4563 DAG.getNode(ISD::SINT_TO_FP, SDLoc(Node), Node->getValueType(0),
4564 Node->getOperand(1));
4565 Results.push_back(DAG.getNode(ISD::FPOW, SDLoc(Node),
4566 Node->getValueType(0),
4567 Node->getOperand(0), Exponent));
4568 }
4569 break;
4570 }
4571 unsigned Offset = Node->isStrictFPOpcode() ? 1 : 0;
4572 bool ExponentHasSizeOfInt =
4573 DAG.getLibInfo().getIntSize() ==
4574 Node->getOperand(1 + Offset).getValueType().getSizeInBits();
4575 if (!ExponentHasSizeOfInt) {
4576 // If the exponent does not match with sizeof(int) a libcall to
4577 // RTLIB::POWI would use the wrong type for the argument.
4578 DAG.getContext()->emitError("POWI exponent does not match sizeof(int)");
4579 Results.push_back(DAG.getUNDEF(Node->getValueType(0)));
4580 break;
4581 }
4582 ExpandFPLibCall(Node, LC, Results);
4583 break;
4584 }
4585 case ISD::FPOW:
4586 case ISD::STRICT_FPOW:
4587 ExpandFPLibCall(Node, RTLIB::POW_F32, RTLIB::POW_F64, RTLIB::POW_F80,
4588 RTLIB::POW_F128, RTLIB::POW_PPCF128, Results);
4589 break;
4590 case ISD::LROUND:
4591 case ISD::STRICT_LROUND:
4592 ExpandArgFPLibCall(Node, RTLIB::LROUND_F32,
4593 RTLIB::LROUND_F64, RTLIB::LROUND_F80,
4594 RTLIB::LROUND_F128,
4595 RTLIB::LROUND_PPCF128, Results);
4596 break;
4597 case ISD::LLROUND:
4599 ExpandArgFPLibCall(Node, RTLIB::LLROUND_F32,
4600 RTLIB::LLROUND_F64, RTLIB::LLROUND_F80,
4601 RTLIB::LLROUND_F128,
4602 RTLIB::LLROUND_PPCF128, Results);
4603 break;
4604 case ISD::LRINT:
4605 case ISD::STRICT_LRINT:
4606 ExpandArgFPLibCall(Node, RTLIB::LRINT_F32,
4607 RTLIB::LRINT_F64, RTLIB::LRINT_F80,
4608 RTLIB::LRINT_F128,
4609 RTLIB::LRINT_PPCF128, Results);
4610 break;
4611 case ISD::LLRINT:
4612 case ISD::STRICT_LLRINT:
4613 ExpandArgFPLibCall(Node, RTLIB::LLRINT_F32,
4614 RTLIB::LLRINT_F64, RTLIB::LLRINT_F80,
4615 RTLIB::LLRINT_F128,
4616 RTLIB::LLRINT_PPCF128, Results);
4617 break;
4618 case ISD::FDIV:
4619 case ISD::STRICT_FDIV:
4620 ExpandFPLibCall(Node, RTLIB::DIV_F32, RTLIB::DIV_F64,
4621 RTLIB::DIV_F80, RTLIB::DIV_F128,
4622 RTLIB::DIV_PPCF128, Results);
4623 break;
4624 case ISD::FREM:
4625 case ISD::STRICT_FREM:
4626 ExpandFPLibCall(Node, RTLIB::REM_F32, RTLIB::REM_F64,
4627 RTLIB::REM_F80, RTLIB::REM_F128,
4628 RTLIB::REM_PPCF128, Results);
4629 break;
4630 case ISD::FMA:
4631 case ISD::STRICT_FMA:
4632 ExpandFPLibCall(Node, RTLIB::FMA_F32, RTLIB::FMA_F64,
4633 RTLIB::FMA_F80, RTLIB::FMA_F128,
4634 RTLIB::FMA_PPCF128, Results);
4635 break;
4636 case ISD::FADD:
4637 case ISD::STRICT_FADD:
4638 ExpandFPLibCall(Node, RTLIB::ADD_F32, RTLIB::ADD_F64,
4639 RTLIB::ADD_F80, RTLIB::ADD_F128,
4640 RTLIB::ADD_PPCF128, Results);
4641 break;
4642 case ISD::FMUL:
4643 case ISD::STRICT_FMUL:
4644 ExpandFPLibCall(Node, RTLIB::MUL_F32, RTLIB::MUL_F64,
4645 RTLIB::MUL_F80, RTLIB::MUL_F128,
4646 RTLIB::MUL_PPCF128, Results);
4647 break;
4648 case ISD::FP16_TO_FP:
4649 if (Node->getValueType(0) == MVT::f32) {
4650 Results.push_back(ExpandLibCall(RTLIB::FPEXT_F16_F32, Node, false).first);
4651 }
4652 break;
4654 if (Node->getValueType(0) == MVT::f32) {
4656 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(
4657 DAG, RTLIB::FPEXT_BF16_F32, MVT::f32, Node->getOperand(1),
4658 CallOptions, SDLoc(Node), Node->getOperand(0));
4659 Results.push_back(Tmp.first);
4660 Results.push_back(Tmp.second);
4661 }
4662 break;
4664 if (Node->getValueType(0) == MVT::f32) {
4666 std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(
4667 DAG, RTLIB::FPEXT_F16_F32, MVT::f32, Node->getOperand(1), CallOptions,
4668 SDLoc(Node), Node->getOperand(0));
4669 Results.push_back(Tmp.first);
4670 Results.push_back(Tmp.second);
4671 }
4672 break;
4673 }
4674 case ISD::FP_TO_FP16: {
4675 RTLIB::Libcall LC =
4676 RTLIB::getFPROUND(Node->getOperand(0).getValueType(), MVT::f16);
4677 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unable to expand fp_to_fp16");
4678 Results.push_back(ExpandLibCall(LC, Node, false).first);
4679 break;
4680 }
4681 case ISD::FP_TO_BF16: {
4682 RTLIB::Libcall LC =
4683 RTLIB::getFPROUND(Node->getOperand(0).getValueType(), MVT::bf16);
4684 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unable to expand fp_to_bf16");
4685 Results.push_back(ExpandLibCall(LC, Node, false).first);
4686 break;
4687 }
4690 case ISD::SINT_TO_FP:
4691 case ISD::UINT_TO_FP: {
4692 // TODO - Common the code with DAGTypeLegalizer::SoftenFloatRes_XINT_TO_FP
4693 bool IsStrict = Node->isStrictFPOpcode();
4694 bool Signed = Node->getOpcode() == ISD::SINT_TO_FP ||
4695 Node->getOpcode() == ISD::STRICT_SINT_TO_FP;
4696 EVT SVT = Node->getOperand(IsStrict ? 1 : 0).getValueType();
4697 EVT RVT = Node->getValueType(0);
4698 EVT NVT = EVT();
4699 SDLoc dl(Node);
4700
4701 // Even if the input is legal, no libcall may exactly match, eg. we don't
4702 // have i1 -> fp conversions. So, it needs to be promoted to a larger type,
4703 // eg: i13 -> fp. Then, look for an appropriate libcall.
4704 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
4705 for (unsigned t = MVT::FIRST_INTEGER_VALUETYPE;
4706 t <= MVT::LAST_INTEGER_VALUETYPE && LC == RTLIB::UNKNOWN_LIBCALL;
4707 ++t) {
4708 NVT = (MVT::SimpleValueType)t;
4709 // The source needs to big enough to hold the operand.
4710 if (NVT.bitsGE(SVT))
4711 LC = Signed ? RTLIB::getSINTTOFP(NVT, RVT)
4712 : RTLIB::getUINTTOFP(NVT, RVT);
4713 }
4714 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unable to legalize as libcall");
4715
4716 SDValue Chain = IsStrict ? Node->getOperand(0) : SDValue();
4717 // Sign/zero extend the argument if the libcall takes a larger type.
4718 SDValue Op = DAG.getNode(Signed ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND, dl,
4719 NVT, Node->getOperand(IsStrict ? 1 : 0));
4721 CallOptions.setSExt(Signed);
4722 std::pair<SDValue, SDValue> Tmp =
4723 TLI.makeLibCall(DAG, LC, RVT, Op, CallOptions, dl, Chain);
4724 Results.push_back(Tmp.first);
4725 if (IsStrict)
4726 Results.push_back(Tmp.second);
4727 break;
4728 }
4729 case ISD::FP_TO_SINT:
4730 case ISD::FP_TO_UINT:
4733 // TODO - Common the code with DAGTypeLegalizer::SoftenFloatOp_FP_TO_XINT.
4734 bool IsStrict = Node->isStrictFPOpcode();
4735 bool Signed = Node->getOpcode() == ISD::FP_TO_SINT ||
4736 Node->getOpcode() == ISD::STRICT_FP_TO_SINT;
4737
4738 SDValue Op = Node->getOperand(IsStrict ? 1 : 0);
4739 EVT SVT = Op.getValueType();
4740 EVT RVT = Node->getValueType(0);
4741 EVT NVT = EVT();
4742 SDLoc dl(Node);
4743
4744 // Even if the result is legal, no libcall may exactly match, eg. we don't
4745 // have fp -> i1 conversions. So, it needs to be promoted to a larger type,
4746 // eg: fp -> i32. Then, look for an appropriate libcall.
4747 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
4748 for (unsigned IntVT = MVT::FIRST_INTEGER_VALUETYPE;
4749 IntVT <= MVT::LAST_INTEGER_VALUETYPE && LC == RTLIB::UNKNOWN_LIBCALL;
4750 ++IntVT) {
4751 NVT = (MVT::SimpleValueType)IntVT;
4752 // The type needs to big enough to hold the result.
4753 if (NVT.bitsGE(RVT))
4754 LC = Signed ? RTLIB::getFPTOSINT(SVT, NVT)
4755 : RTLIB::getFPTOUINT(SVT, NVT);
4756 }
4757 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unable to legalize as libcall");
4758
4759 SDValue Chain = IsStrict ? Node->getOperand(0) : SDValue();
4761 std::pair<SDValue, SDValue> Tmp =
4762 TLI.makeLibCall(DAG, LC, NVT, Op, CallOptions, dl, Chain);
4763
4764 // Truncate the result if the libcall returns a larger type.
4765 Results.push_back(DAG.getNode(ISD::TRUNCATE, dl, RVT, Tmp.first));
4766 if (IsStrict)
4767 Results.push_back(Tmp.second);
4768 break;
4769 }
4770
4771 case ISD::FP_ROUND:
4772 case ISD::STRICT_FP_ROUND: {
4773 // X = FP_ROUND(Y, TRUNC)
4774 // TRUNC is a flag, which is always an integer that is zero or one.
4775 // If TRUNC is 0, this is a normal rounding, if it is 1, this FP_ROUND
4776 // is known to not change the value of Y.
4777 // We can only expand it into libcall if the TRUNC is 0.
4778 bool IsStrict = Node->isStrictFPOpcode();
4779 SDValue Op = Node->getOperand(IsStrict ? 1 : 0);
4780 SDValue Chain = IsStrict ? Node->getOperand(0) : SDValue();
4781 EVT VT = Node->getValueType(0);
4782 assert(cast<ConstantSDNode>(Node->getOperand(IsStrict ? 2 : 1))->isZero() &&
4783 "Unable to expand as libcall if it is not normal rounding");
4784
4785 RTLIB::Libcall LC = RTLIB::getFPROUND(Op.getValueType(), VT);
4786 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unable to legalize as libcall");
4787
4789 std::pair<SDValue, SDValue> Tmp =
4790 TLI.makeLibCall(DAG, LC, VT, Op, CallOptions, SDLoc(Node), Chain);
4791 Results.push_back(Tmp.first);
4792 if (IsStrict)
4793 Results.push_back(Tmp.second);
4794 break;
4795 }
4796 case ISD::FP_EXTEND: {
4797 Results.push_back(
4798 ExpandLibCall(RTLIB::getFPEXT(Node->getOperand(0).getValueType(),
4799 Node->getValueType(0)),
4800 Node, false).first);
4801 break;
4802 }
4806 RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
4807 if (Node->getOpcode() == ISD::STRICT_FP_TO_FP16)
4808 LC = RTLIB::getFPROUND(Node->getOperand(1).getValueType(), MVT::f16);
4809 else if (Node->getOpcode() == ISD::STRICT_FP_TO_BF16)
4810 LC = RTLIB::getFPROUND(Node->getOperand(1).getValueType(), MVT::bf16);
4811 else
4812 LC = RTLIB::getFPEXT(Node->getOperand(1).getValueType(),
4813 Node->getValueType(0));
4814
4815 assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unable to legalize as libcall");
4816
4818 std::pair<SDValue, SDValue> Tmp =
4819 TLI.makeLibCall(DAG, LC, Node->getValueType(0), Node->getOperand(1),
4820 CallOptions, SDLoc(Node), Node->getOperand(0));
4821 Results.push_back(Tmp.first);
4822 Results.push_back(Tmp.second);
4823 break;
4824 }
4825 case ISD::FSUB:
4826 case ISD::STRICT_FSUB:
4827 ExpandFPLibCall(Node, RTLIB::SUB_F32, RTLIB::SUB_F64,
4828 RTLIB::SUB_F80, RTLIB::SUB_F128,
4829 RTLIB::SUB_PPCF128, Results);
4830 break;
4831 case ISD::SREM:
4832 Results.push_back(ExpandIntLibCall(Node, true,
4833 RTLIB::SREM_I8,
4834 RTLIB::SREM_I16, RTLIB::SREM_I32,
4835 RTLIB::SREM_I64, RTLIB::SREM_I128));
4836 break;
4837 case ISD::UREM:
4838 Results.push_back(ExpandIntLibCall(Node, false,
4839 RTLIB::UREM_I8,
4840 RTLIB::UREM_I16, RTLIB::UREM_I32,
4841 RTLIB::UREM_I64, RTLIB::UREM_I128));
4842 break;
4843 case ISD::SDIV:
4844 Results.push_back(ExpandIntLibCall(Node, true,
4845 RTLIB::SDIV_I8,
4846 RTLIB::SDIV_I16, RTLIB::SDIV_I32,
4847 RTLIB::SDIV_I64, RTLIB::SDIV_I128));
4848 break;
4849 case ISD::UDIV:
4850 Results.push_back(ExpandIntLibCall(Node, false,
4851 RTLIB::UDIV_I8,
4852 RTLIB::UDIV_I16, RTLIB::UDIV_I32,
4853 RTLIB::UDIV_I64, RTLIB::UDIV_I128));
4854 break;
4855 case ISD::SDIVREM:
4856 case ISD::UDIVREM:
4857 // Expand into divrem libcall
4858 ExpandDivRemLibCall(Node, Results);
4859 break;
4860 case ISD::MUL:
4861 Results.push_back(ExpandIntLibCall(Node, false,
4862 RTLIB::MUL_I8,
4863 RTLIB::MUL_I16, RTLIB::MUL_I32,
4864 RTLIB::MUL_I64, RTLIB::MUL_I128));
4865 break;
4867 switch (Node->getSimpleValueType(0).SimpleTy) {
4868 default:
4869 llvm_unreachable("LibCall explicitly requested, but not available");
4870 case MVT::i32:
4871 Results.push_back(ExpandLibCall(RTLIB::CTLZ_I32, Node, false).first);
4872 break;
4873 case MVT::i64:
4874 Results.push_back(ExpandLibCall(RTLIB::CTLZ_I64, Node, false).first);
4875 break;
4876 case MVT::i128:
4877 Results.push_back(ExpandLibCall(RTLIB::CTLZ_I128, Node, false).first);
4878 break;
4879 }
4880 break;
4881 case ISD::RESET_FPENV: {
4882 // It is legalized to call 'fesetenv(FE_DFL_ENV)'. On most targets
4883 // FE_DFL_ENV is defined as '((const fenv_t *) -1)' in glibc.
4884 SDValue Ptr = DAG.getIntPtrConstant(-1LL, dl);
4885 SDValue Chain = Node->getOperand(0);
4886 Results.push_back(
4887 DAG.makeStateFunctionCall(RTLIB::FESETENV, Ptr, Chain, dl));
4888 break;
4889 }
4890 case ISD::GET_FPENV_MEM: {
4891 SDValue Chain = Node->getOperand(0);
4892 SDValue EnvPtr = Node->getOperand(1);
4893 Results.push_back(
4894 DAG.makeStateFunctionCall(RTLIB::FEGETENV, EnvPtr, Chain, dl));
4895 break;
4896 }
4897 case ISD::SET_FPENV_MEM: {
4898 SDValue Chain = Node->getOperand(0);
4899 SDValue EnvPtr = Node->getOperand(1);
4900 Results.push_back(
4901 DAG.makeStateFunctionCall(RTLIB::FESETENV, EnvPtr, Chain, dl));
4902 break;
4903 }
4904 case ISD::GET_FPMODE: {
4905 // Call fegetmode, which saves control modes into a stack slot. Then load
4906 // the value to return from the stack.
4907 EVT ModeVT = Node->getValueType(0);
4908 SDValue StackPtr = DAG.CreateStackTemporary(ModeVT);
4909 int SPFI = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex();
4910 SDValue Chain = DAG.makeStateFunctionCall(RTLIB::FEGETMODE, StackPtr,
4911 Node->getOperand(0), dl);
4912 SDValue LdInst = DAG.getLoad(
4913 ModeVT, dl, Chain, StackPtr,
4914 MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), SPFI));
4915 Results.push_back(LdInst);
4916 Results.push_back(LdInst.getValue(1));
4917 break;
4918 }
4919 case ISD::SET_FPMODE: {
4920 // Move control modes to stack slot and then call fesetmode with the pointer
4921 // to the slot as argument.
4922 SDValue Mode = Node->getOperand(1);
4923 EVT ModeVT = Mode.getValueType();
4924 SDValue StackPtr = DAG.CreateStackTemporary(ModeVT);
4925 int SPFI = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex();
4926 SDValue StInst = DAG.getStore(
4927 Node->getOperand(0), dl, Mode, StackPtr,
4928 MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), SPFI));
4929 Results.push_back(
4930 DAG.makeStateFunctionCall(RTLIB::FESETMODE, StackPtr, StInst, dl));
4931 break;
4932 }
4933 case ISD::RESET_FPMODE: {
4934 // It is legalized to a call 'fesetmode(FE_DFL_MODE)'. On most targets
4935 // FE_DFL_MODE is defined as '((const femode_t *) -1)' in glibc. If not, the
4936 // target must provide custom lowering.
4937 const DataLayout &DL = DAG.getDataLayout();
4938 EVT PtrTy = TLI.getPointerTy(DL);
4939 SDValue Mode = DAG.getConstant(-1LL, dl, PtrTy);
4940 Results.push_back(DAG.makeStateFunctionCall(RTLIB::FESETMODE, Mode,
4941 Node->getOperand(0), dl));
4942 break;
4943 }
4944 }
4945
4946 // Replace the original node with the legalized result.
4947 if (!Results.empty()) {
4948 LLVM_DEBUG(dbgs() << "Successfully converted node to libcall\n");
4949 ReplaceNode(Node, Results.data());
4950 } else
4951 LLVM_DEBUG(dbgs() << "Could not convert node to libcall\n");
4952}
4953
4954// Determine the vector type to use in place of an original scalar element when
4955// promoting equally sized vectors.
4957 MVT EltVT, MVT NewEltVT) {
4958 unsigned OldEltsPerNewElt = EltVT.getSizeInBits() / NewEltVT.getSizeInBits();
4959 MVT MidVT = OldEltsPerNewElt == 1
4960 ? NewEltVT
4961 : MVT::getVectorVT(NewEltVT, OldEltsPerNewElt);
4962 assert(TLI.isTypeLegal(MidVT) && "unexpected");
4963 return MidVT;
4964}
4965
4966void SelectionDAGLegalize::PromoteNode(SDNode *Node) {
4967 LLVM_DEBUG(dbgs() << "Trying to promote node\n");
4969 MVT OVT = Node->getSimpleValueType(0);
4970 if (Node->getOpcode() == ISD::UINT_TO_FP ||
4971 Node->getOpcode() == ISD::SINT_TO_FP ||
4972 Node->getOpcode() == ISD::SETCC ||
4973 Node->getOpcode() == ISD::EXTRACT_VECTOR_ELT ||
4974 Node->getOpcode() == ISD::INSERT_VECTOR_ELT) {
4975 OVT = Node->getOperand(0).getSimpleValueType();
4976 }
4977 if (Node->getOpcode() == ISD::STRICT_UINT_TO_FP ||
4978 Node->getOpcode() == ISD::STRICT_SINT_TO_FP ||
4979 Node->getOpcode() == ISD::STRICT_FSETCC ||
4980 Node->getOpcode() == ISD::STRICT_FSETCCS)
4981 OVT = Node->getOperand(1).getSimpleValueType();
4982 if (Node->getOpcode() == ISD::BR_CC ||
4983 Node->getOpcode() == ISD::SELECT_CC)
4984 OVT = Node->getOperand(2).getSimpleValueType();
4985 MVT NVT = TLI.getTypeToPromoteTo(Node->getOpcode(), OVT);
4986 SDLoc dl(Node);
4987 SDValue Tmp1, Tmp2, Tmp3, Tmp4;
4988 switch (Node->getOpcode()) {
4989 case ISD::CTTZ:
4991 case ISD::CTLZ:
4993 case ISD::CTPOP:
4994 // Zero extend the argument unless its cttz, then use any_extend.
4995 if (Node->getOpcode() == ISD::CTTZ ||
4996 Node->getOpcode() == ISD::CTTZ_ZERO_UNDEF)
4997 Tmp1 = DAG.getNode(ISD::ANY_EXTEND, dl, NVT, Node->getOperand(0));
4998 else
4999 Tmp1 = DAG.getNode(ISD::ZERO_EXTEND, dl, NVT, Node->getOperand(0));
5000
5001 if (Node->getOpcode() == ISD::CTTZ) {
5002 // The count is the same in the promoted type except if the original
5003 // value was zero. This can be handled by setting the bit just off
5004 // the top of the original type.
5005 auto TopBit = APInt::getOneBitSet(NVT.getSizeInBits(),
5006 OVT.getSizeInBits());
5007 Tmp1 = DAG.getNode(ISD::OR, dl, NVT, Tmp1,
5008 DAG.getConstant(TopBit, dl, NVT));
5009 }
5010 // Perform the larger operation. For CTPOP and CTTZ_ZERO_UNDEF, this is
5011 // already the correct result.
5012 Tmp1 = DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1);
5013 if (Node->getOpcode() == ISD::CTLZ ||
5014 Node->getOpcode() == ISD::CTLZ_ZERO_UNDEF) {
5015 // Tmp1 = Tmp1 - (sizeinbits(NVT) - sizeinbits(Old VT))
5016 Tmp1 = DAG.getNode(ISD::SUB, dl, NVT, Tmp1,
5017 DAG.getConstant(NVT.getSizeInBits() -
5018 OVT.getSizeInBits(), dl, NVT));
5019 }
5020 Results.push_back(DAG.getNode(ISD::TRUNCATE, dl, OVT, Tmp1));
5021 break;
5022 case ISD::BITREVERSE:
5023 case ISD::BSWAP: {
5024 unsigned DiffBits = NVT.getSizeInBits() - OVT.getSizeInBits();
5025 Tmp1 = DAG.getNode(ISD::ZERO_EXTEND, dl, NVT, Node->getOperand(0));
5026 Tmp1 = DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1);
5027 Tmp1 = DAG.getNode(
5028 ISD::SRL, dl, NVT, Tmp1,
5029 DAG.getConstant(DiffBits, dl,
5030 TLI.getShiftAmountTy(NVT, DAG.getDataLayout())));
5031
5032 Results.push_back(DAG.getNode(ISD::TRUNCATE, dl, OVT, Tmp1));
5033 break;
5034 }
5035 case ISD::FP_TO_UINT:
5037 case ISD::FP_TO_SINT:
5039 PromoteLegalFP_TO_INT(Node, dl, Results);
5040 break;
5043 Results.push_back(PromoteLegalFP_TO_INT_SAT(Node, dl));
5044 break;
5045 case ISD::UINT_TO_FP:
5047 case ISD::SINT_TO_FP:
5049 PromoteLegalINT_TO_FP(Node, dl, Results);
5050 break;
5051 case ISD::VAARG: {
5052 SDValue Chain = Node->getOperand(0); // Get the chain.
5053 SDValue Ptr = Node->getOperand(1); // Get the pointer.
5054
5055 unsigned TruncOp;
5056 if (OVT.isVector()) {
5057 TruncOp = ISD::BITCAST;
5058 } else {
5059 assert(OVT.isInteger()
5060 && "VAARG promotion is supported only for vectors or integer types");
5061 TruncOp = ISD::TRUNCATE;
5062 }
5063
5064 // Perform the larger operation, then convert back
5065 Tmp1 = DAG.getVAArg(NVT, dl, Chain, Ptr, Node->getOperand(2),
5066 Node->getConstantOperandVal(3));
5067 Chain = Tmp1.getValue(1);
5068
5069 Tmp2 = DAG.getNode(TruncOp, dl, OVT, Tmp1);
5070
5071 // Modified the chain result - switch anything that used the old chain to
5072 // use the new one.
5073 DAG.ReplaceAllUsesOfValueWith(SDValue(Node, 0), Tmp2);
5074 DAG.ReplaceAllUsesOfValueWith(SDValue(Node, 1), Chain);
5075 if (UpdatedNodes) {
5076 UpdatedNodes->insert(Tmp2.getNode());
5077 UpdatedNodes->insert(Chain.getNode());
5078 }
5079 ReplacedNode(Node);
5080 break;
5081 }
5082 case ISD::MUL:
5083 case ISD::SDIV:
5084 case ISD::SREM:
5085 case ISD::UDIV:
5086 case ISD::UREM:
5087 case ISD::SMIN:
5088 case ISD::SMAX:
5089 case ISD::UMIN:
5090 case ISD::UMAX:
5091 case ISD::AND:
5092 case ISD::OR:
5093 case ISD::XOR: {
5094 unsigned ExtOp, TruncOp;
5095 if (OVT.isVector()) {
5096 ExtOp = ISD::BITCAST;
5097 TruncOp = ISD::BITCAST;
5098 } else {
5099 assert(OVT.isInteger() && "Cannot promote logic operation");
5100
5101 switch (Node->getOpcode()) {
5102 default:
5103 ExtOp = ISD::ANY_EXTEND;
5104 break;
5105 case ISD::SDIV:
5106 case ISD::SREM:
5107 case ISD::SMIN:
5108 case ISD::SMAX:
5109 ExtOp = ISD::SIGN_EXTEND;
5110 break;
5111 case ISD::UDIV:
5112 case ISD::UREM:
5113 ExtOp = ISD::ZERO_EXTEND;
5114 break;
5115 case ISD::UMIN:
5116 case ISD::UMAX:
5117 if (TLI.isSExtCheaperThanZExt(OVT, NVT))
5118 ExtOp = ISD::SIGN_EXTEND;
5119 else
5120 ExtOp = ISD::ZERO_EXTEND;
5121 break;
5122 }
5123 TruncOp = ISD::TRUNCATE;
5124 }
5125 // Promote each of the values to the new type.
5126 Tmp1 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(0));
5127 Tmp2 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(1));
5128 // Perform the larger operation, then convert back
5129 Tmp1 = DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1, Tmp2);
5130 Results.push_back(DAG.getNode(TruncOp, dl, OVT, Tmp1));
5131 break;
5132 }
5133 case ISD::UMUL_LOHI:
5134 case ISD::SMUL_LOHI: {
5135 // Promote to a multiply in a wider integer type.
5136 unsigned ExtOp = Node->getOpcode() == ISD::UMUL_LOHI ? ISD::ZERO_EXTEND
5138 Tmp1 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(0));
5139 Tmp2 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(1));
5140 Tmp1 = DAG.getNode(ISD::MUL, dl, NVT, Tmp1, Tmp2);
5141
5142 auto &DL = DAG.getDataLayout();
5143 unsigned OriginalSize = OVT.getScalarSizeInBits();
5144 Tmp2 = DAG.getNode(
5145 ISD::SRL, dl, NVT, Tmp1,
5146 DAG.getConstant(OriginalSize, dl, TLI.getScalarShiftAmountTy(DL, NVT)));
5147 Results.push_back(DAG.getNode(ISD::TRUNCATE, dl, OVT, Tmp1));
5148 Results.push_back(DAG.getNode(ISD::TRUNCATE, dl, OVT, Tmp2));
5149 break;
5150 }
5151 case ISD::SELECT: {
5152 unsigned ExtOp, TruncOp;
5153 if (Node->getValueType(0).isVector() ||
5154 Node->getValueType(0).getSizeInBits() == NVT.getSizeInBits()) {
5155 ExtOp = ISD::BITCAST;
5156 TruncOp = ISD::BITCAST;
5157 } else if (Node->getValueType(0).isInteger()) {
5158 ExtOp = ISD::ANY_EXTEND;
5159 TruncOp = ISD::TRUNCATE;
5160 } else {
5161 ExtOp = ISD::FP_EXTEND;
5162 TruncOp = ISD::FP_ROUND;
5163 }
5164 Tmp1 = Node->getOperand(0);
5165 // Promote each of the values to the new type.
5166 Tmp2 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(1));
5167 Tmp3 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(2));
5168 // Perform the larger operation, then round down.
5169 Tmp1 = DAG.getSelect(dl, NVT, Tmp1, Tmp2, Tmp3);
5170 Tmp1->setFlags(Node->getFlags());
5171 if (TruncOp != ISD::FP_ROUND)
5172 Tmp1 = DAG.getNode(TruncOp, dl, Node->getValueType(0), Tmp1);
5173 else
5174 Tmp1 = DAG.getNode(TruncOp, dl, Node->getValueType(0), Tmp1,
5175 DAG.getIntPtrConstant(0, dl));
5176 Results.push_back(Tmp1);
5177 break;
5178 }
5179 case ISD::VECTOR_SHUFFLE: {
5180 ArrayRef<int> Mask = cast<ShuffleVectorSDNode>(Node)->getMask();
5181
5182 // Cast the two input vectors.
5183 Tmp1 = DAG.getNode(ISD::BITCAST, dl, NVT, Node->getOperand(0));
5184 Tmp2 = DAG.getNode(ISD::BITCAST, dl, NVT, Node->getOperand(1));
5185
5186 // Convert the shuffle mask to the right # elements.
5187 Tmp1 = ShuffleWithNarrowerEltType(NVT, OVT, dl, Tmp1, Tmp2, Mask);
5188 Tmp1 = DAG.getNode(ISD::BITCAST, dl, OVT, Tmp1);
5189 Results.push_back(Tmp1);
5190 break;
5191 }
5192 case ISD::VECTOR_SPLICE: {
5193 Tmp1 = DAG.getNode(ISD::ANY_EXTEND, dl, NVT, Node->getOperand(0));
5194 Tmp2 = DAG.getNode(ISD::ANY_EXTEND, dl, NVT, Node->getOperand(1));
5195 Tmp3 = DAG.getNode(ISD::VECTOR_SPLICE, dl, NVT, Tmp1, Tmp2,
5196 Node->getOperand(2));
5197 Results.push_back(DAG.getNode(ISD::TRUNCATE, dl, OVT, Tmp3));
5198 break;
5199 }
5200 case ISD::SELECT_CC: {
5201 SDValue Cond = Node->getOperand(4);
5202 ISD::CondCode CCCode = cast<CondCodeSDNode>(Cond)->get();
5203 // Type of the comparison operands.
5204 MVT CVT = Node->getSimpleValueType(0);
5205 assert(CVT == OVT && "not handled");
5206
5207 unsigned ExtOp = ISD::FP_EXTEND;
5208 if (NVT.isInteger()) {
5210 }
5211
5212 // Promote the comparison operands, if needed.
5213 if (TLI.isCondCodeLegal(CCCode, CVT)) {
5214 Tmp1 = Node->getOperand(0);
5215 Tmp2 = Node->getOperand(1);
5216 } else {
5217 Tmp1 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(0));
5218 Tmp2 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(1));
5219 }
5220 // Cast the true/false operands.
5221 Tmp3 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(2));
5222 Tmp4 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(3));
5223
5224 Tmp1 = DAG.getNode(ISD::SELECT_CC, dl, NVT, {Tmp1, Tmp2, Tmp3, Tmp4, Cond},
5225 Node->getFlags());
5226
5227 // Cast the result back to the original type.
5228 if (ExtOp != ISD::FP_EXTEND)
5229 Tmp1 = DAG.getNode(ISD::TRUNCATE, dl, OVT, Tmp1);
5230 else
5231 Tmp1 = DAG.getNode(ISD::FP_ROUND, dl, OVT, Tmp1,
5232 DAG.getIntPtrConstant(0, dl, /*isTarget=*/true));
5233
5234 Results.push_back(Tmp1);
5235 break;
5236 }
5237 case ISD::SETCC:
5238 case ISD::STRICT_FSETCC:
5239 case ISD::STRICT_FSETCCS: {
5240 unsigned ExtOp = ISD::FP_EXTEND;
5241 if (NVT.isInteger()) {
5242 ISD::CondCode CCCode = cast<CondCodeSDNode>(Node->getOperand(2))->get();
5243 if (isSignedIntSetCC(CCCode) ||
5244 TLI.isSExtCheaperThanZExt(Node->getOperand(0).getValueType(), NVT))
5245 ExtOp = ISD::SIGN_EXTEND;
5246 else
5247 ExtOp = ISD::ZERO_EXTEND;
5248 }
5249 if (Node->isStrictFPOpcode()) {
5250 SDValue InChain = Node->getOperand(0);
5251 std::tie(Tmp1, std::ignore) =
5252 DAG.getStrictFPExtendOrRound(Node->getOperand(1), InChain, dl, NVT);
5253 std::tie(Tmp2, std::ignore) =
5254 DAG.getStrictFPExtendOrRound(Node->getOperand(2), InChain, dl, NVT);
5255 SmallVector<SDValue, 2> TmpChains = {Tmp1.getValue(1), Tmp2.getValue(1)};
5256 SDValue OutChain = DAG.getTokenFactor(dl, TmpChains);
5257 SDVTList VTs = DAG.getVTList(Node->getValueType(0), MVT::Other);
5258 Results.push_back(DAG.getNode(Node->getOpcode(), dl, VTs,
5259 {OutChain, Tmp1, Tmp2, Node->getOperand(3)},
5260 Node->getFlags()));
5261 Results.push_back(Results.back().getValue(1));
5262 break;
5263 }
5264 Tmp1 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(0));
5265 Tmp2 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(1));
5266 Results.push_back(DAG.getNode(ISD::SETCC, dl, Node->getValueType(0), Tmp1,
5267 Tmp2, Node->getOperand(2), Node->getFlags()));
5268 break;
5269 }
5270 case ISD::BR_CC: {
5271 unsigned ExtOp = ISD::FP_EXTEND;
5272 if (NVT.isInteger()) {
5273 ISD::CondCode CCCode =
5274 cast<CondCodeSDNode>(Node->getOperand(1))->get();
5276 }
5277 Tmp1 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(2));
5278 Tmp2 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(3));
5279 Results.push_back(DAG.getNode(ISD::BR_CC, dl, Node->getValueType(0),
5280 Node->getOperand(0), Node->getOperand(1),
5281 Tmp1, Tmp2, Node->getOperand(4)));
5282 break;
5283 }
5284 case ISD::FADD:
5285 case ISD::FSUB:
5286 case ISD::FMUL:
5287 case ISD::FDIV:
5288 case ISD::FREM:
5289 case ISD::FMINNUM:
5290 case ISD::FMAXNUM:
5291 case ISD::FMINIMUM:
5292 case ISD::FMAXIMUM:
5293 case ISD::FPOW:
5294 Tmp1 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(0));
5295 Tmp2 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(1));
5296 Tmp3 = DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1, Tmp2,
5297 Node->getFlags());
5298 Results.push_back(
5299 DAG.getNode(ISD::FP_ROUND, dl, OVT, Tmp3,
5300 DAG.getIntPtrConstant(0, dl, /*isTarget=*/true)));
5301 break;
5302 case ISD::STRICT_FADD:
5303 case ISD::STRICT_FSUB:
5304 case ISD::STRICT_FMUL:
5305 case ISD::STRICT_FDIV:
5308 case ISD::STRICT_FREM:
5309 case ISD::STRICT_FPOW:
5310 Tmp1 = DAG.getNode(ISD::STRICT_FP_EXTEND, dl, {NVT, MVT::Other},
5311 {Node->getOperand(0), Node->getOperand(1)});
5312 Tmp2 = DAG.getNode(ISD::STRICT_FP_EXTEND, dl, {NVT, MVT::Other},
5313 {Node->getOperand(0), Node->getOperand(2)});
5314 Tmp3 = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Tmp1.getValue(1),
5315 Tmp2.getValue(1));
5316 Tmp1 = DAG.getNode(Node->getOpcode(), dl, {NVT, MVT::Other},
5317 {Tmp3, Tmp1, Tmp2});
5318 Tmp1 = DAG.getNode(ISD::STRICT_FP_ROUND, dl, {OVT, MVT::Other},
5319 {Tmp1.getValue(1), Tmp1, DAG.getIntPtrConstant(0, dl)});
5320 Results.push_back(Tmp1);
5321 Results.push_back(Tmp1.getValue(1));
5322 break;
5323 case ISD::FMA:
5324 Tmp1 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(0));
5325 Tmp2 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(1));
5326 Tmp3 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(2));
5327 Results.push_back(
5328 DAG.getNode(ISD::FP_ROUND, dl, OVT,
5329 DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1, Tmp2, Tmp3),
5330 DAG.getIntPtrConstant(0, dl, /*isTarget=*/true)));
5331 break;
5332 case ISD::STRICT_FMA:
5333 Tmp1 = DAG.getNode(ISD::STRICT_FP_EXTEND, dl, {NVT, MVT::Other},
5334 {Node->getOperand(0), Node->getOperand(1)});
5335 Tmp2 = DAG.getNode(ISD::STRICT_FP_EXTEND, dl, {NVT, MVT::Other},
5336 {Node->getOperand(0), Node->getOperand(2)});
5337 Tmp3 = DAG.getNode(ISD::STRICT_FP_EXTEND, dl, {NVT, MVT::Other},
5338 {Node->getOperand(0), Node->getOperand(3)});
5339 Tmp4 = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Tmp1.getValue(1),
5340 Tmp2.getValue(1), Tmp3.getValue(1));
5341 Tmp4 = DAG.getNode(Node->getOpcode(), dl, {NVT, MVT::Other},
5342 {Tmp4, Tmp1, Tmp2, Tmp3});
5343 Tmp4 = DAG.getNode(ISD::STRICT_FP_ROUND, dl, {OVT, MVT::Other},
5344 {Tmp4.getValue(1), Tmp4, DAG.getIntPtrConstant(0, dl)});
5345 Results.push_back(Tmp4);
5346 Results.push_back(Tmp4.getValue(1));
5347 break;
5348 case ISD::FCOPYSIGN:
5349 case ISD::FLDEXP:
5350 case ISD::FPOWI: {
5351 Tmp1 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(0));
5352 Tmp2 = Node->getOperand(1);
5353 Tmp3 = DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1, Tmp2);
5354
5355 // fcopysign doesn't change anything but the sign bit, so
5356 // (fp_round (fcopysign (fpext a), b))
5357 // is as precise as
5358 // (fp_round (fpext a))
5359 // which is a no-op. Mark it as a TRUNCating FP_ROUND.
5360 const bool isTrunc = (Node->getOpcode() == ISD::FCOPYSIGN);
5361 Results.push_back(
5362 DAG.getNode(ISD::FP_ROUND, dl, OVT, Tmp3,
5363 DAG.getIntPtrConstant(isTrunc, dl, /*isTarget=*/true)));
5364 break;
5365 }
5366 case ISD::STRICT_FPOWI:
5367 Tmp1 = DAG.getNode(ISD::STRICT_FP_EXTEND, dl, {NVT, MVT::Other},
5368 {Node->getOperand(0), Node->getOperand(1)});
5369 Tmp2 = DAG.getNode(Node->getOpcode(), dl, {NVT, MVT::Other},
5370 {Tmp1.getValue(1), Tmp1, Node->getOperand(2)});
5371 Tmp3 = DAG.getNode(ISD::STRICT_FP_ROUND, dl, {OVT, MVT::Other},
5372 {Tmp2.getValue(1), Tmp2, DAG.getIntPtrConstant(0, dl)});
5373 Results.push_back(Tmp3);
5374 Results.push_back(Tmp3.getValue(1));
5375 break;
5376 case ISD::FFREXP: {
5377 Tmp1 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(0));
5378 Tmp2 = DAG.getNode(ISD::FFREXP, dl, {NVT, Node->getValueType(1)}, Tmp1);
5379
5380 Results.push_back(
5381 DAG.getNode(ISD::FP_ROUND, dl, OVT, Tmp2,
5382 DAG.getIntPtrConstant(0, dl, /*isTarget=*/true)));
5383
5384 Results.push_back(Tmp2.getValue(1));
5385 break;
5386 }
5387 case ISD::FFLOOR:
5388 case ISD::FCEIL:
5389 case ISD::FRINT:
5390 case ISD::FNEARBYINT:
5391 case ISD::FROUND:
5392 case ISD::FROUNDEVEN:
5393 case ISD::FTRUNC:
5394 case ISD::FNEG:
5395 case ISD::FSQRT:
5396 case ISD::FSIN:
5397 case ISD::FCOS:
5398 case ISD::FLOG:
5399 case ISD::FLOG2:
5400 case ISD::FLOG10:
5401 case ISD::FABS:
5402 case ISD::FEXP:
5403 case ISD::FEXP2:
5404 case ISD::FEXP10:
5405 case ISD::FCANONICALIZE:
5406 Tmp1 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(0));
5407 Tmp2 = DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1);
5408 Results.push_back(
5409 DAG.getNode(ISD::FP_ROUND, dl, OVT, Tmp2,
5410 DAG.getIntPtrConstant(0, dl, /*isTarget=*/true)));
5411 break;
5412 case ISD::STRICT_FFLOOR:
5413 case ISD::STRICT_FCEIL:
5414 case ISD::STRICT_FRINT:
5416 case ISD::STRICT_FROUND:
5418 case ISD::STRICT_FTRUNC:
5419 case ISD::STRICT_FSQRT:
5420 case ISD::STRICT_FSIN:
5421 case ISD::STRICT_FCOS:
5422 case ISD::STRICT_FLOG:
5423 case ISD::STRICT_FLOG2:
5424 case ISD::STRICT_FLOG10:
5425 case ISD::STRICT_FEXP:
5426 case ISD::STRICT_FEXP2:
5427 Tmp1 = DAG.getNode(ISD::STRICT_FP_EXTEND, dl, {NVT, MVT::Other},
5428 {Node->getOperand(0), Node->getOperand(1)});
5429 Tmp2 = DAG.getNode(Node->getOpcode(), dl, {NVT, MVT::Other},
5430 {Tmp1.getValue(1), Tmp1});
5431 Tmp3 = DAG.getNode(ISD::STRICT_FP_ROUND, dl, {OVT, MVT::Other},
5432 {Tmp2.getValue(1), Tmp2, DAG.getIntPtrConstant(0, dl)});
5433 Results.push_back(Tmp3);
5434 Results.push_back(Tmp3.getValue(1));
5435 break;
5436 case ISD::BUILD_VECTOR: {
5437 MVT EltVT = OVT.getVectorElementType();
5438 MVT NewEltVT = NVT.getVectorElementType();
5439
5440 // Handle bitcasts to a different vector type with the same total bit size
5441 //
5442 // e.g. v2i64 = build_vector i64:x, i64:y => v4i32
5443 // =>
5444 // v4i32 = concat_vectors (v2i32 (bitcast i64:x)), (v2i32 (bitcast i64:y))
5445
5446 assert(NVT.isVector() && OVT.getSizeInBits() == NVT.getSizeInBits() &&
5447 "Invalid promote type for build_vector");
5448 assert(NewEltVT.bitsLE(EltVT) && "not handled");
5449
5450 MVT MidVT = getPromotedVectorElementType(TLI, EltVT, NewEltVT);
5451
5453 for (unsigned I = 0, E = Node->getNumOperands(); I != E; ++I) {
5454 SDValue Op = Node->getOperand(I);
5455 NewOps.push_back(DAG.getNode(ISD::BITCAST, SDLoc(Op), MidVT, Op));
5456 }
5457
5458 SDLoc SL(Node);
5459 SDValue Concat =
5460 DAG.getNode(MidVT == NewEltVT ? ISD::BUILD_VECTOR : ISD::CONCAT_VECTORS,
5461 SL, NVT, NewOps);
5462 SDValue CvtVec = DAG.getNode(ISD::BITCAST, SL, OVT, Concat);
5463 Results.push_back(CvtVec);
5464 break;
5465 }
5467 MVT EltVT = OVT.getVectorElementType();
5468 MVT NewEltVT = NVT.getVectorElementType();
5469
5470 // Handle bitcasts to a different vector type with the same total bit size.
5471 //
5472 // e.g. v2i64 = extract_vector_elt x:v2i64, y:i32
5473 // =>
5474 // v4i32:castx = bitcast x:v2i64
5475 //
5476 // i64 = bitcast
5477 // (v2i32 build_vector (i32 (extract_vector_elt castx, (2 * y))),
5478 // (i32 (extract_vector_elt castx, (2 * y + 1)))
5479 //
5480
5481 assert(NVT.isVector() && OVT.getSizeInBits() == NVT.getSizeInBits() &&
5482 "Invalid promote type for extract_vector_elt");
5483 assert(NewEltVT.bitsLT(EltVT) && "not handled");
5484
5485 MVT MidVT = getPromotedVectorElementType(TLI, EltVT, NewEltVT);
5486 unsigned NewEltsPerOldElt = MidVT.getVectorNumElements();
5487
5488 SDValue Idx = Node->getOperand(1);
5489 EVT IdxVT = Idx.getValueType();
5490 SDLoc SL(Node);
5491 SDValue Factor = DAG.getConstant(NewEltsPerOldElt, SL, IdxVT);
5492 SDValue NewBaseIdx = DAG.getNode(ISD::MUL, SL, IdxVT, Idx, Factor);
5493
5494 SDValue CastVec = DAG.getNode(ISD::BITCAST, SL, NVT, Node->getOperand(0));
5495
5497 for (unsigned I = 0; I < NewEltsPerOldElt; ++I) {
5498 SDValue IdxOffset = DAG.getConstant(I, SL, IdxVT);
5499 SDValue TmpIdx = DAG.getNode(ISD::ADD, SL, IdxVT, NewBaseIdx, IdxOffset);
5500
5501 SDValue Elt = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, NewEltVT,
5502 CastVec, TmpIdx);
5503 NewOps.push_back(Elt);
5504 }
5505
5506 SDValue NewVec = DAG.getBuildVector(MidVT, SL, NewOps);
5507 Results.push_back(DAG.getNode(ISD::BITCAST, SL, EltVT, NewVec));
5508 break;
5509 }
5511 MVT EltVT = OVT.getVectorElementType();
5512 MVT NewEltVT = NVT.getVectorElementType();
5513
5514 // Handle bitcasts to a different vector type with the same total bit size
5515 //
5516 // e.g. v2i64 = insert_vector_elt x:v2i64, y:i64, z:i32
5517 // =>
5518 // v4i32:castx = bitcast x:v2i64
5519 // v2i32:casty = bitcast y:i64
5520 //
5521 // v2i64 = bitcast
5522 // (v4i32 insert_vector_elt
5523 // (v4i32 insert_vector_elt v4i32:castx,
5524 // (extract_vector_elt casty, 0), 2 * z),
5525 // (extract_vector_elt casty, 1), (2 * z + 1))
5526
5527 assert(NVT.isVector() && OVT.getSizeInBits() == NVT.getSizeInBits() &&
5528 "Invalid promote type for insert_vector_elt");
5529 assert(NewEltVT.bitsLT(EltVT) && "not handled");
5530
5531 MVT MidVT = getPromotedVectorElementType(TLI, EltVT, NewEltVT);
5532 unsigned NewEltsPerOldElt = MidVT.getVectorNumElements();
5533
5534 SDValue Val = Node->getOperand(1);
5535 SDValue Idx = Node->getOperand(2);
5536 EVT IdxVT = Idx.getValueType();
5537 SDLoc SL(Node);
5538
5539 SDValue Factor = DAG.getConstant(NewEltsPerOldElt, SDLoc(), IdxVT);
5540 SDValue NewBaseIdx = DAG.getNode(ISD::MUL, SL, IdxVT, Idx, Factor);
5541
5542 SDValue CastVec = DAG.getNode(ISD::BITCAST, SL, NVT, Node->getOperand(0));
5543 SDValue CastVal = DAG.getNode(ISD::BITCAST, SL, MidVT, Val);
5544
5545 SDValue NewVec = CastVec;
5546 for (unsigned I = 0; I < NewEltsPerOldElt; ++I) {
5547 SDValue IdxOffset = DAG.getConstant(I, SL, IdxVT);
5548 SDValue InEltIdx = DAG.getNode(ISD::ADD, SL, IdxVT, NewBaseIdx, IdxOffset);
5549
5550 SDValue Elt = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, NewEltVT,
5551 CastVal, IdxOffset);
5552
5553 NewVec = DAG.getNode(ISD::INSERT_VECTOR_ELT, SL, NVT,
5554 NewVec, Elt, InEltIdx);
5555 }
5556
5557 Results.push_back(DAG.getNode(ISD::BITCAST, SL, OVT, NewVec));
5558 break;
5559 }
5560 case ISD::SCALAR_TO_VECTOR: {
5561 MVT EltVT = OVT.getVectorElementType();
5562 MVT NewEltVT = NVT.getVectorElementType();
5563
5564 // Handle bitcasts to different vector type with the same total bit size.
5565 //
5566 // e.g. v2i64 = scalar_to_vector x:i64
5567 // =>
5568 // concat_vectors (v2i32 bitcast x:i64), (v2i32 undef)
5569 //
5570
5571 MVT MidVT = getPromotedVectorElementType(TLI, EltVT, NewEltVT);
5572 SDValue Val = Node->getOperand(0);
5573 SDLoc SL(Node);
5574
5575 SDValue CastVal = DAG.getNode(ISD::BITCAST, SL, MidVT, Val);
5576 SDValue Undef = DAG.getUNDEF(MidVT);
5577
5579 NewElts.push_back(CastVal);
5580 for (unsigned I = 1, NElts = OVT.getVectorNumElements(); I != NElts; ++I)
5581 NewElts.push_back(Undef);
5582
5583 SDValue Concat = DAG.getNode(ISD::CONCAT_VECTORS, SL, NVT, NewElts);
5584 SDValue CvtVec = DAG.getNode(ISD::BITCAST, SL, OVT, Concat);
5585 Results.push_back(CvtVec);
5586 break;
5587 }
5588 case ISD::ATOMIC_SWAP: {
5589 AtomicSDNode *AM = cast<AtomicSDNode>(Node);
5590 SDLoc SL(Node);
5591 SDValue CastVal = DAG.getNode(ISD::BITCAST, SL, NVT, AM->getVal());
5592 assert(NVT.getSizeInBits() == OVT.getSizeInBits() &&
5593 "unexpected promotion type");
5594 assert(AM->getMemoryVT().getSizeInBits() == NVT.getSizeInBits() &&
5595 "unexpected atomic_swap with illegal type");
5596
5597 SDValue NewAtomic
5598 = DAG.getAtomic(ISD::ATOMIC_SWAP, SL, NVT,
5599 DAG.getVTList(NVT, MVT::Other),
5600 { AM->getChain(), AM->getBasePtr(), CastVal },
5601 AM->getMemOperand());
5602 Results.push_back(DAG.getNode(ISD::BITCAST, SL, OVT, NewAtomic));
5603 Results.push_back(NewAtomic.getValue(1));
5604 break;
5605 }
5606 case ISD::SPLAT_VECTOR: {
5607 SDValue Scalar = Node->getOperand(0);
5608 MVT ScalarType = Scalar.getSimpleValueType();
5609 MVT NewScalarType = NVT.getVectorElementType();
5610 if (ScalarType.isInteger()) {
5611 Tmp1 = DAG.getNode(ISD::ANY_EXTEND, dl, NewScalarType, Scalar);
5612 Tmp2 = DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1);
5613 Results.push_back(DAG.getNode(ISD::TRUNCATE, dl, OVT, Tmp2));
5614 break;
5615 }
5616 Tmp1 = DAG.getNode(ISD::FP_EXTEND, dl, NewScalarType, Scalar);
5617 Tmp2 = DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1);
5618 Results.push_back(
5619 DAG.getNode(ISD::FP_ROUND, dl, OVT, Tmp2,
5620 DAG.getIntPtrConstant(0, dl, /*isTarget=*/true)));
5621 break;
5622 }
5623 }
5624
5625 // Replace the original node with the legalized result.
5626 if (!Results.empty()) {
5627 LLVM_DEBUG(dbgs() << "Successfully promoted node\n");
5628 ReplaceNode(Node, Results.data());
5629 } else
5630 LLVM_DEBUG(dbgs() << "Could not promote node\n");
5631}
5632
5633/// This is the entry point for the file.
5636
5637 SmallPtrSet<SDNode *, 16> LegalizedNodes;
5638 // Use a delete listener to remove nodes which were deleted during
5639 // legalization from LegalizeNodes. This is needed to handle the situation
5640 // where a new node is allocated by the object pool to the same address of a
5641 // previously deleted node.
5642 DAGNodeDeletedListener DeleteListener(
5643 *this,
5644 [&LegalizedNodes](SDNode *N, SDNode *E) { LegalizedNodes.erase(N); });
5645
5646 SelectionDAGLegalize Legalizer(*this, LegalizedNodes);
5647
5648 // Visit all the nodes. We start in topological order, so that we see
5649 // nodes with their original operands intact. Legalization can produce
5650 // new nodes which may themselves need to be legalized. Iterate until all
5651 // nodes have been legalized.
5652 while (true) {
5653 bool AnyLegalized = false;
5654 for (auto NI = allnodes_end(); NI != allnodes_begin();) {
5655 --NI;
5656
5657 SDNode *N = &*NI;
5658 if (N->use_empty() && N != getRoot().getNode()) {
5659 ++NI;
5660 DeleteNode(N);
5661 continue;
5662 }
5663
5664 if (LegalizedNodes.insert(N).second) {
5665 AnyLegalized = true;
5666 Legalizer.LegalizeOp(N);
5667
5668 if (N->use_empty() && N != getRoot().getNode()) {
5669 ++NI;
5670 DeleteNode(N);
5671 }
5672 }
5673 }
5674 if (!AnyLegalized)
5675 break;
5676
5677 }
5678
5679 // Remove dead nodes now.
5681}
5682
5684 SmallSetVector<SDNode *, 16> &UpdatedNodes) {
5685 SmallPtrSet<SDNode *, 16> LegalizedNodes;
5686 SelectionDAGLegalize Legalizer(*this, LegalizedNodes, &UpdatedNodes);
5687
5688 // Directly insert the node in question, and legalize it. This will recurse
5689 // as needed through operands.
5690 LegalizedNodes.insert(N);
5691 Legalizer.LegalizeOp(N);
5692
5693 return LegalizedNodes.count(N);
5694}
#define Success
aarch64 falkor hwpf fix Falkor HW Prefetch Fix Late Phase
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
static bool isConstant(const MachineInstr &MI)
This file declares a class to represent arbitrary precision floating point values and provide a varie...
This file implements a class to represent arbitrary precision integral constant values and operations...
Function Alias Analysis Results
#define LLVM_FALLTHROUGH
LLVM_FALLTHROUGH - Mark fallthrough cases in switch statements.
Definition: Compiler.h:301
This file contains the declarations for the subclasses of Constant, which represent the different fla...
return RetTy
Returns the sub type a function will return at a given Idx Should correspond to the result type of an ExtractValue instruction executed with just that one unsigned Idx
#define LLVM_DEBUG(X)
Definition: Debug.h:101
uint64_t Addr
uint64_t Size
static GCMetadataPrinterRegistry::Add< ErlangGCPrinter > X("erlang", "erlang-compatible garbage collector")
static bool isSigned(unsigned int Opcode)
Utilities for dealing with flags related to floating point properties and mode controls.
static MaybeAlign getAlign(Value *Ptr)
Definition: IRBuilder.cpp:530
static bool ExpandBVWithShuffles(SDNode *Node, SelectionDAG &DAG, const TargetLowering &TLI, SDValue &Res)
static bool isSinCosLibcallAvailable(SDNode *Node, const TargetLowering &TLI)
Return true if sincos libcall is available.
static bool useSinCos(SDNode *Node)
Only issue sincos libcall if both sin and cos are needed.
static MachineMemOperand * getStackAlignedMMO(SDValue StackPtr, MachineFunction &MF, bool isObjectScalable)
static MVT getPromotedVectorElementType(const TargetLowering &TLI, MVT EltVT, MVT NewEltVT)
#define F(x, y, z)
Definition: MD5.cpp:55
#define I(x, y, z)
Definition: MD5.cpp:58
std::pair< MCSymbol *, MachineModuleInfoImpl::StubValueTy > PairTy
This file contains the declarations for metadata subclasses.
PowerPC Reduce CR logical Operation
const char LLVMTargetMachineRef TM
const SmallVectorImpl< MachineOperand > & Cond
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
This file implements a set that has insertion order iteration characteristics.
This file defines the SmallPtrSet class.
This file defines the SmallSet class.
This file defines the SmallVector class.
This file describes how to lower LLVM code to machine code.
static constexpr int Concat[]
Value * RHS
Value * LHS
support::ulittle16_t & Lo
Definition: aarch32.cpp:206
support::ulittle16_t & Hi
Definition: aarch32.cpp:205
DEMANGLE_DUMP_METHOD void dump() const
bool isSignaling() const
Definition: APFloat.h:1297
static APFloat getSmallestNormalized(const fltSemantics &Sem, bool Negative=false)
Returns the smallest (by magnitude) normalized finite number in the given semantics.
Definition: APFloat.h:1026
APInt bitcastToAPInt() const
Definition: APFloat.h:1210
static APFloat getInf(const fltSemantics &Sem, bool Negative=false)
Factory for Positive and Negative Infinity.
Definition: APFloat.h:966
Class for arbitrary precision integers.
Definition: APInt.h:76
static APInt getSignMask(unsigned BitWidth)
Get the SignMask for a specific bit width.
Definition: APInt.h:207
void setBit(unsigned BitPosition)
Set the given bit to 1 whose position is given as "bitPosition".
Definition: APInt.h:1308
static APInt getBitsSet(unsigned numBits, unsigned loBit, unsigned hiBit)
Get a value with a block of bits set.
Definition: APInt.h:236
static APInt getSignedMaxValue(unsigned numBits)
Gets maximum signed value of APInt for a specific bit width.
Definition: APInt.h:187
static APInt getOneBitSet(unsigned numBits, unsigned BitNo)
Return an APInt with exactly one bit set in the result.
Definition: APInt.h:217
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
Definition: ArrayRef.h:41
This is an SDNode representing atomic operations.
const SDValue & getVal() const
static bool isValueValidForType(EVT VT, const APFloat &Val)
const APFloat & getValueAPF() const
const ConstantFP * getConstantFPValue() const
ConstantFP - Floating Point Values [float, double].
Definition: Constants.h:268
const APFloat & getValueAPF() const
Definition: Constants.h:311
This is the shared class of boolean and integer constants.
Definition: Constants.h:80
uint64_t getZExtValue() const
Return the constant as a 64-bit unsigned integer value after it has been zero extended as appropriate...
Definition: Constants.h:154
static Constant * get(ArrayRef< Constant * > V)
Definition: Constants.cpp:1398
This is an important base class in LLVM.
Definition: Constant.h:41
This class represents an Operation in the Expression.
A parsed version of the target data layout string in and methods for querying it.
Definition: DataLayout.h:110
bool isBigEndian() const
Definition: DataLayout.h:239
bool empty() const
Definition: Function.h:804
const BasicBlock & back() const
Definition: Function.h:807
This class is used to form a handle around another node that is persistent and is updated across invo...
This class is used to represent ISD::LOAD nodes.
static LocationSize precise(uint64_t Value)
static constexpr LocationSize beforeOrAfterPointer()
Any location before or after the base pointer (but still within the underlying object).
Machine Value Type.
SimpleValueType SimpleTy
uint64_t getScalarSizeInBits() const
bool bitsLE(MVT VT) const
Return true if this has no more bits than VT.
unsigned getVectorNumElements() const
bool isVector() const
Return true if this is a vector value type.
bool isInteger() const
Return true if this is an integer or a vector integer type.
bool bitsLT(MVT VT) const
Return true if this has less bits than VT.
TypeSize getSizeInBits() const
Returns the size of the specified MVT in bits.
static MVT getVectorVT(MVT VT, unsigned NumElements)
MVT getVectorElementType() const
MachineMemOperand * getMachineMemOperand(MachinePointerInfo PtrInfo, MachineMemOperand::Flags f, LLT MemTy, Align base_alignment, const AAMDNodes &AAInfo=AAMDNodes(), const MDNode *Ranges=nullptr, SyncScope::ID SSID=SyncScope::System, AtomicOrdering Ordering=AtomicOrdering::NotAtomic, AtomicOrdering FailureOrdering=AtomicOrdering::NotAtomic)
getMachineMemOperand - Allocate a new MachineMemOperand.
MachineFrameInfo & getFrameInfo()
getFrameInfo - Return the frame info object for the current function.
A description of a memory reference used in the backend.
Flags
Flags values. These may be or'd together.
@ MOStore
The memory access writes data.
MachineMemOperand * getMemOperand() const
Return a MachineMemOperand object describing the memory reference performed by operation.
EVT getMemoryVT() const
Return the type of the in-memory value.
Wrapper class representing virtual and physical registers.
Definition: Register.h:19
Wrapper class for IR location info (IR ordering and DebugLoc) to be passed into SDNode creation funct...
Represents one node in the SelectionDAG.
void dump() const
Dump this node, for debugging.
iterator_range< use_iterator > uses()
static bool hasPredecessorHelper(const SDNode *N, SmallPtrSetImpl< const SDNode * > &Visited, SmallVectorImpl< const SDNode * > &Worklist, unsigned int MaxSteps=0, bool TopologicalPrune=false)
Returns true if N is a predecessor of any node in Worklist.
unsigned getNumValues() const
Return the number of values defined/returned by this operator.
EVT getValueType(unsigned ResNo) const
Return the type of a specified result.
void setFlags(SDNodeFlags NewFlags)
op_iterator op_end() const
op_iterator op_begin() const
Unlike LLVM values, Selection DAG nodes may return multiple values as the result of a computation.
bool isUndef() const
SDNode * getNode() const
get the SDNode which holds the desired result
SDValue getValue(unsigned R) const
EVT getValueType() const
Return the ValueType of the referenced return value.
const SDValue & getOperand(unsigned i) const
uint64_t getScalarValueSizeInBits() const
unsigned getResNo() const
get the index which selects a specific result in the SDNode
MVT getSimpleValueType() const
Return the simple ValueType of the referenced return value.
unsigned getOpcode() const
This is used to represent a portion of an LLVM function in a low-level Data Dependence DAG representa...
Definition: SelectionDAG.h:225
const SDValue & getRoot() const
Return the root tag of the SelectionDAG.
Definition: SelectionDAG.h:551
const TargetLowering & getTargetLoweringInfo() const
Definition: SelectionDAG.h:478
allnodes_const_iterator allnodes_begin() const
Definition: SelectionDAG.h:531
SDValue getUNDEF(EVT VT)
Return an UNDEF node. UNDEF does not have a useful SDLoc.
allnodes_const_iterator allnodes_end() const
Definition: SelectionDAG.h:532
void DeleteNode(SDNode *N)
Remove the specified node from the system.
const DataLayout & getDataLayout() const
Definition: SelectionDAG.h:472
void Legalize()
This transforms the SelectionDAG into a SelectionDAG that is compatible with the target instruction s...
bool LegalizeOp(SDNode *N, SmallSetVector< SDNode *, 16 > &UpdatedNodes)
Transforms a SelectionDAG node and any operands to it into a node that is compatible with the target ...
void ReplaceAllUsesWith(SDValue From, SDValue To)
Modify anything using 'From' to use 'To' instead.
void RemoveDeadNodes()
This method deletes all unreachable nodes in the SelectionDAG.
static const fltSemantics & EVTToAPFloatSemantics(EVT VT)
Returns an APFloat semantics tag appropriate for the given type.
const TargetMachine & getTarget() const
Definition: SelectionDAG.h:473
SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT, ArrayRef< SDUse > Ops)
Gets or creates the specified node.
unsigned AssignTopologicalOrder()
Topological-sort the AllNodes list and a assign a unique node id for each node in the DAG based on th...
void ReplaceAllUsesOfValueWith(SDValue From, SDValue To)
Replace any uses of From with To, leaving uses of other values produced by From.getNode() alone.
LLVMContext * getContext() const
Definition: SelectionDAG.h:485
SDValue getVectorShuffle(EVT VT, const SDLoc &dl, SDValue N1, SDValue N2, ArrayRef< int > Mask)
Return an ISD::VECTOR_SHUFFLE node.
bool insert(const value_type &X)
Insert a new element into the SetVector.
Definition: SetVector.h:162
A templated base class for SmallPtrSet which provides the typesafe interface that is common across al...
Definition: SmallPtrSet.h:321
bool erase(PtrType Ptr)
erase - If the set contains the specified pointer, remove it and return true, otherwise return false.
Definition: SmallPtrSet.h:356
size_type count(ConstPtrType Ptr) const
count - Return 1 if the specified pointer is in the set, 0 otherwise.
Definition: SmallPtrSet.h:360
std::pair< iterator, bool > insert(PtrType Ptr)
Inserts Ptr if and only if there is no element in the container equal to Ptr.
Definition: SmallPtrSet.h:342
SmallPtrSet - This class implements a set which is optimized for holding SmallSize or less elements.
Definition: SmallPtrSet.h:427
A SetVector that performs no allocations if smaller than a certain size.
Definition: SetVector.h:370
SmallSet - This maintains a set of unique values, optimizing for the case when the set is small (less...
Definition: SmallSet.h:135
std::pair< const_iterator, bool > insert(const T &V)
insert - Insert an element into the set if it isn't already there.
Definition: SmallSet.h:179
size_type size() const
Definition: SmallSet.h:161
bool empty() const
Definition: SmallVector.h:94
size_t size() const
Definition: SmallVector.h:91
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
Definition: SmallVector.h:586
void reserve(size_type N)
Definition: SmallVector.h:676
void append(ItTy in_start, ItTy in_end)
Add the specified range to the end of the SmallVector.
Definition: SmallVector.h:696
void swap(SmallVectorImpl &RHS)
Definition: SmallVector.h:981
void push_back(const T &Elt)
Definition: SmallVector.h:426
pointer data()
Return a pointer to the vector's buffer, even if empty().
Definition: SmallVector.h:299
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
Definition: SmallVector.h:1209
This class is used to represent ISD::STORE nodes.
Information about stack frame layout on the target.
Align getStackAlign() const
getStackAlignment - This method returns the number of bytes to which the stack pointer must be aligne...
StackDirection getStackGrowthDirection() const
getStackGrowthDirection - Return the direction the stack grows
virtual bool isShuffleMaskLegal(ArrayRef< int >, EVT) const
Targets can use this to indicate that they only support some VECTOR_SHUFFLE operations,...
LegalizeAction
This enum indicates whether operations are valid for a target, and if not, what action should be used...
virtual EVT getSetCCResultType(const DataLayout &DL, LLVMContext &Context, EVT VT) const
Return the ValueType of the result of SETCC operations.
bool isTypeLegal(EVT VT) const
Return true if the target has native support for the specified value type.
const char * getLibcallName(RTLIB::Libcall Call) const
Get the libcall routine name for the specified libcall.
std::vector< ArgListEntry > ArgListTy
This class defines information used to lower LLVM code to legal SelectionDAG operators that the targe...
Primary interface to the complete machine description for the target machine.
Definition: TargetMachine.h:76
static constexpr TypeSize getFixed(ScalarTy ExactSize)
Definition: TypeSize.h:330
The instances of the Type class are immutable: once they are created, they are never changed.
Definition: Type.h:45
static Type * getVoidTy(LLVMContext &C)
static IntegerType * getInt64Ty(LLVMContext &C)
static UndefValue * get(Type *T)
Static factory methods - Return an 'undef' object of the specified type.
Definition: Constants.cpp:1808
LLVM Value Representation.
Definition: Value.h:74
constexpr ScalarTy getFixedValue() const
Definition: TypeSize.h:187
constexpr ScalarTy getKnownMinValue() const
Returns the minimum value this quantity can represent.
Definition: TypeSize.h:168
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
constexpr char Args[]
Key for Kernel::Metadata::mArgs.
constexpr std::underlying_type_t< E > Mask()
Get a bitmask with 1s in all places up to the high-order bit of E's largest value.
Definition: BitmaskEnum.h:121
@ Fast
Attempts to make calls as fast as possible (e.g.
Definition: CallingConv.h:41
@ C
The default llvm calling convention, compatible with C.
Definition: CallingConv.h:34
@ SETCC
SetCC operator - This evaluates to a true value iff the condition is true.
Definition: ISDOpcodes.h:750
@ MERGE_VALUES
MERGE_VALUES - This node takes multiple discrete operands and returns them all as its individual resu...
Definition: ISDOpcodes.h:236
@ STACKRESTORE
STACKRESTORE has two operands, an input chain and a pointer to restore to it returns an output chain.
Definition: ISDOpcodes.h:1126
@ STACKSAVE
STACKSAVE - STACKSAVE has one operand, an input chain.
Definition: ISDOpcodes.h:1122
@ CTLZ_ZERO_UNDEF
Definition: ISDOpcodes.h:723
@ STRICT_FSETCC
STRICT_FSETCC/STRICT_FSETCCS - Constrained versions of SETCC, used for floating-point operands only.
Definition: ISDOpcodes.h:476
@ SET_FPENV
Sets the current floating-point environment.
Definition: ISDOpcodes.h:998
@ VECREDUCE_SEQ_FADD
Generic reduction nodes.
Definition: ISDOpcodes.h:1339
@ VECREDUCE_SMIN
Definition: ISDOpcodes.h:1370
@ EH_SJLJ_LONGJMP
OUTCHAIN = EH_SJLJ_LONGJMP(INCHAIN, buffer) This corresponds to the eh.sjlj.longjmp intrinsic.
Definition: ISDOpcodes.h:147
@ SMUL_LOHI
SMUL_LOHI/UMUL_LOHI - Multiply two integers of type iN, producing a signed/unsigned value of type i[2...
Definition: ISDOpcodes.h:250
@ ATOMIC_LOAD_NAND
Definition: ISDOpcodes.h:1269
@ INSERT_SUBVECTOR
INSERT_SUBVECTOR(VECTOR1, VECTOR2, IDX) - Returns a vector with VECTOR2 inserted into VECTOR1.
Definition: ISDOpcodes.h:559
@ BSWAP
Byte Swap and Counting operators.
Definition: ISDOpcodes.h:714
@ SMULFIX
RESULT = [US]MULFIX(LHS, RHS, SCALE) - Perform fixed point multiplication on 2 integers with the same...
Definition: ISDOpcodes.h:367
@ VAEND
VAEND, VASTART - VAEND and VASTART have three operands: an input chain, pointer, and a SRCVALUE.
Definition: ISDOpcodes.h:1155
@ ConstantFP
Definition: ISDOpcodes.h:77
@ ATOMIC_LOAD_MAX
Definition: ISDOpcodes.h:1271
@ ATOMIC_STORE
OUTCHAIN = ATOMIC_STORE(INCHAIN, ptr, val) This corresponds to "store atomic" instruction.
Definition: ISDOpcodes.h:1241
@ STRICT_FCEIL
Definition: ISDOpcodes.h:426
@ ATOMIC_LOAD_UMIN
Definition: ISDOpcodes.h:1272
@ FRAME_TO_ARGS_OFFSET
FRAME_TO_ARGS_OFFSET - This node represents offset from frame pointer to first (possible) on-stack ar...
Definition: ISDOpcodes.h:124
@ RESET_FPENV
Set floating-point environment to default state.
Definition: ISDOpcodes.h:1002
@ FMAD
FMAD - Perform a * b + c, while getting the same result as the separately rounded operations.
Definition: ISDOpcodes.h:487
@ ADD
Simple integer binary arithmetic operators.
Definition: ISDOpcodes.h:239
@ LOAD
LOAD and STORE have token chains as their first operand, then the same operands as an LLVM load/store...
Definition: ISDOpcodes.h:1031
@ SMULFIXSAT
Same as the corresponding unsaturated fixed point instructions, but the result is clamped between the...
Definition: ISDOpcodes.h:373
@ SET_FPMODE
Sets the current dynamic floating-point control modes.
Definition: ISDOpcodes.h:1021
@ ANY_EXTEND
ANY_EXTEND - Used for integer types. The high bits are undefined.
Definition: ISDOpcodes.h:783
@ FMA
FMA - Perform a * b + c with no intermediate rounding step.
Definition: ISDOpcodes.h:483
@ INTRINSIC_VOID
OUTCHAIN = INTRINSIC_VOID(INCHAIN, INTRINSICID, arg1, arg2, ...) This node represents a target intrin...
Definition: ISDOpcodes.h:199
@ RETURNADDR
Definition: ISDOpcodes.h:95
@ EH_SJLJ_SETUP_DISPATCH
OUTCHAIN = EH_SJLJ_SETUP_DISPATCH(INCHAIN) The target initializes the dispatch table here.
Definition: ISDOpcodes.h:151
@ GlobalAddress
Definition: ISDOpcodes.h:78
@ ATOMIC_CMP_SWAP_WITH_SUCCESS
Val, Success, OUTCHAIN = ATOMIC_CMP_SWAP_WITH_SUCCESS(INCHAIN, ptr, cmp, swap) N.b.
Definition: ISDOpcodes.h:1254
@ SINT_TO_FP
[SU]INT_TO_FP - These operators convert integers (whose interpreted sign depends on the first letter)...
Definition: ISDOpcodes.h:790
@ CONCAT_VECTORS
CONCAT_VECTORS(VECTOR0, VECTOR1, ...) - Given a number of values of vector type with the same length ...
Definition: ISDOpcodes.h:543
@ VECREDUCE_FMAX
FMIN/FMAX nodes can have flags, for NaN/NoNaN variants.
Definition: ISDOpcodes.h:1355
@ FADD
Simple binary floating point operators.
Definition: ISDOpcodes.h:390
@ VECREDUCE_FMAXIMUM
FMINIMUM/FMAXIMUM nodes propatate NaNs and signed zeroes using the llvm.minimum and llvm....
Definition: ISDOpcodes.h:1359
@ ABS
ABS - Determine the unsigned absolute value of a signed integer value of the same bitwidth.
Definition: ISDOpcodes.h:688
@ ATOMIC_FENCE
OUTCHAIN = ATOMIC_FENCE(INCHAIN, ordering, scope) This corresponds to the fence instruction.
Definition: ISDOpcodes.h:1233
@ RESET_FPMODE
Sets default dynamic floating-point control modes.
Definition: ISDOpcodes.h:1025
@ SDIVREM
SDIVREM/UDIVREM - Divide two integers and produce both a quotient and remainder result.
Definition: ISDOpcodes.h:255
@ VECREDUCE_SMAX
Definition: ISDOpcodes.h:1369
@ STRICT_FSETCCS
Definition: ISDOpcodes.h:477
@ FP16_TO_FP
FP16_TO_FP, FP_TO_FP16 - These operators are used to perform promotions and truncation for half-preci...
Definition: ISDOpcodes.h:913
@ STRICT_FLOG2
Definition: ISDOpcodes.h:421
@ ATOMIC_LOAD_OR
Definition: ISDOpcodes.h:1267
@ BITCAST
BITCAST - This operator converts between integer, vector and FP values, as if the value was stored to...
Definition: ISDOpcodes.h:903
@ BUILD_PAIR
BUILD_PAIR - This is the opposite of EXTRACT_ELEMENT in some ways.
Definition: ISDOpcodes.h:229
@ ATOMIC_LOAD_XOR
Definition: ISDOpcodes.h:1268
@ INIT_TRAMPOLINE
INIT_TRAMPOLINE - This corresponds to the init_trampoline intrinsic.
Definition: ISDOpcodes.h:1199
@ FLDEXP
FLDEXP - ldexp, inspired by libm (op0 * 2**op1).
Definition: ISDOpcodes.h:939
@ SDIVFIX
RESULT = [US]DIVFIX(LHS, RHS, SCALE) - Perform fixed point division on 2 integers with the same width...
Definition: ISDOpcodes.h:380
@ STRICT_FSQRT
Constrained versions of libm-equivalent floating point intrinsics.
Definition: ISDOpcodes.h:411
@ BUILTIN_OP_END
BUILTIN_OP_END - This must be the last enum value in this list.
Definition: ISDOpcodes.h:1400
@ GlobalTLSAddress
Definition: ISDOpcodes.h:79
@ EH_LABEL
EH_LABEL - Represents a label in mid basic block used to track locations needed for debug and excepti...
Definition: ISDOpcodes.h:1102
@ EH_RETURN
OUTCHAIN = EH_RETURN(INCHAIN, OFFSET, HANDLER) - This node represents 'eh_return' gcc dwarf builtin,...
Definition: ISDOpcodes.h:135
@ SIGN_EXTEND
Conversion operators.
Definition: ISDOpcodes.h:774
@ STRICT_UINT_TO_FP
Definition: ISDOpcodes.h:450
@ SCALAR_TO_VECTOR
SCALAR_TO_VECTOR(VAL) - This represents the operation of loading a scalar value into element 0 of the...
Definition: ISDOpcodes.h:620
@ READSTEADYCOUNTER
READSTEADYCOUNTER - This corresponds to the readfixedcounter intrinsic.
Definition: ISDOpcodes.h:1188
@ ADDROFRETURNADDR
ADDROFRETURNADDR - Represents the llvm.addressofreturnaddress intrinsic.
Definition: ISDOpcodes.h:101
@ VECREDUCE_FADD
These reductions have relaxed evaluation order semantics, and have a single vector operand.
Definition: ISDOpcodes.h:1352
@ CTTZ_ZERO_UNDEF
Bit counting operators with an undefined result for zero inputs.
Definition: ISDOpcodes.h:722
@ WRITE_REGISTER
Definition: ISDOpcodes.h:119
@ PREFETCH
PREFETCH - This corresponds to a prefetch intrinsic.
Definition: ISDOpcodes.h:1221
@ VECREDUCE_FMIN
Definition: ISDOpcodes.h:1356
@ FSINCOS
FSINCOS - Compute both fsin and fcos as a single operation.
Definition: ISDOpcodes.h:988
@ SETCCCARRY
Like SetCC, ops #0 and #1 are the LHS and RHS operands to compare, but op #2 is a boolean indicating ...
Definition: ISDOpcodes.h:758
@ STRICT_LROUND
Definition: ISDOpcodes.h:431
@ FNEG
Perform various unary floating-point operations inspired by libm.
Definition: ISDOpcodes.h:930
@ BR_CC
BR_CC - Conditional branch.
Definition: ISDOpcodes.h:1077
@ SSUBO
Same for subtraction.
Definition: ISDOpcodes.h:327
@ ATOMIC_LOAD_MIN
Definition: ISDOpcodes.h:1270
@ BR_JT
BR_JT - Jumptable branch.
Definition: ISDOpcodes.h:1056
@ FCANONICALIZE
Returns platform specific canonical encoding of a floating point number.
Definition: ISDOpcodes.h:500
@ IS_FPCLASS
Performs a check of floating point class property, defined by IEEE-754.
Definition: ISDOpcodes.h:507
@ SSUBSAT
RESULT = [US]SUBSAT(LHS, RHS) - Perform saturation subtraction on 2 integers with the same bit width ...
Definition: ISDOpcodes.h:349
@ SELECT
Select(COND, TRUEVAL, FALSEVAL).
Definition: ISDOpcodes.h:727
@ STRICT_FPOWI
Definition: ISDOpcodes.h:413
@ ATOMIC_LOAD
Val, OUTCHAIN = ATOMIC_LOAD(INCHAIN, ptr) This corresponds to "load atomic" instruction.
Definition: ISDOpcodes.h:1237
@ UNDEF
UNDEF - An undefined node.
Definition: ISDOpcodes.h:211
@ VECREDUCE_UMAX
Definition: ISDOpcodes.h:1371
@ EXTRACT_ELEMENT
EXTRACT_ELEMENT - This is used to get the lower or upper (determined by a Constant,...
Definition: ISDOpcodes.h:222
@ SPLAT_VECTOR
SPLAT_VECTOR(VAL) - Returns a vector with the scalar value VAL duplicated in all lanes.
Definition: ISDOpcodes.h:627
@ VACOPY
VACOPY - VACOPY has 5 operands: an input chain, a destination pointer, a source pointer,...
Definition: ISDOpcodes.h:1151
@ SADDO
RESULT, BOOL = [SU]ADDO(LHS, RHS) - Overflow-aware nodes for addition.
Definition: ISDOpcodes.h:323
@ STRICT_FTRUNC
Definition: ISDOpcodes.h:430
@ VECREDUCE_ADD
Integer reductions may have a result type larger than the vector element type.
Definition: ISDOpcodes.h:1364
@ GET_ROUNDING
Returns current rounding mode: -1 Undefined 0 Round to 0 1 Round to nearest, ties to even 2 Round to ...
Definition: ISDOpcodes.h:880
@ STRICT_FP_TO_FP16
Definition: ISDOpcodes.h:916
@ MULHU
MULHU/MULHS - Multiply high - Multiply two integers of type iN, producing an unsigned/signed value of...
Definition: ISDOpcodes.h:651
@ GET_FPMODE
Reads the current dynamic floating-point control modes.
Definition: ISDOpcodes.h:1016
@ STRICT_FP16_TO_FP
Definition: ISDOpcodes.h:915
@ SHL
Shift and rotation operations.
Definition: ISDOpcodes.h:705
@ ATOMIC_LOAD_CLR
Definition: ISDOpcodes.h:1266
@ VECTOR_SHUFFLE
VECTOR_SHUFFLE(VEC1, VEC2) - Returns a vector, of the same type as VEC1/VEC2.
Definition: ISDOpcodes.h:600
@ ATOMIC_LOAD_AND
Definition: ISDOpcodes.h:1265
@ EXTRACT_SUBVECTOR
EXTRACT_SUBVECTOR(VECTOR, IDX) - Returns a subvector from VECTOR.
Definition: ISDOpcodes.h:573
@ STRICT_FMAXNUM
Definition: ISDOpcodes.h:424
@ READ_REGISTER
READ_REGISTER, WRITE_REGISTER - This node represents llvm.register on the DAG, which implements the n...
Definition: ISDOpcodes.h:118
@ EXTRACT_VECTOR_ELT
EXTRACT_VECTOR_ELT(VECTOR, IDX) - Returns a single element from VECTOR identified by the (potentially...
Definition: ISDOpcodes.h:535
@ ZERO_EXTEND
ZERO_EXTEND - Used for integer types, zeroing the new bits.
Definition: ISDOpcodes.h:780
@ TargetConstantFP
Definition: ISDOpcodes.h:159
@ DEBUGTRAP
DEBUGTRAP - Trap intended to get the attention of a debugger.
Definition: ISDOpcodes.h:1211
@ FP_TO_UINT_SAT
Definition: ISDOpcodes.h:856
@ STRICT_FMINNUM
Definition: ISDOpcodes.h:425
@ SELECT_CC
Select with condition operator - This selects between a true value and a false value (ops #2 and #3) ...
Definition: ISDOpcodes.h:742
@ ATOMIC_CMP_SWAP
Val, OUTCHAIN = ATOMIC_CMP_SWAP(INCHAIN, ptr, cmp, swap) For double-word atomic operations: ValLo,...
Definition: ISDOpcodes.h:1248
@ ATOMIC_LOAD_UMAX
Definition: ISDOpcodes.h:1273
@ FMINNUM
FMINNUM/FMAXNUM - Perform floating-point minimum or maximum on two values.
Definition: ISDOpcodes.h:971
@ UBSANTRAP
UBSANTRAP - Trap with an immediate describing the kind of sanitizer failure.
Definition: ISDOpcodes.h:1215
@ SSHLSAT
RESULT = [US]SHLSAT(LHS, RHS) - Perform saturation left shift.
Definition: ISDOpcodes.h:359
@ SMULO
Same for multiplication.
Definition: ISDOpcodes.h:331
@ DYNAMIC_STACKALLOC
DYNAMIC_STACKALLOC - Allocate some number of bytes on the stack aligned to a specified boundary.
Definition: ISDOpcodes.h:1041
@ STRICT_LRINT
Definition: ISDOpcodes.h:433
@ ConstantPool
Definition: ISDOpcodes.h:82
@ SIGN_EXTEND_INREG
SIGN_EXTEND_INREG - This operator atomically performs a SHL/SRA pair to sign extend a small value in ...
Definition: ISDOpcodes.h:798
@ SMIN
[US]{MIN/MAX} - Binary minimum or maximum of signed or unsigned integers.
Definition: ISDOpcodes.h:674
@ SDIVFIXSAT
Same as the corresponding unsaturated fixed point instructions, but the result is clamped between the...
Definition: ISDOpcodes.h:386
@ FP_EXTEND
X = FP_EXTEND(Y) - Extend a smaller FP type into a larger FP type.
Definition: ISDOpcodes.h:888
@ GLOBAL_OFFSET_TABLE
The address of the GOT.
Definition: ISDOpcodes.h:87
@ STRICT_FROUND
Definition: ISDOpcodes.h:428
@ UADDO_CARRY
Carry-using nodes for multiple precision addition and subtraction.
Definition: ISDOpcodes.h:303
@ STRICT_SINT_TO_FP
STRICT_[US]INT_TO_FP - Convert a signed or unsigned integer to a floating point value.
Definition: ISDOpcodes.h:449
@ STRICT_BF16_TO_FP
Definition: ISDOpcodes.h:924
@ VECREDUCE_UMIN
Definition: ISDOpcodes.h:1372
@ STRICT_FFLOOR
Definition: ISDOpcodes.h:427
@ STRICT_FROUNDEVEN
Definition: ISDOpcodes.h:429
@ EH_DWARF_CFA
EH_DWARF_CFA - This node represents the pointer to the DWARF Canonical Frame Address (CFA),...
Definition: ISDOpcodes.h:129
@ BF16_TO_FP
BF16_TO_FP, FP_TO_BF16 - These operators are used to perform promotions and truncation for bfloat16.
Definition: ISDOpcodes.h:922
@ FRAMEADDR
FRAMEADDR, RETURNADDR - These nodes represent llvm.frameaddress and llvm.returnaddress on the DAG.
Definition: ISDOpcodes.h:94
@ ATOMIC_LOAD_ADD
Definition: ISDOpcodes.h:1263
@ STRICT_FP_TO_UINT
Definition: ISDOpcodes.h:443
@ STRICT_FP_ROUND
X = STRICT_FP_ROUND(Y, TRUNC) - Rounding 'Y' from a larger floating point type down to the precision ...
Definition: ISDOpcodes.h:465
@ STRICT_FP_TO_SINT
STRICT_FP_TO_[US]INT - Convert a floating point value to a signed or unsigned integer.
Definition: ISDOpcodes.h:442
@ FMINIMUM
FMINIMUM/FMAXIMUM - NaN-propagating minimum/maximum that also treat -0.0 as less than 0....
Definition: ISDOpcodes.h:984
@ ATOMIC_LOAD_SUB
Definition: ISDOpcodes.h:1264
@ FP_TO_SINT
FP_TO_[US]INT - Convert a floating point value to a signed or unsigned integer.
Definition: ISDOpcodes.h:836
@ READCYCLECOUNTER
READCYCLECOUNTER - This corresponds to the readcyclecounter intrinsic.
Definition: ISDOpcodes.h:1182
@ TargetConstant
TargetConstant* - Like Constant*, but the DAG does not do any folding, simplification,...
Definition: ISDOpcodes.h:158
@ STRICT_FP_EXTEND
X = STRICT_FP_EXTEND(Y) - Extend a smaller FP type into a larger FP type.
Definition: ISDOpcodes.h:470
@ AND
Bitwise operators - logical and, logical or, logical xor.
Definition: ISDOpcodes.h:680
@ TRAP
TRAP - Trapping instruction.
Definition: ISDOpcodes.h:1208
@ INTRINSIC_WO_CHAIN
RESULT = INTRINSIC_WO_CHAIN(INTRINSICID, arg1, arg2, ...) This node represents a target intrinsic fun...
Definition: ISDOpcodes.h:184
@ GET_FPENV_MEM
Gets the current floating-point environment.
Definition: ISDOpcodes.h:1007
@ STRICT_FP_TO_BF16
Definition: ISDOpcodes.h:925
@ VECREDUCE_FMUL
Definition: ISDOpcodes.h:1353
@ STRICT_FADD
Constrained versions of the binary floating point operators.
Definition: ISDOpcodes.h:400
@ STRICT_FLOG10
Definition: ISDOpcodes.h:420
@ INSERT_VECTOR_ELT
INSERT_VECTOR_ELT(VECTOR, VAL, IDX) - Returns VECTOR with the element at IDX replaced with VAL.
Definition: ISDOpcodes.h:524
@ TokenFactor
TokenFactor - This node takes multiple tokens as input and produces a single token result.
Definition: ISDOpcodes.h:52
@ STRICT_LLRINT
Definition: ISDOpcodes.h:434
@ VECTOR_SPLICE
VECTOR_SPLICE(VEC1, VEC2, IMM) - Returns a subvector of the same type as VEC1/VEC2 from CONCAT_VECTOR...
Definition: ISDOpcodes.h:612
@ STRICT_FEXP2
Definition: ISDOpcodes.h:418
@ ATOMIC_SWAP
Val, OUTCHAIN = ATOMIC_SWAP(INCHAIN, ptr, amt) Val, OUTCHAIN = ATOMIC_LOAD_[OpName](INCHAIN,...
Definition: ISDOpcodes.h:1262
@ ExternalSymbol
Definition: ISDOpcodes.h:83
@ FFREXP
FFREXP - frexp, extract fractional and exponent component of a floating-point value.
Definition: ISDOpcodes.h:944
@ FP_ROUND
X = FP_ROUND(Y, TRUNC) - Rounding 'Y' from a larger floating point type down to the precision of the ...
Definition: ISDOpcodes.h:869
@ SPONENTRY
SPONENTRY - Represents the llvm.sponentry intrinsic.
Definition: ISDOpcodes.h:106
@ STRICT_FLDEXP
Definition: ISDOpcodes.h:414
@ STRICT_LLROUND
Definition: ISDOpcodes.h:432
@ STRICT_FNEARBYINT
Definition: ISDOpcodes.h:423
@ FP_TO_SINT_SAT
FP_TO_[US]INT_SAT - Convert floating point value in operand 0 to a signed or unsigned scalar integer ...
Definition: ISDOpcodes.h:855
@ VECREDUCE_FMINIMUM
Definition: ISDOpcodes.h:1360
@ EH_SJLJ_SETJMP
RESULT, OUTCHAIN = EH_SJLJ_SETJMP(INCHAIN, buffer) This corresponds to the eh.sjlj....
Definition: ISDOpcodes.h:141
@ TRUNCATE
TRUNCATE - Completely drop the high bits.
Definition: ISDOpcodes.h:786
@ VAARG
VAARG - VAARG has four operands: an input chain, a pointer, a SRCVALUE, and the alignment.
Definition: ISDOpcodes.h:1146
@ BRCOND
BRCOND - Conditional branch.
Definition: ISDOpcodes.h:1070
@ VECREDUCE_SEQ_FMUL
Definition: ISDOpcodes.h:1340
@ SHL_PARTS
SHL_PARTS/SRA_PARTS/SRL_PARTS - These operators are used for expanded integer shift operations.
Definition: ISDOpcodes.h:763
@ AssertSext
AssertSext, AssertZext - These nodes record if a register contains a value that has already been zero...
Definition: ISDOpcodes.h:61
@ FCOPYSIGN
FCOPYSIGN(X, Y) - Return the value of X with the sign of Y.
Definition: ISDOpcodes.h:493
@ SADDSAT
RESULT = [US]ADDSAT(LHS, RHS) - Perform saturation addition on 2 integers with the same bit width (W)...
Definition: ISDOpcodes.h:340
@ AssertZext
Definition: ISDOpcodes.h:62
@ CALLSEQ_START
CALLSEQ_START/CALLSEQ_END - These operators mark the beginning and end of a call sequence,...
Definition: ISDOpcodes.h:1140
@ STRICT_FRINT
Definition: ISDOpcodes.h:422
@ GET_DYNAMIC_AREA_OFFSET
GET_DYNAMIC_AREA_OFFSET - get offset from native SP to the address of the most recent dynamic alloca.
Definition: ISDOpcodes.h:1320
@ SET_FPENV_MEM
Sets the current floating point environment.
Definition: ISDOpcodes.h:1012
@ ADJUST_TRAMPOLINE
ADJUST_TRAMPOLINE - This corresponds to the adjust_trampoline intrinsic.
Definition: ISDOpcodes.h:1205
@ INTRINSIC_W_CHAIN
RESULT,OUTCHAIN = INTRINSIC_W_CHAIN(INCHAIN, INTRINSICID, arg1, ...) This node represents a target in...
Definition: ISDOpcodes.h:192
@ BUILD_VECTOR
BUILD_VECTOR(ELT0, ELT1, ELT2, ELT3,...) - Return a fixed-width vector with the specified,...
Definition: ISDOpcodes.h:515
NodeType getExtForLoadExtType(bool IsFP, LoadExtType)
bool isNormalStore(const SDNode *N)
Returns true if the specified node is a non-truncating and unindexed store.
CondCode getSetCCInverse(CondCode Operation, EVT Type)
Return the operation corresponding to !(X op Y), where 'op' is a valid SetCC operation.
CondCode getSetCCSwappedOperands(CondCode Operation)
Return the operation corresponding to (Y op X) when given the operation for (X op Y).
bool isSignedIntSetCC(CondCode Code)
Return true if this is a setcc instruction that performs a signed comparison when used with integer o...
Definition: ISDOpcodes.h:1556
CondCode
ISD::CondCode enum - These are ordered carefully to make the bitfields below work out,...
Definition: ISDOpcodes.h:1523
LoadExtType
LoadExtType enum - This enum defines the three variants of LOADEXT (load with extension).
Definition: ISDOpcodes.h:1503
Libcall getPOWI(EVT RetVT)
getPOWI - Return the POWI_* value for the given types, or UNKNOWN_LIBCALL if there is none.
Libcall getSINTTOFP(EVT OpVT, EVT RetVT)
getSINTTOFP - Return the SINTTOFP_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
Libcall getSYNC(unsigned Opc, MVT VT)
Return the SYNC_FETCH_AND_* value for the given opcode and type, or UNKNOWN_LIBCALL if there is none.
Libcall getLDEXP(EVT RetVT)
getLDEXP - Return the LDEXP_* value for the given types, or UNKNOWN_LIBCALL if there is none.
Libcall getUINTTOFP(EVT OpVT, EVT RetVT)
getUINTTOFP - Return the UINTTOFP_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
Libcall getFREXP(EVT RetVT)
getFREXP - Return the FREXP_* value for the given types, or UNKNOWN_LIBCALL if there is none.
Libcall
RTLIB::Libcall enum - This enum defines all of the runtime library calls the backend can emit.
Libcall getFPTOUINT(EVT OpVT, EVT RetVT)
getFPTOUINT - Return the FPTOUINT_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
Libcall getFPTOSINT(EVT OpVT, EVT RetVT)
getFPTOSINT - Return the FPTOSINT_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
Libcall getOUTLINE_ATOMIC(unsigned Opc, AtomicOrdering Order, MVT VT)
Return the outline atomics value for the given opcode, atomic ordering and type, or UNKNOWN_LIBCALL i...
Libcall getFPEXT(EVT OpVT, EVT RetVT)
getFPEXT - Return the FPEXT_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
Libcall getFPROUND(EVT OpVT, EVT RetVT)
getFPROUND - Return the FPROUND_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
Libcall getFPLibCall(EVT VT, Libcall Call_F32, Libcall Call_F64, Libcall Call_F80, Libcall Call_F128, Libcall Call_PPCF128)
GetFPLibCall - Helper to return the right libcall for the given floating point type,...
@ Undef
Value of the register doesn't matter.
ManagedStatic< cl::opt< FnT >, OptCreatorT > Action
constexpr double e
Definition: MathExtras.h:31
This is an optimization pass for GlobalISel generic memory operations.
Definition: AddressRanges.h:18
auto drop_begin(T &&RangeOrContainer, size_t N=1)
Return a range covering RangeOrContainer with the first N elements excluded.
Definition: STLExtras.h:329
void dump(const SparseBitVector< ElementSize > &LHS, raw_ostream &out)
unsigned Log2_32_Ceil(uint32_t Value)
Return the ceil log base 2 of the specified value, 32 if the value is zero.
Definition: MathExtras.h:326
@ Offset
Definition: DWP.cpp:456
constexpr bool isPowerOf2_64(uint64_t Value)
Return true if the argument is a power of two > 0 (64 bit edition.)
Definition: MathExtras.h:269
unsigned Log2_32(uint32_t Value)
Return the floor log base 2 of the specified value, -1 if the value is zero.
Definition: MathExtras.h:313
constexpr bool isPowerOf2_32(uint32_t Value)
Return true if the argument is a power of two > 0.
Definition: MathExtras.h:264
FPClassTest
Floating-point class tests, supported by 'is_fpclass' intrinsic.
APFloat scalbn(APFloat X, int Exp, APFloat::roundingMode RM)
Definition: APFloat.h:1361
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
Definition: Debug.cpp:163
Constant * ConstantFoldCastOperand(unsigned Opcode, Constant *C, Type *DestTy, const DataLayout &DL)
Attempt to constant fold a cast with the specified operand.
AtomicOrdering
Atomic ordering for LLVM's memory model.
@ Or
Bitwise or logical OR of integers.
@ And
Bitwise or logical AND of integers.
DWARFExpression::Operation Op
bool isOneConstant(SDValue V)
Returns true if V is a constant integer one.
Align commonAlignment(Align A, uint64_t Offset)
Returns the alignment that satisfies both alignments.
Definition: Alignment.h:212
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
Definition: BitVector.h:860
#define N
A collection of metadata nodes that might be associated with a memory access used by the alias-analys...
Definition: Metadata.h:760
int32_t ExponentType
A signed type to represent a floating point numbers unbiased exponent.
Definition: APFloat.h:147
This struct is a compact representation of a valid (non-zero power of two) alignment.
Definition: Alignment.h:39
uint64_t value() const
This is a hole in the type system and should not be abused.
Definition: Alignment.h:85
Extended Value Type.
Definition: ValueTypes.h:34
TypeSize getStoreSize() const
Return the number of bytes overwritten by a store of the specified value type.
Definition: ValueTypes.h:380
static EVT getVectorVT(LLVMContext &Context, EVT VT, unsigned NumElements, bool IsScalable=false)
Returns the EVT that represents a vector NumElements in length, where each element is of type VT.
Definition: ValueTypes.h:73
EVT changeTypeToInteger() const
Return the type converted to an equivalently sized integer or vector with integer element type.
Definition: ValueTypes.h:120
bool bitsGT(EVT VT) const
Return true if this has more bits than VT.
Definition: ValueTypes.h:274
bool bitsLT(EVT VT) const
Return true if this has less bits than VT.
Definition: ValueTypes.h:290
bool isFloatingPoint() const
Return true if this is a FP or a vector FP type.
Definition: ValueTypes.h:146
TypeSize getSizeInBits() const
Return the size of the specified value type in bits.
Definition: ValueTypes.h:358
bool isByteSized() const
Return true if the bit size is a multiple of 8.
Definition: ValueTypes.h:233
uint64_t getScalarSizeInBits() const
Definition: ValueTypes.h:370
EVT getHalfSizedIntegerVT(LLVMContext &Context) const
Finds the smallest simple value type that is greater than or equal to half the width of this EVT.
Definition: ValueTypes.h:415
TypeSize getStoreSizeInBits() const
Return the number of bits overwritten by a store of the specified value type.
Definition: ValueTypes.h:397
MVT getSimpleVT() const
Return the SimpleValueType held in the specified simple EVT.
Definition: ValueTypes.h:306
static EVT getIntegerVT(LLVMContext &Context, unsigned BitWidth)
Returns the EVT that represents an integer with the given number of bits.
Definition: ValueTypes.h:64
bool isVector() const
Return true if this is a vector value type.
Definition: ValueTypes.h:167
EVT getScalarType() const
If this is a vector type, return the element type, otherwise return this.
Definition: ValueTypes.h:313
bool bitsGE(EVT VT) const
Return true if this has no less bits than VT.
Definition: ValueTypes.h:282
bool bitsEq(EVT VT) const
Return true if this has the same number of bits as VT.
Definition: ValueTypes.h:246
Type * getTypeForEVT(LLVMContext &Context) const
This method returns an LLVM type corresponding to the specified EVT.
Definition: ValueTypes.cpp:202
bool isScalableVector() const
Return true if this is a vector type where the runtime length is machine dependent.
Definition: ValueTypes.h:173
EVT getVectorElementType() const
Given a vector type, return the type of each element.
Definition: ValueTypes.h:318
bool isScalarInteger() const
Return true if this is an integer, but not a vector.
Definition: ValueTypes.h:156
unsigned getVectorNumElements() const
Given a vector type, return the number of elements it contains.
Definition: ValueTypes.h:326
bool bitsLE(EVT VT) const
Return true if this has no more bits than VT.
Definition: ValueTypes.h:298
bool isInteger() const
Return true if this is an integer or a vector integer type.
Definition: ValueTypes.h:151
This class contains a discriminated union of information about pointers in memory operands,...
static MachinePointerInfo getJumpTable(MachineFunction &MF)
Return a MachinePointerInfo record that refers to a jump table entry.
static MachinePointerInfo getConstantPool(MachineFunction &MF)
Return a MachinePointerInfo record that refers to the constant pool.
MachinePointerInfo getWithOffset(int64_t O) const
static MachinePointerInfo getUnknownStack(MachineFunction &MF)
Stack memory without other information.
static MachinePointerInfo getFixedStack(MachineFunction &MF, int FI, int64_t Offset=0)
Return a MachinePointerInfo record that refers to the specified FrameIndex.
These are IR-level optimization flags that may be propagated to SDNodes.
void setNoUnsignedWrap(bool b)
void setNoSignedWrap(bool b)
This represents a list of ValueType's that has been intern'd by a SelectionDAG.
This structure contains all information that is necessary for lowering calls.
This structure is used to pass arguments to makeLibCall function.
MakeLibCallOptions & setSExt(bool Value=true)