LLVM  13.0.0git
LegalizeDAG.cpp
Go to the documentation of this file.
1 //===- LegalizeDAG.cpp - Implement SelectionDAG::Legalize -----------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file implements the SelectionDAG::Legalize method.
10 //
11 //===----------------------------------------------------------------------===//
12 
13 #include "llvm/ADT/APFloat.h"
14 #include "llvm/ADT/APInt.h"
15 #include "llvm/ADT/ArrayRef.h"
16 #include "llvm/ADT/SetVector.h"
17 #include "llvm/ADT/SmallPtrSet.h"
18 #include "llvm/ADT/SmallSet.h"
19 #include "llvm/ADT/SmallVector.h"
32 #include "llvm/IR/CallingConv.h"
33 #include "llvm/IR/Constants.h"
34 #include "llvm/IR/DataLayout.h"
35 #include "llvm/IR/DerivedTypes.h"
36 #include "llvm/IR/Function.h"
37 #include "llvm/IR/Metadata.h"
38 #include "llvm/IR/Type.h"
39 #include "llvm/Support/Casting.h"
40 #include "llvm/Support/Compiler.h"
41 #include "llvm/Support/Debug.h"
48 #include <algorithm>
49 #include <cassert>
50 #include <cstdint>
51 #include <tuple>
52 #include <utility>
53 
54 using namespace llvm;
55 
56 #define DEBUG_TYPE "legalizedag"
57 
58 namespace {
59 
60 /// Keeps track of state when getting the sign of a floating-point value as an
61 /// integer.
62 struct FloatSignAsInt {
63  EVT FloatVT;
64  SDValue Chain;
65  SDValue FloatPtr;
66  SDValue IntPtr;
67  MachinePointerInfo IntPointerInfo;
68  MachinePointerInfo FloatPointerInfo;
69  SDValue IntValue;
70  APInt SignMask;
71  uint8_t SignBit;
72 };
73 
74 //===----------------------------------------------------------------------===//
75 /// This takes an arbitrary SelectionDAG as input and
76 /// hacks on it until the target machine can handle it. This involves
77 /// eliminating value sizes the machine cannot handle (promoting small sizes to
78 /// large sizes or splitting up large values into small values) as well as
79 /// eliminating operations the machine cannot handle.
80 ///
81 /// This code also does a small amount of optimization and recognition of idioms
82 /// as part of its processing. For example, if a target does not support a
83 /// 'setcc' instruction efficiently, but does support 'brcc' instruction, this
84 /// will attempt merge setcc and brc instructions into brcc's.
85 class SelectionDAGLegalize {
86  const TargetMachine &TM;
87  const TargetLowering &TLI;
88  SelectionDAG &DAG;
89 
90  /// The set of nodes which have already been legalized. We hold a
91  /// reference to it in order to update as necessary on node deletion.
92  SmallPtrSetImpl<SDNode *> &LegalizedNodes;
93 
94  /// A set of all the nodes updated during legalization.
95  SmallSetVector<SDNode *, 16> *UpdatedNodes;
96 
97  EVT getSetCCResultType(EVT VT) const {
98  return TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT);
99  }
100 
101  // Libcall insertion helpers.
102 
103 public:
104  SelectionDAGLegalize(SelectionDAG &DAG,
105  SmallPtrSetImpl<SDNode *> &LegalizedNodes,
106  SmallSetVector<SDNode *, 16> *UpdatedNodes = nullptr)
107  : TM(DAG.getTarget()), TLI(DAG.getTargetLoweringInfo()), DAG(DAG),
108  LegalizedNodes(LegalizedNodes), UpdatedNodes(UpdatedNodes) {}
109 
110  /// Legalizes the given operation.
111  void LegalizeOp(SDNode *Node);
112 
113 private:
114  SDValue OptimizeFloatStore(StoreSDNode *ST);
115 
116  void LegalizeLoadOps(SDNode *Node);
117  void LegalizeStoreOps(SDNode *Node);
118 
119  /// Some targets cannot handle a variable
120  /// insertion index for the INSERT_VECTOR_ELT instruction. In this case, it
121  /// is necessary to spill the vector being inserted into to memory, perform
122  /// the insert there, and then read the result back.
123  SDValue PerformInsertVectorEltInMemory(SDValue Vec, SDValue Val, SDValue Idx,
124  const SDLoc &dl);
125  SDValue ExpandINSERT_VECTOR_ELT(SDValue Vec, SDValue Val, SDValue Idx,
126  const SDLoc &dl);
127 
128  /// Return a vector shuffle operation which
129  /// performs the same shuffe in terms of order or result bytes, but on a type
130  /// whose vector element type is narrower than the original shuffle type.
131  /// e.g. <v4i32> <0, 1, 0, 1> -> v8i16 <0, 1, 2, 3, 0, 1, 2, 3>
132  SDValue ShuffleWithNarrowerEltType(EVT NVT, EVT VT, const SDLoc &dl,
133  SDValue N1, SDValue N2,
134  ArrayRef<int> Mask) const;
135 
136  SDValue ExpandLibCall(RTLIB::Libcall LC, SDNode *Node, bool isSigned);
137 
138  void ExpandFPLibCall(SDNode *Node, RTLIB::Libcall Call_F32,
139  RTLIB::Libcall Call_F64, RTLIB::Libcall Call_F80,
140  RTLIB::Libcall Call_F128,
141  RTLIB::Libcall Call_PPCF128,
143  SDValue ExpandIntLibCall(SDNode *Node, bool isSigned,
144  RTLIB::Libcall Call_I8,
145  RTLIB::Libcall Call_I16,
146  RTLIB::Libcall Call_I32,
147  RTLIB::Libcall Call_I64,
148  RTLIB::Libcall Call_I128);
149  void ExpandArgFPLibCall(SDNode *Node,
150  RTLIB::Libcall Call_F32, RTLIB::Libcall Call_F64,
151  RTLIB::Libcall Call_F80, RTLIB::Libcall Call_F128,
152  RTLIB::Libcall Call_PPCF128,
154  void ExpandDivRemLibCall(SDNode *Node, SmallVectorImpl<SDValue> &Results);
155  void ExpandSinCosLibCall(SDNode *Node, SmallVectorImpl<SDValue> &Results);
156 
157  SDValue EmitStackConvert(SDValue SrcOp, EVT SlotVT, EVT DestVT,
158  const SDLoc &dl);
159  SDValue EmitStackConvert(SDValue SrcOp, EVT SlotVT, EVT DestVT,
160  const SDLoc &dl, SDValue ChainIn);
161  SDValue ExpandBUILD_VECTOR(SDNode *Node);
162  SDValue ExpandSPLAT_VECTOR(SDNode *Node);
163  SDValue ExpandSCALAR_TO_VECTOR(SDNode *Node);
164  void ExpandDYNAMIC_STACKALLOC(SDNode *Node,
166  void getSignAsIntValue(FloatSignAsInt &State, const SDLoc &DL,
167  SDValue Value) const;
168  SDValue modifySignAsInt(const FloatSignAsInt &State, const SDLoc &DL,
169  SDValue NewIntValue) const;
170  SDValue ExpandFCOPYSIGN(SDNode *Node) const;
171  SDValue ExpandFABS(SDNode *Node) const;
172  SDValue ExpandFNEG(SDNode *Node) const;
173  SDValue ExpandLegalINT_TO_FP(SDNode *Node, SDValue &Chain);
174  void PromoteLegalINT_TO_FP(SDNode *N, const SDLoc &dl,
176  void PromoteLegalFP_TO_INT(SDNode *N, const SDLoc &dl,
178  SDValue PromoteLegalFP_TO_INT_SAT(SDNode *Node, const SDLoc &dl);
179 
180  SDValue ExpandPARITY(SDValue Op, const SDLoc &dl);
181 
182  SDValue ExpandExtractFromVectorThroughStack(SDValue Op);
183  SDValue ExpandInsertToVectorThroughStack(SDValue Op);
184  SDValue ExpandVectorBuildThroughStack(SDNode* Node);
185 
186  SDValue ExpandConstantFP(ConstantFPSDNode *CFP, bool UseCP);
187  SDValue ExpandConstant(ConstantSDNode *CP);
188 
189  // if ExpandNode returns false, LegalizeOp falls back to ConvertNodeToLibcall
190  bool ExpandNode(SDNode *Node);
191  void ConvertNodeToLibcall(SDNode *Node);
192  void PromoteNode(SDNode *Node);
193 
194 public:
195  // Node replacement helpers
196 
197  void ReplacedNode(SDNode *N) {
198  LegalizedNodes.erase(N);
199  if (UpdatedNodes)
200  UpdatedNodes->insert(N);
201  }
202 
203  void ReplaceNode(SDNode *Old, SDNode *New) {
204  LLVM_DEBUG(dbgs() << " ... replacing: "; Old->dump(&DAG);
205  dbgs() << " with: "; New->dump(&DAG));
206 
207  assert(Old->getNumValues() == New->getNumValues() &&
208  "Replacing one node with another that produces a different number "
209  "of values!");
210  DAG.ReplaceAllUsesWith(Old, New);
211  if (UpdatedNodes)
212  UpdatedNodes->insert(New);
213  ReplacedNode(Old);
214  }
215 
216  void ReplaceNode(SDValue Old, SDValue New) {
217  LLVM_DEBUG(dbgs() << " ... replacing: "; Old->dump(&DAG);
218  dbgs() << " with: "; New->dump(&DAG));
219 
220  DAG.ReplaceAllUsesWith(Old, New);
221  if (UpdatedNodes)
222  UpdatedNodes->insert(New.getNode());
223  ReplacedNode(Old.getNode());
224  }
225 
226  void ReplaceNode(SDNode *Old, const SDValue *New) {
227  LLVM_DEBUG(dbgs() << " ... replacing: "; Old->dump(&DAG));
228 
229  DAG.ReplaceAllUsesWith(Old, New);
230  for (unsigned i = 0, e = Old->getNumValues(); i != e; ++i) {
231  LLVM_DEBUG(dbgs() << (i == 0 ? " with: " : " and: ");
232  New[i]->dump(&DAG));
233  if (UpdatedNodes)
234  UpdatedNodes->insert(New[i].getNode());
235  }
236  ReplacedNode(Old);
237  }
238 
239  void ReplaceNodeWithValue(SDValue Old, SDValue New) {
240  LLVM_DEBUG(dbgs() << " ... replacing: "; Old->dump(&DAG);
241  dbgs() << " with: "; New->dump(&DAG));
242 
243  DAG.ReplaceAllUsesOfValueWith(Old, New);
244  if (UpdatedNodes)
245  UpdatedNodes->insert(New.getNode());
246  ReplacedNode(Old.getNode());
247  }
248 };
249 
250 } // end anonymous namespace
251 
252 /// Return a vector shuffle operation which
253 /// performs the same shuffle in terms of order or result bytes, but on a type
254 /// whose vector element type is narrower than the original shuffle type.
255 /// e.g. <v4i32> <0, 1, 0, 1> -> v8i16 <0, 1, 2, 3, 0, 1, 2, 3>
256 SDValue SelectionDAGLegalize::ShuffleWithNarrowerEltType(
257  EVT NVT, EVT VT, const SDLoc &dl, SDValue N1, SDValue N2,
258  ArrayRef<int> Mask) const {
259  unsigned NumMaskElts = VT.getVectorNumElements();
260  unsigned NumDestElts = NVT.getVectorNumElements();
261  unsigned NumEltsGrowth = NumDestElts / NumMaskElts;
262 
263  assert(NumEltsGrowth && "Cannot promote to vector type with fewer elts!");
264 
265  if (NumEltsGrowth == 1)
266  return DAG.getVectorShuffle(NVT, dl, N1, N2, Mask);
267 
268  SmallVector<int, 8> NewMask;
269  for (unsigned i = 0; i != NumMaskElts; ++i) {
270  int Idx = Mask[i];
271  for (unsigned j = 0; j != NumEltsGrowth; ++j) {
272  if (Idx < 0)
273  NewMask.push_back(-1);
274  else
275  NewMask.push_back(Idx * NumEltsGrowth + j);
276  }
277  }
278  assert(NewMask.size() == NumDestElts && "Non-integer NumEltsGrowth?");
279  assert(TLI.isShuffleMaskLegal(NewMask, NVT) && "Shuffle not legal?");
280  return DAG.getVectorShuffle(NVT, dl, N1, N2, NewMask);
281 }
282 
283 /// Expands the ConstantFP node to an integer constant or
284 /// a load from the constant pool.
285 SDValue
286 SelectionDAGLegalize::ExpandConstantFP(ConstantFPSDNode *CFP, bool UseCP) {
287  bool Extend = false;
288  SDLoc dl(CFP);
289 
290  // If a FP immediate is precise when represented as a float and if the
291  // target can do an extending load from float to double, we put it into
292  // the constant pool as a float, even if it's is statically typed as a
293  // double. This shrinks FP constants and canonicalizes them for targets where
294  // an FP extending load is the same cost as a normal load (such as on the x87
295  // fp stack or PPC FP unit).
296  EVT VT = CFP->getValueType(0);
297  ConstantFP *LLVMC = const_cast<ConstantFP*>(CFP->getConstantFPValue());
298  if (!UseCP) {
299  assert((VT == MVT::f64 || VT == MVT::f32) && "Invalid type expansion");
300  return DAG.getConstant(LLVMC->getValueAPF().bitcastToAPInt(), dl,
301  (VT == MVT::f64) ? MVT::i64 : MVT::i32);
302  }
303 
304  APFloat APF = CFP->getValueAPF();
305  EVT OrigVT = VT;
306  EVT SVT = VT;
307 
308  // We don't want to shrink SNaNs. Converting the SNaN back to its real type
309  // can cause it to be changed into a QNaN on some platforms (e.g. on SystemZ).
310  if (!APF.isSignaling()) {
311  while (SVT != MVT::f32 && SVT != MVT::f16) {
312  SVT = (MVT::SimpleValueType)(SVT.getSimpleVT().SimpleTy - 1);
314  // Only do this if the target has a native EXTLOAD instruction from
315  // smaller type.
316  TLI.isLoadExtLegal(ISD::EXTLOAD, OrigVT, SVT) &&
317  TLI.ShouldShrinkFPConstant(OrigVT)) {
318  Type *SType = SVT.getTypeForEVT(*DAG.getContext());
319  LLVMC = cast<ConstantFP>(ConstantExpr::getFPTrunc(LLVMC, SType));
320  VT = SVT;
321  Extend = true;
322  }
323  }
324  }
325 
326  SDValue CPIdx =
327  DAG.getConstantPool(LLVMC, TLI.getPointerTy(DAG.getDataLayout()));
328  Align Alignment = cast<ConstantPoolSDNode>(CPIdx)->getAlign();
329  if (Extend) {
330  SDValue Result = DAG.getExtLoad(
331  ISD::EXTLOAD, dl, OrigVT, DAG.getEntryNode(), CPIdx,
332  MachinePointerInfo::getConstantPool(DAG.getMachineFunction()), VT,
333  Alignment);
334  return Result;
335  }
336  SDValue Result = DAG.getLoad(
337  OrigVT, dl, DAG.getEntryNode(), CPIdx,
338  MachinePointerInfo::getConstantPool(DAG.getMachineFunction()), Alignment);
339  return Result;
340 }
341 
342 /// Expands the Constant node to a load from the constant pool.
343 SDValue SelectionDAGLegalize::ExpandConstant(ConstantSDNode *CP) {
344  SDLoc dl(CP);
345  EVT VT = CP->getValueType(0);
346  SDValue CPIdx = DAG.getConstantPool(CP->getConstantIntValue(),
347  TLI.getPointerTy(DAG.getDataLayout()));
348  Align Alignment = cast<ConstantPoolSDNode>(CPIdx)->getAlign();
349  SDValue Result = DAG.getLoad(
350  VT, dl, DAG.getEntryNode(), CPIdx,
351  MachinePointerInfo::getConstantPool(DAG.getMachineFunction()), Alignment);
352  return Result;
353 }
354 
355 /// Some target cannot handle a variable insertion index for the
356 /// INSERT_VECTOR_ELT instruction. In this case, it
357 /// is necessary to spill the vector being inserted into to memory, perform
358 /// the insert there, and then read the result back.
359 SDValue SelectionDAGLegalize::PerformInsertVectorEltInMemory(SDValue Vec,
360  SDValue Val,
361  SDValue Idx,
362  const SDLoc &dl) {
363  SDValue Tmp1 = Vec;
364  SDValue Tmp2 = Val;
365  SDValue Tmp3 = Idx;
366 
367  // If the target doesn't support this, we have to spill the input vector
368  // to a temporary stack slot, update the element, then reload it. This is
369  // badness. We could also load the value into a vector register (either
370  // with a "move to register" or "extload into register" instruction, then
371  // permute it into place, if the idx is a constant and if the idx is
372  // supported by the target.
373  EVT VT = Tmp1.getValueType();
374  EVT EltVT = VT.getVectorElementType();
375  SDValue StackPtr = DAG.CreateStackTemporary(VT);
376 
377  int SPFI = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex();
378 
379  // Store the vector.
380  SDValue Ch = DAG.getStore(
381  DAG.getEntryNode(), dl, Tmp1, StackPtr,
382  MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), SPFI));
383 
384  SDValue StackPtr2 = TLI.getVectorElementPointer(DAG, StackPtr, VT, Tmp3);
385 
386  // Store the scalar value.
387  Ch = DAG.getTruncStore(
388  Ch, dl, Tmp2, StackPtr2,
389  MachinePointerInfo::getUnknownStack(DAG.getMachineFunction()), EltVT);
390  // Load the updated vector.
391  return DAG.getLoad(VT, dl, Ch, StackPtr, MachinePointerInfo::getFixedStack(
392  DAG.getMachineFunction(), SPFI));
393 }
394 
395 SDValue SelectionDAGLegalize::ExpandINSERT_VECTOR_ELT(SDValue Vec, SDValue Val,
396  SDValue Idx,
397  const SDLoc &dl) {
398  if (ConstantSDNode *InsertPos = dyn_cast<ConstantSDNode>(Idx)) {
399  // SCALAR_TO_VECTOR requires that the type of the value being inserted
400  // match the element type of the vector being created, except for
401  // integers in which case the inserted value can be over width.
402  EVT EltVT = Vec.getValueType().getVectorElementType();
403  if (Val.getValueType() == EltVT ||
404  (EltVT.isInteger() && Val.getValueType().bitsGE(EltVT))) {
405  SDValue ScVec = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl,
406  Vec.getValueType(), Val);
407 
408  unsigned NumElts = Vec.getValueType().getVectorNumElements();
409  // We generate a shuffle of InVec and ScVec, so the shuffle mask
410  // should be 0,1,2,3,4,5... with the appropriate element replaced with
411  // elt 0 of the RHS.
412  SmallVector<int, 8> ShufOps;
413  for (unsigned i = 0; i != NumElts; ++i)
414  ShufOps.push_back(i != InsertPos->getZExtValue() ? i : NumElts);
415 
416  return DAG.getVectorShuffle(Vec.getValueType(), dl, Vec, ScVec, ShufOps);
417  }
418  }
419  return PerformInsertVectorEltInMemory(Vec, Val, Idx, dl);
420 }
421 
422 SDValue SelectionDAGLegalize::OptimizeFloatStore(StoreSDNode* ST) {
423  if (!ISD::isNormalStore(ST))
424  return SDValue();
425 
426  LLVM_DEBUG(dbgs() << "Optimizing float store operations\n");
427  // Turn 'store float 1.0, Ptr' -> 'store int 0x12345678, Ptr'
428  // FIXME: move this to the DAG Combiner! Note that we can't regress due
429  // to phase ordering between legalized code and the dag combiner. This
430  // probably means that we need to integrate dag combiner and legalizer
431  // together.
432  // We generally can't do this one for long doubles.
433  SDValue Chain = ST->getChain();
434  SDValue Ptr = ST->getBasePtr();
435  SDValue Value = ST->getValue();
436  MachineMemOperand::Flags MMOFlags = ST->getMemOperand()->getFlags();
437  AAMDNodes AAInfo = ST->getAAInfo();
438  SDLoc dl(ST);
439 
440  // Don't optimise TargetConstantFP
441  if (Value.getOpcode() == ISD::TargetConstantFP)
442  return SDValue();
443 
444  if (ConstantFPSDNode *CFP = dyn_cast<ConstantFPSDNode>(Value)) {
445  if (CFP->getValueType(0) == MVT::f32 &&
446  TLI.isTypeLegal(MVT::i32)) {
447  SDValue Con = DAG.getConstant(CFP->getValueAPF().
448  bitcastToAPInt().zextOrTrunc(32),
449  SDLoc(CFP), MVT::i32);
450  return DAG.getStore(Chain, dl, Con, Ptr, ST->getPointerInfo(),
451  ST->getOriginalAlign(), MMOFlags, AAInfo);
452  }
453 
454  if (CFP->getValueType(0) == MVT::f64) {
455  // If this target supports 64-bit registers, do a single 64-bit store.
456  if (TLI.isTypeLegal(MVT::i64)) {
457  SDValue Con = DAG.getConstant(CFP->getValueAPF().bitcastToAPInt().
458  zextOrTrunc(64), SDLoc(CFP), MVT::i64);
459  return DAG.getStore(Chain, dl, Con, Ptr, ST->getPointerInfo(),
460  ST->getOriginalAlign(), MMOFlags, AAInfo);
461  }
462 
463  if (TLI.isTypeLegal(MVT::i32) && !ST->isVolatile()) {
464  // Otherwise, if the target supports 32-bit registers, use 2 32-bit
465  // stores. If the target supports neither 32- nor 64-bits, this
466  // xform is certainly not worth it.
467  const APInt &IntVal = CFP->getValueAPF().bitcastToAPInt();
468  SDValue Lo = DAG.getConstant(IntVal.trunc(32), dl, MVT::i32);
469  SDValue Hi = DAG.getConstant(IntVal.lshr(32).trunc(32), dl, MVT::i32);
470  if (DAG.getDataLayout().isBigEndian())
471  std::swap(Lo, Hi);
472 
473  Lo = DAG.getStore(Chain, dl, Lo, Ptr, ST->getPointerInfo(),
474  ST->getOriginalAlign(), MMOFlags, AAInfo);
475  Ptr = DAG.getMemBasePlusOffset(Ptr, TypeSize::Fixed(4), dl);
476  Hi = DAG.getStore(Chain, dl, Hi, Ptr,
477  ST->getPointerInfo().getWithOffset(4),
478  ST->getOriginalAlign(), MMOFlags, AAInfo);
479 
480  return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo, Hi);
481  }
482  }
483  }
484  return SDValue();
485 }
486 
487 void SelectionDAGLegalize::LegalizeStoreOps(SDNode *Node) {
488  StoreSDNode *ST = cast<StoreSDNode>(Node);
489  SDValue Chain = ST->getChain();
490  SDValue Ptr = ST->getBasePtr();
491  SDLoc dl(Node);
492 
493  MachineMemOperand::Flags MMOFlags = ST->getMemOperand()->getFlags();
494  AAMDNodes AAInfo = ST->getAAInfo();
495 
496  if (!ST->isTruncatingStore()) {
497  LLVM_DEBUG(dbgs() << "Legalizing store operation\n");
498  if (SDNode *OptStore = OptimizeFloatStore(ST).getNode()) {
499  ReplaceNode(ST, OptStore);
500  return;
501  }
502 
503  SDValue Value = ST->getValue();
504  MVT VT = Value.getSimpleValueType();
505  switch (TLI.getOperationAction(ISD::STORE, VT)) {
506  default: llvm_unreachable("This action is not supported yet!");
507  case TargetLowering::Legal: {
508  // If this is an unaligned store and the target doesn't support it,
509  // expand it.
510  EVT MemVT = ST->getMemoryVT();
511  const DataLayout &DL = DAG.getDataLayout();
512  if (!TLI.allowsMemoryAccessForAlignment(*DAG.getContext(), DL, MemVT,
513  *ST->getMemOperand())) {
514  LLVM_DEBUG(dbgs() << "Expanding unsupported unaligned store\n");
515  SDValue Result = TLI.expandUnalignedStore(ST, DAG);
516  ReplaceNode(SDValue(ST, 0), Result);
517  } else
518  LLVM_DEBUG(dbgs() << "Legal store\n");
519  break;
520  }
521  case TargetLowering::Custom: {
522  LLVM_DEBUG(dbgs() << "Trying custom lowering\n");
523  SDValue Res = TLI.LowerOperation(SDValue(Node, 0), DAG);
524  if (Res && Res != SDValue(Node, 0))
525  ReplaceNode(SDValue(Node, 0), Res);
526  return;
527  }
529  MVT NVT = TLI.getTypeToPromoteTo(ISD::STORE, VT);
530  assert(NVT.getSizeInBits() == VT.getSizeInBits() &&
531  "Can only promote stores to same size type");
532  Value = DAG.getNode(ISD::BITCAST, dl, NVT, Value);
533  SDValue Result = DAG.getStore(Chain, dl, Value, Ptr, ST->getPointerInfo(),
534  ST->getOriginalAlign(), MMOFlags, AAInfo);
535  ReplaceNode(SDValue(Node, 0), Result);
536  break;
537  }
538  }
539  return;
540  }
541 
542  LLVM_DEBUG(dbgs() << "Legalizing truncating store operations\n");
543  SDValue Value = ST->getValue();
544  EVT StVT = ST->getMemoryVT();
545  TypeSize StWidth = StVT.getSizeInBits();
546  TypeSize StSize = StVT.getStoreSizeInBits();
547  auto &DL = DAG.getDataLayout();
548 
549  if (StWidth != StSize) {
550  // Promote to a byte-sized store with upper bits zero if not
551  // storing an integral number of bytes. For example, promote
552  // TRUNCSTORE:i1 X -> TRUNCSTORE:i8 (and X, 1)
553  EVT NVT = EVT::getIntegerVT(*DAG.getContext(), StSize.getFixedSize());
554  Value = DAG.getZeroExtendInReg(Value, dl, StVT);
555  SDValue Result =
556  DAG.getTruncStore(Chain, dl, Value, Ptr, ST->getPointerInfo(), NVT,
557  ST->getOriginalAlign(), MMOFlags, AAInfo);
558  ReplaceNode(SDValue(Node, 0), Result);
559  } else if (!StVT.isVector() && !isPowerOf2_64(StWidth.getFixedSize())) {
560  // If not storing a power-of-2 number of bits, expand as two stores.
561  assert(!StVT.isVector() && "Unsupported truncstore!");
562  unsigned StWidthBits = StWidth.getFixedSize();
563  unsigned LogStWidth = Log2_32(StWidthBits);
564  assert(LogStWidth < 32);
565  unsigned RoundWidth = 1 << LogStWidth;
566  assert(RoundWidth < StWidthBits);
567  unsigned ExtraWidth = StWidthBits - RoundWidth;
568  assert(ExtraWidth < RoundWidth);
569  assert(!(RoundWidth % 8) && !(ExtraWidth % 8) &&
570  "Store size not an integral number of bytes!");
571  EVT RoundVT = EVT::getIntegerVT(*DAG.getContext(), RoundWidth);
572  EVT ExtraVT = EVT::getIntegerVT(*DAG.getContext(), ExtraWidth);
573  SDValue Lo, Hi;
574  unsigned IncrementSize;
575 
576  if (DL.isLittleEndian()) {
577  // TRUNCSTORE:i24 X -> TRUNCSTORE:i16 X, TRUNCSTORE@+2:i8 (srl X, 16)
578  // Store the bottom RoundWidth bits.
579  Lo = DAG.getTruncStore(Chain, dl, Value, Ptr, ST->getPointerInfo(),
580  RoundVT, ST->getOriginalAlign(), MMOFlags, AAInfo);
581 
582  // Store the remaining ExtraWidth bits.
583  IncrementSize = RoundWidth / 8;
584  Ptr = DAG.getMemBasePlusOffset(Ptr, TypeSize::Fixed(IncrementSize), dl);
585  Hi = DAG.getNode(
586  ISD::SRL, dl, Value.getValueType(), Value,
587  DAG.getConstant(RoundWidth, dl,
588  TLI.getShiftAmountTy(Value.getValueType(), DL)));
589  Hi = DAG.getTruncStore(Chain, dl, Hi, Ptr,
590  ST->getPointerInfo().getWithOffset(IncrementSize),
591  ExtraVT, ST->getOriginalAlign(), MMOFlags, AAInfo);
592  } else {
593  // Big endian - avoid unaligned stores.
594  // TRUNCSTORE:i24 X -> TRUNCSTORE:i16 (srl X, 8), TRUNCSTORE@+2:i8 X
595  // Store the top RoundWidth bits.
596  Hi = DAG.getNode(
597  ISD::SRL, dl, Value.getValueType(), Value,
598  DAG.getConstant(ExtraWidth, dl,
599  TLI.getShiftAmountTy(Value.getValueType(), DL)));
600  Hi = DAG.getTruncStore(Chain, dl, Hi, Ptr, ST->getPointerInfo(), RoundVT,
601  ST->getOriginalAlign(), MMOFlags, AAInfo);
602 
603  // Store the remaining ExtraWidth bits.
604  IncrementSize = RoundWidth / 8;
605  Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr,
606  DAG.getConstant(IncrementSize, dl,
607  Ptr.getValueType()));
608  Lo = DAG.getTruncStore(Chain, dl, Value, Ptr,
609  ST->getPointerInfo().getWithOffset(IncrementSize),
610  ExtraVT, ST->getOriginalAlign(), MMOFlags, AAInfo);
611  }
612 
613  // The order of the stores doesn't matter.
614  SDValue Result = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo, Hi);
615  ReplaceNode(SDValue(Node, 0), Result);
616  } else {
617  switch (TLI.getTruncStoreAction(ST->getValue().getValueType(), StVT)) {
618  default: llvm_unreachable("This action is not supported yet!");
619  case TargetLowering::Legal: {
620  EVT MemVT = ST->getMemoryVT();
621  // If this is an unaligned store and the target doesn't support it,
622  // expand it.
623  if (!TLI.allowsMemoryAccessForAlignment(*DAG.getContext(), DL, MemVT,
624  *ST->getMemOperand())) {
625  SDValue Result = TLI.expandUnalignedStore(ST, DAG);
626  ReplaceNode(SDValue(ST, 0), Result);
627  }
628  break;
629  }
630  case TargetLowering::Custom: {
631  SDValue Res = TLI.LowerOperation(SDValue(Node, 0), DAG);
632  if (Res && Res != SDValue(Node, 0))
633  ReplaceNode(SDValue(Node, 0), Res);
634  return;
635  }
637  assert(!StVT.isVector() &&
638  "Vector Stores are handled in LegalizeVectorOps");
639 
640  SDValue Result;
641 
642  // TRUNCSTORE:i16 i32 -> STORE i16
643  if (TLI.isTypeLegal(StVT)) {
644  Value = DAG.getNode(ISD::TRUNCATE, dl, StVT, Value);
645  Result = DAG.getStore(Chain, dl, Value, Ptr, ST->getPointerInfo(),
646  ST->getOriginalAlign(), MMOFlags, AAInfo);
647  } else {
648  // The in-memory type isn't legal. Truncate to the type it would promote
649  // to, and then do a truncstore.
650  Value = DAG.getNode(ISD::TRUNCATE, dl,
651  TLI.getTypeToTransformTo(*DAG.getContext(), StVT),
652  Value);
653  Result =
654  DAG.getTruncStore(Chain, dl, Value, Ptr, ST->getPointerInfo(), StVT,
655  ST->getOriginalAlign(), MMOFlags, AAInfo);
656  }
657 
658  ReplaceNode(SDValue(Node, 0), Result);
659  break;
660  }
661  }
662 }
663 
664 void SelectionDAGLegalize::LegalizeLoadOps(SDNode *Node) {
665  LoadSDNode *LD = cast<LoadSDNode>(Node);
666  SDValue Chain = LD->getChain(); // The chain.
667  SDValue Ptr = LD->getBasePtr(); // The base pointer.
668  SDValue Value; // The value returned by the load op.
669  SDLoc dl(Node);
670 
671  ISD::LoadExtType ExtType = LD->getExtensionType();
672  if (ExtType == ISD::NON_EXTLOAD) {
673  LLVM_DEBUG(dbgs() << "Legalizing non-extending load operation\n");
674  MVT VT = Node->getSimpleValueType(0);
675  SDValue RVal = SDValue(Node, 0);
676  SDValue RChain = SDValue(Node, 1);
677 
678  switch (TLI.getOperationAction(Node->getOpcode(), VT)) {
679  default: llvm_unreachable("This action is not supported yet!");
680  case TargetLowering::Legal: {
681  EVT MemVT = LD->getMemoryVT();
682  const DataLayout &DL = DAG.getDataLayout();
683  // If this is an unaligned load and the target doesn't support it,
684  // expand it.
685  if (!TLI.allowsMemoryAccessForAlignment(*DAG.getContext(), DL, MemVT,
686  *LD->getMemOperand())) {
687  std::tie(RVal, RChain) = TLI.expandUnalignedLoad(LD, DAG);
688  }
689  break;
690  }
692  if (SDValue Res = TLI.LowerOperation(RVal, DAG)) {
693  RVal = Res;
694  RChain = Res.getValue(1);
695  }
696  break;
697 
699  MVT NVT = TLI.getTypeToPromoteTo(Node->getOpcode(), VT);
700  assert(NVT.getSizeInBits() == VT.getSizeInBits() &&
701  "Can only promote loads to same size type");
702 
703  SDValue Res = DAG.getLoad(NVT, dl, Chain, Ptr, LD->getMemOperand());
704  RVal = DAG.getNode(ISD::BITCAST, dl, VT, Res);
705  RChain = Res.getValue(1);
706  break;
707  }
708  }
709  if (RChain.getNode() != Node) {
710  assert(RVal.getNode() != Node && "Load must be completely replaced");
711  DAG.ReplaceAllUsesOfValueWith(SDValue(Node, 0), RVal);
712  DAG.ReplaceAllUsesOfValueWith(SDValue(Node, 1), RChain);
713  if (UpdatedNodes) {
714  UpdatedNodes->insert(RVal.getNode());
715  UpdatedNodes->insert(RChain.getNode());
716  }
717  ReplacedNode(Node);
718  }
719  return;
720  }
721 
722  LLVM_DEBUG(dbgs() << "Legalizing extending load operation\n");
723  EVT SrcVT = LD->getMemoryVT();
724  TypeSize SrcWidth = SrcVT.getSizeInBits();
725  MachineMemOperand::Flags MMOFlags = LD->getMemOperand()->getFlags();
726  AAMDNodes AAInfo = LD->getAAInfo();
727 
728  if (SrcWidth != SrcVT.getStoreSizeInBits() &&
729  // Some targets pretend to have an i1 loading operation, and actually
730  // load an i8. This trick is correct for ZEXTLOAD because the top 7
731  // bits are guaranteed to be zero; it helps the optimizers understand
732  // that these bits are zero. It is also useful for EXTLOAD, since it
733  // tells the optimizers that those bits are undefined. It would be
734  // nice to have an effective generic way of getting these benefits...
735  // Until such a way is found, don't insist on promoting i1 here.
736  (SrcVT != MVT::i1 ||
737  TLI.getLoadExtAction(ExtType, Node->getValueType(0), MVT::i1) ==
739  // Promote to a byte-sized load if not loading an integral number of
740  // bytes. For example, promote EXTLOAD:i20 -> EXTLOAD:i24.
741  unsigned NewWidth = SrcVT.getStoreSizeInBits();
742  EVT NVT = EVT::getIntegerVT(*DAG.getContext(), NewWidth);
743  SDValue Ch;
744 
745  // The extra bits are guaranteed to be zero, since we stored them that
746  // way. A zext load from NVT thus automatically gives zext from SrcVT.
747 
748  ISD::LoadExtType NewExtType =
750 
751  SDValue Result = DAG.getExtLoad(NewExtType, dl, Node->getValueType(0),
752  Chain, Ptr, LD->getPointerInfo(), NVT,
753  LD->getOriginalAlign(), MMOFlags, AAInfo);
754 
755  Ch = Result.getValue(1); // The chain.
756 
757  if (ExtType == ISD::SEXTLOAD)
758  // Having the top bits zero doesn't help when sign extending.
759  Result = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl,
760  Result.getValueType(),
761  Result, DAG.getValueType(SrcVT));
762  else if (ExtType == ISD::ZEXTLOAD || NVT == Result.getValueType())
763  // All the top bits are guaranteed to be zero - inform the optimizers.
764  Result = DAG.getNode(ISD::AssertZext, dl,
765  Result.getValueType(), Result,
766  DAG.getValueType(SrcVT));
767 
768  Value = Result;
769  Chain = Ch;
770  } else if (!isPowerOf2_64(SrcWidth.getKnownMinSize())) {
771  // If not loading a power-of-2 number of bits, expand as two loads.
772  assert(!SrcVT.isVector() && "Unsupported extload!");
773  unsigned SrcWidthBits = SrcWidth.getFixedSize();
774  unsigned LogSrcWidth = Log2_32(SrcWidthBits);
775  assert(LogSrcWidth < 32);
776  unsigned RoundWidth = 1 << LogSrcWidth;
777  assert(RoundWidth < SrcWidthBits);
778  unsigned ExtraWidth = SrcWidthBits - RoundWidth;
779  assert(ExtraWidth < RoundWidth);
780  assert(!(RoundWidth % 8) && !(ExtraWidth % 8) &&
781  "Load size not an integral number of bytes!");
782  EVT RoundVT = EVT::getIntegerVT(*DAG.getContext(), RoundWidth);
783  EVT ExtraVT = EVT::getIntegerVT(*DAG.getContext(), ExtraWidth);
784  SDValue Lo, Hi, Ch;
785  unsigned IncrementSize;
786  auto &DL = DAG.getDataLayout();
787 
788  if (DL.isLittleEndian()) {
789  // EXTLOAD:i24 -> ZEXTLOAD:i16 | (shl EXTLOAD@+2:i8, 16)
790  // Load the bottom RoundWidth bits.
791  Lo = DAG.getExtLoad(ISD::ZEXTLOAD, dl, Node->getValueType(0), Chain, Ptr,
792  LD->getPointerInfo(), RoundVT, LD->getOriginalAlign(),
793  MMOFlags, AAInfo);
794 
795  // Load the remaining ExtraWidth bits.
796  IncrementSize = RoundWidth / 8;
797  Ptr = DAG.getMemBasePlusOffset(Ptr, TypeSize::Fixed(IncrementSize), dl);
798  Hi = DAG.getExtLoad(ExtType, dl, Node->getValueType(0), Chain, Ptr,
799  LD->getPointerInfo().getWithOffset(IncrementSize),
800  ExtraVT, LD->getOriginalAlign(), MMOFlags, AAInfo);
801 
802  // Build a factor node to remember that this load is independent of
803  // the other one.
804  Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1),
805  Hi.getValue(1));
806 
807  // Move the top bits to the right place.
808  Hi = DAG.getNode(
809  ISD::SHL, dl, Hi.getValueType(), Hi,
810  DAG.getConstant(RoundWidth, dl,
811  TLI.getShiftAmountTy(Hi.getValueType(), DL)));
812 
813  // Join the hi and lo parts.
814  Value = DAG.getNode(ISD::OR, dl, Node->getValueType(0), Lo, Hi);
815  } else {
816  // Big endian - avoid unaligned loads.
817  // EXTLOAD:i24 -> (shl EXTLOAD:i16, 8) | ZEXTLOAD@+2:i8
818  // Load the top RoundWidth bits.
819  Hi = DAG.getExtLoad(ExtType, dl, Node->getValueType(0), Chain, Ptr,
820  LD->getPointerInfo(), RoundVT, LD->getOriginalAlign(),
821  MMOFlags, AAInfo);
822 
823  // Load the remaining ExtraWidth bits.
824  IncrementSize = RoundWidth / 8;
825  Ptr = DAG.getMemBasePlusOffset(Ptr, TypeSize::Fixed(IncrementSize), dl);
826  Lo = DAG.getExtLoad(ISD::ZEXTLOAD, dl, Node->getValueType(0), Chain, Ptr,
827  LD->getPointerInfo().getWithOffset(IncrementSize),
828  ExtraVT, LD->getOriginalAlign(), MMOFlags, AAInfo);
829 
830  // Build a factor node to remember that this load is independent of
831  // the other one.
832  Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1),
833  Hi.getValue(1));
834 
835  // Move the top bits to the right place.
836  Hi = DAG.getNode(
837  ISD::SHL, dl, Hi.getValueType(), Hi,
838  DAG.getConstant(ExtraWidth, dl,
839  TLI.getShiftAmountTy(Hi.getValueType(), DL)));
840 
841  // Join the hi and lo parts.
842  Value = DAG.getNode(ISD::OR, dl, Node->getValueType(0), Lo, Hi);
843  }
844 
845  Chain = Ch;
846  } else {
847  bool isCustom = false;
848  switch (TLI.getLoadExtAction(ExtType, Node->getValueType(0),
849  SrcVT.getSimpleVT())) {
850  default: llvm_unreachable("This action is not supported yet!");
852  isCustom = true;
855  Value = SDValue(Node, 0);
856  Chain = SDValue(Node, 1);
857 
858  if (isCustom) {
859  if (SDValue Res = TLI.LowerOperation(SDValue(Node, 0), DAG)) {
860  Value = Res;
861  Chain = Res.getValue(1);
862  }
863  } else {
864  // If this is an unaligned load and the target doesn't support it,
865  // expand it.
866  EVT MemVT = LD->getMemoryVT();
867  const DataLayout &DL = DAG.getDataLayout();
868  if (!TLI.allowsMemoryAccess(*DAG.getContext(), DL, MemVT,
869  *LD->getMemOperand())) {
870  std::tie(Value, Chain) = TLI.expandUnalignedLoad(LD, DAG);
871  }
872  }
873  break;
874 
875  case TargetLowering::Expand: {
876  EVT DestVT = Node->getValueType(0);
877  if (!TLI.isLoadExtLegal(ISD::EXTLOAD, DestVT, SrcVT)) {
878  // If the source type is not legal, see if there is a legal extload to
879  // an intermediate type that we can then extend further.
880  EVT LoadVT = TLI.getRegisterType(SrcVT.getSimpleVT());
881  if (TLI.isTypeLegal(SrcVT) || // Same as SrcVT == LoadVT?
882  TLI.isLoadExtLegal(ExtType, LoadVT, SrcVT)) {
883  // If we are loading a legal type, this is a non-extload followed by a
884  // full extend.
885  ISD::LoadExtType MidExtType =
886  (LoadVT == SrcVT) ? ISD::NON_EXTLOAD : ExtType;
887 
888  SDValue Load = DAG.getExtLoad(MidExtType, dl, LoadVT, Chain, Ptr,
889  SrcVT, LD->getMemOperand());
890  unsigned ExtendOp =
891  ISD::getExtForLoadExtType(SrcVT.isFloatingPoint(), ExtType);
892  Value = DAG.getNode(ExtendOp, dl, Node->getValueType(0), Load);
893  Chain = Load.getValue(1);
894  break;
895  }
896 
897  // Handle the special case of fp16 extloads. EXTLOAD doesn't have the
898  // normal undefined upper bits behavior to allow using an in-reg extend
899  // with the illegal FP type, so load as an integer and do the
900  // from-integer conversion.
901  if (SrcVT.getScalarType() == MVT::f16) {
902  EVT ISrcVT = SrcVT.changeTypeToInteger();
903  EVT IDestVT = DestVT.changeTypeToInteger();
904  EVT ILoadVT = TLI.getRegisterType(IDestVT.getSimpleVT());
905 
906  SDValue Result = DAG.getExtLoad(ISD::ZEXTLOAD, dl, ILoadVT, Chain,
907  Ptr, ISrcVT, LD->getMemOperand());
908  Value = DAG.getNode(ISD::FP16_TO_FP, dl, DestVT, Result);
909  Chain = Result.getValue(1);
910  break;
911  }
912  }
913 
914  assert(!SrcVT.isVector() &&
915  "Vector Loads are handled in LegalizeVectorOps");
916 
917  // FIXME: This does not work for vectors on most targets. Sign-
918  // and zero-extend operations are currently folded into extending
919  // loads, whether they are legal or not, and then we end up here
920  // without any support for legalizing them.
921  assert(ExtType != ISD::EXTLOAD &&
922  "EXTLOAD should always be supported!");
923  // Turn the unsupported load into an EXTLOAD followed by an
924  // explicit zero/sign extend inreg.
925  SDValue Result = DAG.getExtLoad(ISD::EXTLOAD, dl,
926  Node->getValueType(0),
927  Chain, Ptr, SrcVT,
928  LD->getMemOperand());
929  SDValue ValRes;
930  if (ExtType == ISD::SEXTLOAD)
931  ValRes = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl,
932  Result.getValueType(),
933  Result, DAG.getValueType(SrcVT));
934  else
935  ValRes = DAG.getZeroExtendInReg(Result, dl, SrcVT);
936  Value = ValRes;
937  Chain = Result.getValue(1);
938  break;
939  }
940  }
941  }
942 
943  // Since loads produce two values, make sure to remember that we legalized
944  // both of them.
945  if (Chain.getNode() != Node) {
946  assert(Value.getNode() != Node && "Load must be completely replaced");
947  DAG.ReplaceAllUsesOfValueWith(SDValue(Node, 0), Value);
948  DAG.ReplaceAllUsesOfValueWith(SDValue(Node, 1), Chain);
949  if (UpdatedNodes) {
950  UpdatedNodes->insert(Value.getNode());
951  UpdatedNodes->insert(Chain.getNode());
952  }
953  ReplacedNode(Node);
954  }
955 }
956 
957 /// Return a legal replacement for the given operation, with all legal operands.
958 void SelectionDAGLegalize::LegalizeOp(SDNode *Node) {
959  LLVM_DEBUG(dbgs() << "\nLegalizing: "; Node->dump(&DAG));
960 
961  // Allow illegal target nodes and illegal registers.
962  if (Node->getOpcode() == ISD::TargetConstant ||
963  Node->getOpcode() == ISD::Register)
964  return;
965 
966 #ifndef NDEBUG
967  for (unsigned i = 0, e = Node->getNumValues(); i != e; ++i)
968  assert(TLI.getTypeAction(*DAG.getContext(), Node->getValueType(i)) ==
970  "Unexpected illegal type!");
971 
972  for (const SDValue &Op : Node->op_values())
973  assert((TLI.getTypeAction(*DAG.getContext(), Op.getValueType()) ==
975  Op.getOpcode() == ISD::TargetConstant ||
976  Op.getOpcode() == ISD::Register) &&
977  "Unexpected illegal type!");
978 #endif
979 
980  // Figure out the correct action; the way to query this varies by opcode
982  bool SimpleFinishLegalizing = true;
983  switch (Node->getOpcode()) {
986  case ISD::INTRINSIC_VOID:
987  case ISD::STACKSAVE:
988  Action = TLI.getOperationAction(Node->getOpcode(), MVT::Other);
989  break;
991  Action = TLI.getOperationAction(Node->getOpcode(),
992  Node->getValueType(0));
993  break;
994  case ISD::VAARG:
995  Action = TLI.getOperationAction(Node->getOpcode(),
996  Node->getValueType(0));
997  if (Action != TargetLowering::Promote)
998  Action = TLI.getOperationAction(Node->getOpcode(), MVT::Other);
999  break;
1000  case ISD::FP_TO_FP16:
1001  case ISD::SINT_TO_FP:
1002  case ISD::UINT_TO_FP:
1004  case ISD::LROUND:
1005  case ISD::LLROUND:
1006  case ISD::LRINT:
1007  case ISD::LLRINT:
1008  Action = TLI.getOperationAction(Node->getOpcode(),
1009  Node->getOperand(0).getValueType());
1010  break;
1014  case ISD::STRICT_LRINT:
1015  case ISD::STRICT_LLRINT:
1016  case ISD::STRICT_LROUND:
1017  case ISD::STRICT_LLROUND:
1018  // These pseudo-ops are the same as the other STRICT_ ops except
1019  // they are registered with setOperationAction() using the input type
1020  // instead of the output type.
1021  Action = TLI.getOperationAction(Node->getOpcode(),
1022  Node->getOperand(1).getValueType());
1023  break;
1024  case ISD::SIGN_EXTEND_INREG: {
1025  EVT InnerType = cast<VTSDNode>(Node->getOperand(1))->getVT();
1026  Action = TLI.getOperationAction(Node->getOpcode(), InnerType);
1027  break;
1028  }
1029  case ISD::ATOMIC_STORE:
1030  Action = TLI.getOperationAction(Node->getOpcode(),
1031  Node->getOperand(2).getValueType());
1032  break;
1033  case ISD::SELECT_CC:
1034  case ISD::STRICT_FSETCC:
1035  case ISD::STRICT_FSETCCS:
1036  case ISD::SETCC:
1037  case ISD::BR_CC: {
1038  unsigned CCOperand = Node->getOpcode() == ISD::SELECT_CC ? 4 :
1039  Node->getOpcode() == ISD::STRICT_FSETCC ? 3 :
1040  Node->getOpcode() == ISD::STRICT_FSETCCS ? 3 :
1041  Node->getOpcode() == ISD::SETCC ? 2 : 1;
1042  unsigned CompareOperand = Node->getOpcode() == ISD::BR_CC ? 2 :
1043  Node->getOpcode() == ISD::STRICT_FSETCC ? 1 :
1044  Node->getOpcode() == ISD::STRICT_FSETCCS ? 1 : 0;
1045  MVT OpVT = Node->getOperand(CompareOperand).getSimpleValueType();
1046  ISD::CondCode CCCode =
1047  cast<CondCodeSDNode>(Node->getOperand(CCOperand))->get();
1048  Action = TLI.getCondCodeAction(CCCode, OpVT);
1049  if (Action == TargetLowering::Legal) {
1050  if (Node->getOpcode() == ISD::SELECT_CC)
1051  Action = TLI.getOperationAction(Node->getOpcode(),
1052  Node->getValueType(0));
1053  else
1054  Action = TLI.getOperationAction(Node->getOpcode(), OpVT);
1055  }
1056  break;
1057  }
1058  case ISD::LOAD:
1059  case ISD::STORE:
1060  // FIXME: Model these properly. LOAD and STORE are complicated, and
1061  // STORE expects the unlegalized operand in some cases.
1062  SimpleFinishLegalizing = false;
1063  break;
1064  case ISD::CALLSEQ_START:
1065  case ISD::CALLSEQ_END:
1066  // FIXME: This shouldn't be necessary. These nodes have special properties
1067  // dealing with the recursive nature of legalization. Removing this
1068  // special case should be done as part of making LegalizeDAG non-recursive.
1069  SimpleFinishLegalizing = false;
1070  break;
1071  case ISD::EXTRACT_ELEMENT:
1072  case ISD::FLT_ROUNDS_:
1073  case ISD::MERGE_VALUES:
1074  case ISD::EH_RETURN:
1076  case ISD::EH_DWARF_CFA:
1077  case ISD::EH_SJLJ_SETJMP:
1078  case ISD::EH_SJLJ_LONGJMP:
1080  // These operations lie about being legal: when they claim to be legal,
1081  // they should actually be expanded.
1082  Action = TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0));
1083  if (Action == TargetLowering::Legal)
1084  Action = TargetLowering::Expand;
1085  break;
1086  case ISD::INIT_TRAMPOLINE:
1088  case ISD::FRAMEADDR:
1089  case ISD::RETURNADDR:
1090  case ISD::ADDROFRETURNADDR:
1091  case ISD::SPONENTRY:
1092  // These operations lie about being legal: when they claim to be legal,
1093  // they should actually be custom-lowered.
1094  Action = TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0));
1095  if (Action == TargetLowering::Legal)
1096  Action = TargetLowering::Custom;
1097  break;
1098  case ISD::READCYCLECOUNTER:
1099  // READCYCLECOUNTER returns an i64, even if type legalization might have
1100  // expanded that to several smaller types.
1101  Action = TLI.getOperationAction(Node->getOpcode(), MVT::i64);
1102  break;
1103  case ISD::READ_REGISTER:
1104  case ISD::WRITE_REGISTER:
1105  // Named register is legal in the DAG, but blocked by register name
1106  // selection if not implemented by target (to chose the correct register)
1107  // They'll be converted to Copy(To/From)Reg.
1108  Action = TargetLowering::Legal;
1109  break;
1110  case ISD::UBSANTRAP:
1111  Action = TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0));
1112  if (Action == TargetLowering::Expand) {
1113  // replace ISD::UBSANTRAP with ISD::TRAP
1114  SDValue NewVal;
1115  NewVal = DAG.getNode(ISD::TRAP, SDLoc(Node), Node->getVTList(),
1116  Node->getOperand(0));
1117  ReplaceNode(Node, NewVal.getNode());
1118  LegalizeOp(NewVal.getNode());
1119  return;
1120  }
1121  break;
1122  case ISD::DEBUGTRAP:
1123  Action = TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0));
1124  if (Action == TargetLowering::Expand) {
1125  // replace ISD::DEBUGTRAP with ISD::TRAP
1126  SDValue NewVal;
1127  NewVal = DAG.getNode(ISD::TRAP, SDLoc(Node), Node->getVTList(),
1128  Node->getOperand(0));
1129  ReplaceNode(Node, NewVal.getNode());
1130  LegalizeOp(NewVal.getNode());
1131  return;
1132  }
1133  break;
1134  case ISD::SADDSAT:
1135  case ISD::UADDSAT:
1136  case ISD::SSUBSAT:
1137  case ISD::USUBSAT:
1138  case ISD::SSHLSAT:
1139  case ISD::USHLSAT:
1140  case ISD::FP_TO_SINT_SAT:
1141  case ISD::FP_TO_UINT_SAT:
1142  Action = TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0));
1143  break;
1144  case ISD::SMULFIX:
1145  case ISD::SMULFIXSAT:
1146  case ISD::UMULFIX:
1147  case ISD::UMULFIXSAT:
1148  case ISD::SDIVFIX:
1149  case ISD::SDIVFIXSAT:
1150  case ISD::UDIVFIX:
1151  case ISD::UDIVFIXSAT: {
1152  unsigned Scale = Node->getConstantOperandVal(2);
1153  Action = TLI.getFixedPointOperationAction(Node->getOpcode(),
1154  Node->getValueType(0), Scale);
1155  break;
1156  }
1157  case ISD::MSCATTER:
1158  Action = TLI.getOperationAction(Node->getOpcode(),
1159  cast<MaskedScatterSDNode>(Node)->getValue().getValueType());
1160  break;
1161  case ISD::MSTORE:
1162  Action = TLI.getOperationAction(Node->getOpcode(),
1163  cast<MaskedStoreSDNode>(Node)->getValue().getValueType());
1164  break;
1165  case ISD::VECREDUCE_FADD:
1166  case ISD::VECREDUCE_FMUL:
1167  case ISD::VECREDUCE_ADD:
1168  case ISD::VECREDUCE_MUL:
1169  case ISD::VECREDUCE_AND:
1170  case ISD::VECREDUCE_OR:
1171  case ISD::VECREDUCE_XOR:
1172  case ISD::VECREDUCE_SMAX:
1173  case ISD::VECREDUCE_SMIN:
1174  case ISD::VECREDUCE_UMAX:
1175  case ISD::VECREDUCE_UMIN:
1176  case ISD::VECREDUCE_FMAX:
1177  case ISD::VECREDUCE_FMIN:
1178  Action = TLI.getOperationAction(
1179  Node->getOpcode(), Node->getOperand(0).getValueType());
1180  break;
1182  Action = TLI.getOperationAction(
1183  Node->getOpcode(), Node->getOperand(1).getValueType());
1184  break;
1185  default:
1186  if (Node->getOpcode() >= ISD::BUILTIN_OP_END) {
1187  Action = TargetLowering::Legal;
1188  } else {
1189  Action = TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0));
1190  }
1191  break;
1192  }
1193 
1194  if (SimpleFinishLegalizing) {
1195  SDNode *NewNode = Node;
1196  switch (Node->getOpcode()) {
1197  default: break;
1198  case ISD::SHL:
1199  case ISD::SRL:
1200  case ISD::SRA:
1201  case ISD::ROTL:
1202  case ISD::ROTR: {
1203  // Legalizing shifts/rotates requires adjusting the shift amount
1204  // to the appropriate width.
1205  SDValue Op0 = Node->getOperand(0);
1206  SDValue Op1 = Node->getOperand(1);
1207  if (!Op1.getValueType().isVector()) {
1208  SDValue SAO = DAG.getShiftAmountOperand(Op0.getValueType(), Op1);
1209  // The getShiftAmountOperand() may create a new operand node or
1210  // return the existing one. If new operand is created we need
1211  // to update the parent node.
1212  // Do not try to legalize SAO here! It will be automatically legalized
1213  // in the next round.
1214  if (SAO != Op1)
1215  NewNode = DAG.UpdateNodeOperands(Node, Op0, SAO);
1216  }
1217  }
1218  break;
1219  case ISD::FSHL:
1220  case ISD::FSHR:
1221  case ISD::SRL_PARTS:
1222  case ISD::SRA_PARTS:
1223  case ISD::SHL_PARTS: {
1224  // Legalizing shifts/rotates requires adjusting the shift amount
1225  // to the appropriate width.
1226  SDValue Op0 = Node->getOperand(0);
1227  SDValue Op1 = Node->getOperand(1);
1228  SDValue Op2 = Node->getOperand(2);
1229  if (!Op2.getValueType().isVector()) {
1230  SDValue SAO = DAG.getShiftAmountOperand(Op0.getValueType(), Op2);
1231  // The getShiftAmountOperand() may create a new operand node or
1232  // return the existing one. If new operand is created we need
1233  // to update the parent node.
1234  if (SAO != Op2)
1235  NewNode = DAG.UpdateNodeOperands(Node, Op0, Op1, SAO);
1236  }
1237  break;
1238  }
1239  }
1240 
1241  if (NewNode != Node) {
1242  ReplaceNode(Node, NewNode);
1243  Node = NewNode;
1244  }
1245  switch (Action) {
1246  case TargetLowering::Legal:
1247  LLVM_DEBUG(dbgs() << "Legal node: nothing to do\n");
1248  return;
1250  LLVM_DEBUG(dbgs() << "Trying custom legalization\n");
1251  // FIXME: The handling for custom lowering with multiple results is
1252  // a complete mess.
1253  if (SDValue Res = TLI.LowerOperation(SDValue(Node, 0), DAG)) {
1254  if (!(Res.getNode() != Node || Res.getResNo() != 0))
1255  return;
1256 
1257  if (Node->getNumValues() == 1) {
1258  // Verify the new types match the original. Glue is waived because
1259  // ISD::ADDC can be legalized by replacing Glue with an integer type.
1260  assert((Res.getValueType() == Node->getValueType(0) ||
1261  Node->getValueType(0) == MVT::Glue) &&
1262  "Type mismatch for custom legalized operation");
1263  LLVM_DEBUG(dbgs() << "Successfully custom legalized node\n");
1264  // We can just directly replace this node with the lowered value.
1265  ReplaceNode(SDValue(Node, 0), Res);
1266  return;
1267  }
1268 
1269  SmallVector<SDValue, 8> ResultVals;
1270  for (unsigned i = 0, e = Node->getNumValues(); i != e; ++i) {
1271  // Verify the new types match the original. Glue is waived because
1272  // ISD::ADDC can be legalized by replacing Glue with an integer type.
1273  assert((Res->getValueType(i) == Node->getValueType(i) ||
1274  Node->getValueType(i) == MVT::Glue) &&
1275  "Type mismatch for custom legalized operation");
1276  ResultVals.push_back(Res.getValue(i));
1277  }
1278  LLVM_DEBUG(dbgs() << "Successfully custom legalized node\n");
1279  ReplaceNode(Node, ResultVals.data());
1280  return;
1281  }
1282  LLVM_DEBUG(dbgs() << "Could not custom legalize node\n");
1285  if (ExpandNode(Node))
1286  return;
1289  ConvertNodeToLibcall(Node);
1290  return;
1292  PromoteNode(Node);
1293  return;
1294  }
1295  }
1296 
1297  switch (Node->getOpcode()) {
1298  default:
1299 #ifndef NDEBUG
1300  dbgs() << "NODE: ";
1301  Node->dump( &DAG);
1302  dbgs() << "\n";
1303 #endif
1304  llvm_unreachable("Do not know how to legalize this operator!");
1305 
1306  case ISD::CALLSEQ_START:
1307  case ISD::CALLSEQ_END:
1308  break;
1309  case ISD::LOAD:
1310  return LegalizeLoadOps(Node);
1311  case ISD::STORE:
1312  return LegalizeStoreOps(Node);
1313  }
1314 }
1315 
1316 SDValue SelectionDAGLegalize::ExpandExtractFromVectorThroughStack(SDValue Op) {
1317  SDValue Vec = Op.getOperand(0);
1318  SDValue Idx = Op.getOperand(1);
1319  SDLoc dl(Op);
1320 
1321  // Before we generate a new store to a temporary stack slot, see if there is
1322  // already one that we can use. There often is because when we scalarize
1323  // vector operations (using SelectionDAG::UnrollVectorOp for example) a whole
1324  // series of EXTRACT_VECTOR_ELT nodes are generated, one for each element in
1325  // the vector. If all are expanded here, we don't want one store per vector
1326  // element.
1327 
1328  // Caches for hasPredecessorHelper
1331  Visited.insert(Op.getNode());
1332  Worklist.push_back(Idx.getNode());
1333  SDValue StackPtr, Ch;
1334  for (SDNode::use_iterator UI = Vec.getNode()->use_begin(),
1335  UE = Vec.getNode()->use_end(); UI != UE; ++UI) {
1336  SDNode *User = *UI;
1337  if (StoreSDNode *ST = dyn_cast<StoreSDNode>(User)) {
1338  if (ST->isIndexed() || ST->isTruncatingStore() ||
1339  ST->getValue() != Vec)
1340  continue;
1341 
1342  // Make sure that nothing else could have stored into the destination of
1343  // this store.
1344  if (!ST->getChain().reachesChainWithoutSideEffects(DAG.getEntryNode()))
1345  continue;
1346 
1347  // If the index is dependent on the store we will introduce a cycle when
1348  // creating the load (the load uses the index, and by replacing the chain
1349  // we will make the index dependent on the load). Also, the store might be
1350  // dependent on the extractelement and introduce a cycle when creating
1351  // the load.
1352  if (SDNode::hasPredecessorHelper(ST, Visited, Worklist) ||
1353  ST->hasPredecessor(Op.getNode()))
1354  continue;
1355 
1356  StackPtr = ST->getBasePtr();
1357  Ch = SDValue(ST, 0);
1358  break;
1359  }
1360  }
1361 
1362  EVT VecVT = Vec.getValueType();
1363 
1364  if (!Ch.getNode()) {
1365  // Store the value to a temporary stack slot, then LOAD the returned part.
1366  StackPtr = DAG.CreateStackTemporary(VecVT);
1367  Ch = DAG.getStore(DAG.getEntryNode(), dl, Vec, StackPtr,
1368  MachinePointerInfo());
1369  }
1370 
1371  StackPtr = TLI.getVectorElementPointer(DAG, StackPtr, VecVT, Idx);
1372 
1373  SDValue NewLoad;
1374 
1375  if (Op.getValueType().isVector())
1376  NewLoad =
1377  DAG.getLoad(Op.getValueType(), dl, Ch, StackPtr, MachinePointerInfo());
1378  else
1379  NewLoad = DAG.getExtLoad(ISD::EXTLOAD, dl, Op.getValueType(), Ch, StackPtr,
1381  VecVT.getVectorElementType());
1382 
1383  // Replace the chain going out of the store, by the one out of the load.
1384  DAG.ReplaceAllUsesOfValueWith(Ch, SDValue(NewLoad.getNode(), 1));
1385 
1386  // We introduced a cycle though, so update the loads operands, making sure
1387  // to use the original store's chain as an incoming chain.
1388  SmallVector<SDValue, 6> NewLoadOperands(NewLoad->op_begin(),
1389  NewLoad->op_end());
1390  NewLoadOperands[0] = Ch;
1391  NewLoad =
1392  SDValue(DAG.UpdateNodeOperands(NewLoad.getNode(), NewLoadOperands), 0);
1393  return NewLoad;
1394 }
1395 
1396 SDValue SelectionDAGLegalize::ExpandInsertToVectorThroughStack(SDValue Op) {
1397  assert(Op.getValueType().isVector() && "Non-vector insert subvector!");
1398 
1399  SDValue Vec = Op.getOperand(0);
1400  SDValue Part = Op.getOperand(1);
1401  SDValue Idx = Op.getOperand(2);
1402  SDLoc dl(Op);
1403 
1404  // Store the value to a temporary stack slot, then LOAD the returned part.
1405  EVT VecVT = Vec.getValueType();
1406  SDValue StackPtr = DAG.CreateStackTemporary(VecVT);
1407  int FI = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex();
1408  MachinePointerInfo PtrInfo =
1409  MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), FI);
1410 
1411  // First store the whole vector.
1412  SDValue Ch = DAG.getStore(DAG.getEntryNode(), dl, Vec, StackPtr, PtrInfo);
1413 
1414  // Then store the inserted part.
1415  SDValue SubStackPtr = TLI.getVectorElementPointer(DAG, StackPtr, VecVT, Idx);
1416 
1417  // Store the subvector.
1418  Ch = DAG.getStore(
1419  Ch, dl, Part, SubStackPtr,
1420  MachinePointerInfo::getUnknownStack(DAG.getMachineFunction()));
1421 
1422  // Finally, load the updated vector.
1423  return DAG.getLoad(Op.getValueType(), dl, Ch, StackPtr, PtrInfo);
1424 }
1425 
1426 SDValue SelectionDAGLegalize::ExpandVectorBuildThroughStack(SDNode* Node) {
1427  assert((Node->getOpcode() == ISD::BUILD_VECTOR ||
1428  Node->getOpcode() == ISD::CONCAT_VECTORS) &&
1429  "Unexpected opcode!");
1430 
1431  // We can't handle this case efficiently. Allocate a sufficiently
1432  // aligned object on the stack, store each operand into it, then load
1433  // the result as a vector.
1434  // Create the stack frame object.
1435  EVT VT = Node->getValueType(0);
1436  EVT MemVT = isa<BuildVectorSDNode>(Node) ? VT.getVectorElementType()
1437  : Node->getOperand(0).getValueType();
1438  SDLoc dl(Node);
1439  SDValue FIPtr = DAG.CreateStackTemporary(VT);
1440  int FI = cast<FrameIndexSDNode>(FIPtr.getNode())->getIndex();
1441  MachinePointerInfo PtrInfo =
1442  MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), FI);
1443 
1444  // Emit a store of each element to the stack slot.
1445  SmallVector<SDValue, 8> Stores;
1446  unsigned TypeByteSize = MemVT.getSizeInBits() / 8;
1447  assert(TypeByteSize > 0 && "Vector element type too small for stack store!");
1448 
1449  // If the destination vector element type of a BUILD_VECTOR is narrower than
1450  // the source element type, only store the bits necessary.
1451  bool Truncate = isa<BuildVectorSDNode>(Node) &&
1452  MemVT.bitsLT(Node->getOperand(0).getValueType());
1453 
1454  // Store (in the right endianness) the elements to memory.
1455  for (unsigned i = 0, e = Node->getNumOperands(); i != e; ++i) {
1456  // Ignore undef elements.
1457  if (Node->getOperand(i).isUndef()) continue;
1458 
1459  unsigned Offset = TypeByteSize*i;
1460 
1461  SDValue Idx = DAG.getMemBasePlusOffset(FIPtr, TypeSize::Fixed(Offset), dl);
1462 
1463  if (Truncate)
1464  Stores.push_back(DAG.getTruncStore(DAG.getEntryNode(), dl,
1465  Node->getOperand(i), Idx,
1466  PtrInfo.getWithOffset(Offset), MemVT));
1467  else
1468  Stores.push_back(DAG.getStore(DAG.getEntryNode(), dl, Node->getOperand(i),
1469  Idx, PtrInfo.getWithOffset(Offset)));
1470  }
1471 
1472  SDValue StoreChain;
1473  if (!Stores.empty()) // Not all undef elements?
1474  StoreChain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Stores);
1475  else
1476  StoreChain = DAG.getEntryNode();
1477 
1478  // Result is a load from the stack slot.
1479  return DAG.getLoad(VT, dl, StoreChain, FIPtr, PtrInfo);
1480 }
1481 
1482 /// Bitcast a floating-point value to an integer value. Only bitcast the part
1483 /// containing the sign bit if the target has no integer value capable of
1484 /// holding all bits of the floating-point value.
1485 void SelectionDAGLegalize::getSignAsIntValue(FloatSignAsInt &State,
1486  const SDLoc &DL,
1487  SDValue Value) const {
1488  EVT FloatVT = Value.getValueType();
1489  unsigned NumBits = FloatVT.getScalarSizeInBits();
1490  State.FloatVT = FloatVT;
1491  EVT IVT = EVT::getIntegerVT(*DAG.getContext(), NumBits);
1492  // Convert to an integer of the same size.
1493  if (TLI.isTypeLegal(IVT)) {
1494  State.IntValue = DAG.getNode(ISD::BITCAST, DL, IVT, Value);
1495  State.SignMask = APInt::getSignMask(NumBits);
1496  State.SignBit = NumBits - 1;
1497  return;
1498  }
1499 
1500  auto &DataLayout = DAG.getDataLayout();
1501  // Store the float to memory, then load the sign part out as an integer.
1502  MVT LoadTy = TLI.getRegisterType(*DAG.getContext(), MVT::i8);
1503  // First create a temporary that is aligned for both the load and store.
1504  SDValue StackPtr = DAG.CreateStackTemporary(FloatVT, LoadTy);
1505  int FI = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex();
1506  // Then store the float to it.
1507  State.FloatPtr = StackPtr;
1508  MachineFunction &MF = DAG.getMachineFunction();
1509  State.FloatPointerInfo = MachinePointerInfo::getFixedStack(MF, FI);
1510  State.Chain = DAG.getStore(DAG.getEntryNode(), DL, Value, State.FloatPtr,
1511  State.FloatPointerInfo);
1512 
1513  SDValue IntPtr;
1514  if (DataLayout.isBigEndian()) {
1515  assert(FloatVT.isByteSized() && "Unsupported floating point type!");
1516  // Load out a legal integer with the same sign bit as the float.
1517  IntPtr = StackPtr;
1518  State.IntPointerInfo = State.FloatPointerInfo;
1519  } else {
1520  // Advance the pointer so that the loaded byte will contain the sign bit.
1521  unsigned ByteOffset = (NumBits / 8) - 1;
1522  IntPtr =
1523  DAG.getMemBasePlusOffset(StackPtr, TypeSize::Fixed(ByteOffset), DL);
1524  State.IntPointerInfo = MachinePointerInfo::getFixedStack(MF, FI,
1525  ByteOffset);
1526  }
1527 
1528  State.IntPtr = IntPtr;
1529  State.IntValue = DAG.getExtLoad(ISD::EXTLOAD, DL, LoadTy, State.Chain, IntPtr,
1530  State.IntPointerInfo, MVT::i8);
1531  State.SignMask = APInt::getOneBitSet(LoadTy.getScalarSizeInBits(), 7);
1532  State.SignBit = 7;
1533 }
1534 
1535 /// Replace the integer value produced by getSignAsIntValue() with a new value
1536 /// and cast the result back to a floating-point type.
1537 SDValue SelectionDAGLegalize::modifySignAsInt(const FloatSignAsInt &State,
1538  const SDLoc &DL,
1539  SDValue NewIntValue) const {
1540  if (!State.Chain)
1541  return DAG.getNode(ISD::BITCAST, DL, State.FloatVT, NewIntValue);
1542 
1543  // Override the part containing the sign bit in the value stored on the stack.
1544  SDValue Chain = DAG.getTruncStore(State.Chain, DL, NewIntValue, State.IntPtr,
1545  State.IntPointerInfo, MVT::i8);
1546  return DAG.getLoad(State.FloatVT, DL, Chain, State.FloatPtr,
1547  State.FloatPointerInfo);
1548 }
1549 
1550 SDValue SelectionDAGLegalize::ExpandFCOPYSIGN(SDNode *Node) const {
1551  SDLoc DL(Node);
1552  SDValue Mag = Node->getOperand(0);
1553  SDValue Sign = Node->getOperand(1);
1554 
1555  // Get sign bit into an integer value.
1556  FloatSignAsInt SignAsInt;
1557  getSignAsIntValue(SignAsInt, DL, Sign);
1558 
1559  EVT IntVT = SignAsInt.IntValue.getValueType();
1560  SDValue SignMask = DAG.getConstant(SignAsInt.SignMask, DL, IntVT);
1561  SDValue SignBit = DAG.getNode(ISD::AND, DL, IntVT, SignAsInt.IntValue,
1562  SignMask);
1563 
1564  // If FABS is legal transform FCOPYSIGN(x, y) => sign(x) ? -FABS(x) : FABS(X)
1565  EVT FloatVT = Mag.getValueType();
1566  if (TLI.isOperationLegalOrCustom(ISD::FABS, FloatVT) &&
1567  TLI.isOperationLegalOrCustom(ISD::FNEG, FloatVT)) {
1568  SDValue AbsValue = DAG.getNode(ISD::FABS, DL, FloatVT, Mag);
1569  SDValue NegValue = DAG.getNode(ISD::FNEG, DL, FloatVT, AbsValue);
1570  SDValue Cond = DAG.getSetCC(DL, getSetCCResultType(IntVT), SignBit,
1571  DAG.getConstant(0, DL, IntVT), ISD::SETNE);
1572  return DAG.getSelect(DL, FloatVT, Cond, NegValue, AbsValue);
1573  }
1574 
1575  // Transform Mag value to integer, and clear the sign bit.
1576  FloatSignAsInt MagAsInt;
1577  getSignAsIntValue(MagAsInt, DL, Mag);
1578  EVT MagVT = MagAsInt.IntValue.getValueType();
1579  SDValue ClearSignMask = DAG.getConstant(~MagAsInt.SignMask, DL, MagVT);
1580  SDValue ClearedSign = DAG.getNode(ISD::AND, DL, MagVT, MagAsInt.IntValue,
1581  ClearSignMask);
1582 
1583  // Get the signbit at the right position for MagAsInt.
1584  int ShiftAmount = SignAsInt.SignBit - MagAsInt.SignBit;
1585  EVT ShiftVT = IntVT;
1586  if (SignBit.getScalarValueSizeInBits() <
1587  ClearedSign.getScalarValueSizeInBits()) {
1588  SignBit = DAG.getNode(ISD::ZERO_EXTEND, DL, MagVT, SignBit);
1589  ShiftVT = MagVT;
1590  }
1591  if (ShiftAmount > 0) {
1592  SDValue ShiftCnst = DAG.getConstant(ShiftAmount, DL, ShiftVT);
1593  SignBit = DAG.getNode(ISD::SRL, DL, ShiftVT, SignBit, ShiftCnst);
1594  } else if (ShiftAmount < 0) {
1595  SDValue ShiftCnst = DAG.getConstant(-ShiftAmount, DL, ShiftVT);
1596  SignBit = DAG.getNode(ISD::SHL, DL, ShiftVT, SignBit, ShiftCnst);
1597  }
1598  if (SignBit.getScalarValueSizeInBits() >
1599  ClearedSign.getScalarValueSizeInBits()) {
1600  SignBit = DAG.getNode(ISD::TRUNCATE, DL, MagVT, SignBit);
1601  }
1602 
1603  // Store the part with the modified sign and convert back to float.
1604  SDValue CopiedSign = DAG.getNode(ISD::OR, DL, MagVT, ClearedSign, SignBit);
1605  return modifySignAsInt(MagAsInt, DL, CopiedSign);
1606 }
1607 
1608 SDValue SelectionDAGLegalize::ExpandFNEG(SDNode *Node) const {
1609  // Get the sign bit as an integer.
1610  SDLoc DL(Node);
1611  FloatSignAsInt SignAsInt;
1612  getSignAsIntValue(SignAsInt, DL, Node->getOperand(0));
1613  EVT IntVT = SignAsInt.IntValue.getValueType();
1614 
1615  // Flip the sign.
1616  SDValue SignMask = DAG.getConstant(SignAsInt.SignMask, DL, IntVT);
1617  SDValue SignFlip =
1618  DAG.getNode(ISD::XOR, DL, IntVT, SignAsInt.IntValue, SignMask);
1619 
1620  // Convert back to float.
1621  return modifySignAsInt(SignAsInt, DL, SignFlip);
1622 }
1623 
1624 SDValue SelectionDAGLegalize::ExpandFABS(SDNode *Node) const {
1625  SDLoc DL(Node);
1626  SDValue Value = Node->getOperand(0);
1627 
1628  // Transform FABS(x) => FCOPYSIGN(x, 0.0) if FCOPYSIGN is legal.
1629  EVT FloatVT = Value.getValueType();
1630  if (TLI.isOperationLegalOrCustom(ISD::FCOPYSIGN, FloatVT)) {
1631  SDValue Zero = DAG.getConstantFP(0.0, DL, FloatVT);
1632  return DAG.getNode(ISD::FCOPYSIGN, DL, FloatVT, Value, Zero);
1633  }
1634 
1635  // Transform value to integer, clear the sign bit and transform back.
1636  FloatSignAsInt ValueAsInt;
1637  getSignAsIntValue(ValueAsInt, DL, Value);
1638  EVT IntVT = ValueAsInt.IntValue.getValueType();
1639  SDValue ClearSignMask = DAG.getConstant(~ValueAsInt.SignMask, DL, IntVT);
1640  SDValue ClearedSign = DAG.getNode(ISD::AND, DL, IntVT, ValueAsInt.IntValue,
1641  ClearSignMask);
1642  return modifySignAsInt(ValueAsInt, DL, ClearedSign);
1643 }
1644 
1645 void SelectionDAGLegalize::ExpandDYNAMIC_STACKALLOC(SDNode* Node,
1647  Register SPReg = TLI.getStackPointerRegisterToSaveRestore();
1648  assert(SPReg && "Target cannot require DYNAMIC_STACKALLOC expansion and"
1649  " not tell us which reg is the stack pointer!");
1650  SDLoc dl(Node);
1651  EVT VT = Node->getValueType(0);
1652  SDValue Tmp1 = SDValue(Node, 0);
1653  SDValue Tmp2 = SDValue(Node, 1);
1654  SDValue Tmp3 = Node->getOperand(2);
1655  SDValue Chain = Tmp1.getOperand(0);
1656 
1657  // Chain the dynamic stack allocation so that it doesn't modify the stack
1658  // pointer when other instructions are using the stack.
1659  Chain = DAG.getCALLSEQ_START(Chain, 0, 0, dl);
1660 
1661  SDValue Size = Tmp2.getOperand(1);
1662  SDValue SP = DAG.getCopyFromReg(Chain, dl, SPReg, VT);
1663  Chain = SP.getValue(1);
1664  Align Alignment = cast<ConstantSDNode>(Tmp3)->getAlignValue();
1665  const TargetFrameLowering *TFL = DAG.getSubtarget().getFrameLowering();
1666  unsigned Opc =
1668  ISD::ADD : ISD::SUB;
1669 
1670  Align StackAlign = TFL->getStackAlign();
1671  Tmp1 = DAG.getNode(Opc, dl, VT, SP, Size); // Value
1672  if (Alignment > StackAlign)
1673  Tmp1 = DAG.getNode(ISD::AND, dl, VT, Tmp1,
1674  DAG.getConstant(-Alignment.value(), dl, VT));
1675  Chain = DAG.getCopyToReg(Chain, dl, SPReg, Tmp1); // Output chain
1676 
1677  Tmp2 = DAG.getCALLSEQ_END(Chain, DAG.getIntPtrConstant(0, dl, true),
1678  DAG.getIntPtrConstant(0, dl, true), SDValue(), dl);
1679 
1680  Results.push_back(Tmp1);
1681  Results.push_back(Tmp2);
1682 }
1683 
1684 /// Emit a store/load combination to the stack. This stores
1685 /// SrcOp to a stack slot of type SlotVT, truncating it if needed. It then does
1686 /// a load from the stack slot to DestVT, extending it if needed.
1687 /// The resultant code need not be legal.
1688 SDValue SelectionDAGLegalize::EmitStackConvert(SDValue SrcOp, EVT SlotVT,
1689  EVT DestVT, const SDLoc &dl) {
1690  return EmitStackConvert(SrcOp, SlotVT, DestVT, dl, DAG.getEntryNode());
1691 }
1692 
1693 SDValue SelectionDAGLegalize::EmitStackConvert(SDValue SrcOp, EVT SlotVT,
1694  EVT DestVT, const SDLoc &dl,
1695  SDValue Chain) {
1696  unsigned SrcSize = SrcOp.getValueSizeInBits();
1697  unsigned SlotSize = SlotVT.getSizeInBits();
1698  unsigned DestSize = DestVT.getSizeInBits();
1699  Type *DestType = DestVT.getTypeForEVT(*DAG.getContext());
1700  Align DestAlign = DAG.getDataLayout().getPrefTypeAlign(DestType);
1701 
1702  // Don't convert with stack if the load/store is expensive.
1703  if ((SrcSize > SlotSize &&
1704  !TLI.isTruncStoreLegalOrCustom(SrcOp.getValueType(), SlotVT)) ||
1705  (SlotSize < DestSize &&
1706  !TLI.isLoadExtLegalOrCustom(ISD::EXTLOAD, DestVT, SlotVT)))
1707  return SDValue();
1708 
1709  // Create the stack frame object.
1710  Align SrcAlign = DAG.getDataLayout().getPrefTypeAlign(
1711  SrcOp.getValueType().getTypeForEVT(*DAG.getContext()));
1712  SDValue FIPtr = DAG.CreateStackTemporary(SlotVT.getStoreSize(), SrcAlign);
1713 
1714  FrameIndexSDNode *StackPtrFI = cast<FrameIndexSDNode>(FIPtr);
1715  int SPFI = StackPtrFI->getIndex();
1716  MachinePointerInfo PtrInfo =
1717  MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), SPFI);
1718 
1719  // Emit a store to the stack slot. Use a truncstore if the input value is
1720  // later than DestVT.
1721  SDValue Store;
1722 
1723  if (SrcSize > SlotSize)
1724  Store = DAG.getTruncStore(Chain, dl, SrcOp, FIPtr, PtrInfo,
1725  SlotVT, SrcAlign);
1726  else {
1727  assert(SrcSize == SlotSize && "Invalid store");
1728  Store =
1729  DAG.getStore(Chain, dl, SrcOp, FIPtr, PtrInfo, SrcAlign);
1730  }
1731 
1732  // Result is a load from the stack slot.
1733  if (SlotSize == DestSize)
1734  return DAG.getLoad(DestVT, dl, Store, FIPtr, PtrInfo, DestAlign);
1735 
1736  assert(SlotSize < DestSize && "Unknown extension!");
1737  return DAG.getExtLoad(ISD::EXTLOAD, dl, DestVT, Store, FIPtr, PtrInfo, SlotVT,
1738  DestAlign);
1739 }
1740 
1741 SDValue SelectionDAGLegalize::ExpandSCALAR_TO_VECTOR(SDNode *Node) {
1742  SDLoc dl(Node);
1743  // Create a vector sized/aligned stack slot, store the value to element #0,
1744  // then load the whole vector back out.
1745  SDValue StackPtr = DAG.CreateStackTemporary(Node->getValueType(0));
1746 
1747  FrameIndexSDNode *StackPtrFI = cast<FrameIndexSDNode>(StackPtr);
1748  int SPFI = StackPtrFI->getIndex();
1749 
1750  SDValue Ch = DAG.getTruncStore(
1751  DAG.getEntryNode(), dl, Node->getOperand(0), StackPtr,
1752  MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), SPFI),
1753  Node->getValueType(0).getVectorElementType());
1754  return DAG.getLoad(
1755  Node->getValueType(0), dl, Ch, StackPtr,
1756  MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), SPFI));
1757 }
1758 
1759 static bool
1761  const TargetLowering &TLI, SDValue &Res) {
1762  unsigned NumElems = Node->getNumOperands();
1763  SDLoc dl(Node);
1764  EVT VT = Node->getValueType(0);
1765 
1766  // Try to group the scalars into pairs, shuffle the pairs together, then
1767  // shuffle the pairs of pairs together, etc. until the vector has
1768  // been built. This will work only if all of the necessary shuffle masks
1769  // are legal.
1770 
1771  // We do this in two phases; first to check the legality of the shuffles,
1772  // and next, assuming that all shuffles are legal, to create the new nodes.
1773  for (int Phase = 0; Phase < 2; ++Phase) {
1775  NewIntermedVals;
1776  for (unsigned i = 0; i < NumElems; ++i) {
1777  SDValue V = Node->getOperand(i);
1778  if (V.isUndef())
1779  continue;
1780 
1781  SDValue Vec;
1782  if (Phase)
1783  Vec = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, V);
1784  IntermedVals.push_back(std::make_pair(Vec, SmallVector<int, 16>(1, i)));
1785  }
1786 
1787  while (IntermedVals.size() > 2) {
1788  NewIntermedVals.clear();
1789  for (unsigned i = 0, e = (IntermedVals.size() & ~1u); i < e; i += 2) {
1790  // This vector and the next vector are shuffled together (simply to
1791  // append the one to the other).
1792  SmallVector<int, 16> ShuffleVec(NumElems, -1);
1793 
1794  SmallVector<int, 16> FinalIndices;
1795  FinalIndices.reserve(IntermedVals[i].second.size() +
1796  IntermedVals[i+1].second.size());
1797 
1798  int k = 0;
1799  for (unsigned j = 0, f = IntermedVals[i].second.size(); j != f;
1800  ++j, ++k) {
1801  ShuffleVec[k] = j;
1802  FinalIndices.push_back(IntermedVals[i].second[j]);
1803  }
1804  for (unsigned j = 0, f = IntermedVals[i+1].second.size(); j != f;
1805  ++j, ++k) {
1806  ShuffleVec[k] = NumElems + j;
1807  FinalIndices.push_back(IntermedVals[i+1].second[j]);
1808  }
1809 
1810  SDValue Shuffle;
1811  if (Phase)
1812  Shuffle = DAG.getVectorShuffle(VT, dl, IntermedVals[i].first,
1813  IntermedVals[i+1].first,
1814  ShuffleVec);
1815  else if (!TLI.isShuffleMaskLegal(ShuffleVec, VT))
1816  return false;
1817  NewIntermedVals.push_back(
1818  std::make_pair(Shuffle, std::move(FinalIndices)));
1819  }
1820 
1821  // If we had an odd number of defined values, then append the last
1822  // element to the array of new vectors.
1823  if ((IntermedVals.size() & 1) != 0)
1824  NewIntermedVals.push_back(IntermedVals.back());
1825 
1826  IntermedVals.swap(NewIntermedVals);
1827  }
1828 
1829  assert(IntermedVals.size() <= 2 && IntermedVals.size() > 0 &&
1830  "Invalid number of intermediate vectors");
1831  SDValue Vec1 = IntermedVals[0].first;
1832  SDValue Vec2;
1833  if (IntermedVals.size() > 1)
1834  Vec2 = IntermedVals[1].first;
1835  else if (Phase)
1836  Vec2 = DAG.getUNDEF(VT);
1837 
1838  SmallVector<int, 16> ShuffleVec(NumElems, -1);
1839  for (unsigned i = 0, e = IntermedVals[0].second.size(); i != e; ++i)
1840  ShuffleVec[IntermedVals[0].second[i]] = i;
1841  for (unsigned i = 0, e = IntermedVals[1].second.size(); i != e; ++i)
1842  ShuffleVec[IntermedVals[1].second[i]] = NumElems + i;
1843 
1844  if (Phase)
1845  Res = DAG.getVectorShuffle(VT, dl, Vec1, Vec2, ShuffleVec);
1846  else if (!TLI.isShuffleMaskLegal(ShuffleVec, VT))
1847  return false;
1848  }
1849 
1850  return true;
1851 }
1852 
1853 /// Expand a BUILD_VECTOR node on targets that don't
1854 /// support the operation, but do support the resultant vector type.
1855 SDValue SelectionDAGLegalize::ExpandBUILD_VECTOR(SDNode *Node) {
1856  unsigned NumElems = Node->getNumOperands();
1857  SDValue Value1, Value2;
1858  SDLoc dl(Node);
1859  EVT VT = Node->getValueType(0);
1860  EVT OpVT = Node->getOperand(0).getValueType();
1861  EVT EltVT = VT.getVectorElementType();
1862 
1863  // If the only non-undef value is the low element, turn this into a
1864  // SCALAR_TO_VECTOR node. If this is { X, X, X, X }, determine X.
1865  bool isOnlyLowElement = true;
1866  bool MoreThanTwoValues = false;
1867  bool isConstant = true;
1868  for (unsigned i = 0; i < NumElems; ++i) {
1869  SDValue V = Node->getOperand(i);
1870  if (V.isUndef())
1871  continue;
1872  if (i > 0)
1873  isOnlyLowElement = false;
1874  if (!isa<ConstantFPSDNode>(V) && !isa<ConstantSDNode>(V))
1875  isConstant = false;
1876 
1877  if (!Value1.getNode()) {
1878  Value1 = V;
1879  } else if (!Value2.getNode()) {
1880  if (V != Value1)
1881  Value2 = V;
1882  } else if (V != Value1 && V != Value2) {
1883  MoreThanTwoValues = true;
1884  }
1885  }
1886 
1887  if (!Value1.getNode())
1888  return DAG.getUNDEF(VT);
1889 
1890  if (isOnlyLowElement)
1891  return DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, Node->getOperand(0));
1892 
1893  // If all elements are constants, create a load from the constant pool.
1894  if (isConstant) {
1896  for (unsigned i = 0, e = NumElems; i != e; ++i) {
1897  if (ConstantFPSDNode *V =
1898  dyn_cast<ConstantFPSDNode>(Node->getOperand(i))) {
1899  CV.push_back(const_cast<ConstantFP *>(V->getConstantFPValue()));
1900  } else if (ConstantSDNode *V =
1901  dyn_cast<ConstantSDNode>(Node->getOperand(i))) {
1902  if (OpVT==EltVT)
1903  CV.push_back(const_cast<ConstantInt *>(V->getConstantIntValue()));
1904  else {
1905  // If OpVT and EltVT don't match, EltVT is not legal and the
1906  // element values have been promoted/truncated earlier. Undo this;
1907  // we don't want a v16i8 to become a v16i32 for example.
1908  const ConstantInt *CI = V->getConstantIntValue();
1909  CV.push_back(ConstantInt::get(EltVT.getTypeForEVT(*DAG.getContext()),
1910  CI->getZExtValue()));
1911  }
1912  } else {
1913  assert(Node->getOperand(i).isUndef());
1914  Type *OpNTy = EltVT.getTypeForEVT(*DAG.getContext());
1915  CV.push_back(UndefValue::get(OpNTy));
1916  }
1917  }
1919  SDValue CPIdx =
1920  DAG.getConstantPool(CP, TLI.getPointerTy(DAG.getDataLayout()));
1921  Align Alignment = cast<ConstantPoolSDNode>(CPIdx)->getAlign();
1922  return DAG.getLoad(
1923  VT, dl, DAG.getEntryNode(), CPIdx,
1924  MachinePointerInfo::getConstantPool(DAG.getMachineFunction()),
1925  Alignment);
1926  }
1927 
1928  SmallSet<SDValue, 16> DefinedValues;
1929  for (unsigned i = 0; i < NumElems; ++i) {
1930  if (Node->getOperand(i).isUndef())
1931  continue;
1932  DefinedValues.insert(Node->getOperand(i));
1933  }
1934 
1935  if (TLI.shouldExpandBuildVectorWithShuffles(VT, DefinedValues.size())) {
1936  if (!MoreThanTwoValues) {
1937  SmallVector<int, 8> ShuffleVec(NumElems, -1);
1938  for (unsigned i = 0; i < NumElems; ++i) {
1939  SDValue V = Node->getOperand(i);
1940  if (V.isUndef())
1941  continue;
1942  ShuffleVec[i] = V == Value1 ? 0 : NumElems;
1943  }
1944  if (TLI.isShuffleMaskLegal(ShuffleVec, Node->getValueType(0))) {
1945  // Get the splatted value into the low element of a vector register.
1946  SDValue Vec1 = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, Value1);
1947  SDValue Vec2;
1948  if (Value2.getNode())
1949  Vec2 = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, Value2);
1950  else
1951  Vec2 = DAG.getUNDEF(VT);
1952 
1953  // Return shuffle(LowValVec, undef, <0,0,0,0>)
1954  return DAG.getVectorShuffle(VT, dl, Vec1, Vec2, ShuffleVec);
1955  }
1956  } else {
1957  SDValue Res;
1958  if (ExpandBVWithShuffles(Node, DAG, TLI, Res))
1959  return Res;
1960  }
1961  }
1962 
1963  // Otherwise, we can't handle this case efficiently.
1964  return ExpandVectorBuildThroughStack(Node);
1965 }
1966 
1967 SDValue SelectionDAGLegalize::ExpandSPLAT_VECTOR(SDNode *Node) {
1968  SDLoc DL(Node);
1969  EVT VT = Node->getValueType(0);
1970  SDValue SplatVal = Node->getOperand(0);
1971 
1972  return DAG.getSplatBuildVector(VT, DL, SplatVal);
1973 }
1974 
1975 // Expand a node into a call to a libcall. If the result value
1976 // does not fit into a register, return the lo part and set the hi part to the
1977 // by-reg argument. If it does fit into a single register, return the result
1978 // and leave the Hi part unset.
1979 SDValue SelectionDAGLegalize::ExpandLibCall(RTLIB::Libcall LC, SDNode *Node,
1980  bool isSigned) {
1982  TargetLowering::ArgListEntry Entry;
1983  for (const SDValue &Op : Node->op_values()) {
1984  EVT ArgVT = Op.getValueType();
1985  Type *ArgTy = ArgVT.getTypeForEVT(*DAG.getContext());
1986  Entry.Node = Op;
1987  Entry.Ty = ArgTy;
1988  Entry.IsSExt = TLI.shouldSignExtendTypeInLibCall(ArgVT, isSigned);
1989  Entry.IsZExt = !TLI.shouldSignExtendTypeInLibCall(ArgVT, isSigned);
1990  Args.push_back(Entry);
1991  }
1992  SDValue Callee = DAG.getExternalSymbol(TLI.getLibcallName(LC),
1993  TLI.getPointerTy(DAG.getDataLayout()));
1994 
1995  EVT RetVT = Node->getValueType(0);
1996  Type *RetTy = RetVT.getTypeForEVT(*DAG.getContext());
1997 
1998  // By default, the input chain to this libcall is the entry node of the
1999  // function. If the libcall is going to be emitted as a tail call then
2000  // TLI.isUsedByReturnOnly will change it to the right chain if the return
2001  // node which is being folded has a non-entry input chain.
2002  SDValue InChain = DAG.getEntryNode();
2003 
2004  // isTailCall may be true since the callee does not reference caller stack
2005  // frame. Check if it's in the right position and that the return types match.
2006  SDValue TCChain = InChain;
2007  const Function &F = DAG.getMachineFunction().getFunction();
2008  bool isTailCall =
2009  TLI.isInTailCallPosition(DAG, Node, TCChain) &&
2010  (RetTy == F.getReturnType() || F.getReturnType()->isVoidTy());
2011  if (isTailCall)
2012  InChain = TCChain;
2013 
2015  bool signExtend = TLI.shouldSignExtendTypeInLibCall(RetVT, isSigned);
2016  CLI.setDebugLoc(SDLoc(Node))
2017  .setChain(InChain)
2018  .setLibCallee(TLI.getLibcallCallingConv(LC), RetTy, Callee,
2019  std::move(Args))
2020  .setTailCall(isTailCall)
2021  .setSExtResult(signExtend)
2022  .setZExtResult(!signExtend)
2023  .setIsPostTypeLegalization(true);
2024 
2025  std::pair<SDValue, SDValue> CallInfo = TLI.LowerCallTo(CLI);
2026 
2027  if (!CallInfo.second.getNode()) {
2028  LLVM_DEBUG(dbgs() << "Created tailcall: "; DAG.getRoot().dump(&DAG));
2029  // It's a tailcall, return the chain (which is the DAG root).
2030  return DAG.getRoot();
2031  }
2032 
2033  LLVM_DEBUG(dbgs() << "Created libcall: "; CallInfo.first.dump(&DAG));
2034  return CallInfo.first;
2035 }
2036 
2037 void SelectionDAGLegalize::ExpandFPLibCall(SDNode* Node,
2038  RTLIB::Libcall Call_F32,
2039  RTLIB::Libcall Call_F64,
2040  RTLIB::Libcall Call_F80,
2041  RTLIB::Libcall Call_F128,
2042  RTLIB::Libcall Call_PPCF128,
2044  RTLIB::Libcall LC;
2045  switch (Node->getSimpleValueType(0).SimpleTy) {
2046  default: llvm_unreachable("Unexpected request for libcall!");
2047  case MVT::f32: LC = Call_F32; break;
2048  case MVT::f64: LC = Call_F64; break;
2049  case MVT::f80: LC = Call_F80; break;
2050  case MVT::f128: LC = Call_F128; break;
2051  case MVT::ppcf128: LC = Call_PPCF128; break;
2052  }
2053 
2054  if (Node->isStrictFPOpcode()) {
2055  EVT RetVT = Node->getValueType(0);
2056  SmallVector<SDValue, 4> Ops(drop_begin(Node->ops()));
2058  // FIXME: This doesn't support tail calls.
2059  std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, RetVT,
2060  Ops, CallOptions,
2061  SDLoc(Node),
2062  Node->getOperand(0));
2063  Results.push_back(Tmp.first);
2064  Results.push_back(Tmp.second);
2065  } else {
2066  SDValue Tmp = ExpandLibCall(LC, Node, false);
2067  Results.push_back(Tmp);
2068  }
2069 }
2070 
2071 SDValue SelectionDAGLegalize::ExpandIntLibCall(SDNode* Node, bool isSigned,
2072  RTLIB::Libcall Call_I8,
2073  RTLIB::Libcall Call_I16,
2074  RTLIB::Libcall Call_I32,
2075  RTLIB::Libcall Call_I64,
2076  RTLIB::Libcall Call_I128) {
2077  RTLIB::Libcall LC;
2078  switch (Node->getSimpleValueType(0).SimpleTy) {
2079  default: llvm_unreachable("Unexpected request for libcall!");
2080  case MVT::i8: LC = Call_I8; break;
2081  case MVT::i16: LC = Call_I16; break;
2082  case MVT::i32: LC = Call_I32; break;
2083  case MVT::i64: LC = Call_I64; break;
2084  case MVT::i128: LC = Call_I128; break;
2085  }
2086  return ExpandLibCall(LC, Node, isSigned);
2087 }
2088 
2089 /// Expand the node to a libcall based on first argument type (for instance
2090 /// lround and its variant).
2091 void SelectionDAGLegalize::ExpandArgFPLibCall(SDNode* Node,
2092  RTLIB::Libcall Call_F32,
2093  RTLIB::Libcall Call_F64,
2094  RTLIB::Libcall Call_F80,
2095  RTLIB::Libcall Call_F128,
2096  RTLIB::Libcall Call_PPCF128,
2098  EVT InVT = Node->getOperand(Node->isStrictFPOpcode() ? 1 : 0).getValueType();
2099 
2100  RTLIB::Libcall LC;
2101  switch (InVT.getSimpleVT().SimpleTy) {
2102  default: llvm_unreachable("Unexpected request for libcall!");
2103  case MVT::f32: LC = Call_F32; break;
2104  case MVT::f64: LC = Call_F64; break;
2105  case MVT::f80: LC = Call_F80; break;
2106  case MVT::f128: LC = Call_F128; break;
2107  case MVT::ppcf128: LC = Call_PPCF128; break;
2108  }
2109 
2110  if (Node->isStrictFPOpcode()) {
2111  EVT RetVT = Node->getValueType(0);
2112  SmallVector<SDValue, 4> Ops(Node->op_begin() + 1, Node->op_end());
2114  // FIXME: This doesn't support tail calls.
2115  std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, RetVT,
2116  Ops, CallOptions,
2117  SDLoc(Node),
2118  Node->getOperand(0));
2119  Results.push_back(Tmp.first);
2120  Results.push_back(Tmp.second);
2121  } else {
2122  SDValue Tmp = ExpandLibCall(LC, Node, false);
2123  Results.push_back(Tmp);
2124  }
2125 }
2126 
2127 /// Issue libcalls to __{u}divmod to compute div / rem pairs.
2128 void
2129 SelectionDAGLegalize::ExpandDivRemLibCall(SDNode *Node,
2131  unsigned Opcode = Node->getOpcode();
2132  bool isSigned = Opcode == ISD::SDIVREM;
2133 
2134  RTLIB::Libcall LC;
2135  switch (Node->getSimpleValueType(0).SimpleTy) {
2136  default: llvm_unreachable("Unexpected request for libcall!");
2137  case MVT::i8: LC= isSigned ? RTLIB::SDIVREM_I8 : RTLIB::UDIVREM_I8; break;
2138  case MVT::i16: LC= isSigned ? RTLIB::SDIVREM_I16 : RTLIB::UDIVREM_I16; break;
2139  case MVT::i32: LC= isSigned ? RTLIB::SDIVREM_I32 : RTLIB::UDIVREM_I32; break;
2140  case MVT::i64: LC= isSigned ? RTLIB::SDIVREM_I64 : RTLIB::UDIVREM_I64; break;
2141  case MVT::i128: LC= isSigned ? RTLIB::SDIVREM_I128:RTLIB::UDIVREM_I128; break;
2142  }
2143 
2144  // The input chain to this libcall is the entry node of the function.
2145  // Legalizing the call will automatically add the previous call to the
2146  // dependence.
2147  SDValue InChain = DAG.getEntryNode();
2148 
2149  EVT RetVT = Node->getValueType(0);
2150  Type *RetTy = RetVT.getTypeForEVT(*DAG.getContext());
2151 
2153  TargetLowering::ArgListEntry Entry;
2154  for (const SDValue &Op : Node->op_values()) {
2155  EVT ArgVT = Op.getValueType();
2156  Type *ArgTy = ArgVT.getTypeForEVT(*DAG.getContext());
2157  Entry.Node = Op;
2158  Entry.Ty = ArgTy;
2159  Entry.IsSExt = isSigned;
2160  Entry.IsZExt = !isSigned;
2161  Args.push_back(Entry);
2162  }
2163 
2164  // Also pass the return address of the remainder.
2165  SDValue FIPtr = DAG.CreateStackTemporary(RetVT);
2166  Entry.Node = FIPtr;
2167  Entry.Ty = RetTy->getPointerTo();
2168  Entry.IsSExt = isSigned;
2169  Entry.IsZExt = !isSigned;
2170  Args.push_back(Entry);
2171 
2172  SDValue Callee = DAG.getExternalSymbol(TLI.getLibcallName(LC),
2173  TLI.getPointerTy(DAG.getDataLayout()));
2174 
2175  SDLoc dl(Node);
2177  CLI.setDebugLoc(dl)
2178  .setChain(InChain)
2179  .setLibCallee(TLI.getLibcallCallingConv(LC), RetTy, Callee,
2180  std::move(Args))
2181  .setSExtResult(isSigned)
2182  .setZExtResult(!isSigned);
2183 
2184  std::pair<SDValue, SDValue> CallInfo = TLI.LowerCallTo(CLI);
2185 
2186  // Remainder is loaded back from the stack frame.
2187  SDValue Rem =
2188  DAG.getLoad(RetVT, dl, CallInfo.second, FIPtr, MachinePointerInfo());
2189  Results.push_back(CallInfo.first);
2190  Results.push_back(Rem);
2191 }
2192 
2193 /// Return true if sincos libcall is available.
2194 static bool isSinCosLibcallAvailable(SDNode *Node, const TargetLowering &TLI) {
2195  RTLIB::Libcall LC;
2196  switch (Node->getSimpleValueType(0).SimpleTy) {
2197  default: llvm_unreachable("Unexpected request for libcall!");
2198  case MVT::f32: LC = RTLIB::SINCOS_F32; break;
2199  case MVT::f64: LC = RTLIB::SINCOS_F64; break;
2200  case MVT::f80: LC = RTLIB::SINCOS_F80; break;
2201  case MVT::f128: LC = RTLIB::SINCOS_F128; break;
2202  case MVT::ppcf128: LC = RTLIB::SINCOS_PPCF128; break;
2203  }
2204  return TLI.getLibcallName(LC) != nullptr;
2205 }
2206 
2207 /// Only issue sincos libcall if both sin and cos are needed.
2208 static bool useSinCos(SDNode *Node) {
2209  unsigned OtherOpcode = Node->getOpcode() == ISD::FSIN
2210  ? ISD::FCOS : ISD::FSIN;
2211 
2212  SDValue Op0 = Node->getOperand(0);
2213  for (SDNode::use_iterator UI = Op0.getNode()->use_begin(),
2214  UE = Op0.getNode()->use_end(); UI != UE; ++UI) {
2215  SDNode *User = *UI;
2216  if (User == Node)
2217  continue;
2218  // The other user might have been turned into sincos already.
2219  if (User->getOpcode() == OtherOpcode || User->getOpcode() == ISD::FSINCOS)
2220  return true;
2221  }
2222  return false;
2223 }
2224 
2225 /// Issue libcalls to sincos to compute sin / cos pairs.
2226 void
2227 SelectionDAGLegalize::ExpandSinCosLibCall(SDNode *Node,
2229  RTLIB::Libcall LC;
2230  switch (Node->getSimpleValueType(0).SimpleTy) {
2231  default: llvm_unreachable("Unexpected request for libcall!");
2232  case MVT::f32: LC = RTLIB::SINCOS_F32; break;
2233  case MVT::f64: LC = RTLIB::SINCOS_F64; break;
2234  case MVT::f80: LC = RTLIB::SINCOS_F80; break;
2235  case MVT::f128: LC = RTLIB::SINCOS_F128; break;
2236  case MVT::ppcf128: LC = RTLIB::SINCOS_PPCF128; break;
2237  }
2238 
2239  // The input chain to this libcall is the entry node of the function.
2240  // Legalizing the call will automatically add the previous call to the
2241  // dependence.
2242  SDValue InChain = DAG.getEntryNode();
2243 
2244  EVT RetVT = Node->getValueType(0);
2245  Type *RetTy = RetVT.getTypeForEVT(*DAG.getContext());
2246 
2248  TargetLowering::ArgListEntry Entry;
2249 
2250  // Pass the argument.
2251  Entry.Node = Node->getOperand(0);
2252  Entry.Ty = RetTy;
2253  Entry.IsSExt = false;
2254  Entry.IsZExt = false;
2255  Args.push_back(Entry);
2256 
2257  // Pass the return address of sin.
2258  SDValue SinPtr = DAG.CreateStackTemporary(RetVT);
2259  Entry.Node = SinPtr;
2260  Entry.Ty = RetTy->getPointerTo();
2261  Entry.IsSExt = false;
2262  Entry.IsZExt = false;
2263  Args.push_back(Entry);
2264 
2265  // Also pass the return address of the cos.
2266  SDValue CosPtr = DAG.CreateStackTemporary(RetVT);
2267  Entry.Node = CosPtr;
2268  Entry.Ty = RetTy->getPointerTo();
2269  Entry.IsSExt = false;
2270  Entry.IsZExt = false;
2271  Args.push_back(Entry);
2272 
2273  SDValue Callee = DAG.getExternalSymbol(TLI.getLibcallName(LC),
2274  TLI.getPointerTy(DAG.getDataLayout()));
2275 
2276  SDLoc dl(Node);
2278  CLI.setDebugLoc(dl).setChain(InChain).setLibCallee(
2279  TLI.getLibcallCallingConv(LC), Type::getVoidTy(*DAG.getContext()), Callee,
2280  std::move(Args));
2281 
2282  std::pair<SDValue, SDValue> CallInfo = TLI.LowerCallTo(CLI);
2283 
2284  Results.push_back(
2285  DAG.getLoad(RetVT, dl, CallInfo.second, SinPtr, MachinePointerInfo()));
2286  Results.push_back(
2287  DAG.getLoad(RetVT, dl, CallInfo.second, CosPtr, MachinePointerInfo()));
2288 }
2289 
2290 /// This function is responsible for legalizing a
2291 /// INT_TO_FP operation of the specified operand when the target requests that
2292 /// we expand it. At this point, we know that the result and operand types are
2293 /// legal for the target.
2294 SDValue SelectionDAGLegalize::ExpandLegalINT_TO_FP(SDNode *Node,
2295  SDValue &Chain) {
2296  bool isSigned = (Node->getOpcode() == ISD::STRICT_SINT_TO_FP ||
2297  Node->getOpcode() == ISD::SINT_TO_FP);
2298  EVT DestVT = Node->getValueType(0);
2299  SDLoc dl(Node);
2300  unsigned OpNo = Node->isStrictFPOpcode() ? 1 : 0;
2301  SDValue Op0 = Node->getOperand(OpNo);
2302  EVT SrcVT = Op0.getValueType();
2303 
2304  // TODO: Should any fast-math-flags be set for the created nodes?
2305  LLVM_DEBUG(dbgs() << "Legalizing INT_TO_FP\n");
2306  if (SrcVT == MVT::i32 && TLI.isTypeLegal(MVT::f64) &&
2307  (DestVT.bitsLE(MVT::f64) ||
2308  TLI.isOperationLegal(Node->isStrictFPOpcode() ? ISD::STRICT_FP_EXTEND
2309  : ISD::FP_EXTEND,
2310  DestVT))) {
2311  LLVM_DEBUG(dbgs() << "32-bit [signed|unsigned] integer to float/double "
2312  "expansion\n");
2313 
2314  // Get the stack frame index of a 8 byte buffer.
2315  SDValue StackSlot = DAG.CreateStackTemporary(MVT::f64);
2316 
2317  SDValue Lo = Op0;
2318  // if signed map to unsigned space
2319  if (isSigned) {
2320  // Invert sign bit (signed to unsigned mapping).
2321  Lo = DAG.getNode(ISD::XOR, dl, MVT::i32, Lo,
2322  DAG.getConstant(0x80000000u, dl, MVT::i32));
2323  }
2324  // Initial hi portion of constructed double.
2325  SDValue Hi = DAG.getConstant(0x43300000u, dl, MVT::i32);
2326 
2327  // If this a big endian target, swap the lo and high data.
2328  if (DAG.getDataLayout().isBigEndian())
2329  std::swap(Lo, Hi);
2330 
2331  SDValue MemChain = DAG.getEntryNode();
2332 
2333  // Store the lo of the constructed double.
2334  SDValue Store1 = DAG.getStore(MemChain, dl, Lo, StackSlot,
2335  MachinePointerInfo());
2336  // Store the hi of the constructed double.
2337  SDValue HiPtr = DAG.getMemBasePlusOffset(StackSlot, TypeSize::Fixed(4), dl);
2338  SDValue Store2 =
2339  DAG.getStore(MemChain, dl, Hi, HiPtr, MachinePointerInfo());
2340  MemChain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Store1, Store2);
2341 
2342  // load the constructed double
2343  SDValue Load =
2344  DAG.getLoad(MVT::f64, dl, MemChain, StackSlot, MachinePointerInfo());
2345  // FP constant to bias correct the final result
2346  SDValue Bias = DAG.getConstantFP(isSigned ?
2347  BitsToDouble(0x4330000080000000ULL) :
2348  BitsToDouble(0x4330000000000000ULL),
2349  dl, MVT::f64);
2350  // Subtract the bias and get the final result.
2351  SDValue Sub;
2352  SDValue Result;
2353  if (Node->isStrictFPOpcode()) {
2354  Sub = DAG.getNode(ISD::STRICT_FSUB, dl, {MVT::f64, MVT::Other},
2355  {Node->getOperand(0), Load, Bias});
2356  Chain = Sub.getValue(1);
2357  if (DestVT != Sub.getValueType()) {
2358  std::pair<SDValue, SDValue> ResultPair;
2359  ResultPair =
2360  DAG.getStrictFPExtendOrRound(Sub, Chain, dl, DestVT);
2361  Result = ResultPair.first;
2362  Chain = ResultPair.second;
2363  }
2364  else
2365  Result = Sub;
2366  } else {
2367  Sub = DAG.getNode(ISD::FSUB, dl, MVT::f64, Load, Bias);
2368  Result = DAG.getFPExtendOrRound(Sub, dl, DestVT);
2369  }
2370  return Result;
2371  }
2372 
2373  if (isSigned)
2374  return SDValue();
2375 
2376  // TODO: Generalize this for use with other types.
2377  if (((SrcVT == MVT::i32 || SrcVT == MVT::i64) && DestVT == MVT::f32) ||
2378  (SrcVT == MVT::i64 && DestVT == MVT::f64)) {
2379  LLVM_DEBUG(dbgs() << "Converting unsigned i32/i64 to f32/f64\n");
2380  // For unsigned conversions, convert them to signed conversions using the
2381  // algorithm from the x86_64 __floatundisf in compiler_rt. That method
2382  // should be valid for i32->f32 as well.
2383 
2384  // More generally this transform should be valid if there are 3 more bits
2385  // in the integer type than the significand. Rounding uses the first bit
2386  // after the width of the significand and the OR of all bits after that. So
2387  // we need to be able to OR the shifted out bit into one of the bits that
2388  // participate in the OR.
2389 
2390  // TODO: This really should be implemented using a branch rather than a
2391  // select. We happen to get lucky and machinesink does the right
2392  // thing most of the time. This would be a good candidate for a
2393  // pseudo-op, or, even better, for whole-function isel.
2394  EVT SetCCVT = getSetCCResultType(SrcVT);
2395 
2396  SDValue SignBitTest = DAG.getSetCC(
2397  dl, SetCCVT, Op0, DAG.getConstant(0, dl, SrcVT), ISD::SETLT);
2398 
2399  EVT ShiftVT = TLI.getShiftAmountTy(SrcVT, DAG.getDataLayout());
2400  SDValue ShiftConst = DAG.getConstant(1, dl, ShiftVT);
2401  SDValue Shr = DAG.getNode(ISD::SRL, dl, SrcVT, Op0, ShiftConst);
2402  SDValue AndConst = DAG.getConstant(1, dl, SrcVT);
2403  SDValue And = DAG.getNode(ISD::AND, dl, SrcVT, Op0, AndConst);
2404  SDValue Or = DAG.getNode(ISD::OR, dl, SrcVT, And, Shr);
2405 
2406  SDValue Slow, Fast;
2407  if (Node->isStrictFPOpcode()) {
2408  // In strict mode, we must avoid spurious exceptions, and therefore
2409  // must make sure to only emit a single STRICT_SINT_TO_FP.
2410  SDValue InCvt = DAG.getSelect(dl, SrcVT, SignBitTest, Or, Op0);
2411  Fast = DAG.getNode(ISD::STRICT_SINT_TO_FP, dl, { DestVT, MVT::Other },
2412  { Node->getOperand(0), InCvt });
2413  Slow = DAG.getNode(ISD::STRICT_FADD, dl, { DestVT, MVT::Other },
2414  { Fast.getValue(1), Fast, Fast });
2415  Chain = Slow.getValue(1);
2416  // The STRICT_SINT_TO_FP inherits the exception mode from the
2417  // incoming STRICT_UINT_TO_FP node; the STRICT_FADD node can
2418  // never raise any exception.
2419  SDNodeFlags Flags;
2420  Flags.setNoFPExcept(Node->getFlags().hasNoFPExcept());
2421  Fast->setFlags(Flags);
2422  Flags.setNoFPExcept(true);
2423  Slow->setFlags(Flags);
2424  } else {
2425  SDValue SignCvt = DAG.getNode(ISD::SINT_TO_FP, dl, DestVT, Or);
2426  Slow = DAG.getNode(ISD::FADD, dl, DestVT, SignCvt, SignCvt);
2427  Fast = DAG.getNode(ISD::SINT_TO_FP, dl, DestVT, Op0);
2428  }
2429 
2430  return DAG.getSelect(dl, DestVT, SignBitTest, Slow, Fast);
2431  }
2432 
2433  // Don't expand it if there isn't cheap fadd.
2434  if (!TLI.isOperationLegalOrCustom(
2435  Node->isStrictFPOpcode() ? ISD::STRICT_FADD : ISD::FADD, DestVT))
2436  return SDValue();
2437 
2438  // The following optimization is valid only if every value in SrcVT (when
2439  // treated as signed) is representable in DestVT. Check that the mantissa
2440  // size of DestVT is >= than the number of bits in SrcVT -1.
2441  assert(APFloat::semanticsPrecision(DAG.EVTToAPFloatSemantics(DestVT)) >=
2442  SrcVT.getSizeInBits() - 1 &&
2443  "Cannot perform lossless SINT_TO_FP!");
2444 
2445  SDValue Tmp1;
2446  if (Node->isStrictFPOpcode()) {
2447  Tmp1 = DAG.getNode(ISD::STRICT_SINT_TO_FP, dl, { DestVT, MVT::Other },
2448  { Node->getOperand(0), Op0 });
2449  } else
2450  Tmp1 = DAG.getNode(ISD::SINT_TO_FP, dl, DestVT, Op0);
2451 
2452  SDValue SignSet = DAG.getSetCC(dl, getSetCCResultType(SrcVT), Op0,
2453  DAG.getConstant(0, dl, SrcVT), ISD::SETLT);
2454  SDValue Zero = DAG.getIntPtrConstant(0, dl),
2455  Four = DAG.getIntPtrConstant(4, dl);
2456  SDValue CstOffset = DAG.getSelect(dl, Zero.getValueType(),
2457  SignSet, Four, Zero);
2458 
2459  // If the sign bit of the integer is set, the large number will be treated
2460  // as a negative number. To counteract this, the dynamic code adds an
2461  // offset depending on the data type.
2462  uint64_t FF;
2463  switch (SrcVT.getSimpleVT().SimpleTy) {
2464  default:
2465  return SDValue();
2466  case MVT::i8 : FF = 0x43800000ULL; break; // 2^8 (as a float)
2467  case MVT::i16: FF = 0x47800000ULL; break; // 2^16 (as a float)
2468  case MVT::i32: FF = 0x4F800000ULL; break; // 2^32 (as a float)
2469  case MVT::i64: FF = 0x5F800000ULL; break; // 2^64 (as a float)
2470  }
2471  if (DAG.getDataLayout().isLittleEndian())
2472  FF <<= 32;
2473  Constant *FudgeFactor = ConstantInt::get(
2474  Type::getInt64Ty(*DAG.getContext()), FF);
2475 
2476  SDValue CPIdx =
2477  DAG.getConstantPool(FudgeFactor, TLI.getPointerTy(DAG.getDataLayout()));
2478  Align Alignment = cast<ConstantPoolSDNode>(CPIdx)->getAlign();
2479  CPIdx = DAG.getNode(ISD::ADD, dl, CPIdx.getValueType(), CPIdx, CstOffset);
2480  Alignment = commonAlignment(Alignment, 4);
2481  SDValue FudgeInReg;
2482  if (DestVT == MVT::f32)
2483  FudgeInReg = DAG.getLoad(
2484  MVT::f32, dl, DAG.getEntryNode(), CPIdx,
2485  MachinePointerInfo::getConstantPool(DAG.getMachineFunction()),
2486  Alignment);
2487  else {
2488  SDValue Load = DAG.getExtLoad(
2489  ISD::EXTLOAD, dl, DestVT, DAG.getEntryNode(), CPIdx,
2490  MachinePointerInfo::getConstantPool(DAG.getMachineFunction()), MVT::f32,
2491  Alignment);
2492  HandleSDNode Handle(Load);
2493  LegalizeOp(Load.getNode());
2494  FudgeInReg = Handle.getValue();
2495  }
2496 
2497  if (Node->isStrictFPOpcode()) {
2498  SDValue Result = DAG.getNode(ISD::STRICT_FADD, dl, { DestVT, MVT::Other },
2499  { Tmp1.getValue(1), Tmp1, FudgeInReg });
2500  Chain = Result.getValue(1);
2501  return Result;
2502  }
2503 
2504  return DAG.getNode(ISD::FADD, dl, DestVT, Tmp1, FudgeInReg);
2505 }
2506 
2507 /// This function is responsible for legalizing a
2508 /// *INT_TO_FP operation of the specified operand when the target requests that
2509 /// we promote it. At this point, we know that the result and operand types are
2510 /// legal for the target, and that there is a legal UINT_TO_FP or SINT_TO_FP
2511 /// operation that takes a larger input.
2512 void SelectionDAGLegalize::PromoteLegalINT_TO_FP(
2513  SDNode *N, const SDLoc &dl, SmallVectorImpl<SDValue> &Results) {
2514  bool IsStrict = N->isStrictFPOpcode();
2515  bool IsSigned = N->getOpcode() == ISD::SINT_TO_FP ||
2516  N->getOpcode() == ISD::STRICT_SINT_TO_FP;
2517  EVT DestVT = N->getValueType(0);
2518  SDValue LegalOp = N->getOperand(IsStrict ? 1 : 0);
2519  unsigned UIntOp = IsStrict ? ISD::STRICT_UINT_TO_FP : ISD::UINT_TO_FP;
2520  unsigned SIntOp = IsStrict ? ISD::STRICT_SINT_TO_FP : ISD::SINT_TO_FP;
2521 
2522  // First step, figure out the appropriate *INT_TO_FP operation to use.
2523  EVT NewInTy = LegalOp.getValueType();
2524 
2525  unsigned OpToUse = 0;
2526 
2527  // Scan for the appropriate larger type to use.
2528  while (true) {
2529  NewInTy = (MVT::SimpleValueType)(NewInTy.getSimpleVT().SimpleTy+1);
2530  assert(NewInTy.isInteger() && "Ran out of possibilities!");
2531 
2532  // If the target supports SINT_TO_FP of this type, use it.
2533  if (TLI.isOperationLegalOrCustom(SIntOp, NewInTy)) {
2534  OpToUse = SIntOp;
2535  break;
2536  }
2537  if (IsSigned)
2538  continue;
2539 
2540  // If the target supports UINT_TO_FP of this type, use it.
2541  if (TLI.isOperationLegalOrCustom(UIntOp, NewInTy)) {
2542  OpToUse = UIntOp;
2543  break;
2544  }
2545 
2546  // Otherwise, try a larger type.
2547  }
2548 
2549  // Okay, we found the operation and type to use. Zero extend our input to the
2550  // desired type then run the operation on it.
2551  if (IsStrict) {
2552  SDValue Res =
2553  DAG.getNode(OpToUse, dl, {DestVT, MVT::Other},
2554  {N->getOperand(0),
2555  DAG.getNode(IsSigned ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND,
2556  dl, NewInTy, LegalOp)});
2557  Results.push_back(Res);
2558  Results.push_back(Res.getValue(1));
2559  return;
2560  }
2561 
2562  Results.push_back(
2563  DAG.getNode(OpToUse, dl, DestVT,
2564  DAG.getNode(IsSigned ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND,
2565  dl, NewInTy, LegalOp)));
2566 }
2567 
2568 /// This function is responsible for legalizing a
2569 /// FP_TO_*INT operation of the specified operand when the target requests that
2570 /// we promote it. At this point, we know that the result and operand types are
2571 /// legal for the target, and that there is a legal FP_TO_UINT or FP_TO_SINT
2572 /// operation that returns a larger result.
2573 void SelectionDAGLegalize::PromoteLegalFP_TO_INT(SDNode *N, const SDLoc &dl,
2575  bool IsStrict = N->isStrictFPOpcode();
2576  bool IsSigned = N->getOpcode() == ISD::FP_TO_SINT ||
2577  N->getOpcode() == ISD::STRICT_FP_TO_SINT;
2578  EVT DestVT = N->getValueType(0);
2579  SDValue LegalOp = N->getOperand(IsStrict ? 1 : 0);
2580  // First step, figure out the appropriate FP_TO*INT operation to use.
2581  EVT NewOutTy = DestVT;
2582 
2583  unsigned OpToUse = 0;
2584 
2585  // Scan for the appropriate larger type to use.
2586  while (true) {
2587  NewOutTy = (MVT::SimpleValueType)(NewOutTy.getSimpleVT().SimpleTy+1);
2588  assert(NewOutTy.isInteger() && "Ran out of possibilities!");
2589 
2590  // A larger signed type can hold all unsigned values of the requested type,
2591  // so using FP_TO_SINT is valid
2592  OpToUse = IsStrict ? ISD::STRICT_FP_TO_SINT : ISD::FP_TO_SINT;
2593  if (TLI.isOperationLegalOrCustom(OpToUse, NewOutTy))
2594  break;
2595 
2596  // However, if the value may be < 0.0, we *must* use some FP_TO_SINT.
2597  OpToUse = IsStrict ? ISD::STRICT_FP_TO_UINT : ISD::FP_TO_UINT;
2598  if (!IsSigned && TLI.isOperationLegalOrCustom(OpToUse, NewOutTy))
2599  break;
2600 
2601  // Otherwise, try a larger type.
2602  }
2603 
2604  // Okay, we found the operation and type to use.
2606  if (IsStrict) {
2607  SDVTList VTs = DAG.getVTList(NewOutTy, MVT::Other);
2608  Operation = DAG.getNode(OpToUse, dl, VTs, N->getOperand(0), LegalOp);
2609  } else
2610  Operation = DAG.getNode(OpToUse, dl, NewOutTy, LegalOp);
2611 
2612  // Truncate the result of the extended FP_TO_*INT operation to the desired
2613  // size.
2614  SDValue Trunc = DAG.getNode(ISD::TRUNCATE, dl, DestVT, Operation);
2615  Results.push_back(Trunc);
2616  if (IsStrict)
2617  Results.push_back(Operation.getValue(1));
2618 }
2619 
2620 /// Promote FP_TO_*INT_SAT operation to a larger result type. At this point
2621 /// the result and operand types are legal and there must be a legal
2622 /// FP_TO_*INT_SAT operation for a larger result type.
2623 SDValue SelectionDAGLegalize::PromoteLegalFP_TO_INT_SAT(SDNode *Node,
2624  const SDLoc &dl) {
2625  unsigned Opcode = Node->getOpcode();
2626 
2627  // Scan for the appropriate larger type to use.
2628  EVT NewOutTy = Node->getValueType(0);
2629  while (true) {
2630  NewOutTy = (MVT::SimpleValueType)(NewOutTy.getSimpleVT().SimpleTy + 1);
2631  assert(NewOutTy.isInteger() && "Ran out of possibilities!");
2632 
2633  if (TLI.isOperationLegalOrCustom(Opcode, NewOutTy))
2634  break;
2635  }
2636 
2637  // Saturation width is determined by second operand, so we don't have to
2638  // perform any fixup and can directly truncate the result.
2639  SDValue Result = DAG.getNode(Opcode, dl, NewOutTy, Node->getOperand(0),
2640  Node->getOperand(1));
2641  return DAG.getNode(ISD::TRUNCATE, dl, Node->getValueType(0), Result);
2642 }
2643 
2644 /// Open code the operations for PARITY of the specified operation.
2645 SDValue SelectionDAGLegalize::ExpandPARITY(SDValue Op, const SDLoc &dl) {
2646  EVT VT = Op.getValueType();
2647  EVT ShVT = TLI.getShiftAmountTy(VT, DAG.getDataLayout());
2648  unsigned Sz = VT.getScalarSizeInBits();
2649 
2650  // If CTPOP is legal, use it. Otherwise use shifts and xor.
2651  SDValue Result;
2652  if (TLI.isOperationLegal(ISD::CTPOP, VT)) {
2653  Result = DAG.getNode(ISD::CTPOP, dl, VT, Op);
2654  } else {
2655  Result = Op;
2656  for (unsigned i = Log2_32_Ceil(Sz); i != 0;) {
2657  SDValue Shift = DAG.getNode(ISD::SRL, dl, VT, Result,
2658  DAG.getConstant(1ULL << (--i), dl, ShVT));
2659  Result = DAG.getNode(ISD::XOR, dl, VT, Result, Shift);
2660  }
2661  }
2662 
2663  return DAG.getNode(ISD::AND, dl, VT, Result, DAG.getConstant(1, dl, VT));
2664 }
2665 
2666 bool SelectionDAGLegalize::ExpandNode(SDNode *Node) {
2667  LLVM_DEBUG(dbgs() << "Trying to expand node\n");
2669  SDLoc dl(Node);
2670  SDValue Tmp1, Tmp2, Tmp3, Tmp4;
2671  bool NeedInvert;
2672  switch (Node->getOpcode()) {
2673  case ISD::ABS:
2674  if (TLI.expandABS(Node, Tmp1, DAG))
2675  Results.push_back(Tmp1);
2676  break;
2677  case ISD::CTPOP:
2678  if (TLI.expandCTPOP(Node, Tmp1, DAG))
2679  Results.push_back(Tmp1);
2680  break;
2681  case ISD::CTLZ:
2682  case ISD::CTLZ_ZERO_UNDEF:
2683  if (TLI.expandCTLZ(Node, Tmp1, DAG))
2684  Results.push_back(Tmp1);
2685  break;
2686  case ISD::CTTZ:
2687  case ISD::CTTZ_ZERO_UNDEF:
2688  if (TLI.expandCTTZ(Node, Tmp1, DAG))
2689  Results.push_back(Tmp1);
2690  break;
2691  case ISD::BITREVERSE:
2692  if ((Tmp1 = TLI.expandBITREVERSE(Node, DAG)))
2693  Results.push_back(Tmp1);
2694  break;
2695  case ISD::BSWAP:
2696  if ((Tmp1 = TLI.expandBSWAP(Node, DAG)))
2697  Results.push_back(Tmp1);
2698  break;
2699  case ISD::PARITY:
2700  Results.push_back(ExpandPARITY(Node->getOperand(0), dl));
2701  break;
2702  case ISD::FRAMEADDR:
2703  case ISD::RETURNADDR:
2705  Results.push_back(DAG.getConstant(0, dl, Node->getValueType(0)));
2706  break;
2707  case ISD::EH_DWARF_CFA: {
2708  SDValue CfaArg = DAG.getSExtOrTrunc(Node->getOperand(0), dl,
2709  TLI.getPointerTy(DAG.getDataLayout()));
2710  SDValue Offset = DAG.getNode(ISD::ADD, dl,
2711  CfaArg.getValueType(),
2712  DAG.getNode(ISD::FRAME_TO_ARGS_OFFSET, dl,
2713  CfaArg.getValueType()),
2714  CfaArg);
2715  SDValue FA = DAG.getNode(
2716  ISD::FRAMEADDR, dl, TLI.getPointerTy(DAG.getDataLayout()),
2717  DAG.getConstant(0, dl, TLI.getPointerTy(DAG.getDataLayout())));
2718  Results.push_back(DAG.getNode(ISD::ADD, dl, FA.getValueType(),
2719  FA, Offset));
2720  break;
2721  }
2722  case ISD::FLT_ROUNDS_:
2723  Results.push_back(DAG.getConstant(1, dl, Node->getValueType(0)));
2724  Results.push_back(Node->getOperand(0));
2725  break;
2726  case ISD::EH_RETURN:
2727  case ISD::EH_LABEL:
2728  case ISD::PREFETCH:
2729  case ISD::VAEND:
2730  case ISD::EH_SJLJ_LONGJMP:
2731  // If the target didn't expand these, there's nothing to do, so just
2732  // preserve the chain and be done.
2733  Results.push_back(Node->getOperand(0));
2734  break;
2735  case ISD::READCYCLECOUNTER:
2736  // If the target didn't expand this, just return 'zero' and preserve the
2737  // chain.
2738  Results.append(Node->getNumValues() - 1,
2739  DAG.getConstant(0, dl, Node->getValueType(0)));
2740  Results.push_back(Node->getOperand(0));
2741  break;
2742  case ISD::EH_SJLJ_SETJMP:
2743  // If the target didn't expand this, just return 'zero' and preserve the
2744  // chain.
2745  Results.push_back(DAG.getConstant(0, dl, MVT::i32));
2746  Results.push_back(Node->getOperand(0));
2747  break;
2748  case ISD::ATOMIC_LOAD: {
2749  // There is no libcall for atomic load; fake it with ATOMIC_CMP_SWAP.
2750  SDValue Zero = DAG.getConstant(0, dl, Node->getValueType(0));
2751  SDVTList VTs = DAG.getVTList(Node->getValueType(0), MVT::Other);
2752  SDValue Swap = DAG.getAtomicCmpSwap(
2753  ISD::ATOMIC_CMP_SWAP, dl, cast<AtomicSDNode>(Node)->getMemoryVT(), VTs,
2754  Node->getOperand(0), Node->getOperand(1), Zero, Zero,
2755  cast<AtomicSDNode>(Node)->getMemOperand());
2756  Results.push_back(Swap.getValue(0));
2757  Results.push_back(Swap.getValue(1));
2758  break;
2759  }
2760  case ISD::ATOMIC_STORE: {
2761  // There is no libcall for atomic store; fake it with ATOMIC_SWAP.
2762  SDValue Swap = DAG.getAtomic(ISD::ATOMIC_SWAP, dl,
2763  cast<AtomicSDNode>(Node)->getMemoryVT(),
2764  Node->getOperand(0),
2765  Node->getOperand(1), Node->getOperand(2),
2766  cast<AtomicSDNode>(Node)->getMemOperand());
2767  Results.push_back(Swap.getValue(1));
2768  break;
2769  }
2771  // Expanding an ATOMIC_CMP_SWAP_WITH_SUCCESS produces an ATOMIC_CMP_SWAP and
2772  // splits out the success value as a comparison. Expanding the resulting
2773  // ATOMIC_CMP_SWAP will produce a libcall.
2774  SDVTList VTs = DAG.getVTList(Node->getValueType(0), MVT::Other);
2775  SDValue Res = DAG.getAtomicCmpSwap(
2776  ISD::ATOMIC_CMP_SWAP, dl, cast<AtomicSDNode>(Node)->getMemoryVT(), VTs,
2777  Node->getOperand(0), Node->getOperand(1), Node->getOperand(2),
2778  Node->getOperand(3), cast<MemSDNode>(Node)->getMemOperand());
2779 
2780  SDValue ExtRes = Res;
2781  SDValue LHS = Res;
2782  SDValue RHS = Node->getOperand(1);
2783 
2784  EVT AtomicType = cast<AtomicSDNode>(Node)->getMemoryVT();
2785  EVT OuterType = Node->getValueType(0);
2786  switch (TLI.getExtendForAtomicOps()) {
2787  case ISD::SIGN_EXTEND:
2788  LHS = DAG.getNode(ISD::AssertSext, dl, OuterType, Res,
2789  DAG.getValueType(AtomicType));
2790  RHS = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, OuterType,
2791  Node->getOperand(2), DAG.getValueType(AtomicType));
2792  ExtRes = LHS;
2793  break;
2794  case ISD::ZERO_EXTEND:
2795  LHS = DAG.getNode(ISD::AssertZext, dl, OuterType, Res,
2796  DAG.getValueType(AtomicType));
2797  RHS = DAG.getZeroExtendInReg(Node->getOperand(2), dl, AtomicType);
2798  ExtRes = LHS;
2799  break;
2800  case ISD::ANY_EXTEND:
2801  LHS = DAG.getZeroExtendInReg(Res, dl, AtomicType);
2802  RHS = DAG.getZeroExtendInReg(Node->getOperand(2), dl, AtomicType);
2803  break;
2804  default:
2805  llvm_unreachable("Invalid atomic op extension");
2806  }
2807 
2808  SDValue Success =
2809  DAG.getSetCC(dl, Node->getValueType(1), LHS, RHS, ISD::SETEQ);
2810 
2811  Results.push_back(ExtRes.getValue(0));
2812  Results.push_back(Success);
2813  Results.push_back(Res.getValue(1));
2814  break;
2815  }
2817  ExpandDYNAMIC_STACKALLOC(Node, Results);
2818  break;
2819  case ISD::MERGE_VALUES:
2820  for (unsigned i = 0; i < Node->getNumValues(); i++)
2821  Results.push_back(Node->getOperand(i));
2822  break;
2823  case ISD::UNDEF: {
2824  EVT VT = Node->getValueType(0);
2825  if (VT.isInteger())
2826  Results.push_back(DAG.getConstant(0, dl, VT));
2827  else {
2828  assert(VT.isFloatingPoint() && "Unknown value type!");
2829  Results.push_back(DAG.getConstantFP(0, dl, VT));
2830  }
2831  break;
2832  }
2833  case ISD::STRICT_FP_ROUND:
2834  // When strict mode is enforced we can't do expansion because it
2835  // does not honor the "strict" properties. Only libcall is allowed.
2836  if (TLI.isStrictFPEnabled())
2837  break;
2838  // We might as well mutate to FP_ROUND when FP_ROUND operation is legal
2839  // since this operation is more efficient than stack operation.
2840  if (TLI.getStrictFPOperationAction(Node->getOpcode(),
2841  Node->getValueType(0))
2843  break;
2844  // We fall back to use stack operation when the FP_ROUND operation
2845  // isn't available.
2846  if ((Tmp1 = EmitStackConvert(Node->getOperand(1), Node->getValueType(0),
2847  Node->getValueType(0), dl,
2848  Node->getOperand(0)))) {
2849  ReplaceNode(Node, Tmp1.getNode());
2850  LLVM_DEBUG(dbgs() << "Successfully expanded STRICT_FP_ROUND node\n");
2851  return true;
2852  }
2853  break;
2854  case ISD::FP_ROUND:
2855  case ISD::BITCAST:
2856  if ((Tmp1 = EmitStackConvert(Node->getOperand(0), Node->getValueType(0),
2857  Node->getValueType(0), dl)))
2858  Results.push_back(Tmp1);
2859  break;
2860  case ISD::STRICT_FP_EXTEND:
2861  // When strict mode is enforced we can't do expansion because it
2862  // does not honor the "strict" properties. Only libcall is allowed.
2863  if (TLI.isStrictFPEnabled())
2864  break;
2865  // We might as well mutate to FP_EXTEND when FP_EXTEND operation is legal
2866  // since this operation is more efficient than stack operation.
2867  if (TLI.getStrictFPOperationAction(Node->getOpcode(),
2868  Node->getValueType(0))
2870  break;
2871  // We fall back to use stack operation when the FP_EXTEND operation
2872  // isn't available.
2873  if ((Tmp1 = EmitStackConvert(
2874  Node->getOperand(1), Node->getOperand(1).getValueType(),
2875  Node->getValueType(0), dl, Node->getOperand(0)))) {
2876  ReplaceNode(Node, Tmp1.getNode());
2877  LLVM_DEBUG(dbgs() << "Successfully expanded STRICT_FP_EXTEND node\n");
2878  return true;
2879  }
2880  break;
2881  case ISD::FP_EXTEND:
2882  if ((Tmp1 = EmitStackConvert(Node->getOperand(0),
2883  Node->getOperand(0).getValueType(),
2884  Node->getValueType(0), dl)))
2885  Results.push_back(Tmp1);
2886  break;
2887  case ISD::SIGN_EXTEND_INREG: {
2888  EVT ExtraVT = cast<VTSDNode>(Node->getOperand(1))->getVT();
2889  EVT VT = Node->getValueType(0);
2890 
2891  // An in-register sign-extend of a boolean is a negation:
2892  // 'true' (1) sign-extended is -1.
2893  // 'false' (0) sign-extended is 0.
2894  // However, we must mask the high bits of the source operand because the
2895  // SIGN_EXTEND_INREG does not guarantee that the high bits are already zero.
2896 
2897  // TODO: Do this for vectors too?
2898  if (ExtraVT.getSizeInBits() == 1) {
2899  SDValue One = DAG.getConstant(1, dl, VT);
2900  SDValue And = DAG.getNode(ISD::AND, dl, VT, Node->getOperand(0), One);
2901  SDValue Zero = DAG.getConstant(0, dl, VT);
2902  SDValue Neg = DAG.getNode(ISD::SUB, dl, VT, Zero, And);
2903  Results.push_back(Neg);
2904  break;
2905  }
2906 
2907  // NOTE: we could fall back on load/store here too for targets without
2908  // SRA. However, it is doubtful that any exist.
2909  EVT ShiftAmountTy = TLI.getShiftAmountTy(VT, DAG.getDataLayout());
2910  unsigned BitsDiff = VT.getScalarSizeInBits() -
2911  ExtraVT.getScalarSizeInBits();
2912  SDValue ShiftCst = DAG.getConstant(BitsDiff, dl, ShiftAmountTy);
2913  Tmp1 = DAG.getNode(ISD::SHL, dl, Node->getValueType(0),
2914  Node->getOperand(0), ShiftCst);
2915  Tmp1 = DAG.getNode(ISD::SRA, dl, Node->getValueType(0), Tmp1, ShiftCst);
2916  Results.push_back(Tmp1);
2917  break;
2918  }
2919  case ISD::UINT_TO_FP:
2921  if (TLI.expandUINT_TO_FP(Node, Tmp1, Tmp2, DAG)) {
2922  Results.push_back(Tmp1);
2923  if (Node->isStrictFPOpcode())
2924  Results.push_back(Tmp2);
2925  break;
2926  }
2928  case ISD::SINT_TO_FP:
2930  if ((Tmp1 = ExpandLegalINT_TO_FP(Node, Tmp2))) {
2931  Results.push_back(Tmp1);
2932  if (Node->isStrictFPOpcode())
2933  Results.push_back(Tmp2);
2934  }
2935  break;
2936  case ISD::FP_TO_SINT:
2937  if (TLI.expandFP_TO_SINT(Node, Tmp1, DAG))
2938  Results.push_back(Tmp1);
2939  break;
2941  if (TLI.expandFP_TO_SINT(Node, Tmp1, DAG)) {
2942  ReplaceNode(Node, Tmp1.getNode());
2943  LLVM_DEBUG(dbgs() << "Successfully expanded STRICT_FP_TO_SINT node\n");
2944  return true;
2945  }
2946  break;
2947  case ISD::FP_TO_UINT:
2948  if (TLI.expandFP_TO_UINT(Node, Tmp1, Tmp2, DAG))
2949  Results.push_back(Tmp1);
2950  break;
2952  if (TLI.expandFP_TO_UINT(Node, Tmp1, Tmp2, DAG)) {
2953  // Relink the chain.
2954  DAG.ReplaceAllUsesOfValueWith(SDValue(Node,1), Tmp2);
2955  // Replace the new UINT result.
2956  ReplaceNodeWithValue(SDValue(Node, 0), Tmp1);
2957  LLVM_DEBUG(dbgs() << "Successfully expanded STRICT_FP_TO_UINT node\n");
2958  return true;
2959  }
2960  break;
2961  case ISD::FP_TO_SINT_SAT:
2962  case ISD::FP_TO_UINT_SAT:
2963  Results.push_back(TLI.expandFP_TO_INT_SAT(Node, DAG));
2964  break;
2965  case ISD::VAARG:
2966  Results.push_back(DAG.expandVAArg(Node));
2967  Results.push_back(Results[0].getValue(1));
2968  break;
2969  case ISD::VACOPY:
2970  Results.push_back(DAG.expandVACopy(Node));
2971  break;
2973  if (Node->getOperand(0).getValueType().getVectorNumElements() == 1)
2974  // This must be an access of the only element. Return it.
2975  Tmp1 = DAG.getNode(ISD::BITCAST, dl, Node->getValueType(0),
2976  Node->getOperand(0));
2977  else
2978  Tmp1 = ExpandExtractFromVectorThroughStack(SDValue(Node, 0));
2979  Results.push_back(Tmp1);
2980  break;
2982  Results.push_back(ExpandExtractFromVectorThroughStack(SDValue(Node, 0)));
2983  break;
2984  case ISD::INSERT_SUBVECTOR:
2985  Results.push_back(ExpandInsertToVectorThroughStack(SDValue(Node, 0)));
2986  break;
2987  case ISD::CONCAT_VECTORS:
2988  Results.push_back(ExpandVectorBuildThroughStack(Node));
2989  break;
2990  case ISD::SCALAR_TO_VECTOR:
2991  Results.push_back(ExpandSCALAR_TO_VECTOR(Node));
2992  break;
2994  Results.push_back(ExpandINSERT_VECTOR_ELT(Node->getOperand(0),
2995  Node->getOperand(1),
2996  Node->getOperand(2), dl));
2997  break;
2998  case ISD::VECTOR_SHUFFLE: {
2999  SmallVector<int, 32> NewMask;
3000  ArrayRef<int> Mask = cast<ShuffleVectorSDNode>(Node)->getMask();
3001 
3002  EVT VT = Node->getValueType(0);
3003  EVT EltVT = VT.getVectorElementType();
3004  SDValue Op0 = Node->getOperand(0);
3005  SDValue Op1 = Node->getOperand(1);
3006  if (!TLI.isTypeLegal(EltVT)) {
3007  EVT NewEltVT = TLI.getTypeToTransformTo(*DAG.getContext(), EltVT);
3008 
3009  // BUILD_VECTOR operands are allowed to be wider than the element type.
3010  // But if NewEltVT is smaller that EltVT the BUILD_VECTOR does not accept
3011  // it.
3012  if (NewEltVT.bitsLT(EltVT)) {
3013  // Convert shuffle node.
3014  // If original node was v4i64 and the new EltVT is i32,
3015  // cast operands to v8i32 and re-build the mask.
3016 
3017  // Calculate new VT, the size of the new VT should be equal to original.
3018  EVT NewVT =
3019  EVT::getVectorVT(*DAG.getContext(), NewEltVT,
3020  VT.getSizeInBits() / NewEltVT.getSizeInBits());
3021  assert(NewVT.bitsEq(VT));
3022 
3023  // cast operands to new VT
3024  Op0 = DAG.getNode(ISD::BITCAST, dl, NewVT, Op0);
3025  Op1 = DAG.getNode(ISD::BITCAST, dl, NewVT, Op1);
3026 
3027  // Convert the shuffle mask
3028  unsigned int factor =
3030 
3031  // EltVT gets smaller
3032  assert(factor > 0);
3033 
3034  for (unsigned i = 0; i < VT.getVectorNumElements(); ++i) {
3035  if (Mask[i] < 0) {
3036  for (unsigned fi = 0; fi < factor; ++fi)
3037  NewMask.push_back(Mask[i]);
3038  }
3039  else {
3040  for (unsigned fi = 0; fi < factor; ++fi)
3041  NewMask.push_back(Mask[i]*factor+fi);
3042  }
3043  }
3044  Mask = NewMask;
3045  VT = NewVT;
3046  }
3047  EltVT = NewEltVT;
3048  }
3049  unsigned NumElems = VT.getVectorNumElements();
3051  for (unsigned i = 0; i != NumElems; ++i) {
3052  if (Mask[i] < 0) {
3053  Ops.push_back(DAG.getUNDEF(EltVT));
3054  continue;
3055  }
3056  unsigned Idx = Mask[i];
3057  if (Idx < NumElems)
3058  Ops.push_back(DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT, Op0,
3059  DAG.getVectorIdxConstant(Idx, dl)));
3060  else
3061  Ops.push_back(
3062  DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT, Op1,
3063  DAG.getVectorIdxConstant(Idx - NumElems, dl)));
3064  }
3065 
3066  Tmp1 = DAG.getBuildVector(VT, dl, Ops);
3067  // We may have changed the BUILD_VECTOR type. Cast it back to the Node type.
3068  Tmp1 = DAG.getNode(ISD::BITCAST, dl, Node->getValueType(0), Tmp1);
3069  Results.push_back(Tmp1);
3070  break;
3071  }
3072  case ISD::VECTOR_SPLICE: {
3073  Results.push_back(TLI.expandVectorSplice(Node, DAG));
3074  break;
3075  }
3076  case ISD::EXTRACT_ELEMENT: {
3077  EVT OpTy = Node->getOperand(0).getValueType();
3078  if (cast<ConstantSDNode>(Node->getOperand(1))->getZExtValue()) {
3079  // 1 -> Hi
3080  Tmp1 = DAG.getNode(ISD::SRL, dl, OpTy, Node->getOperand(0),
3081  DAG.getConstant(OpTy.getSizeInBits() / 2, dl,
3082  TLI.getShiftAmountTy(
3083  Node->getOperand(0).getValueType(),
3084  DAG.getDataLayout())));
3085  Tmp1 = DAG.getNode(ISD::TRUNCATE, dl, Node->getValueType(0), Tmp1);
3086  } else {
3087  // 0 -> Lo
3088  Tmp1 = DAG.getNode(ISD::TRUNCATE, dl, Node->getValueType(0),
3089  Node->getOperand(0));
3090  }
3091  Results.push_back(Tmp1);
3092  break;
3093  }
3094  case ISD::STACKSAVE:
3095  // Expand to CopyFromReg if the target set
3096  // StackPointerRegisterToSaveRestore.
3097  if (Register SP = TLI.getStackPointerRegisterToSaveRestore()) {
3098  Results.push_back(DAG.getCopyFromReg(Node->getOperand(0), dl, SP,
3099  Node->getValueType(0)));
3100  Results.push_back(Results[0].getValue(1));
3101  } else {
3102  Results.push_back(DAG.getUNDEF(Node->getValueType(0)));
3103  Results.push_back(Node->getOperand(0));
3104  }
3105  break;
3106  case ISD::STACKRESTORE:
3107  // Expand to CopyToReg if the target set
3108  // StackPointerRegisterToSaveRestore.
3109  if (Register SP = TLI.getStackPointerRegisterToSaveRestore()) {
3110  Results.push_back(DAG.getCopyToReg(Node->getOperand(0), dl, SP,
3111  Node->getOperand(1)));
3112  } else {
3113  Results.push_back(Node->getOperand(0));
3114  }
3115  break;
3117  Results.push_back(DAG.getConstant(0, dl, Node->getValueType(0)));
3118  Results.push_back(Results[0].getValue(0));
3119  break;
3120  case ISD::FCOPYSIGN:
3121  Results.push_back(ExpandFCOPYSIGN(Node));
3122  break;
3123  case ISD::FNEG:
3124  Results.push_back(ExpandFNEG(Node));
3125  break;
3126  case ISD::FABS:
3127  Results.push_back(ExpandFABS(Node));
3128  break;
3129  case ISD::SMIN:
3130  case ISD::SMAX:
3131  case ISD::UMIN:
3132  case ISD::UMAX: {
3133  // Expand Y = MAX(A, B) -> Y = (A > B) ? A : B
3134  ISD::CondCode Pred;
3135  switch (Node->getOpcode()) {
3136  default: llvm_unreachable("How did we get here?");
3137  case ISD::SMAX: Pred = ISD::SETGT; break;
3138  case ISD::SMIN: Pred = ISD::SETLT; break;
3139  case ISD::UMAX: Pred = ISD::SETUGT; break;
3140  case ISD::UMIN: Pred = ISD::SETULT; break;
3141  }
3142  Tmp1 = Node->getOperand(0);
3143  Tmp2 = Node->getOperand(1);
3144  Tmp1 = DAG.getSelectCC(dl, Tmp1, Tmp2, Tmp1, Tmp2, Pred);
3145  Results.push_back(Tmp1);
3146  break;
3147  }
3148  case ISD::FMINNUM:
3149  case ISD::FMAXNUM: {
3150  if (SDValue Expanded = TLI.expandFMINNUM_FMAXNUM(Node, DAG))
3151  Results.push_back(Expanded);
3152  break;
3153  }
3154  case ISD::FSIN:
3155  case ISD::FCOS: {
3156  EVT VT = Node->getValueType(0);
3157  // Turn fsin / fcos into ISD::FSINCOS node if there are a pair of fsin /
3158  // fcos which share the same operand and both are used.
3159  if ((TLI.isOperationLegalOrCustom(ISD::FSINCOS, VT) ||
3160  isSinCosLibcallAvailable(Node, TLI))
3161  && useSinCos(Node)) {
3162  SDVTList VTs = DAG.getVTList(VT, VT);
3163  Tmp1 = DAG.getNode(ISD::FSINCOS, dl, VTs, Node->getOperand(0));
3164  if (Node->getOpcode() == ISD::FCOS)
3165  Tmp1 = Tmp1.getValue(1);
3166  Results.push_back(Tmp1);
3167  }
3168  break;
3169  }
3170  case ISD::FMAD:
3171  llvm_unreachable("Illegal fmad should never be formed");
3172 
3173  case ISD::FP16_TO_FP:
3174  if (Node->getValueType(0) != MVT::f32) {
3175  // We can extend to types bigger than f32 in two steps without changing
3176  // the result. Since "f16 -> f32" is much more commonly available, give
3177  // CodeGen the option of emitting that before resorting to a libcall.
3178  SDValue Res =
3179  DAG.getNode(ISD::FP16_TO_FP, dl, MVT::f32, Node->getOperand(0));
3180  Results.push_back(
3181  DAG.getNode(ISD::FP_EXTEND, dl, Node->getValueType(0), Res));
3182  }
3183  break;
3185  if (Node->getValueType(0) != MVT::f32) {
3186  // We can extend to types bigger than f32 in two steps without changing
3187  // the result. Since "f16 -> f32" is much more commonly available, give
3188  // CodeGen the option of emitting that before resorting to a libcall.
3189  SDValue Res =
3191  {Node->getOperand(0), Node->getOperand(1)});
3192  Res = DAG.getNode(ISD::STRICT_FP_EXTEND, dl,
3193  {Node->getValueType(0), MVT::Other},
3194  {Res.getValue(1), Res});
3195  Results.push_back(Res);
3196  Results.push_back(Res.getValue(1));
3197  }
3198  break;
3199  case ISD::FP_TO_FP16:
3200  LLVM_DEBUG(dbgs() << "Legalizing FP_TO_FP16\n");
3201  if (!TLI.useSoftFloat() && TM.Options.UnsafeFPMath) {
3202  SDValue Op = Node->getOperand(0);
3203  MVT SVT = Op.getSimpleValueType();
3204  if ((SVT == MVT::f64 || SVT == MVT::f80) &&
3205  TLI.isOperationLegalOrCustom(ISD::FP_TO_FP16, MVT::f32)) {
3206  // Under fastmath, we can expand this node into a fround followed by
3207  // a float-half conversion.
3208  SDValue FloatVal = DAG.getNode(ISD::FP_ROUND, dl, MVT::f32, Op,
3209  DAG.getIntPtrConstant(0, dl));
3210  Results.push_back(
3211  DAG.getNode(ISD::FP_TO_FP16, dl, Node->getValueType(0), FloatVal));
3212  }
3213  }
3214  break;
3215  case ISD::ConstantFP: {
3216  ConstantFPSDNode *CFP = cast<ConstantFPSDNode>(Node);
3217  // Check to see if this FP immediate is already legal.
3218  // If this is a legal constant, turn it into a TargetConstantFP node.
3219  if (!TLI.isFPImmLegal(CFP->getValueAPF(), Node->getValueType(0),
3220  DAG.shouldOptForSize()))
3221  Results.push_back(ExpandConstantFP(CFP, true));
3222  break;
3223  }
3224  case ISD::Constant: {
3225  ConstantSDNode *CP = cast<ConstantSDNode>(Node);
3226  Results.push_back(ExpandConstant(CP));
3227  break;
3228  }
3229  case ISD::FSUB: {
3230  EVT VT = Node->getValueType(0);
3231  if (TLI.isOperationLegalOrCustom(ISD::FADD, VT) &&
3232  TLI.isOperationLegalOrCustom(ISD::FNEG, VT)) {
3233  const SDNodeFlags Flags = Node->getFlags();
3234  Tmp1 = DAG.getNode(ISD::FNEG, dl, VT, Node->getOperand(1));
3235  Tmp1 = DAG.getNode(ISD::FADD, dl, VT, Node->getOperand(0), Tmp1, Flags);
3236  Results.push_back(Tmp1);
3237  }
3238  break;
3239  }
3240  case ISD::SUB: {
3241  EVT VT = Node->getValueType(0);
3242  assert(TLI.isOperationLegalOrCustom(ISD::ADD, VT) &&
3243  TLI.isOperationLegalOrCustom(ISD::XOR, VT) &&
3244  "Don't know how to expand this subtraction!");
3245  Tmp1 = DAG.getNode(ISD::XOR, dl, VT, Node->getOperand(1),
3246  DAG.getConstant(APInt::getAllOnesValue(VT.getSizeInBits()), dl,
3247  VT));
3248  Tmp1 = DAG.getNode(ISD::ADD, dl, VT, Tmp1, DAG.getConstant(1, dl, VT));
3249  Results.push_back(DAG.getNode(ISD::ADD, dl, VT, Node->getOperand(0), Tmp1));
3250  break;
3251  }
3252  case ISD::UREM:
3253  case ISD::SREM:
3254  if (TLI.expandREM(Node, Tmp1, DAG))
3255  Results.push_back(Tmp1);
3256  break;
3257  case ISD::UDIV:
3258  case ISD::SDIV: {
3259  bool isSigned = Node->getOpcode() == ISD::SDIV;
3260  unsigned DivRemOpc = isSigned ? ISD::SDIVREM : ISD::UDIVREM;
3261  EVT VT = Node->getValueType(0);
3262  if (TLI.isOperationLegalOrCustom(DivRemOpc, VT)) {
3263  SDVTList VTs = DAG.getVTList(VT, VT);
3264  Tmp1 = DAG.getNode(DivRemOpc, dl, VTs, Node->getOperand(0),
3265  Node->getOperand(1));
3266  Results.push_back(Tmp1);
3267  }
3268  break;
3269  }
3270  case ISD::MULHU:
3271  case ISD::MULHS: {
3272  unsigned ExpandOpcode =
3273  Node->getOpcode() == ISD::MULHU ? ISD::UMUL_LOHI : ISD::SMUL_LOHI;
3274  EVT VT = Node->getValueType(0);
3275  SDVTList VTs = DAG.getVTList(VT, VT);
3276 
3277  Tmp1 = DAG.getNode(ExpandOpcode, dl, VTs, Node->getOperand(0),
3278  Node->getOperand(1));
3279  Results.push_back(Tmp1.getValue(1));
3280  break;
3281  }
3282  case ISD::UMUL_LOHI:
3283  case ISD::SMUL_LOHI: {
3284  SDValue LHS = Node->getOperand(0);
3285  SDValue RHS = Node->getOperand(1);
3286  MVT VT = LHS.getSimpleValueType();
3287  unsigned MULHOpcode =
3288  Node->getOpcode() == ISD::UMUL_LOHI ? ISD::MULHU : ISD::MULHS;
3289 
3290  if (TLI.isOperationLegalOrCustom(MULHOpcode, VT)) {
3291  Results.push_back(DAG.getNode(ISD::MUL, dl, VT, LHS, RHS));
3292  Results.push_back(DAG.getNode(MULHOpcode, dl, VT, LHS, RHS));
3293  break;
3294  }
3295 
3296  SmallVector<SDValue, 4> Halves;
3297  EVT HalfType = EVT(VT).getHalfSizedIntegerVT(*DAG.getContext());
3298  assert(TLI.isTypeLegal(HalfType));
3299  if (TLI.expandMUL_LOHI(Node->getOpcode(), VT, dl, LHS, RHS, Halves,
3300  HalfType, DAG,
3302  for (unsigned i = 0; i < 2; ++i) {
3303  SDValue Lo = DAG.getNode(ISD::ZERO_EXTEND, dl, VT, Halves[2 * i]);
3304  SDValue Hi = DAG.getNode(ISD::ANY_EXTEND, dl, VT, Halves[2 * i + 1]);
3305  SDValue Shift = DAG.getConstant(
3306  HalfType.getScalarSizeInBits(), dl,
3307  TLI.getShiftAmountTy(HalfType, DAG.getDataLayout()));
3308  Hi = DAG.getNode(ISD::SHL, dl, VT, Hi, Shift);
3309  Results.push_back(DAG.getNode(ISD::OR, dl, VT, Lo, Hi));
3310  }
3311  break;
3312  }
3313  break;
3314  }
3315  case ISD::MUL: {
3316  EVT VT = Node->getValueType(0);
3317  SDVTList VTs = DAG.getVTList(VT, VT);
3318  // See if multiply or divide can be lowered using two-result operations.
3319  // We just need the low half of the multiply; try both the signed
3320  // and unsigned forms. If the target supports both SMUL_LOHI and
3321  // UMUL_LOHI, form a preference by checking which forms of plain
3322  // MULH it supports.
3323  bool HasSMUL_LOHI = TLI.isOperationLegalOrCustom(ISD::SMUL_LOHI, VT);
3324  bool HasUMUL_LOHI = TLI.isOperationLegalOrCustom(ISD::UMUL_LOHI, VT);
3325  bool HasMULHS = TLI.isOperationLegalOrCustom(ISD::MULHS, VT);
3326  bool HasMULHU = TLI.isOperationLegalOrCustom(ISD::MULHU, VT);
3327  unsigned OpToUse = 0;
3328  if (HasSMUL_LOHI && !HasMULHS) {
3329  OpToUse = ISD::SMUL_LOHI;
3330  } else if (HasUMUL_LOHI && !HasMULHU) {
3331  OpToUse = ISD::UMUL_LOHI;
3332  } else if (HasSMUL_LOHI) {
3333  OpToUse = ISD::SMUL_LOHI;
3334  } else if (HasUMUL_LOHI) {
3335  OpToUse = ISD::UMUL_LOHI;
3336  }
3337  if (OpToUse) {
3338  Results.push_back(DAG.getNode(OpToUse, dl, VTs, Node->getOperand(0),
3339  Node->getOperand(1)));
3340  break;
3341  }
3342 
3343  SDValue Lo, Hi;
3344  EVT HalfType = VT.getHalfSizedIntegerVT(*DAG.getContext());
3345  if (TLI.isOperationLegalOrCustom(ISD::ZERO_EXTEND, VT) &&
3346  TLI.isOperationLegalOrCustom(ISD::ANY_EXTEND, VT) &&
3347  TLI.isOperationLegalOrCustom(ISD::SHL, VT) &&
3348  TLI.isOperationLegalOrCustom(ISD::OR, VT) &&
3349  TLI.expandMUL(Node, Lo, Hi, HalfType, DAG,
3351  Lo = DAG.getNode(ISD::ZERO_EXTEND, dl, VT, Lo);
3352  Hi = DAG.getNode(ISD::ANY_EXTEND, dl, VT, Hi);
3353  SDValue Shift =
3354  DAG.getConstant(HalfType.getSizeInBits(), dl,
3355  TLI.getShiftAmountTy(HalfType, DAG.getDataLayout()));
3356  Hi = DAG.getNode(ISD::SHL, dl, VT, Hi, Shift);
3357  Results.push_back(DAG.getNode(ISD::OR, dl, VT, Lo, Hi));
3358  }
3359  break;
3360  }
3361  case ISD::FSHL:
3362  case ISD::FSHR:
3363  if (TLI.expandFunnelShift(Node, Tmp1, DAG))
3364  Results.push_back(Tmp1);
3365  break;
3366  case ISD::ROTL:
3367  case ISD::ROTR:
3368  if (TLI.expandROT(Node, true /*AllowVectorOps*/, Tmp1, DAG))
3369  Results.push_back(Tmp1);
3370  break;
3371  case ISD::SADDSAT:
3372  case ISD::UADDSAT:
3373  case ISD::SSUBSAT:
3374  case ISD::USUBSAT:
3375  Results.push_back(TLI.expandAddSubSat(Node, DAG));
3376  break;
3377  case ISD::SSHLSAT:
3378  case ISD::USHLSAT:
3379  Results.push_back(TLI.expandShlSat(Node, DAG));
3380  break;
3381  case ISD::SMULFIX:
3382  case ISD::SMULFIXSAT:
3383  case ISD::UMULFIX:
3384  case ISD::UMULFIXSAT:
3385  Results.push_back(TLI.expandFixedPointMul(Node, DAG));
3386  break;
3387  case ISD::SDIVFIX:
3388  case ISD::SDIVFIXSAT:
3389  case ISD::UDIVFIX:
3390  case ISD::UDIVFIXSAT:
3391  if (SDValue V = TLI.expandFixedPointDiv(Node->getOpcode(), SDLoc(Node),
3392  Node->getOperand(0),
3393  Node->getOperand(1),
3394  Node->getConstantOperandVal(2),
3395  DAG)) {
3396  Results.push_back(V);
3397  break;
3398  }
3399  // FIXME: We might want to retry here with a wider type if we fail, if that
3400  // type is legal.
3401  // FIXME: Technically, so long as we only have sdivfixes where BW+Scale is
3402  // <= 128 (which is the case for all of the default Embedded-C types),
3403  // we will only get here with types and scales that we could always expand
3404  // if we were allowed to generate libcalls to division functions of illegal
3405  // type. But we cannot do that.
3406  llvm_unreachable("Cannot expand DIVFIX!");
3407  case ISD::ADDCARRY:
3408  case ISD::SUBCARRY: {
3409  SDValue LHS = Node->getOperand(0);
3410  SDValue RHS = Node->getOperand(1);
3411  SDValue Carry = Node->getOperand(2);
3412 
3413  bool IsAdd = Node->getOpcode() == ISD::ADDCARRY;
3414 
3415  // Initial add of the 2 operands.
3416  unsigned Op = IsAdd ? ISD::ADD : ISD::SUB;
3417  EVT VT = LHS.getValueType();
3418  SDValue Sum = DAG.getNode(Op, dl, VT, LHS, RHS);
3419 
3420  // Initial check for overflow.
3421  EVT CarryType = Node->getValueType(1);
3422  EVT SetCCType = getSetCCResultType(Node->getValueType(0));
3423  ISD::CondCode CC = IsAdd ? ISD::SETULT : ISD::SETUGT;
3424  SDValue Overflow = DAG.getSetCC(dl, SetCCType, Sum, LHS, CC);
3425 
3426  // Add of the sum and the carry.
3427  SDValue One = DAG.getConstant(1, dl, VT);
3428  SDValue CarryExt =
3429  DAG.getNode(ISD::AND, dl, VT, DAG.getZExtOrTrunc(Carry, dl, VT), One);
3430  SDValue Sum2 = DAG.getNode(Op, dl, VT, Sum, CarryExt);
3431 
3432  // Second check for overflow. If we are adding, we can only overflow if the
3433  // initial sum is all 1s ang the carry is set, resulting in a new sum of 0.
3434  // If we are subtracting, we can only overflow if the initial sum is 0 and
3435  // the carry is set, resulting in a new sum of all 1s.
3436  SDValue Zero = DAG.getConstant(0, dl, VT);
3437  SDValue Overflow2 =
3438  IsAdd ? DAG.getSetCC(dl, SetCCType, Sum2, Zero, ISD::SETEQ)
3439  : DAG.getSetCC(dl, SetCCType, Sum, Zero, ISD::SETEQ);
3440  Overflow2 = DAG.getNode(ISD::AND, dl, SetCCType, Overflow2,
3441  DAG.getZExtOrTrunc(Carry, dl, SetCCType));
3442 
3443  SDValue ResultCarry =
3444  DAG.getNode(ISD::OR, dl, SetCCType, Overflow, Overflow2);
3445 
3446  Results.push_back(Sum2);
3447  Results.push_back(DAG.getBoolExtOrTrunc(ResultCarry, dl, CarryType, VT));
3448  break;
3449  }
3450  case ISD::SADDO:
3451  case ISD::SSUBO: {
3452  SDValue Result, Overflow;
3453  TLI.expandSADDSUBO(Node, Result, Overflow, DAG);
3454  Results.push_back(Result);
3455  Results.push_back(Overflow);
3456  break;
3457  }
3458  case ISD::UADDO:
3459  case ISD::USUBO: {
3460  SDValue Result, Overflow;
3461  TLI.expandUADDSUBO(Node, Result, Overflow, DAG);
3462  Results.push_back(Result);
3463  Results.push_back(Overflow);
3464  break;
3465  }
3466  case ISD::UMULO:
3467  case ISD::SMULO: {
3468  SDValue Result, Overflow;
3469  if (TLI.expandMULO(Node, Result, Overflow, DAG)) {
3470  Results.push_back(Result);
3471  Results.push_back(Overflow);
3472  }
3473  break;
3474  }
3475  case ISD::BUILD_PAIR: {
3476  EVT PairTy = Node->getValueType(0);
3477  Tmp1 = DAG.getNode(ISD::ZERO_EXTEND, dl, PairTy, Node->getOperand(0));
3478  Tmp2 = DAG.getNode(ISD::ANY_EXTEND, dl, PairTy, Node->getOperand(1));
3479  Tmp2 = DAG.getNode(
3480  ISD::SHL, dl, PairTy, Tmp2,
3481  DAG.getConstant(PairTy.getSizeInBits() / 2, dl,
3482  TLI.getShiftAmountTy(PairTy, DAG.getDataLayout())));
3483  Results.push_back(DAG.getNode(ISD::OR, dl, PairTy, Tmp1, Tmp2));
3484  break;
3485  }
3486  case ISD::SELECT:
3487  Tmp1 = Node->getOperand(0);
3488  Tmp2 = Node->getOperand(1);
3489  Tmp3 = Node->getOperand(2);
3490  if (Tmp1.getOpcode() == ISD::SETCC) {
3491  Tmp1 = DAG.getSelectCC(dl, Tmp1.getOperand(0), Tmp1.getOperand(1),
3492  Tmp2, Tmp3,
3493  cast<CondCodeSDNode>(Tmp1.getOperand(2))->get());
3494  } else {
3495  Tmp1 = DAG.getSelectCC(dl, Tmp1,
3496  DAG.getConstant(0, dl, Tmp1.getValueType()),
3497  Tmp2, Tmp3, ISD::SETNE);
3498  }
3499  Tmp1->setFlags(Node->getFlags());
3500  Results.push_back(Tmp1);
3501  break;
3502  case ISD::BR_JT: {
3503  SDValue Chain = Node->getOperand(0);
3504  SDValue Table = Node->getOperand(1);
3505  SDValue Index = Node->getOperand(2);
3506 
3507  const DataLayout &TD = DAG.getDataLayout();
3508  EVT PTy = TLI.getPointerTy(TD);
3509 
3510  unsigned EntrySize =
3511  DAG.getMachineFunction().getJumpTableInfo()->getEntrySize(TD);
3512 
3513  // For power-of-two jumptable entry sizes convert multiplication to a shift.
3514  // This transformation needs to be done here since otherwise the MIPS
3515  // backend will end up emitting a three instruction multiply sequence
3516  // instead of a single shift and MSP430 will call a runtime function.
3517  if (llvm::isPowerOf2_32(EntrySize))
3518  Index = DAG.getNode(
3519  ISD::SHL, dl, Index.getValueType(), Index,
3520  DAG.getConstant(llvm::Log2_32(EntrySize), dl, Index.getValueType()));
3521  else
3522  Index = DAG.getNode(ISD::MUL, dl, Index.getValueType(), Index,
3523  DAG.getConstant(EntrySize, dl, Index.getValueType()));
3524  SDValue Addr = DAG.getNode(ISD::ADD, dl, Index.getValueType(),
3525  Index, Table);
3526 
3527  EVT MemVT = EVT::getIntegerVT(*DAG.getContext(), EntrySize * 8);
3528  SDValue LD = DAG.getExtLoad(
3529  ISD::SEXTLOAD, dl, PTy, Chain, Addr,
3530  MachinePointerInfo::getJumpTable(DAG.getMachineFunction()), MemVT);
3531  Addr = LD;
3532  if (TLI.isJumpTableRelative()) {
3533  // For PIC, the sequence is:
3534  // BRIND(load(Jumptable + index) + RelocBase)
3535  // RelocBase can be JumpTable, GOT or some sort of global base.
3536  Addr = DAG.getNode(ISD::ADD, dl, PTy, Addr,
3537  TLI.getPICJumpTableRelocBase(Table, DAG));
3538  }
3539 
3540  Tmp1 = TLI.expandIndirectJTBranch(dl, LD.getValue(1), Addr, DAG);
3541  Results.push_back(Tmp1);
3542  break;
3543  }
3544  case ISD::BRCOND:
3545  // Expand brcond's setcc into its constituent parts and create a BR_CC
3546  // Node.
3547  Tmp1 = Node->getOperand(0);
3548  Tmp2 = Node->getOperand(1);
3549  if (Tmp2.getOpcode() == ISD::SETCC) {
3550  Tmp1 = DAG.getNode(ISD::BR_CC, dl, MVT::Other,
3551  Tmp1, Tmp2.getOperand(2),
3552  Tmp2.getOperand(0), Tmp2.getOperand(1),
3553  Node->getOperand(2));
3554  } else {
3555  // We test only the i1 bit. Skip the AND if UNDEF or another AND.
3556  if (Tmp2.isUndef() ||
3557  (Tmp2.getOpcode() == ISD::AND &&
3558  isa<ConstantSDNode>(Tmp2.getOperand(1)) &&
3559  cast<ConstantSDNode>(Tmp2.getOperand(1))->getZExtValue() == 1))
3560  Tmp3 = Tmp2;
3561  else
3562  Tmp3 = DAG.getNode(ISD::AND, dl, Tmp2.getValueType(), Tmp2,
3563  DAG.getConstant(1, dl, Tmp2.getValueType()));
3564  Tmp1 = DAG.getNode(ISD::BR_CC, dl, MVT::Other, Tmp1,
3565  DAG.getCondCode(ISD::SETNE), Tmp3,
3566  DAG.getConstant(0, dl, Tmp3.getValueType()),
3567  Node->getOperand(2));
3568  }
3569  Results.push_back(Tmp1);
3570  break;
3571  case ISD::SETCC:
3572  case ISD::STRICT_FSETCC:
3573  case ISD::STRICT_FSETCCS: {
3574  bool IsStrict = Node->getOpcode() != ISD::SETCC;
3575  bool IsSignaling = Node->getOpcode() == ISD::STRICT_FSETCCS;
3576  SDValue Chain = IsStrict ? Node->getOperand(0) : SDValue();
3577  unsigned Offset = IsStrict ? 1 : 0;
3578  Tmp1 = Node->getOperand(0 + Offset);
3579  Tmp2 = Node->getOperand(1 + Offset);
3580  Tmp3 = Node->getOperand(2 + Offset);
3581  bool Legalized =
3582  TLI.LegalizeSetCCCondCode(DAG, Node->getValueType(0), Tmp1, Tmp2, Tmp3,
3583  NeedInvert, dl, Chain, IsSignaling);
3584 
3585  if (Legalized) {
3586  // If we expanded the SETCC by swapping LHS and RHS, or by inverting the
3587  // condition code, create a new SETCC node.
3588  if (Tmp3.getNode())
3589  Tmp1 = DAG.getNode(ISD::SETCC, dl, Node->getValueType(0),
3590  Tmp1, Tmp2, Tmp3, Node->getFlags());
3591 
3592  // If we expanded the SETCC by inverting the condition code, then wrap
3593  // the existing SETCC in a NOT to restore the intended condition.
3594  if (NeedInvert)
3595  Tmp1 = DAG.getLogicalNOT(dl, Tmp1, Tmp1->getValueType(0));
3596 
3597  Results.push_back(Tmp1);
3598  if (IsStrict)
3599  Results.push_back(Chain);
3600 
3601  break;
3602  }
3603 
3604  // FIXME: It seems Legalized is false iff CCCode is Legal. I don't
3605  // understand if this code is useful for strict nodes.
3606  assert(!IsStrict && "Don't know how to expand for strict nodes.");
3607 
3608  // Otherwise, SETCC for the given comparison type must be completely
3609  // illegal; expand it into a SELECT_CC.
3610  EVT VT = Node->getValueType(0);
3611  int TrueValue;
3612  switch (TLI.getBooleanContents(Tmp1.getValueType())) {
3615  TrueValue = 1;
3616  break;
3618  TrueValue = -1;
3619  break;
3620  }
3621  Tmp1 = DAG.getNode(ISD::SELECT_CC, dl, VT, Tmp1, Tmp2,
3622  DAG.getConstant(TrueValue, dl, VT),
3623  DAG.getConstant(0, dl, VT),
3624  Tmp3);
3625  Tmp1->setFlags(Node->getFlags());
3626  Results.push_back(Tmp1);
3627  break;
3628  }
3629  case ISD::SELECT_CC: {
3630  // TODO: need to add STRICT_SELECT_CC and STRICT_SELECT_CCS
3631  Tmp1 = Node->getOperand(0); // LHS
3632  Tmp2 = Node->getOperand(1); // RHS
3633  Tmp3 = Node->getOperand(2); // True
3634  Tmp4 = Node->getOperand(3); // False
3635  EVT VT = Node->getValueType(0);
3636  SDValue Chain;
3637  SDValue CC = Node->getOperand(4);
3638  ISD::CondCode CCOp = cast<CondCodeSDNode>(CC)->get();
3639 
3640  if (TLI.isCondCodeLegalOrCustom(CCOp, Tmp1.getSimpleValueType())) {
3641  // If the condition code is legal, then we need to expand this
3642  // node using SETCC and SELECT.
3643  EVT CmpVT = Tmp1.getValueType();
3644  assert(!TLI.isOperationExpand(ISD::SELECT, VT) &&
3645  "Cannot expand ISD::SELECT_CC when ISD::SELECT also needs to be "
3646  "expanded.");
3647  EVT CCVT = getSetCCResultType(CmpVT);
3648  SDValue Cond = DAG.getNode(ISD::SETCC, dl, CCVT, Tmp1, Tmp2, CC, Node->getFlags());
3649  Results.push_back(DAG.getSelect(dl, VT, Cond, Tmp3, Tmp4));
3650  break;
3651  }
3652 
3653  // SELECT_CC is legal, so the condition code must not be.
3654  bool Legalized = false;
3655  // Try to legalize by inverting the condition. This is for targets that
3656  // might support an ordered version of a condition, but not the unordered
3657  // version (or vice versa).
3658  ISD::CondCode InvCC = ISD::getSetCCInverse(CCOp, Tmp1.getValueType());
3659  if (TLI.isCondCodeLegalOrCustom(InvCC, Tmp1.getSimpleValueType())) {
3660  // Use the new condition code and swap true and false
3661  Legalized = true;
3662  Tmp1 = DAG.getSelectCC(dl, Tmp1, Tmp2, Tmp4, Tmp3, InvCC);
3663  Tmp1->setFlags(Node->getFlags());
3664  } else {
3665  // If The inverse is not legal, then try to swap the arguments using
3666  // the inverse condition code.
3667  ISD::CondCode SwapInvCC = ISD::getSetCCSwappedOperands(InvCC);
3668  if (TLI.isCondCodeLegalOrCustom(SwapInvCC, Tmp1.getSimpleValueType())) {
3669  // The swapped inverse condition is legal, so swap true and false,
3670  // lhs and rhs.
3671  Legalized = true;
3672  Tmp1 = DAG.getSelectCC(dl, Tmp2, Tmp1, Tmp4, Tmp3, SwapInvCC);
3673  Tmp1->setFlags(Node->getFlags());
3674  }
3675  }
3676 
3677  if (!Legalized) {
3678  Legalized = TLI.LegalizeSetCCCondCode(
3679  DAG, getSetCCResultType(Tmp1.getValueType()), Tmp1, Tmp2, CC,
3680  NeedInvert, dl, Chain);
3681 
3682  assert(Legalized && "Can't legalize SELECT_CC with legal condition!");
3683 
3684  // If we expanded the SETCC by inverting the condition code, then swap
3685  // the True/False operands to match.
3686  if (NeedInvert)
3687  std::swap(Tmp3, Tmp4);
3688 
3689  // If we expanded the SETCC by swapping LHS and RHS, or by inverting the
3690  // condition code, create a new SELECT_CC node.
3691  if (CC.getNode()) {
3692  Tmp1 = DAG.getNode(ISD::SELECT_CC, dl, Node->getValueType(0),
3693  Tmp1, Tmp2, Tmp3, Tmp4, CC);
3694  } else {
3695  Tmp2 = DAG.getConstant(0, dl, Tmp1.getValueType());
3696  CC = DAG.getCondCode(ISD::SETNE);
3697  Tmp1 = DAG.getNode(ISD::SELECT_CC, dl, Node->getValueType(0), Tmp1,
3698  Tmp2, Tmp3, Tmp4, CC);
3699  }
3700  Tmp1->setFlags(Node->getFlags());
3701  }
3702  Results.push_back(Tmp1);
3703  break;
3704  }
3705  case ISD::BR_CC: {
3706  // TODO: need to add STRICT_BR_CC and STRICT_BR_CCS
3707  SDValue Chain;
3708  Tmp1 = Node->getOperand(0); // Chain
3709  Tmp2 = Node->getOperand(2); // LHS
3710  Tmp3 = Node->getOperand(3); // RHS
3711  Tmp4 = Node->getOperand(1); // CC
3712 
3713  bool Legalized =
3714  TLI.LegalizeSetCCCondCode(DAG, getSetCCResultType(Tmp2.getValueType()),
3715  Tmp2, Tmp3, Tmp4, NeedInvert, dl, Chain);
3716  (void)Legalized;
3717  assert(Legalized && "Can't legalize BR_CC with legal condition!");
3718 
3719  // If we expanded the SETCC by swapping LHS and RHS, create a new BR_CC
3720  // node.
3721  if (Tmp4.getNode()) {
3722  assert(!NeedInvert && "Don't know how to invert BR_CC!");
3723 
3724  Tmp1 = DAG.getNode(ISD::BR_CC, dl, Node->getValueType(0), Tmp1,
3725  Tmp4, Tmp2, Tmp3, Node->getOperand(4));
3726  } else {
3727  Tmp3 = DAG.getConstant(0, dl, Tmp2.getValueType());
3728  Tmp4 = DAG.getCondCode(NeedInvert ? ISD::SETEQ : ISD::SETNE);
3729  Tmp1 = DAG.getNode(ISD::BR_CC, dl, Node->getValueType(0), Tmp1, Tmp4,
3730  Tmp2, Tmp3, Node->getOperand(4));
3731  }
3732  Results.push_back(Tmp1);
3733  break;
3734  }
3735  case ISD::BUILD_VECTOR:
3736  Results.push_back(ExpandBUILD_VECTOR(Node));
3737  break;
3738  case ISD::SPLAT_VECTOR:
3739  Results.push_back(ExpandSPLAT_VECTOR(Node));
3740  break;
3741  case ISD::SRA:
3742  case ISD::SRL:
3743  case ISD::SHL: {
3744  // Scalarize vector SRA/SRL/SHL.
3745  EVT VT = Node->getValueType(0);
3746  assert(VT.isVector() && "Unable to legalize non-vector shift");
3747  assert(TLI.isTypeLegal(VT.getScalarType())&& "Element type must be legal");
3748  unsigned NumElem = VT.getVectorNumElements();
3749 
3750  SmallVector<SDValue, 8> Scalars;
3751  for (unsigned Idx = 0; Idx < NumElem; Idx++) {
3752  SDValue Ex =
3754  Node->getOperand(0), DAG.getVectorIdxConstant(Idx, dl));
3755  SDValue Sh =
3757  Node->getOperand(1), DAG.getVectorIdxConstant(Idx, dl));
3758  Scalars.push_back(DAG.getNode(Node->getOpcode(), dl,
3759  VT.getScalarType(), Ex, Sh));
3760  }
3761 
3762  SDValue Result = DAG.getBuildVector(Node->getValueType(0), dl, Scalars);
3763  Results.push_back(Result);
3764  break;
3765  }
3766  case ISD::VECREDUCE_FADD:
3767  case ISD::VECREDUCE_FMUL:
3768  case ISD::VECREDUCE_ADD:
3769  case ISD::VECREDUCE_MUL:
3770  case ISD::VECREDUCE_AND:
3771  case ISD::VECREDUCE_OR:
3772  case ISD::VECREDUCE_XOR:
3773  case ISD::VECREDUCE_SMAX:
3774  case ISD::VECREDUCE_SMIN:
3775  case ISD::VECREDUCE_UMAX:
3776  case ISD::VECREDUCE_UMIN:
3777  case ISD::VECREDUCE_FMAX:
3778  case ISD::VECREDUCE_FMIN:
3779  Results.push_back(TLI.expandVecReduce(Node, DAG));
3780  break;
3782  case ISD::GlobalAddress:
3783  case ISD::GlobalTLSAddress:
3784  case ISD::ExternalSymbol:
3785  case ISD::ConstantPool:
3786  case ISD::JumpTable:
3789  case ISD::INTRINSIC_VOID:
3790  // FIXME: Custom lowering for these operations shouldn't return null!
3791  // Return true so that we don't call ConvertNodeToLibcall which also won't
3792  // do anything.
3793  return true;
3794  }
3795 
3796  if (!TLI.isStrictFPEnabled() && Results.empty() && Node->isStrictFPOpcode()) {
3797  // FIXME: We were asked to expand a strict floating-point operation,
3798  // but there is currently no expansion implemented that would preserve
3799  // the "strict" properties. For now, we just fall back to the non-strict
3800  // version if that is legal on the target. The actual mutation of the
3801  // operation will happen in SelectionDAGISel::DoInstructionSelection.
3802  switch (Node->getOpcode()) {
3803  default:
3804  if (TLI.getStrictFPOperationAction(Node->getOpcode(),
3805  Node->getValueType(0))
3807  return true;
3808  break;
3809  case ISD::STRICT_FSUB: {
3810  if (TLI.getStrictFPOperationAction(
3811  ISD::STRICT_FSUB, Node->getValueType(0)) == TargetLowering::Legal)
3812  return true;
3813  if (TLI.getStrictFPOperationAction(
3814  ISD::STRICT_FADD, Node->getValueType(0)) != TargetLowering::Legal)
3815  break;
3816 
3817  EVT VT = Node->getValueType(0);
3818  const SDNodeFlags Flags = Node->getFlags();
3819  SDValue Neg = DAG.getNode(ISD::FNEG, dl, VT, Node->getOperand(2), Flags);
3820  SDValue Fadd = DAG.getNode(ISD::STRICT_FADD, dl, Node->getVTList(),
3821  {Node->getOperand(0), Node->getOperand(1), Neg},
3822  Flags);
3823 
3824  Results.push_back(Fadd);
3825  Results.push_back(Fadd.getValue(1));
3826  break;
3827  }
3830  case ISD::STRICT_LRINT:
3831  case ISD::STRICT_LLRINT:
3832  case ISD::STRICT_LROUND:
3833  case ISD::STRICT_LLROUND:
3834  // These are registered by the operand type instead of the value
3835  // type. Reflect that here.
3836  if (TLI.getStrictFPOperationAction(Node->getOpcode(),
3837  Node->getOperand(1).getValueType())
3839  return true;
3840  break;
3841  }
3842  }
3843 
3844  // Replace the original node with the legalized result.
3845  if (Results.empty()) {
3846  LLVM_DEBUG(dbgs() << "Cannot expand node\n");
3847  return false;
3848  }
3849 
3850  LLVM_DEBUG(dbgs() << "Successfully expanded node\n");
3851  ReplaceNode(Node, Results.data());
3852  return true;
3853 }
3854 
3855 void SelectionDAGLegalize::ConvertNodeToLibcall(SDNode *Node) {
3856  LLVM_DEBUG(dbgs() << "Trying to convert node to libcall\n");
3858  SDLoc dl(Node);
3859  // FIXME: Check flags on the node to see if we can use a finite call.
3860  unsigned Opc = Node->getOpcode();
3861  switch (Opc) {
3862  case ISD::ATOMIC_FENCE: {
3863  // If the target didn't lower this, lower it to '__sync_synchronize()' call
3864  // FIXME: handle "fence singlethread" more efficiently.
3866 
3868  CLI.setDebugLoc(dl)
3869  .setChain(Node->getOperand(0))
3870  .setLibCallee(
3871  CallingConv::C, Type::getVoidTy(*DAG.getContext()),
3872  DAG.getExternalSymbol("__sync_synchronize",
3873  TLI.getPointerTy(DAG.getDataLayout())),
3874  std::move(Args));
3875 
3876  std::pair<SDValue, SDValue> CallResult = TLI.LowerCallTo(CLI);
3877 
3878  Results.push_back(CallResult.second);
3879  break;
3880  }
3881  // By default, atomic intrinsics are marked Legal and lowered. Targets
3882  // which don't support them directly, however, may want libcalls, in which
3883  // case they mark them Expand, and we get here.
3884  case ISD::ATOMIC_SWAP:
3885  case ISD::ATOMIC_LOAD_ADD:
3886  case ISD::ATOMIC_LOAD_SUB:
3887  case ISD::ATOMIC_LOAD_AND:
3888  case ISD::ATOMIC_LOAD_CLR:
3889  case ISD::ATOMIC_LOAD_OR:
3890  case ISD::ATOMIC_LOAD_XOR:
3891  case ISD::ATOMIC_LOAD_NAND:
3892  case ISD::ATOMIC_LOAD_MIN:
3893  case ISD::ATOMIC_LOAD_MAX:
3894  case ISD::ATOMIC_LOAD_UMIN:
3895  case ISD::ATOMIC_LOAD_UMAX:
3896  case ISD::ATOMIC_CMP_SWAP: {
3897  MVT VT = cast<AtomicSDNode>(Node)->getMemoryVT().getSimpleVT();
3898  AtomicOrdering Order = cast<AtomicSDNode>(Node)->getOrdering();
3899  RTLIB::Libcall LC = RTLIB::getOUTLINE_ATOMIC(Opc, Order, VT);
3900  EVT RetVT = Node->getValueType(0);
3903  if (TLI.getLibcallName(LC)) {
3904  // If outline atomic available, prepare its arguments and expand.
3905  Ops.append(Node->op_begin() + 2, Node->op_end());
3906  Ops.push_back(Node->getOperand(1));
3907 
3908  } else {
3909  LC = RTLIB::getSYNC(Opc, VT);
3910  assert(LC != RTLIB::UNKNOWN_LIBCALL &&
3911  "Unexpected atomic op or value type!");
3912  // Arguments for expansion to sync libcall
3913  Ops.append(Node->op_begin() + 1, Node->op_end());
3914  }
3915  std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, RetVT,
3916  Ops, CallOptions,
3917  SDLoc(Node),
3918  Node->getOperand(0));
3919  Results.push_back(Tmp.first);
3920  Results.push_back(Tmp.second);
3921  break;
3922  }
3923  case ISD::TRAP: {
3924  // If this operation is not supported, lower it to 'abort()' call
3927  CLI.setDebugLoc(dl)
3928  .setChain(Node->getOperand(0))
3929  .setLibCallee(CallingConv::C, Type::getVoidTy(*DAG.getContext()),
3930  DAG.getExternalSymbol(
3931  "abort", TLI.getPointerTy(DAG.getDataLayout())),
3932  std::move(Args));
3933  std::pair<SDValue, SDValue> CallResult = TLI.LowerCallTo(CLI);
3934 
3935  Results.push_back(CallResult.second);
3936  break;
3937  }
3938  case ISD::FMINNUM:
3939  case ISD::STRICT_FMINNUM:
3940  ExpandFPLibCall(Node, RTLIB::FMIN_F32, RTLIB::FMIN_F64,
3941  RTLIB::FMIN_F80, RTLIB::FMIN_F128,
3942  RTLIB::FMIN_PPCF128, Results);
3943  break;
3944  case ISD::FMAXNUM:
3945  case ISD::STRICT_FMAXNUM:
3946  ExpandFPLibCall(Node, RTLIB::FMAX_F32, RTLIB::FMAX_F64,
3947  RTLIB::FMAX_F80, RTLIB::FMAX_F128,
3948  RTLIB::FMAX_PPCF128, Results);
3949  break;
3950  case ISD::FSQRT:
3951  case ISD::STRICT_FSQRT:
3952  ExpandFPLibCall(Node, RTLIB::SQRT_F32, RTLIB::SQRT_F64,
3953  RTLIB::SQRT_F80, RTLIB::SQRT_F128,
3954  RTLIB::SQRT_PPCF128, Results);
3955  break;
3956  case ISD::FCBRT:
3957  ExpandFPLibCall(Node, RTLIB::CBRT_F32, RTLIB::CBRT_F64,
3958  RTLIB::CBRT_F80, RTLIB::CBRT_F128,
3959  RTLIB::CBRT_PPCF128, Results);
3960  break;
3961  case ISD::FSIN:
3962  case ISD::STRICT_FSIN:
3963  ExpandFPLibCall(Node, RTLIB::SIN_F32, RTLIB::SIN_F64,
3964  RTLIB::SIN_F80, RTLIB::SIN_F128,
3965  RTLIB::SIN_PPCF128, Results);
3966  break;
3967  case ISD::FCOS:
3968  case ISD::STRICT_FCOS:
3969  ExpandFPLibCall(Node, RTLIB::COS_F32, RTLIB::COS_F64,
3970  RTLIB::COS_F80, RTLIB::COS_F128,
3971  RTLIB::COS_PPCF128, Results);
3972  break;
3973  case ISD::FSINCOS:
3974  // Expand into sincos libcall.
3975  ExpandSinCosLibCall(Node, Results);
3976  break;
3977  case ISD::FLOG:
3978  case ISD::STRICT_FLOG:
3979  ExpandFPLibCall(Node, RTLIB::LOG_F32, RTLIB::LOG_F64, RTLIB::LOG_F80,
3980  RTLIB::LOG_F128, RTLIB::LOG_PPCF128, Results);
3981  break;
3982  case ISD::FLOG2:
3983  case ISD::STRICT_FLOG2:
3984  ExpandFPLibCall(Node, RTLIB::LOG2_F32, RTLIB::LOG2_F64, RTLIB::LOG2_F80,
3985  RTLIB::LOG2_F128, RTLIB::LOG2_PPCF128, Results);
3986  break;
3987  case ISD::FLOG10:
3988  case ISD::STRICT_FLOG10:
3989  ExpandFPLibCall(Node, RTLIB::LOG10_F32, RTLIB::LOG10_F64, RTLIB::LOG10_F80,
3990  RTLIB::LOG10_F128, RTLIB::LOG10_PPCF128, Results);
3991  break;
3992  case ISD::FEXP:
3993  case ISD::STRICT_FEXP:
3994  ExpandFPLibCall(Node, RTLIB::EXP_F32, RTLIB::EXP_F64, RTLIB::EXP_F80,
3995  RTLIB::EXP_F128, RTLIB::EXP_PPCF128, Results);
3996  break;
3997  case ISD::FEXP2:
3998  case ISD::STRICT_FEXP2:
3999  ExpandFPLibCall(Node, RTLIB::EXP2_F32, RTLIB::EXP2_F64, RTLIB::EXP2_F80,
4000  RTLIB::EXP2_F128, RTLIB::EXP2_PPCF128, Results);
4001  break;
4002  case ISD::FTRUNC:
4003  case ISD::STRICT_FTRUNC:
4004  ExpandFPLibCall(Node, RTLIB::TRUNC_F32, RTLIB::TRUNC_F64,
4005  RTLIB::TRUNC_F80, RTLIB::TRUNC_F128,
4006  RTLIB::TRUNC_PPCF128, Results);
4007  break;
4008  case ISD::FFLOOR:
4009  case ISD::STRICT_FFLOOR:
4010  ExpandFPLibCall(Node, RTLIB::FLOOR_F32, RTLIB::FLOOR_F64,
4011  RTLIB::FLOOR_F80, RTLIB::FLOOR_F128,
4012  RTLIB::FLOOR_PPCF128, Results);
4013  break;
4014  case ISD::FCEIL:
4015  case ISD::STRICT_FCEIL:
4016  ExpandFPLibCall(Node, RTLIB::CEIL_F32, RTLIB::CEIL_F64,
4017  RTLIB::CEIL_F80, RTLIB::CEIL_F128,
4018  RTLIB::CEIL_PPCF128, Results);
4019  break;
4020  case ISD::FRINT:
4021  case ISD::STRICT_FRINT:
4022  ExpandFPLibCall(Node, RTLIB::RINT_F32, RTLIB::RINT_F64,
4023  RTLIB::RINT_F80, RTLIB::RINT_F128,
4024  RTLIB::RINT_PPCF128, Results);
4025  break;
4026  case ISD::FNEARBYINT:
4028  ExpandFPLibCall(Node, RTLIB::NEARBYINT_F32,
4029  RTLIB::NEARBYINT_F64,
4030  RTLIB::NEARBYINT_F80,
4031  RTLIB::NEARBYINT_F128,
4032  RTLIB::NEARBYINT_PPCF128, Results);
4033  break;
4034  case ISD::FROUND:
4035  case ISD::STRICT_FROUND:
4036  ExpandFPLibCall(Node, RTLIB::ROUND_F32,
4037  RTLIB::ROUND_F64,
4038  RTLIB::ROUND_F80,
4039  RTLIB::ROUND_F128,
4040  RTLIB::ROUND_PPCF128, Results);
4041  break;
4042  case ISD::FROUNDEVEN:
4044  ExpandFPLibCall(Node, RTLIB::ROUNDEVEN_F32,
4045  RTLIB::ROUNDEVEN_F64,
4046  RTLIB::ROUNDEVEN_F80,
4047  RTLIB::ROUNDEVEN_F128,
4048  RTLIB::ROUNDEVEN_PPCF128, Results);
4049  break;
4050  case ISD::FPOWI:
4051  case ISD::STRICT_FPOWI: {
4052  RTLIB::Libcall LC;
4053  switch (Node->getSimpleValueType(0).SimpleTy) {
4054  default: llvm_unreachable("Unexpected request for libcall!");
4055  case MVT::f32: LC = RTLIB::POWI_F32; break;
4056  case MVT::f64: LC = RTLIB::POWI_F64; break;
4057  case MVT::f80: LC = RTLIB::POWI_F80; break;
4058  case MVT::f128: LC = RTLIB::POWI_F128; break;
4059  case MVT::ppcf128: LC = RTLIB::POWI_PPCF128; break;
4060  }
4061  if (!TLI.getLibcallName(LC)) {
4062  // Some targets don't have a powi libcall; use pow instead.
4063  SDValue Exponent = DAG.getNode(ISD::SINT_TO_FP, SDLoc(Node),
4064  Node->getValueType(0),
4065  Node->getOperand(1));
4066  Results.push_back(DAG.getNode(ISD::FPOW, SDLoc(Node),
4067  Node->getValueType(0), Node->getOperand(0),
4068  Exponent));
4069  break;
4070  }
4071  ExpandFPLibCall(Node, RTLIB::POWI_F32, RTLIB::POWI_F64,
4072  RTLIB::POWI_F80, RTLIB::POWI_F128,
4073  RTLIB::POWI_PPCF128, Results);
4074  break;
4075  }
4076  case ISD::FPOW:
4077  case ISD::STRICT_FPOW:
4078  ExpandFPLibCall(Node, RTLIB::POW_F32, RTLIB::POW_F64, RTLIB::POW_F80,
4079  RTLIB::POW_F128, RTLIB::POW_PPCF128, Results);
4080  break;
4081  case ISD::LROUND:
4082  case ISD::STRICT_LROUND:
4083  ExpandArgFPLibCall(Node, RTLIB::LROUND_F32,
4084  RTLIB::LROUND_F64, RTLIB::LROUND_F80,
4085  RTLIB::LROUND_F128,
4086  RTLIB::LROUND_PPCF128, Results);
4087  break;
4088  case ISD::LLROUND:
4089  case ISD::STRICT_LLROUND:
4090  ExpandArgFPLibCall(Node, RTLIB::LLROUND_F32,
4091  RTLIB::LLROUND_F64, RTLIB::LLROUND_F80,
4092  RTLIB::LLROUND_F128,
4093  RTLIB::LLROUND_PPCF128, Results);
4094  break;
4095  case ISD::LRINT:
4096  case ISD::STRICT_LRINT:
4097  ExpandArgFPLibCall(Node, RTLIB::LRINT_F32,
4098  RTLIB::LRINT_F64, RTLIB::LRINT_F80,
4099  RTLIB::LRINT_F128,
4100  RTLIB::LRINT_PPCF128, Results);
4101  break;
4102  case ISD::LLRINT:
4103  case ISD::STRICT_LLRINT:
4104  ExpandArgFPLibCall(Node, RTLIB::LLRINT_F32,
4105  RTLIB::LLRINT_F64, RTLIB::LLRINT_F80,
4106  RTLIB::LLRINT_F128,
4107  RTLIB::LLRINT_PPCF128, Results);
4108  break;
4109  case ISD::FDIV:
4110  case ISD::STRICT_FDIV:
4111  ExpandFPLibCall(Node, RTLIB::DIV_F32, RTLIB::DIV_F64,
4112  RTLIB::DIV_F80, RTLIB::DIV_F128,
4113  RTLIB::DIV_PPCF128, Results);
4114  break;
4115  case ISD::FREM:
4116  case ISD::STRICT_FREM:
4117  ExpandFPLibCall(Node, RTLIB::REM_F32, RTLIB::REM_F64,
4118  RTLIB::REM_F80, RTLIB::REM_F128,
4119  RTLIB::REM_PPCF128, Results);
4120  break;
4121  case ISD::FMA:
4122  case ISD::STRICT_FMA:
4123  ExpandFPLibCall(Node, RTLIB::FMA_F32, RTLIB::FMA_F64,
4124  RTLIB::FMA_F80, RTLIB::FMA_F128,
4125  RTLIB::FMA_PPCF128, Results);
4126  break;
4127  case ISD::FADD:
4128  case ISD::STRICT_FADD:
4129  ExpandFPLibCall(Node, RTLIB::ADD_F32, RTLIB::ADD_F64,
4130  RTLIB::ADD_F80, RTLIB::ADD_F128,
4131  RTLIB::ADD_PPCF128, Results);
4132  break;
4133  case ISD::FMUL:
4134  case ISD::STRICT_FMUL:
4135  ExpandFPLibCall(Node, RTLIB::MUL_F32, RTLIB::MUL_F64,
4136  RTLIB::MUL_F80, RTLIB::MUL_F128,
4137  RTLIB::MUL_PPCF128, Results);
4138  break;
4139  case ISD::FP16_TO_FP:
4140  if (Node->getValueType(0) == MVT::f32) {
4141  Results.push_back(ExpandLibCall(RTLIB::FPEXT_F16_F32, Node, false));
4142  }
4143  break;
4144  case ISD::STRICT_FP16_TO_FP: {
4145  if (Node->getValueType(0) == MVT::f32) {
4147  std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(
4148  DAG, RTLIB::FPEXT_F16_F32, MVT::f32, Node->getOperand(1), CallOptions,
4149  SDLoc(Node), Node->getOperand(0));
4150  Results.push_back(Tmp.first);
4151  Results.push_back(Tmp.second);
4152  }
4153  break;
4154  }
4155  case ISD::FP_TO_FP16: {
4156  RTLIB::Libcall LC =
4157  RTLIB::getFPROUND(Node->getOperand(0).getValueType(), MVT::f16);
4158  assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unable to expand fp_to_fp16");
4159  Results.push_back(ExpandLibCall(LC, Node, false));
4160  break;
4161  }
4164  case ISD::SINT_TO_FP:
4165  case ISD::UINT_TO_FP: {
4166  // TODO - Common the code with DAGTypeLegalizer::SoftenFloatRes_XINT_TO_FP
4167  bool IsStrict = Node->isStrictFPOpcode();
4168  bool Signed = Node->getOpcode() == ISD::SINT_TO_FP ||
4169  Node->getOpcode() == ISD::STRICT_SINT_TO_FP;
4170  EVT SVT = Node->getOperand(IsStrict ? 1 : 0).getValueType();
4171  EVT RVT = Node->getValueType(0);
4172  EVT NVT = EVT();
4173  SDLoc dl(Node);
4174 
4175  // Even if the input is legal, no libcall may exactly match, eg. we don't
4176  // have i1 -> fp conversions. So, it needs to be promoted to a larger type,
4177  // eg: i13 -> fp. Then, look for an appropriate libcall.
4178  RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
4179  for (unsigned t = MVT::FIRST_INTEGER_VALUETYPE;
4180  t <= MVT::LAST_INTEGER_VALUETYPE && LC == RTLIB::UNKNOWN_LIBCALL;
4181  ++t) {
4182  NVT = (MVT::SimpleValueType)t;
4183  // The source needs to big enough to hold the operand.
4184  if (NVT.bitsGE(SVT))
4185  LC = Signed ? RTLIB::getSINTTOFP(NVT, RVT)
4186  : RTLIB::getUINTTOFP(NVT, RVT);
4187  }
4188  assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unable to legalize as libcall");
4189 
4190  SDValue Chain = IsStrict ? Node->getOperand(0) : SDValue();
4191  // Sign/zero extend the argument if the libcall takes a larger type.
4192  SDValue Op = DAG.getNode(Signed ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND, dl,
4193  NVT, Node->getOperand(IsStrict ? 1 : 0));
4195  CallOptions.setSExt(Signed);
4196  std::pair<SDValue, SDValue> Tmp =
4197  TLI.makeLibCall(DAG, LC, RVT, Op, CallOptions, dl, Chain);
4198  Results.push_back(Tmp.first);
4199  if (IsStrict)
4200  Results.push_back(Tmp.second);
4201  break;
4202  }
4203  case ISD::FP_TO_SINT:
4204  case ISD::FP_TO_UINT:
4206  case ISD::STRICT_FP_TO_UINT: {
4207  // TODO - Common the code with DAGTypeLegalizer::SoftenFloatOp_FP_TO_XINT.
4208  bool IsStrict = Node->isStrictFPOpcode();
4209  bool Signed = Node->getOpcode() == ISD::FP_TO_SINT ||
4210  Node->getOpcode() == ISD::STRICT_FP_TO_SINT;
4211 
4212  SDValue Op = Node->getOperand(IsStrict ? 1 : 0);
4213  EVT SVT = Op.getValueType();
4214  EVT RVT = Node->getValueType(0);
4215  EVT NVT = EVT();
4216  SDLoc dl(Node);
4217 
4218  // Even if the result is legal, no libcall may exactly match, eg. we don't
4219  // have fp -> i1 conversions. So, it needs to be promoted to a larger type,
4220  // eg: fp -> i32. Then, look for an appropriate libcall.
4221  RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
4222  for (unsigned IntVT = MVT::FIRST_INTEGER_VALUETYPE;
4223  IntVT <= MVT::LAST_INTEGER_VALUETYPE && LC == RTLIB::UNKNOWN_LIBCALL;
4224  ++IntVT) {
4225  NVT = (MVT::SimpleValueType)IntVT;
4226  // The type needs to big enough to hold the result.
4227  if (NVT.bitsGE(RVT))
4228  LC = Signed ? RTLIB::getFPTOSINT(SVT, NVT)
4229  : RTLIB::getFPTOUINT(SVT, NVT);
4230  }
4231  assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unable to legalize as libcall");
4232 
4233  SDValue Chain = IsStrict ? Node->getOperand(0) : SDValue();
4235  std::pair<SDValue, SDValue> Tmp =
4236  TLI.makeLibCall(DAG, LC, NVT, Op, CallOptions, dl, Chain);
4237 
4238  // Truncate the result if the libcall returns a larger type.
4239  Results.push_back(DAG.getNode(ISD::TRUNCATE, dl, RVT, Tmp.first));
4240  if (IsStrict)
4241  Results.push_back(Tmp.second);
4242  break;
4243  }
4244 
4245  case ISD::FP_ROUND:
4246  case ISD::STRICT_FP_ROUND: {
4247  // X = FP_ROUND(Y, TRUNC)
4248  // TRUNC is a flag, which is always an integer that is zero or one.
4249  // If TRUNC is 0, this is a normal rounding, if it is 1, this FP_ROUND
4250  // is known to not change the value of Y.
4251  // We can only expand it into libcall if the TRUNC is 0.
4252  bool IsStrict = Node->isStrictFPOpcode();
4253  SDValue Op = Node->getOperand(IsStrict ? 1 : 0);
4254  SDValue Chain = IsStrict ? Node->getOperand(0) : SDValue();
4255  EVT VT = Node->getValueType(0);
4256  assert(cast<ConstantSDNode>(Node->getOperand(IsStrict ? 2 : 1))
4257  ->isNullValue() &&
4258  "Unable to expand as libcall if it is not normal rounding");
4259 
4260  RTLIB::Libcall LC = RTLIB::getFPROUND(Op.getValueType(), VT);
4261  assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unable to legalize as libcall");
4262 
4264  std::pair<SDValue, SDValue> Tmp =
4265  TLI.makeLibCall(DAG, LC, VT, Op, CallOptions, SDLoc(Node), Chain);
4266  Results.push_back(Tmp.first);
4267  if (IsStrict)
4268  Results.push_back(Tmp.second);
4269  break;
4270  }
4271  case ISD::FP_EXTEND: {
4272  Results.push_back(
4273  ExpandLibCall(RTLIB::getFPEXT(Node->getOperand(0).getValueType(),
4274  Node->getValueType(0)),
4275  Node, false));
4276  break;
4277  }
4278  case ISD::STRICT_FP_EXTEND:
4279  case ISD::STRICT_FP_TO_FP16: {
4280  RTLIB::Libcall LC =
4281  Node->getOpcode() == ISD::STRICT_FP_TO_FP16
4282  ? RTLIB::getFPROUND(Node->getOperand(1).getValueType(), MVT::f16)
4283  : RTLIB::getFPEXT(Node->getOperand(1).getValueType(),
4284  Node->getValueType(0));
4285  assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unable to legalize as libcall");
4286 
4288  std::pair<SDValue, SDValue> Tmp =
4289  TLI.makeLibCall(DAG, LC, Node->getValueType(0), Node->getOperand(1),
4290  CallOptions, SDLoc(Node), Node->getOperand(0));
4291  Results.push_back(Tmp.first);
4292  Results.push_back(Tmp.second);
4293  break;
4294  }
4295  case ISD::FSUB:
4296  case ISD::STRICT_FSUB:
4297  ExpandFPLibCall(Node, RTLIB::SUB_F32, RTLIB::SUB_F64,
4298  RTLIB::SUB_F80, RTLIB::SUB_F128,
4299  RTLIB::SUB_PPCF128, Results);
4300  break;
4301  case ISD::SREM:
4302  Results.push_back(ExpandIntLibCall(Node, true,
4303  RTLIB::SREM_I8,
4304  RTLIB::SREM_I16, RTLIB::SREM_I32,
4305  RTLIB::SREM_I64, RTLIB::SREM_I128));
4306  break;
4307  case ISD::UREM:
4308  Results.push_back(ExpandIntLibCall(Node, false,
4309  RTLIB::UREM_I8,
4310  RTLIB::UREM_I16, RTLIB::UREM_I32,
4311  RTLIB::UREM_I64, RTLIB::UREM_I128));
4312  break;
4313  case ISD::SDIV:
4314  Results.push_back(ExpandIntLibCall(Node, true,
4315  RTLIB::SDIV_I8,
4316  RTLIB::SDIV_I16, RTLIB::SDIV_I32,
4317  RTLIB::SDIV_I64, RTLIB::SDIV_I128));
4318  break;
4319  case ISD::UDIV:
4320  Results.push_back(ExpandIntLibCall(Node, false,
4321  RTLIB::UDIV_I8,
4322  RTLIB::UDIV_I16, RTLIB::UDIV_I32,
4323  RTLIB::UDIV_I64, RTLIB::UDIV_I128));
4324  break;
4325  case ISD::SDIVREM:
4326  case ISD::UDIVREM:
4327  // Expand into divrem libcall
4328  ExpandDivRemLibCall(Node, Results);
4329  break;
4330  case ISD::MUL:
4331  Results.push_back(ExpandIntLibCall(Node, false,
4332  RTLIB::MUL_I8,
4333  RTLIB::MUL_I16, RTLIB::MUL_I32,
4334  RTLIB::MUL_I64, RTLIB::MUL_I128));
4335  break;
4336  case ISD::CTLZ_ZERO_UNDEF:
4337  switch (Node->getSimpleValueType(0).SimpleTy) {
4338  default:
4339  llvm_unreachable("LibCall explicitly requested, but not available");
4340  case MVT::i32:
4341  Results.push_back(ExpandLibCall(RTLIB::CTLZ_I32, Node, false));
4342  break;
4343  case MVT::i64:
4344  Results.push_back(ExpandLibCall(RTLIB::CTLZ_I64, Node, false));
4345  break;
4346  case MVT::i128:
4347  Results.push_back(ExpandLibCall(RTLIB::CTLZ_I128, Node, false));
4348  break;
4349  }
4350  break;
4351  }
4352 
4353  // Replace the original node with the legalized result.
4354  if (!Results.empty()) {
4355  LLVM_DEBUG(dbgs() << "Successfully converted node to libcall\n");
4356  ReplaceNode(Node, Results.data());
4357  } else
4358  LLVM_DEBUG(dbgs() << "Could not convert node to libcall\n");
4359 }
4360 
4361 // Determine the vector type to use in place of an original scalar element when
4362 // promoting equally sized vectors.
4364  MVT EltVT, MVT NewEltVT) {
4365  unsigned OldEltsPerNewElt = EltVT.getSizeInBits() / NewEltVT.getSizeInBits();
4366  MVT MidVT = MVT::getVectorVT(NewEltVT, OldEltsPerNewElt);
4367  assert(TLI.isTypeLegal(MidVT) && "unexpected");
4368  return MidVT;
4369 }
4370 
4371 void SelectionDAGLegalize::PromoteNode(SDNode *Node) {
4372  LLVM_DEBUG(dbgs() << "Trying to promote node\n");
4374  MVT OVT = Node->getSimpleValueType(0);
4375  if (Node->getOpcode() == ISD::UINT_TO_FP ||
4376  Node->getOpcode() == ISD::SINT_TO_FP ||
4377  Node->getOpcode() == ISD::SETCC ||
4378  Node->getOpcode() == ISD::EXTRACT_VECTOR_ELT ||
4379  Node->getOpcode() == ISD::INSERT_VECTOR_ELT) {
4380  OVT = Node->getOperand(0).getSimpleValueType();
4381  }
4382  if (Node->getOpcode() == ISD::STRICT_UINT_TO_FP ||
4383  Node->getOpcode() == ISD::STRICT_SINT_TO_FP ||
4384  Node->getOpcode() == ISD::STRICT_FSETCC ||
4385  Node->getOpcode() == ISD::STRICT_FSETCCS)
4386  OVT = Node->getOperand(1).getSimpleValueType();
4387  if (Node->getOpcode() == ISD::BR_CC ||
4388  Node->getOpcode() == ISD::SELECT_CC)
4389  OVT = Node->getOperand(2).getSimpleValueType();
4390  MVT NVT = TLI.getTypeToPromoteTo(Node->getOpcode(), OVT);
4391  SDLoc dl(Node);
4392  SDValue Tmp1, Tmp2, Tmp3, Tmp4;
4393  switch (Node->getOpcode()) {
4394  case ISD::CTTZ:
4395  case ISD::CTTZ_ZERO_UNDEF:
4396  case ISD::CTLZ:
4397  case ISD::CTLZ_ZERO_UNDEF:
4398  case ISD::CTPOP:
4399  // Zero extend the argument unless its cttz, then use any_extend.
4400  if (Node->getOpcode() == ISD::CTTZ ||
4401  Node->getOpcode() == ISD::CTTZ_ZERO_UNDEF)
4402  Tmp1 = DAG.getNode(ISD::ANY_EXTEND, dl, NVT, Node->getOperand(0));
4403  else
4404  Tmp1 = DAG.getNode(ISD::ZERO_EXTEND, dl, NVT, Node->getOperand(0));
4405 
4406  if (Node->getOpcode() == ISD::CTTZ) {
4407  // The count is the same in the promoted type except if the original
4408  // value was zero. This can be handled by setting the bit just off
4409  // the top of the original type.
4410  auto TopBit = APInt::getOneBitSet(NVT.getSizeInBits(),
4411  OVT.getSizeInBits());
4412  Tmp1 = DAG.getNode(ISD::OR, dl, NVT, Tmp1,
4413  DAG.getConstant(TopBit, dl, NVT));
4414  }
4415  // Perform the larger operation. For CTPOP and CTTZ_ZERO_UNDEF, this is
4416  // already the correct result.
4417  Tmp1 = DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1);
4418  if (Node->getOpcode() == ISD::CTLZ ||
4419  Node->getOpcode() == ISD::CTLZ_ZERO_UNDEF) {
4420  // Tmp1 = Tmp1 - (sizeinbits(NVT) - sizeinbits(Old VT))
4421  Tmp1 = DAG.getNode(ISD::SUB, dl, NVT, Tmp1,
4422  DAG.getConstant(NVT.getSizeInBits() -
4423  OVT.getSizeInBits(), dl, NVT));
4424  }
4425  Results.push_back(DAG.getNode(ISD::TRUNCATE, dl, OVT, Tmp1));
4426  break;
4427  case ISD::BITREVERSE:
4428  case ISD::BSWAP: {
4429  unsigned DiffBits = NVT.getSizeInBits() - OVT.getSizeInBits();
4430  Tmp1 = DAG.getNode(ISD::ZERO_EXTEND, dl, NVT, Node->getOperand(0));
4431  Tmp1 = DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1);
4432  Tmp1 = DAG.getNode(
4433  ISD::SRL, dl, NVT, Tmp1,
4434  DAG.getConstant(DiffBits, dl,
4435  TLI.getShiftAmountTy(NVT, DAG.getDataLayout())));
4436 
4437  Results.push_back(DAG.getNode(ISD::TRUNCATE, dl, OVT, Tmp1));
4438  break;
4439  }
4440  case ISD::FP_TO_UINT:
4442  case ISD::FP_TO_SINT:
4444  PromoteLegalFP_TO_INT(Node, dl, Results);
4445  break;
4446  case ISD::FP_TO_UINT_SAT:
4447  case ISD::FP_TO_SINT_SAT:
4448  Results.push_back(PromoteLegalFP_TO_INT_SAT(Node, dl));
4449  break;
4450  case ISD::UINT_TO_FP:
4452  case ISD::SINT_TO_FP:
4454  PromoteLegalINT_TO_FP(Node, dl, Results);
4455  break;
4456  case ISD::VAARG: {
4457  SDValue Chain = Node->getOperand(0); // Get the chain.
4458  SDValue Ptr = Node->getOperand(1); // Get the pointer.
4459 
4460  unsigned TruncOp;
4461  if (OVT.isVector()) {
4462  TruncOp = ISD::BITCAST;
4463  } else {
4464  assert(OVT.isInteger()
4465  && "VAARG promotion is supported only for vectors or integer types");
4466  TruncOp = ISD::TRUNCATE;
4467  }
4468 
4469  // Perform the larger operation, then convert back
4470  Tmp1 = DAG.getVAArg(NVT, dl, Chain, Ptr, Node->getOperand(2),
4471  Node->getConstantOperandVal(3));
4472  Chain = Tmp1.getValue(1);
4473 
4474  Tmp2 = DAG.getNode(TruncOp, dl, OVT, Tmp1);
4475 
4476  // Modified the chain result - switch anything that used the old chain to
4477  // use the new one.
4478  DAG.ReplaceAllUsesOfValueWith(SDValue(Node, 0), Tmp2);
4479  DAG.ReplaceAllUsesOfValueWith(SDValue(Node, 1), Chain);
4480  if (UpdatedNodes) {
4481  UpdatedNodes->insert(Tmp2.getNode());
4482  UpdatedNodes->insert(Chain.getNode());
4483  }
4484  ReplacedNode(Node);
4485  break;
4486  }
4487  case ISD::MUL:
4488  case ISD::SDIV:
4489  case ISD::SREM:
4490  case ISD::UDIV:
4491  case ISD::UREM:
4492  case ISD::AND:
4493  case ISD::OR:
4494  case ISD::XOR: {
4495  unsigned ExtOp, TruncOp;
4496  if (OVT.isVector()) {
4497  ExtOp = ISD::BITCAST;
4498  TruncOp = ISD::BITCAST;
4499  } else {
4500  assert(OVT.isInteger() && "Cannot promote logic operation");
4501 
4502  switch (Node->getOpcode()) {
4503  default:
4504  ExtOp = ISD::ANY_EXTEND;
4505  break;
4506  case ISD::SDIV:
4507  case ISD::SREM:
4508  ExtOp = ISD::SIGN_EXTEND;
4509  break;
4510  case ISD::UDIV:
4511  case ISD::UREM:
4512  ExtOp = ISD::ZERO_EXTEND;
4513  break;
4514  }
4515  TruncOp = ISD::TRUNCATE;
4516  }
4517  // Promote each of the values to the new type.
4518  Tmp1 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(0));
4519  Tmp2 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(1));
4520  // Perform the larger operation, then convert back
4521  Tmp1 = DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1, Tmp2);
4522  Results.push_back(DAG.getNode(TruncOp, dl, OVT, Tmp1));
4523  break;
4524  }
4525  case ISD::UMUL_LOHI:
4526  case ISD::SMUL_LOHI: {
4527  // Promote to a multiply in a wider integer type.
4528  unsigned ExtOp = Node->getOpcode() == ISD::UMUL_LOHI ? ISD::ZERO_EXTEND
4529  : ISD::SIGN_EXTEND;
4530  Tmp1 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(0));
4531  Tmp2 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(1));
4532  Tmp1 = DAG.getNode(ISD::MUL, dl, NVT, Tmp1, Tmp2);
4533 
4534  auto &DL = DAG.getDataLayout();
4535  unsigned OriginalSize = OVT.getScalarSizeInBits();
4536  Tmp2 = DAG.getNode(
4537  ISD::SRL, dl, NVT, Tmp1,
4538  DAG.getConstant(OriginalSize, dl, TLI.getScalarShiftAmountTy(DL, NVT)));
4539  Results.push_back(DAG.getNode(ISD::TRUNCATE, dl, OVT, Tmp1));
4540  Results.push_back(DAG.getNode(ISD::TRUNCATE, dl, OVT, Tmp2));
4541  break;
4542  }
4543  case ISD::SELECT: {
4544  unsigned ExtOp, TruncOp;
4545  if (Node->getValueType(0).isVector() ||
4546  Node->getValueType(0).getSizeInBits() == NVT.getSizeInBits()) {
4547  ExtOp = ISD::BITCAST;
4548  TruncOp = ISD::BITCAST;
4549  } else if (Node->getValueType(0).isInteger()) {
4550  ExtOp = ISD::ANY_EXTEND;
4551  TruncOp = ISD::TRUNCATE;
4552  } else {
4553  ExtOp = ISD::FP_EXTEND;
4554  TruncOp = ISD::FP_ROUND;
4555  }
4556  Tmp1 = Node->getOperand(0);
4557  // Promote each of the values to the new type.
4558  Tmp2 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(1));
4559  Tmp3 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(2));
4560  // Perform the larger operation, then round down.
4561  Tmp1 = DAG.getSelect(dl, NVT, Tmp1, Tmp2, Tmp3);
4562  Tmp1->setFlags(Node->getFlags());
4563  if (TruncOp != ISD::FP_ROUND)
4564  Tmp1 = DAG.getNode(TruncOp, dl, Node->getValueType(0), Tmp1);
4565  else
4566  Tmp1 = DAG.getNode(TruncOp, dl, Node->getValueType(0), Tmp1,
4567  DAG.getIntPtrConstant(0, dl));
4568  Results.push_back(Tmp1);
4569  break;
4570  }
4571  case ISD::VECTOR_SHUFFLE: {
4572  ArrayRef<int> Mask = cast<ShuffleVectorSDNode>(Node)->getMask();
4573 
4574  // Cast the two input vectors.
4575  Tmp1 = DAG.getNode(ISD::BITCAST, dl, NVT, Node->getOperand(0));
4576  Tmp2 = DAG.getNode(ISD::BITCAST, dl, NVT, Node->getOperand(1));
4577 
4578  // Convert the shuffle mask to the right # elements.
4579  Tmp1 = ShuffleWithNarrowerEltType(NVT, OVT, dl, Tmp1, Tmp2, Mask);
4580  Tmp1 = DAG.getNode(ISD::BITCAST, dl, OVT, Tmp1);
4581  Results.push_back(Tmp1);
4582  break;
4583  }
4584  case ISD::VECTOR_SPLICE: {
4585  Tmp1 = DAG.getNode(ISD::ANY_EXTEND, dl, NVT, Node->getOperand(0));
4586  Tmp2 = DAG.getNode(ISD::ANY_EXTEND, dl, NVT, Node->getOperand(1));
4587  Tmp3 = DAG.getNode(ISD::VECTOR_SPLICE, dl, NVT, Tmp1, Tmp2,
4588  Node->getOperand(2));
4589  Results.push_back(DAG.getNode(ISD::TRUNCATE, dl, OVT, Tmp3));
4590  break;
4591  }
4592  case ISD::SELECT_CC: {
4593  SDValue Cond = Node->getOperand(4);
4594  ISD::CondCode CCCode = cast<CondCodeSDNode>(Cond)->get();
4595  // Type of the comparison operands.
4596  MVT CVT = Node->getSimpleValueType(0);
4597  assert(CVT == OVT && "not handled");
4598 
4599  unsigned ExtOp = ISD::FP_EXTEND;
4600  if (NVT.isInteger()) {
4602  }
4603 
4604  // Promote the comparison operands, if needed.
4605  if (TLI.isCondCodeLegal(CCCode, CVT)) {
4606  Tmp1 = Node->getOperand(0);
4607  Tmp2 = Node->getOperand(1);
4608  } else {
4609  Tmp1 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(0));
4610  Tmp2 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(1));
4611  }
4612  // Cast the true/false operands.
4613  Tmp3 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(2));
4614  Tmp4 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(3));
4615 
4616  Tmp1 = DAG.getNode(ISD::SELECT_CC, dl, NVT, {Tmp1, Tmp2, Tmp3, Tmp4, Cond},
4617  Node->getFlags());
4618 
4619  // Cast the result back to the original type.
4620  if (ExtOp != ISD::FP_EXTEND)
4621  Tmp1 = DAG.getNode(ISD::TRUNCATE, dl, OVT, Tmp1);
4622  else
4623  Tmp1 = DAG.getNode(ISD::FP_ROUND, dl, OVT, Tmp1,
4624  DAG.getIntPtrConstant(0, dl));
4625 
4626  Results.push_back(Tmp1);
4627  break;
4628  }
4629  case ISD::SETCC:
4630  case ISD::STRICT_FSETCC:
4631  case ISD::STRICT_FSETCCS: {
4632  unsigned ExtOp = ISD::FP_EXTEND;
4633  if (NVT.isInteger()) {
4634  ISD::CondCode CCCode = cast<CondCodeSDNode>(Node->getOperand(2))->get();
4636  }
4637  if (Node->isStrictFPOpcode()) {
4638  SDValue InChain = Node->getOperand(0);
4639  std::tie(Tmp1, std::ignore) =
4640  DAG.getStrictFPExtendOrRound(Node->getOperand(1), InChain, dl, NVT);
4641  std::tie(Tmp2, std::ignore) =
4642  DAG.getStrictFPExtendOrRound(Node->getOperand(2), InChain, dl, NVT);
4643  SmallVector<SDValue, 2> TmpChains = {Tmp1.getValue(1), Tmp2.getValue(1)};
4644  SDValue OutChain = DAG.getTokenFactor(dl, TmpChains);
4645  SDVTList VTs = DAG.getVTList(Node->getValueType(0), MVT::Other);
4646  Results.push_back(DAG.getNode(Node->getOpcode(), dl, VTs,
4647  {OutChain, Tmp1, Tmp2, Node->getOperand(3)},
4648  Node->getFlags()));
4649  Results.push_back(Results.back().getValue(1));
4650  break;
4651  }
4652  Tmp1 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(0));
4653  Tmp2 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(1));
4654  Results.push_back(DAG.getNode(ISD::SETCC, dl, Node->getValueType(0), Tmp1,
4655  Tmp2, Node->getOperand(2), Node->getFlags()));
4656  break;
4657  }
4658  case ISD::BR_CC: {
4659  unsigned ExtOp = ISD::FP_EXTEND;
4660  if (NVT.isInteger()) {
4661  ISD::CondCode CCCode =
4662  cast<CondCodeSDNode>(Node->getOperand(1))->get();
4664  }
4665  Tmp1 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(2));
4666  Tmp2 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(3));
4667  Results.push_back(DAG.getNode(ISD::BR_CC, dl, Node->getValueType(0),
4668  Node->getOperand(0), Node->getOperand(1),
4669  Tmp1, Tmp2, Node->getOperand(4)));
4670  break;
4671  }
4672  case ISD::FADD:
4673  case ISD::FSUB:
4674  case ISD::FMUL:
4675  case ISD::FDIV:
4676  case ISD::FREM:
4677  case ISD::FMINNUM:
4678  case ISD::FMAXNUM:
4679  case ISD::FPOW:
4680  Tmp1 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(0));
4681  Tmp2 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(1));
4682  Tmp3 = DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1, Tmp2,
4683  Node->getFlags());
4684  Results.push_back(DAG.getNode(ISD::FP_ROUND, dl, OVT,
4685  Tmp3, DAG.getIntPtrConstant(0, dl)));
4686  break;
4687  case ISD::STRICT_FREM:
4688  case ISD::STRICT_FPOW:
4689  Tmp1 = DAG.getNode(ISD::STRICT_FP_EXTEND, dl, {NVT, MVT::Other},
4690  {Node->getOperand(0), Node->getOperand(1)});
4691  Tmp2 = DAG.getNode(ISD::STRICT_FP_EXTEND, dl, {NVT, MVT::Other},
4692  {Node->getOperand(0), Node->getOperand(2)});
4693  Tmp3 = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Tmp1.getValue(1),
4694  Tmp2.getValue(1));
4695  Tmp1 = DAG.getNode(Node->getOpcode(), dl, {NVT, MVT::Other},
4696  {Tmp3, Tmp1, Tmp2});
4697  Tmp1 = DAG.getNode(ISD::STRICT_FP_ROUND, dl, {OVT, MVT::Other},
4698  {Tmp1.getValue(1), Tmp1, DAG.getIntPtrConstant(0, dl)});
4699  Results.push_back(Tmp1);
4700  Results.push_back(Tmp1.getValue(1));
4701  break;
4702  case ISD::FMA:
4703  Tmp1 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(0));
4704  Tmp2 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(1));
4705  Tmp3 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(2));
4706  Results.push_back(
4707  DAG.getNode(ISD::FP_ROUND, dl, OVT,
4708  DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1, Tmp2, Tmp3),
4709  DAG.getIntPtrConstant(0, dl)));
4710  break;
4711  case ISD::FCOPYSIGN:
4712  case ISD::FPOWI: {
4713  Tmp1 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(0));
4714  Tmp2 = Node->getOperand(1);
4715  Tmp3 = DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1, Tmp2);
4716 
4717  // fcopysign doesn't change anything but the sign bit, so
4718  // (fp_round (fcopysign (fpext a), b))
4719  // is as precise as
4720  // (fp_round (fpext a))
4721  // which is a no-op. Mark it as a TRUNCating FP_ROUND.
4722  const bool isTrunc = (Node->getOpcode() == ISD::FCOPYSIGN);
4723  Results.push_back(DAG.getNode(ISD::FP_ROUND, dl, OVT,
4724  Tmp3, DAG.getIntPtrConstant(isTrunc, dl)));
4725  break;
4726  }
4727  case ISD::FFLOOR:
4728  case ISD::FCEIL:
4729  case ISD::FRINT:
4730  case ISD::FNEARBYINT:
4731  case ISD::FROUND:
4732  case ISD::FROUNDEVEN:
4733  case ISD::FTRUNC:
4734  case ISD::FNEG:
4735  case ISD::FSQRT:
4736  case ISD::FSIN:
4737  case ISD::FCOS:
4738  case ISD::FLOG:
4739  case ISD::FLOG2:
4740  case ISD::FLOG10:
4741  case ISD::FABS:
4742  case ISD::FEXP:
4743  case ISD::FEXP2:
4744  Tmp1 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(0));
4745  Tmp2 = DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1);
4746  Results.push_back(DAG.getNode(ISD::FP_ROUND, dl, OVT,
4747  Tmp2, DAG.getIntPtrConstant(0, dl)));
4748  break;
4749  case ISD::STRICT_FFLOOR:
4750  case ISD::STRICT_FCEIL:
4751  case ISD::STRICT_FSIN:
4752  case ISD::STRICT_FCOS:
4753  case ISD::STRICT_FLOG:
4754  case ISD::STRICT_FLOG10:
4755  case ISD::STRICT_FEXP:
4756  Tmp1 = DAG.getNode(ISD::STRICT_FP_EXTEND, dl, {NVT, MVT::Other},
4757  {Node->getOperand(0), Node->getOperand(1)});
4758  Tmp2 = DAG.getNode(Node->getOpcode(), dl, {NVT, MVT::Other},
4759  {Tmp1.getValue(1), Tmp1});
4760  Tmp3 = DAG.getNode(ISD::STRICT_FP_ROUND, dl, {OVT, MVT::Other},
4761  {Tmp2.getValue(1), Tmp2, DAG.getIntPtrConstant(0, dl)});
4762  Results.push_back(Tmp3);
4763  Results.push_back(Tmp3.getValue(1));
4764  break;
4765  case ISD::BUILD_VECTOR: {
4766  MVT EltVT = OVT.getVectorElementType();
4767  MVT NewEltVT = NVT.getVectorElementType();
4768 
4769  // Handle bitcasts to a different vector type with the same total bit size
4770  //
4771  // e.g. v2i64 = build_vector i64:x, i64:y => v4i32
4772  // =>
4773  // v4i32 = concat_vectors (v2i32 (bitcast i64:x)), (v2i32 (bitcast i64:y))
4774 
4775  assert(NVT.isVector() && OVT.getSizeInBits() == NVT.getSizeInBits() &&
4776  "Invalid promote type for build_vector");
4777  assert(NewEltVT.bitsLT(EltVT) && "not handled");
4778 
4779  MVT MidVT = getPromotedVectorElementType(TLI, EltVT, NewEltVT);
4780 
4781  SmallVector<SDValue, 8> NewOps;
4782  for (unsigned I = 0, E = Node->getNumOperands(); I != E; ++I) {
4783  SDValue Op = Node->getOperand(I);
4784  NewOps.push_back(DAG.getNode(ISD::BITCAST, SDLoc(Op), MidVT, Op));
4785  }
4786 
4787  SDLoc SL(Node);
4788  SDValue Concat = DAG.getNode(ISD::CONCAT_VECTORS, SL, NVT, NewOps);
4789  SDValue CvtVec = DAG.getNode(ISD::BITCAST, SL, OVT, Concat);
4790  Results.push_back(CvtVec);
4791  break;
4792  }
4793  case ISD::EXTRACT_VECTOR_ELT: {
4794  MVT EltVT = OVT.getVectorElementType();
4795  MVT NewEltVT = NVT.getVectorElementType();
4796 
4797  // Handle bitcasts to a different vector type with the same total bit size.
4798  //
4799  // e.g. v2i64 = extract_vector_elt x:v2i64, y:i32
4800  // =>
4801  // v4i32:castx = bitcast x:v2i64
4802  //
4803  // i64 = bitcast
4804  // (v2i32 build_vector (i32 (extract_vector_elt castx, (2 * y))),
4805  // (i32 (extract_vector_elt castx, (2 * y + 1)))
4806  //
4807 
4808  assert(NVT.isVector() && OVT.getSizeInBits() == NVT.getSizeInBits() &&
4809  "Invalid promote type for extract_vector_elt");
4810  assert(NewEltVT.bitsLT(EltVT) && "not handled");
4811 
4812  MVT MidVT = getPromotedVectorElementType(TLI, EltVT, NewEltVT);
4813  unsigned NewEltsPerOldElt = MidVT.getVectorNumElements();
4814 
4815  SDValue Idx = Node->getOperand(1);
4816  EVT IdxVT = Idx.getValueType();
4817  SDLoc SL(Node);
4818  SDValue Factor = DAG.getConstant(NewEltsPerOldElt, SL, IdxVT);
4819  SDValue NewBaseIdx = DAG.getNode(ISD::MUL, SL, IdxVT, Idx, Factor);
4820 
4821  SDValue CastVec = DAG.getNode(ISD::BITCAST, SL, NVT, Node->getOperand(0));
4822 
4823  SmallVector<SDValue, 8> NewOps;
4824  for (unsigned I = 0; I < NewEltsPerOldElt; ++I) {
4825  SDValue IdxOffset = DAG.getConstant(I, SL, IdxVT);
4826  SDValue TmpIdx = DAG.getNode(ISD::ADD, SL, IdxVT, NewBaseIdx, IdxOffset);
4827 
4828  SDValue Elt = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, NewEltVT,
4829  CastVec, TmpIdx);
4830  NewOps.push_back(Elt);
4831  }
4832 
4833  SDValue NewVec = DAG.getBuildVector(MidVT, SL, NewOps);
4834  Results.push_back(DAG.getNode(ISD::BITCAST, SL, EltVT, NewVec));
4835  break;
4836  }
4837  case ISD::INSERT_VECTOR_ELT: {
4838  MVT EltVT = OVT.getVectorElementType();
4839  MVT NewEltVT = NVT.getVectorElementType();
4840 
4841  // Handle bitcasts to a different vector type with the same total bit size
4842  //
4843  // e.g. v2i64 = insert_vector_elt x:v2i64, y:i64, z:i32
4844  // =>
4845  // v4i32:castx = bitcast x:v2i64
4846  // v2i32:casty = bitcast y:i64
4847  //
4848  // v2i64 = bitcast
4849  // (v4i32 insert_vector_elt
4850  // (v4i32 insert_vector_elt v4i32:castx,
4851  // (extract_vector_elt casty, 0), 2 * z),
4852  // (extract_vector_elt casty, 1), (2 * z + 1))
4853 
4854  assert(NVT.isVector() && OVT.getSizeInBits() == NVT.getSizeInBits() &&
4855  "Invalid promote type for insert_vector_elt");
4856  assert(NewEltVT.bitsLT(EltVT) && "not handled");
4857 
4858  MVT MidVT = getPromotedVectorElementType(TLI, EltVT, NewEltVT);
4859  unsigned NewEltsPerOldElt = MidVT.getVectorNumElements();
4860 
4861  SDValue Val = Node->getOperand(1);
4862  SDValue Idx = Node->getOperand(2);
4863  EVT IdxVT = Idx.getValueType();
4864  SDLoc SL(Node);
4865 
4866  SDValue Factor = DAG.getConstant(NewEltsPerOldElt, SDLoc(), IdxVT);
4867  SDValue NewBaseIdx = DAG.getNode(ISD::MUL, SL, IdxVT, Idx, Factor);
4868 
4869  SDValue CastVec = DAG.getNode(ISD::BITCAST, SL, NVT, Node->getOperand(0));
4870  SDValue CastVal = DAG.getNode(ISD::BITCAST, SL, MidVT, Val);
4871 
4872  SDValue NewVec = CastVec;
4873  for (unsigned I = 0; I < NewEltsPerOldElt; ++I) {
4874  SDValue IdxOffset = DAG.getConstant(I, SL, IdxVT);
4875  SDValue InEltIdx = DAG.getNode(ISD::ADD, SL, IdxVT, NewBaseIdx, IdxOffset);
4876 
4877  SDValue Elt = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, NewEltVT,
4878  CastVal, IdxOffset);
4879 
4880  NewVec = DAG.getNode(ISD::INSERT_VECTOR_ELT, SL, NVT,
4881  NewVec, Elt, InEltIdx);
4882  }
4883 
4884  Results.push_back(DAG.getNode(ISD::BITCAST, SL, OVT, NewVec));
4885  break;
4886  }
4887  case ISD::SCALAR_TO_VECTOR: {
4888  MVT EltVT = OVT.getVectorElementType();
4889  MVT NewEltVT = NVT.getVectorElementType();
4890 
4891  // Handle bitcasts to different vector type with the same total bit size.
4892  //
4893  // e.g. v2i64 = scalar_to_vector x:i64
4894  // =>
4895  // concat_vectors (v2i32 bitcast x:i64), (v2i32 undef)
4896  //
4897 
4898  MVT MidVT = getPromotedVectorElementType(TLI, EltVT, NewEltVT);
4899  SDValue Val = Node->getOperand(0);
4900  SDLoc SL(Node);
4901 
4902  SDValue CastVal = DAG.getNode(ISD::BITCAST, SL, MidVT, Val);
4903  SDValue Undef = DAG.getUNDEF(MidVT);
4904 
4905  SmallVector<SDValue, 8> NewElts;
4906  NewElts.push_back(CastVal);
4907  for (unsigned I = 1, NElts = OVT.getVectorNumElements(); I != NElts; ++I)
4908  NewElts.push_back(Undef);
4909 
4910  SDValue Concat = DAG.getNode(ISD::CONCAT_VECTORS, SL, NVT, NewElts);
4911  SDValue CvtVec = DAG.getNode(ISD::BITCAST, SL, OVT, Concat);
4912  Results.push_back(CvtVec);
4913  break;
4914  }
4915  case ISD::ATOMIC_SWAP: {
4916  AtomicSDNode *AM = cast<AtomicSDNode>(Node);
4917  SDLoc SL(Node);
4918  SDValue CastVal = DAG.getNode(ISD::BITCAST, SL, NVT, AM->getVal());
4919  assert(NVT.getSizeInBits() == OVT.getSizeInBits() &&
4920  "unexpected promotion type");
4921  assert(AM->getMemoryVT().getSizeInBits() == NVT.getSizeInBits() &&
4922  "unexpected atomic_swap with illegal type");
4923 
4924  SDValue NewAtomic
4925  = DAG.getAtomic(ISD::ATOMIC_SWAP, SL, NVT,
4926  DAG.getVTList(NVT, MVT::Other),
4927  { AM->getChain(), AM->getBasePtr(), CastVal },
4928  AM->getMemOperand());
4929  Results.push_back(DAG.getNode(ISD::BITCAST, SL, OVT, NewAtomic));
4930  Results.push_back(NewAtomic.getValue(1));
4931  break;
4932  }
4933  }
4934 
4935  // Replace the original node with the legalized result.
4936  if (!Results.empty()) {
4937  LLVM_DEBUG(dbgs() << "Successfully promoted node\n");
4938  ReplaceNode(Node, Results.data());
4939  } else
4940  LLVM_DEBUG(dbgs() << "Could not promote node\n");
4941 }
4942 
4943 /// This is the entry point for the file.
4946 
4947  SmallPtrSet<SDNode *, 16> LegalizedNodes;
4948  // Use a delete listener to remove nodes which were deleted during
4949  // legalization from LegalizeNodes. This is needed to handle the situation
4950  // where a new node is allocated by the object pool to the same address of a
4951  // previously deleted node.
4952  DAGNodeDeletedListener DeleteListener(
4953  *this,
4954  [&LegalizedNodes](SDNode *N, SDNode *E) { LegalizedNodes.erase(N); });
4955 
4956  SelectionDAGLegalize Legalizer(*this, LegalizedNodes);
4957 
4958  // Visit all the nodes. We start in topological order, so that we see
4959  // nodes with their original operands intact. Legalization can produce
4960  // new nodes which may themselves need to be legalized. Iterate until all
4961  // nodes have been legalized.
4962  while (true) {
4963  bool AnyLegalized = false;
4964  for (auto NI = allnodes_end(); NI != allnodes_begin();) {
4965  --NI;
4966 
4967  SDNode *N = &*NI;
4968  if (N->use_empty() && N != getRoot().getNode()) {
4969  ++NI;
4970  DeleteNode(N);
4971  continue;
4972  }
4973 
4974  if (LegalizedNodes.insert(N).second) {
4975  AnyLegalized = true;
4976  Legalizer.LegalizeOp(N);
4977 
4978  if (N->use_empty() && N != getRoot().getNode()) {
4979  ++NI;
4980  DeleteNode(N);
4981  }
4982  }
4983  }
4984  if (!AnyLegalized)
4985  break;
4986 
4987  }
4988 
4989  // Remove dead nodes now.
4990  RemoveDeadNodes();
4991 }
4992 
4994  SmallSetVector<SDNode *, 16> &UpdatedNodes) {
4995  SmallPtrSet<SDNode *, 16> LegalizedNodes;
4996  SelectionDAGLegalize Legalizer(*this, LegalizedNodes, &UpdatedNodes);
4997 
4998  // Directly insert the node in question, and legalize it. This will recurse
4999  // as needed through operands.
5000  LegalizedNodes.insert(N);
5001  Legalizer.LegalizeOp(N);
5002 
5003  return LegalizedNodes.count(N);
5004 }
llvm::ISD::SUB
@ SUB
Definition: ISDOpcodes.h:233
llvm::Check::Size
@ Size
Definition: FileCheck.h:73
llvm::ISD::FPOWI
@ FPOWI
Definition: ISDOpcodes.h:862
llvm::ISD::FROUNDEVEN
@ FROUNDEVEN
Definition: ISDOpcodes.h:874
i
i
Definition: README.txt:29
llvm::ISD::ExternalSymbol
@ ExternalSymbol
Definition: ISDOpcodes.h:76
llvm::ISD::STRICT_FP_ROUND
@ STRICT_FP_ROUND
X = STRICT_FP_ROUND(Y, TRUNC) - Rounding 'Y' from a larger floating point type down to the precision ...
Definition: ISDOpcodes.h:445
ValueTypes.h