LLVM  14.0.0git
LegalizeDAG.cpp
Go to the documentation of this file.
1 //===- LegalizeDAG.cpp - Implement SelectionDAG::Legalize -----------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file implements the SelectionDAG::Legalize method.
10 //
11 //===----------------------------------------------------------------------===//
12 
13 #include "llvm/ADT/APFloat.h"
14 #include "llvm/ADT/APInt.h"
15 #include "llvm/ADT/ArrayRef.h"
16 #include "llvm/ADT/SetVector.h"
17 #include "llvm/ADT/SmallPtrSet.h"
18 #include "llvm/ADT/SmallSet.h"
19 #include "llvm/ADT/SmallVector.h"
32 #include "llvm/IR/CallingConv.h"
33 #include "llvm/IR/Constants.h"
34 #include "llvm/IR/DataLayout.h"
35 #include "llvm/IR/DerivedTypes.h"
36 #include "llvm/IR/Function.h"
37 #include "llvm/IR/Metadata.h"
38 #include "llvm/IR/Type.h"
39 #include "llvm/Support/Casting.h"
40 #include "llvm/Support/Compiler.h"
41 #include "llvm/Support/Debug.h"
48 #include <algorithm>
49 #include <cassert>
50 #include <cstdint>
51 #include <tuple>
52 #include <utility>
53 
54 using namespace llvm;
55 
56 #define DEBUG_TYPE "legalizedag"
57 
58 namespace {
59 
60 /// Keeps track of state when getting the sign of a floating-point value as an
61 /// integer.
62 struct FloatSignAsInt {
63  EVT FloatVT;
64  SDValue Chain;
65  SDValue FloatPtr;
66  SDValue IntPtr;
67  MachinePointerInfo IntPointerInfo;
68  MachinePointerInfo FloatPointerInfo;
69  SDValue IntValue;
70  APInt SignMask;
71  uint8_t SignBit;
72 };
73 
74 //===----------------------------------------------------------------------===//
75 /// This takes an arbitrary SelectionDAG as input and
76 /// hacks on it until the target machine can handle it. This involves
77 /// eliminating value sizes the machine cannot handle (promoting small sizes to
78 /// large sizes or splitting up large values into small values) as well as
79 /// eliminating operations the machine cannot handle.
80 ///
81 /// This code also does a small amount of optimization and recognition of idioms
82 /// as part of its processing. For example, if a target does not support a
83 /// 'setcc' instruction efficiently, but does support 'brcc' instruction, this
84 /// will attempt merge setcc and brc instructions into brcc's.
85 class SelectionDAGLegalize {
86  const TargetMachine &TM;
87  const TargetLowering &TLI;
88  SelectionDAG &DAG;
89 
90  /// The set of nodes which have already been legalized. We hold a
91  /// reference to it in order to update as necessary on node deletion.
92  SmallPtrSetImpl<SDNode *> &LegalizedNodes;
93 
94  /// A set of all the nodes updated during legalization.
95  SmallSetVector<SDNode *, 16> *UpdatedNodes;
96 
97  EVT getSetCCResultType(EVT VT) const {
98  return TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT);
99  }
100 
101  // Libcall insertion helpers.
102 
103 public:
104  SelectionDAGLegalize(SelectionDAG &DAG,
105  SmallPtrSetImpl<SDNode *> &LegalizedNodes,
106  SmallSetVector<SDNode *, 16> *UpdatedNodes = nullptr)
107  : TM(DAG.getTarget()), TLI(DAG.getTargetLoweringInfo()), DAG(DAG),
108  LegalizedNodes(LegalizedNodes), UpdatedNodes(UpdatedNodes) {}
109 
110  /// Legalizes the given operation.
111  void LegalizeOp(SDNode *Node);
112 
113 private:
114  SDValue OptimizeFloatStore(StoreSDNode *ST);
115 
116  void LegalizeLoadOps(SDNode *Node);
117  void LegalizeStoreOps(SDNode *Node);
118 
119  /// Some targets cannot handle a variable
120  /// insertion index for the INSERT_VECTOR_ELT instruction. In this case, it
121  /// is necessary to spill the vector being inserted into to memory, perform
122  /// the insert there, and then read the result back.
123  SDValue PerformInsertVectorEltInMemory(SDValue Vec, SDValue Val, SDValue Idx,
124  const SDLoc &dl);
125  SDValue ExpandINSERT_VECTOR_ELT(SDValue Vec, SDValue Val, SDValue Idx,
126  const SDLoc &dl);
127 
128  /// Return a vector shuffle operation which
129  /// performs the same shuffe in terms of order or result bytes, but on a type
130  /// whose vector element type is narrower than the original shuffle type.
131  /// e.g. <v4i32> <0, 1, 0, 1> -> v8i16 <0, 1, 2, 3, 0, 1, 2, 3>
132  SDValue ShuffleWithNarrowerEltType(EVT NVT, EVT VT, const SDLoc &dl,
133  SDValue N1, SDValue N2,
134  ArrayRef<int> Mask) const;
135 
136  SDValue ExpandLibCall(RTLIB::Libcall LC, SDNode *Node, bool isSigned);
137 
138  void ExpandFPLibCall(SDNode *Node, RTLIB::Libcall LC,
140  void ExpandFPLibCall(SDNode *Node, RTLIB::Libcall Call_F32,
141  RTLIB::Libcall Call_F64, RTLIB::Libcall Call_F80,
142  RTLIB::Libcall Call_F128,
143  RTLIB::Libcall Call_PPCF128,
145  SDValue ExpandIntLibCall(SDNode *Node, bool isSigned,
146  RTLIB::Libcall Call_I8,
147  RTLIB::Libcall Call_I16,
148  RTLIB::Libcall Call_I32,
149  RTLIB::Libcall Call_I64,
150  RTLIB::Libcall Call_I128);
151  void ExpandArgFPLibCall(SDNode *Node,
152  RTLIB::Libcall Call_F32, RTLIB::Libcall Call_F64,
153  RTLIB::Libcall Call_F80, RTLIB::Libcall Call_F128,
154  RTLIB::Libcall Call_PPCF128,
156  void ExpandDivRemLibCall(SDNode *Node, SmallVectorImpl<SDValue> &Results);
157  void ExpandSinCosLibCall(SDNode *Node, SmallVectorImpl<SDValue> &Results);
158 
159  SDValue EmitStackConvert(SDValue SrcOp, EVT SlotVT, EVT DestVT,
160  const SDLoc &dl);
161  SDValue EmitStackConvert(SDValue SrcOp, EVT SlotVT, EVT DestVT,
162  const SDLoc &dl, SDValue ChainIn);
163  SDValue ExpandBUILD_VECTOR(SDNode *Node);
164  SDValue ExpandSPLAT_VECTOR(SDNode *Node);
165  SDValue ExpandSCALAR_TO_VECTOR(SDNode *Node);
166  void ExpandDYNAMIC_STACKALLOC(SDNode *Node,
168  void getSignAsIntValue(FloatSignAsInt &State, const SDLoc &DL,
169  SDValue Value) const;
170  SDValue modifySignAsInt(const FloatSignAsInt &State, const SDLoc &DL,
171  SDValue NewIntValue) const;
172  SDValue ExpandFCOPYSIGN(SDNode *Node) const;
173  SDValue ExpandFABS(SDNode *Node) const;
174  SDValue ExpandFNEG(SDNode *Node) const;
175  SDValue ExpandLegalINT_TO_FP(SDNode *Node, SDValue &Chain);
176  void PromoteLegalINT_TO_FP(SDNode *N, const SDLoc &dl,
178  void PromoteLegalFP_TO_INT(SDNode *N, const SDLoc &dl,
180  SDValue PromoteLegalFP_TO_INT_SAT(SDNode *Node, const SDLoc &dl);
181 
182  SDValue ExpandPARITY(SDValue Op, const SDLoc &dl);
183 
184  SDValue ExpandExtractFromVectorThroughStack(SDValue Op);
185  SDValue ExpandInsertToVectorThroughStack(SDValue Op);
186  SDValue ExpandVectorBuildThroughStack(SDNode* Node);
187 
188  SDValue ExpandConstantFP(ConstantFPSDNode *CFP, bool UseCP);
189  SDValue ExpandConstant(ConstantSDNode *CP);
190 
191  // if ExpandNode returns false, LegalizeOp falls back to ConvertNodeToLibcall
192  bool ExpandNode(SDNode *Node);
193  void ConvertNodeToLibcall(SDNode *Node);
194  void PromoteNode(SDNode *Node);
195 
196 public:
197  // Node replacement helpers
198 
199  void ReplacedNode(SDNode *N) {
200  LegalizedNodes.erase(N);
201  if (UpdatedNodes)
202  UpdatedNodes->insert(N);
203  }
204 
205  void ReplaceNode(SDNode *Old, SDNode *New) {
206  LLVM_DEBUG(dbgs() << " ... replacing: "; Old->dump(&DAG);
207  dbgs() << " with: "; New->dump(&DAG));
208 
209  assert(Old->getNumValues() == New->getNumValues() &&
210  "Replacing one node with another that produces a different number "
211  "of values!");
212  DAG.ReplaceAllUsesWith(Old, New);
213  if (UpdatedNodes)
214  UpdatedNodes->insert(New);
215  ReplacedNode(Old);
216  }
217 
218  void ReplaceNode(SDValue Old, SDValue New) {
219  LLVM_DEBUG(dbgs() << " ... replacing: "; Old->dump(&DAG);
220  dbgs() << " with: "; New->dump(&DAG));
221 
222  DAG.ReplaceAllUsesWith(Old, New);
223  if (UpdatedNodes)
224  UpdatedNodes->insert(New.getNode());
225  ReplacedNode(Old.getNode());
226  }
227 
228  void ReplaceNode(SDNode *Old, const SDValue *New) {
229  LLVM_DEBUG(dbgs() << " ... replacing: "; Old->dump(&DAG));
230 
231  DAG.ReplaceAllUsesWith(Old, New);
232  for (unsigned i = 0, e = Old->getNumValues(); i != e; ++i) {
233  LLVM_DEBUG(dbgs() << (i == 0 ? " with: " : " and: ");
234  New[i]->dump(&DAG));
235  if (UpdatedNodes)
236  UpdatedNodes->insert(New[i].getNode());
237  }
238  ReplacedNode(Old);
239  }
240 
241  void ReplaceNodeWithValue(SDValue Old, SDValue New) {
242  LLVM_DEBUG(dbgs() << " ... replacing: "; Old->dump(&DAG);
243  dbgs() << " with: "; New->dump(&DAG));
244 
245  DAG.ReplaceAllUsesOfValueWith(Old, New);
246  if (UpdatedNodes)
247  UpdatedNodes->insert(New.getNode());
248  ReplacedNode(Old.getNode());
249  }
250 };
251 
252 } // end anonymous namespace
253 
254 /// Return a vector shuffle operation which
255 /// performs the same shuffle in terms of order or result bytes, but on a type
256 /// whose vector element type is narrower than the original shuffle type.
257 /// e.g. <v4i32> <0, 1, 0, 1> -> v8i16 <0, 1, 2, 3, 0, 1, 2, 3>
258 SDValue SelectionDAGLegalize::ShuffleWithNarrowerEltType(
259  EVT NVT, EVT VT, const SDLoc &dl, SDValue N1, SDValue N2,
260  ArrayRef<int> Mask) const {
261  unsigned NumMaskElts = VT.getVectorNumElements();
262  unsigned NumDestElts = NVT.getVectorNumElements();
263  unsigned NumEltsGrowth = NumDestElts / NumMaskElts;
264 
265  assert(NumEltsGrowth && "Cannot promote to vector type with fewer elts!");
266 
267  if (NumEltsGrowth == 1)
268  return DAG.getVectorShuffle(NVT, dl, N1, N2, Mask);
269 
270  SmallVector<int, 8> NewMask;
271  for (unsigned i = 0; i != NumMaskElts; ++i) {
272  int Idx = Mask[i];
273  for (unsigned j = 0; j != NumEltsGrowth; ++j) {
274  if (Idx < 0)
275  NewMask.push_back(-1);
276  else
277  NewMask.push_back(Idx * NumEltsGrowth + j);
278  }
279  }
280  assert(NewMask.size() == NumDestElts && "Non-integer NumEltsGrowth?");
281  assert(TLI.isShuffleMaskLegal(NewMask, NVT) && "Shuffle not legal?");
282  return DAG.getVectorShuffle(NVT, dl, N1, N2, NewMask);
283 }
284 
285 /// Expands the ConstantFP node to an integer constant or
286 /// a load from the constant pool.
287 SDValue
288 SelectionDAGLegalize::ExpandConstantFP(ConstantFPSDNode *CFP, bool UseCP) {
289  bool Extend = false;
290  SDLoc dl(CFP);
291 
292  // If a FP immediate is precise when represented as a float and if the
293  // target can do an extending load from float to double, we put it into
294  // the constant pool as a float, even if it's is statically typed as a
295  // double. This shrinks FP constants and canonicalizes them for targets where
296  // an FP extending load is the same cost as a normal load (such as on the x87
297  // fp stack or PPC FP unit).
298  EVT VT = CFP->getValueType(0);
299  ConstantFP *LLVMC = const_cast<ConstantFP*>(CFP->getConstantFPValue());
300  if (!UseCP) {
301  assert((VT == MVT::f64 || VT == MVT::f32) && "Invalid type expansion");
302  return DAG.getConstant(LLVMC->getValueAPF().bitcastToAPInt(), dl,
303  (VT == MVT::f64) ? MVT::i64 : MVT::i32);
304  }
305 
306  APFloat APF = CFP->getValueAPF();
307  EVT OrigVT = VT;
308  EVT SVT = VT;
309 
310  // We don't want to shrink SNaNs. Converting the SNaN back to its real type
311  // can cause it to be changed into a QNaN on some platforms (e.g. on SystemZ).
312  if (!APF.isSignaling()) {
313  while (SVT != MVT::f32 && SVT != MVT::f16) {
314  SVT = (MVT::SimpleValueType)(SVT.getSimpleVT().SimpleTy - 1);
316  // Only do this if the target has a native EXTLOAD instruction from
317  // smaller type.
318  TLI.isLoadExtLegal(ISD::EXTLOAD, OrigVT, SVT) &&
319  TLI.ShouldShrinkFPConstant(OrigVT)) {
320  Type *SType = SVT.getTypeForEVT(*DAG.getContext());
321  LLVMC = cast<ConstantFP>(ConstantExpr::getFPTrunc(LLVMC, SType));
322  VT = SVT;
323  Extend = true;
324  }
325  }
326  }
327 
328  SDValue CPIdx =
329  DAG.getConstantPool(LLVMC, TLI.getPointerTy(DAG.getDataLayout()));
330  Align Alignment = cast<ConstantPoolSDNode>(CPIdx)->getAlign();
331  if (Extend) {
332  SDValue Result = DAG.getExtLoad(
333  ISD::EXTLOAD, dl, OrigVT, DAG.getEntryNode(), CPIdx,
334  MachinePointerInfo::getConstantPool(DAG.getMachineFunction()), VT,
335  Alignment);
336  return Result;
337  }
338  SDValue Result = DAG.getLoad(
339  OrigVT, dl, DAG.getEntryNode(), CPIdx,
340  MachinePointerInfo::getConstantPool(DAG.getMachineFunction()), Alignment);
341  return Result;
342 }
343 
344 /// Expands the Constant node to a load from the constant pool.
345 SDValue SelectionDAGLegalize::ExpandConstant(ConstantSDNode *CP) {
346  SDLoc dl(CP);
347  EVT VT = CP->getValueType(0);
348  SDValue CPIdx = DAG.getConstantPool(CP->getConstantIntValue(),
349  TLI.getPointerTy(DAG.getDataLayout()));
350  Align Alignment = cast<ConstantPoolSDNode>(CPIdx)->getAlign();
351  SDValue Result = DAG.getLoad(
352  VT, dl, DAG.getEntryNode(), CPIdx,
353  MachinePointerInfo::getConstantPool(DAG.getMachineFunction()), Alignment);
354  return Result;
355 }
356 
357 /// Some target cannot handle a variable insertion index for the
358 /// INSERT_VECTOR_ELT instruction. In this case, it
359 /// is necessary to spill the vector being inserted into to memory, perform
360 /// the insert there, and then read the result back.
361 SDValue SelectionDAGLegalize::PerformInsertVectorEltInMemory(SDValue Vec,
362  SDValue Val,
363  SDValue Idx,
364  const SDLoc &dl) {
365  SDValue Tmp1 = Vec;
366  SDValue Tmp2 = Val;
367  SDValue Tmp3 = Idx;
368 
369  // If the target doesn't support this, we have to spill the input vector
370  // to a temporary stack slot, update the element, then reload it. This is
371  // badness. We could also load the value into a vector register (either
372  // with a "move to register" or "extload into register" instruction, then
373  // permute it into place, if the idx is a constant and if the idx is
374  // supported by the target.
375  EVT VT = Tmp1.getValueType();
376  EVT EltVT = VT.getVectorElementType();
377  SDValue StackPtr = DAG.CreateStackTemporary(VT);
378 
379  int SPFI = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex();
380 
381  // Store the vector.
382  SDValue Ch = DAG.getStore(
383  DAG.getEntryNode(), dl, Tmp1, StackPtr,
384  MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), SPFI));
385 
386  SDValue StackPtr2 = TLI.getVectorElementPointer(DAG, StackPtr, VT, Tmp3);
387 
388  // Store the scalar value.
389  Ch = DAG.getTruncStore(
390  Ch, dl, Tmp2, StackPtr2,
391  MachinePointerInfo::getUnknownStack(DAG.getMachineFunction()), EltVT);
392  // Load the updated vector.
393  return DAG.getLoad(VT, dl, Ch, StackPtr, MachinePointerInfo::getFixedStack(
394  DAG.getMachineFunction(), SPFI));
395 }
396 
397 SDValue SelectionDAGLegalize::ExpandINSERT_VECTOR_ELT(SDValue Vec, SDValue Val,
398  SDValue Idx,
399  const SDLoc &dl) {
400  if (ConstantSDNode *InsertPos = dyn_cast<ConstantSDNode>(Idx)) {
401  // SCALAR_TO_VECTOR requires that the type of the value being inserted
402  // match the element type of the vector being created, except for
403  // integers in which case the inserted value can be over width.
404  EVT EltVT = Vec.getValueType().getVectorElementType();
405  if (Val.getValueType() == EltVT ||
406  (EltVT.isInteger() && Val.getValueType().bitsGE(EltVT))) {
407  SDValue ScVec = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl,
408  Vec.getValueType(), Val);
409 
410  unsigned NumElts = Vec.getValueType().getVectorNumElements();
411  // We generate a shuffle of InVec and ScVec, so the shuffle mask
412  // should be 0,1,2,3,4,5... with the appropriate element replaced with
413  // elt 0 of the RHS.
414  SmallVector<int, 8> ShufOps;
415  for (unsigned i = 0; i != NumElts; ++i)
416  ShufOps.push_back(i != InsertPos->getZExtValue() ? i : NumElts);
417 
418  return DAG.getVectorShuffle(Vec.getValueType(), dl, Vec, ScVec, ShufOps);
419  }
420  }
421  return PerformInsertVectorEltInMemory(Vec, Val, Idx, dl);
422 }
423 
424 SDValue SelectionDAGLegalize::OptimizeFloatStore(StoreSDNode* ST) {
425  if (!ISD::isNormalStore(ST))
426  return SDValue();
427 
428  LLVM_DEBUG(dbgs() << "Optimizing float store operations\n");
429  // Turn 'store float 1.0, Ptr' -> 'store int 0x12345678, Ptr'
430  // FIXME: move this to the DAG Combiner! Note that we can't regress due
431  // to phase ordering between legalized code and the dag combiner. This
432  // probably means that we need to integrate dag combiner and legalizer
433  // together.
434  // We generally can't do this one for long doubles.
435  SDValue Chain = ST->getChain();
436  SDValue Ptr = ST->getBasePtr();
437  SDValue Value = ST->getValue();
438  MachineMemOperand::Flags MMOFlags = ST->getMemOperand()->getFlags();
439  AAMDNodes AAInfo = ST->getAAInfo();
440  SDLoc dl(ST);
441 
442  // Don't optimise TargetConstantFP
443  if (Value.getOpcode() == ISD::TargetConstantFP)
444  return SDValue();
445 
446  if (ConstantFPSDNode *CFP = dyn_cast<ConstantFPSDNode>(Value)) {
447  if (CFP->getValueType(0) == MVT::f32 &&
448  TLI.isTypeLegal(MVT::i32)) {
449  SDValue Con = DAG.getConstant(CFP->getValueAPF().
450  bitcastToAPInt().zextOrTrunc(32),
451  SDLoc(CFP), MVT::i32);
452  return DAG.getStore(Chain, dl, Con, Ptr, ST->getPointerInfo(),
453  ST->getOriginalAlign(), MMOFlags, AAInfo);
454  }
455 
456  if (CFP->getValueType(0) == MVT::f64) {
457  // If this target supports 64-bit registers, do a single 64-bit store.
458  if (TLI.isTypeLegal(MVT::i64)) {
459  SDValue Con = DAG.getConstant(CFP->getValueAPF().bitcastToAPInt().
460  zextOrTrunc(64), SDLoc(CFP), MVT::i64);
461  return DAG.getStore(Chain, dl, Con, Ptr, ST->getPointerInfo(),
462  ST->getOriginalAlign(), MMOFlags, AAInfo);
463  }
464 
465  if (TLI.isTypeLegal(MVT::i32) && !ST->isVolatile()) {
466  // Otherwise, if the target supports 32-bit registers, use 2 32-bit
467  // stores. If the target supports neither 32- nor 64-bits, this
468  // xform is certainly not worth it.
469  const APInt &IntVal = CFP->getValueAPF().bitcastToAPInt();
470  SDValue Lo = DAG.getConstant(IntVal.trunc(32), dl, MVT::i32);
471  SDValue Hi = DAG.getConstant(IntVal.lshr(32).trunc(32), dl, MVT::i32);
472  if (DAG.getDataLayout().isBigEndian())
473  std::swap(Lo, Hi);
474 
475  Lo = DAG.getStore(Chain, dl, Lo, Ptr, ST->getPointerInfo(),
476  ST->getOriginalAlign(), MMOFlags, AAInfo);
477  Ptr = DAG.getMemBasePlusOffset(Ptr, TypeSize::Fixed(4), dl);
478  Hi = DAG.getStore(Chain, dl, Hi, Ptr,
479  ST->getPointerInfo().getWithOffset(4),
480  ST->getOriginalAlign(), MMOFlags, AAInfo);
481 
482  return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo, Hi);
483  }
484  }
485  }
486  return SDValue();
487 }
488 
489 void SelectionDAGLegalize::LegalizeStoreOps(SDNode *Node) {
490  StoreSDNode *ST = cast<StoreSDNode>(Node);
491  SDValue Chain = ST->getChain();
492  SDValue Ptr = ST->getBasePtr();
493  SDLoc dl(Node);
494 
495  MachineMemOperand::Flags MMOFlags = ST->getMemOperand()->getFlags();
496  AAMDNodes AAInfo = ST->getAAInfo();
497 
498  if (!ST->isTruncatingStore()) {
499  LLVM_DEBUG(dbgs() << "Legalizing store operation\n");
500  if (SDNode *OptStore = OptimizeFloatStore(ST).getNode()) {
501  ReplaceNode(ST, OptStore);
502  return;
503  }
504 
505  SDValue Value = ST->getValue();
506  MVT VT = Value.getSimpleValueType();
507  switch (TLI.getOperationAction(ISD::STORE, VT)) {
508  default: llvm_unreachable("This action is not supported yet!");
509  case TargetLowering::Legal: {
510  // If this is an unaligned store and the target doesn't support it,
511  // expand it.
512  EVT MemVT = ST->getMemoryVT();
513  const DataLayout &DL = DAG.getDataLayout();
514  if (!TLI.allowsMemoryAccessForAlignment(*DAG.getContext(), DL, MemVT,
515  *ST->getMemOperand())) {
516  LLVM_DEBUG(dbgs() << "Expanding unsupported unaligned store\n");
517  SDValue Result = TLI.expandUnalignedStore(ST, DAG);
518  ReplaceNode(SDValue(ST, 0), Result);
519  } else
520  LLVM_DEBUG(dbgs() << "Legal store\n");
521  break;
522  }
523  case TargetLowering::Custom: {
524  LLVM_DEBUG(dbgs() << "Trying custom lowering\n");
525  SDValue Res = TLI.LowerOperation(SDValue(Node, 0), DAG);
526  if (Res && Res != SDValue(Node, 0))
527  ReplaceNode(SDValue(Node, 0), Res);
528  return;
529  }
531  MVT NVT = TLI.getTypeToPromoteTo(ISD::STORE, VT);
532  assert(NVT.getSizeInBits() == VT.getSizeInBits() &&
533  "Can only promote stores to same size type");
534  Value = DAG.getNode(ISD::BITCAST, dl, NVT, Value);
535  SDValue Result = DAG.getStore(Chain, dl, Value, Ptr, ST->getPointerInfo(),
536  ST->getOriginalAlign(), MMOFlags, AAInfo);
537  ReplaceNode(SDValue(Node, 0), Result);
538  break;
539  }
540  }
541  return;
542  }
543 
544  LLVM_DEBUG(dbgs() << "Legalizing truncating store operations\n");
545  SDValue Value = ST->getValue();
546  EVT StVT = ST->getMemoryVT();
547  TypeSize StWidth = StVT.getSizeInBits();
548  TypeSize StSize = StVT.getStoreSizeInBits();
549  auto &DL = DAG.getDataLayout();
550 
551  if (StWidth != StSize) {
552  // Promote to a byte-sized store with upper bits zero if not
553  // storing an integral number of bytes. For example, promote
554  // TRUNCSTORE:i1 X -> TRUNCSTORE:i8 (and X, 1)
555  EVT NVT = EVT::getIntegerVT(*DAG.getContext(), StSize.getFixedSize());
556  Value = DAG.getZeroExtendInReg(Value, dl, StVT);
557  SDValue Result =
558  DAG.getTruncStore(Chain, dl, Value, Ptr, ST->getPointerInfo(), NVT,
559  ST->getOriginalAlign(), MMOFlags, AAInfo);
560  ReplaceNode(SDValue(Node, 0), Result);
561  } else if (!StVT.isVector() && !isPowerOf2_64(StWidth.getFixedSize())) {
562  // If not storing a power-of-2 number of bits, expand as two stores.
563  assert(!StVT.isVector() && "Unsupported truncstore!");
564  unsigned StWidthBits = StWidth.getFixedSize();
565  unsigned LogStWidth = Log2_32(StWidthBits);
566  assert(LogStWidth < 32);
567  unsigned RoundWidth = 1 << LogStWidth;
568  assert(RoundWidth < StWidthBits);
569  unsigned ExtraWidth = StWidthBits - RoundWidth;
570  assert(ExtraWidth < RoundWidth);
571  assert(!(RoundWidth % 8) && !(ExtraWidth % 8) &&
572  "Store size not an integral number of bytes!");
573  EVT RoundVT = EVT::getIntegerVT(*DAG.getContext(), RoundWidth);
574  EVT ExtraVT = EVT::getIntegerVT(*DAG.getContext(), ExtraWidth);
575  SDValue Lo, Hi;
576  unsigned IncrementSize;
577 
578  if (DL.isLittleEndian()) {
579  // TRUNCSTORE:i24 X -> TRUNCSTORE:i16 X, TRUNCSTORE@+2:i8 (srl X, 16)
580  // Store the bottom RoundWidth bits.
581  Lo = DAG.getTruncStore(Chain, dl, Value, Ptr, ST->getPointerInfo(),
582  RoundVT, ST->getOriginalAlign(), MMOFlags, AAInfo);
583 
584  // Store the remaining ExtraWidth bits.
585  IncrementSize = RoundWidth / 8;
586  Ptr = DAG.getMemBasePlusOffset(Ptr, TypeSize::Fixed(IncrementSize), dl);
587  Hi = DAG.getNode(
588  ISD::SRL, dl, Value.getValueType(), Value,
589  DAG.getConstant(RoundWidth, dl,
590  TLI.getShiftAmountTy(Value.getValueType(), DL)));
591  Hi = DAG.getTruncStore(Chain, dl, Hi, Ptr,
592  ST->getPointerInfo().getWithOffset(IncrementSize),
593  ExtraVT, ST->getOriginalAlign(), MMOFlags, AAInfo);
594  } else {
595  // Big endian - avoid unaligned stores.
596  // TRUNCSTORE:i24 X -> TRUNCSTORE:i16 (srl X, 8), TRUNCSTORE@+2:i8 X
597  // Store the top RoundWidth bits.
598  Hi = DAG.getNode(
599  ISD::SRL, dl, Value.getValueType(), Value,
600  DAG.getConstant(ExtraWidth, dl,
601  TLI.getShiftAmountTy(Value.getValueType(), DL)));
602  Hi = DAG.getTruncStore(Chain, dl, Hi, Ptr, ST->getPointerInfo(), RoundVT,
603  ST->getOriginalAlign(), MMOFlags, AAInfo);
604 
605  // Store the remaining ExtraWidth bits.
606  IncrementSize = RoundWidth / 8;
607  Ptr = DAG.getNode(ISD::ADD, dl, Ptr.getValueType(), Ptr,
608  DAG.getConstant(IncrementSize, dl,
609  Ptr.getValueType()));
610  Lo = DAG.getTruncStore(Chain, dl, Value, Ptr,
611  ST->getPointerInfo().getWithOffset(IncrementSize),
612  ExtraVT, ST->getOriginalAlign(), MMOFlags, AAInfo);
613  }
614 
615  // The order of the stores doesn't matter.
616  SDValue Result = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo, Hi);
617  ReplaceNode(SDValue(Node, 0), Result);
618  } else {
619  switch (TLI.getTruncStoreAction(ST->getValue().getValueType(), StVT)) {
620  default: llvm_unreachable("This action is not supported yet!");
621  case TargetLowering::Legal: {
622  EVT MemVT = ST->getMemoryVT();
623  // If this is an unaligned store and the target doesn't support it,
624  // expand it.
625  if (!TLI.allowsMemoryAccessForAlignment(*DAG.getContext(), DL, MemVT,
626  *ST->getMemOperand())) {
627  SDValue Result = TLI.expandUnalignedStore(ST, DAG);
628  ReplaceNode(SDValue(ST, 0), Result);
629  }
630  break;
631  }
632  case TargetLowering::Custom: {
633  SDValue Res = TLI.LowerOperation(SDValue(Node, 0), DAG);
634  if (Res && Res != SDValue(Node, 0))
635  ReplaceNode(SDValue(Node, 0), Res);
636  return;
637  }
639  assert(!StVT.isVector() &&
640  "Vector Stores are handled in LegalizeVectorOps");
641 
642  SDValue Result;
643 
644  // TRUNCSTORE:i16 i32 -> STORE i16
645  if (TLI.isTypeLegal(StVT)) {
646  Value = DAG.getNode(ISD::TRUNCATE, dl, StVT, Value);
647  Result = DAG.getStore(Chain, dl, Value, Ptr, ST->getPointerInfo(),
648  ST->getOriginalAlign(), MMOFlags, AAInfo);
649  } else {
650  // The in-memory type isn't legal. Truncate to the type it would promote
651  // to, and then do a truncstore.
652  Value = DAG.getNode(ISD::TRUNCATE, dl,
653  TLI.getTypeToTransformTo(*DAG.getContext(), StVT),
654  Value);
655  Result =
656  DAG.getTruncStore(Chain, dl, Value, Ptr, ST->getPointerInfo(), StVT,
657  ST->getOriginalAlign(), MMOFlags, AAInfo);
658  }
659 
660  ReplaceNode(SDValue(Node, 0), Result);
661  break;
662  }
663  }
664 }
665 
666 void SelectionDAGLegalize::LegalizeLoadOps(SDNode *Node) {
667  LoadSDNode *LD = cast<LoadSDNode>(Node);
668  SDValue Chain = LD->getChain(); // The chain.
669  SDValue Ptr = LD->getBasePtr(); // The base pointer.
670  SDValue Value; // The value returned by the load op.
671  SDLoc dl(Node);
672 
673  ISD::LoadExtType ExtType = LD->getExtensionType();
674  if (ExtType == ISD::NON_EXTLOAD) {
675  LLVM_DEBUG(dbgs() << "Legalizing non-extending load operation\n");
676  MVT VT = Node->getSimpleValueType(0);
677  SDValue RVal = SDValue(Node, 0);
678  SDValue RChain = SDValue(Node, 1);
679 
680  switch (TLI.getOperationAction(Node->getOpcode(), VT)) {
681  default: llvm_unreachable("This action is not supported yet!");
682  case TargetLowering::Legal: {
683  EVT MemVT = LD->getMemoryVT();
684  const DataLayout &DL = DAG.getDataLayout();
685  // If this is an unaligned load and the target doesn't support it,
686  // expand it.
687  if (!TLI.allowsMemoryAccessForAlignment(*DAG.getContext(), DL, MemVT,
688  *LD->getMemOperand())) {
689  std::tie(RVal, RChain) = TLI.expandUnalignedLoad(LD, DAG);
690  }
691  break;
692  }
694  if (SDValue Res = TLI.LowerOperation(RVal, DAG)) {
695  RVal = Res;
696  RChain = Res.getValue(1);
697  }
698  break;
699 
701  MVT NVT = TLI.getTypeToPromoteTo(Node->getOpcode(), VT);
702  assert(NVT.getSizeInBits() == VT.getSizeInBits() &&
703  "Can only promote loads to same size type");
704 
705  SDValue Res = DAG.getLoad(NVT, dl, Chain, Ptr, LD->getMemOperand());
706  RVal = DAG.getNode(ISD::BITCAST, dl, VT, Res);
707  RChain = Res.getValue(1);
708  break;
709  }
710  }
711  if (RChain.getNode() != Node) {
712  assert(RVal.getNode() != Node && "Load must be completely replaced");
713  DAG.ReplaceAllUsesOfValueWith(SDValue(Node, 0), RVal);
714  DAG.ReplaceAllUsesOfValueWith(SDValue(Node, 1), RChain);
715  if (UpdatedNodes) {
716  UpdatedNodes->insert(RVal.getNode());
717  UpdatedNodes->insert(RChain.getNode());
718  }
719  ReplacedNode(Node);
720  }
721  return;
722  }
723 
724  LLVM_DEBUG(dbgs() << "Legalizing extending load operation\n");
725  EVT SrcVT = LD->getMemoryVT();
726  TypeSize SrcWidth = SrcVT.getSizeInBits();
727  MachineMemOperand::Flags MMOFlags = LD->getMemOperand()->getFlags();
728  AAMDNodes AAInfo = LD->getAAInfo();
729 
730  if (SrcWidth != SrcVT.getStoreSizeInBits() &&
731  // Some targets pretend to have an i1 loading operation, and actually
732  // load an i8. This trick is correct for ZEXTLOAD because the top 7
733  // bits are guaranteed to be zero; it helps the optimizers understand
734  // that these bits are zero. It is also useful for EXTLOAD, since it
735  // tells the optimizers that those bits are undefined. It would be
736  // nice to have an effective generic way of getting these benefits...
737  // Until such a way is found, don't insist on promoting i1 here.
738  (SrcVT != MVT::i1 ||
739  TLI.getLoadExtAction(ExtType, Node->getValueType(0), MVT::i1) ==
741  // Promote to a byte-sized load if not loading an integral number of
742  // bytes. For example, promote EXTLOAD:i20 -> EXTLOAD:i24.
743  unsigned NewWidth = SrcVT.getStoreSizeInBits();
744  EVT NVT = EVT::getIntegerVT(*DAG.getContext(), NewWidth);
745  SDValue Ch;
746 
747  // The extra bits are guaranteed to be zero, since we stored them that
748  // way. A zext load from NVT thus automatically gives zext from SrcVT.
749 
750  ISD::LoadExtType NewExtType =
752 
753  SDValue Result = DAG.getExtLoad(NewExtType, dl, Node->getValueType(0),
754  Chain, Ptr, LD->getPointerInfo(), NVT,
755  LD->getOriginalAlign(), MMOFlags, AAInfo);
756 
757  Ch = Result.getValue(1); // The chain.
758 
759  if (ExtType == ISD::SEXTLOAD)
760  // Having the top bits zero doesn't help when sign extending.
761  Result = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl,
762  Result.getValueType(),
763  Result, DAG.getValueType(SrcVT));
764  else if (ExtType == ISD::ZEXTLOAD || NVT == Result.getValueType())
765  // All the top bits are guaranteed to be zero - inform the optimizers.
766  Result = DAG.getNode(ISD::AssertZext, dl,
767  Result.getValueType(), Result,
768  DAG.getValueType(SrcVT));
769 
770  Value = Result;
771  Chain = Ch;
772  } else if (!isPowerOf2_64(SrcWidth.getKnownMinSize())) {
773  // If not loading a power-of-2 number of bits, expand as two loads.
774  assert(!SrcVT.isVector() && "Unsupported extload!");
775  unsigned SrcWidthBits = SrcWidth.getFixedSize();
776  unsigned LogSrcWidth = Log2_32(SrcWidthBits);
777  assert(LogSrcWidth < 32);
778  unsigned RoundWidth = 1 << LogSrcWidth;
779  assert(RoundWidth < SrcWidthBits);
780  unsigned ExtraWidth = SrcWidthBits - RoundWidth;
781  assert(ExtraWidth < RoundWidth);
782  assert(!(RoundWidth % 8) && !(ExtraWidth % 8) &&
783  "Load size not an integral number of bytes!");
784  EVT RoundVT = EVT::getIntegerVT(*DAG.getContext(), RoundWidth);
785  EVT ExtraVT = EVT::getIntegerVT(*DAG.getContext(), ExtraWidth);
786  SDValue Lo, Hi, Ch;
787  unsigned IncrementSize;
788  auto &DL = DAG.getDataLayout();
789 
790  if (DL.isLittleEndian()) {
791  // EXTLOAD:i24 -> ZEXTLOAD:i16 | (shl EXTLOAD@+2:i8, 16)
792  // Load the bottom RoundWidth bits.
793  Lo = DAG.getExtLoad(ISD::ZEXTLOAD, dl, Node->getValueType(0), Chain, Ptr,
794  LD->getPointerInfo(), RoundVT, LD->getOriginalAlign(),
795  MMOFlags, AAInfo);
796 
797  // Load the remaining ExtraWidth bits.
798  IncrementSize = RoundWidth / 8;
799  Ptr = DAG.getMemBasePlusOffset(Ptr, TypeSize::Fixed(IncrementSize), dl);
800  Hi = DAG.getExtLoad(ExtType, dl, Node->getValueType(0), Chain, Ptr,
801  LD->getPointerInfo().getWithOffset(IncrementSize),
802  ExtraVT, LD->getOriginalAlign(), MMOFlags, AAInfo);
803 
804  // Build a factor node to remember that this load is independent of
805  // the other one.
806  Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1),
807  Hi.getValue(1));
808 
809  // Move the top bits to the right place.
810  Hi = DAG.getNode(
811  ISD::SHL, dl, Hi.getValueType(), Hi,
812  DAG.getConstant(RoundWidth, dl,
813  TLI.getShiftAmountTy(Hi.getValueType(), DL)));
814 
815  // Join the hi and lo parts.
816  Value = DAG.getNode(ISD::OR, dl, Node->getValueType(0), Lo, Hi);
817  } else {
818  // Big endian - avoid unaligned loads.
819  // EXTLOAD:i24 -> (shl EXTLOAD:i16, 8) | ZEXTLOAD@+2:i8
820  // Load the top RoundWidth bits.
821  Hi = DAG.getExtLoad(ExtType, dl, Node->getValueType(0), Chain, Ptr,
822  LD->getPointerInfo(), RoundVT, LD->getOriginalAlign(),
823  MMOFlags, AAInfo);
824 
825  // Load the remaining ExtraWidth bits.
826  IncrementSize = RoundWidth / 8;
827  Ptr = DAG.getMemBasePlusOffset(Ptr, TypeSize::Fixed(IncrementSize), dl);
828  Lo = DAG.getExtLoad(ISD::ZEXTLOAD, dl, Node->getValueType(0), Chain, Ptr,
829  LD->getPointerInfo().getWithOffset(IncrementSize),
830  ExtraVT, LD->getOriginalAlign(), MMOFlags, AAInfo);
831 
832  // Build a factor node to remember that this load is independent of
833  // the other one.
834  Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1),
835  Hi.getValue(1));
836 
837  // Move the top bits to the right place.
838  Hi = DAG.getNode(
839  ISD::SHL, dl, Hi.getValueType(), Hi,
840  DAG.getConstant(ExtraWidth, dl,
841  TLI.getShiftAmountTy(Hi.getValueType(), DL)));
842 
843  // Join the hi and lo parts.
844  Value = DAG.getNode(ISD::OR, dl, Node->getValueType(0), Lo, Hi);
845  }
846 
847  Chain = Ch;
848  } else {
849  bool isCustom = false;
850  switch (TLI.getLoadExtAction(ExtType, Node->getValueType(0),
851  SrcVT.getSimpleVT())) {
852  default: llvm_unreachable("This action is not supported yet!");
854  isCustom = true;
857  Value = SDValue(Node, 0);
858  Chain = SDValue(Node, 1);
859 
860  if (isCustom) {
861  if (SDValue Res = TLI.LowerOperation(SDValue(Node, 0), DAG)) {
862  Value = Res;
863  Chain = Res.getValue(1);
864  }
865  } else {
866  // If this is an unaligned load and the target doesn't support it,
867  // expand it.
868  EVT MemVT = LD->getMemoryVT();
869  const DataLayout &DL = DAG.getDataLayout();
870  if (!TLI.allowsMemoryAccess(*DAG.getContext(), DL, MemVT,
871  *LD->getMemOperand())) {
872  std::tie(Value, Chain) = TLI.expandUnalignedLoad(LD, DAG);
873  }
874  }
875  break;
876 
877  case TargetLowering::Expand: {
878  EVT DestVT = Node->getValueType(0);
879  if (!TLI.isLoadExtLegal(ISD::EXTLOAD, DestVT, SrcVT)) {
880  // If the source type is not legal, see if there is a legal extload to
881  // an intermediate type that we can then extend further.
882  EVT LoadVT = TLI.getRegisterType(SrcVT.getSimpleVT());
883  if (TLI.isTypeLegal(SrcVT) || // Same as SrcVT == LoadVT?
884  TLI.isLoadExtLegal(ExtType, LoadVT, SrcVT)) {
885  // If we are loading a legal type, this is a non-extload followed by a
886  // full extend.
887  ISD::LoadExtType MidExtType =
888  (LoadVT == SrcVT) ? ISD::NON_EXTLOAD : ExtType;
889 
890  SDValue Load = DAG.getExtLoad(MidExtType, dl, LoadVT, Chain, Ptr,
891  SrcVT, LD->getMemOperand());
892  unsigned ExtendOp =
893  ISD::getExtForLoadExtType(SrcVT.isFloatingPoint(), ExtType);
894  Value = DAG.getNode(ExtendOp, dl, Node->getValueType(0), Load);
895  Chain = Load.getValue(1);
896  break;
897  }
898 
899  // Handle the special case of fp16 extloads. EXTLOAD doesn't have the
900  // normal undefined upper bits behavior to allow using an in-reg extend
901  // with the illegal FP type, so load as an integer and do the
902  // from-integer conversion.
903  if (SrcVT.getScalarType() == MVT::f16) {
904  EVT ISrcVT = SrcVT.changeTypeToInteger();
905  EVT IDestVT = DestVT.changeTypeToInteger();
906  EVT ILoadVT = TLI.getRegisterType(IDestVT.getSimpleVT());
907 
908  SDValue Result = DAG.getExtLoad(ISD::ZEXTLOAD, dl, ILoadVT, Chain,
909  Ptr, ISrcVT, LD->getMemOperand());
910  Value = DAG.getNode(ISD::FP16_TO_FP, dl, DestVT, Result);
911  Chain = Result.getValue(1);
912  break;
913  }
914  }
915 
916  assert(!SrcVT.isVector() &&
917  "Vector Loads are handled in LegalizeVectorOps");
918 
919  // FIXME: This does not work for vectors on most targets. Sign-
920  // and zero-extend operations are currently folded into extending
921  // loads, whether they are legal or not, and then we end up here
922  // without any support for legalizing them.
923  assert(ExtType != ISD::EXTLOAD &&
924  "EXTLOAD should always be supported!");
925  // Turn the unsupported load into an EXTLOAD followed by an
926  // explicit zero/sign extend inreg.
927  SDValue Result = DAG.getExtLoad(ISD::EXTLOAD, dl,
928  Node->getValueType(0),
929  Chain, Ptr, SrcVT,
930  LD->getMemOperand());
931  SDValue ValRes;
932  if (ExtType == ISD::SEXTLOAD)
933  ValRes = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl,
934  Result.getValueType(),
935  Result, DAG.getValueType(SrcVT));
936  else
937  ValRes = DAG.getZeroExtendInReg(Result, dl, SrcVT);
938  Value = ValRes;
939  Chain = Result.getValue(1);
940  break;
941  }
942  }
943  }
944 
945  // Since loads produce two values, make sure to remember that we legalized
946  // both of them.
947  if (Chain.getNode() != Node) {
948  assert(Value.getNode() != Node && "Load must be completely replaced");
949  DAG.ReplaceAllUsesOfValueWith(SDValue(Node, 0), Value);
950  DAG.ReplaceAllUsesOfValueWith(SDValue(Node, 1), Chain);
951  if (UpdatedNodes) {
952  UpdatedNodes->insert(Value.getNode());
953  UpdatedNodes->insert(Chain.getNode());
954  }
955  ReplacedNode(Node);
956  }
957 }
958 
959 /// Return a legal replacement for the given operation, with all legal operands.
960 void SelectionDAGLegalize::LegalizeOp(SDNode *Node) {
961  LLVM_DEBUG(dbgs() << "\nLegalizing: "; Node->dump(&DAG));
962 
963  // Allow illegal target nodes and illegal registers.
964  if (Node->getOpcode() == ISD::TargetConstant ||
965  Node->getOpcode() == ISD::Register)
966  return;
967 
968 #ifndef NDEBUG
969  for (unsigned i = 0, e = Node->getNumValues(); i != e; ++i)
970  assert(TLI.getTypeAction(*DAG.getContext(), Node->getValueType(i)) ==
972  "Unexpected illegal type!");
973 
974  for (const SDValue &Op : Node->op_values())
975  assert((TLI.getTypeAction(*DAG.getContext(), Op.getValueType()) ==
977  Op.getOpcode() == ISD::TargetConstant ||
978  Op.getOpcode() == ISD::Register) &&
979  "Unexpected illegal type!");
980 #endif
981 
982  // Figure out the correct action; the way to query this varies by opcode
984  bool SimpleFinishLegalizing = true;
985  switch (Node->getOpcode()) {
988  case ISD::INTRINSIC_VOID:
989  case ISD::STACKSAVE:
990  Action = TLI.getOperationAction(Node->getOpcode(), MVT::Other);
991  break;
993  Action = TLI.getOperationAction(Node->getOpcode(),
994  Node->getValueType(0));
995  break;
996  case ISD::VAARG:
997  Action = TLI.getOperationAction(Node->getOpcode(),
998  Node->getValueType(0));
999  if (Action != TargetLowering::Promote)
1000  Action = TLI.getOperationAction(Node->getOpcode(), MVT::Other);
1001  break;
1002  case ISD::FP_TO_FP16:
1003  case ISD::SINT_TO_FP:
1004  case ISD::UINT_TO_FP:
1006  case ISD::LROUND:
1007  case ISD::LLROUND:
1008  case ISD::LRINT:
1009  case ISD::LLRINT:
1010  Action = TLI.getOperationAction(Node->getOpcode(),
1011  Node->getOperand(0).getValueType());
1012  break;
1016  case ISD::STRICT_LRINT:
1017  case ISD::STRICT_LLRINT:
1018  case ISD::STRICT_LROUND:
1019  case ISD::STRICT_LLROUND:
1020  // These pseudo-ops are the same as the other STRICT_ ops except
1021  // they are registered with setOperationAction() using the input type
1022  // instead of the output type.
1023  Action = TLI.getOperationAction(Node->getOpcode(),
1024  Node->getOperand(1).getValueType());
1025  break;
1026  case ISD::SIGN_EXTEND_INREG: {
1027  EVT InnerType = cast<VTSDNode>(Node->getOperand(1))->getVT();
1028  Action = TLI.getOperationAction(Node->getOpcode(), InnerType);
1029  break;
1030  }
1031  case ISD::ATOMIC_STORE:
1032  Action = TLI.getOperationAction(Node->getOpcode(),
1033  Node->getOperand(2).getValueType());
1034  break;
1035  case ISD::SELECT_CC:
1036  case ISD::STRICT_FSETCC:
1037  case ISD::STRICT_FSETCCS:
1038  case ISD::SETCC:
1039  case ISD::BR_CC: {
1040  unsigned CCOperand = Node->getOpcode() == ISD::SELECT_CC ? 4 :
1041  Node->getOpcode() == ISD::STRICT_FSETCC ? 3 :
1042  Node->getOpcode() == ISD::STRICT_FSETCCS ? 3 :
1043  Node->getOpcode() == ISD::SETCC ? 2 : 1;
1044  unsigned CompareOperand = Node->getOpcode() == ISD::BR_CC ? 2 :
1045  Node->getOpcode() == ISD::STRICT_FSETCC ? 1 :
1046  Node->getOpcode() == ISD::STRICT_FSETCCS ? 1 : 0;
1047  MVT OpVT = Node->getOperand(CompareOperand).getSimpleValueType();
1048  ISD::CondCode CCCode =
1049  cast<CondCodeSDNode>(Node->getOperand(CCOperand))->get();
1050  Action = TLI.getCondCodeAction(CCCode, OpVT);
1051  if (Action == TargetLowering::Legal) {
1052  if (Node->getOpcode() == ISD::SELECT_CC)
1053  Action = TLI.getOperationAction(Node->getOpcode(),
1054  Node->getValueType(0));
1055  else
1056  Action = TLI.getOperationAction(Node->getOpcode(), OpVT);
1057  }
1058  break;
1059  }
1060  case ISD::LOAD:
1061  case ISD::STORE:
1062  // FIXME: Model these properly. LOAD and STORE are complicated, and
1063  // STORE expects the unlegalized operand in some cases.
1064  SimpleFinishLegalizing = false;
1065  break;
1066  case ISD::CALLSEQ_START:
1067  case ISD::CALLSEQ_END:
1068  // FIXME: This shouldn't be necessary. These nodes have special properties
1069  // dealing with the recursive nature of legalization. Removing this
1070  // special case should be done as part of making LegalizeDAG non-recursive.
1071  SimpleFinishLegalizing = false;
1072  break;
1073  case ISD::EXTRACT_ELEMENT:
1074  case ISD::FLT_ROUNDS_:
1075  case ISD::MERGE_VALUES:
1076  case ISD::EH_RETURN:
1078  case ISD::EH_DWARF_CFA:
1079  case ISD::EH_SJLJ_SETJMP:
1080  case ISD::EH_SJLJ_LONGJMP:
1082  // These operations lie about being legal: when they claim to be legal,
1083  // they should actually be expanded.
1084  Action = TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0));
1085  if (Action == TargetLowering::Legal)
1086  Action = TargetLowering::Expand;
1087  break;
1088  case ISD::INIT_TRAMPOLINE:
1090  case ISD::FRAMEADDR:
1091  case ISD::RETURNADDR:
1092  case ISD::ADDROFRETURNADDR:
1093  case ISD::SPONENTRY:
1094  // These operations lie about being legal: when they claim to be legal,
1095  // they should actually be custom-lowered.
1096  Action = TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0));
1097  if (Action == TargetLowering::Legal)
1098  Action = TargetLowering::Custom;
1099  break;
1100  case ISD::READCYCLECOUNTER:
1101  // READCYCLECOUNTER returns an i64, even if type legalization might have
1102  // expanded that to several smaller types.
1103  Action = TLI.getOperationAction(Node->getOpcode(), MVT::i64);
1104  break;
1105  case ISD::READ_REGISTER:
1106  case ISD::WRITE_REGISTER:
1107  // Named register is legal in the DAG, but blocked by register name
1108  // selection if not implemented by target (to chose the correct register)
1109  // They'll be converted to Copy(To/From)Reg.
1110  Action = TargetLowering::Legal;
1111  break;
1112  case ISD::UBSANTRAP:
1113  Action = TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0));
1114  if (Action == TargetLowering::Expand) {
1115  // replace ISD::UBSANTRAP with ISD::TRAP
1116  SDValue NewVal;
1117  NewVal = DAG.getNode(ISD::TRAP, SDLoc(Node), Node->getVTList(),
1118  Node->getOperand(0));
1119  ReplaceNode(Node, NewVal.getNode());
1120  LegalizeOp(NewVal.getNode());
1121  return;
1122  }
1123  break;
1124  case ISD::DEBUGTRAP:
1125  Action = TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0));
1126  if (Action == TargetLowering::Expand) {
1127  // replace ISD::DEBUGTRAP with ISD::TRAP
1128  SDValue NewVal;
1129  NewVal = DAG.getNode(ISD::TRAP, SDLoc(Node), Node->getVTList(),
1130  Node->getOperand(0));
1131  ReplaceNode(Node, NewVal.getNode());
1132  LegalizeOp(NewVal.getNode());
1133  return;
1134  }
1135  break;
1136  case ISD::SADDSAT:
1137  case ISD::UADDSAT:
1138  case ISD::SSUBSAT:
1139  case ISD::USUBSAT:
1140  case ISD::SSHLSAT:
1141  case ISD::USHLSAT:
1142  case ISD::FP_TO_SINT_SAT:
1143  case ISD::FP_TO_UINT_SAT:
1144  Action = TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0));
1145  break;
1146  case ISD::SMULFIX:
1147  case ISD::SMULFIXSAT:
1148  case ISD::UMULFIX:
1149  case ISD::UMULFIXSAT:
1150  case ISD::SDIVFIX:
1151  case ISD::SDIVFIXSAT:
1152  case ISD::UDIVFIX:
1153  case ISD::UDIVFIXSAT: {
1154  unsigned Scale = Node->getConstantOperandVal(2);
1155  Action = TLI.getFixedPointOperationAction(Node->getOpcode(),
1156  Node->getValueType(0), Scale);
1157  break;
1158  }
1159  case ISD::MSCATTER:
1160  Action = TLI.getOperationAction(Node->getOpcode(),
1161  cast<MaskedScatterSDNode>(Node)->getValue().getValueType());
1162  break;
1163  case ISD::MSTORE:
1164  Action = TLI.getOperationAction(Node->getOpcode(),
1165  cast<MaskedStoreSDNode>(Node)->getValue().getValueType());
1166  break;
1167  case ISD::VP_SCATTER:
1168  Action = TLI.getOperationAction(
1169  Node->getOpcode(),
1170  cast<VPScatterSDNode>(Node)->getValue().getValueType());
1171  break;
1172  case ISD::VP_STORE:
1173  Action = TLI.getOperationAction(
1174  Node->getOpcode(),
1175  cast<VPStoreSDNode>(Node)->getValue().getValueType());
1176  break;
1177  case ISD::VECREDUCE_FADD:
1178  case ISD::VECREDUCE_FMUL:
1179  case ISD::VECREDUCE_ADD:
1180  case ISD::VECREDUCE_MUL:
1181  case ISD::VECREDUCE_AND:
1182  case ISD::VECREDUCE_OR:
1183  case ISD::VECREDUCE_XOR:
1184  case ISD::VECREDUCE_SMAX:
1185  case ISD::VECREDUCE_SMIN:
1186  case ISD::VECREDUCE_UMAX:
1187  case ISD::VECREDUCE_UMIN:
1188  case ISD::VECREDUCE_FMAX:
1189  case ISD::VECREDUCE_FMIN:
1190  Action = TLI.getOperationAction(
1191  Node->getOpcode(), Node->getOperand(0).getValueType());
1192  break;
1195  case ISD::VP_REDUCE_FADD:
1196  case ISD::VP_REDUCE_FMUL:
1197  case ISD::VP_REDUCE_ADD:
1198  case ISD::VP_REDUCE_MUL:
1199  case ISD::VP_REDUCE_AND:
1200  case ISD::VP_REDUCE_OR:
1201  case ISD::VP_REDUCE_XOR:
1202  case ISD::VP_REDUCE_SMAX:
1203  case ISD::VP_REDUCE_SMIN:
1204  case ISD::VP_REDUCE_UMAX:
1205  case ISD::VP_REDUCE_UMIN:
1206  case ISD::VP_REDUCE_FMAX:
1207  case ISD::VP_REDUCE_FMIN:
1208  case ISD::VP_REDUCE_SEQ_FADD:
1209  case ISD::VP_REDUCE_SEQ_FMUL:
1210  Action = TLI.getOperationAction(
1211  Node->getOpcode(), Node->getOperand(1).getValueType());
1212  break;
1213  default:
1214  if (Node->getOpcode() >= ISD::BUILTIN_OP_END) {
1215  Action = TargetLowering::Legal;
1216  } else {
1217  Action = TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0));
1218  }
1219  break;
1220  }
1221 
1222  if (SimpleFinishLegalizing) {
1223  SDNode *NewNode = Node;
1224  switch (Node->getOpcode()) {
1225  default: break;
1226  case ISD::SHL:
1227  case ISD::SRL:
1228  case ISD::SRA:
1229  case ISD::ROTL:
1230  case ISD::ROTR: {
1231  // Legalizing shifts/rotates requires adjusting the shift amount
1232  // to the appropriate width.
1233  SDValue Op0 = Node->getOperand(0);
1234  SDValue Op1 = Node->getOperand(1);
1235  if (!Op1.getValueType().isVector()) {
1236  SDValue SAO = DAG.getShiftAmountOperand(Op0.getValueType(), Op1);
1237  // The getShiftAmountOperand() may create a new operand node or
1238  // return the existing one. If new operand is created we need
1239  // to update the parent node.
1240  // Do not try to legalize SAO here! It will be automatically legalized
1241  // in the next round.
1242  if (SAO != Op1)
1243  NewNode = DAG.UpdateNodeOperands(Node, Op0, SAO);
1244  }
1245  }
1246  break;
1247  case ISD::FSHL:
1248  case ISD::FSHR:
1249  case ISD::SRL_PARTS:
1250  case ISD::SRA_PARTS:
1251  case ISD::SHL_PARTS: {
1252  // Legalizing shifts/rotates requires adjusting the shift amount
1253  // to the appropriate width.
1254  SDValue Op0 = Node->getOperand(0);
1255  SDValue Op1 = Node->getOperand(1);
1256  SDValue Op2 = Node->getOperand(2);
1257  if (!Op2.getValueType().isVector()) {
1258  SDValue SAO = DAG.getShiftAmountOperand(Op0.getValueType(), Op2);
1259  // The getShiftAmountOperand() may create a new operand node or
1260  // return the existing one. If new operand is created we need
1261  // to update the parent node.
1262  if (SAO != Op2)
1263  NewNode = DAG.UpdateNodeOperands(Node, Op0, Op1, SAO);
1264  }
1265  break;
1266  }
1267  }
1268 
1269  if (NewNode != Node) {
1270  ReplaceNode(Node, NewNode);
1271  Node = NewNode;
1272  }
1273  switch (Action) {
1274  case TargetLowering::Legal:
1275  LLVM_DEBUG(dbgs() << "Legal node: nothing to do\n");
1276  return;
1278  LLVM_DEBUG(dbgs() << "Trying custom legalization\n");
1279  // FIXME: The handling for custom lowering with multiple results is
1280  // a complete mess.
1281  if (SDValue Res = TLI.LowerOperation(SDValue(Node, 0), DAG)) {
1282  if (!(Res.getNode() != Node || Res.getResNo() != 0))
1283  return;
1284 
1285  if (Node->getNumValues() == 1) {
1286  // Verify the new types match the original. Glue is waived because
1287  // ISD::ADDC can be legalized by replacing Glue with an integer type.
1288  assert((Res.getValueType() == Node->getValueType(0) ||
1289  Node->getValueType(0) == MVT::Glue) &&
1290  "Type mismatch for custom legalized operation");
1291  LLVM_DEBUG(dbgs() << "Successfully custom legalized node\n");
1292  // We can just directly replace this node with the lowered value.
1293  ReplaceNode(SDValue(Node, 0), Res);
1294  return;
1295  }
1296 
1297  SmallVector<SDValue, 8> ResultVals;
1298  for (unsigned i = 0, e = Node->getNumValues(); i != e; ++i) {
1299  // Verify the new types match the original. Glue is waived because
1300  // ISD::ADDC can be legalized by replacing Glue with an integer type.
1301  assert((Res->getValueType(i) == Node->getValueType(i) ||
1302  Node->getValueType(i) == MVT::Glue) &&
1303  "Type mismatch for custom legalized operation");
1304  ResultVals.push_back(Res.getValue(i));
1305  }
1306  LLVM_DEBUG(dbgs() << "Successfully custom legalized node\n");
1307  ReplaceNode(Node, ResultVals.data());
1308  return;
1309  }
1310  LLVM_DEBUG(dbgs() << "Could not custom legalize node\n");
1313  if (ExpandNode(Node))
1314  return;
1317  ConvertNodeToLibcall(Node);
1318  return;
1320  PromoteNode(Node);
1321  return;
1322  }
1323  }
1324 
1325  switch (Node->getOpcode()) {
1326  default:
1327 #ifndef NDEBUG
1328  dbgs() << "NODE: ";
1329  Node->dump( &DAG);
1330  dbgs() << "\n";
1331 #endif
1332  llvm_unreachable("Do not know how to legalize this operator!");
1333 
1334  case ISD::CALLSEQ_START:
1335  case ISD::CALLSEQ_END:
1336  break;
1337  case ISD::LOAD:
1338  return LegalizeLoadOps(Node);
1339  case ISD::STORE:
1340  return LegalizeStoreOps(Node);
1341  }
1342 }
1343 
1344 SDValue SelectionDAGLegalize::ExpandExtractFromVectorThroughStack(SDValue Op) {
1345  SDValue Vec = Op.getOperand(0);
1346  SDValue Idx = Op.getOperand(1);
1347  SDLoc dl(Op);
1348 
1349  // Before we generate a new store to a temporary stack slot, see if there is
1350  // already one that we can use. There often is because when we scalarize
1351  // vector operations (using SelectionDAG::UnrollVectorOp for example) a whole
1352  // series of EXTRACT_VECTOR_ELT nodes are generated, one for each element in
1353  // the vector. If all are expanded here, we don't want one store per vector
1354  // element.
1355 
1356  // Caches for hasPredecessorHelper
1359  Visited.insert(Op.getNode());
1360  Worklist.push_back(Idx.getNode());
1361  SDValue StackPtr, Ch;
1362  for (SDNode::use_iterator UI = Vec.getNode()->use_begin(),
1363  UE = Vec.getNode()->use_end(); UI != UE; ++UI) {
1364  SDNode *User = *UI;
1365  if (StoreSDNode *ST = dyn_cast<StoreSDNode>(User)) {
1366  if (ST->isIndexed() || ST->isTruncatingStore() ||
1367  ST->getValue() != Vec)
1368  continue;
1369 
1370  // Make sure that nothing else could have stored into the destination of
1371  // this store.
1372  if (!ST->getChain().reachesChainWithoutSideEffects(DAG.getEntryNode()))
1373  continue;
1374 
1375  // If the index is dependent on the store we will introduce a cycle when
1376  // creating the load (the load uses the index, and by replacing the chain
1377  // we will make the index dependent on the load). Also, the store might be
1378  // dependent on the extractelement and introduce a cycle when creating
1379  // the load.
1380  if (SDNode::hasPredecessorHelper(ST, Visited, Worklist) ||
1381  ST->hasPredecessor(Op.getNode()))
1382  continue;
1383 
1384  StackPtr = ST->getBasePtr();
1385  Ch = SDValue(ST, 0);
1386  break;
1387  }
1388  }
1389 
1390  EVT VecVT = Vec.getValueType();
1391 
1392  if (!Ch.getNode()) {
1393  // Store the value to a temporary stack slot, then LOAD the returned part.
1394  StackPtr = DAG.CreateStackTemporary(VecVT);
1395  Ch = DAG.getStore(DAG.getEntryNode(), dl, Vec, StackPtr,
1396  MachinePointerInfo());
1397  }
1398 
1399  SDValue NewLoad;
1400 
1401  if (Op.getValueType().isVector()) {
1402  StackPtr = TLI.getVectorSubVecPointer(DAG, StackPtr, VecVT,
1403  Op.getValueType(), Idx);
1404  NewLoad =
1405  DAG.getLoad(Op.getValueType(), dl, Ch, StackPtr, MachinePointerInfo());
1406  } else {
1407  StackPtr = TLI.getVectorElementPointer(DAG, StackPtr, VecVT, Idx);
1408  NewLoad = DAG.getExtLoad(ISD::EXTLOAD, dl, Op.getValueType(), Ch, StackPtr,
1410  VecVT.getVectorElementType());
1411  }
1412 
1413  // Replace the chain going out of the store, by the one out of the load.
1414  DAG.ReplaceAllUsesOfValueWith(Ch, SDValue(NewLoad.getNode(), 1));
1415 
1416  // We introduced a cycle though, so update the loads operands, making sure
1417  // to use the original store's chain as an incoming chain.
1418  SmallVector<SDValue, 6> NewLoadOperands(NewLoad->op_begin(),
1419  NewLoad->op_end());
1420  NewLoadOperands[0] = Ch;
1421  NewLoad =
1422  SDValue(DAG.UpdateNodeOperands(NewLoad.getNode(), NewLoadOperands), 0);
1423  return NewLoad;
1424 }
1425 
1426 SDValue SelectionDAGLegalize::ExpandInsertToVectorThroughStack(SDValue Op) {
1427  assert(Op.getValueType().isVector() && "Non-vector insert subvector!");
1428 
1429  SDValue Vec = Op.getOperand(0);
1430  SDValue Part = Op.getOperand(1);
1431  SDValue Idx = Op.getOperand(2);
1432  SDLoc dl(Op);
1433 
1434  // Store the value to a temporary stack slot, then LOAD the returned part.
1435  EVT VecVT = Vec.getValueType();
1436  EVT SubVecVT = Part.getValueType();
1437  SDValue StackPtr = DAG.CreateStackTemporary(VecVT);
1438  int FI = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex();
1439  MachinePointerInfo PtrInfo =
1440  MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), FI);
1441 
1442  // First store the whole vector.
1443  SDValue Ch = DAG.getStore(DAG.getEntryNode(), dl, Vec, StackPtr, PtrInfo);
1444 
1445  // Then store the inserted part.
1446  SDValue SubStackPtr =
1447  TLI.getVectorSubVecPointer(DAG, StackPtr, VecVT, SubVecVT, Idx);
1448 
1449  // Store the subvector.
1450  Ch = DAG.getStore(
1451  Ch, dl, Part, SubStackPtr,
1452  MachinePointerInfo::getUnknownStack(DAG.getMachineFunction()));
1453 
1454  // Finally, load the updated vector.
1455  return DAG.getLoad(Op.getValueType(), dl, Ch, StackPtr, PtrInfo);
1456 }
1457 
1458 SDValue SelectionDAGLegalize::ExpandVectorBuildThroughStack(SDNode* Node) {
1459  assert((Node->getOpcode() == ISD::BUILD_VECTOR ||
1460  Node->getOpcode() == ISD::CONCAT_VECTORS) &&
1461  "Unexpected opcode!");
1462 
1463  // We can't handle this case efficiently. Allocate a sufficiently
1464  // aligned object on the stack, store each operand into it, then load
1465  // the result as a vector.
1466  // Create the stack frame object.
1467  EVT VT = Node->getValueType(0);
1468  EVT MemVT = isa<BuildVectorSDNode>(Node) ? VT.getVectorElementType()
1469  : Node->getOperand(0).getValueType();
1470  SDLoc dl(Node);
1471  SDValue FIPtr = DAG.CreateStackTemporary(VT);
1472  int FI = cast<FrameIndexSDNode>(FIPtr.getNode())->getIndex();
1473  MachinePointerInfo PtrInfo =
1474  MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), FI);
1475 
1476  // Emit a store of each element to the stack slot.
1477  SmallVector<SDValue, 8> Stores;
1478  unsigned TypeByteSize = MemVT.getSizeInBits() / 8;
1479  assert(TypeByteSize > 0 && "Vector element type too small for stack store!");
1480 
1481  // If the destination vector element type of a BUILD_VECTOR is narrower than
1482  // the source element type, only store the bits necessary.
1483  bool Truncate = isa<BuildVectorSDNode>(Node) &&
1484  MemVT.bitsLT(Node->getOperand(0).getValueType());
1485 
1486  // Store (in the right endianness) the elements to memory.
1487  for (unsigned i = 0, e = Node->getNumOperands(); i != e; ++i) {
1488  // Ignore undef elements.
1489  if (Node->getOperand(i).isUndef()) continue;
1490 
1491  unsigned Offset = TypeByteSize*i;
1492 
1493  SDValue Idx = DAG.getMemBasePlusOffset(FIPtr, TypeSize::Fixed(Offset), dl);
1494 
1495  if (Truncate)
1496  Stores.push_back(DAG.getTruncStore(DAG.getEntryNode(), dl,
1497  Node->getOperand(i), Idx,
1498  PtrInfo.getWithOffset(Offset), MemVT));
1499  else
1500  Stores.push_back(DAG.getStore(DAG.getEntryNode(), dl, Node->getOperand(i),
1501  Idx, PtrInfo.getWithOffset(Offset)));
1502  }
1503 
1504  SDValue StoreChain;
1505  if (!Stores.empty()) // Not all undef elements?
1506  StoreChain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Stores);
1507  else
1508  StoreChain = DAG.getEntryNode();
1509 
1510  // Result is a load from the stack slot.
1511  return DAG.getLoad(VT, dl, StoreChain, FIPtr, PtrInfo);
1512 }
1513 
1514 /// Bitcast a floating-point value to an integer value. Only bitcast the part
1515 /// containing the sign bit if the target has no integer value capable of
1516 /// holding all bits of the floating-point value.
1517 void SelectionDAGLegalize::getSignAsIntValue(FloatSignAsInt &State,
1518  const SDLoc &DL,
1519  SDValue Value) const {
1520  EVT FloatVT = Value.getValueType();
1521  unsigned NumBits = FloatVT.getScalarSizeInBits();
1522  State.FloatVT = FloatVT;
1523  EVT IVT = EVT::getIntegerVT(*DAG.getContext(), NumBits);
1524  // Convert to an integer of the same size.
1525  if (TLI.isTypeLegal(IVT)) {
1526  State.IntValue = DAG.getNode(ISD::BITCAST, DL, IVT, Value);
1527  State.SignMask = APInt::getSignMask(NumBits);
1528  State.SignBit = NumBits - 1;
1529  return;
1530  }
1531 
1532  auto &DataLayout = DAG.getDataLayout();
1533  // Store the float to memory, then load the sign part out as an integer.
1534  MVT LoadTy = TLI.getRegisterType(*DAG.getContext(), MVT::i8);
1535  // First create a temporary that is aligned for both the load and store.
1536  SDValue StackPtr = DAG.CreateStackTemporary(FloatVT, LoadTy);
1537  int FI = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex();
1538  // Then store the float to it.
1539  State.FloatPtr = StackPtr;
1540  MachineFunction &MF = DAG.getMachineFunction();
1541  State.FloatPointerInfo = MachinePointerInfo::getFixedStack(MF, FI);
1542  State.Chain = DAG.getStore(DAG.getEntryNode(), DL, Value, State.FloatPtr,
1543  State.FloatPointerInfo);
1544 
1545  SDValue IntPtr;
1546  if (DataLayout.isBigEndian()) {
1547  assert(FloatVT.isByteSized() && "Unsupported floating point type!");
1548  // Load out a legal integer with the same sign bit as the float.
1549  IntPtr = StackPtr;
1550  State.IntPointerInfo = State.FloatPointerInfo;
1551  } else {
1552  // Advance the pointer so that the loaded byte will contain the sign bit.
1553  unsigned ByteOffset = (NumBits / 8) - 1;
1554  IntPtr =
1555  DAG.getMemBasePlusOffset(StackPtr, TypeSize::Fixed(ByteOffset), DL);
1556  State.IntPointerInfo = MachinePointerInfo::getFixedStack(MF, FI,
1557  ByteOffset);
1558  }
1559 
1560  State.IntPtr = IntPtr;
1561  State.IntValue = DAG.getExtLoad(ISD::EXTLOAD, DL, LoadTy, State.Chain, IntPtr,
1562  State.IntPointerInfo, MVT::i8);
1563  State.SignMask = APInt::getOneBitSet(LoadTy.getScalarSizeInBits(), 7);
1564  State.SignBit = 7;
1565 }
1566 
1567 /// Replace the integer value produced by getSignAsIntValue() with a new value
1568 /// and cast the result back to a floating-point type.
1569 SDValue SelectionDAGLegalize::modifySignAsInt(const FloatSignAsInt &State,
1570  const SDLoc &DL,
1571  SDValue NewIntValue) const {
1572  if (!State.Chain)
1573  return DAG.getNode(ISD::BITCAST, DL, State.FloatVT, NewIntValue);
1574 
1575  // Override the part containing the sign bit in the value stored on the stack.
1576  SDValue Chain = DAG.getTruncStore(State.Chain, DL, NewIntValue, State.IntPtr,
1577  State.IntPointerInfo, MVT::i8);
1578  return DAG.getLoad(State.FloatVT, DL, Chain, State.FloatPtr,
1579  State.FloatPointerInfo);
1580 }
1581 
1582 SDValue SelectionDAGLegalize::ExpandFCOPYSIGN(SDNode *Node) const {
1583  SDLoc DL(Node);
1584  SDValue Mag = Node->getOperand(0);
1585  SDValue Sign = Node->getOperand(1);
1586 
1587  // Get sign bit into an integer value.
1588  FloatSignAsInt SignAsInt;
1589  getSignAsIntValue(SignAsInt, DL, Sign);
1590 
1591  EVT IntVT = SignAsInt.IntValue.getValueType();
1592  SDValue SignMask = DAG.getConstant(SignAsInt.SignMask, DL, IntVT);
1593  SDValue SignBit = DAG.getNode(ISD::AND, DL, IntVT, SignAsInt.IntValue,
1594  SignMask);
1595 
1596  // If FABS is legal transform FCOPYSIGN(x, y) => sign(x) ? -FABS(x) : FABS(X)
1597  EVT FloatVT = Mag.getValueType();
1598  if (TLI.isOperationLegalOrCustom(ISD::FABS, FloatVT) &&
1599  TLI.isOperationLegalOrCustom(ISD::FNEG, FloatVT)) {
1600  SDValue AbsValue = DAG.getNode(ISD::FABS, DL, FloatVT, Mag);
1601  SDValue NegValue = DAG.getNode(ISD::FNEG, DL, FloatVT, AbsValue);
1602  SDValue Cond = DAG.getSetCC(DL, getSetCCResultType(IntVT), SignBit,
1603  DAG.getConstant(0, DL, IntVT), ISD::SETNE);
1604  return DAG.getSelect(DL, FloatVT, Cond, NegValue, AbsValue);
1605  }
1606 
1607  // Transform Mag value to integer, and clear the sign bit.
1608  FloatSignAsInt MagAsInt;
1609  getSignAsIntValue(MagAsInt, DL, Mag);
1610  EVT MagVT = MagAsInt.IntValue.getValueType();
1611  SDValue ClearSignMask = DAG.getConstant(~MagAsInt.SignMask, DL, MagVT);
1612  SDValue ClearedSign = DAG.getNode(ISD::AND, DL, MagVT, MagAsInt.IntValue,
1613  ClearSignMask);
1614 
1615  // Get the signbit at the right position for MagAsInt.
1616  int ShiftAmount = SignAsInt.SignBit - MagAsInt.SignBit;
1617  EVT ShiftVT = IntVT;
1618  if (SignBit.getScalarValueSizeInBits() <
1619  ClearedSign.getScalarValueSizeInBits()) {
1620  SignBit = DAG.getNode(ISD::ZERO_EXTEND, DL, MagVT, SignBit);
1621  ShiftVT = MagVT;
1622  }
1623  if (ShiftAmount > 0) {
1624  SDValue ShiftCnst = DAG.getConstant(ShiftAmount, DL, ShiftVT);
1625  SignBit = DAG.getNode(ISD::SRL, DL, ShiftVT, SignBit, ShiftCnst);
1626  } else if (ShiftAmount < 0) {
1627  SDValue ShiftCnst = DAG.getConstant(-ShiftAmount, DL, ShiftVT);
1628  SignBit = DAG.getNode(ISD::SHL, DL, ShiftVT, SignBit, ShiftCnst);
1629  }
1630  if (SignBit.getScalarValueSizeInBits() >
1631  ClearedSign.getScalarValueSizeInBits()) {
1632  SignBit = DAG.getNode(ISD::TRUNCATE, DL, MagVT, SignBit);
1633  }
1634 
1635  // Store the part with the modified sign and convert back to float.
1636  SDValue CopiedSign = DAG.getNode(ISD::OR, DL, MagVT, ClearedSign, SignBit);
1637  return modifySignAsInt(MagAsInt, DL, CopiedSign);
1638 }
1639 
1640 SDValue SelectionDAGLegalize::ExpandFNEG(SDNode *Node) const {
1641  // Get the sign bit as an integer.
1642  SDLoc DL(Node);
1643  FloatSignAsInt SignAsInt;
1644  getSignAsIntValue(SignAsInt, DL, Node->getOperand(0));
1645  EVT IntVT = SignAsInt.IntValue.getValueType();
1646 
1647  // Flip the sign.
1648  SDValue SignMask = DAG.getConstant(SignAsInt.SignMask, DL, IntVT);
1649  SDValue SignFlip =
1650  DAG.getNode(ISD::XOR, DL, IntVT, SignAsInt.IntValue, SignMask);
1651 
1652  // Convert back to float.
1653  return modifySignAsInt(SignAsInt, DL, SignFlip);
1654 }
1655 
1656 SDValue SelectionDAGLegalize::ExpandFABS(SDNode *Node) const {
1657  SDLoc DL(Node);
1658  SDValue Value = Node->getOperand(0);
1659 
1660  // Transform FABS(x) => FCOPYSIGN(x, 0.0) if FCOPYSIGN is legal.
1661  EVT FloatVT = Value.getValueType();
1662  if (TLI.isOperationLegalOrCustom(ISD::FCOPYSIGN, FloatVT)) {
1663  SDValue Zero = DAG.getConstantFP(0.0, DL, FloatVT);
1664  return DAG.getNode(ISD::FCOPYSIGN, DL, FloatVT, Value, Zero);
1665  }
1666 
1667  // Transform value to integer, clear the sign bit and transform back.
1668  FloatSignAsInt ValueAsInt;
1669  getSignAsIntValue(ValueAsInt, DL, Value);
1670  EVT IntVT = ValueAsInt.IntValue.getValueType();
1671  SDValue ClearSignMask = DAG.getConstant(~ValueAsInt.SignMask, DL, IntVT);
1672  SDValue ClearedSign = DAG.getNode(ISD::AND, DL, IntVT, ValueAsInt.IntValue,
1673  ClearSignMask);
1674  return modifySignAsInt(ValueAsInt, DL, ClearedSign);
1675 }
1676 
1677 void SelectionDAGLegalize::ExpandDYNAMIC_STACKALLOC(SDNode* Node,
1679  Register SPReg = TLI.getStackPointerRegisterToSaveRestore();
1680  assert(SPReg && "Target cannot require DYNAMIC_STACKALLOC expansion and"
1681  " not tell us which reg is the stack pointer!");
1682  SDLoc dl(Node);
1683  EVT VT = Node->getValueType(0);
1684  SDValue Tmp1 = SDValue(Node, 0);
1685  SDValue Tmp2 = SDValue(Node, 1);
1686  SDValue Tmp3 = Node->getOperand(2);
1687  SDValue Chain = Tmp1.getOperand(0);
1688 
1689  // Chain the dynamic stack allocation so that it doesn't modify the stack
1690  // pointer when other instructions are using the stack.
1691  Chain = DAG.getCALLSEQ_START(Chain, 0, 0, dl);
1692 
1693  SDValue Size = Tmp2.getOperand(1);
1694  SDValue SP = DAG.getCopyFromReg(Chain, dl, SPReg, VT);
1695  Chain = SP.getValue(1);
1696  Align Alignment = cast<ConstantSDNode>(Tmp3)->getAlignValue();
1697  const TargetFrameLowering *TFL = DAG.getSubtarget().getFrameLowering();
1698  unsigned Opc =
1700  ISD::ADD : ISD::SUB;
1701 
1702  Align StackAlign = TFL->getStackAlign();
1703  Tmp1 = DAG.getNode(Opc, dl, VT, SP, Size); // Value
1704  if (Alignment > StackAlign)
1705  Tmp1 = DAG.getNode(ISD::AND, dl, VT, Tmp1,
1706  DAG.getConstant(-Alignment.value(), dl, VT));
1707  Chain = DAG.getCopyToReg(Chain, dl, SPReg, Tmp1); // Output chain
1708 
1709  Tmp2 = DAG.getCALLSEQ_END(Chain, DAG.getIntPtrConstant(0, dl, true),
1710  DAG.getIntPtrConstant(0, dl, true), SDValue(), dl);
1711 
1712  Results.push_back(Tmp1);
1713  Results.push_back(Tmp2);
1714 }
1715 
1716 /// Emit a store/load combination to the stack. This stores
1717 /// SrcOp to a stack slot of type SlotVT, truncating it if needed. It then does
1718 /// a load from the stack slot to DestVT, extending it if needed.
1719 /// The resultant code need not be legal.
1720 SDValue SelectionDAGLegalize::EmitStackConvert(SDValue SrcOp, EVT SlotVT,
1721  EVT DestVT, const SDLoc &dl) {
1722  return EmitStackConvert(SrcOp, SlotVT, DestVT, dl, DAG.getEntryNode());
1723 }
1724 
1725 SDValue SelectionDAGLegalize::EmitStackConvert(SDValue SrcOp, EVT SlotVT,
1726  EVT DestVT, const SDLoc &dl,
1727  SDValue Chain) {
1728  unsigned SrcSize = SrcOp.getValueSizeInBits();
1729  unsigned SlotSize = SlotVT.getSizeInBits();
1730  unsigned DestSize = DestVT.getSizeInBits();
1731  Type *DestType = DestVT.getTypeForEVT(*DAG.getContext());
1732  Align DestAlign = DAG.getDataLayout().getPrefTypeAlign(DestType);
1733 
1734  // Don't convert with stack if the load/store is expensive.
1735  if ((SrcSize > SlotSize &&
1736  !TLI.isTruncStoreLegalOrCustom(SrcOp.getValueType(), SlotVT)) ||
1737  (SlotSize < DestSize &&
1738  !TLI.isLoadExtLegalOrCustom(ISD::EXTLOAD, DestVT, SlotVT)))
1739  return SDValue();
1740 
1741  // Create the stack frame object.
1742  Align SrcAlign = DAG.getDataLayout().getPrefTypeAlign(
1743  SrcOp.getValueType().getTypeForEVT(*DAG.getContext()));
1744  SDValue FIPtr = DAG.CreateStackTemporary(SlotVT.getStoreSize(), SrcAlign);
1745 
1746  FrameIndexSDNode *StackPtrFI = cast<FrameIndexSDNode>(FIPtr);
1747  int SPFI = StackPtrFI->getIndex();
1748  MachinePointerInfo PtrInfo =
1749  MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), SPFI);
1750 
1751  // Emit a store to the stack slot. Use a truncstore if the input value is
1752  // later than DestVT.
1753  SDValue Store;
1754 
1755  if (SrcSize > SlotSize)
1756  Store = DAG.getTruncStore(Chain, dl, SrcOp, FIPtr, PtrInfo,
1757  SlotVT, SrcAlign);
1758  else {
1759  assert(SrcSize == SlotSize && "Invalid store");
1760  Store =
1761  DAG.getStore(Chain, dl, SrcOp, FIPtr, PtrInfo, SrcAlign);
1762  }
1763 
1764  // Result is a load from the stack slot.
1765  if (SlotSize == DestSize)
1766  return DAG.getLoad(DestVT, dl, Store, FIPtr, PtrInfo, DestAlign);
1767 
1768  assert(SlotSize < DestSize && "Unknown extension!");
1769  return DAG.getExtLoad(ISD::EXTLOAD, dl, DestVT, Store, FIPtr, PtrInfo, SlotVT,
1770  DestAlign);
1771 }
1772 
1773 SDValue SelectionDAGLegalize::ExpandSCALAR_TO_VECTOR(SDNode *Node) {
1774  SDLoc dl(Node);
1775  // Create a vector sized/aligned stack slot, store the value to element #0,
1776  // then load the whole vector back out.
1777  SDValue StackPtr = DAG.CreateStackTemporary(Node->getValueType(0));
1778 
1779  FrameIndexSDNode *StackPtrFI = cast<FrameIndexSDNode>(StackPtr);
1780  int SPFI = StackPtrFI->getIndex();
1781 
1782  SDValue Ch = DAG.getTruncStore(
1783  DAG.getEntryNode(), dl, Node->getOperand(0), StackPtr,
1784  MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), SPFI),
1785  Node->getValueType(0).getVectorElementType());
1786  return DAG.getLoad(
1787  Node->getValueType(0), dl, Ch, StackPtr,
1788  MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), SPFI));
1789 }
1790 
1791 static bool
1793  const TargetLowering &TLI, SDValue &Res) {
1794  unsigned NumElems = Node->getNumOperands();
1795  SDLoc dl(Node);
1796  EVT VT = Node->getValueType(0);
1797 
1798  // Try to group the scalars into pairs, shuffle the pairs together, then
1799  // shuffle the pairs of pairs together, etc. until the vector has
1800  // been built. This will work only if all of the necessary shuffle masks
1801  // are legal.
1802 
1803  // We do this in two phases; first to check the legality of the shuffles,
1804  // and next, assuming that all shuffles are legal, to create the new nodes.
1805  for (int Phase = 0; Phase < 2; ++Phase) {
1807  NewIntermedVals;
1808  for (unsigned i = 0; i < NumElems; ++i) {
1809  SDValue V = Node->getOperand(i);
1810  if (V.isUndef())
1811  continue;
1812 
1813  SDValue Vec;
1814  if (Phase)
1815  Vec = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, V);
1816  IntermedVals.push_back(std::make_pair(Vec, SmallVector<int, 16>(1, i)));
1817  }
1818 
1819  while (IntermedVals.size() > 2) {
1820  NewIntermedVals.clear();
1821  for (unsigned i = 0, e = (IntermedVals.size() & ~1u); i < e; i += 2) {
1822  // This vector and the next vector are shuffled together (simply to
1823  // append the one to the other).
1824  SmallVector<int, 16> ShuffleVec(NumElems, -1);
1825 
1826  SmallVector<int, 16> FinalIndices;
1827  FinalIndices.reserve(IntermedVals[i].second.size() +
1828  IntermedVals[i+1].second.size());
1829 
1830  int k = 0;
1831  for (unsigned j = 0, f = IntermedVals[i].second.size(); j != f;
1832  ++j, ++k) {
1833  ShuffleVec[k] = j;
1834  FinalIndices.push_back(IntermedVals[i].second[j]);
1835  }
1836  for (unsigned j = 0, f = IntermedVals[i+1].second.size(); j != f;
1837  ++j, ++k) {
1838  ShuffleVec[k] = NumElems + j;
1839  FinalIndices.push_back(IntermedVals[i+1].second[j]);
1840  }
1841 
1842  SDValue Shuffle;
1843  if (Phase)
1844  Shuffle = DAG.getVectorShuffle(VT, dl, IntermedVals[i].first,
1845  IntermedVals[i+1].first,
1846  ShuffleVec);
1847  else if (!TLI.isShuffleMaskLegal(ShuffleVec, VT))
1848  return false;
1849  NewIntermedVals.push_back(
1850  std::make_pair(Shuffle, std::move(FinalIndices)));
1851  }
1852 
1853  // If we had an odd number of defined values, then append the last
1854  // element to the array of new vectors.
1855  if ((IntermedVals.size() & 1) != 0)
1856  NewIntermedVals.push_back(IntermedVals.back());
1857 
1858  IntermedVals.swap(NewIntermedVals);
1859  }
1860 
1861  assert(IntermedVals.size() <= 2 && IntermedVals.size() > 0 &&
1862  "Invalid number of intermediate vectors");
1863  SDValue Vec1 = IntermedVals[0].first;
1864  SDValue Vec2;
1865  if (IntermedVals.size() > 1)
1866  Vec2 = IntermedVals[1].first;
1867  else if (Phase)
1868  Vec2 = DAG.getUNDEF(VT);
1869 
1870  SmallVector<int, 16> ShuffleVec(NumElems, -1);
1871  for (unsigned i = 0, e = IntermedVals[0].second.size(); i != e; ++i)
1872  ShuffleVec[IntermedVals[0].second[i]] = i;
1873  for (unsigned i = 0, e = IntermedVals[1].second.size(); i != e; ++i)
1874  ShuffleVec[IntermedVals[1].second[i]] = NumElems + i;
1875 
1876  if (Phase)
1877  Res = DAG.getVectorShuffle(VT, dl, Vec1, Vec2, ShuffleVec);
1878  else if (!TLI.isShuffleMaskLegal(ShuffleVec, VT))
1879  return false;
1880  }
1881 
1882  return true;
1883 }
1884 
1885 /// Expand a BUILD_VECTOR node on targets that don't
1886 /// support the operation, but do support the resultant vector type.
1887 SDValue SelectionDAGLegalize::ExpandBUILD_VECTOR(SDNode *Node) {
1888  unsigned NumElems = Node->getNumOperands();
1889  SDValue Value1, Value2;
1890  SDLoc dl(Node);
1891  EVT VT = Node->getValueType(0);
1892  EVT OpVT = Node->getOperand(0).getValueType();
1893  EVT EltVT = VT.getVectorElementType();
1894 
1895  // If the only non-undef value is the low element, turn this into a
1896  // SCALAR_TO_VECTOR node. If this is { X, X, X, X }, determine X.
1897  bool isOnlyLowElement = true;
1898  bool MoreThanTwoValues = false;
1899  bool isConstant = true;
1900  for (unsigned i = 0; i < NumElems; ++i) {
1901  SDValue V = Node->getOperand(i);
1902  if (V.isUndef())
1903  continue;
1904  if (i > 0)
1905  isOnlyLowElement = false;
1906  if (!isa<ConstantFPSDNode>(V) && !isa<ConstantSDNode>(V))
1907  isConstant = false;
1908 
1909  if (!Value1.getNode()) {
1910  Value1 = V;
1911  } else if (!Value2.getNode()) {
1912  if (V != Value1)
1913  Value2 = V;
1914  } else if (V != Value1 && V != Value2) {
1915  MoreThanTwoValues = true;
1916  }
1917  }
1918 
1919  if (!Value1.getNode())
1920  return DAG.getUNDEF(VT);
1921 
1922  if (isOnlyLowElement)
1923  return DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, Node->getOperand(0));
1924 
1925  // If all elements are constants, create a load from the constant pool.
1926  if (isConstant) {
1928  for (unsigned i = 0, e = NumElems; i != e; ++i) {
1929  if (ConstantFPSDNode *V =
1930  dyn_cast<ConstantFPSDNode>(Node->getOperand(i))) {
1931  CV.push_back(const_cast<ConstantFP *>(V->getConstantFPValue()));
1932  } else if (ConstantSDNode *V =
1933  dyn_cast<ConstantSDNode>(Node->getOperand(i))) {
1934  if (OpVT==EltVT)
1935  CV.push_back(const_cast<ConstantInt *>(V->getConstantIntValue()));
1936  else {
1937  // If OpVT and EltVT don't match, EltVT is not legal and the
1938  // element values have been promoted/truncated earlier. Undo this;
1939  // we don't want a v16i8 to become a v16i32 for example.
1940  const ConstantInt *CI = V->getConstantIntValue();
1941  CV.push_back(ConstantInt::get(EltVT.getTypeForEVT(*DAG.getContext()),
1942  CI->getZExtValue()));
1943  }
1944  } else {
1945  assert(Node->getOperand(i).isUndef());
1946  Type *OpNTy = EltVT.getTypeForEVT(*DAG.getContext());
1947  CV.push_back(UndefValue::get(OpNTy));
1948  }
1949  }
1951  SDValue CPIdx =
1952  DAG.getConstantPool(CP, TLI.getPointerTy(DAG.getDataLayout()));
1953  Align Alignment = cast<ConstantPoolSDNode>(CPIdx)->getAlign();
1954  return DAG.getLoad(
1955  VT, dl, DAG.getEntryNode(), CPIdx,
1956  MachinePointerInfo::getConstantPool(DAG.getMachineFunction()),
1957  Alignment);
1958  }
1959 
1960  SmallSet<SDValue, 16> DefinedValues;
1961  for (unsigned i = 0; i < NumElems; ++i) {
1962  if (Node->getOperand(i).isUndef())
1963  continue;
1964  DefinedValues.insert(Node->getOperand(i));
1965  }
1966 
1967  if (TLI.shouldExpandBuildVectorWithShuffles(VT, DefinedValues.size())) {
1968  if (!MoreThanTwoValues) {
1969  SmallVector<int, 8> ShuffleVec(NumElems, -1);
1970  for (unsigned i = 0; i < NumElems; ++i) {
1971  SDValue V = Node->getOperand(i);
1972  if (V.isUndef())
1973  continue;
1974  ShuffleVec[i] = V == Value1 ? 0 : NumElems;
1975  }
1976  if (TLI.isShuffleMaskLegal(ShuffleVec, Node->getValueType(0))) {
1977  // Get the splatted value into the low element of a vector register.
1978  SDValue Vec1 = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, Value1);
1979  SDValue Vec2;
1980  if (Value2.getNode())
1981  Vec2 = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, Value2);
1982  else
1983  Vec2 = DAG.getUNDEF(VT);
1984 
1985  // Return shuffle(LowValVec, undef, <0,0,0,0>)
1986  return DAG.getVectorShuffle(VT, dl, Vec1, Vec2, ShuffleVec);
1987  }
1988  } else {
1989  SDValue Res;
1990  if (ExpandBVWithShuffles(Node, DAG, TLI, Res))
1991  return Res;
1992  }
1993  }
1994 
1995  // Otherwise, we can't handle this case efficiently.
1996  return ExpandVectorBuildThroughStack(Node);
1997 }
1998 
1999 SDValue SelectionDAGLegalize::ExpandSPLAT_VECTOR(SDNode *Node) {
2000  SDLoc DL(Node);
2001  EVT VT = Node->getValueType(0);
2002  SDValue SplatVal = Node->getOperand(0);
2003 
2004  return DAG.getSplatBuildVector(VT, DL, SplatVal);
2005 }
2006 
2007 // Expand a node into a call to a libcall. If the result value
2008 // does not fit into a register, return the lo part and set the hi part to the
2009 // by-reg argument. If it does fit into a single register, return the result
2010 // and leave the Hi part unset.
2011 SDValue SelectionDAGLegalize::ExpandLibCall(RTLIB::Libcall LC, SDNode *Node,
2012  bool isSigned) {
2014  TargetLowering::ArgListEntry Entry;
2015  for (const SDValue &Op : Node->op_values()) {
2016  EVT ArgVT = Op.getValueType();
2017  Type *ArgTy = ArgVT.getTypeForEVT(*DAG.getContext());
2018  Entry.Node = Op;
2019  Entry.Ty = ArgTy;
2020  Entry.IsSExt = TLI.shouldSignExtendTypeInLibCall(ArgVT, isSigned);
2021  Entry.IsZExt = !TLI.shouldSignExtendTypeInLibCall(ArgVT, isSigned);
2022  Args.push_back(Entry);
2023  }
2024  SDValue Callee = DAG.getExternalSymbol(TLI.getLibcallName(LC),
2025  TLI.getPointerTy(DAG.getDataLayout()));
2026 
2027  EVT RetVT = Node->getValueType(0);
2028  Type *RetTy = RetVT.getTypeForEVT(*DAG.getContext());
2029 
2030  // By default, the input chain to this libcall is the entry node of the
2031  // function. If the libcall is going to be emitted as a tail call then
2032  // TLI.isUsedByReturnOnly will change it to the right chain if the return
2033  // node which is being folded has a non-entry input chain.
2034  SDValue InChain = DAG.getEntryNode();
2035 
2036  // isTailCall may be true since the callee does not reference caller stack
2037  // frame. Check if it's in the right position and that the return types match.
2038  SDValue TCChain = InChain;
2039  const Function &F = DAG.getMachineFunction().getFunction();
2040  bool isTailCall =
2041  TLI.isInTailCallPosition(DAG, Node, TCChain) &&
2042  (RetTy == F.getReturnType() || F.getReturnType()->isVoidTy());
2043  if (isTailCall)
2044  InChain = TCChain;
2045 
2047  bool signExtend = TLI.shouldSignExtendTypeInLibCall(RetVT, isSigned);
2048  CLI.setDebugLoc(SDLoc(Node))
2049  .setChain(InChain)
2050  .setLibCallee(TLI.getLibcallCallingConv(LC), RetTy, Callee,
2051  std::move(Args))
2052  .setTailCall(isTailCall)
2053  .setSExtResult(signExtend)
2054  .setZExtResult(!signExtend)
2055  .setIsPostTypeLegalization(true);
2056 
2057  std::pair<SDValue, SDValue> CallInfo = TLI.LowerCallTo(CLI);
2058 
2059  if (!CallInfo.second.getNode()) {
2060  LLVM_DEBUG(dbgs() << "Created tailcall: "; DAG.getRoot().dump(&DAG));
2061  // It's a tailcall, return the chain (which is the DAG root).
2062  return DAG.getRoot();
2063  }
2064 
2065  LLVM_DEBUG(dbgs() << "Created libcall: "; CallInfo.first.dump(&DAG));
2066  return CallInfo.first;
2067 }
2068 
2069 void SelectionDAGLegalize::ExpandFPLibCall(SDNode* Node,
2070  RTLIB::Libcall LC,
2072  if (LC == RTLIB::UNKNOWN_LIBCALL)
2073  llvm_unreachable("Can't create an unknown libcall!");
2074 
2075  if (Node->isStrictFPOpcode()) {
2076  EVT RetVT = Node->getValueType(0);
2077  SmallVector<SDValue, 4> Ops(drop_begin(Node->ops()));
2079  // FIXME: This doesn't support tail calls.
2080  std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, RetVT,
2081  Ops, CallOptions,
2082  SDLoc(Node),
2083  Node->getOperand(0));
2084  Results.push_back(Tmp.first);
2085  Results.push_back(Tmp.second);
2086  } else {
2087  SDValue Tmp = ExpandLibCall(LC, Node, false);
2088  Results.push_back(Tmp);
2089  }
2090 }
2091 
2092 /// Expand the node to a libcall based on the result type.
2093 void SelectionDAGLegalize::ExpandFPLibCall(SDNode* Node,
2094  RTLIB::Libcall Call_F32,
2095  RTLIB::Libcall Call_F64,
2096  RTLIB::Libcall Call_F80,
2097  RTLIB::Libcall Call_F128,
2098  RTLIB::Libcall Call_PPCF128,
2100  RTLIB::Libcall LC = RTLIB::getFPLibCall(Node->getSimpleValueType(0),
2101  Call_F32, Call_F64, Call_F80,
2102  Call_F128, Call_PPCF128);
2103  ExpandFPLibCall(Node, LC, Results);
2104 }
2105 
2106 SDValue SelectionDAGLegalize::ExpandIntLibCall(SDNode* Node, bool isSigned,
2107  RTLIB::Libcall Call_I8,
2108  RTLIB::Libcall Call_I16,
2109  RTLIB::Libcall Call_I32,
2110  RTLIB::Libcall Call_I64,
2111  RTLIB::Libcall Call_I128) {
2112  RTLIB::Libcall LC;
2113  switch (Node->getSimpleValueType(0).SimpleTy) {
2114  default: llvm_unreachable("Unexpected request for libcall!");
2115  case MVT::i8: LC = Call_I8; break;
2116  case MVT::i16: LC = Call_I16; break;
2117  case MVT::i32: LC = Call_I32; break;
2118  case MVT::i64: LC = Call_I64; break;
2119  case MVT::i128: LC = Call_I128; break;
2120  }
2121  return ExpandLibCall(LC, Node, isSigned);
2122 }
2123 
2124 /// Expand the node to a libcall based on first argument type (for instance
2125 /// lround and its variant).
2126 void SelectionDAGLegalize::ExpandArgFPLibCall(SDNode* Node,
2127  RTLIB::Libcall Call_F32,
2128  RTLIB::Libcall Call_F64,
2129  RTLIB::Libcall Call_F80,
2130  RTLIB::Libcall Call_F128,
2131  RTLIB::Libcall Call_PPCF128,
2133  EVT InVT = Node->getOperand(Node->isStrictFPOpcode() ? 1 : 0).getValueType();
2135  Call_F32, Call_F64, Call_F80,
2136  Call_F128, Call_PPCF128);
2137  ExpandFPLibCall(Node, LC, Results);
2138 }
2139 
2140 /// Issue libcalls to __{u}divmod to compute div / rem pairs.
2141 void
2142 SelectionDAGLegalize::ExpandDivRemLibCall(SDNode *Node,
2144  unsigned Opcode = Node->getOpcode();
2145  bool isSigned = Opcode == ISD::SDIVREM;
2146 
2147  RTLIB::Libcall LC;
2148  switch (Node->getSimpleValueType(0).SimpleTy) {
2149  default: llvm_unreachable("Unexpected request for libcall!");
2150  case MVT::i8: LC= isSigned ? RTLIB::SDIVREM_I8 : RTLIB::UDIVREM_I8; break;
2151  case MVT::i16: LC= isSigned ? RTLIB::SDIVREM_I16 : RTLIB::UDIVREM_I16; break;
2152  case MVT::i32: LC= isSigned ? RTLIB::SDIVREM_I32 : RTLIB::UDIVREM_I32; break;
2153  case MVT::i64: LC= isSigned ? RTLIB::SDIVREM_I64 : RTLIB::UDIVREM_I64; break;
2154  case MVT::i128: LC= isSigned ? RTLIB::SDIVREM_I128:RTLIB::UDIVREM_I128; break;
2155  }
2156 
2157  // The input chain to this libcall is the entry node of the function.
2158  // Legalizing the call will automatically add the previous call to the
2159  // dependence.
2160  SDValue InChain = DAG.getEntryNode();
2161 
2162  EVT RetVT = Node->getValueType(0);
2163  Type *RetTy = RetVT.getTypeForEVT(*DAG.getContext());
2164 
2166  TargetLowering::ArgListEntry Entry;
2167  for (const SDValue &Op : Node->op_values()) {
2168  EVT ArgVT = Op.getValueType();
2169  Type *ArgTy = ArgVT.getTypeForEVT(*DAG.getContext());
2170  Entry.Node = Op;
2171  Entry.Ty = ArgTy;
2172  Entry.IsSExt = isSigned;
2173  Entry.IsZExt = !isSigned;
2174  Args.push_back(Entry);
2175  }
2176 
2177  // Also pass the return address of the remainder.
2178  SDValue FIPtr = DAG.CreateStackTemporary(RetVT);
2179  Entry.Node = FIPtr;
2180  Entry.Ty = RetTy->getPointerTo();
2181  Entry.IsSExt = isSigned;
2182  Entry.IsZExt = !isSigned;
2183  Args.push_back(Entry);
2184 
2185  SDValue Callee = DAG.getExternalSymbol(TLI.getLibcallName(LC),
2186  TLI.getPointerTy(DAG.getDataLayout()));
2187 
2188  SDLoc dl(Node);
2190  CLI.setDebugLoc(dl)
2191  .setChain(InChain)
2192  .setLibCallee(TLI.getLibcallCallingConv(LC), RetTy, Callee,
2193  std::move(Args))
2194  .setSExtResult(isSigned)
2195  .setZExtResult(!isSigned);
2196 
2197  std::pair<SDValue, SDValue> CallInfo = TLI.LowerCallTo(CLI);
2198 
2199  // Remainder is loaded back from the stack frame.
2200  SDValue Rem =
2201  DAG.getLoad(RetVT, dl, CallInfo.second, FIPtr, MachinePointerInfo());
2202  Results.push_back(CallInfo.first);
2203  Results.push_back(Rem);
2204 }
2205 
2206 /// Return true if sincos libcall is available.
2207 static bool isSinCosLibcallAvailable(SDNode *Node, const TargetLowering &TLI) {
2208  RTLIB::Libcall LC;
2209  switch (Node->getSimpleValueType(0).SimpleTy) {
2210  default: llvm_unreachable("Unexpected request for libcall!");
2211  case MVT::f32: LC = RTLIB::SINCOS_F32; break;
2212  case MVT::f64: LC = RTLIB::SINCOS_F64; break;
2213  case MVT::f80: LC = RTLIB::SINCOS_F80; break;
2214  case MVT::f128: LC = RTLIB::SINCOS_F128; break;
2215  case MVT::ppcf128: LC = RTLIB::SINCOS_PPCF128; break;
2216  }
2217  return TLI.getLibcallName(LC) != nullptr;
2218 }
2219 
2220 /// Only issue sincos libcall if both sin and cos are needed.
2221 static bool useSinCos(SDNode *Node) {
2222  unsigned OtherOpcode = Node->getOpcode() == ISD::FSIN
2223  ? ISD::FCOS : ISD::FSIN;
2224 
2225  SDValue Op0 = Node->getOperand(0);
2226  for (SDNode::use_iterator UI = Op0.getNode()->use_begin(),
2227  UE = Op0.getNode()->use_end(); UI != UE; ++UI) {
2228  SDNode *User = *UI;
2229  if (User == Node)
2230  continue;
2231  // The other user might have been turned into sincos already.
2232  if (User->getOpcode() == OtherOpcode || User->getOpcode() == ISD::FSINCOS)
2233  return true;
2234  }
2235  return false;
2236 }
2237 
2238 /// Issue libcalls to sincos to compute sin / cos pairs.
2239 void
2240 SelectionDAGLegalize::ExpandSinCosLibCall(SDNode *Node,
2242  RTLIB::Libcall LC;
2243  switch (Node->getSimpleValueType(0).SimpleTy) {
2244  default: llvm_unreachable("Unexpected request for libcall!");
2245  case MVT::f32: LC = RTLIB::SINCOS_F32; break;
2246  case MVT::f64: LC = RTLIB::SINCOS_F64; break;
2247  case MVT::f80: LC = RTLIB::SINCOS_F80; break;
2248  case MVT::f128: LC = RTLIB::SINCOS_F128; break;
2249  case MVT::ppcf128: LC = RTLIB::SINCOS_PPCF128; break;
2250  }
2251 
2252  // The input chain to this libcall is the entry node of the function.
2253  // Legalizing the call will automatically add the previous call to the
2254  // dependence.
2255  SDValue InChain = DAG.getEntryNode();
2256 
2257  EVT RetVT = Node->getValueType(0);
2258  Type *RetTy = RetVT.getTypeForEVT(*DAG.getContext());
2259 
2261  TargetLowering::ArgListEntry Entry;
2262 
2263  // Pass the argument.
2264  Entry.Node = Node->getOperand(0);
2265  Entry.Ty = RetTy;
2266  Entry.IsSExt = false;
2267  Entry.IsZExt = false;
2268  Args.push_back(Entry);
2269 
2270  // Pass the return address of sin.
2271  SDValue SinPtr = DAG.CreateStackTemporary(RetVT);
2272  Entry.Node = SinPtr;
2273  Entry.Ty = RetTy->getPointerTo();
2274  Entry.IsSExt = false;
2275  Entry.IsZExt = false;
2276  Args.push_back(Entry);
2277 
2278  // Also pass the return address of the cos.
2279  SDValue CosPtr = DAG.CreateStackTemporary(RetVT);
2280  Entry.Node = CosPtr;
2281  Entry.Ty = RetTy->getPointerTo();
2282  Entry.IsSExt = false;
2283  Entry.IsZExt = false;
2284  Args.push_back(Entry);
2285 
2286  SDValue Callee = DAG.getExternalSymbol(TLI.getLibcallName(LC),
2287  TLI.getPointerTy(DAG.getDataLayout()));
2288 
2289  SDLoc dl(Node);
2291  CLI.setDebugLoc(dl).setChain(InChain).setLibCallee(
2292  TLI.getLibcallCallingConv(LC), Type::getVoidTy(*DAG.getContext()), Callee,
2293  std::move(Args));
2294 
2295  std::pair<SDValue, SDValue> CallInfo = TLI.LowerCallTo(CLI);
2296 
2297  Results.push_back(
2298  DAG.getLoad(RetVT, dl, CallInfo.second, SinPtr, MachinePointerInfo()));
2299  Results.push_back(
2300  DAG.getLoad(RetVT, dl, CallInfo.second, CosPtr, MachinePointerInfo()));
2301 }
2302 
2303 /// This function is responsible for legalizing a
2304 /// INT_TO_FP operation of the specified operand when the target requests that
2305 /// we expand it. At this point, we know that the result and operand types are
2306 /// legal for the target.
2307 SDValue SelectionDAGLegalize::ExpandLegalINT_TO_FP(SDNode *Node,
2308  SDValue &Chain) {
2309  bool isSigned = (Node->getOpcode() == ISD::STRICT_SINT_TO_FP ||
2310  Node->getOpcode() == ISD::SINT_TO_FP);
2311  EVT DestVT = Node->getValueType(0);
2312  SDLoc dl(Node);
2313  unsigned OpNo = Node->isStrictFPOpcode() ? 1 : 0;
2314  SDValue Op0 = Node->getOperand(OpNo);
2315  EVT SrcVT = Op0.getValueType();
2316 
2317  // TODO: Should any fast-math-flags be set for the created nodes?
2318  LLVM_DEBUG(dbgs() << "Legalizing INT_TO_FP\n");
2319  if (SrcVT == MVT::i32 && TLI.isTypeLegal(MVT::f64) &&
2320  (DestVT.bitsLE(MVT::f64) ||
2321  TLI.isOperationLegal(Node->isStrictFPOpcode() ? ISD::STRICT_FP_EXTEND
2322  : ISD::FP_EXTEND,
2323  DestVT))) {
2324  LLVM_DEBUG(dbgs() << "32-bit [signed|unsigned] integer to float/double "
2325  "expansion\n");
2326 
2327  // Get the stack frame index of a 8 byte buffer.
2328  SDValue StackSlot = DAG.CreateStackTemporary(MVT::f64);
2329 
2330  SDValue Lo = Op0;
2331  // if signed map to unsigned space
2332  if (isSigned) {
2333  // Invert sign bit (signed to unsigned mapping).
2334  Lo = DAG.getNode(ISD::XOR, dl, MVT::i32, Lo,
2335  DAG.getConstant(0x80000000u, dl, MVT::i32));
2336  }
2337  // Initial hi portion of constructed double.
2338  SDValue Hi = DAG.getConstant(0x43300000u, dl, MVT::i32);
2339 
2340  // If this a big endian target, swap the lo and high data.
2341  if (DAG.getDataLayout().isBigEndian())
2342  std::swap(Lo, Hi);
2343 
2344  SDValue MemChain = DAG.getEntryNode();
2345 
2346  // Store the lo of the constructed double.
2347  SDValue Store1 = DAG.getStore(MemChain, dl, Lo, StackSlot,
2348  MachinePointerInfo());
2349  // Store the hi of the constructed double.
2350  SDValue HiPtr = DAG.getMemBasePlusOffset(StackSlot, TypeSize::Fixed(4), dl);
2351  SDValue Store2 =
2352  DAG.getStore(MemChain, dl, Hi, HiPtr, MachinePointerInfo());
2353  MemChain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Store1, Store2);
2354 
2355  // load the constructed double
2356  SDValue Load =
2357  DAG.getLoad(MVT::f64, dl, MemChain, StackSlot, MachinePointerInfo());
2358  // FP constant to bias correct the final result
2359  SDValue Bias = DAG.getConstantFP(isSigned ?
2360  BitsToDouble(0x4330000080000000ULL) :
2361  BitsToDouble(0x4330000000000000ULL),
2362  dl, MVT::f64);
2363  // Subtract the bias and get the final result.
2364  SDValue Sub;
2365  SDValue Result;
2366  if (Node->isStrictFPOpcode()) {
2367  Sub = DAG.getNode(ISD::STRICT_FSUB, dl, {MVT::f64, MVT::Other},
2368  {Node->getOperand(0), Load, Bias});
2369  Chain = Sub.getValue(1);
2370  if (DestVT != Sub.getValueType()) {
2371  std::pair<SDValue, SDValue> ResultPair;
2372  ResultPair =
2373  DAG.getStrictFPExtendOrRound(Sub, Chain, dl, DestVT);
2374  Result = ResultPair.first;
2375  Chain = ResultPair.second;
2376  }
2377  else
2378  Result = Sub;
2379  } else {
2380  Sub = DAG.getNode(ISD::FSUB, dl, MVT::f64, Load, Bias);
2381  Result = DAG.getFPExtendOrRound(Sub, dl, DestVT);
2382  }
2383  return Result;
2384  }
2385 
2386  if (isSigned)
2387  return SDValue();
2388 
2389  // TODO: Generalize this for use with other types.
2390  if (((SrcVT == MVT::i32 || SrcVT == MVT::i64) && DestVT == MVT::f32) ||
2391  (SrcVT == MVT::i64 && DestVT == MVT::f64)) {
2392  LLVM_DEBUG(dbgs() << "Converting unsigned i32/i64 to f32/f64\n");
2393  // For unsigned conversions, convert them to signed conversions using the
2394  // algorithm from the x86_64 __floatundisf in compiler_rt. That method
2395  // should be valid for i32->f32 as well.
2396 
2397  // More generally this transform should be valid if there are 3 more bits
2398  // in the integer type than the significand. Rounding uses the first bit
2399  // after the width of the significand and the OR of all bits after that. So
2400  // we need to be able to OR the shifted out bit into one of the bits that
2401  // participate in the OR.
2402 
2403  // TODO: This really should be implemented using a branch rather than a
2404  // select. We happen to get lucky and machinesink does the right
2405  // thing most of the time. This would be a good candidate for a
2406  // pseudo-op, or, even better, for whole-function isel.
2407  EVT SetCCVT = getSetCCResultType(SrcVT);
2408 
2409  SDValue SignBitTest = DAG.getSetCC(
2410  dl, SetCCVT, Op0, DAG.getConstant(0, dl, SrcVT), ISD::SETLT);
2411 
2412  EVT ShiftVT = TLI.getShiftAmountTy(SrcVT, DAG.getDataLayout());
2413  SDValue ShiftConst = DAG.getConstant(1, dl, ShiftVT);
2414  SDValue Shr = DAG.getNode(ISD::SRL, dl, SrcVT, Op0, ShiftConst);
2415  SDValue AndConst = DAG.getConstant(1, dl, SrcVT);
2416  SDValue And = DAG.getNode(ISD::AND, dl, SrcVT, Op0, AndConst);
2417  SDValue Or = DAG.getNode(ISD::OR, dl, SrcVT, And, Shr);
2418 
2419  SDValue Slow, Fast;
2420  if (Node->isStrictFPOpcode()) {
2421  // In strict mode, we must avoid spurious exceptions, and therefore
2422  // must make sure to only emit a single STRICT_SINT_TO_FP.
2423  SDValue InCvt = DAG.getSelect(dl, SrcVT, SignBitTest, Or, Op0);
2424  Fast = DAG.getNode(ISD::STRICT_SINT_TO_FP, dl, { DestVT, MVT::Other },
2425  { Node->getOperand(0), InCvt });
2426  Slow = DAG.getNode(ISD::STRICT_FADD, dl, { DestVT, MVT::Other },
2427  { Fast.getValue(1), Fast, Fast });
2428  Chain = Slow.getValue(1);
2429  // The STRICT_SINT_TO_FP inherits the exception mode from the
2430  // incoming STRICT_UINT_TO_FP node; the STRICT_FADD node can
2431  // never raise any exception.
2432  SDNodeFlags Flags;
2433  Flags.setNoFPExcept(Node->getFlags().hasNoFPExcept());
2434  Fast->setFlags(Flags);
2435  Flags.setNoFPExcept(true);
2436  Slow->setFlags(Flags);
2437  } else {
2438  SDValue SignCvt = DAG.getNode(ISD::SINT_TO_FP, dl, DestVT, Or);
2439  Slow = DAG.getNode(ISD::FADD, dl, DestVT, SignCvt, SignCvt);
2440  Fast = DAG.getNode(ISD::SINT_TO_FP, dl, DestVT, Op0);
2441  }
2442 
2443  return DAG.getSelect(dl, DestVT, SignBitTest, Slow, Fast);
2444  }
2445 
2446  // Don't expand it if there isn't cheap fadd.
2447  if (!TLI.isOperationLegalOrCustom(
2448  Node->isStrictFPOpcode() ? ISD::STRICT_FADD : ISD::FADD, DestVT))
2449  return SDValue();
2450 
2451  // The following optimization is valid only if every value in SrcVT (when
2452  // treated as signed) is representable in DestVT. Check that the mantissa
2453  // size of DestVT is >= than the number of bits in SrcVT -1.
2454  assert(APFloat::semanticsPrecision(DAG.EVTToAPFloatSemantics(DestVT)) >=
2455  SrcVT.getSizeInBits() - 1 &&
2456  "Cannot perform lossless SINT_TO_FP!");
2457 
2458  SDValue Tmp1;
2459  if (Node->isStrictFPOpcode()) {
2460  Tmp1 = DAG.getNode(ISD::STRICT_SINT_TO_FP, dl, { DestVT, MVT::Other },
2461  { Node->getOperand(0), Op0 });
2462  } else
2463  Tmp1 = DAG.getNode(ISD::SINT_TO_FP, dl, DestVT, Op0);
2464 
2465  SDValue SignSet = DAG.getSetCC(dl, getSetCCResultType(SrcVT), Op0,
2466  DAG.getConstant(0, dl, SrcVT), ISD::SETLT);
2467  SDValue Zero = DAG.getIntPtrConstant(0, dl),
2468  Four = DAG.getIntPtrConstant(4, dl);
2469  SDValue CstOffset = DAG.getSelect(dl, Zero.getValueType(),
2470  SignSet, Four, Zero);
2471 
2472  // If the sign bit of the integer is set, the large number will be treated
2473  // as a negative number. To counteract this, the dynamic code adds an
2474  // offset depending on the data type.
2475  uint64_t FF;
2476  switch (SrcVT.getSimpleVT().SimpleTy) {
2477  default:
2478  return SDValue();
2479  case MVT::i8 : FF = 0x43800000ULL; break; // 2^8 (as a float)
2480  case MVT::i16: FF = 0x47800000ULL; break; // 2^16 (as a float)
2481  case MVT::i32: FF = 0x4F800000ULL; break; // 2^32 (as a float)
2482  case MVT::i64: FF = 0x5F800000ULL; break; // 2^64 (as a float)
2483  }
2484  if (DAG.getDataLayout().isLittleEndian())
2485  FF <<= 32;
2486  Constant *FudgeFactor = ConstantInt::get(
2487  Type::getInt64Ty(*DAG.getContext()), FF);
2488 
2489  SDValue CPIdx =
2490  DAG.getConstantPool(FudgeFactor, TLI.getPointerTy(DAG.getDataLayout()));
2491  Align Alignment = cast<ConstantPoolSDNode>(CPIdx)->getAlign();
2492  CPIdx = DAG.getNode(ISD::ADD, dl, CPIdx.getValueType(), CPIdx, CstOffset);
2493  Alignment = commonAlignment(Alignment, 4);
2494  SDValue FudgeInReg;
2495  if (DestVT == MVT::f32)
2496  FudgeInReg = DAG.getLoad(
2497  MVT::f32, dl, DAG.getEntryNode(), CPIdx,
2498  MachinePointerInfo::getConstantPool(DAG.getMachineFunction()),
2499  Alignment);
2500  else {
2501  SDValue Load = DAG.getExtLoad(
2502  ISD::EXTLOAD, dl, DestVT, DAG.getEntryNode(), CPIdx,
2503  MachinePointerInfo::getConstantPool(DAG.getMachineFunction()), MVT::f32,
2504  Alignment);
2505  HandleSDNode Handle(Load);
2506  LegalizeOp(Load.getNode());
2507  FudgeInReg = Handle.getValue();
2508  }
2509 
2510  if (Node->isStrictFPOpcode()) {
2511  SDValue Result = DAG.getNode(ISD::STRICT_FADD, dl, { DestVT, MVT::Other },
2512  { Tmp1.getValue(1), Tmp1, FudgeInReg });
2513  Chain = Result.getValue(1);
2514  return Result;
2515  }
2516 
2517  return DAG.getNode(ISD::FADD, dl, DestVT, Tmp1, FudgeInReg);
2518 }
2519 
2520 /// This function is responsible for legalizing a
2521 /// *INT_TO_FP operation of the specified operand when the target requests that
2522 /// we promote it. At this point, we know that the result and operand types are
2523 /// legal for the target, and that there is a legal UINT_TO_FP or SINT_TO_FP
2524 /// operation that takes a larger input.
2525 void SelectionDAGLegalize::PromoteLegalINT_TO_FP(
2526  SDNode *N, const SDLoc &dl, SmallVectorImpl<SDValue> &Results) {
2527  bool IsStrict = N->isStrictFPOpcode();
2528  bool IsSigned = N->getOpcode() == ISD::SINT_TO_FP ||
2529  N->getOpcode() == ISD::STRICT_SINT_TO_FP;
2530  EVT DestVT = N->getValueType(0);
2531  SDValue LegalOp = N->getOperand(IsStrict ? 1 : 0);
2532  unsigned UIntOp = IsStrict ? ISD::STRICT_UINT_TO_FP : ISD::UINT_TO_FP;
2533  unsigned SIntOp = IsStrict ? ISD::STRICT_SINT_TO_FP : ISD::SINT_TO_FP;
2534 
2535  // First step, figure out the appropriate *INT_TO_FP operation to use.
2536  EVT NewInTy = LegalOp.getValueType();
2537 
2538  unsigned OpToUse = 0;
2539 
2540  // Scan for the appropriate larger type to use.
2541  while (true) {
2542  NewInTy = (MVT::SimpleValueType)(NewInTy.getSimpleVT().SimpleTy+1);
2543  assert(NewInTy.isInteger() && "Ran out of possibilities!");
2544 
2545  // If the target supports SINT_TO_FP of this type, use it.
2546  if (TLI.isOperationLegalOrCustom(SIntOp, NewInTy)) {
2547  OpToUse = SIntOp;
2548  break;
2549  }
2550  if (IsSigned)
2551  continue;
2552 
2553  // If the target supports UINT_TO_FP of this type, use it.
2554  if (TLI.isOperationLegalOrCustom(UIntOp, NewInTy)) {
2555  OpToUse = UIntOp;
2556  break;
2557  }
2558 
2559  // Otherwise, try a larger type.
2560  }
2561 
2562  // Okay, we found the operation and type to use. Zero extend our input to the
2563  // desired type then run the operation on it.
2564  if (IsStrict) {
2565  SDValue Res =
2566  DAG.getNode(OpToUse, dl, {DestVT, MVT::Other},
2567  {N->getOperand(0),
2568  DAG.getNode(IsSigned ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND,
2569  dl, NewInTy, LegalOp)});
2570  Results.push_back(Res);
2571  Results.push_back(Res.getValue(1));
2572  return;
2573  }
2574 
2575  Results.push_back(
2576  DAG.getNode(OpToUse, dl, DestVT,
2577  DAG.getNode(IsSigned ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND,
2578  dl, NewInTy, LegalOp)));
2579 }
2580 
2581 /// This function is responsible for legalizing a
2582 /// FP_TO_*INT operation of the specified operand when the target requests that
2583 /// we promote it. At this point, we know that the result and operand types are
2584 /// legal for the target, and that there is a legal FP_TO_UINT or FP_TO_SINT
2585 /// operation that returns a larger result.
2586 void SelectionDAGLegalize::PromoteLegalFP_TO_INT(SDNode *N, const SDLoc &dl,
2588  bool IsStrict = N->isStrictFPOpcode();
2589  bool IsSigned = N->getOpcode() == ISD::FP_TO_SINT ||
2590  N->getOpcode() == ISD::STRICT_FP_TO_SINT;
2591  EVT DestVT = N->getValueType(0);
2592  SDValue LegalOp = N->getOperand(IsStrict ? 1 : 0);
2593  // First step, figure out the appropriate FP_TO*INT operation to use.
2594  EVT NewOutTy = DestVT;
2595 
2596  unsigned OpToUse = 0;
2597 
2598  // Scan for the appropriate larger type to use.
2599  while (true) {
2600  NewOutTy = (MVT::SimpleValueType)(NewOutTy.getSimpleVT().SimpleTy+1);
2601  assert(NewOutTy.isInteger() && "Ran out of possibilities!");
2602 
2603  // A larger signed type can hold all unsigned values of the requested type,
2604  // so using FP_TO_SINT is valid
2605  OpToUse = IsStrict ? ISD::STRICT_FP_TO_SINT : ISD::FP_TO_SINT;
2606  if (TLI.isOperationLegalOrCustom(OpToUse, NewOutTy))
2607  break;
2608 
2609  // However, if the value may be < 0.0, we *must* use some FP_TO_SINT.
2610  OpToUse = IsStrict ? ISD::STRICT_FP_TO_UINT : ISD::FP_TO_UINT;
2611  if (!IsSigned && TLI.isOperationLegalOrCustom(OpToUse, NewOutTy))
2612  break;
2613 
2614  // Otherwise, try a larger type.
2615  }
2616 
2617  // Okay, we found the operation and type to use.
2619  if (IsStrict) {
2620  SDVTList VTs = DAG.getVTList(NewOutTy, MVT::Other);
2621  Operation = DAG.getNode(OpToUse, dl, VTs, N->getOperand(0), LegalOp);
2622  } else
2623  Operation = DAG.getNode(OpToUse, dl, NewOutTy, LegalOp);
2624 
2625  // Truncate the result of the extended FP_TO_*INT operation to the desired
2626  // size.
2627  SDValue Trunc = DAG.getNode(ISD::TRUNCATE, dl, DestVT, Operation);
2628  Results.push_back(Trunc);
2629  if (IsStrict)
2630  Results.push_back(Operation.getValue(1));
2631 }
2632 
2633 /// Promote FP_TO_*INT_SAT operation to a larger result type. At this point
2634 /// the result and operand types are legal and there must be a legal
2635 /// FP_TO_*INT_SAT operation for a larger result type.
2636 SDValue SelectionDAGLegalize::PromoteLegalFP_TO_INT_SAT(SDNode *Node,
2637  const SDLoc &dl) {
2638  unsigned Opcode = Node->getOpcode();
2639 
2640  // Scan for the appropriate larger type to use.
2641  EVT NewOutTy = Node->getValueType(0);
2642  while (true) {
2643  NewOutTy = (MVT::SimpleValueType)(NewOutTy.getSimpleVT().SimpleTy + 1);
2644  assert(NewOutTy.isInteger() && "Ran out of possibilities!");
2645 
2646  if (TLI.isOperationLegalOrCustom(Opcode, NewOutTy))
2647  break;
2648  }
2649 
2650  // Saturation width is determined by second operand, so we don't have to
2651  // perform any fixup and can directly truncate the result.
2652  SDValue Result = DAG.getNode(Opcode, dl, NewOutTy, Node->getOperand(0),
2653  Node->getOperand(1));
2654  return DAG.getNode(ISD::TRUNCATE, dl, Node->getValueType(0), Result);
2655 }
2656 
2657 /// Open code the operations for PARITY of the specified operation.
2658 SDValue SelectionDAGLegalize::ExpandPARITY(SDValue Op, const SDLoc &dl) {
2659  EVT VT = Op.getValueType();
2660  EVT ShVT = TLI.getShiftAmountTy(VT, DAG.getDataLayout());
2661  unsigned Sz = VT.getScalarSizeInBits();
2662 
2663  // If CTPOP is legal, use it. Otherwise use shifts and xor.
2664  SDValue Result;
2665  if (TLI.isOperationLegal(ISD::CTPOP, VT)) {
2666  Result = DAG.getNode(ISD::CTPOP, dl, VT, Op);
2667  } else {
2668  Result = Op;
2669  for (unsigned i = Log2_32_Ceil(Sz); i != 0;) {
2670  SDValue Shift = DAG.getNode(ISD::SRL, dl, VT, Result,
2671  DAG.getConstant(1ULL << (--i), dl, ShVT));
2672  Result = DAG.getNode(ISD::XOR, dl, VT, Result, Shift);
2673  }
2674  }
2675 
2676  return DAG.getNode(ISD::AND, dl, VT, Result, DAG.getConstant(1, dl, VT));
2677 }
2678 
2679 bool SelectionDAGLegalize::ExpandNode(SDNode *Node) {
2680  LLVM_DEBUG(dbgs() << "Trying to expand node\n");
2682  SDLoc dl(Node);
2683  SDValue Tmp1, Tmp2, Tmp3, Tmp4;
2684  bool NeedInvert;
2685  switch (Node->getOpcode()) {
2686  case ISD::ABS:
2687  if (TLI.expandABS(Node, Tmp1, DAG))
2688  Results.push_back(Tmp1);
2689  break;
2690  case ISD::CTPOP:
2691  if (TLI.expandCTPOP(Node, Tmp1, DAG))
2692  Results.push_back(Tmp1);
2693  break;
2694  case ISD::CTLZ:
2695  case ISD::CTLZ_ZERO_UNDEF:
2696  if (TLI.expandCTLZ(Node, Tmp1, DAG))
2697  Results.push_back(Tmp1);
2698  break;
2699  case ISD::CTTZ:
2700  case ISD::CTTZ_ZERO_UNDEF:
2701  if (TLI.expandCTTZ(Node, Tmp1, DAG))
2702  Results.push_back(Tmp1);
2703  break;
2704  case ISD::BITREVERSE:
2705  if ((Tmp1 = TLI.expandBITREVERSE(Node, DAG)))
2706  Results.push_back(Tmp1);
2707  break;
2708  case ISD::BSWAP:
2709  if ((Tmp1 = TLI.expandBSWAP(Node, DAG)))
2710  Results.push_back(Tmp1);
2711  break;
2712  case ISD::PARITY:
2713  Results.push_back(ExpandPARITY(Node->getOperand(0), dl));
2714  break;
2715  case ISD::FRAMEADDR:
2716  case ISD::RETURNADDR:
2718  Results.push_back(DAG.getConstant(0, dl, Node->getValueType(0)));
2719  break;
2720  case ISD::EH_DWARF_CFA: {
2721  SDValue CfaArg = DAG.getSExtOrTrunc(Node->getOperand(0), dl,
2722  TLI.getPointerTy(DAG.getDataLayout()));
2723  SDValue Offset = DAG.getNode(ISD::ADD, dl,
2724  CfaArg.getValueType(),
2725  DAG.getNode(ISD::FRAME_TO_ARGS_OFFSET, dl,
2726  CfaArg.getValueType()),
2727  CfaArg);
2728  SDValue FA = DAG.getNode(
2729  ISD::FRAMEADDR, dl, TLI.getPointerTy(DAG.getDataLayout()),
2730  DAG.getConstant(0, dl, TLI.getPointerTy(DAG.getDataLayout())));
2731  Results.push_back(DAG.getNode(ISD::ADD, dl, FA.getValueType(),
2732  FA, Offset));
2733  break;
2734  }
2735  case ISD::FLT_ROUNDS_:
2736  Results.push_back(DAG.getConstant(1, dl, Node->getValueType(0)));
2737  Results.push_back(Node->getOperand(0));
2738  break;
2739  case ISD::EH_RETURN:
2740  case ISD::EH_LABEL:
2741  case ISD::PREFETCH:
2742  case ISD::VAEND:
2743  case ISD::EH_SJLJ_LONGJMP:
2744  // If the target didn't expand these, there's nothing to do, so just
2745  // preserve the chain and be done.
2746  Results.push_back(Node->getOperand(0));
2747  break;
2748  case ISD::READCYCLECOUNTER:
2749  // If the target didn't expand this, just return 'zero' and preserve the
2750  // chain.
2751  Results.append(Node->getNumValues() - 1,
2752  DAG.getConstant(0, dl, Node->getValueType(0)));
2753  Results.push_back(Node->getOperand(0));
2754  break;
2755  case ISD::EH_SJLJ_SETJMP:
2756  // If the target didn't expand this, just return 'zero' and preserve the
2757  // chain.
2758  Results.push_back(DAG.getConstant(0, dl, MVT::i32));
2759  Results.push_back(Node->getOperand(0));
2760  break;
2761  case ISD::ATOMIC_LOAD: {
2762  // There is no libcall for atomic load; fake it with ATOMIC_CMP_SWAP.
2763  SDValue Zero = DAG.getConstant(0, dl, Node->getValueType(0));
2764  SDVTList VTs = DAG.getVTList(Node->getValueType(0), MVT::Other);
2765  SDValue Swap = DAG.getAtomicCmpSwap(
2766  ISD::ATOMIC_CMP_SWAP, dl, cast<AtomicSDNode>(Node)->getMemoryVT(), VTs,
2767  Node->getOperand(0), Node->getOperand(1), Zero, Zero,
2768  cast<AtomicSDNode>(Node)->getMemOperand());
2769  Results.push_back(Swap.getValue(0));
2770  Results.push_back(Swap.getValue(1));
2771  break;
2772  }
2773  case ISD::ATOMIC_STORE: {
2774  // There is no libcall for atomic store; fake it with ATOMIC_SWAP.
2775  SDValue Swap = DAG.getAtomic(ISD::ATOMIC_SWAP, dl,
2776  cast<AtomicSDNode>(Node)->getMemoryVT(),
2777  Node->getOperand(0),
2778  Node->getOperand(1), Node->getOperand(2),
2779  cast<AtomicSDNode>(Node)->getMemOperand());
2780  Results.push_back(Swap.getValue(1));
2781  break;
2782  }
2784  // Expanding an ATOMIC_CMP_SWAP_WITH_SUCCESS produces an ATOMIC_CMP_SWAP and
2785  // splits out the success value as a comparison. Expanding the resulting
2786  // ATOMIC_CMP_SWAP will produce a libcall.
2787  SDVTList VTs = DAG.getVTList(Node->getValueType(0), MVT::Other);
2788  SDValue Res = DAG.getAtomicCmpSwap(
2789  ISD::ATOMIC_CMP_SWAP, dl, cast<AtomicSDNode>(Node)->getMemoryVT(), VTs,
2790  Node->getOperand(0), Node->getOperand(1), Node->getOperand(2),
2791  Node->getOperand(3), cast<MemSDNode>(Node)->getMemOperand());
2792 
2793  SDValue ExtRes = Res;
2794  SDValue LHS = Res;
2795  SDValue RHS = Node->getOperand(1);
2796 
2797  EVT AtomicType = cast<AtomicSDNode>(Node)->getMemoryVT();
2798  EVT OuterType = Node->getValueType(0);
2799  switch (TLI.getExtendForAtomicOps()) {
2800  case ISD::SIGN_EXTEND:
2801  LHS = DAG.getNode(ISD::AssertSext, dl, OuterType, Res,
2802  DAG.getValueType(AtomicType));
2803  RHS = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, OuterType,
2804  Node->getOperand(2), DAG.getValueType(AtomicType));
2805  ExtRes = LHS;
2806  break;
2807  case ISD::ZERO_EXTEND:
2808  LHS = DAG.getNode(ISD::AssertZext, dl, OuterType, Res,
2809  DAG.getValueType(AtomicType));
2810  RHS = DAG.getZeroExtendInReg(Node->getOperand(2), dl, AtomicType);
2811  ExtRes = LHS;
2812  break;
2813  case ISD::ANY_EXTEND:
2814  LHS = DAG.getZeroExtendInReg(Res, dl, AtomicType);
2815  RHS = DAG.getZeroExtendInReg(Node->getOperand(2), dl, AtomicType);
2816  break;
2817  default:
2818  llvm_unreachable("Invalid atomic op extension");
2819  }
2820 
2821  SDValue Success =
2822  DAG.getSetCC(dl, Node->getValueType(1), LHS, RHS, ISD::SETEQ);
2823 
2824  Results.push_back(ExtRes.getValue(0));
2825  Results.push_back(Success);
2826  Results.push_back(Res.getValue(1));
2827  break;
2828  }
2830  ExpandDYNAMIC_STACKALLOC(Node, Results);
2831  break;
2832  case ISD::MERGE_VALUES:
2833  for (unsigned i = 0; i < Node->getNumValues(); i++)
2834  Results.push_back(Node->getOperand(i));
2835  break;
2836  case ISD::UNDEF: {
2837  EVT VT = Node->getValueType(0);
2838  if (VT.isInteger())
2839  Results.push_back(DAG.getConstant(0, dl, VT));
2840  else {
2841  assert(VT.isFloatingPoint() && "Unknown value type!");
2842  Results.push_back(DAG.getConstantFP(0, dl, VT));
2843  }
2844  break;
2845  }
2846  case ISD::STRICT_FP_ROUND:
2847  // When strict mode is enforced we can't do expansion because it
2848  // does not honor the "strict" properties. Only libcall is allowed.
2849  if (TLI.isStrictFPEnabled())
2850  break;
2851  // We might as well mutate to FP_ROUND when FP_ROUND operation is legal
2852  // since this operation is more efficient than stack operation.
2853  if (TLI.getStrictFPOperationAction(Node->getOpcode(),
2854  Node->getValueType(0))
2856  break;
2857  // We fall back to use stack operation when the FP_ROUND operation
2858  // isn't available.
2859  if ((Tmp1 = EmitStackConvert(Node->getOperand(1), Node->getValueType(0),
2860  Node->getValueType(0), dl,
2861  Node->getOperand(0)))) {
2862  ReplaceNode(Node, Tmp1.getNode());
2863  LLVM_DEBUG(dbgs() << "Successfully expanded STRICT_FP_ROUND node\n");
2864  return true;
2865  }
2866  break;
2867  case ISD::FP_ROUND:
2868  case ISD::BITCAST:
2869  if ((Tmp1 = EmitStackConvert(Node->getOperand(0), Node->getValueType(0),
2870  Node->getValueType(0), dl)))
2871  Results.push_back(Tmp1);
2872  break;
2873  case ISD::STRICT_FP_EXTEND:
2874  // When strict mode is enforced we can't do expansion because it
2875  // does not honor the "strict" properties. Only libcall is allowed.
2876  if (TLI.isStrictFPEnabled())
2877  break;
2878  // We might as well mutate to FP_EXTEND when FP_EXTEND operation is legal
2879  // since this operation is more efficient than stack operation.
2880  if (TLI.getStrictFPOperationAction(Node->getOpcode(),
2881  Node->getValueType(0))
2883  break;
2884  // We fall back to use stack operation when the FP_EXTEND operation
2885  // isn't available.
2886  if ((Tmp1 = EmitStackConvert(
2887  Node->getOperand(1), Node->getOperand(1).getValueType(),
2888  Node->getValueType(0), dl, Node->getOperand(0)))) {
2889  ReplaceNode(Node, Tmp1.getNode());
2890  LLVM_DEBUG(dbgs() << "Successfully expanded STRICT_FP_EXTEND node\n");
2891  return true;
2892  }
2893  break;
2894  case ISD::FP_EXTEND:
2895  if ((Tmp1 = EmitStackConvert(Node->getOperand(0),
2896  Node->getOperand(0).getValueType(),
2897  Node->getValueType(0), dl)))
2898  Results.push_back(Tmp1);
2899  break;
2900  case ISD::SIGN_EXTEND_INREG: {
2901  EVT ExtraVT = cast<VTSDNode>(Node->getOperand(1))->getVT();
2902  EVT VT = Node->getValueType(0);
2903 
2904  // An in-register sign-extend of a boolean is a negation:
2905  // 'true' (1) sign-extended is -1.
2906  // 'false' (0) sign-extended is 0.
2907  // However, we must mask the high bits of the source operand because the
2908  // SIGN_EXTEND_INREG does not guarantee that the high bits are already zero.
2909 
2910  // TODO: Do this for vectors too?
2911  if (ExtraVT.getSizeInBits() == 1) {
2912  SDValue One = DAG.getConstant(1, dl, VT);
2913  SDValue And = DAG.getNode(ISD::AND, dl, VT, Node->getOperand(0), One);
2914  SDValue Zero = DAG.getConstant(0, dl, VT);
2915  SDValue Neg = DAG.getNode(ISD::SUB, dl, VT, Zero, And);
2916  Results.push_back(Neg);
2917  break;
2918  }
2919 
2920  // NOTE: we could fall back on load/store here too for targets without
2921  // SRA. However, it is doubtful that any exist.
2922  EVT ShiftAmountTy = TLI.getShiftAmountTy(VT, DAG.getDataLayout());
2923  unsigned BitsDiff = VT.getScalarSizeInBits() -
2924  ExtraVT.getScalarSizeInBits();
2925  SDValue ShiftCst = DAG.getConstant(BitsDiff, dl, ShiftAmountTy);
2926  Tmp1 = DAG.getNode(ISD::SHL, dl, Node->getValueType(0),
2927  Node->getOperand(0), ShiftCst);
2928  Tmp1 = DAG.getNode(ISD::SRA, dl, Node->getValueType(0), Tmp1, ShiftCst);
2929  Results.push_back(Tmp1);
2930  break;
2931  }
2932  case ISD::UINT_TO_FP:
2934  if (TLI.expandUINT_TO_FP(Node, Tmp1, Tmp2, DAG)) {
2935  Results.push_back(Tmp1);
2936  if (Node->isStrictFPOpcode())
2937  Results.push_back(Tmp2);
2938  break;
2939  }
2941  case ISD::SINT_TO_FP:
2943  if ((Tmp1 = ExpandLegalINT_TO_FP(Node, Tmp2))) {
2944  Results.push_back(Tmp1);
2945  if (Node->isStrictFPOpcode())
2946  Results.push_back(Tmp2);
2947  }
2948  break;
2949  case ISD::FP_TO_SINT:
2950  if (TLI.expandFP_TO_SINT(Node, Tmp1, DAG))
2951  Results.push_back(Tmp1);
2952  break;
2954  if (TLI.expandFP_TO_SINT(Node, Tmp1, DAG)) {
2955  ReplaceNode(Node, Tmp1.getNode());
2956  LLVM_DEBUG(dbgs() << "Successfully expanded STRICT_FP_TO_SINT node\n");
2957  return true;
2958  }
2959  break;
2960  case ISD::FP_TO_UINT:
2961  if (TLI.expandFP_TO_UINT(Node, Tmp1, Tmp2, DAG))
2962  Results.push_back(Tmp1);
2963  break;
2965  if (TLI.expandFP_TO_UINT(Node, Tmp1, Tmp2, DAG)) {
2966  // Relink the chain.
2967  DAG.ReplaceAllUsesOfValueWith(SDValue(Node,1), Tmp2);
2968  // Replace the new UINT result.
2969  ReplaceNodeWithValue(SDValue(Node, 0), Tmp1);
2970  LLVM_DEBUG(dbgs() << "Successfully expanded STRICT_FP_TO_UINT node\n");
2971  return true;
2972  }
2973  break;
2974  case ISD::FP_TO_SINT_SAT:
2975  case ISD::FP_TO_UINT_SAT:
2976  Results.push_back(TLI.expandFP_TO_INT_SAT(Node, DAG));
2977  break;
2978  case ISD::VAARG:
2979  Results.push_back(DAG.expandVAArg(Node));
2980  Results.push_back(Results[0].getValue(1));
2981  break;
2982  case ISD::VACOPY:
2983  Results.push_back(DAG.expandVACopy(Node));
2984  break;
2986  if (Node->getOperand(0).getValueType().getVectorNumElements() == 1)
2987  // This must be an access of the only element. Return it.
2988  Tmp1 = DAG.getNode(ISD::BITCAST, dl, Node->getValueType(0),
2989  Node->getOperand(0));
2990  else
2991  Tmp1 = ExpandExtractFromVectorThroughStack(SDValue(Node, 0));
2992  Results.push_back(Tmp1);
2993  break;
2995  Results.push_back(ExpandExtractFromVectorThroughStack(SDValue(Node, 0)));
2996  break;
2997  case ISD::INSERT_SUBVECTOR:
2998  Results.push_back(ExpandInsertToVectorThroughStack(SDValue(Node, 0)));
2999  break;
3000  case ISD::CONCAT_VECTORS:
3001  Results.push_back(ExpandVectorBuildThroughStack(Node));
3002  break;
3003  case ISD::SCALAR_TO_VECTOR:
3004  Results.push_back(ExpandSCALAR_TO_VECTOR(Node));
3005  break;
3007  Results.push_back(ExpandINSERT_VECTOR_ELT(Node->getOperand(0),
3008  Node->getOperand(1),
3009  Node->getOperand(2), dl));
3010  break;
3011  case ISD::VECTOR_SHUFFLE: {
3012  SmallVector<int, 32> NewMask;
3013  ArrayRef<int> Mask = cast<ShuffleVectorSDNode>(Node)->getMask();
3014 
3015  EVT VT = Node->getValueType(0);
3016  EVT EltVT = VT.getVectorElementType();
3017  SDValue Op0 = Node->getOperand(0);
3018  SDValue Op1 = Node->getOperand(1);
3019  if (!TLI.isTypeLegal(EltVT)) {
3020  EVT NewEltVT = TLI.getTypeToTransformTo(*DAG.getContext(), EltVT);
3021 
3022  // BUILD_VECTOR operands are allowed to be wider than the element type.
3023  // But if NewEltVT is smaller that EltVT the BUILD_VECTOR does not accept
3024  // it.
3025  if (NewEltVT.bitsLT(EltVT)) {
3026  // Convert shuffle node.
3027  // If original node was v4i64 and the new EltVT is i32,
3028  // cast operands to v8i32 and re-build the mask.
3029 
3030  // Calculate new VT, the size of the new VT should be equal to original.
3031  EVT NewVT =
3032  EVT::getVectorVT(*DAG.getContext(), NewEltVT,
3033  VT.getSizeInBits() / NewEltVT.getSizeInBits());
3034  assert(NewVT.bitsEq(VT));
3035 
3036  // cast operands to new VT
3037  Op0 = DAG.getNode(ISD::BITCAST, dl, NewVT, Op0);
3038  Op1 = DAG.getNode(ISD::BITCAST, dl, NewVT, Op1);
3039 
3040  // Convert the shuffle mask
3041  unsigned int factor =
3043 
3044  // EltVT gets smaller
3045  assert(factor > 0);
3046 
3047  for (unsigned i = 0; i < VT.getVectorNumElements(); ++i) {
3048  if (Mask[i] < 0) {
3049  for (unsigned fi = 0; fi < factor; ++fi)
3050  NewMask.push_back(Mask[i]);
3051  }
3052  else {
3053  for (unsigned fi = 0; fi < factor; ++fi)
3054  NewMask.push_back(Mask[i]*factor+fi);
3055  }
3056  }
3057  Mask = NewMask;
3058  VT = NewVT;
3059  }
3060  EltVT = NewEltVT;
3061  }
3062  unsigned NumElems = VT.getVectorNumElements();
3064  for (unsigned i = 0; i != NumElems; ++i) {
3065  if (Mask[i] < 0) {
3066  Ops.push_back(DAG.getUNDEF(EltVT));
3067  continue;
3068  }
3069  unsigned Idx = Mask[i];
3070  if (Idx < NumElems)
3071  Ops.push_back(DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT, Op0,
3072  DAG.getVectorIdxConstant(Idx, dl)));
3073  else
3074  Ops.push_back(
3075  DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT, Op1,
3076  DAG.getVectorIdxConstant(Idx - NumElems, dl)));
3077  }
3078 
3079  Tmp1 = DAG.getBuildVector(VT, dl, Ops);
3080  // We may have changed the BUILD_VECTOR type. Cast it back to the Node type.
3081  Tmp1 = DAG.getNode(ISD::BITCAST, dl, Node->getValueType(0), Tmp1);
3082  Results.push_back(Tmp1);
3083  break;
3084  }
3085  case ISD::VECTOR_SPLICE: {
3086  Results.push_back(TLI.expandVectorSplice(Node, DAG));
3087  break;
3088  }
3089  case ISD::EXTRACT_ELEMENT: {
3090  EVT OpTy = Node->getOperand(0).getValueType();
3091  if (cast<ConstantSDNode>(Node->getOperand(1))->getZExtValue()) {
3092  // 1 -> Hi
3093  Tmp1 = DAG.getNode(ISD::SRL, dl, OpTy, Node->getOperand(0),
3094  DAG.getConstant(OpTy.getSizeInBits() / 2, dl,
3095  TLI.getShiftAmountTy(
3096  Node->getOperand(0).getValueType(),
3097  DAG.getDataLayout())));
3098  Tmp1 = DAG.getNode(ISD::TRUNCATE, dl, Node->getValueType(0), Tmp1);
3099  } else {
3100  // 0 -> Lo
3101  Tmp1 = DAG.getNode(ISD::TRUNCATE, dl, Node->getValueType(0),
3102  Node->getOperand(0));
3103  }
3104  Results.push_back(Tmp1);
3105  break;
3106  }
3107  case ISD::STACKSAVE:
3108  // Expand to CopyFromReg if the target set
3109  // StackPointerRegisterToSaveRestore.
3110  if (Register SP = TLI.getStackPointerRegisterToSaveRestore()) {
3111  Results.push_back(DAG.getCopyFromReg(Node->getOperand(0), dl, SP,
3112  Node->getValueType(0)));
3113  Results.push_back(Results[0].getValue(1));
3114  } else {
3115  Results.push_back(DAG.getUNDEF(Node->getValueType(0)));
3116  Results.push_back(Node->getOperand(0));
3117  }
3118  break;
3119  case ISD::STACKRESTORE:
3120  // Expand to CopyToReg if the target set
3121  // StackPointerRegisterToSaveRestore.
3122  if (Register SP = TLI.getStackPointerRegisterToSaveRestore()) {
3123  Results.push_back(DAG.getCopyToReg(Node->getOperand(0), dl, SP,
3124  Node->getOperand(1)));
3125  } else {
3126  Results.push_back(Node->getOperand(0));
3127  }
3128  break;
3130  Results.push_back(DAG.getConstant(0, dl, Node->getValueType(0)));
3131  Results.push_back(Results[0].getValue(0));
3132  break;
3133  case ISD::FCOPYSIGN:
3134  Results.push_back(ExpandFCOPYSIGN(Node));
3135  break;
3136  case ISD::FNEG:
3137  Results.push_back(ExpandFNEG(Node));
3138  break;
3139  case ISD::FABS:
3140  Results.push_back(ExpandFABS(Node));
3141  break;
3142  case ISD::SMIN:
3143  case ISD::SMAX:
3144  case ISD::UMIN:
3145  case ISD::UMAX: {
3146  // Expand Y = MAX(A, B) -> Y = (A > B) ? A : B
3147  ISD::CondCode Pred;
3148  switch (Node->getOpcode()) {
3149  default: llvm_unreachable("How did we get here?");
3150  case ISD::SMAX: Pred = ISD::SETGT; break;
3151  case ISD::SMIN: Pred = ISD::SETLT; break;
3152  case ISD::UMAX: Pred = ISD::SETUGT; break;
3153  case ISD::UMIN: Pred = ISD::SETULT; break;
3154  }
3155  Tmp1 = Node->getOperand(0);
3156  Tmp2 = Node->getOperand(1);
3157  Tmp1 = DAG.getSelectCC(dl, Tmp1, Tmp2, Tmp1, Tmp2, Pred);
3158  Results.push_back(Tmp1);
3159  break;
3160  }
3161  case ISD::FMINNUM:
3162  case ISD::FMAXNUM: {
3163  if (SDValue Expanded = TLI.expandFMINNUM_FMAXNUM(Node, DAG))
3164  Results.push_back(Expanded);
3165  break;
3166  }
3167  case ISD::FSIN:
3168  case ISD::FCOS: {
3169  EVT VT = Node->getValueType(0);
3170  // Turn fsin / fcos into ISD::FSINCOS node if there are a pair of fsin /
3171  // fcos which share the same operand and both are used.
3172  if ((TLI.isOperationLegalOrCustom(ISD::FSINCOS, VT) ||
3173  isSinCosLibcallAvailable(Node, TLI))
3174  && useSinCos(Node)) {
3175  SDVTList VTs = DAG.getVTList(VT, VT);
3176  Tmp1 = DAG.getNode(ISD::FSINCOS, dl, VTs, Node->getOperand(0));
3177  if (Node->getOpcode() == ISD::FCOS)
3178  Tmp1 = Tmp1.getValue(1);
3179  Results.push_back(Tmp1);
3180  }
3181  break;
3182  }
3183  case ISD::FMAD:
3184  llvm_unreachable("Illegal fmad should never be formed");
3185 
3186  case ISD::FP16_TO_FP:
3187  if (Node->getValueType(0) != MVT::f32) {
3188  // We can extend to types bigger than f32 in two steps without changing
3189  // the result. Since "f16 -> f32" is much more commonly available, give
3190  // CodeGen the option of emitting that before resorting to a libcall.
3191  SDValue Res =
3192  DAG.getNode(ISD::FP16_TO_FP, dl, MVT::f32, Node->getOperand(0));
3193  Results.push_back(
3194  DAG.getNode(ISD::FP_EXTEND, dl, Node->getValueType(0), Res));
3195  }
3196  break;
3198  if (Node->getValueType(0) != MVT::f32) {
3199  // We can extend to types bigger than f32 in two steps without changing
3200  // the result. Since "f16 -> f32" is much more commonly available, give
3201  // CodeGen the option of emitting that before resorting to a libcall.
3202  SDValue Res =
3204  {Node->getOperand(0), Node->getOperand(1)});
3205  Res = DAG.getNode(ISD::STRICT_FP_EXTEND, dl,
3206  {Node->getValueType(0), MVT::Other},
3207  {Res.getValue(1), Res});
3208  Results.push_back(Res);
3209  Results.push_back(Res.getValue(1));
3210  }
3211  break;
3212  case ISD::FP_TO_FP16:
3213  LLVM_DEBUG(dbgs() << "Legalizing FP_TO_FP16\n");
3214  if (!TLI.useSoftFloat() && TM.Options.UnsafeFPMath) {
3215  SDValue Op = Node->getOperand(0);
3216  MVT SVT = Op.getSimpleValueType();
3217  if ((SVT == MVT::f64 || SVT == MVT::f80) &&
3218  TLI.isOperationLegalOrCustom(ISD::FP_TO_FP16, MVT::f32)) {
3219  // Under fastmath, we can expand this node into a fround followed by
3220  // a float-half conversion.
3221  SDValue FloatVal = DAG.getNode(ISD::FP_ROUND, dl, MVT::f32, Op,
3222  DAG.getIntPtrConstant(0, dl));
3223  Results.push_back(
3224  DAG.getNode(ISD::FP_TO_FP16, dl, Node->getValueType(0), FloatVal));
3225  }
3226  }
3227  break;
3228  case ISD::ConstantFP: {
3229  ConstantFPSDNode *CFP = cast<ConstantFPSDNode>(Node);
3230  // Check to see if this FP immediate is already legal.
3231  // If this is a legal constant, turn it into a TargetConstantFP node.
3232  if (!TLI.isFPImmLegal(CFP->getValueAPF(), Node->getValueType(0),
3233  DAG.shouldOptForSize()))
3234  Results.push_back(ExpandConstantFP(CFP, true));
3235  break;
3236  }
3237  case ISD::Constant: {
3238  ConstantSDNode *CP = cast<ConstantSDNode>(Node);
3239  Results.push_back(ExpandConstant(CP));
3240  break;
3241  }
3242  case ISD::FSUB: {
3243  EVT VT = Node->getValueType(0);
3244  if (TLI.isOperationLegalOrCustom(ISD::FADD, VT) &&
3245  TLI.isOperationLegalOrCustom(ISD::FNEG, VT)) {
3246  const SDNodeFlags Flags = Node->getFlags();
3247  Tmp1 = DAG.getNode(ISD::FNEG, dl, VT, Node->getOperand(1));
3248  Tmp1 = DAG.getNode(ISD::FADD, dl, VT, Node->getOperand(0), Tmp1, Flags);
3249  Results.push_back(Tmp1);
3250  }
3251  break;
3252  }
3253  case ISD::SUB: {
3254  EVT VT = Node->getValueType(0);
3255  assert(TLI.isOperationLegalOrCustom(ISD::ADD, VT) &&
3256  TLI.isOperationLegalOrCustom(ISD::XOR, VT) &&
3257  "Don't know how to expand this subtraction!");
3258  Tmp1 = DAG.getNOT(dl, Node->getOperand(1), VT);
3259  Tmp1 = DAG.getNode(ISD::ADD, dl, VT, Tmp1, DAG.getConstant(1, dl, VT));
3260  Results.push_back(DAG.getNode(ISD::ADD, dl, VT, Node->getOperand(0), Tmp1));
3261  break;
3262  }
3263  case ISD::UREM:
3264  case ISD::SREM:
3265  if (TLI.expandREM(Node, Tmp1, DAG))
3266  Results.push_back(Tmp1);
3267  break;
3268  case ISD::UDIV:
3269  case ISD::SDIV: {
3270  bool isSigned = Node->getOpcode() == ISD::SDIV;
3271  unsigned DivRemOpc = isSigned ? ISD::SDIVREM : ISD::UDIVREM;
3272  EVT VT = Node->getValueType(0);
3273  if (TLI.isOperationLegalOrCustom(DivRemOpc, VT)) {
3274  SDVTList VTs = DAG.getVTList(VT, VT);
3275  Tmp1 = DAG.getNode(DivRemOpc, dl, VTs, Node->getOperand(0),
3276  Node->getOperand(1));
3277  Results.push_back(Tmp1);
3278  }
3279  break;
3280  }
3281  case ISD::MULHU:
3282  case ISD::MULHS: {
3283  unsigned ExpandOpcode =
3284  Node->getOpcode() == ISD::MULHU ? ISD::UMUL_LOHI : ISD::SMUL_LOHI;
3285  EVT VT = Node->getValueType(0);
3286  SDVTList VTs = DAG.getVTList(VT, VT);
3287 
3288  Tmp1 = DAG.getNode(ExpandOpcode, dl, VTs, Node->getOperand(0),
3289  Node->getOperand(1));
3290  Results.push_back(Tmp1.getValue(1));
3291  break;
3292  }
3293  case ISD::UMUL_LOHI:
3294  case ISD::SMUL_LOHI: {
3295  SDValue LHS = Node->getOperand(0);
3296  SDValue RHS = Node->getOperand(1);
3297  MVT VT = LHS.getSimpleValueType();
3298  unsigned MULHOpcode =
3299  Node->getOpcode() == ISD::UMUL_LOHI ? ISD::MULHU : ISD::MULHS;
3300 
3301  if (TLI.isOperationLegalOrCustom(MULHOpcode, VT)) {
3302  Results.push_back(DAG.getNode(ISD::MUL, dl, VT, LHS, RHS));
3303  Results.push_back(DAG.getNode(MULHOpcode, dl, VT, LHS, RHS));
3304  break;
3305  }
3306 
3307  SmallVector<SDValue, 4> Halves;
3308  EVT HalfType = EVT(VT).getHalfSizedIntegerVT(*DAG.getContext());
3309  assert(TLI.isTypeLegal(HalfType));
3310  if (TLI.expandMUL_LOHI(Node->getOpcode(), VT, dl, LHS, RHS, Halves,
3311  HalfType, DAG,
3313  for (unsigned i = 0; i < 2; ++i) {
3314  SDValue Lo = DAG.getNode(ISD::ZERO_EXTEND, dl, VT, Halves[2 * i]);
3315  SDValue Hi = DAG.getNode(ISD::ANY_EXTEND, dl, VT, Halves[2 * i + 1]);
3316  SDValue Shift = DAG.getConstant(
3317  HalfType.getScalarSizeInBits(), dl,
3318  TLI.getShiftAmountTy(HalfType, DAG.getDataLayout()));
3319  Hi = DAG.getNode(ISD::SHL, dl, VT, Hi, Shift);
3320  Results.push_back(DAG.getNode(ISD::OR, dl, VT, Lo, Hi));
3321  }
3322  break;
3323  }
3324  break;
3325  }
3326  case ISD::MUL: {
3327  EVT VT = Node->getValueType(0);
3328  SDVTList VTs = DAG.getVTList(VT, VT);
3329  // See if multiply or divide can be lowered using two-result operations.
3330  // We just need the low half of the multiply; try both the signed
3331  // and unsigned forms. If the target supports both SMUL_LOHI and
3332  // UMUL_LOHI, form a preference by checking which forms of plain
3333  // MULH it supports.
3334  bool HasSMUL_LOHI = TLI.isOperationLegalOrCustom(ISD::SMUL_LOHI, VT);
3335  bool HasUMUL_LOHI = TLI.isOperationLegalOrCustom(ISD::UMUL_LOHI, VT);
3336  bool HasMULHS = TLI.isOperationLegalOrCustom(ISD::MULHS, VT);
3337  bool HasMULHU = TLI.isOperationLegalOrCustom(ISD::MULHU, VT);
3338  unsigned OpToUse = 0;
3339  if (HasSMUL_LOHI && !HasMULHS) {
3340  OpToUse = ISD::SMUL_LOHI;
3341  } else if (HasUMUL_LOHI && !HasMULHU) {
3342  OpToUse = ISD::UMUL_LOHI;
3343  } else if (HasSMUL_LOHI) {
3344  OpToUse = ISD::SMUL_LOHI;
3345  } else if (HasUMUL_LOHI) {
3346  OpToUse = ISD::UMUL_LOHI;
3347  }
3348  if (OpToUse) {
3349  Results.push_back(DAG.getNode(OpToUse, dl, VTs, Node->getOperand(0),
3350  Node->getOperand(1)));
3351  break;
3352  }
3353 
3354  SDValue Lo, Hi;
3355  EVT HalfType = VT.getHalfSizedIntegerVT(*DAG.getContext());
3356  if (TLI.isOperationLegalOrCustom(ISD::ZERO_EXTEND, VT) &&
3357  TLI.isOperationLegalOrCustom(ISD::ANY_EXTEND, VT) &&
3358  TLI.isOperationLegalOrCustom(ISD::SHL, VT) &&
3359  TLI.isOperationLegalOrCustom(ISD::OR, VT) &&
3360  TLI.expandMUL(Node, Lo, Hi, HalfType, DAG,
3362  Lo = DAG.getNode(ISD::ZERO_EXTEND, dl, VT, Lo);
3363  Hi = DAG.getNode(ISD::ANY_EXTEND, dl, VT, Hi);
3364  SDValue Shift =
3365  DAG.getConstant(HalfType.getSizeInBits(), dl,
3366  TLI.getShiftAmountTy(HalfType, DAG.getDataLayout()));
3367  Hi = DAG.getNode(ISD::SHL, dl, VT, Hi, Shift);
3368  Results.push_back(DAG.getNode(ISD::OR, dl, VT, Lo, Hi));
3369  }
3370  break;
3371  }
3372  case ISD::FSHL:
3373  case ISD::FSHR:
3374  if (TLI.expandFunnelShift(Node, Tmp1, DAG))
3375  Results.push_back(Tmp1);
3376  break;
3377  case ISD::ROTL:
3378  case ISD::ROTR:
3379  if (TLI.expandROT(Node, true /*AllowVectorOps*/, Tmp1, DAG))
3380  Results.push_back(Tmp1);
3381  break;
3382  case ISD::SADDSAT:
3383  case ISD::UADDSAT:
3384  case ISD::SSUBSAT:
3385  case ISD::USUBSAT:
3386  Results.push_back(TLI.expandAddSubSat(Node, DAG));
3387  break;
3388  case ISD::SSHLSAT:
3389  case ISD::USHLSAT:
3390  Results.push_back(TLI.expandShlSat(Node, DAG));
3391  break;
3392  case ISD::SMULFIX:
3393  case ISD::SMULFIXSAT:
3394  case ISD::UMULFIX:
3395  case ISD::UMULFIXSAT:
3396  Results.push_back(TLI.expandFixedPointMul(Node, DAG));
3397  break;
3398  case ISD::SDIVFIX:
3399  case ISD::SDIVFIXSAT:
3400  case ISD::UDIVFIX:
3401  case ISD::UDIVFIXSAT:
3402  if (SDValue V = TLI.expandFixedPointDiv(Node->getOpcode(), SDLoc(Node),
3403  Node->getOperand(0),
3404  Node->getOperand(1),
3405  Node->getConstantOperandVal(2),
3406  DAG)) {
3407  Results.push_back(V);
3408  break;
3409  }
3410  // FIXME: We might want to retry here with a wider type if we fail, if that
3411  // type is legal.
3412  // FIXME: Technically, so long as we only have sdivfixes where BW+Scale is
3413  // <= 128 (which is the case for all of the default Embedded-C types),
3414  // we will only get here with types and scales that we could always expand
3415  // if we were allowed to generate libcalls to division functions of illegal
3416  // type. But we cannot do that.
3417  llvm_unreachable("Cannot expand DIVFIX!");
3418  case ISD::ADDCARRY:
3419  case ISD::SUBCARRY: {
3420  SDValue LHS = Node->getOperand(0);
3421  SDValue RHS = Node->getOperand(1);
3422  SDValue Carry = Node->getOperand(2);
3423 
3424  bool IsAdd = Node->getOpcode() == ISD::ADDCARRY;
3425 
3426  // Initial add of the 2 operands.
3427  unsigned Op = IsAdd ? ISD::ADD : ISD::SUB;
3428  EVT VT = LHS.getValueType();
3429  SDValue Sum = DAG.getNode(Op, dl, VT, LHS, RHS);
3430 
3431  // Initial check for overflow.
3432  EVT CarryType = Node->getValueType(1);
3433  EVT SetCCType = getSetCCResultType(Node->getValueType(0));
3434  ISD::CondCode CC = IsAdd ? ISD::SETULT : ISD::SETUGT;
3435  SDValue Overflow = DAG.getSetCC(dl, SetCCType, Sum, LHS, CC);
3436 
3437  // Add of the sum and the carry.
3438  SDValue One = DAG.getConstant(1, dl, VT);
3439  SDValue CarryExt =
3440  DAG.getNode(ISD::AND, dl, VT, DAG.getZExtOrTrunc(Carry, dl, VT), One);
3441  SDValue Sum2 = DAG.getNode(Op, dl, VT, Sum, CarryExt);
3442 
3443  // Second check for overflow. If we are adding, we can only overflow if the
3444  // initial sum is all 1s ang the carry is set, resulting in a new sum of 0.
3445  // If we are subtracting, we can only overflow if the initial sum is 0 and
3446  // the carry is set, resulting in a new sum of all 1s.
3447  SDValue Zero = DAG.getConstant(0, dl, VT);
3448  SDValue Overflow2 =
3449  IsAdd ? DAG.getSetCC(dl, SetCCType, Sum2, Zero, ISD::SETEQ)
3450  : DAG.getSetCC(dl, SetCCType, Sum, Zero, ISD::SETEQ);
3451  Overflow2 = DAG.getNode(ISD::AND, dl, SetCCType, Overflow2,
3452  DAG.getZExtOrTrunc(Carry, dl, SetCCType));
3453 
3454  SDValue ResultCarry =
3455  DAG.getNode(ISD::OR, dl, SetCCType, Overflow, Overflow2);
3456 
3457  Results.push_back(Sum2);
3458  Results.push_back(DAG.getBoolExtOrTrunc(ResultCarry, dl, CarryType, VT));
3459  break;
3460  }
3461  case ISD::SADDO:
3462  case ISD::SSUBO: {
3463  SDValue Result, Overflow;
3464  TLI.expandSADDSUBO(Node, Result, Overflow, DAG);
3465  Results.push_back(Result);
3466  Results.push_back(Overflow);
3467  break;
3468  }
3469  case ISD::UADDO:
3470  case ISD::USUBO: {
3471  SDValue Result, Overflow;
3472  TLI.expandUADDSUBO(Node, Result, Overflow, DAG);
3473  Results.push_back(Result);
3474  Results.push_back(Overflow);
3475  break;
3476  }
3477  case ISD::UMULO:
3478  case ISD::SMULO: {
3479  SDValue Result, Overflow;
3480  if (TLI.expandMULO(Node, Result, Overflow, DAG)) {
3481  Results.push_back(Result);
3482  Results.push_back(Overflow);
3483  }
3484  break;
3485  }
3486  case ISD::BUILD_PAIR: {
3487  EVT PairTy = Node->getValueType(0);
3488  Tmp1 = DAG.getNode(ISD::ZERO_EXTEND, dl, PairTy, Node->getOperand(0));
3489  Tmp2 = DAG.getNode(ISD::ANY_EXTEND, dl, PairTy, Node->getOperand(1));
3490  Tmp2 = DAG.getNode(
3491  ISD::SHL, dl, PairTy, Tmp2,
3492  DAG.getConstant(PairTy.getSizeInBits() / 2, dl,
3493  TLI.getShiftAmountTy(PairTy, DAG.getDataLayout())));
3494  Results.push_back(DAG.getNode(ISD::OR, dl, PairTy, Tmp1, Tmp2));
3495  break;
3496  }
3497  case ISD::SELECT:
3498  Tmp1 = Node->getOperand(0);
3499  Tmp2 = Node->getOperand(1);
3500  Tmp3 = Node->getOperand(2);
3501  if (Tmp1.getOpcode() == ISD::SETCC) {
3502  Tmp1 = DAG.getSelectCC(dl, Tmp1.getOperand(0), Tmp1.getOperand(1),
3503  Tmp2, Tmp3,
3504  cast<CondCodeSDNode>(Tmp1.getOperand(2))->get());
3505  } else {
3506  Tmp1 = DAG.getSelectCC(dl, Tmp1,
3507  DAG.getConstant(0, dl, Tmp1.getValueType()),
3508  Tmp2, Tmp3, ISD::SETNE);
3509  }
3510  Tmp1->setFlags(Node->getFlags());
3511  Results.push_back(Tmp1);
3512  break;
3513  case ISD::BR_JT: {
3514  SDValue Chain = Node->getOperand(0);
3515  SDValue Table = Node->getOperand(1);
3516  SDValue Index = Node->getOperand(2);
3517 
3518  const DataLayout &TD = DAG.getDataLayout();
3519  EVT PTy = TLI.getPointerTy(TD);
3520 
3521  unsigned EntrySize =
3522  DAG.getMachineFunction().getJumpTableInfo()->getEntrySize(TD);
3523 
3524  // For power-of-two jumptable entry sizes convert multiplication to a shift.
3525  // This transformation needs to be done here since otherwise the MIPS
3526  // backend will end up emitting a three instruction multiply sequence
3527  // instead of a single shift and MSP430 will call a runtime function.
3528  if (llvm::isPowerOf2_32(EntrySize))
3529  Index = DAG.getNode(
3530  ISD::SHL, dl, Index.getValueType(), Index,
3531  DAG.getConstant(llvm::Log2_32(EntrySize), dl, Index.getValueType()));
3532  else
3533  Index = DAG.getNode(ISD::MUL, dl, Index.getValueType(), Index,
3534  DAG.getConstant(EntrySize, dl, Index.getValueType()));
3535  SDValue Addr = DAG.getNode(ISD::ADD, dl, Index.getValueType(),
3536  Index, Table);
3537 
3538  EVT MemVT = EVT::getIntegerVT(*DAG.getContext(), EntrySize * 8);
3539  SDValue LD = DAG.getExtLoad(
3540  ISD::SEXTLOAD, dl, PTy, Chain, Addr,
3541  MachinePointerInfo::getJumpTable(DAG.getMachineFunction()), MemVT);
3542  Addr = LD;
3543  if (TLI.isJumpTableRelative()) {
3544  // For PIC, the sequence is:
3545  // BRIND(load(Jumptable + index) + RelocBase)
3546  // RelocBase can be JumpTable, GOT or some sort of global base.
3547  Addr = DAG.getNode(ISD::ADD, dl, PTy, Addr,
3548  TLI.getPICJumpTableRelocBase(Table, DAG));
3549  }
3550 
3551  Tmp1 = TLI.expandIndirectJTBranch(dl, LD.getValue(1), Addr, DAG);
3552  Results.push_back(Tmp1);
3553  break;
3554  }
3555  case ISD::BRCOND:
3556  // Expand brcond's setcc into its constituent parts and create a BR_CC
3557  // Node.
3558  Tmp1 = Node->getOperand(0);
3559  Tmp2 = Node->getOperand(1);
3560  if (Tmp2.getOpcode() == ISD::SETCC) {
3561  Tmp1 = DAG.getNode(ISD::BR_CC, dl, MVT::Other,
3562  Tmp1, Tmp2.getOperand(2),
3563  Tmp2.getOperand(0), Tmp2.getOperand(1),
3564  Node->getOperand(2));
3565  } else {
3566  // We test only the i1 bit. Skip the AND if UNDEF or another AND.
3567  if (Tmp2.isUndef() ||
3568  (Tmp2.getOpcode() == ISD::AND &&
3569  isa<ConstantSDNode>(Tmp2.getOperand(1)) &&
3570  cast<ConstantSDNode>(Tmp2.getOperand(1))->getZExtValue() == 1))
3571  Tmp3 = Tmp2;
3572  else
3573  Tmp3 = DAG.getNode(ISD::AND, dl, Tmp2.getValueType(), Tmp2,
3574  DAG.getConstant(1, dl, Tmp2.getValueType()));
3575  Tmp1 = DAG.getNode(ISD::BR_CC, dl, MVT::Other, Tmp1,
3576  DAG.getCondCode(ISD::SETNE), Tmp3,
3577  DAG.getConstant(0, dl, Tmp3.getValueType()),
3578  Node->getOperand(2));
3579  }
3580  Results.push_back(Tmp1);
3581  break;
3582  case ISD::SETCC:
3583  case ISD::STRICT_FSETCC:
3584  case ISD::STRICT_FSETCCS: {
3585  bool IsStrict = Node->getOpcode() != ISD::SETCC;
3586  bool IsSignaling = Node->getOpcode() == ISD::STRICT_FSETCCS;
3587  SDValue Chain = IsStrict ? Node->getOperand(0) : SDValue();
3588  unsigned Offset = IsStrict ? 1 : 0;
3589  Tmp1 = Node->getOperand(0 + Offset);
3590  Tmp2 = Node->getOperand(1 + Offset);
3591  Tmp3 = Node->getOperand(2 + Offset);
3592  bool Legalized =
3593  TLI.LegalizeSetCCCondCode(DAG, Node->getValueType(0), Tmp1, Tmp2, Tmp3,
3594  NeedInvert, dl, Chain, IsSignaling);
3595 
3596  if (Legalized) {
3597  // If we expanded the SETCC by swapping LHS and RHS, or by inverting the
3598  // condition code, create a new SETCC node.
3599  if (Tmp3.getNode())
3600  Tmp1 = DAG.getNode(ISD::SETCC, dl, Node->getValueType(0),
3601  Tmp1, Tmp2, Tmp3, Node->getFlags());
3602 
3603  // If we expanded the SETCC by inverting the condition code, then wrap
3604  // the existing SETCC in a NOT to restore the intended condition.
3605  if (NeedInvert)
3606  Tmp1 = DAG.getLogicalNOT(dl, Tmp1, Tmp1->getValueType(0));
3607 
3608  Results.push_back(Tmp1);
3609  if (IsStrict)
3610  Results.push_back(Chain);
3611 
3612  break;
3613  }
3614 
3615  // FIXME: It seems Legalized is false iff CCCode is Legal. I don't
3616  // understand if this code is useful for strict nodes.
3617  assert(!IsStrict && "Don't know how to expand for strict nodes.");
3618 
3619  // Otherwise, SETCC for the given comparison type must be completely
3620  // illegal; expand it into a SELECT_CC.
3621  EVT VT = Node->getValueType(0);
3622  int TrueValue;
3623  switch (TLI.getBooleanContents(Tmp1.getValueType())) {
3626  TrueValue = 1;
3627  break;
3629  TrueValue = -1;
3630  break;
3631  }
3632  Tmp1 = DAG.getNode(ISD::SELECT_CC, dl, VT, Tmp1, Tmp2,
3633  DAG.getConstant(TrueValue, dl, VT),
3634  DAG.getConstant(0, dl, VT),
3635  Tmp3);
3636  Tmp1->setFlags(Node->getFlags());
3637  Results.push_back(Tmp1);
3638  break;
3639  }
3640  case ISD::SELECT_CC: {
3641  // TODO: need to add STRICT_SELECT_CC and STRICT_SELECT_CCS
3642  Tmp1 = Node->getOperand(0); // LHS
3643  Tmp2 = Node->getOperand(1); // RHS
3644  Tmp3 = Node->getOperand(2); // True
3645  Tmp4 = Node->getOperand(3); // False
3646  EVT VT = Node->getValueType(0);
3647  SDValue Chain;
3648  SDValue CC = Node->getOperand(4);
3649  ISD::CondCode CCOp = cast<CondCodeSDNode>(CC)->get();
3650 
3651  if (TLI.isCondCodeLegalOrCustom(CCOp, Tmp1.getSimpleValueType())) {
3652  // If the condition code is legal, then we need to expand this
3653  // node using SETCC and SELECT.
3654  EVT CmpVT = Tmp1.getValueType();
3655  assert(!TLI.isOperationExpand(ISD::SELECT, VT) &&
3656  "Cannot expand ISD::SELECT_CC when ISD::SELECT also needs to be "
3657  "expanded.");
3658  EVT CCVT = getSetCCResultType(CmpVT);
3659  SDValue Cond = DAG.getNode(ISD::SETCC, dl, CCVT, Tmp1, Tmp2, CC, Node->getFlags());
3660  Results.push_back(DAG.getSelect(dl, VT, Cond, Tmp3, Tmp4));
3661  break;
3662  }
3663 
3664  // SELECT_CC is legal, so the condition code must not be.
3665  bool Legalized = false;
3666  // Try to legalize by inverting the condition. This is for targets that
3667  // might support an ordered version of a condition, but not the unordered
3668  // version (or vice versa).
3669  ISD::CondCode InvCC = ISD::getSetCCInverse(CCOp, Tmp1.getValueType());
3670  if (TLI.isCondCodeLegalOrCustom(InvCC, Tmp1.getSimpleValueType())) {
3671  // Use the new condition code and swap true and false
3672  Legalized = true;
3673  Tmp1 = DAG.getSelectCC(dl, Tmp1, Tmp2, Tmp4, Tmp3, InvCC);
3674  Tmp1->setFlags(Node->getFlags());
3675  } else {
3676  // If The inverse is not legal, then try to swap the arguments using
3677  // the inverse condition code.
3678  ISD::CondCode SwapInvCC = ISD::getSetCCSwappedOperands(InvCC);
3679  if (TLI.isCondCodeLegalOrCustom(SwapInvCC, Tmp1.getSimpleValueType())) {
3680  // The swapped inverse condition is legal, so swap true and false,
3681  // lhs and rhs.
3682  Legalized = true;
3683  Tmp1 = DAG.getSelectCC(dl, Tmp2, Tmp1, Tmp4, Tmp3, SwapInvCC);
3684  Tmp1->setFlags(Node->getFlags());
3685  }
3686  }
3687 
3688  if (!Legalized) {
3689  Legalized = TLI.LegalizeSetCCCondCode(
3690  DAG, getSetCCResultType(Tmp1.getValueType()), Tmp1, Tmp2, CC,
3691  NeedInvert, dl, Chain);
3692 
3693  assert(Legalized && "Can't legalize SELECT_CC with legal condition!");
3694 
3695  // If we expanded the SETCC by inverting the condition code, then swap
3696  // the True/False operands to match.
3697  if (NeedInvert)
3698  std::swap(Tmp3, Tmp4);
3699 
3700  // If we expanded the SETCC by swapping LHS and RHS, or by inverting the
3701  // condition code, create a new SELECT_CC node.
3702  if (CC.getNode()) {
3703  Tmp1 = DAG.getNode(ISD::SELECT_CC, dl, Node->getValueType(0),
3704  Tmp1, Tmp2, Tmp3, Tmp4, CC);
3705  } else {
3706  Tmp2 = DAG.getConstant(0, dl, Tmp1.getValueType());
3707  CC = DAG.getCondCode(ISD::SETNE);
3708  Tmp1 = DAG.getNode(ISD::SELECT_CC, dl, Node->getValueType(0), Tmp1,
3709  Tmp2, Tmp3, Tmp4, CC);
3710  }
3711  Tmp1->setFlags(Node->getFlags());
3712  }
3713  Results.push_back(Tmp1);
3714  break;
3715  }
3716  case ISD::BR_CC: {
3717  // TODO: need to add STRICT_BR_CC and STRICT_BR_CCS
3718  SDValue Chain;
3719  Tmp1 = Node->getOperand(0); // Chain
3720  Tmp2 = Node->getOperand(2); // LHS
3721  Tmp3 = Node->getOperand(3); // RHS
3722  Tmp4 = Node->getOperand(1); // CC
3723 
3724  bool Legalized =
3725  TLI.LegalizeSetCCCondCode(DAG, getSetCCResultType(Tmp2.getValueType()),
3726  Tmp2, Tmp3, Tmp4, NeedInvert, dl, Chain);
3727  (void)Legalized;
3728  assert(Legalized && "Can't legalize BR_CC with legal condition!");
3729 
3730  // If we expanded the SETCC by swapping LHS and RHS, create a new BR_CC
3731  // node.
3732  if (Tmp4.getNode()) {
3733  assert(!NeedInvert && "Don't know how to invert BR_CC!");
3734 
3735  Tmp1 = DAG.getNode(ISD::BR_CC, dl, Node->getValueType(0), Tmp1,
3736  Tmp4, Tmp2, Tmp3, Node->getOperand(4));
3737  } else {
3738  Tmp3 = DAG.getConstant(0, dl, Tmp2.getValueType());
3739  Tmp4 = DAG.getCondCode(NeedInvert ? ISD::SETEQ : ISD::SETNE);
3740  Tmp1 = DAG.getNode(ISD::BR_CC, dl, Node->getValueType(0), Tmp1, Tmp4,
3741  Tmp2, Tmp3, Node->getOperand(4));
3742  }
3743  Results.push_back(Tmp1);
3744  break;
3745  }
3746  case ISD::BUILD_VECTOR:
3747  Results.push_back(ExpandBUILD_VECTOR(Node));
3748  break;
3749  case ISD::SPLAT_VECTOR:
3750  Results.push_back(ExpandSPLAT_VECTOR(Node));
3751  break;
3752  case ISD::SRA:
3753  case ISD::SRL:
3754  case ISD::SHL: {
3755  // Scalarize vector SRA/SRL/SHL.
3756  EVT VT = Node->getValueType(0);
3757  assert(VT.isVector() && "Unable to legalize non-vector shift");
3758  assert(TLI.isTypeLegal(VT.getScalarType())&& "Element type must be legal");
3759  unsigned NumElem = VT.getVectorNumElements();
3760 
3761  SmallVector<SDValue, 8> Scalars;
3762  for (unsigned Idx = 0; Idx < NumElem; Idx++) {
3763  SDValue Ex =
3765  Node->getOperand(0), DAG.getVectorIdxConstant(Idx, dl));
3766  SDValue Sh =
3768  Node->getOperand(1), DAG.getVectorIdxConstant(Idx, dl));
3769  Scalars.push_back(DAG.getNode(Node->getOpcode(), dl,
3770  VT.getScalarType(), Ex, Sh));
3771  }
3772 
3773  SDValue Result = DAG.getBuildVector(Node->getValueType(0), dl, Scalars);
3774  Results.push_back(Result);
3775  break;
3776  }
3777  case ISD::VECREDUCE_FADD:
3778  case ISD::VECREDUCE_FMUL:
3779  case ISD::VECREDUCE_ADD:
3780  case ISD::VECREDUCE_MUL:
3781  case ISD::VECREDUCE_AND:
3782  case ISD::VECREDUCE_OR:
3783  case ISD::VECREDUCE_XOR:
3784  case ISD::VECREDUCE_SMAX:
3785  case ISD::VECREDUCE_SMIN:
3786  case ISD::VECREDUCE_UMAX:
3787  case ISD::VECREDUCE_UMIN:
3788  case ISD::VECREDUCE_FMAX:
3789  case ISD::VECREDUCE_FMIN:
3790  Results.push_back(TLI.expandVecReduce(Node, DAG));
3791  break;
3793  case ISD::GlobalAddress:
3794  case ISD::GlobalTLSAddress:
3795  case ISD::ExternalSymbol:
3796  case ISD::ConstantPool:
3797  case ISD::JumpTable:
3800  case ISD::INTRINSIC_VOID:
3801  // FIXME: Custom lowering for these operations shouldn't return null!
3802  // Return true so that we don't call ConvertNodeToLibcall which also won't
3803  // do anything.
3804  return true;
3805  }
3806 
3807  if (!TLI.isStrictFPEnabled() && Results.empty() && Node->isStrictFPOpcode()) {
3808  // FIXME: We were asked to expand a strict floating-point operation,
3809  // but there is currently no expansion implemented that would preserve
3810  // the "strict" properties. For now, we just fall back to the non-strict
3811  // version if that is legal on the target. The actual mutation of the
3812  // operation will happen in SelectionDAGISel::DoInstructionSelection.
3813  switch (Node->getOpcode()) {
3814  default:
3815  if (TLI.getStrictFPOperationAction(Node->getOpcode(),
3816  Node->getValueType(0))
3818  return true;
3819  break;
3820  case ISD::STRICT_FSUB: {
3821  if (TLI.getStrictFPOperationAction(
3822  ISD::STRICT_FSUB, Node->getValueType(0)) == TargetLowering::Legal)
3823  return true;
3824  if (TLI.getStrictFPOperationAction(
3825  ISD::STRICT_FADD, Node->getValueType(0)) != TargetLowering::Legal)
3826  break;
3827 
3828  EVT VT = Node->getValueType(0);
3829  const SDNodeFlags Flags = Node->getFlags();
3830  SDValue Neg = DAG.getNode(ISD::FNEG, dl, VT, Node->getOperand(2), Flags);
3831  SDValue Fadd = DAG.getNode(ISD::STRICT_FADD, dl, Node->getVTList(),
3832  {Node->getOperand(0), Node->getOperand(1), Neg},
3833  Flags);
3834 
3835  Results.push_back(Fadd);
3836  Results.push_back(Fadd.getValue(1));
3837  break;
3838  }
3841  case ISD::STRICT_LRINT:
3842  case ISD::STRICT_LLRINT:
3843  case ISD::STRICT_LROUND:
3844  case ISD::STRICT_LLROUND:
3845  // These are registered by the operand type instead of the value
3846  // type. Reflect that here.
3847  if (TLI.getStrictFPOperationAction(Node->getOpcode(),
3848  Node->getOperand(1).getValueType())
3850  return true;
3851  break;
3852  }
3853  }
3854 
3855  // Replace the original node with the legalized result.
3856  if (Results.empty()) {
3857  LLVM_DEBUG(dbgs() << "Cannot expand node\n");
3858  return false;
3859  }
3860 
3861  LLVM_DEBUG(dbgs() << "Successfully expanded node\n");
3862  ReplaceNode(Node, Results.data());
3863  return true;
3864 }
3865 
3866 void SelectionDAGLegalize::ConvertNodeToLibcall(SDNode *Node) {
3867  LLVM_DEBUG(dbgs() << "Trying to convert node to libcall\n");
3869  SDLoc dl(Node);
3870  // FIXME: Check flags on the node to see if we can use a finite call.
3871  unsigned Opc = Node->getOpcode();
3872  switch (Opc) {
3873  case ISD::ATOMIC_FENCE: {
3874  // If the target didn't lower this, lower it to '__sync_synchronize()' call
3875  // FIXME: handle "fence singlethread" more efficiently.
3877 
3879  CLI.setDebugLoc(dl)
3880  .setChain(Node->getOperand(0))
3881  .setLibCallee(
3882  CallingConv::C, Type::getVoidTy(*DAG.getContext()),
3883  DAG.getExternalSymbol("__sync_synchronize",
3884  TLI.getPointerTy(DAG.getDataLayout())),
3885  std::move(Args));
3886 
3887  std::pair<SDValue, SDValue> CallResult = TLI.LowerCallTo(CLI);
3888 
3889  Results.push_back(CallResult.second);
3890  break;
3891  }
3892  // By default, atomic intrinsics are marked Legal and lowered. Targets
3893  // which don't support them directly, however, may want libcalls, in which
3894  // case they mark them Expand, and we get here.
3895  case ISD::ATOMIC_SWAP:
3896  case ISD::ATOMIC_LOAD_ADD:
3897  case ISD::ATOMIC_LOAD_SUB:
3898  case ISD::ATOMIC_LOAD_AND:
3899  case ISD::ATOMIC_LOAD_CLR:
3900  case ISD::ATOMIC_LOAD_OR:
3901  case ISD::ATOMIC_LOAD_XOR:
3902  case ISD::ATOMIC_LOAD_NAND:
3903  case ISD::ATOMIC_LOAD_MIN:
3904  case ISD::ATOMIC_LOAD_MAX:
3905  case ISD::ATOMIC_LOAD_UMIN:
3906  case ISD::ATOMIC_LOAD_UMAX:
3907  case ISD::ATOMIC_CMP_SWAP: {
3908  MVT VT = cast<AtomicSDNode>(Node)->getMemoryVT().getSimpleVT();
3909  AtomicOrdering Order = cast<AtomicSDNode>(Node)->getMergedOrdering();
3910  RTLIB::Libcall LC = RTLIB::getOUTLINE_ATOMIC(Opc, Order, VT);
3911  EVT RetVT = Node->getValueType(0);
3914  if (TLI.getLibcallName(LC)) {
3915  // If outline atomic available, prepare its arguments and expand.
3916  Ops.append(Node->op_begin() + 2, Node->op_end());
3917  Ops.push_back(Node->getOperand(1));
3918 
3919  } else {
3920  LC = RTLIB::getSYNC(Opc, VT);
3921  assert(LC != RTLIB::UNKNOWN_LIBCALL &&
3922  "Unexpected atomic op or value type!");
3923  // Arguments for expansion to sync libcall
3924  Ops.append(Node->op_begin() + 1, Node->op_end());
3925  }
3926  std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(DAG, LC, RetVT,
3927  Ops, CallOptions,
3928  SDLoc(Node),
3929  Node->getOperand(0));
3930  Results.push_back(Tmp.first);
3931  Results.push_back(Tmp.second);
3932  break;
3933  }
3934  case ISD::TRAP: {
3935  // If this operation is not supported, lower it to 'abort()' call
3938  CLI.setDebugLoc(dl)
3939  .setChain(Node->getOperand(0))
3940  .setLibCallee(CallingConv::C, Type::getVoidTy(*DAG.getContext()),
3941  DAG.getExternalSymbol(
3942  "abort", TLI.getPointerTy(DAG.getDataLayout())),
3943  std::move(Args));
3944  std::pair<SDValue, SDValue> CallResult = TLI.LowerCallTo(CLI);
3945 
3946  Results.push_back(CallResult.second);
3947  break;
3948  }
3949  case ISD::FMINNUM:
3950  case ISD::STRICT_FMINNUM:
3951  ExpandFPLibCall(Node, RTLIB::FMIN_F32, RTLIB::FMIN_F64,
3952  RTLIB::FMIN_F80, RTLIB::FMIN_F128,
3953  RTLIB::FMIN_PPCF128, Results);
3954  break;
3955  case ISD::FMAXNUM:
3956  case ISD::STRICT_FMAXNUM:
3957  ExpandFPLibCall(Node, RTLIB::FMAX_F32, RTLIB::FMAX_F64,
3958  RTLIB::FMAX_F80, RTLIB::FMAX_F128,
3959  RTLIB::FMAX_PPCF128, Results);
3960  break;
3961  case ISD::FSQRT:
3962  case ISD::STRICT_FSQRT:
3963  ExpandFPLibCall(Node, RTLIB::SQRT_F32, RTLIB::SQRT_F64,
3964  RTLIB::SQRT_F80, RTLIB::SQRT_F128,
3965  RTLIB::SQRT_PPCF128, Results);
3966  break;
3967  case ISD::FCBRT:
3968  ExpandFPLibCall(Node, RTLIB::CBRT_F32, RTLIB::CBRT_F64,
3969  RTLIB::CBRT_F80, RTLIB::CBRT_F128,
3970  RTLIB::CBRT_PPCF128, Results);
3971  break;
3972  case ISD::FSIN:
3973  case ISD::STRICT_FSIN:
3974  ExpandFPLibCall(Node, RTLIB::SIN_F32, RTLIB::SIN_F64,
3975  RTLIB::SIN_F80, RTLIB::SIN_F128,
3976  RTLIB::SIN_PPCF128, Results);
3977  break;
3978  case ISD::FCOS:
3979  case ISD::STRICT_FCOS:
3980  ExpandFPLibCall(Node, RTLIB::COS_F32, RTLIB::COS_F64,
3981  RTLIB::COS_F80, RTLIB::COS_F128,
3982  RTLIB::COS_PPCF128, Results);
3983  break;
3984  case ISD::FSINCOS:
3985  // Expand into sincos libcall.
3986  ExpandSinCosLibCall(Node, Results);
3987  break;
3988  case ISD::FLOG:
3989  case ISD::STRICT_FLOG:
3990  ExpandFPLibCall(Node, RTLIB::LOG_F32, RTLIB::LOG_F64, RTLIB::LOG_F80,
3991  RTLIB::LOG_F128, RTLIB::LOG_PPCF128, Results);
3992  break;
3993  case ISD::FLOG2:
3994  case ISD::STRICT_FLOG2:
3995  ExpandFPLibCall(Node, RTLIB::LOG2_F32, RTLIB::LOG2_F64, RTLIB::LOG2_F80,
3996  RTLIB::LOG2_F128, RTLIB::LOG2_PPCF128, Results);
3997  break;
3998  case ISD::FLOG10:
3999  case ISD::STRICT_FLOG10:
4000  ExpandFPLibCall(Node, RTLIB::LOG10_F32, RTLIB::LOG10_F64, RTLIB::LOG10_F80,
4001  RTLIB::LOG10_F128, RTLIB::LOG10_PPCF128, Results);
4002  break;
4003  case ISD::FEXP:
4004  case ISD::STRICT_FEXP:
4005  ExpandFPLibCall(Node, RTLIB::EXP_F32, RTLIB::EXP_F64, RTLIB::EXP_F80,
4006  RTLIB::EXP_F128, RTLIB::EXP_PPCF128, Results);
4007  break;
4008  case ISD::FEXP2:
4009  case ISD::STRICT_FEXP2:
4010  ExpandFPLibCall(Node, RTLIB::EXP2_F32, RTLIB::EXP2_F64, RTLIB::EXP2_F80,
4011  RTLIB::EXP2_F128, RTLIB::EXP2_PPCF128, Results);
4012  break;
4013  case ISD::FTRUNC:
4014  case ISD::STRICT_FTRUNC:
4015  ExpandFPLibCall(Node, RTLIB::TRUNC_F32, RTLIB::TRUNC_F64,
4016  RTLIB::TRUNC_F80, RTLIB::TRUNC_F128,
4017  RTLIB::TRUNC_PPCF128, Results);
4018  break;
4019  case ISD::FFLOOR:
4020  case ISD::STRICT_FFLOOR:
4021  ExpandFPLibCall(Node, RTLIB::FLOOR_F32, RTLIB::FLOOR_F64,
4022  RTLIB::FLOOR_F80, RTLIB::FLOOR_F128,
4023  RTLIB::FLOOR_PPCF128, Results);
4024  break;
4025  case ISD::FCEIL:
4026  case ISD::STRICT_FCEIL:
4027  ExpandFPLibCall(Node, RTLIB::CEIL_F32, RTLIB::CEIL_F64,
4028  RTLIB::CEIL_F80, RTLIB::CEIL_F128,
4029  RTLIB::CEIL_PPCF128, Results);
4030  break;
4031  case ISD::FRINT:
4032  case ISD::STRICT_FRINT:
4033  ExpandFPLibCall(Node, RTLIB::RINT_F32, RTLIB::RINT_F64,
4034  RTLIB::RINT_F80, RTLIB::RINT_F128,
4035  RTLIB::RINT_PPCF128, Results);
4036  break;
4037  case ISD::FNEARBYINT:
4039  ExpandFPLibCall(Node, RTLIB::NEARBYINT_F32,
4040  RTLIB::NEARBYINT_F64,
4041  RTLIB::NEARBYINT_F80,
4042  RTLIB::NEARBYINT_F128,
4043  RTLIB::NEARBYINT_PPCF128, Results);
4044  break;
4045  case ISD::FROUND:
4046  case ISD::STRICT_FROUND:
4047  ExpandFPLibCall(Node, RTLIB::ROUND_F32,
4048  RTLIB::ROUND_F64,
4049  RTLIB::ROUND_F80,
4050  RTLIB::ROUND_F128,
4051  RTLIB::ROUND_PPCF128, Results);
4052  break;
4053  case ISD::FROUNDEVEN:
4055  ExpandFPLibCall(Node, RTLIB::ROUNDEVEN_F32,
4056  RTLIB::ROUNDEVEN_F64,
4057  RTLIB::ROUNDEVEN_F80,
4058  RTLIB::ROUNDEVEN_F128,
4059  RTLIB::ROUNDEVEN_PPCF128, Results);
4060  break;
4061  case ISD::FPOWI:
4062  case ISD::STRICT_FPOWI: {
4063  RTLIB::Libcall LC = RTLIB::getPOWI(Node->getSimpleValueType(0));
4064  assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected fpowi.");
4065  if (!TLI.getLibcallName(LC)) {
4066  // Some targets don't have a powi libcall; use pow instead.
4067  SDValue Exponent = DAG.getNode(ISD::SINT_TO_FP, SDLoc(Node),
4068  Node->getValueType(0),
4069  Node->getOperand(1));
4070  Results.push_back(DAG.getNode(ISD::FPOW, SDLoc(Node),
4071  Node->getValueType(0), Node->getOperand(0),
4072  Exponent));
4073  break;
4074  }
4075  unsigned Offset = Node->isStrictFPOpcode() ? 1 : 0;
4076  bool ExponentHasSizeOfInt =
4077  DAG.getLibInfo().getIntSize() ==
4078  Node->getOperand(1 + Offset).getValueType().getSizeInBits();
4079  if (!ExponentHasSizeOfInt) {
4080  // If the exponent does not match with sizeof(int) a libcall to
4081  // RTLIB::POWI would use the wrong type for the argument.
4082  DAG.getContext()->emitError("POWI exponent does not match sizeof(int)");
4083  Results.push_back(DAG.getUNDEF(Node->getValueType(0)));
4084  break;
4085  }
4086  ExpandFPLibCall(Node, LC, Results);
4087  break;
4088  }
4089  case ISD::FPOW:
4090  case ISD::STRICT_FPOW:
4091  ExpandFPLibCall(Node, RTLIB::POW_F32, RTLIB::POW_F64, RTLIB::POW_F80,
4092  RTLIB::POW_F128, RTLIB::POW_PPCF128, Results);
4093  break;
4094  case ISD::LROUND:
4095  case ISD::STRICT_LROUND:
4096  ExpandArgFPLibCall(Node, RTLIB::LROUND_F32,
4097  RTLIB::LROUND_F64, RTLIB::LROUND_F80,
4098  RTLIB::LROUND_F128,
4099  RTLIB::LROUND_PPCF128, Results);
4100  break;
4101  case ISD::LLROUND:
4102  case ISD::STRICT_LLROUND:
4103  ExpandArgFPLibCall(Node, RTLIB::LLROUND_F32,
4104  RTLIB::LLROUND_F64, RTLIB::LLROUND_F80,
4105  RTLIB::LLROUND_F128,
4106  RTLIB::LLROUND_PPCF128, Results);
4107  break;
4108  case ISD::LRINT:
4109  case ISD::STRICT_LRINT:
4110  ExpandArgFPLibCall(Node, RTLIB::LRINT_F32,
4111  RTLIB::LRINT_F64, RTLIB::LRINT_F80,
4112  RTLIB::LRINT_F128,
4113  RTLIB::LRINT_PPCF128, Results);
4114  break;
4115  case ISD::LLRINT:
4116  case ISD::STRICT_LLRINT:
4117  ExpandArgFPLibCall(Node, RTLIB::LLRINT_F32,
4118  RTLIB::LLRINT_F64, RTLIB::LLRINT_F80,
4119  RTLIB::LLRINT_F128,
4120  RTLIB::LLRINT_PPCF128, Results);
4121  break;
4122  case ISD::FDIV:
4123  case ISD::STRICT_FDIV:
4124  ExpandFPLibCall(Node, RTLIB::DIV_F32, RTLIB::DIV_F64,
4125  RTLIB::DIV_F80, RTLIB::DIV_F128,
4126  RTLIB::DIV_PPCF128, Results);
4127  break;
4128  case ISD::FREM:
4129  case ISD::STRICT_FREM:
4130  ExpandFPLibCall(Node, RTLIB::REM_F32, RTLIB::REM_F64,
4131  RTLIB::REM_F80, RTLIB::REM_F128,
4132  RTLIB::REM_PPCF128, Results);
4133  break;
4134  case ISD::FMA:
4135  case ISD::STRICT_FMA:
4136  ExpandFPLibCall(Node, RTLIB::FMA_F32, RTLIB::FMA_F64,
4137  RTLIB::FMA_F80, RTLIB::FMA_F128,
4138  RTLIB::FMA_PPCF128, Results);
4139  break;
4140  case ISD::FADD:
4141  case ISD::STRICT_FADD:
4142  ExpandFPLibCall(Node, RTLIB::ADD_F32, RTLIB::ADD_F64,
4143  RTLIB::ADD_F80, RTLIB::ADD_F128,
4144  RTLIB::ADD_PPCF128, Results);
4145  break;
4146  case ISD::FMUL:
4147  case ISD::STRICT_FMUL:
4148  ExpandFPLibCall(Node, RTLIB::MUL_F32, RTLIB::MUL_F64,
4149  RTLIB::MUL_F80, RTLIB::MUL_F128,
4150  RTLIB::MUL_PPCF128, Results);
4151  break;
4152  case ISD::FP16_TO_FP:
4153  if (Node->getValueType(0) == MVT::f32) {
4154  Results.push_back(ExpandLibCall(RTLIB::FPEXT_F16_F32, Node, false));
4155  }
4156  break;
4157  case ISD::STRICT_FP16_TO_FP: {
4158  if (Node->getValueType(0) == MVT::f32) {
4160  std::pair<SDValue, SDValue> Tmp = TLI.makeLibCall(
4161  DAG, RTLIB::FPEXT_F16_F32, MVT::f32, Node->getOperand(1), CallOptions,
4162  SDLoc(Node), Node->getOperand(0));
4163  Results.push_back(Tmp.first);
4164  Results.push_back(Tmp.second);
4165  }
4166  break;
4167  }
4168  case ISD::FP_TO_FP16: {
4169  RTLIB::Libcall LC =
4170  RTLIB::getFPROUND(Node->getOperand(0).getValueType(), MVT::f16);
4171  assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unable to expand fp_to_fp16");
4172  Results.push_back(ExpandLibCall(LC, Node, false));
4173  break;
4174  }
4177  case ISD::SINT_TO_FP:
4178  case ISD::UINT_TO_FP: {
4179  // TODO - Common the code with DAGTypeLegalizer::SoftenFloatRes_XINT_TO_FP
4180  bool IsStrict = Node->isStrictFPOpcode();
4181  bool Signed = Node->getOpcode() == ISD::SINT_TO_FP ||
4182  Node->getOpcode() == ISD::STRICT_SINT_TO_FP;
4183  EVT SVT = Node->getOperand(IsStrict ? 1 : 0).getValueType();
4184  EVT RVT = Node->getValueType(0);
4185  EVT NVT = EVT();
4186  SDLoc dl(Node);
4187 
4188  // Even if the input is legal, no libcall may exactly match, eg. we don't
4189  // have i1 -> fp conversions. So, it needs to be promoted to a larger type,
4190  // eg: i13 -> fp. Then, look for an appropriate libcall.
4191  RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
4192  for (unsigned t = MVT::FIRST_INTEGER_VALUETYPE;
4193  t <= MVT::LAST_INTEGER_VALUETYPE && LC == RTLIB::UNKNOWN_LIBCALL;
4194  ++t) {
4195  NVT = (MVT::SimpleValueType)t;
4196  // The source needs to big enough to hold the operand.
4197  if (NVT.bitsGE(SVT))
4198  LC = Signed ? RTLIB::getSINTTOFP(NVT, RVT)
4199  : RTLIB::getUINTTOFP(NVT, RVT);
4200  }
4201  assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unable to legalize as libcall");
4202 
4203  SDValue Chain = IsStrict ? Node->getOperand(0) : SDValue();
4204  // Sign/zero extend the argument if the libcall takes a larger type.
4205  SDValue Op = DAG.getNode(Signed ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND, dl,
4206  NVT, Node->getOperand(IsStrict ? 1 : 0));
4208  CallOptions.setSExt(Signed);
4209  std::pair<SDValue, SDValue> Tmp =
4210  TLI.makeLibCall(DAG, LC, RVT, Op, CallOptions, dl, Chain);
4211  Results.push_back(Tmp.first);
4212  if (IsStrict)
4213  Results.push_back(Tmp.second);
4214  break;
4215  }
4216  case ISD::FP_TO_SINT:
4217  case ISD::FP_TO_UINT:
4219  case ISD::STRICT_FP_TO_UINT: {
4220  // TODO - Common the code with DAGTypeLegalizer::SoftenFloatOp_FP_TO_XINT.
4221  bool IsStrict = Node->isStrictFPOpcode();
4222  bool Signed = Node->getOpcode() == ISD::FP_TO_SINT ||
4223  Node->getOpcode() == ISD::STRICT_FP_TO_SINT;
4224 
4225  SDValue Op = Node->getOperand(IsStrict ? 1 : 0);
4226  EVT SVT = Op.getValueType();
4227  EVT RVT = Node->getValueType(0);
4228  EVT NVT = EVT();
4229  SDLoc dl(Node);
4230 
4231  // Even if the result is legal, no libcall may exactly match, eg. we don't
4232  // have fp -> i1 conversions. So, it needs to be promoted to a larger type,
4233  // eg: fp -> i32. Then, look for an appropriate libcall.
4234  RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL;
4235  for (unsigned IntVT = MVT::FIRST_INTEGER_VALUETYPE;
4236  IntVT <= MVT::LAST_INTEGER_VALUETYPE && LC == RTLIB::UNKNOWN_LIBCALL;
4237  ++IntVT) {
4238  NVT = (MVT::SimpleValueType)IntVT;
4239  // The type needs to big enough to hold the result.
4240  if (NVT.bitsGE(RVT))
4241  LC = Signed ? RTLIB::getFPTOSINT(SVT, NVT)
4242  : RTLIB::getFPTOUINT(SVT, NVT);
4243  }
4244  assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unable to legalize as libcall");
4245 
4246  SDValue Chain = IsStrict ? Node->getOperand(0) : SDValue();
4248  std::pair<SDValue, SDValue> Tmp =
4249  TLI.makeLibCall(DAG, LC, NVT, Op, CallOptions, dl, Chain);
4250 
4251  // Truncate the result if the libcall returns a larger type.
4252  Results.push_back(DAG.getNode(ISD::TRUNCATE, dl, RVT, Tmp.first));
4253  if (IsStrict)
4254  Results.push_back(Tmp.second);
4255  break;
4256  }
4257 
4258  case ISD::FP_ROUND:
4259  case ISD::STRICT_FP_ROUND: {
4260  // X = FP_ROUND(Y, TRUNC)
4261  // TRUNC is a flag, which is always an integer that is zero or one.
4262  // If TRUNC is 0, this is a normal rounding, if it is 1, this FP_ROUND
4263  // is known to not change the value of Y.
4264  // We can only expand it into libcall if the TRUNC is 0.
4265  bool IsStrict = Node->isStrictFPOpcode();
4266  SDValue Op = Node->getOperand(IsStrict ? 1 : 0);
4267  SDValue Chain = IsStrict ? Node->getOperand(0) : SDValue();
4268  EVT VT = Node->getValueType(0);
4269  assert(cast<ConstantSDNode>(Node->getOperand(IsStrict ? 2 : 1))->isZero() &&
4270  "Unable to expand as libcall if it is not normal rounding");
4271 
4272  RTLIB::Libcall LC = RTLIB::getFPROUND(Op.getValueType(), VT);
4273  assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unable to legalize as libcall");
4274 
4276  std::pair<SDValue, SDValue> Tmp =
4277  TLI.makeLibCall(DAG, LC, VT, Op, CallOptions, SDLoc(Node), Chain);
4278  Results.push_back(Tmp.first);
4279  if (IsStrict)
4280  Results.push_back(Tmp.second);
4281  break;
4282  }
4283  case ISD::FP_EXTEND: {
4284  Results.push_back(
4285  ExpandLibCall(RTLIB::getFPEXT(Node->getOperand(0).getValueType(),
4286  Node->getValueType(0)),
4287  Node, false));
4288  break;
4289  }
4290  case ISD::STRICT_FP_EXTEND:
4291  case ISD::STRICT_FP_TO_FP16: {
4292  RTLIB::Libcall LC =
4293  Node->getOpcode() == ISD::STRICT_FP_TO_FP16
4294  ? RTLIB::getFPROUND(Node->getOperand(1).getValueType(), MVT::f16)
4295  : RTLIB::getFPEXT(Node->getOperand(1).getValueType(),
4296  Node->getValueType(0));
4297  assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unable to legalize as libcall");
4298 
4300  std::pair<SDValue, SDValue> Tmp =
4301  TLI.makeLibCall(DAG, LC, Node->getValueType(0), Node->getOperand(1),
4302  CallOptions, SDLoc(Node), Node->getOperand(0));
4303  Results.push_back(Tmp.first);
4304  Results.push_back(Tmp.second);
4305  break;
4306  }
4307  case ISD::FSUB:
4308  case ISD::STRICT_FSUB:
4309  ExpandFPLibCall(Node, RTLIB::SUB_F32, RTLIB::SUB_F64,
4310  RTLIB::SUB_F80, RTLIB::SUB_F128,
4311  RTLIB::SUB_PPCF128, Results);
4312  break;
4313  case ISD::SREM:
4314  Results.push_back(ExpandIntLibCall(Node, true,
4315  RTLIB::SREM_I8,
4316  RTLIB::SREM_I16, RTLIB::SREM_I32,
4317  RTLIB::SREM_I64, RTLIB::SREM_I128));
4318  break;
4319  case ISD::UREM:
4320  Results.push_back(ExpandIntLibCall(Node, false,
4321  RTLIB::UREM_I8,
4322  RTLIB::UREM_I16, RTLIB::UREM_I32,
4323  RTLIB::UREM_I64, RTLIB::UREM_I128));
4324  break;
4325  case ISD::SDIV:
4326  Results.push_back(ExpandIntLibCall(Node, true,
4327  RTLIB::SDIV_I8,
4328  RTLIB::SDIV_I16, RTLIB::SDIV_I32,
4329  RTLIB::SDIV_I64, RTLIB::SDIV_I128));
4330  break;
4331  case ISD::UDIV:
4332  Results.push_back(ExpandIntLibCall(Node, false,
4333  RTLIB::UDIV_I8,
4334  RTLIB::UDIV_I16, RTLIB::UDIV_I32,
4335  RTLIB::UDIV_I64, RTLIB::UDIV_I128));
4336  break;
4337  case ISD::SDIVREM:
4338  case ISD::UDIVREM:
4339  // Expand into divrem libcall
4340  ExpandDivRemLibCall(Node, Results);
4341  break;
4342  case ISD::MUL:
4343  Results.push_back(ExpandIntLibCall(Node, false,
4344  RTLIB::MUL_I8,
4345  RTLIB::MUL_I16, RTLIB::MUL_I32,
4346  RTLIB::MUL_I64, RTLIB::MUL_I128));
4347  break;
4348  case ISD::CTLZ_ZERO_UNDEF:
4349  switch (Node->getSimpleValueType(0).SimpleTy) {
4350  default:
4351  llvm_unreachable("LibCall explicitly requested, but not available");
4352  case MVT::i32:
4353  Results.push_back(ExpandLibCall(RTLIB::CTLZ_I32, Node, false));
4354  break;
4355  case MVT::i64:
4356  Results.push_back(ExpandLibCall(RTLIB::CTLZ_I64, Node, false));
4357  break;
4358  case MVT::i128:
4359  Results.push_back(ExpandLibCall(RTLIB::CTLZ_I128, Node, false));
4360  break;
4361  }
4362  break;
4363  }
4364 
4365  // Replace the original node with the legalized result.
4366  if (!Results.empty()) {
4367  LLVM_DEBUG(dbgs() << "Successfully converted node to libcall\n");
4368  ReplaceNode(Node, Results.data());
4369  } else
4370  LLVM_DEBUG(dbgs() << "Could not convert node to libcall\n");
4371 }
4372 
4373 // Determine the vector type to use in place of an original scalar element when
4374 // promoting equally sized vectors.
4376  MVT EltVT, MVT NewEltVT) {
4377  unsigned OldEltsPerNewElt = EltVT.getSizeInBits() / NewEltVT.getSizeInBits();
4378  MVT MidVT = MVT::getVectorVT(NewEltVT, OldEltsPerNewElt);
4379  assert(TLI.isTypeLegal(MidVT) && "unexpected");
4380  return MidVT;
4381 }
4382 
4383 void SelectionDAGLegalize::PromoteNode(SDNode *Node) {
4384  LLVM_DEBUG(dbgs() << "Trying to promote node\n");
4386  MVT OVT = Node->getSimpleValueType(0);
4387  if (Node->getOpcode() == ISD::UINT_TO_FP ||
4388  Node->getOpcode() == ISD::SINT_TO_FP ||
4389  Node->getOpcode() == ISD::SETCC ||
4390  Node->getOpcode() == ISD::EXTRACT_VECTOR_ELT ||
4391  Node->getOpcode() == ISD::INSERT_VECTOR_ELT) {
4392  OVT = Node->getOperand(0).getSimpleValueType();
4393  }
4394  if (Node->getOpcode() == ISD::STRICT_UINT_TO_FP ||
4395  Node->getOpcode() == ISD::STRICT_SINT_TO_FP ||
4396  Node->getOpcode() == ISD::STRICT_FSETCC ||
4397  Node->getOpcode() == ISD::STRICT_FSETCCS)
4398  OVT = Node->getOperand(1).getSimpleValueType();
4399  if (Node->getOpcode() == ISD::BR_CC ||
4400  Node->getOpcode() == ISD::SELECT_CC)
4401  OVT = Node->getOperand(2).getSimpleValueType();
4402  MVT NVT = TLI.getTypeToPromoteTo(Node->getOpcode(), OVT);
4403  SDLoc dl(Node);
4404  SDValue Tmp1, Tmp2, Tmp3, Tmp4;
4405  switch (Node->getOpcode()) {
4406  case ISD::CTTZ:
4407  case ISD::CTTZ_ZERO_UNDEF:
4408  case ISD::CTLZ:
4409  case ISD::CTLZ_ZERO_UNDEF:
4410  case ISD::CTPOP:
4411  // Zero extend the argument unless its cttz, then use any_extend.
4412  if (Node->getOpcode() == ISD::CTTZ ||
4413  Node->getOpcode() == ISD::CTTZ_ZERO_UNDEF)
4414  Tmp1 = DAG.getNode(ISD::ANY_EXTEND, dl, NVT, Node->getOperand(0));
4415  else
4416  Tmp1 = DAG.getNode(ISD::ZERO_EXTEND, dl, NVT, Node->getOperand(0));
4417 
4418  if (Node->getOpcode() == ISD::CTTZ) {
4419  // The count is the same in the promoted type except if the original
4420  // value was zero. This can be handled by setting the bit just off
4421  // the top of the original type.
4422  auto TopBit = APInt::getOneBitSet(NVT.getSizeInBits(),
4423  OVT.getSizeInBits());
4424  Tmp1 = DAG.getNode(ISD::OR, dl, NVT, Tmp1,
4425  DAG.getConstant(TopBit, dl, NVT));
4426  }
4427  // Perform the larger operation. For CTPOP and CTTZ_ZERO_UNDEF, this is
4428  // already the correct result.
4429  Tmp1 = DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1);
4430  if (Node->getOpcode() == ISD::CTLZ ||
4431  Node->getOpcode() == ISD::CTLZ_ZERO_UNDEF) {
4432  // Tmp1 = Tmp1 - (sizeinbits(NVT) - sizeinbits(Old VT))
4433  Tmp1 = DAG.getNode(ISD::SUB, dl, NVT, Tmp1,
4434  DAG.getConstant(NVT.getSizeInBits() -
4435  OVT.getSizeInBits(), dl, NVT));
4436  }
4437  Results.push_back(DAG.getNode(ISD::TRUNCATE, dl, OVT, Tmp1));
4438  break;
4439  case ISD::BITREVERSE:
4440  case ISD::BSWAP: {
4441  unsigned DiffBits = NVT.getSizeInBits() - OVT.getSizeInBits();
4442  Tmp1 = DAG.getNode(ISD::ZERO_EXTEND, dl, NVT, Node->getOperand(0));
4443  Tmp1 = DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1);
4444  Tmp1 = DAG.getNode(
4445  ISD::SRL, dl, NVT, Tmp1,
4446  DAG.getConstant(DiffBits, dl,
4447  TLI.getShiftAmountTy(NVT, DAG.getDataLayout())));
4448 
4449  Results.push_back(DAG.getNode(ISD::TRUNCATE, dl, OVT, Tmp1));
4450  break;
4451  }
4452  case ISD::FP_TO_UINT:
4454  case ISD::FP_TO_SINT:
4456  PromoteLegalFP_TO_INT(Node, dl, Results);
4457  break;
4458  case ISD::FP_TO_UINT_SAT:
4459  case ISD::FP_TO_SINT_SAT:
4460  Results.push_back(PromoteLegalFP_TO_INT_SAT(Node, dl));
4461  break;
4462  case ISD::UINT_TO_FP:
4464  case ISD::SINT_TO_FP:
4466  PromoteLegalINT_TO_FP(Node, dl, Results);
4467  break;
4468  case ISD::VAARG: {
4469  SDValue Chain = Node->getOperand(0); // Get the chain.
4470  SDValue Ptr = Node->getOperand(1); // Get the pointer.
4471 
4472  unsigned TruncOp;
4473  if (OVT.isVector()) {
4474  TruncOp = ISD::BITCAST;
4475  } else {
4476  assert(OVT.isInteger()
4477  && "VAARG promotion is supported only for vectors or integer types");
4478  TruncOp = ISD::TRUNCATE;
4479  }
4480 
4481  // Perform the larger operation, then convert back
4482  Tmp1 = DAG.getVAArg(NVT, dl, Chain, Ptr, Node->getOperand(2),
4483  Node->getConstantOperandVal(3));
4484  Chain = Tmp1.getValue(1);
4485 
4486  Tmp2 = DAG.getNode(TruncOp, dl, OVT, Tmp1);
4487 
4488  // Modified the chain result - switch anything that used the old chain to
4489  // use the new one.
4490  DAG.ReplaceAllUsesOfValueWith(SDValue(Node, 0), Tmp2);
4491  DAG.ReplaceAllUsesOfValueWith(SDValue(Node, 1), Chain);
4492  if (UpdatedNodes) {
4493  UpdatedNodes->insert(Tmp2.getNode());
4494  UpdatedNodes->insert(Chain.getNode());
4495  }
4496  ReplacedNode(Node);
4497  break;
4498  }
4499  case ISD::MUL:
4500  case ISD::SDIV:
4501  case ISD::SREM:
4502  case ISD::UDIV:
4503  case ISD::UREM:
4504  case ISD::AND:
4505  case ISD::OR:
4506  case ISD::XOR: {
4507  unsigned ExtOp, TruncOp;
4508  if (OVT.isVector()) {
4509  ExtOp = ISD::BITCAST;
4510  TruncOp = ISD::BITCAST;
4511  } else {
4512  assert(OVT.isInteger() && "Cannot promote logic operation");
4513 
4514  switch (Node->getOpcode()) {
4515  default:
4516  ExtOp = ISD::ANY_EXTEND;
4517  break;
4518  case ISD::SDIV:
4519  case ISD::SREM:
4520  ExtOp = ISD::SIGN_EXTEND;
4521  break;
4522  case ISD::UDIV:
4523  case ISD::UREM:
4524  ExtOp = ISD::ZERO_EXTEND;
4525  break;
4526  }
4527  TruncOp = ISD::TRUNCATE;
4528  }
4529  // Promote each of the values to the new type.
4530  Tmp1 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(0));
4531  Tmp2 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(1));
4532  // Perform the larger operation, then convert back
4533  Tmp1 = DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1, Tmp2);
4534  Results.push_back(DAG.getNode(TruncOp, dl, OVT, Tmp1));
4535  break;
4536  }
4537  case ISD::UMUL_LOHI:
4538  case ISD::SMUL_LOHI: {
4539  // Promote to a multiply in a wider integer type.
4540  unsigned ExtOp = Node->getOpcode() == ISD::UMUL_LOHI ? ISD::ZERO_EXTEND
4541  : ISD::SIGN_EXTEND;
4542  Tmp1 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(0));
4543  Tmp2 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(1));
4544  Tmp1 = DAG.getNode(ISD::MUL, dl, NVT, Tmp1, Tmp2);
4545 
4546  auto &DL = DAG.getDataLayout();
4547  unsigned OriginalSize = OVT.getScalarSizeInBits();
4548  Tmp2 = DAG.getNode(
4549  ISD::SRL, dl, NVT, Tmp1,
4550  DAG.getConstant(OriginalSize, dl, TLI.getScalarShiftAmountTy(DL, NVT)));
4551  Results.push_back(DAG.getNode(ISD::TRUNCATE, dl, OVT, Tmp1));
4552  Results.push_back(DAG.getNode(ISD::TRUNCATE, dl, OVT, Tmp2));
4553  break;
4554  }
4555  case ISD::SELECT: {
4556  unsigned ExtOp, TruncOp;
4557  if (Node->getValueType(0).isVector() ||
4558  Node->getValueType(0).getSizeInBits() == NVT.getSizeInBits()) {
4559  ExtOp = ISD::BITCAST;
4560  TruncOp = ISD::BITCAST;
4561  } else if (Node->getValueType(0).isInteger()) {
4562  ExtOp = ISD::ANY_EXTEND;
4563  TruncOp = ISD::TRUNCATE;
4564  } else {
4565  ExtOp = ISD::FP_EXTEND;
4566  TruncOp = ISD::FP_ROUND;
4567  }
4568  Tmp1 = Node->getOperand(0);
4569  // Promote each of the values to the new type.
4570  Tmp2 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(1));
4571  Tmp3 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(2));
4572  // Perform the larger operation, then round down.
4573  Tmp1 = DAG.getSelect(dl, NVT, Tmp1, Tmp2, Tmp3);
4574  Tmp1->setFlags(Node->getFlags());
4575  if (TruncOp != ISD::FP_ROUND)
4576  Tmp1 = DAG.getNode(TruncOp, dl, Node->getValueType(0), Tmp1);
4577  else
4578  Tmp1 = DAG.getNode(TruncOp, dl, Node->getValueType(0), Tmp1,
4579  DAG.getIntPtrConstant(0, dl));
4580  Results.push_back(Tmp1);
4581  break;
4582  }
4583  case ISD::VECTOR_SHUFFLE: {
4584  ArrayRef<int> Mask = cast<ShuffleVectorSDNode>(Node)->getMask();
4585 
4586  // Cast the two input vectors.
4587  Tmp1 = DAG.getNode(ISD::BITCAST, dl, NVT, Node->getOperand(0));
4588  Tmp2 = DAG.getNode(ISD::BITCAST, dl, NVT, Node->getOperand(1));
4589 
4590  // Convert the shuffle mask to the right # elements.
4591  Tmp1 = ShuffleWithNarrowerEltType(NVT, OVT, dl, Tmp1, Tmp2, Mask);
4592  Tmp1 = DAG.getNode(ISD::BITCAST, dl, OVT, Tmp1);
4593  Results.push_back(Tmp1);
4594  break;
4595  }
4596  case ISD::VECTOR_SPLICE: {
4597  Tmp1 = DAG.getNode(ISD::ANY_EXTEND, dl, NVT, Node->getOperand(0));
4598  Tmp2 = DAG.getNode(ISD::ANY_EXTEND, dl, NVT, Node->getOperand(1));
4599  Tmp3 = DAG.getNode(ISD::VECTOR_SPLICE, dl, NVT, Tmp1, Tmp2,
4600  Node->getOperand(2));
4601  Results.push_back(DAG.getNode(ISD::TRUNCATE, dl, OVT, Tmp3));
4602  break;
4603  }
4604  case ISD::SELECT_CC: {
4605  SDValue Cond = Node->getOperand(4);
4606  ISD::CondCode CCCode = cast<CondCodeSDNode>(Cond)->get();
4607  // Type of the comparison operands.
4608  MVT CVT = Node->getSimpleValueType(0);
4609  assert(CVT == OVT && "not handled");
4610 
4611  unsigned ExtOp = ISD::FP_EXTEND;
4612  if (NVT.isInteger()) {
4614  }
4615 
4616  // Promote the comparison operands, if needed.
4617  if (TLI.isCondCodeLegal(CCCode, CVT)) {
4618  Tmp1 = Node->getOperand(0);
4619  Tmp2 = Node->getOperand(1);
4620  } else {
4621  Tmp1 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(0));
4622  Tmp2 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(1));
4623  }
4624  // Cast the true/false operands.
4625  Tmp3 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(2));
4626  Tmp4 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(3));
4627 
4628  Tmp1 = DAG.getNode(ISD::SELECT_CC, dl, NVT, {Tmp1, Tmp2, Tmp3, Tmp4, Cond},
4629  Node->getFlags());
4630 
4631  // Cast the result back to the original type.
4632  if (ExtOp != ISD::FP_EXTEND)
4633  Tmp1 = DAG.getNode(ISD::TRUNCATE, dl, OVT, Tmp1);
4634  else
4635  Tmp1 = DAG.getNode(ISD::FP_ROUND, dl, OVT, Tmp1,
4636  DAG.getIntPtrConstant(0, dl));
4637 
4638  Results.push_back(Tmp1);
4639  break;
4640  }
4641  case ISD::SETCC:
4642  case ISD::STRICT_FSETCC:
4643  case ISD::STRICT_FSETCCS: {
4644  unsigned ExtOp = ISD::FP_EXTEND;
4645  if (NVT.isInteger()) {
4646  ISD::CondCode CCCode = cast<CondCodeSDNode>(Node->getOperand(2))->get();
4648  }
4649  if (Node->isStrictFPOpcode()) {
4650  SDValue InChain = Node->getOperand(0);
4651  std::tie(Tmp1, std::ignore) =
4652  DAG.getStrictFPExtendOrRound(Node->getOperand(1), InChain, dl, NVT);
4653  std::tie(Tmp2, std::ignore) =
4654  DAG.getStrictFPExtendOrRound(Node->getOperand(2), InChain, dl, NVT);
4655  SmallVector<SDValue, 2> TmpChains = {Tmp1.getValue(1), Tmp2.getValue(1)};
4656  SDValue OutChain = DAG.getTokenFactor(dl, TmpChains);
4657  SDVTList VTs = DAG.getVTList(Node->getValueType(0), MVT::Other);
4658  Results.push_back(DAG.getNode(Node->getOpcode(), dl, VTs,
4659  {OutChain, Tmp1, Tmp2, Node->getOperand(3)},
4660  Node->getFlags()));
4661  Results.push_back(Results.back().getValue(1));
4662  break;
4663  }
4664  Tmp1 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(0));
4665  Tmp2 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(1));
4666  Results.push_back(DAG.getNode(ISD::SETCC, dl, Node->getValueType(0), Tmp1,
4667  Tmp2, Node->getOperand(2), Node->getFlags()));
4668  break;
4669  }
4670  case ISD::BR_CC: {
4671  unsigned ExtOp = ISD::FP_EXTEND;
4672  if (NVT.isInteger()) {
4673  ISD::CondCode CCCode =
4674  cast<CondCodeSDNode>(Node->getOperand(1))->get();
4676  }
4677  Tmp1 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(2));
4678  Tmp2 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(3));
4679  Results.push_back(DAG.getNode(ISD::BR_CC, dl, Node->getValueType(0),
4680  Node->getOperand(0), Node->getOperand(1),
4681  Tmp1, Tmp2, Node->getOperand(4)));
4682  break;
4683  }
4684  case ISD::FADD:
4685  case ISD::FSUB:
4686  case ISD::FMUL:
4687  case ISD::FDIV:
4688  case ISD::FREM:
4689  case ISD::FMINNUM:
4690  case ISD::FMAXNUM:
4691  case ISD::FPOW:
4692  Tmp1 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(0));
4693  Tmp2 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(1));
4694  Tmp3 = DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1, Tmp2,
4695  Node->getFlags());
4696  Results.push_back(DAG.getNode(ISD::FP_ROUND, dl, OVT,
4697  Tmp3, DAG.getIntPtrConstant(0, dl)));
4698  break;
4699  case ISD::STRICT_FREM:
4700  case ISD::STRICT_FPOW:
4701  Tmp1 = DAG.getNode(ISD::STRICT_FP_EXTEND, dl, {NVT, MVT::Other},
4702  {Node->getOperand(0), Node->getOperand(1)});
4703  Tmp2 = DAG.getNode(ISD::STRICT_FP_EXTEND, dl, {NVT, MVT::Other},
4704  {Node->getOperand(0), Node->getOperand(2)});
4705  Tmp3 = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Tmp1.getValue(1),
4706  Tmp2.getValue(1));
4707  Tmp1 = DAG.getNode(Node->getOpcode(), dl, {NVT, MVT::Other},
4708  {Tmp3, Tmp1, Tmp2});
4709  Tmp1 = DAG.getNode(ISD::STRICT_FP_ROUND, dl, {OVT, MVT::Other},
4710  {Tmp1.getValue(1), Tmp1, DAG.getIntPtrConstant(0, dl)});
4711  Results.push_back(Tmp1);
4712  Results.push_back(Tmp1.getValue(1));
4713  break;
4714  case ISD::FMA:
4715  Tmp1 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(0));
4716  Tmp2 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(1));
4717  Tmp3 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(2));
4718  Results.push_back(
4719  DAG.getNode(ISD::FP_ROUND, dl, OVT,
4720  DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1, Tmp2, Tmp3),
4721  DAG.getIntPtrConstant(0, dl)));
4722  break;
4723  case ISD::FCOPYSIGN:
4724  case ISD::FPOWI: {
4725  Tmp1 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(0));
4726  Tmp2 = Node->getOperand(1);
4727  Tmp3 = DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1, Tmp2);
4728 
4729  // fcopysign doesn't change anything but the sign bit, so
4730  // (fp_round (fcopysign (fpext a), b))
4731  // is as precise as
4732  // (fp_round (fpext a))
4733  // which is a no-op. Mark it as a TRUNCating FP_ROUND.
4734  const bool isTrunc = (Node->getOpcode() == ISD::FCOPYSIGN);
4735  Results.push_back(DAG.getNode(ISD::FP_ROUND, dl, OVT,
4736  Tmp3, DAG.getIntPtrConstant(isTrunc, dl)));
4737  break;
4738  }
4739  case ISD::FFLOOR:
4740  case ISD::FCEIL:
4741  case ISD::FRINT:
4742  case ISD::FNEARBYINT:
4743  case ISD::FROUND:
4744  case ISD::FROUNDEVEN:
4745  case ISD::FTRUNC:
4746  case ISD::FNEG:
4747  case ISD::FSQRT:
4748  case ISD::FSIN:
4749  case ISD::FCOS:
4750  case ISD::FLOG:
4751  case ISD::FLOG2:
4752  case ISD::FLOG10:
4753  case ISD::FABS:
4754  case ISD::FEXP:
4755  case ISD::FEXP2:
4756  Tmp1 = DAG.getNode(ISD::FP_EXTEND, dl, NVT, Node->getOperand(0));
4757  Tmp2 = DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1);
4758  Results.push_back(DAG.getNode(ISD::FP_ROUND, dl, OVT,
4759  Tmp2, DAG.getIntPtrConstant(0, dl)));
4760  break;
4761  case ISD::STRICT_FFLOOR:
4762  case ISD::STRICT_FCEIL:
4763  case ISD::STRICT_FSIN:
4764  case ISD::STRICT_FCOS:
4765  case ISD::STRICT_FLOG:
4766  case ISD::STRICT_FLOG10:
4767  case ISD::STRICT_FEXP:
4768  Tmp1 = DAG.getNode(ISD::STRICT_FP_EXTEND, dl, {NVT, MVT::Other},
4769  {Node->getOperand(0), Node->getOperand(1)});
4770  Tmp2 = DAG.getNode(Node->getOpcode(), dl, {NVT, MVT::Other},
4771  {Tmp1.getValue(1), Tmp1});
4772  Tmp3 = DAG.getNode(ISD::STRICT_FP_ROUND, dl, {OVT, MVT::Other},
4773  {Tmp2.getValue(1), Tmp2, DAG.getIntPtrConstant(0, dl)});
4774  Results.push_back(Tmp3);
4775  Results.push_back(Tmp3.getValue(1));
4776  break;
4777  case ISD::BUILD_VECTOR: {
4778  MVT EltVT = OVT.getVectorElementType();
4779  MVT NewEltVT = NVT.getVectorElementType();
4780 
4781  // Handle bitcasts to a different vector type with the same total bit size
4782  //
4783  // e.g. v2i64 = build_vector i64:x, i64:y => v4i32
4784  // =>
4785  // v4i32 = concat_vectors (v2i32 (bitcast i64:x)), (v2i32 (bitcast i64:y))
4786 
4787  assert(NVT.isVector() && OVT.getSizeInBits() == NVT.getSizeInBits() &&
4788  "Invalid promote type for build_vector");
4789  assert(NewEltVT.bitsLT(EltVT) && "not handled");
4790 
4791  MVT MidVT = getPromotedVectorElementType(TLI, EltVT, NewEltVT);
4792 
4793  SmallVector<SDValue, 8> NewOps;
4794  for (unsigned I = 0, E = Node->getNumOperands(); I != E; ++I) {
4795  SDValue Op = Node->getOperand(I);
4796  NewOps.push_back(DAG.getNode(ISD::BITCAST, SDLoc(Op), MidVT, Op));
4797  }
4798 
4799  SDLoc SL(Node);
4800  SDValue Concat = DAG.getNode(ISD::CONCAT_VECTORS, SL, NVT, NewOps);
4801  SDValue CvtVec = DAG.getNode(ISD::BITCAST, SL, OVT, Concat);
4802  Results.push_back(CvtVec);
4803  break;
4804  }
4805  case ISD::EXTRACT_VECTOR_ELT: {
4806  MVT EltVT = OVT.getVectorElementType();
4807  MVT NewEltVT = NVT.getVectorElementType();
4808 
4809  // Handle bitcasts to a different vector type with the same total bit size.
4810  //
4811  // e.g. v2i64 = extract_vector_elt x:v2i64, y:i32
4812  // =>
4813  // v4i32:castx = bitcast x:v2i64
4814  //
4815  // i64 = bitcast
4816  // (v2i32 build_vector (i32 (extract_vector_elt castx, (2 * y))),
4817  // (i32 (extract_vector_elt castx, (2 * y + 1)))
4818  //
4819 
4820  assert(NVT.isVector() && OVT.getSizeInBits() == NVT.getSizeInBits() &&
4821  "Invalid promote type for extract_vector_elt");
4822  assert(NewEltVT.bitsLT(EltVT) && "not handled");
4823 
4824  MVT MidVT = getPromotedVectorElementType(TLI, EltVT, NewEltVT);
4825  unsigned NewEltsPerOldElt = MidVT.getVectorNumElements();
4826 
4827  SDValue Idx = Node->getOperand(1);
4828  EVT IdxVT = Idx.getValueType();
4829  SDLoc SL(Node);
4830  SDValue Factor = DAG.getConstant(NewEltsPerOldElt, SL, IdxVT);
4831  SDValue NewBaseIdx = DAG.getNode(ISD::MUL, SL, IdxVT, Idx, Factor);
4832 
4833  SDValue CastVec = DAG.getNode(ISD::BITCAST, SL, NVT, Node->getOperand(0));
4834 
4835  SmallVector<SDValue, 8> NewOps;
4836  for (unsigned I = 0; I < NewEltsPerOldElt; ++I) {
4837  SDValue IdxOffset = DAG.getConstant(I, SL, IdxVT);
4838  SDValue TmpIdx = DAG.getNode(ISD::ADD, SL, IdxVT, NewBaseIdx, IdxOffset);
4839 
4840  SDValue Elt = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, NewEltVT,
4841  CastVec, TmpIdx);
4842  NewOps.push_back(Elt);
4843  }
4844 
4845  SDValue NewVec = DAG.getBuildVector(MidVT, SL, NewOps);
4846  Results.push_back(DAG.getNode(ISD::BITCAST, SL, EltVT, NewVec));
4847  break;
4848  }
4849  case ISD::INSERT_VECTOR_ELT: {
4850  MVT EltVT = OVT.getVectorElementType();
4851  MVT NewEltVT = NVT.getVectorElementType();
4852 
4853  // Handle bitcasts to a different vector type with the same total bit size
4854  //
4855  // e.g. v2i64 = insert_vector_elt x:v2i64, y:i64, z:i32
4856  // =>
4857  // v4i32:castx = bitcast x:v2i64
4858  // v2i32:casty = bitcast y:i64
4859  //
4860  // v2i64 = bitcast
4861  // (v4i32 insert_vector_elt
4862  // (v4i32 insert_vector_elt v4i32:castx,
4863  // (extract_vector_elt casty, 0), 2 * z),
4864  // (extract_vector_elt casty, 1), (2 * z + 1))
4865 
4866  assert(NVT.isVector() && OVT.getSizeInBits() == NVT.getSizeInBits() &&
4867  "Invalid promote type for insert_vector_elt");
4868  assert(NewEltVT.bitsLT(EltVT) && "not handled");
4869 
4870  MVT MidVT = getPromotedVectorElementType(TLI, EltVT, NewEltVT);
4871  unsigned NewEltsPerOldElt = MidVT.getVectorNumElements();
4872 
4873  SDValue Val = Node->getOperand(1);
4874  SDValue Idx = Node->getOperand(2);
4875  EVT IdxVT = Idx.getValueType();
4876  SDLoc SL(Node);
4877 
4878  SDValue Factor = DAG.getConstant(NewEltsPerOldElt, SDLoc(), IdxVT);
4879  SDValue NewBaseIdx = DAG.getNode(ISD::MUL, SL, IdxVT, Idx, Factor);
4880 
4881  SDValue CastVec = DAG.getNode(ISD::BITCAST, SL, NVT, Node->getOperand(0));
4882  SDValue CastVal = DAG.getNode(ISD::BITCAST, SL, MidVT, Val);
4883 
4884  SDValue NewVec = CastVec;
4885  for (unsigned I = 0; I < NewEltsPerOldElt; ++I) {
4886  SDValue IdxOffset = DAG.getConstant(I, SL, IdxVT);
4887  SDValue InEltIdx = DAG.getNode(ISD::ADD, SL, IdxVT, NewBaseIdx, IdxOffset);
4888 
4889  SDValue Elt = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, NewEltVT,
4890  CastVal, IdxOffset);
4891 
4892  NewVec = DAG.getNode(ISD::INSERT_VECTOR_ELT, SL, NVT,
4893  NewVec, Elt, InEltIdx);
4894  }
4895 
4896  Results.push_back(DAG.getNode(ISD::BITCAST, SL, OVT, NewVec));
4897  break;
4898  }
4899  case ISD::SCALAR_TO_VECTOR: {
4900  MVT EltVT = OVT.getVectorElementType();
4901  MVT NewEltVT = NVT.getVectorElementType();
4902 
4903  // Handle bitcasts to different vector type with the same total bit size.
4904  //
4905  // e.g. v2i64 = scalar_to_vector x:i64
4906  // =>
4907  // concat_vectors (v2i32 bitcast x:i64), (v2i32 undef)
4908  //
4909 
4910  MVT MidVT = getPromotedVectorElementType(TLI, EltVT, NewEltVT);
4911  SDValue Val = Node->getOperand(0);
4912  SDLoc SL(Node);
4913 
4914  SDValue CastVal = DAG.getNode(ISD::BITCAST, SL, MidVT, Val);
4915  SDValue Undef = DAG.getUNDEF(MidVT);
4916 
4917  SmallVector<SDValue, 8> NewElts;
4918  NewElts.push_back(CastVal);
4919  for (unsigned I = 1, NElts = OVT.getVectorNumElements(); I != NElts; ++I)
4920  NewElts.push_back(Undef);
4921 
4922  SDValue Concat = DAG.getNode(ISD::CONCAT_VECTORS, SL, NVT, NewElts);
4923  SDValue CvtVec = DAG.getNode(ISD::BITCAST, SL, OVT, Concat);
4924  Results.push_back(CvtVec);
4925  break;
4926  }
4927  case ISD::ATOMIC_SWAP: {
4928  AtomicSDNode *AM = cast<AtomicSDNode>(Node);
4929  SDLoc SL(Node);
4930  SDValue CastVal = DAG.getNode(ISD::BITCAST, SL, NVT, AM->getVal());
4931  assert(NVT.getSizeInBits() == OVT.getSizeInBits() &&
4932  "unexpected promotion type");
4933  assert(AM->getMemoryVT().getSizeInBits() == NVT.getSizeInBits() &&
4934  "unexpected atomic_swap with illegal type");
4935 
4936  SDValue NewAtomic
4937  = DAG.getAtomic(ISD::ATOMIC_SWAP, SL, NVT,
4938  DAG.getVTList(NVT, MVT::Other),
4939  { AM->getChain(), AM->getBasePtr(), CastVal },
4940  AM->getMemOperand());
4941  Results.push_back(DAG.getNode(ISD::BITCAST, SL, OVT, NewAtomic));
4942  Results.push_back(NewAtomic.getValue(1));
4943  break;
4944  }
4945  }
4946 
4947  // Replace the original node with the legalized result.
4948  if (!Results.empty()) {
4949  LLVM_DEBUG(dbgs() << "Successfully promoted node\n");
4950  ReplaceNode(Node, Results.data());
4951  } else
4952  LLVM_DEBUG(dbgs() << "Could not promote node\n");
4953 }
4954 
4955 /// This is the entry point for the file.
4958 
4959  SmallPtrSet<SDNode *, 16> LegalizedNodes;
4960  // Use a delete listener to remove nodes which were deleted during
4961  // legalization from LegalizeNodes. This is needed to handle the situation
4962  // where a new node is allocated by the object pool to the same address of a
4963  // previously deleted node.
4964  DAGNodeDeletedListener DeleteListener(
4965  *this,
4966  [&LegalizedNodes](SDNode *N, SDNode *E) { LegalizedNodes.erase(N); });
4967 
4968  SelectionDAGLegalize Legalizer(*this, LegalizedNodes);
4969 
4970  // Visit all the nodes. We start in topological order, so that we see
4971  // nodes with their original operands intact. Legalization can produce
4972  // new nodes which may themselves need to be legalized. Iterate until all
4973  // nodes have been legalized.
4974  while (true) {
4975  bool AnyLegalized = false;
4976  for (auto NI = allnodes_end(); NI != allnodes_begin();) {
4977  --NI;
4978 
4979  SDNode *N = &*NI;
4980  if (N->use_empty() && N != getRoot().getNode()) {
4981  ++NI;
4982  DeleteNode(N);
4983  continue;
4984  }
4985 
4986  if (LegalizedNodes.insert(N).second) {
4987  AnyLegalized = true;
4988  Legalizer.LegalizeOp(N);
4989 
4990  if (N->use_empty() && N != getRoot().getNode()) {
4991  ++NI;
4992  DeleteNode(N);
4993  }
4994  }
4995  }
4996  if (!AnyLegalized)
4997  break;
4998 
4999  }
5000 
5001  // Remove dead nodes now.
5002  RemoveDeadNodes();
5003 }
5004 
5006  SmallSetVector<SDNode *, 16> &UpdatedNodes) {
5007  SmallPtrSet<SDNode *, 16> LegalizedNodes;
5008  SelectionDAGLegalize Legalizer(*this, LegalizedNodes, &UpdatedNodes);
5009 
5010  // Directly insert the node in question, and legalize it. This will recurse
5011  // as needed through operands.
5012  LegalizedNodes.insert(N);
5013  Legalizer.LegalizeOp(N);
5014 
5015  return LegalizedNodes.count(N);
5016 }
llvm::CallingConv::C
@ C
C - The default llvm calling convention, compatible with C.
Definition: CallingConv.h:34
llvm::ISD::SUB
@ SUB
Definition: ISDOpcodes.h:240
llvm::Check::Size
@ Size
Definition: FileCheck.h:73
llvm::ISD::FPOWI
@ FPOWI
Definition: ISDOpcodes.h:872
llvm::ISD::FROUNDEVEN
@ FROUNDEVEN
Definition: ISDOpcodes.h:884
i
i
Definition: README.txt:29
llvm::ISD::ExternalSymbol
@ ExternalSymbol
Definition: ISDOpcodes.h:83